Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions flaml/automl/automl.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,33 @@ def size(learner_classes: dict, config: dict) -> float:
return learner_class.size(config)


def _estimator_constraint(estimator: str, func: Callable[[dict], float], sign: str) -> Callable[[dict], float]:
"""Wrap an estimator config-constraint function for the joint search space.

The wrapped function only applies ``func`` to configs whose selected
learner is ``estimator``; for any other learner it returns a value that
trivially satisfies the constraint.
"""
satisfying = float("-inf") if sign in ("<=", "<") else float("inf")

def constraint(config: dict) -> float:
config = config.get("ml", config)
if config.get("learner") != estimator:
return satisfying
return func(config)

return constraint


def _joint_config_constraints(learner_classes: dict) -> list:
"""Collect per-estimator config constraints for a joint search space."""
constraints = []
for estimator, learner_class in learner_classes.items():
for func, sign, threshold in learner_class.config_constraints():
constraints.append((_estimator_constraint(estimator, func, sign), sign, threshold))
return constraints


# metrics that sklearn only computes for binary targets
BINARY_ONLY_METRICS = ("roc_auc", "roc_auc_weighted", "ap", "f1")

Expand Down Expand Up @@ -2847,7 +2874,8 @@ def _search_parallel(self):
resource_attr=self.resource_attr,
min_resource=min_resource_all_estimator,
max_resource=self.max_resource,
config_constraints=[(partial(size, self._state.learner_classes), "<=", self._mem_thres)],
config_constraints=[(partial(size, self._state.learner_classes), "<=", self._mem_thres)]
+ _joint_config_constraints(self._state.learner_classes),
metric_constraints=self.metric_constraints,
seed=self._seed,
time_budget_s=time_budget_s,
Expand Down Expand Up @@ -3086,7 +3114,8 @@ def _search_sequential(self):
resource_attr=resource_attr,
min_resource=min_resource,
max_resource=max_resource,
config_constraints=[(learner_class.size, "<=", self._mem_thres)],
config_constraints=[(learner_class.size, "<=", self._mem_thres)]
+ list(learner_class.config_constraints()),
metric_constraints=self.metric_constraints,
seed=self._seed,
allow_empty_config=True,
Expand Down
24 changes: 21 additions & 3 deletions flaml/automl/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,9 @@ def get_params(self, deep=False):
params["_estimator_type"] = self._estimator_type
return params

def __sklearn_is_fitted__(self):
return self._model is not None

def __sklearn_is_fitted__(self):
return self._model is not None
def __sklearn_tags__(self):
"""Override sklearn tags to respect the _estimator_type attribute.

Expand Down Expand Up @@ -465,6 +465,24 @@ def size(cls, config: dict) -> float:
"""
return 1.0

@classmethod
def config_constraints(cls) -> list:
"""[optional method] config constraints for the estimator.

Override this to restrict the search to configs that satisfy
estimator-specific constraints, expressed in the same syntax as
``flaml.tune`` config_constraints.

Returns:
A list of ``(function, operator, threshold)`` tuples, where
``function`` maps a hyperparameter-config dict to a float,
``operator`` is one of ``"<="``, ``">="``, ``"<"`` or ``">"``,
and ``threshold`` is a float. A config that violates any
constraint is skipped during search. Returns an empty list
(no constraints) by default.
"""
return []

@classmethod
def cost_relative2lgbm(cls) -> float:
"""[optional method] relative cost compared to lightgbm."""
Expand Down
49 changes: 49 additions & 0 deletions test/automl/test_constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,55 @@ def test_metric_constraints_custom():
print(analysis.trials[-1])


def test_config_constraints():
from sklearn.datasets import make_classification

from flaml.automl.model import LGBMEstimator
from flaml.automl.training_log import training_log_reader

MAX_N_ESTIMATORS = 16

class ConstrainedLGBM(LGBMEstimator):
"""LGBM estimator that caps n_estimators via config_constraints."""

@classmethod
def config_constraints(cls):
return [(lambda config: config["n_estimators"], "<=", MAX_N_ESTIMATORS)]

automl = AutoML()
automl.add_learner("constrained_lgbm", ConstrainedLGBM)

log_file_name = f"test/config_constraints_{dataset}.log"
automl_settings = {
"estimator_list": ["constrained_lgbm"],
"task": "classification",
"log_file_name": log_file_name,
"n_jobs": 1,
"log_type": "all",
"keep_search_state": True,
"time_budget": 10,
}

X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)

# The best config must honor the estimator's constraint.
assert automl.best_config["n_estimators"] <= MAX_N_ESTIMATORS, automl.best_config

# Deterministic guard: every config that was actually trained (and therefore
# logged) must satisfy the constraint, i.e. no violating config was explored.
n_records = 0
with training_log_reader(log_file_name) as reader:
for record in reader.records():
n_records += 1
assert (
record.config["n_estimators"] <= MAX_N_ESTIMATORS
), f"a trained config violates the constraint: n_estimators={record.config['n_estimators']}"
assert n_records > 0, "no configs were trained; the test did not exercise the search"


if __name__ == "__main__":
test_metric_constraints()
test_metric_constraints_custom()
test_config_constraints()