From c2af83f761cd2531b2dce8ba164dbb1d812b8b3a Mon Sep 17 00:00:00 2001 From: Ankit Dhandharia <44242184+ankitdhandharia@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:28:30 -0500 Subject: [PATCH] feat: honor estimator config_constraints during AutoML search (#840) --- flaml/automl/automl.py | 33 ++++++++++++++++++++-- flaml/automl/model.py | 24 ++++++++++++++-- test/automl/test_constraints.py | 49 +++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py index b05f98b24d..cdd7bf8e38 100644 --- a/flaml/automl/automl.py +++ b/flaml/automl/automl.py @@ -92,6 +92,33 @@ def size(learner_classes: dict, config: dict) -> float: return learner_class.size(config) +def _estimator_constraint(estimator: str, func: Callable[[dict], float], sign: str) -> Callable[[dict], float]: + """Wrap an estimator config-constraint function for the joint search space. + + The wrapped function only applies ``func`` to configs whose selected + learner is ``estimator``; for any other learner it returns a value that + trivially satisfies the constraint. + """ + satisfying = float("-inf") if sign in ("<=", "<") else float("inf") + + def constraint(config: dict) -> float: + config = config.get("ml", config) + if config.get("learner") != estimator: + return satisfying + return func(config) + + return constraint + + +def _joint_config_constraints(learner_classes: dict) -> list: + """Collect per-estimator config constraints for a joint search space.""" + constraints = [] + for estimator, learner_class in learner_classes.items(): + for func, sign, threshold in learner_class.config_constraints(): + constraints.append((_estimator_constraint(estimator, func, sign), sign, threshold)) + return constraints + + # metrics that sklearn only computes for binary targets BINARY_ONLY_METRICS = ("roc_auc", "roc_auc_weighted", "ap", "f1") @@ -2847,7 +2874,8 @@ def _search_parallel(self): resource_attr=self.resource_attr, min_resource=min_resource_all_estimator, max_resource=self.max_resource, - config_constraints=[(partial(size, self._state.learner_classes), "<=", self._mem_thres)], + config_constraints=[(partial(size, self._state.learner_classes), "<=", self._mem_thres)] + + _joint_config_constraints(self._state.learner_classes), metric_constraints=self.metric_constraints, seed=self._seed, time_budget_s=time_budget_s, @@ -3086,7 +3114,8 @@ def _search_sequential(self): resource_attr=resource_attr, min_resource=min_resource, max_resource=max_resource, - config_constraints=[(learner_class.size, "<=", self._mem_thres)], + config_constraints=[(learner_class.size, "<=", self._mem_thres)] + + list(learner_class.config_constraints()), metric_constraints=self.metric_constraints, seed=self._seed, allow_empty_config=True, diff --git a/flaml/automl/model.py b/flaml/automl/model.py index d863dd6fec..09c454926f 100644 --- a/flaml/automl/model.py +++ b/flaml/automl/model.py @@ -155,9 +155,9 @@ def get_params(self, deep=False): params["_estimator_type"] = self._estimator_type return params - def __sklearn_is_fitted__(self): - return self._model is not None - + def __sklearn_is_fitted__(self): + return self._model is not None + def __sklearn_tags__(self): """Override sklearn tags to respect the _estimator_type attribute. @@ -465,6 +465,24 @@ def size(cls, config: dict) -> float: """ return 1.0 + @classmethod + def config_constraints(cls) -> list: + """[optional method] config constraints for the estimator. + + Override this to restrict the search to configs that satisfy + estimator-specific constraints, expressed in the same syntax as + ``flaml.tune`` config_constraints. + + Returns: + A list of ``(function, operator, threshold)`` tuples, where + ``function`` maps a hyperparameter-config dict to a float, + ``operator`` is one of ``"<="``, ``">="``, ``"<"`` or ``">"``, + and ``threshold`` is a float. A config that violates any + constraint is skipped during search. Returns an empty list + (no constraints) by default. + """ + return [] + @classmethod def cost_relative2lgbm(cls) -> float: """[optional method] relative cost compared to lightgbm.""" diff --git a/test/automl/test_constraints.py b/test/automl/test_constraints.py index 9d2a288550..1a6eb8a976 100644 --- a/test/automl/test_constraints.py +++ b/test/automl/test_constraints.py @@ -161,6 +161,55 @@ def test_metric_constraints_custom(): print(analysis.trials[-1]) +def test_config_constraints(): + from sklearn.datasets import make_classification + + from flaml.automl.model import LGBMEstimator + from flaml.automl.training_log import training_log_reader + + MAX_N_ESTIMATORS = 16 + + class ConstrainedLGBM(LGBMEstimator): + """LGBM estimator that caps n_estimators via config_constraints.""" + + @classmethod + def config_constraints(cls): + return [(lambda config: config["n_estimators"], "<=", MAX_N_ESTIMATORS)] + + automl = AutoML() + automl.add_learner("constrained_lgbm", ConstrainedLGBM) + + log_file_name = f"test/config_constraints_{dataset}.log" + automl_settings = { + "estimator_list": ["constrained_lgbm"], + "task": "classification", + "log_file_name": log_file_name, + "n_jobs": 1, + "log_type": "all", + "keep_search_state": True, + "time_budget": 10, + } + + X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) + automl.fit(X_train=X_train, y_train=y_train, **automl_settings) + + # The best config must honor the estimator's constraint. + assert automl.best_config["n_estimators"] <= MAX_N_ESTIMATORS, automl.best_config + + # Deterministic guard: every config that was actually trained (and therefore + # logged) must satisfy the constraint, i.e. no violating config was explored. + n_records = 0 + with training_log_reader(log_file_name) as reader: + for record in reader.records(): + n_records += 1 + assert ( + record.config["n_estimators"] <= MAX_N_ESTIMATORS + ), f"a trained config violates the constraint: n_estimators={record.config['n_estimators']}" + assert n_records > 0, "no configs were trained; the test did not exercise the search" + + if __name__ == "__main__": test_metric_constraints() test_metric_constraints_custom() + test_config_constraints()