在BayesSearchCV上使用RandomForestClassifier时出现的错误

Question

这是再现错误的代码：

from sklearn.ensemble        import RandomForestClassifier
from sklearn.linear_model    import LogisticRegression
from scipy.stats import loguniform
from skopt import BayesSearchCV
from sklearn.datasets import load_iris
import numpy as np

X, y = load_iris(return_X_y=True)

grid = {
     
     'LogisticRegression' : {
          'C':             loguniform.rvs(0.1, 10000, size = 50),
          'solver':        ['lbfgs','saga'],
          'penalty':       ['l2'],
          'warm_start':    [False, True],
          'class_weight' : [None, 'balanced'],
          'max_iter':      [100, 1000],
          'n_jobs':        [ 10 ]
        },
     
     'RandomForestClassifier' : {
         'n_estimators': np.random.randint(5, 200, size=10),
         'criterion'   : [ 'gini', 'entropy' ],
         'max_depth'   : np.random.randint(5, 50, size=10),
         'min_samples_split': np.random.randint(5, 50, size=10),
         'min_samples_leaf':  np.random.randint(5, 50, size=10),
         'max_features' :     loguniform.rvs(0.2, 1.0, size=5),
         'n_jobs'  : [ 10 ]
        }
}

tuner_params = {
    'cv':      2, 
    'n_jobs':  10, 
    'scoring': 'roc_auc_ovr', 
    'return_train_score': True, 
    'refit': True,
    'n_iter':3
}

clf = 'LogisticRegression'
search_cv = BayesSearchCV( estimator = eval(clf)(), search_spaces = grid[clf],  **tuner_params) 
search_cv.fit(X,y)

clf = 'RandomForestClassifier'
search_cv = BayesSearchCV( estimator = eval(clf)(), search_spaces = grid[clf],  **tuner_params) 
search_cv.fit(X,y)

使用BayesSearchCV on LogisticRegression作为分类器不会产生错误，而使用RandomForestClassifier则会产生以下错误：

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Input In [8], in <cell line: 2>()
      1 search_cv = BayesSearchCV( estimator = eval(clf)(), search_spaces = grid[clf],  **tuner_params) 
----> 2 search_cv.fit(X,y)

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/searchcv.py:466, in BayesSearchCV.fit(self, X, y, groups, callback, **fit_params)
    463 else:
    464     self.optimizer_kwargs_ = dict(self.optimizer_kwargs)
--> 466 super().fit(X=X, y=y, groups=groups, **fit_params)
    468 # BaseSearchCV never ranked train scores,
    469 # but apparently we used to ship this (back-compat)
    470 if self.return_train_score:

File ~/.conda/envs/meth/lib/python3.9/site-packages/sklearn/model_selection/_search.py:875, in BaseSearchCV.fit(self, X, y, groups, **fit_params)
    869     results = self._format_results(
    870         all_candidate_params, n_splits, all_out, all_more_results
    871     )
    873     return results
--> 875 self._run_search(evaluate_candidates)
    877 # multimetric is determined here because in the case of a callable
    878 # self.scoring the return type is only known after calling
    879 first_test_score = all_out[0]["test_scores"]

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/searchcv.py:512, in BayesSearchCV._run_search(self, evaluate_candidates)
    508 while n_iter > 0:
    509     # when n_iter < n_points points left for evaluation
    510     n_points_adjusted = min(n_iter, n_points)
--> 512     optim_result = self._step(
    513         search_space, optimizer,
    514         evaluate_candidates, n_points=n_points_adjusted
    515     )
    516     n_iter -= n_points
    518     if eval_callbacks(callbacks, optim_result):

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/searchcv.py:400, in BayesSearchCV._step(self, search_space, optimizer, evaluate_candidates, n_points)
    397 """Generate n_jobs parameters and evaluate them in parallel.
    398 """
    399 # get parameter values to evaluate
--> 400 params = optimizer.ask(n_points=n_points)
    402 # convert parameters to python native types
    403 params = [[np.array(v).item() for v in p] for p in params]

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/optimizer/optimizer.py:395, in Optimizer.ask(self, n_points, strategy)
    393 X = []
    394 for i in range(n_points):
--> 395     x = opt.ask()
    396     X.append(x)
    398     ti_available = "ps" in self.acq_func and len(opt.yi) > 0

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/optimizer/optimizer.py:367, in Optimizer.ask(self, n_points, strategy)
    336 """Query point or multiple points at which objective should be evaluated.
    337 
    338 n_points : int or None, default: None
   (...)
    364 
    365 """
    366 if n_points is None:
--> 367     return self._ask()
    369 supported_strategies = ["cl_min", "cl_mean", "cl_max"]
    371 if not (isinstance(n_points, int) and n_points > 0):

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/optimizer/optimizer.py:434, in Optimizer._ask(self)
    430 if self._n_initial_points > 0 or self.base_estimator_ is None:
    431     # this will not make a copy of `self.rng` and hence keep advancing
    432     # our random state.
    433     if self._initial_samples is None:
--> 434         return self.space.rvs(random_state=self.rng)[0]
    435     else:
    436         # The samples are evaluated starting form initial_samples[0]
    437         return self._initial_samples[
    438             len(self._initial_samples) - self._n_initial_points]

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/space.py:900, in Space.rvs(self, n_samples, random_state)
    897 columns = []
    899 for dim in self.dimensions:
--> 900     columns.append(dim.rvs(n_samples=n_samples, random_state=rng))
    902 # Transpose
    903 return _transpose_list_array(columns)

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/space.py:698, in Categorical.rvs(self, n_samples, random_state)
    696     return self.inverse_transform([(choices)])
    697 elif self.transform_ == "normalize":
--> 698     return self.inverse_transform(list(choices))
    699 else:
    700     return [self.categories[c] for c in choices]

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/space.py:685, in Categorical.inverse_transform(self, Xt)
    680 """Inverse transform samples from the warped space back into the
    681    original space.
    682 """
    683 # The concatenation of all transformed dimensions makes Xt to be
    684 # of type float, hence the required cast back to int.
--> 685 inv_transform = super(Categorical, self).inverse_transform(Xt)
    686 if isinstance(inv_transform, list):
    687     inv_transform = np.array(inv_transform)

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/space.py:168, in Dimension.inverse_transform(self, Xt)
    164 def inverse_transform(self, Xt):
    165     """Inverse transform samples from the warped space back into the
    166        original space.
    167     """
--> 168     return self.transformer.inverse_transform(Xt)

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/transformers.py:309, in Pipeline.inverse_transform(self, X)
    307 def inverse_transform(self, X):
    308     for transformer in self.transformers[::-1]:
--> 309         X = transformer.inverse_transform(X)
    310     return X

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/transformers.py:216, in LabelEncoder.inverse_transform(self, Xt)
    214 else:
    215     Xt = np.asarray(Xt)
--> 216 return [
    217     self.inverse_mapping_[int(np.round(i))] for i in Xt
    218 ]

File ~/.conda/envs/meth/lib/python3.9/site-packages/skopt/space/transformers.py:217, in <listcomp>(.0)
    214 else:
    215     Xt = np.asarray(Xt)
    216 return [
--> 217     self.inverse_mapping_[int(np.round(i))] for i in Xt
    218 ]

KeyError: 9

我的版本：

python: 3.9.12 sklearn: 1.1.1 skopt: 0.9.0

同样的错误发生在使用XGBClassifier或GradientBoostingClassifier时，而使用SVC或KNeighborsClassifier时没有错误。

Answer 1

我相信这与skopt如何对超参数空间进行编码有关:似乎需要有由随机列表生成的相同点来触发错误，尽管有时它是合适的。要么存在冲突，要么使网格被错误地处理。

至少在将所有随机列表更改为list(range(...))之后，这个问题就不再为我复制了。

也许值得一份窃听器报告。