nirdizati_light.hyperparameter_optimisation.common

  1from enum import Enum
  2from typing import Optional, Union
  3from predictive_model.predictive_model import PredictiveModel
  4import hyperopt
  5import numpy as np
  6from hyperopt import Trials, hp, fmin
  7from hyperopt.pyll import scope
  8
  9from nirdizati_light.predictive_model.common import ClassificationMethods, RegressionMethods
 10
 11
 12class HyperoptTarget(Enum):
 13    """
 14    Available targets for hyperparameter optimisation
 15    """
 16    AUC = 'auc'
 17    F1 = 'f1_score'
 18    MAE = 'mae'
 19    ACCURACY = 'accuracy'
 20
 21
 22def _get_space(model_type: Union[ClassificationMethods, RegressionMethods]) -> dict:
 23    if model_type is ClassificationMethods.RANDOM_FOREST.value:
 24        return {
 25            'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)),
 26            'max_depth': scope.int(hp.quniform('max_depth', 4, 30, 1)),
 27            'max_features': hp.choice('max_features', ['sqrt', 'log2', None]),
 28            'criterion': hp.choice('criterion',['gini','entropy']),
 29            'warm_start': True
 30        }
 31    elif model_type is ClassificationMethods.DT.value:
 32        return {
 33            'max_depth': hp.choice('max_depth', range(1, 6)),
 34            'max_features': hp.choice('max_features', range(7, 50)),
 35            'criterion': hp.choice('criterion', ["gini", "entropy"]),
 36            'min_samples_split': hp.choice('min_samples_split', range(2, 10)),
 37            'min_samples_leaf': hp.choice('min_samples_leaf', range(1, 10)),
 38
 39        }
 40    elif model_type is ClassificationMethods.KNN.value:
 41        return {
 42            'n_neighbors': hp.choice('n_neighbors', np.arange(1, 20, dtype=int)),
 43            'weights': hp.choice('weights', ['uniform', 'distance']),
 44        }
 45
 46    elif model_type is ClassificationMethods.XGBOOST.value:
 47        return {
 48            'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)),
 49            'max_depth': scope.int(hp.quniform('max_depth', 3, 30, 1)),
 50        }
 51
 52    elif model_type is ClassificationMethods.SGDCLASSIFIER.value:
 53        return {
 54            'loss': hp.choice('loss', ['hinge', 'log_loss', 'modified_huber', 'squared_hinge', 'perceptron', 'squared_error',
 55                                       'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive']),
 56            'penalty': hp.choice('penalty', [None, 'l1', 'l2', 'elasticnet']),
 57            'alpha': hp.uniform('alpha', 0.0001, 0.5),
 58            'fit_intercept': hp.choice('fit_intercept', [True, False]),
 59            'tol': hp.uniform('tol', 1e-3, 0.5),
 60            'shuffle': hp.choice('shuffle', [True, False]),
 61            'eta0': hp.quniform('eta0', 0, 5, 1),
 62            # 'early_stopping': hp.choice('early_stopping', [True, False]), #needs to be false with partial_fit
 63            'validation_fraction': 0.1,
 64            'n_iter_no_change': scope.int(hp.quniform('n_iter_no_change', 1, 30, 5))
 65        }
 66    elif model_type is ClassificationMethods.SVM.value:
 67        return{
 68            'kernel' : hp.choice('kernel',['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']),
 69            'C' : hp.uniform('C',0.1,1)
 70        }
 71    elif model_type is ClassificationMethods.PERCEPTRON.value:
 72        return {
 73            'penalty': hp.choice('penalty', [None, 'l1', 'l2', 'elasticnet']),
 74            'alpha': hp.uniform('alpha', 0.0001, 0.5),
 75            'fit_intercept': hp.choice('fit_intercept', [True, False]),
 76            'tol': hp.uniform('tol', 1e-3, 0.5),
 77            'shuffle': hp.choice('shuffle', [True, False]),
 78            'eta0': scope.int(hp.quniform('eta0', 4, 30, 1)),
 79            # 'early_stopping': hp.choice('early_stopping', [True, False]), #needs to be false with partial_fit
 80            'validation_fraction': 0.1,
 81            'n_iter_no_change': scope.int(hp.quniform('n_iter_no_change', 5, 30, 5))
 82        }
 83    elif model_type is ClassificationMethods.MLP.value:
 84        return {
 85            'hidden_layer_sizes': scope.int(hp.uniform('hidden_layer_sizes',10,100)),
 86            'alpha': hp.uniform('alpha', 0.0001, 0.5),
 87            'shuffle': hp.choice('shuffle', [True, False]),
 88#            'eta0': scope.int(hp.quniform('eta0', 4, 30, 1)),
 89            # 'early_stopping': hp.choice('early_stopping', [True, False]), #needs to be false with partial_fit
 90            'validation_fraction': 0.1,
 91            'n_iter_no_change': scope.int(hp.quniform('n_iter_no_change', 5, 30, 5))
 92        }
 93    elif model_type is ClassificationMethods.RANDOM_FOREST.value:
 94        return {
 95            'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)),
 96            'max_depth': scope.int(hp.quniform('max_depth', 4, 30, 1)),
 97            'max_features': hp.choice('max_features', ['sqrt', 'log2', 'auto', None]),
 98            'warm_start': True
 99        }
100
101    elif model_type in [ClassificationMethods.LSTM.value, ClassificationMethods.CUSTOM_PYTORCH.value]:
102        return {
103            'max_num_epochs': 200,
104            'lr': hp.uniform('lr', 0.0001, 0.1),
105            'lstm_hidden_size': hp.choice('lstm_hidden_size', np.arange(50, 200, dtype=int)),
106            'lstm_num_layers': hp.choice('lstm_num_layers', np.arange(1, 3, dtype=int)),
107            'early_stop_patience': scope.int(hp.quniform('early_stop_patience', 5, 30, 5)),
108            'early_stop_min_delta': hp.uniform('early_stop_min_delta', 0.005, 0.05),
109            'batch_size': 128
110        }
111    elif model_type is RegressionMethods.RANDOM_FOREST.value:
112        return {
113            'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)),
114            'max_depth': scope.int(hp.quniform('max_depth', 4, 30, 1)),
115            'max_features': hp.choice('max_features', ['sqrt', 'log2', None]),
116            'criterion': hp.choice('criterion', ['squared_error', 'friedman_mse', 'poisson', 'absolute_error']),
117            'warm_start': True
118        }
119    else:
120        raise Exception('Unsupported model_type')
121
122
123def retrieve_best_model(
124    predictive_models: list[PredictiveModel],
125    max_evaluations: int,
126    target: HyperoptTarget,
127    seed: Optional[int]=None
128):
129    """
130    Perform hyperparameter optimization on the given model.
131
132    Args:
133        predictive_models (list[PredictiveModel]): List of models to perform optimization on (each model must be of class PredictiveModel).
134        max_evaluations (int): Maximum number of hyperparameter configurations to try.
135        target (HyperoptTarget): Which target score to optimize for.
136        seed (Optional[int]): Optional seed value for reproducibility.
137
138    Returns:
139        tuple: A tuple containing the best candidates, the best model index, the best model, and the best hyperparameter configuration.
140    """
141
142    best_candidates = []
143    best_target_per_model = []
144
145    if type(predictive_models) is not list:
146        predictive_models = [predictive_models]
147
148    for predictive_model in predictive_models:
149        print(f'Running hyperparameter optimization on model {predictive_model.model_type}...')
150
151        space = _get_space(predictive_model.model_type)
152
153        # update hyperopt space with the user provided one, if available
154        if predictive_model.hyperopt_space is not None:
155            space.update(predictive_model.hyperopt_space)
156
157        trials = Trials()
158
159        fmin(
160            lambda x: predictive_model.train_and_evaluate_configuration(config=x, target=target),
161            space,
162            algo=hyperopt.tpe.suggest,
163            max_evals=max_evaluations,
164            trials=trials,rstate=np.random.default_rng(seed)
165        )
166        best_candidate = trials.best_trial['result']
167
168        best_candidates.append(best_candidate)
169        best_target_per_model.append(best_candidate['result'][target])
170
171    # Find the best performing model
172    best_model_idx = best_target_per_model.index(max(best_target_per_model))
173
174    return best_candidates, best_model_idx, best_candidates[best_model_idx]['model'], best_candidates[best_model_idx]['config']