Expectile Regression¶
Imports¶
In [12]:
Copied!
from xgboostlss.model import *
from xgboostlss.distributions.Expectile import *
from xgboostlss.datasets.data_loader import load_simulated_gaussian_data
import multiprocessing
import plotnine
from plotnine import *
plotnine.options.figure_size = (20, 10)
from xgboostlss.model import *
from xgboostlss.distributions.Expectile import *
from xgboostlss.datasets.data_loader import load_simulated_gaussian_data
import multiprocessing
import plotnine
from plotnine import *
plotnine.options.figure_size = (20, 10)
Data¶
In [2]:
Copied!
# The data is a simulated Gaussian as follows, where x is the only true feature and all others are noise variables
# loc = 10
# scale = 1 + 4*((0.3 < x) & (x < 0.5)) + 2*(x > 0.7)
train, test = load_simulated_gaussian_data()
n_cpu = multiprocessing.cpu_count()
X_train, y_train = train.filter(regex="x"), train["y"].values
X_test, y_test = test.filter(regex="x"), test["y"].values
dtrain = xgb.DMatrix(X_train, label=y_train, nthread=n_cpu)
dtest = xgb.DMatrix(X_test, nthread=n_cpu)
# The data is a simulated Gaussian as follows, where x is the only true feature and all others are noise variables
# loc = 10
# scale = 1 + 4*((0.3 < x) & (x < 0.5)) + 2*(x > 0.7)
train, test = load_simulated_gaussian_data()
n_cpu = multiprocessing.cpu_count()
X_train, y_train = train.filter(regex="x"), train["y"].values
X_test, y_test = test.filter(regex="x"), test["y"].values
dtrain = xgb.DMatrix(X_train, label=y_train, nthread=n_cpu)
dtest = xgb.DMatrix(X_test, nthread=n_cpu)
Expectile Specification¶
In [3]:
Copied!
xgblss = XGBoostLSS(
Expectile(stabilization="None", # Options are "None", "MAD", "L2".
expectiles = [0.05, 0.95], # List of expectiles to be estimated, in increasing order.
penalize_crossing = True # Whether to include a penalty term to discourage crossing of expectiles.
)
)
xgblss = XGBoostLSS(
Expectile(stabilization="None", # Options are "None", "MAD", "L2".
expectiles = [0.05, 0.95], # List of expectiles to be estimated, in increasing order.
penalize_crossing = True # Whether to include a penalty term to discourage crossing of expectiles.
)
)
Hyper-Parameter Optimization¶
Any XGBoost hyperparameter can be tuned, where the structure of the parameter dictionary needs to be as follows:
- Float/Int sample_type
- {"param_name": ["sample_type", low, high, log]}
- sample_type: str, Type of sampling, e.g., "float" or "int"
- low: int, Lower endpoint of the range of suggested values
- high: int, Upper endpoint of the range of suggested values
- log: bool, Flag to sample the value from the log domain or not
- Example: {"eta": "float", low=1e-5, high=1, log=True]}
- Categorical sample_type
- {"param_name": ["sample_type", ["choice1", "choice2", "choice3", "..."]]}
- sample_type: str, Type of sampling, either "categorical"
- choice1, choice2, choice3, ...: str, Possible choices for the parameter
- Example: {"booster": ["categorical", ["gbtree", "dart"]]}
- For parameters without tunable choice (this is needed if tree_method = "gpu_hist" and gpu_id needs to be specified)
- {"param_name": ["none", [value]]},
- param_name: str, Name of the parameter
- value: int, Value of the parameter
- Example: {"gpu_id": ["none", [0]]}Depending on which parameters are optimized, it might happen that some of them are not used, e.g., when {"booster": ["categorical", ["gbtree", "gblinear"]]} and {"max_depth": ["int", 1, 10, False]} are specified, max_depth is not used when gblinear is sampled, since it has no such argument. it has no such argument.
In [4]:
Copied!
param_dict = {
"eta": ["float", {"low": 1e-5, "high": 1, "log": True}],
"max_depth": ["int", {"low": 1, "high": 10, "log": False}],
"gamma": ["float", {"low": 1e-8, "high": 40, "log": True}],
"subsample": ["float", {"low": 0.2, "high": 1.0, "log": False}],
"colsample_bytree": ["float", {"low": 0.2, "high": 1.0, "log": False}],
"min_child_weight": ["float", {"low": 1e-8, "high": 500, "log": True}],
"booster": ["categorical", ["gbtree"]],
# "tree_method": ["categorical", ["auto", "approx", "hist", "gpu_hist"]],
# "gpu_id": ["none", [0]]
}
np.random.seed(123)
opt_param = xgblss.hyper_opt(param_dict,
dtrain,
num_boost_round=100, # Number of boosting iterations.
nfold=5, # Number of cv-folds.
early_stopping_rounds=20, # Number of early-stopping rounds
max_minutes=10, # Time budget in minutes, i.e., stop study after the given number of minutes.
n_trials=None, # The number of trials. If this argument is set to None, there is no limitation on the number of trials.
silence=False, # Controls the verbosity of the trail, i.e., user can silence the outputs of the trail.
seed=123, # Seed used to generate cv-folds.
hp_seed=None # Seed for random number generator used in the Bayesian hyperparameter search.
)
param_dict = {
"eta": ["float", {"low": 1e-5, "high": 1, "log": True}],
"max_depth": ["int", {"low": 1, "high": 10, "log": False}],
"gamma": ["float", {"low": 1e-8, "high": 40, "log": True}],
"subsample": ["float", {"low": 0.2, "high": 1.0, "log": False}],
"colsample_bytree": ["float", {"low": 0.2, "high": 1.0, "log": False}],
"min_child_weight": ["float", {"low": 1e-8, "high": 500, "log": True}],
"booster": ["categorical", ["gbtree"]],
# "tree_method": ["categorical", ["auto", "approx", "hist", "gpu_hist"]],
# "gpu_id": ["none", [0]]
}
np.random.seed(123)
opt_param = xgblss.hyper_opt(param_dict,
dtrain,
num_boost_round=100, # Number of boosting iterations.
nfold=5, # Number of cv-folds.
early_stopping_rounds=20, # Number of early-stopping rounds
max_minutes=10, # Time budget in minutes, i.e., stop study after the given number of minutes.
n_trials=None, # The number of trials. If this argument is set to None, there is no limitation on the number of trials.
silence=False, # Controls the verbosity of the trail, i.e., user can silence the outputs of the trail.
seed=123, # Seed used to generate cv-folds.
hp_seed=None # Seed for random number generator used in the Bayesian hyperparameter search.
)
[I 2023-05-18 07:44:34,783] A new study created in memory with name: XGBoostLSS Hyper-Parameter Optimization
C:\Users\maerzale\.virtualenvs\XGBoostLSS-vIPRRz-M\lib\site-packages\optuna\progress_bar.py:56: ExperimentalWarning: Progress bar is experimental (supported from v1.2.0). The interface can change in the future.
0%| | 00:00/10:00
[I 2023-05-18 07:44:39,300] Trial 0 finished with value: 3849.6137206000008 and parameters: {'eta': 0.08271379019528409, 'max_depth': 2, 'gamma': 3.75303243021132e-08, 'subsample': 0.7583110772385693, 'colsample_bytree': 0.7712601401265677, 'min_child_weight': 12.05931834108896, 'booster': 'gbtree'}. Best is trial 0 with value: 3849.6137206000008. [I 2023-05-18 07:44:45,975] Trial 1 finished with value: 4910.7698244 and parameters: {'eta': 1.7544244180335518e-05, 'max_depth': 6, 'gamma': 0.004467947764901467, 'subsample': 0.3135576617792821, 'colsample_bytree': 0.9569717279128798, 'min_child_weight': 0.017485979149742367, 'booster': 'gbtree'}. Best is trial 0 with value: 3849.6137206000008. [I 2023-05-18 07:44:52,558] Trial 2 finished with value: 4094.982373 and parameters: {'eta': 0.011635867248966446, 'max_depth': 4, 'gamma': 1.6807209198451994e-06, 'subsample': 0.29862014858022196, 'colsample_bytree': 0.6775444032099875, 'min_child_weight': 0.0716153752583224, 'booster': 'gbtree'}. Best is trial 0 with value: 3849.6137206000008. [I 2023-05-18 07:44:54,976] Trial 3 finished with value: 4746.1577148 and parameters: {'eta': 0.1606777662987379, 'max_depth': 8, 'gamma': 2.100109322159124e-06, 'subsample': 0.48424713514211837, 'colsample_bytree': 0.21402102945557547, 'min_child_weight': 0.0009708779043907101, 'booster': 'gbtree'}. Best is trial 0 with value: 3849.6137206000008. [I 2023-05-18 07:45:00,842] Trial 4 finished with value: 3873.440625 and parameters: {'eta': 0.05652951954174257, 'max_depth': 3, 'gamma': 0.0005963395249999443, 'subsample': 0.7867032983610922, 'colsample_bytree': 0.621496749144236, 'min_child_weight': 3.920866081718485e-08, 'booster': 'gbtree'}. Best is trial 0 with value: 3849.6137206000008. [I 2023-05-18 07:45:07,158] Trial 5 finished with value: 4412.164159800001 and parameters: {'eta': 0.01026333766663507, 'max_depth': 3, 'gamma': 0.6643631124022648, 'subsample': 0.3709759999147997, 'colsample_bytree': 0.7923492167364432, 'min_child_weight': 108.77927015566222, 'booster': 'gbtree'}. Best is trial 0 with value: 3849.6137206000008. [I 2023-05-18 07:45:09,683] Trial 6 finished with value: 3831.8999510000003 and parameters: {'eta': 0.29641843929975825, 'max_depth': 2, 'gamma': 0.0066813332633983424, 'subsample': 0.9494802838287351, 'colsample_bytree': 0.8535938721318461, 'min_child_weight': 13.488897426001607, 'booster': 'gbtree'}. Best is trial 6 with value: 3831.8999510000003. [I 2023-05-18 07:45:11,682] Trial 7 finished with value: 4662.612109600001 and parameters: {'eta': 0.9791107988028825, 'max_depth': 4, 'gamma': 0.013492779338623239, 'subsample': 0.38801042583640505, 'colsample_bytree': 0.674829405253886, 'min_child_weight': 8.439814742331992, 'booster': 'gbtree'}. Best is trial 6 with value: 3831.8999510000003. [I 2023-05-18 07:45:22,533] Trial 8 finished with value: 4846.8927732 and parameters: {'eta': 0.0015722868014909582, 'max_depth': 10, 'gamma': 1.2941893878110945e-07, 'subsample': 0.5501798357222414, 'colsample_bytree': 0.33290949864285163, 'min_child_weight': 0.28962973623146615, 'booster': 'gbtree'}. Best is trial 6 with value: 3831.8999510000003. [I 2023-05-18 07:45:29,062] Trial 9 finished with value: 4279.644287000001 and parameters: {'eta': 0.034838837021592906, 'max_depth': 8, 'gamma': 0.0007605540443170603, 'subsample': 0.2008909175469631, 'colsample_bytree': 0.6408230363371883, 'min_child_weight': 7.835728434675012, 'booster': 'gbtree'}. Best is trial 6 with value: 3831.8999510000003. [I 2023-05-18 07:45:32,163] Trial 10 finished with value: 3872.1007322000005 and parameters: {'eta': 0.946092052380456, 'max_depth': 1, 'gamma': 23.05265186576724, 'subsample': 0.978220750473458, 'colsample_bytree': 0.9657778696197045, 'min_child_weight': 262.8439261869978, 'booster': 'gbtree'}. Best is trial 6 with value: 3831.8999510000003. [I 2023-05-18 07:45:33,796] Trial 11 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:45:38,109] Trial 12 finished with value: 3829.7966309999997 and parameters: {'eta': 0.11518948340699091, 'max_depth': 2, 'gamma': 2.7060981811336582e-05, 'subsample': 0.9950562472314343, 'colsample_bytree': 0.8395188911604501, 'min_child_weight': 1.6756750654141501, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:45:40,633] Trial 13 finished with value: 3972.9110354000004 and parameters: {'eta': 0.3327861289786935, 'max_depth': 5, 'gamma': 8.417487599479588e-05, 'subsample': 0.9759484295164087, 'colsample_bytree': 0.8829773356786005, 'min_child_weight': 0.000710799139819674, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:45:42,415] Trial 14 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:45:49,610] Trial 15 finished with value: 4052.4110352 and parameters: {'eta': 0.03127782133309635, 'max_depth': 6, 'gamma': 0.03856011842518284, 'subsample': 0.8605268485474115, 'colsample_bytree': 0.9802754842939604, 'min_child_weight': 0.7494701524781796, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:45:56,452] Trial 16 finished with value: 3897.8366212 and parameters: {'eta': 0.3248165738732373, 'max_depth': 1, 'gamma': 0.1447715142633139, 'subsample': 0.6686440305901731, 'colsample_bytree': 0.7587273711806963, 'min_child_weight': 0.012176146158160988, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:45:58,418] Trial 17 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:46:01,295] Trial 18 pruned. Trial was pruned at iteration 27. [I 2023-05-18 07:46:03,106] Trial 19 finished with value: 3861.3965332 and parameters: {'eta': 0.9991592793623117, 'max_depth': 2, 'gamma': 0.00032302199815410835, 'subsample': 0.6900045458735542, 'colsample_bytree': 0.898779212376982, 'min_child_weight': 1.4101583404218248, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:46:05,436] Trial 20 pruned. Trial was pruned at iteration 24. [I 2023-05-18 07:46:12,169] Trial 21 finished with value: 3839.3043454 and parameters: {'eta': 0.060014758202272964, 'max_depth': 2, 'gamma': 1.172204905994344e-08, 'subsample': 0.8072273545524364, 'colsample_bytree': 0.804523712203328, 'min_child_weight': 15.03132746070872, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:46:16,198] Trial 22 finished with value: 3880.2305664 and parameters: {'eta': 0.07668855063884375, 'max_depth': 4, 'gamma': 2.541346289239318e-07, 'subsample': 0.8347765094233522, 'colsample_bytree': 0.8265103772854449, 'min_child_weight': 23.130483377588998, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:46:17,828] Trial 23 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:46:21,219] Trial 24 finished with value: 3873.6299805999997 and parameters: {'eta': 0.10470363505869013, 'max_depth': 3, 'gamma': 3.2591364038306743e-07, 'subsample': 0.9431407526628489, 'colsample_bytree': 0.7212823735994254, 'min_child_weight': 2.086355614787028, 'booster': 'gbtree'}. Best is trial 12 with value: 3829.7966309999997. [I 2023-05-18 07:46:22,840] Trial 25 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:46:25,175] Trial 26 finished with value: 3829.0803222000004 and parameters: {'eta': 0.31176831173972425, 'max_depth': 2, 'gamma': 0.00012768235518059143, 'subsample': 0.9976736957019369, 'colsample_bytree': 0.9343791268506586, 'min_child_weight': 0.17633410334968522, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:27,401] Trial 27 finished with value: 3909.7964846000004 and parameters: {'eta': 0.41974670760704924, 'max_depth': 4, 'gamma': 0.00018098410025737228, 'subsample': 0.9358204248422611, 'colsample_bytree': 0.9932715080301568, 'min_child_weight': 0.10117414518107959, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:29,928] Trial 28 finished with value: 3839.4581544000002 and parameters: {'eta': 0.18280121854883766, 'max_depth': 3, 'gamma': 9.419058906747455e-05, 'subsample': 0.9910938881170037, 'colsample_bytree': 0.9109293050412733, 'min_child_weight': 1.6211361973406226, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:35,262] Trial 29 finished with value: 3832.0506344 and parameters: {'eta': 0.09686158058190168, 'max_depth': 2, 'gamma': 0.005141921955426237, 'subsample': 0.9148562338697188, 'colsample_bytree': 0.9175171106536764, 'min_child_weight': 0.09300113208446839, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:39,613] Trial 30 finished with value: 3866.5385742 and parameters: {'eta': 0.49696201469481904, 'max_depth': 1, 'gamma': 0.0019435068988503954, 'subsample': 0.8496305177099301, 'colsample_bytree': 0.8486877326688211, 'min_child_weight': 3.4071836781908664, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:43,521] Trial 31 finished with value: 3842.1080078 and parameters: {'eta': 0.12490453284565363, 'max_depth': 2, 'gamma': 0.0034084347255670024, 'subsample': 0.9430834611957088, 'colsample_bytree': 0.927821082665384, 'min_child_weight': 0.17726958082409275, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:48,321] Trial 32 finished with value: 3836.0010254 and parameters: {'eta': 0.08125276446841985, 'max_depth': 2, 'gamma': 0.005647022294330814, 'subsample': 0.9997744963168509, 'colsample_bytree': 0.9280703976185687, 'min_child_weight': 0.019967919475863467, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:50,264] Trial 33 finished with value: 3849.8502439999997 and parameters: {'eta': 0.4968648252546815, 'max_depth': 3, 'gamma': 0.0002635459046385316, 'subsample': 0.8966028569897085, 'colsample_bytree': 0.9509397808861043, 'min_child_weight': 0.2575526255091068, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:46:51,992] Trial 34 pruned. Trial was pruned at iteration 22. [I 2023-05-18 07:46:54,608] Trial 35 pruned. Trial was pruned at iteration 23. [I 2023-05-18 07:46:57,007] Trial 36 pruned. Trial was pruned at iteration 24. [I 2023-05-18 07:46:58,831] Trial 37 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:02,550] Trial 38 finished with value: 3834.378174 and parameters: {'eta': 0.10100135583324957, 'max_depth': 2, 'gamma': 0.0005549650885496035, 'subsample': 0.947160577895568, 'colsample_bytree': 0.9130753843356371, 'min_child_weight': 0.006229016543699052, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:04,511] Trial 39 pruned. Trial was pruned at iteration 21. [I 2023-05-18 07:47:06,662] Trial 40 pruned. Trial was pruned at iteration 22. [I 2023-05-18 07:47:09,778] Trial 41 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:12,326] Trial 42 finished with value: 3837.595166 and parameters: {'eta': 0.2497037389551113, 'max_depth': 2, 'gamma': 0.00046548258318221376, 'subsample': 0.934933977583335, 'colsample_bytree': 0.9473882468429256, 'min_child_weight': 0.04489229837063145, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:13,985] Trial 43 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:17,015] Trial 44 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:19,642] Trial 45 finished with value: 3841.8095216 and parameters: {'eta': 0.24918763440980632, 'max_depth': 2, 'gamma': 0.00019377279852196002, 'subsample': 0.9623310915834469, 'colsample_bytree': 0.9655102646688085, 'min_child_weight': 0.880784888132132, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:21,253] Trial 46 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:23,910] Trial 47 finished with value: 3848.4883786 and parameters: {'eta': 0.1577822755792656, 'max_depth': 3, 'gamma': 0.06089341136243045, 'subsample': 0.8852415295990044, 'colsample_bytree': 0.8592738199245256, 'min_child_weight': 0.11909618666523691, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:26,518] Trial 48 finished with value: 3868.1118650000003 and parameters: {'eta': 0.318867945915223, 'max_depth': 2, 'gamma': 3.693322190079529e-05, 'subsample': 0.7459710413523042, 'colsample_bytree': 0.9269297617036076, 'min_child_weight': 149.99883489324225, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:30,542] Trial 49 finished with value: 3849.4683103999996 and parameters: {'eta': 0.6132372631868321, 'max_depth': 1, 'gamma': 0.000968871383328273, 'subsample': 0.9199037068824423, 'colsample_bytree': 0.7680994070182107, 'min_child_weight': 0.0009615189967502903, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:33,058] Trial 50 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:34,848] Trial 51 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:36,644] Trial 52 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:38,540] Trial 53 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:41,762] Trial 54 finished with value: 3847.0289060000005 and parameters: {'eta': 0.14798963012797164, 'max_depth': 2, 'gamma': 0.04703637616936393, 'subsample': 0.9207542418442344, 'colsample_bytree': 0.9739740904783275, 'min_child_weight': 0.009243144500511178, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:43,368] Trial 55 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:47:45,675] Trial 56 finished with value: 3838.8463863999996 and parameters: {'eta': 0.2824885965147529, 'max_depth': 3, 'gamma': 0.0011033822642143494, 'subsample': 0.9984569455197705, 'colsample_bytree': 0.876599558428536, 'min_child_weight': 1.3129787718125934, 'booster': 'gbtree'}. Best is trial 26 with value: 3829.0803222000004. [I 2023-05-18 07:47:47,567] Trial 57 finished with value: 3822.0095216 and parameters: {'eta': 0.8681811287628631, 'max_depth': 2, 'gamma': 0.004886016286224786, 'subsample': 0.8325622187584861, 'colsample_bytree': 0.9650403913240058, 'min_child_weight': 0.11080028948229148, 'booster': 'gbtree'}. Best is trial 57 with value: 3822.0095216. [I 2023-05-18 07:47:49,708] Trial 58 finished with value: 3856.3364744 and parameters: {'eta': 0.4102174565945219, 'max_depth': 2, 'gamma': 0.0004958680281475151, 'subsample': 0.8269165311129012, 'colsample_bytree': 0.9699629896707002, 'min_child_weight': 0.1415545417174524, 'booster': 'gbtree'}. Best is trial 57 with value: 3822.0095216. [I 2023-05-18 07:47:51,754] Trial 59 finished with value: 3919.9992676 and parameters: {'eta': 0.7012110913187859, 'max_depth': 4, 'gamma': 0.00014851387809872122, 'subsample': 0.8833626117137573, 'colsample_bytree': 0.8109933547349369, 'min_child_weight': 0.6462615148288308, 'booster': 'gbtree'}. Best is trial 57 with value: 3822.0095216. [I 2023-05-18 07:47:53,860] Trial 60 finished with value: 3827.9895996 and parameters: {'eta': 0.9661691998626719, 'max_depth': 1, 'gamma': 0.002639866992189636, 'subsample': 0.8597026386266264, 'colsample_bytree': 0.8771454187830814, 'min_child_weight': 2.6993491856671614, 'booster': 'gbtree'}. Best is trial 57 with value: 3822.0095216. [I 2023-05-18 07:47:56,483] Trial 61 finished with value: 3861.7959960000007 and parameters: {'eta': 0.8032771463750054, 'max_depth': 1, 'gamma': 0.002422104568932393, 'subsample': 0.867224674315141, 'colsample_bytree': 0.8772066664619169, 'min_child_weight': 2.769767083232375, 'booster': 'gbtree'}. Best is trial 57 with value: 3822.0095216. [I 2023-05-18 07:47:58,941] Trial 62 finished with value: 3814.5947264 and parameters: {'eta': 0.8372379817174161, 'max_depth': 1, 'gamma': 0.00028874513637791523, 'subsample': 0.9414137540756694, 'colsample_bytree': 0.8397232179868871, 'min_child_weight': 25.011728801806715, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:01,319] Trial 63 finished with value: 3828.6137698 and parameters: {'eta': 0.9377903346110351, 'max_depth': 1, 'gamma': 0.0003410860805728561, 'subsample': 0.9219774200582992, 'colsample_bytree': 0.8425081471555455, 'min_child_weight': 14.254351860735044, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:03,541] Trial 64 finished with value: 3831.0185060000003 and parameters: {'eta': 0.9976764017707496, 'max_depth': 1, 'gamma': 0.0003132463504060857, 'subsample': 0.8445083608424314, 'colsample_bytree': 0.80030847753407, 'min_child_weight': 11.520296864306252, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:05,880] Trial 65 finished with value: 3836.6018554 and parameters: {'eta': 0.9116507656118813, 'max_depth': 1, 'gamma': 8.135451583091248e-05, 'subsample': 0.8413887410655331, 'colsample_bytree': 0.8391239297365872, 'min_child_weight': 21.140509902051534, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:07,874] Trial 66 finished with value: 3837.5195314 and parameters: {'eta': 0.984802355483032, 'max_depth': 1, 'gamma': 0.0002572105869798467, 'subsample': 0.8723436435552152, 'colsample_bytree': 0.8104726950642125, 'min_child_weight': 20.03207117409885, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:09,517] Trial 67 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:12,508] Trial 68 finished with value: 3856.5291994 and parameters: {'eta': 0.6004142764734423, 'max_depth': 1, 'gamma': 1.932850739274682e-05, 'subsample': 0.8212008232759541, 'colsample_bytree': 0.7515700983359808, 'min_child_weight': 72.35019898544266, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:14,063] Trial 69 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:16,937] Trial 70 finished with value: 3814.7536132 and parameters: {'eta': 0.7207256974086536, 'max_depth': 1, 'gamma': 5.487916080622071e-05, 'subsample': 0.9691163069579696, 'colsample_bytree': 0.8898076177284568, 'min_child_weight': 47.58140535182517, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:20,227] Trial 71 finished with value: 3815.1328124 and parameters: {'eta': 0.7191145999651261, 'max_depth': 1, 'gamma': 5.501179837779385e-05, 'subsample': 0.9687348262451118, 'colsample_bytree': 0.8361023202354649, 'min_child_weight': 50.15686594597055, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:23,643] Trial 72 finished with value: 3819.584179799999 and parameters: {'eta': 0.6743605603841583, 'max_depth': 1, 'gamma': 2.4698376617646188e-05, 'subsample': 0.9712625313372072, 'colsample_bytree': 0.8937603137973399, 'min_child_weight': 38.32838744532422, 'booster': 'gbtree'}. Best is trial 62 with value: 3814.5947264. [I 2023-05-18 07:48:26,791] Trial 73 finished with value: 3811.5691405999996 and parameters: {'eta': 0.6923029641345312, 'max_depth': 1, 'gamma': 6.680994226181273e-05, 'subsample': 0.9641015620172502, 'colsample_bytree': 0.9618834206880097, 'min_child_weight': 83.88316642031562, 'booster': 'gbtree'}. Best is trial 73 with value: 3811.5691405999996. [I 2023-05-18 07:48:29,778] Trial 74 finished with value: 3810.4062498 and parameters: {'eta': 0.6670024501000067, 'max_depth': 1, 'gamma': 5.407130073705652e-05, 'subsample': 0.9724119994325435, 'colsample_bytree': 0.9530416205131857, 'min_child_weight': 75.385285194199, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:48:32,896] Trial 75 finished with value: 3816.618408 and parameters: {'eta': 0.6389456860329116, 'max_depth': 1, 'gamma': 4.602796469455905e-05, 'subsample': 0.9692110747869698, 'colsample_bytree': 0.9718821444437167, 'min_child_weight': 104.68296712263627, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:48:34,440] Trial 76 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:36,023] Trial 77 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:40,750] Trial 78 finished with value: 3821.8752441999995 and parameters: {'eta': 0.5135369903839971, 'max_depth': 1, 'gamma': 7.516460980718512e-06, 'subsample': 0.9404715343979211, 'colsample_bytree': 0.9952934798868881, 'min_child_weight': 44.233153574764145, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:48:42,309] Trial 79 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:45,832] Trial 80 finished with value: 3813.6013183999994 and parameters: {'eta': 0.5535488775052105, 'max_depth': 1, 'gamma': 6.88311985688877e-06, 'subsample': 0.9727533295056421, 'colsample_bytree': 0.9002765070748767, 'min_child_weight': 44.260925972963356, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:48:50,047] Trial 81 finished with value: 3814.7735352000004 and parameters: {'eta': 0.5041597592847389, 'max_depth': 1, 'gamma': 6.096555445460076e-06, 'subsample': 0.9766473231953711, 'colsample_bytree': 0.897645408234489, 'min_child_weight': 41.05640094333438, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:48:51,548] Trial 82 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:53,030] Trial 83 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:48:55,042] Trial 84 pruned. Trial was pruned at iteration 26. [I 2023-05-18 07:48:58,075] Trial 85 finished with value: 3818.0372558000004 and parameters: {'eta': 0.69322159999975, 'max_depth': 1, 'gamma': 1.6289787441508883e-06, 'subsample': 0.9060092812630418, 'colsample_bytree': 0.9008919491104631, 'min_child_weight': 71.75083844566487, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:48:59,705] Trial 86 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:01,853] Trial 87 finished with value: 3831.3011719999995 and parameters: {'eta': 0.4050154846902294, 'max_depth': 2, 'gamma': 6.777450840896923e-05, 'subsample': 0.9528194852611723, 'colsample_bytree': 0.9307217955159146, 'min_child_weight': 150.75908313510752, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:49:05,286] Trial 88 finished with value: 3825.4711424 and parameters: {'eta': 0.19859627802325924, 'max_depth': 2, 'gamma': 2.0968323024850992e-06, 'subsample': 0.9040343129949975, 'colsample_bytree': 0.9028937766357449, 'min_child_weight': 61.91681704814041, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:49:07,519] Trial 89 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:09,297] Trial 90 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:12,189] Trial 91 finished with value: 3821.7953614 and parameters: {'eta': 0.7131661945097054, 'max_depth': 1, 'gamma': 2.586539205993633e-05, 'subsample': 0.979339102080119, 'colsample_bytree': 0.8886305177085121, 'min_child_weight': 37.336822971363745, 'booster': 'gbtree'}. Best is trial 74 with value: 3810.4062498. [I 2023-05-18 07:49:16,492] Trial 92 finished with value: 3809.2660156000006 and parameters: {'eta': 0.5510214210736533, 'max_depth': 1, 'gamma': 0.00010024714930333905, 'subsample': 0.9800505968689208, 'colsample_bytree': 0.8997613284062348, 'min_child_weight': 112.8703597457607, 'booster': 'gbtree'}. Best is trial 92 with value: 3809.2660156000006. [I 2023-05-18 07:49:18,125] Trial 93 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:22,080] Trial 94 finished with value: 3820.4379882000003 and parameters: {'eta': 0.5380410023835211, 'max_depth': 1, 'gamma': 1.3887893466882647e-05, 'subsample': 0.9303434031368178, 'colsample_bytree': 0.9377933141821073, 'min_child_weight': 77.2828649019065, 'booster': 'gbtree'}. Best is trial 92 with value: 3809.2660156000006. [I 2023-05-18 07:49:23,732] Trial 95 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:26,169] Trial 96 finished with value: 3821.2603517999996 and parameters: {'eta': 0.2766937939373754, 'max_depth': 2, 'gamma': 4.6186600831608094e-05, 'subsample': 0.9592239875071623, 'colsample_bytree': 0.8672068526532577, 'min_child_weight': 12.565387170401696, 'booster': 'gbtree'}. Best is trial 92 with value: 3809.2660156000006. [I 2023-05-18 07:49:27,765] Trial 97 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:29,602] Trial 98 finished with value: 3774.5420408 and parameters: {'eta': 0.7298897353706068, 'max_depth': 2, 'gamma': 5.90940257278992e-06, 'subsample': 0.9810129322454306, 'colsample_bytree': 0.9546244491014185, 'min_child_weight': 113.32324947486019, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:49:31,202] Trial 99 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:32,865] Trial 100 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:35,584] Trial 101 finished with value: 3810.3240235999997 and parameters: {'eta': 0.7718487607033017, 'max_depth': 1, 'gamma': 6.779717187957027e-05, 'subsample': 0.9623137922168719, 'colsample_bytree': 0.9414187092707894, 'min_child_weight': 53.716996172451644, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:49:37,175] Trial 102 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:40,086] Trial 103 finished with value: 3832.2347658 and parameters: {'eta': 0.8031475110715925, 'max_depth': 1, 'gamma': 7.263833317299155e-05, 'subsample': 0.9322171413557389, 'colsample_bytree': 0.943381596939942, 'min_child_weight': 52.61848848120044, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:49:41,707] Trial 104 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:44,842] Trial 105 finished with value: 3830.0519532 and parameters: {'eta': 0.5817540100747453, 'max_depth': 1, 'gamma': 1.334010344440588e-05, 'subsample': 0.959845530632627, 'colsample_bytree': 0.9315503831848274, 'min_child_weight': 10.22592944716146, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:49:48,209] Trial 106 finished with value: 3830.8282225999997 and parameters: {'eta': 0.21219365783189956, 'max_depth': 2, 'gamma': 5.9967864491764954e-05, 'subsample': 0.9223628411279112, 'colsample_bytree': 0.8506150189156021, 'min_child_weight': 5.393849906827747, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:49:50,968] Trial 107 finished with value: 3803.009228600001 and parameters: {'eta': 0.8492323095322724, 'max_depth': 1, 'gamma': 2.9882039495735512e-05, 'subsample': 0.9993436649628783, 'colsample_bytree': 0.9809496707548302, 'min_child_weight': 101.04754575819773, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:49:52,657] Trial 108 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:54,278] Trial 109 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:55,904] Trial 110 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:49:59,023] Trial 111 finished with value: 3821.8586916 and parameters: {'eta': 0.5937697036315519, 'max_depth': 1, 'gamma': 9.51429420290451e-06, 'subsample': 0.9591715417066458, 'colsample_bytree': 0.9616703050636054, 'min_child_weight': 107.00546442048125, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:01,570] Trial 112 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:03,888] Trial 113 finished with value: 3818.9864258000002 and parameters: {'eta': 0.9527537264203723, 'max_depth': 1, 'gamma': 3.7041833026381057e-05, 'subsample': 0.9844320061199026, 'colsample_bytree': 0.9444439576108997, 'min_child_weight': 55.07030497488106, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:05,510] Trial 114 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:07,891] Trial 115 pruned. Trial was pruned at iteration 32. [I 2023-05-18 07:50:10,135] Trial 116 pruned. Trial was pruned at iteration 30. [I 2023-05-18 07:50:11,914] Trial 117 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:13,521] Trial 118 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:15,941] Trial 119 finished with value: 3808.8892578 and parameters: {'eta': 0.36547057708622827, 'max_depth': 2, 'gamma': 6.263578968143105e-06, 'subsample': 0.9967934859398755, 'colsample_bytree': 0.8866591310904918, 'min_child_weight': 87.30066165706317, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:18,278] Trial 120 finished with value: 3821.8129882000003 and parameters: {'eta': 0.34736287133647337, 'max_depth': 2, 'gamma': 6.270771305398735e-06, 'subsample': 0.9997602118343778, 'colsample_bytree': 0.8887804369426145, 'min_child_weight': 7.765826538371717, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:22,034] Trial 121 finished with value: 3817.8001954 and parameters: {'eta': 0.5573940101797304, 'max_depth': 1, 'gamma': 1.2280301021802346e-05, 'subsample': 0.9597696714844648, 'colsample_bytree': 0.8381778279099037, 'min_child_weight': 87.73133311900422, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:24,034] Trial 122 finished with value: 3825.6759278 and parameters: {'eta': 0.9754591040124816, 'max_depth': 2, 'gamma': 1.725098721010096e-05, 'subsample': 0.983028850116411, 'colsample_bytree': 0.9129181871536, 'min_child_weight': 46.815207346306636, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:27,631] Trial 123 finished with value: 3838.4399903999997 and parameters: {'eta': 0.6922537598378491, 'max_depth': 1, 'gamma': 3.7459906060652307e-06, 'subsample': 0.946501386074116, 'colsample_bytree': 0.9535455968408748, 'min_child_weight': 172.13461069113785, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:29,879] Trial 124 finished with value: 3822.1919436 and parameters: {'eta': 0.442443892948403, 'max_depth': 2, 'gamma': 3.2939600063502737e-05, 'subsample': 0.9718884662560698, 'colsample_bytree': 0.895395451540211, 'min_child_weight': 19.924228925877532, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:31,965] Trial 125 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:33,519] Trial 126 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:35,121] Trial 127 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:36,814] Trial 128 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:38,781] Trial 129 finished with value: 3814.9963866 and parameters: {'eta': 0.9911847792473212, 'max_depth': 1, 'gamma': 3.0445251544641683e-06, 'subsample': 0.9385577382639998, 'colsample_bytree': 0.9863871508409975, 'min_child_weight': 33.61514840810867, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:40,788] Trial 130 finished with value: 3829.148242 and parameters: {'eta': 0.9598826925714357, 'max_depth': 1, 'gamma': 2.718776220436818e-06, 'subsample': 0.8804044400036642, 'colsample_bytree': 0.9876640045395425, 'min_child_weight': 32.86727833857164, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:44,555] Trial 131 finished with value: 3824.6124023999996 and parameters: {'eta': 0.6321971346900754, 'max_depth': 1, 'gamma': 4.7376075987343005e-06, 'subsample': 0.940453482542223, 'colsample_bytree': 0.9730283820286341, 'min_child_weight': 60.65014674597619, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:46,171] Trial 132 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:48,729] Trial 133 finished with value: 3808.1213866 and parameters: {'eta': 0.9846834048493902, 'max_depth': 1, 'gamma': 6.548715361620968e-06, 'subsample': 0.9032397792510389, 'colsample_bytree': 0.9655530084261492, 'min_child_weight': 95.65622862788184, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:51,285] Trial 134 pruned. Trial was pruned at iteration 35. [I 2023-05-18 07:50:55,154] Trial 135 finished with value: 3820.4541016000003 and parameters: {'eta': 0.5154829547841965, 'max_depth': 1, 'gamma': 1.4168135493404023e-06, 'subsample': 0.9493734664672733, 'colsample_bytree': 0.938644245878327, 'min_child_weight': 44.386828332893046, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:50:56,851] Trial 136 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:50:59,957] Trial 137 finished with value: 3822.2581055999995 and parameters: {'eta': 0.7098029306875564, 'max_depth': 1, 'gamma': 1.0223813501533075e-06, 'subsample': 0.9303213945720759, 'colsample_bytree': 0.898431284195485, 'min_child_weight': 77.79285405200022, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:01,608] Trial 138 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:03,746] Trial 139 finished with value: 3846.1854004 and parameters: {'eta': 0.9995932570681645, 'max_depth': 2, 'gamma': 1.618243103561258e-05, 'subsample': 0.9611545074013801, 'colsample_bytree': 0.8281551310034145, 'min_child_weight': 5.129791966909028, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:05,376] Trial 140 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:08,889] Trial 141 finished with value: 3812.7474119999997 and parameters: {'eta': 0.6371967237132894, 'max_depth': 1, 'gamma': 3.8310871381617684e-05, 'subsample': 0.9719745513946128, 'colsample_bytree': 0.9604260369361485, 'min_child_weight': 117.19515830350288, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:11,743] Trial 142 finished with value: 3811.3002928 and parameters: {'eta': 0.6309077098538532, 'max_depth': 1, 'gamma': 2.86402313808792e-05, 'subsample': 0.9858031307340481, 'colsample_bytree': 0.9831915685839775, 'min_child_weight': 109.76809286122304, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:14,979] Trial 143 finished with value: 3814.2884766 and parameters: {'eta': 0.6133949901984753, 'max_depth': 1, 'gamma': 1.1292763609459167e-05, 'subsample': 0.9863511199676055, 'colsample_bytree': 0.982393432197663, 'min_child_weight': 169.00058257285153, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:16,614] Trial 144 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:18,230] Trial 145 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:21,160] Trial 146 finished with value: 3810.9458986 and parameters: {'eta': 0.6365507426711319, 'max_depth': 1, 'gamma': 1.972464318493693e-05, 'subsample': 0.977563199370943, 'colsample_bytree': 0.9485916519474965, 'min_child_weight': 93.03818224163311, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:22,951] Trial 147 finished with value: 3794.3759274000004 and parameters: {'eta': 0.776548708790188, 'max_depth': 2, 'gamma': 1.7904611333727794e-05, 'subsample': 0.9556596445443041, 'colsample_bytree': 0.9785218770569155, 'min_child_weight': 100.98447929973676, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:25,592] Trial 148 finished with value: 3807.0750978000005 and parameters: {'eta': 0.23949880724912717, 'max_depth': 3, 'gamma': 2.627988079671551e-05, 'subsample': 0.9509348824899592, 'colsample_bytree': 0.9819457308451747, 'min_child_weight': 104.66984799797362, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:28,020] Trial 149 finished with value: 3807.5563964000003 and parameters: {'eta': 0.2608636700790281, 'max_depth': 3, 'gamma': 1.7060960282581178e-05, 'subsample': 0.9543292089116623, 'colsample_bytree': 0.9756186332904099, 'min_child_weight': 98.20874708252092, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:30,434] Trial 150 finished with value: 3810.3840334 and parameters: {'eta': 0.26170646595541597, 'max_depth': 3, 'gamma': 3.1762105115441814e-05, 'subsample': 0.9606862709256485, 'colsample_bytree': 0.9992932367548514, 'min_child_weight': 112.89551751206567, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:32,984] Trial 151 finished with value: 3822.0263672 and parameters: {'eta': 0.232465383295059, 'max_depth': 3, 'gamma': 2.1110664051871592e-05, 'subsample': 0.9578777856191356, 'colsample_bytree': 0.9996561399728254, 'min_child_weight': 85.56311803853764, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:35,391] Trial 152 finished with value: 3812.938867 and parameters: {'eta': 0.2561775862620712, 'max_depth': 3, 'gamma': 3.375251794241245e-05, 'subsample': 0.9535275313215685, 'colsample_bytree': 0.9725914076005835, 'min_child_weight': 92.29457798499469, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:37,594] Trial 153 finished with value: 3805.7425292 and parameters: {'eta': 0.2649267070447872, 'max_depth': 3, 'gamma': 3.4877974684119234e-05, 'subsample': 0.9518659489318481, 'colsample_bytree': 0.9686720412115923, 'min_child_weight': 115.87217028167655, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:39,316] Trial 154 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:42,484] Trial 155 finished with value: 3823.0128418 and parameters: {'eta': 0.1330426770089759, 'max_depth': 4, 'gamma': 1.8492982857986843e-05, 'subsample': 0.9177443763807207, 'colsample_bytree': 0.9848808284571088, 'min_child_weight': 145.94854240326865, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:44,049] Trial 156 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:46,294] Trial 157 finished with value: 3805.0477052 and parameters: {'eta': 0.28123735065147915, 'max_depth': 3, 'gamma': 2.564674433270389e-05, 'subsample': 0.9485484558781205, 'colsample_bytree': 0.978510523793119, 'min_child_weight': 116.54657144392752, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:47,813] Trial 158 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:49,488] Trial 159 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:51:52,319] Trial 160 finished with value: 3822.3052734 and parameters: {'eta': 0.12649409196434724, 'max_depth': 3, 'gamma': 9.782042778605859e-05, 'subsample': 0.95106950614898, 'colsample_bytree': 0.9442930204999305, 'min_child_weight': 59.869106005247346, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:54,373] Trial 161 finished with value: 3845.9018552 and parameters: {'eta': 0.3800226594494198, 'max_depth': 4, 'gamma': 3.975008662419394e-05, 'subsample': 0.9789609610275153, 'colsample_bytree': 0.9688158112041385, 'min_child_weight': 111.8148900850666, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:56,592] Trial 162 finished with value: 3811.6542482000004 and parameters: {'eta': 0.2769237171802851, 'max_depth': 3, 'gamma': 2.8341751670114232e-05, 'subsample': 0.9667427053340933, 'colsample_bytree': 0.9537240680782355, 'min_child_weight': 120.33814859504652, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:51:58,579] Trial 163 finished with value: 3840.2083008 and parameters: {'eta': 0.27786993789504855, 'max_depth': 3, 'gamma': 9.896138979482861e-06, 'subsample': 0.6529505631008345, 'colsample_bytree': 0.9819169641345262, 'min_child_weight': 68.08318488011541, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:00,344] Trial 164 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:02,573] Trial 165 finished with value: 3810.28667 and parameters: {'eta': 0.3751184451666579, 'max_depth': 3, 'gamma': 6.040003434752994e-05, 'subsample': 0.9456364406643001, 'colsample_bytree': 0.9468841081419181, 'min_child_weight': 92.98658165177622, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:04,744] Trial 166 finished with value: 3843.5580566 and parameters: {'eta': 0.39000014865199906, 'max_depth': 4, 'gamma': 7.150026913697077e-05, 'subsample': 0.9123762052214709, 'colsample_bytree': 0.9369031080830486, 'min_child_weight': 73.74589388677812, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:06,549] Trial 167 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:08,198] Trial 168 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:10,084] Trial 169 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:12,011] Trial 170 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:14,533] Trial 171 finished with value: 3813.0750976000004 and parameters: {'eta': 0.22789542293463796, 'max_depth': 3, 'gamma': 2.857595862872204e-05, 'subsample': 0.9654073106707044, 'colsample_bytree': 0.9496532022875604, 'min_child_weight': 123.59967619820839, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:16,915] Trial 172 finished with value: 3822.8616697999996 and parameters: {'eta': 0.268351069639738, 'max_depth': 3, 'gamma': 2.740316969916391e-05, 'subsample': 0.9471396392039244, 'colsample_bytree': 0.952502751249876, 'min_child_weight': 167.36884772161838, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:19,079] Trial 173 finished with value: 3823.064258 and parameters: {'eta': 0.352671402567924, 'max_depth': 3, 'gamma': 4.478947809489183e-05, 'subsample': 0.9744096151892834, 'colsample_bytree': 0.9743874455316274, 'min_child_weight': 92.88448195007508, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:22,344] Trial 174 finished with value: 3824.6041504 and parameters: {'eta': 0.11676357710330675, 'max_depth': 3, 'gamma': 1.2564483107920085e-05, 'subsample': 0.933877468527901, 'colsample_bytree': 0.9637222257044059, 'min_child_weight': 36.86040384513778, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:24,207] Trial 175 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:26,022] Trial 176 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:27,989] Trial 177 finished with value: 3839.7148435999998 and parameters: {'eta': 0.7853527771950577, 'max_depth': 3, 'gamma': 4.779608381045362e-06, 'subsample': 0.985101312529229, 'colsample_bytree': 0.9223713680281688, 'min_child_weight': 64.34716834093042, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:30,348] Trial 178 finished with value: 3812.7130859999997 and parameters: {'eta': 0.41623311189329876, 'max_depth': 2, 'gamma': 2.54511606232066e-05, 'subsample': 0.9443138130381025, 'colsample_bytree': 0.966592471703635, 'min_child_weight': 134.2816476886315, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:32,864] Trial 179 finished with value: 3829.3199707999993 and parameters: {'eta': 0.22845276838491746, 'max_depth': 3, 'gamma': 0.00010665640776400072, 'subsample': 0.9982076741638872, 'colsample_bytree': 0.9497219258762297, 'min_child_weight': 34.14735216559474, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:34,697] Trial 180 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:37,101] Trial 181 finished with value: 3812.0933594000003 and parameters: {'eta': 0.40490413950436677, 'max_depth': 2, 'gamma': 6.033711975912926e-05, 'subsample': 0.9483147094963252, 'colsample_bytree': 0.9717781264084061, 'min_child_weight': 110.5810611640554, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:39,659] Trial 182 finished with value: 3815.8682128 and parameters: {'eta': 0.290918302223157, 'max_depth': 2, 'gamma': 6.981962034684696e-05, 'subsample': 0.9570487224021158, 'colsample_bytree': 0.9845270090011462, 'min_child_weight': 89.76981750228003, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:43,381] Trial 183 finished with value: 3830.3963868 and parameters: {'eta': 0.15144414316977975, 'max_depth': 2, 'gamma': 3.567490271527175e-05, 'subsample': 0.9261512189611842, 'colsample_bytree': 0.9990131737031909, 'min_child_weight': 142.0686641806102, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:45,567] Trial 184 finished with value: 3802.6825685999997 and parameters: {'eta': 0.4800271142294348, 'max_depth': 2, 'gamma': 0.00013924599701810352, 'subsample': 0.9832917884257444, 'colsample_bytree': 0.9619466367780829, 'min_child_weight': 54.78356986131558, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:47,365] Trial 185 finished with value: 3896.9789063999997 and parameters: {'eta': 0.6820077559084227, 'max_depth': 3, 'gamma': 0.00013721916186962808, 'subsample': 0.4986083509217256, 'colsample_bytree': 0.9555747923208257, 'min_child_weight': 29.070138695094492, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:49,554] Trial 186 finished with value: 3813.3081055999996 and parameters: {'eta': 0.5034544640375731, 'max_depth': 2, 'gamma': 0.0001964462618707981, 'subsample': 0.9791417268253216, 'colsample_bytree': 0.9286179752096622, 'min_child_weight': 55.135630589369256, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:52:51,491] Trial 187 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:53,575] Trial 188 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:55,262] Trial 189 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:57,014] Trial 190 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:52:59,241] Trial 191 finished with value: 3846.0243164 and parameters: {'eta': 0.4132881013136441, 'max_depth': 2, 'gamma': 6.229762858283418e-05, 'subsample': 0.7305577030290495, 'colsample_bytree': 0.9677358134729513, 'min_child_weight': 114.43921054729718, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:00,972] Trial 192 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:53:02,859] Trial 193 finished with value: 3810.9716797999995 and parameters: {'eta': 0.5351300864009915, 'max_depth': 2, 'gamma': 5.503678187799568e-05, 'subsample': 0.9662947738522023, 'colsample_bytree': 0.9494020234712751, 'min_child_weight': 0.38020673165719754, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:04,844] Trial 194 finished with value: 3852.4627442000005 and parameters: {'eta': 0.5456941303856299, 'max_depth': 3, 'gamma': 1.5713625789344887e-05, 'subsample': 0.969084961502616, 'colsample_bytree': 0.9493654385273714, 'min_child_weight': 0.049688708473574573, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:06,816] Trial 195 finished with value: 3852.7620118 and parameters: {'eta': 0.8346862686455253, 'max_depth': 3, 'gamma': 2.5668611457969573e-05, 'subsample': 0.9829114627758831, 'colsample_bytree': 0.9325644258954432, 'min_child_weight': 13.006353489010117, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:08,578] Trial 196 pruned. Trial was pruned at iteration 20. [I 2023-05-18 07:53:10,564] Trial 197 finished with value: 3856.0860352000004 and parameters: {'eta': 0.6420443042970102, 'max_depth': 3, 'gamma': 4.554202321196429e-05, 'subsample': 0.9988640311180044, 'colsample_bytree': 0.9541703038834692, 'min_child_weight': 2.037940155053925, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:12,581] Trial 198 finished with value: 3805.8950683999997 and parameters: {'eta': 0.531536032810686, 'max_depth': 2, 'gamma': 8.973947749581775e-05, 'subsample': 0.9066588671603624, 'colsample_bytree': 0.9433936102014868, 'min_child_weight': 51.24779785256243, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:14,614] Trial 199 finished with value: 3793.5475100000003 and parameters: {'eta': 0.5217717709304456, 'max_depth': 2, 'gamma': 9.281469065300418e-05, 'subsample': 0.8991324034607601, 'colsample_bytree': 0.9873910275825201, 'min_child_weight': 53.24456653904132, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:16,648] Trial 200 finished with value: 3830.9330078 and parameters: {'eta': 0.4933405586110586, 'max_depth': 2, 'gamma': 0.00035288715055804544, 'subsample': 0.9025382013034281, 'colsample_bytree': 0.9999864316195061, 'min_child_weight': 0.7979464505172438, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:18,525] Trial 201 finished with value: 3802.4214356 and parameters: {'eta': 0.6870330799647111, 'max_depth': 2, 'gamma': 0.00012392298583122083, 'subsample': 0.8520194238425267, 'colsample_bytree': 0.9859888561097087, 'min_child_weight': 48.236888077324046, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:20,381] Trial 202 finished with value: 3793.411035 and parameters: {'eta': 0.7405099460213749, 'max_depth': 2, 'gamma': 0.00016758743761623388, 'subsample': 0.8852039828180251, 'colsample_bytree': 0.985121567069678, 'min_child_weight': 50.14691757811314, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:22,289] Trial 203 finished with value: 3801.6028321999997 and parameters: {'eta': 0.7943682732102344, 'max_depth': 2, 'gamma': 0.0002090331992660469, 'subsample': 0.8541491251629988, 'colsample_bytree': 0.9865296026333953, 'min_child_weight': 19.090764518002857, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:24,223] Trial 204 finished with value: 3816.8828614000004 and parameters: {'eta': 0.8228384727946414, 'max_depth': 2, 'gamma': 0.0002252174167955754, 'subsample': 0.8860636365083399, 'colsample_bytree': 0.9863959151909636, 'min_child_weight': 20.96371868133504, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:26,111] Trial 205 finished with value: 3838.0657714 and parameters: {'eta': 0.9950160973754588, 'max_depth': 2, 'gamma': 0.0001428217792969583, 'subsample': 0.8574921539547149, 'colsample_bytree': 0.9998616908778114, 'min_child_weight': 40.63636756801337, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:27,985] Trial 206 finished with value: 3815.1658204 and parameters: {'eta': 0.6995368597417508, 'max_depth': 2, 'gamma': 0.00021721490145040068, 'subsample': 0.8692277400700467, 'colsample_bytree': 0.973333799929004, 'min_child_weight': 45.074040259685994, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:29,854] Trial 207 finished with value: 3830.5267578 and parameters: {'eta': 0.8215845878717334, 'max_depth': 2, 'gamma': 8.796811422242345e-05, 'subsample': 0.8403110863437612, 'colsample_bytree': 0.9843956099596798, 'min_child_weight': 23.050342102669166, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:32,137] Trial 208 finished with value: 3815.612158000001 and parameters: {'eta': 0.4813958217135846, 'max_depth': 2, 'gamma': 0.0006079995887440853, 'subsample': 0.8553745596594611, 'colsample_bytree': 0.9647217417785441, 'min_child_weight': 63.09226010047009, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:34,503] Trial 209 finished with value: 3820.076416 and parameters: {'eta': 0.36480329168547393, 'max_depth': 2, 'gamma': 0.00031095508038011366, 'subsample': 0.8840949479266137, 'colsample_bytree': 0.940509359702044, 'min_child_weight': 68.85602491979893, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:36,513] Trial 210 finished with value: 3791.3450193999997 and parameters: {'eta': 0.6060896179305071, 'max_depth': 2, 'gamma': 0.00010120099706101817, 'subsample': 0.9078297548509381, 'colsample_bytree': 0.9736253557728483, 'min_child_weight': 39.72866438680023, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:38,417] Trial 211 finished with value: 3789.574414 and parameters: {'eta': 0.6159054703673289, 'max_depth': 2, 'gamma': 0.00014314722962526153, 'subsample': 0.8994551284230697, 'colsample_bytree': 0.9719003037891568, 'min_child_weight': 39.637047164191046, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:40,348] Trial 212 finished with value: 3794.5562987999997 and parameters: {'eta': 0.7386069232580048, 'max_depth': 2, 'gamma': 0.00011586186005215479, 'subsample': 0.8965592778727821, 'colsample_bytree': 0.9820553097575544, 'min_child_weight': 40.280623819549604, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:42,143] Trial 213 finished with value: 3799.3002441999997 and parameters: {'eta': 0.776652997561857, 'max_depth': 2, 'gamma': 0.00012103653686842905, 'subsample': 0.9012822257162024, 'colsample_bytree': 0.9851108379681089, 'min_child_weight': 13.893469148350722, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:44,002] Trial 214 finished with value: 3812.255713 and parameters: {'eta': 0.8122786968575143, 'max_depth': 2, 'gamma': 0.00016748425543860394, 'subsample': 0.8971203594388379, 'colsample_bytree': 0.9743061841514913, 'min_child_weight': 11.469942451873193, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:45,834] Trial 215 finished with value: 3781.0590822 and parameters: {'eta': 0.593273255358362, 'max_depth': 2, 'gamma': 0.00011929433079444746, 'subsample': 0.878045606995328, 'colsample_bytree': 0.9804706316592839, 'min_child_weight': 19.845236752721032, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:47,719] Trial 216 finished with value: 3780.9666989999996 and parameters: {'eta': 0.5946296789589034, 'max_depth': 2, 'gamma': 0.0004694174638252085, 'subsample': 0.8799156993004688, 'colsample_bytree': 0.9776416191460644, 'min_child_weight': 19.521984425864964, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:49,755] Trial 217 finished with value: 3807.1967773999995 and parameters: {'eta': 0.5619984832710976, 'max_depth': 2, 'gamma': 0.0003542145075235604, 'subsample': 0.879252262442878, 'colsample_bytree': 0.9850820869562218, 'min_child_weight': 18.036680251017064, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:51,630] Trial 218 finished with value: 3833.3241697999997 and parameters: {'eta': 0.9897172452671651, 'max_depth': 2, 'gamma': 0.0005451250953868507, 'subsample': 0.8699251431132029, 'colsample_bytree': 0.9816930848914066, 'min_child_weight': 8.149570910699383, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:53,580] Trial 219 finished with value: 3786.8766112 and parameters: {'eta': 0.5690062154267753, 'max_depth': 2, 'gamma': 0.0003768910345040364, 'subsample': 0.882161522310125, 'colsample_bytree': 0.9703428975414836, 'min_child_weight': 15.568973878994292, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:55,495] Trial 220 finished with value: 3794.5060546 and parameters: {'eta': 0.6607089049801612, 'max_depth': 2, 'gamma': 0.00035772687767397484, 'subsample': 0.8893933604760615, 'colsample_bytree': 0.9875931826241249, 'min_child_weight': 16.042406789896123, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:57,426] Trial 221 finished with value: 3788.4975586 and parameters: {'eta': 0.6043744503844098, 'max_depth': 2, 'gamma': 0.00029254951861309704, 'subsample': 0.8838343888542909, 'colsample_bytree': 0.9868667361749667, 'min_child_weight': 13.747566791650096, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:53:59,353] Trial 222 finished with value: 3787.6118651999996 and parameters: {'eta': 0.6069662219280358, 'max_depth': 2, 'gamma': 0.00037964985072722827, 'subsample': 0.8808648224104689, 'colsample_bytree': 0.9882392481364234, 'min_child_weight': 13.665614995946294, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:01,241] Trial 223 finished with value: 3781.5255857999996 and parameters: {'eta': 0.5999371146107336, 'max_depth': 2, 'gamma': 0.0003191494868894334, 'subsample': 0.8771656259951647, 'colsample_bytree': 0.9866303935996313, 'min_child_weight': 14.290058908387005, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:03,136] Trial 224 finished with value: 3811.2753416000005 and parameters: {'eta': 0.6732119039605765, 'max_depth': 2, 'gamma': 0.0007366346882825858, 'subsample': 0.8629772226634863, 'colsample_bytree': 0.9901477165096595, 'min_child_weight': 8.025587698598532, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:05,096] Trial 225 finished with value: 3785.7984376 and parameters: {'eta': 0.5881804415060716, 'max_depth': 2, 'gamma': 0.0002940733159900773, 'subsample': 0.885248650430592, 'colsample_bytree': 0.9988541979116115, 'min_child_weight': 12.56600577848275, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:07,084] Trial 226 finished with value: 3787.5686522000005 and parameters: {'eta': 0.6033529943211481, 'max_depth': 2, 'gamma': 0.00047622270181190015, 'subsample': 0.8863982737626793, 'colsample_bytree': 0.9981895568175849, 'min_child_weight': 4.223751115263713, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:09,003] Trial 227 finished with value: 3803.6729006 and parameters: {'eta': 0.6800551333513979, 'max_depth': 2, 'gamma': 0.0004379952606206897, 'subsample': 0.8504875457722848, 'colsample_bytree': 0.9971215685872574, 'min_child_weight': 14.981501559120252, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:10,900] Trial 228 finished with value: 3814.228320200001 and parameters: {'eta': 0.7013170503701218, 'max_depth': 2, 'gamma': 0.0004105001421643814, 'subsample': 0.8553530135132122, 'colsample_bytree': 0.9988865020073269, 'min_child_weight': 4.89861523842058, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:12,819] Trial 229 finished with value: 3787.4480468 and parameters: {'eta': 0.6244705591340458, 'max_depth': 2, 'gamma': 0.001013357166343079, 'subsample': 0.8839034520002744, 'colsample_bytree': 0.9993219587187469, 'min_child_weight': 13.73676350457217, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:14,711] Trial 230 finished with value: 3798.0520508 and parameters: {'eta': 0.7733576763230564, 'max_depth': 2, 'gamma': 0.0012367803108188865, 'subsample': 0.8855371961956392, 'colsample_bytree': 0.9988103492452797, 'min_child_weight': 12.761190655410978, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:16,690] Trial 231 finished with value: 3800.523291 and parameters: {'eta': 0.7881147048202229, 'max_depth': 2, 'gamma': 0.0010081078889777183, 'subsample': 0.8859341604860376, 'colsample_bytree': 0.999604383375097, 'min_child_weight': 14.608941675434805, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:18,556] Trial 232 finished with value: 3805.677246 and parameters: {'eta': 0.799604749170847, 'max_depth': 2, 'gamma': 0.0010599755438954216, 'subsample': 0.8853421381430762, 'colsample_bytree': 0.9998478285621232, 'min_child_weight': 9.780133588729324, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:20,472] Trial 233 finished with value: 3791.0681640000003 and parameters: {'eta': 0.5745100012829601, 'max_depth': 2, 'gamma': 0.0008961924721081646, 'subsample': 0.8789326180387456, 'colsample_bytree': 0.9998195815545119, 'min_child_weight': 13.628943421737363, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:22,346] Trial 234 finished with value: 3779.7146484000004 and parameters: {'eta': 0.6189438157950579, 'max_depth': 2, 'gamma': 0.0008400386789556842, 'subsample': 0.8753458617613022, 'colsample_bytree': 0.9992486075815675, 'min_child_weight': 3.9456980334026284, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:24,249] Trial 235 finished with value: 3790.058545 and parameters: {'eta': 0.583038489957894, 'max_depth': 2, 'gamma': 0.000944666149650731, 'subsample': 0.8726030270575703, 'colsample_bytree': 0.9993520234834433, 'min_child_weight': 3.842438593471165, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:26,294] Trial 236 finished with value: 3788.9561031999997 and parameters: {'eta': 0.5857423245568127, 'max_depth': 2, 'gamma': 0.001404891035707475, 'subsample': 0.8758458319944955, 'colsample_bytree': 0.9997219240059572, 'min_child_weight': 4.818120585418077, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:28,223] Trial 237 finished with value: 3792.697412 and parameters: {'eta': 0.5742117874691034, 'max_depth': 2, 'gamma': 0.0014682736733667929, 'subsample': 0.8780956658353654, 'colsample_bytree': 0.996464624160336, 'min_child_weight': 3.8910855508813817, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:30,158] Trial 238 finished with value: 3798.4584472 and parameters: {'eta': 0.5571274702967427, 'max_depth': 2, 'gamma': 0.0015372305606408678, 'subsample': 0.8740756269061759, 'colsample_bytree': 0.9982772082824873, 'min_child_weight': 5.392911667738512, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:32,308] Trial 239 finished with value: 3789.412158 and parameters: {'eta': 0.5715579993927116, 'max_depth': 2, 'gamma': 0.0012496524295113093, 'subsample': 0.8753167768697987, 'colsample_bytree': 0.9997268430847559, 'min_child_weight': 3.5703091310182953, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:34,566] Trial 240 finished with value: 3805.9568846 and parameters: {'eta': 0.5693131512084756, 'max_depth': 2, 'gamma': 0.0015947740193316797, 'subsample': 0.8733066506086256, 'colsample_bytree': 0.7733171627810891, 'min_child_weight': 3.45213452908631, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. [I 2023-05-18 07:54:36,641] Trial 241 finished with value: 3787.8309082000005 and parameters: {'eta': 0.5888633689954109, 'max_depth': 2, 'gamma': 0.0014576060519186026, 'subsample': 0.8733724615048085, 'colsample_bytree': 0.9971962525237583, 'min_child_weight': 4.7799770425763635, 'booster': 'gbtree'}. Best is trial 98 with value: 3774.5420408. Hyper-Parameter Optimization successfully finished. Number of finished trials: 242 Best trial: Value: 3774.5420408 Params: eta: 0.7298897353706068 max_depth: 2 gamma: 5.90940257278992e-06 subsample: 0.9810129322454306 colsample_bytree: 0.9546244491014185 min_child_weight: 113.32324947486019 booster: gbtree opt_rounds: 3
Model Training¶
In [5]:
Copied!
np.random.seed(123)
opt_params = opt_param.copy()
n_rounds = opt_params["opt_rounds"]
del opt_params["opt_rounds"]
# Train Model with optimized hyperparameters
xgblss.train(opt_params,
dtrain,
num_boost_round=n_rounds
)
np.random.seed(123)
opt_params = opt_param.copy()
n_rounds = opt_params["opt_rounds"]
del opt_params["opt_rounds"]
# Train Model with optimized hyperparameters
xgblss.train(opt_params,
dtrain,
num_boost_round=n_rounds
)
Out[5]:
<xgboost.core.Booster at 0x214ede1d9d0>
Prediction¶
In [6]:
Copied!
# Predicted expectiles
pred_expectile = xgblss.predict(dtest, pred_type="expectiles")
# Predicted expectiles
pred_expectile = xgblss.predict(dtest, pred_type="expectiles")
In [7]:
Copied!
pred_expectile.head()
pred_expectile.head()
Out[7]:
| expectile_0.05 | expectile_0.95 | |
|---|---|---|
| 0 | 6.623859 | 13.174365 |
| 1 | 6.623859 | 13.190308 |
| 2 | 9.022786 | 11.075418 |
| 3 | 4.436906 | 14.815821 |
| 4 | 6.623859 | 13.414122 |
SHAP Interpretability¶
In [8]:
Copied!
# Partial Dependence Plot of how x acts on selected expectile
xgblss.expectile_plot(X_test,
expectile="expectile_0.95",
feature="x_true",
plot_type="Partial_Dependence")
# Partial Dependence Plot of how x acts on selected expectile
xgblss.expectile_plot(X_test,
expectile="expectile_0.95",
feature="x_true",
plot_type="Partial_Dependence")
C:\Users\maerzale\.virtualenvs\XGBoostLSS-vIPRRz-M\lib\site-packages\numpy\lib\function_base.py:2854: RuntimeWarning: invalid value encountered in divide C:\Users\maerzale\.virtualenvs\XGBoostLSS-vIPRRz-M\lib\site-packages\numpy\lib\function_base.py:2855: RuntimeWarning: invalid value encountered in divide
In [9]:
Copied!
# Partial Dependence Plot of how x acts on selected expectile
xgblss.expectile_plot(X_test,
expectile="expectile_0.05",
feature="x_true",
plot_type="Partial_Dependence")
# Partial Dependence Plot of how x acts on selected expectile
xgblss.expectile_plot(X_test,
expectile="expectile_0.05",
feature="x_true",
plot_type="Partial_Dependence")
In [10]:
Copied!
# Global Feature Importance of selected expectile
xgblss.expectile_plot(X_test,
expectile="expectile_0.95",
plot_type="Feature_Importance")
# Global Feature Importance of selected expectile
xgblss.expectile_plot(X_test,
expectile="expectile_0.95",
plot_type="Feature_Importance")
Plot of Actual vs. Predicted Expectiles¶
In [11]:
Copied!
np.random.seed(123)
###
# Actual Expectiles
###
y_loc = np.array([10])
y_scale = np.array([1 + 4*((0.3 < test["x_true"].values) & (test["x_true"].values < 0.5)) + 2*(test["x_true"].values > 0.7)])
tau_lower = np.array([xgblss.dist.tau[0]])
tau_upper = np.array([xgblss.dist.tau[1]])
# Calculates exact expectiles assuming a Normal distribution
expectile_lb = expectile_norm(tau=tau_lower,
m=y_loc,
sd=y_scale).reshape(-1,)
expectile_ub = expectile_norm(tau=tau_upper,
m=y_loc,
sd=y_scale).reshape(-1,)
test["expect"] = np.where(test["y"].values < expectile_lb, 0, np.where(test["y"].values < expectile_ub, 1, 2))
test["alpha"] = np.where(test["y"].values <= expectile_lb, 1, np.where(test["y"].values >= expectile_ub, 1, 0))
df_expectiles = test[test["alpha"] == 1]
# Lower Bound
yl = list(set(expectile_lb))
yl.sort()
yl = [yl[2],yl[0],yl[2],yl[1],yl[1]]
sfunl = pd.DataFrame({"x_true":[0, 0.3, 0.5, 0.7, 1],
"y":yl})
# Upper Bound
yu = list(set(expectile_ub))
yu.sort()
yu = [yu[0],yu[2],yu[0],yu[1],yu[1]]
sfunu = pd.DataFrame({"x_true":[0, 0.3, 0.5, 0.7, 1],
"y":yu})
###
# Forecasted Expectiles
###
test["lb"] = pred_expectile.iloc[:,0]
test["ub"] = pred_expectile.iloc[:,1]
###
# Plot
###
(ggplot(test,
aes("x_true",
"y")) +
geom_point(alpha = 0.2, color = "black", size = 2) +
theme_bw(base_size=15) +
theme(legend_position="bottom",
plot_title = element_text(hjust = 0.5)) +
labs(title = "XGBoostLSS Expectile Regression - Simulated Data Example") +
geom_line(aes("x_true",
"ub"),
size = 1.5,
color = "blue",
alpha = 0.7) +
geom_line(aes("x_true",
"lb"),
size = 1.5,
color = "blue",
alpha = 0.7) +
geom_point(df_expectiles,
aes("x_true",
"y"),
color = "red",
alpha = 0.7,
size = 2) +
geom_step(sfunl,
aes("x_true",
"y"),
size = 1,
linetype = "dashed") +
geom_step(sfunu,
aes("x_true",
"y"),
size = 1,
linetype = "dashed")
)
np.random.seed(123)
###
# Actual Expectiles
###
y_loc = np.array([10])
y_scale = np.array([1 + 4*((0.3 < test["x_true"].values) & (test["x_true"].values < 0.5)) + 2*(test["x_true"].values > 0.7)])
tau_lower = np.array([xgblss.dist.tau[0]])
tau_upper = np.array([xgblss.dist.tau[1]])
# Calculates exact expectiles assuming a Normal distribution
expectile_lb = expectile_norm(tau=tau_lower,
m=y_loc,
sd=y_scale).reshape(-1,)
expectile_ub = expectile_norm(tau=tau_upper,
m=y_loc,
sd=y_scale).reshape(-1,)
test["expect"] = np.where(test["y"].values < expectile_lb, 0, np.where(test["y"].values < expectile_ub, 1, 2))
test["alpha"] = np.where(test["y"].values <= expectile_lb, 1, np.where(test["y"].values >= expectile_ub, 1, 0))
df_expectiles = test[test["alpha"] == 1]
# Lower Bound
yl = list(set(expectile_lb))
yl.sort()
yl = [yl[2],yl[0],yl[2],yl[1],yl[1]]
sfunl = pd.DataFrame({"x_true":[0, 0.3, 0.5, 0.7, 1],
"y":yl})
# Upper Bound
yu = list(set(expectile_ub))
yu.sort()
yu = [yu[0],yu[2],yu[0],yu[1],yu[1]]
sfunu = pd.DataFrame({"x_true":[0, 0.3, 0.5, 0.7, 1],
"y":yu})
###
# Forecasted Expectiles
###
test["lb"] = pred_expectile.iloc[:,0]
test["ub"] = pred_expectile.iloc[:,1]
###
# Plot
###
(ggplot(test,
aes("x_true",
"y")) +
geom_point(alpha = 0.2, color = "black", size = 2) +
theme_bw(base_size=15) +
theme(legend_position="bottom",
plot_title = element_text(hjust = 0.5)) +
labs(title = "XGBoostLSS Expectile Regression - Simulated Data Example") +
geom_line(aes("x_true",
"ub"),
size = 1.5,
color = "blue",
alpha = 0.7) +
geom_line(aes("x_true",
"lb"),
size = 1.5,
color = "blue",
alpha = 0.7) +
geom_point(df_expectiles,
aes("x_true",
"y"),
color = "red",
alpha = 0.7,
size = 2) +
geom_step(sfunl,
aes("x_true",
"y"),
size = 1,
linetype = "dashed") +
geom_step(sfunu,
aes("x_true",
"y"),
size = 1,
linetype = "dashed")
)
Out[11]:
<Figure Size: (2000 x 1000)>