Update model outputs and results for width optimization iterations B29 and B34

Code/models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/cv_detailed_per_output_B29_H45.csv

 output,model,cv_rmse,std_rmse,cv_rmse_dispersion,cv_mae,cv_r2,BEST_PARAMS,fit_time_sec,gpr_kernel
-exymax_tw1,GradientBoosting,0.006493821737049297,0.006697384051332851,1.031347074577389,0.006493821737049297,,"{""learning_rate"": 0.025898788320641172, ""max_depth"": 1, ""max_features"": 0.6746420943668207, ""n_estimators"": 259, ""subsample"": 0.7527625863364908}",29.02,
-exymax_tw1,SVR,0.006585985009931142,0.0064982791163432545,0.9866829497097801,0.006585985009931142,,"{""svr__C"": 1245.9545122809943, ""svr__epsilon"": 0.0012027645559314329, ""svr__gamma"": 0.005025785483194675}",17.74,
-exymax_tw1,FlexibleMLP,0.006937396980222642,0.0068157807075165016,0.9824694661336453,0.006937396980222642,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.0009379840497457353, ""mlp__learning_rate_init"": 0.004342197085610553, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 176}",254.69,
-exymax_tw1,GaussianProcess,0.007006391573544815,0.007763545618619573,1.1080661902959676,0.007006391573544815,,"{""gpr__amplitude"": 12.794825054001086, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 0.13888023693241583, ""gpr__n_restarts_optimizer"": 8, ""gpr__noise"": 3.350098334788488e-05, ""gpr__rq_alpha"": 97.05779613825874}",25.22,"1.43**2 * Matern(length_scale=2.95, nu=1.5) + WhiteKernel(noise_level=3.35e-05)"
-exymax_tw1,RandomForest,0.010079281536114666,0.009245291985595667,0.9172570438150016,0.010079281536114666,,"{""max_depth"": 4, ""max_features"": 0.6684634201495349, ""min_samples_leaf"": 2, ""min_samples_split"": 3, ""n_estimators"": 701}",44.92,
-exymax_tw1,XGBoost,0.011286929222419112,0.010014957220104181,0.8873057518790474,0.011286929222419112,,"{""colsample_bytree"": 0.7772026979143375, ""learning_rate"": 0.010072838879230094, ""max_depth"": 1, ""min_child_weight"": 3, ""n_estimators"": 1060, ""subsample"": 0.7629481902087631}",23.54,
-exymax_tw2,SVR,0.0074343683500841745,0.004806222418653051,0.6464869901958292,0.0074343683500841745,,"{""svr__C"": 1578.3879853890564, ""svr__epsilon"": 0.002061045404501547, ""svr__gamma"": 0.003800674800490907}",18.36,
-exymax_tw2,FlexibleMLP,0.007658610586748594,0.006297387105883743,0.8222623457027408,0.007658610586748594,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 1.0929824065574703e-05, ""mlp__learning_rate_init"": 0.006624003188358499, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 220}",219.18,
-exymax_tw2,GradientBoosting,0.009069898715006377,0.0077238690282264485,0.8515937466255397,0.009069898715006377,,"{""learning_rate"": 0.013130700245599175, ""max_depth"": 3, ""max_features"": 0.6725567287580294, ""n_estimators"": 979, ""subsample"": 0.7716737130174645}",32.52,
-exymax_tw2,GaussianProcess,0.009939632428247051,0.005790324831213219,0.5825491911308432,0.009939632428247051,,"{""gpr__amplitude"": 72.5040766704692, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 6.0343367820082765, ""gpr__n_restarts_optimizer"": 3, ""gpr__noise"": 2.855728226541178e-08, ""gpr__rq_alpha"": 0.022351050575570935}",25.32,"1.59**2 * Matern(length_scale=3.64, nu=1.5) + WhiteKernel(noise_level=2.86e-08)"
-exymax_tw2,RandomForest,0.01190016183112282,0.014154848905866877,1.189466925470485,0.01190016183112282,,"{""max_depth"": 3, ""max_features"": 0.6668808751451677, ""min_samples_leaf"": 2, ""min_samples_split"": 4, ""n_estimators"": 745}",40.66,
-exymax_tw2,XGBoost,0.01344777279254198,0.014678885541959797,1.091547705959204,0.01344777279254198,,"{""colsample_bytree"": 0.7473810557459762, ""learning_rate"": 0.18866984873972462, ""max_depth"": 4, ""min_child_weight"": 3, ""n_estimators"": 614, ""subsample"": 0.729683700209585}",21.82,
-exymax_tw3,SVR,0.006837992343800181,0.004553580884860971,0.6659236594480216,0.006837992343800181,,"{""svr__C"": 448.7880306471758, ""svr__epsilon"": 0.00013403355081127117, ""svr__gamma"": 0.005847007418075073}",17.24,
-exymax_tw3,GradientBoosting,0.008563949492028333,0.005947970551815772,0.6945359214638505,0.008563949492028333,,"{""learning_rate"": 0.06607013348294709, ""max_depth"": 2, ""max_features"": 0.7517846219435218, ""n_estimators"": 396, ""subsample"": 0.7057609458244425}",31.62,
-exymax_tw3,GaussianProcess,0.009885086821969896,0.007845400559850292,0.7936602582400852,0.009885086821969896,,"{""gpr__amplitude"": 91.04050193518773, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 16.603129904873665, ""gpr__n_restarts_optimizer"": 2, ""gpr__noise"": 5.027538398209319e-09, ""gpr__rq_alpha"": 8.130666345000174}",24.0,"2.35**2 * Matern(length_scale=6.54, nu=1.5) + WhiteKernel(noise_level=5.03e-09)"
-exymax_tw3,FlexibleMLP,0.010403826869819305,0.008525822888958094,0.8194891164222315,0.010403826869819305,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.0016236756337057437, ""mlp__learning_rate_init"": 0.007091056527634161, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 114}",133.62,
-exymax_tw3,XGBoost,0.012591324124972523,0.01203655924174753,0.9559407034781418,0.012591324124972523,,"{""colsample_bytree"": 0.9504936269160258, ""learning_rate"": 0.022379620569679962, ""max_depth"": 4, ""min_child_weight"": 3, ""n_estimators"": 1171, ""subsample"": 0.7713136631361498}",23.35,
-exymax_tw3,RandomForest,0.019219946982543132,0.014028424014223698,0.7298887987029969,0.019219946982543132,,"{""max_depth"": 3, ""max_features"": 0.666678136009284, ""min_samples_leaf"": 2, ""min_samples_split"": 4, ""n_estimators"": 202}",42.04,
-tfmmax_frame,SVR,2.619870006205032,2.179555686783801,0.8319327606414179,2.619870006205032,,"{""svr__C"": 5387.889822738673, ""svr__epsilon"": 0.00023269836732785153, ""svr__gamma"": 0.07097468738318204}",16.09,
-tfmmax_frame,GaussianProcess,2.6319634234770533,2.6761700837879463,1.0167960769958164,2.6319634234770533,,"{""gpr__amplitude"": 0.4369339947510315, ""gpr__kernel_type"": ""Matern52"", ""gpr__length_scale"": 53.88550972627239, ""gpr__n_restarts_optimizer"": 10, ""gpr__noise"": 3.3825346762279944e-05, ""gpr__rq_alpha"": 0.014523514317963207}",25.88,"2.04**2 * Matern(length_scale=4.89, nu=2.5) + WhiteKernel(noise_level=2.91e-12)"
-tfmmax_frame,FlexibleMLP,9.118847413164623,7.156604842870021,0.7848146282761823,9.118847413164623,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.008499951197596679, ""mlp__learning_rate_init"": 0.0018647216179401307, ""mlp__n_layers"": 2, ""mlp__n_neurons"": 189}",230.53,
-tfmmax_frame,GradientBoosting,10.342653453427404,9.274970333998832,0.8967689361113846,10.342653453427404,,"{""learning_rate"": 0.02100426043099579, ""max_depth"": 3, ""max_features"": 0.847182811955193, ""n_estimators"": 1273, ""subsample"": 0.8200678637420077}",37.04,
-tfmmax_frame,XGBoost,11.273763220596315,13.025226181092236,1.155357437106372,11.273763220596315,,"{""colsample_bytree"": 0.9663842030420524, ""learning_rate"": 0.0795550204255391, ""max_depth"": 2, ""min_child_weight"": 3, ""n_estimators"": 695, ""subsample"": 0.8013266265338792}",22.69,
-tfmmax_frame,RandomForest,14.753822626899181,12.767048890410846,0.8653383745534499,14.753822626899181,,"{""max_depth"": 4, ""max_features"": 0.680638315287123, ""min_samples_leaf"": 2, ""min_samples_split"": 3, ""n_estimators"": 718}",47.55,
-tfmmax_tw1,FlexibleMLP,9.347145297566009,11.923942423962007,1.275677444221071,9.347145297566009,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 1.9379088771032204e-05, ""mlp__learning_rate_init"": 0.0004224115725246676, ""mlp__n_layers"": 4, ""mlp__n_neurons"": 489}",1060.19,
-tfmmax_tw1,GaussianProcess,9.62698068906717,12.597108758969705,1.3085212452202741,9.62698068906717,,"{""gpr__amplitude"": 7.502493970046291, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 0.8380582958252333, ""gpr__n_restarts_optimizer"": 5, ""gpr__noise"": 2.9263795924381803e-05, ""gpr__rq_alpha"": 0.015640337252370295}",24.59,"1.98**2 * Matern(length_scale=4.66, nu=1.5) + WhiteKernel(noise_level=5.82e-08)"
-tfmmax_tw1,GradientBoosting,10.25379604927948,11.822295469586088,1.1529676826775606,10.25379604927948,,"{""learning_rate"": 0.05944360886648676, ""max_depth"": 3, ""max_features"": 0.6749510567763413, ""n_estimators"": 827, ""subsample"": 0.7051508254376537}",32.47,
-tfmmax_tw1,SVR,10.519809880015345,10.720543782316861,1.0190815142660377,10.519809880015345,,"{""svr__C"": 2821.889984424592, ""svr__epsilon"": 0.0036809826285089673, ""svr__gamma"": 0.05831076296072628}",16.97,
-tfmmax_tw1,RandomForest,20.16973352251655,17.615419387667444,0.8733590539509315,20.16973352251655,,"{""max_depth"": 4, ""max_features"": 0.7045883364886902, ""min_samples_leaf"": 2, ""min_samples_split"": 3, ""n_estimators"": 779}",52.93,
-tfmmax_tw1,XGBoost,20.503049638521674,21.422135827603316,1.044826804074787,20.503049638521674,,"{""colsample_bytree"": 0.8185601354408085, ""learning_rate"": 0.011521005232891813, ""max_depth"": 3, ""min_child_weight"": 3, ""n_estimators"": 402, ""subsample"": 0.7088635494200846}",22.99,
-tfmmax_tw2,GaussianProcess,7.603551373657773,7.080381390450335,0.9311939963975333,7.603551373657773,,"{""gpr__amplitude"": 17.76576664980768, ""gpr__kernel_type"": ""RBF"", ""gpr__length_scale"": 2.467108843522573, ""gpr__n_restarts_optimizer"": 4, ""gpr__noise"": 2.3000512910597964e-12, ""gpr__rq_alpha"": 0.024337729656929853}",27.63,1.91**2 * RBF(length_scale=2.5) + WhiteKernel(noise_level=2.3e-12)
-tfmmax_tw2,FlexibleMLP,7.610330906438847,9.279426886803238,1.219319764263105,7.610330906438847,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.005038476075373874, ""mlp__learning_rate_init"": 0.00014182668289381992, ""mlp__n_layers"": 4, ""mlp__n_neurons"": 308}",441.2,
-tfmmax_tw2,SVR,9.027869677167802,9.88485278689751,1.094926393531919,9.027869677167802,,"{""svr__C"": 716.9454024181599, ""svr__epsilon"": 0.019504725037331867, ""svr__gamma"": 0.037685008549082605}",17.12,
-tfmmax_tw2,GradientBoosting,10.969325006134355,11.03206565345827,1.0057196452187194,10.969325006134355,,"{""learning_rate"": 0.0794000458723623, ""max_depth"": 2, ""max_features"": 0.9523780968960629, ""n_estimators"": 1497, ""subsample"": 0.8095580047092626}",34.65,
-tfmmax_tw2,XGBoost,17.299430461883546,18.49480242483776,1.0690989200822547,17.299430461883546,,"{""colsample_bytree"": 0.7419500396501548, ""learning_rate"": 0.01426193159307184, ""max_depth"": 3, ""min_child_weight"": 3, ""n_estimators"": 1200, ""subsample"": 0.8866577275892942}",22.58,
-tfmmax_tw2,RandomForest,18.388992385201867,19.1309842775827,1.0403497851778947,18.388992385201867,,"{""max_depth"": 4, ""max_features"": 0.7471645822804288, ""min_samples_leaf"": 2, ""min_samples_split"": 2, ""n_estimators"": 799}",50.53,
-tfmmax_tw3,SVR,5.238414599827629,6.046768431344908,1.1543126868086917,5.238414599827629,,"{""svr__C"": 5330.328717919815, ""svr__epsilon"": 0.05582902448723831, ""svr__gamma"": 0.04143240634749279}",16.48,
-tfmmax_tw3,GaussianProcess,6.373824735317225,6.927116943988003,1.0868069380077237,6.373824735317225,,"{""gpr__amplitude"": 5.1006870662557064, ""gpr__kernel_type"": ""RQ"", ""gpr__length_scale"": 0.6011915845322179, ""gpr__n_restarts_optimizer"": 10, ""gpr__noise"": 6.28714484104137e-07, ""gpr__rq_alpha"": 0.010399730643601359}",26.39,"2.47**2 * RationalQuadratic(alpha=1e+03, length_scale=3.84) + WhiteKernel(noise_level=0.00044)"
-tfmmax_tw3,FlexibleMLP,11.19864054900149,10.147086281186624,0.9060998285270772,11.19864054900149,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.003513807659187882, ""mlp__learning_rate_init"": 0.004434665898161324, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 362}",354.79,
-tfmmax_tw3,GradientBoosting,14.737353298887765,16.680981563055898,1.1318844859554815,14.737353298887765,,"{""learning_rate"": 0.012453723661255066, ""max_depth"": 2, ""max_features"": 0.8668530221055493, ""n_estimators"": 911, ""subsample"": 0.7069743934701856}",36.21,
-tfmmax_tw3,XGBoost,21.51369373817444,23.19103567169123,1.0779662457749164,21.51369373817444,,"{""colsample_bytree"": 0.7959588335760514, ""learning_rate"": 0.06303081692903291, ""max_depth"": 4, ""min_child_weight"": 3, ""n_estimators"": 505, ""subsample"": 0.7984053532645751}",21.97,
-tfmmax_tw3,RandomForest,27.9838762364769,26.699362474101843,0.9540980759234242,27.9838762364769,,"{""max_depth"": 4, ""max_features"": 0.6773578522514452, ""min_samples_leaf"": 2, ""min_samples_split"": 4, ""n_estimators"": 727}",47.34,
+exymax_tw1,SVR,0.006001995451920213,0.007236707021472172,1.2057168452463487,0.006001995451920213,,"{""svr__C"": 1.251057115936832, ""svr__epsilon"": 0.00010059140475344428, ""svr__gamma"": 0.046153252899337686}",21.47,
+exymax_tw1,GradientBoosting,0.00603072181438803,0.0070969169841838655,1.176793956446825,0.00603072181438803,,"{""learning_rate"": 0.04253596650346244, ""max_depth"": 3, ""max_features"": 0.7976937056494365, ""n_estimators"": 415, ""subsample"": 0.7}",35.7,
+exymax_tw1,GaussianProcess,0.007032812987993673,0.0077426239625777585,1.100928458611919,0.007032812987993673,,"{""gpr__amplitude"": 15.783879853890564, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 1.2778531518898433, ""gpr__n_restarts_optimizer"": 7, ""gpr__noise"": 1.7014397704982245e-05, ""gpr__rq_alpha"": 7.38115974661544}",31.18,"1.43**2 * Matern(length_scale=2.95, nu=1.5) + WhiteKernel(noise_level=1.7e-05)"
+exymax_tw1,FlexibleMLP,0.008699480230400259,0.00737838245811897,0.8481406087153662,0.008699480230400259,,"{""mlp__activation"": ""tanh"", ""mlp__alpha"": 0.0057547661357822966, ""mlp__learning_rate_init"": 0.0009838602333915323, ""mlp__n_layers"": 4, ""mlp__n_neurons"": 209}",380.76,
+exymax_tw1,RandomForest,0.009211883613481039,0.008746975354716314,0.9495316833916182,0.009211883613481039,,"{""max_depth"": 4, ""max_features"": 1.0, ""min_samples_leaf"": 2, ""min_samples_split"": 2, ""n_estimators"": 450}",56.59,
+exymax_tw1,XGBoost,0.010795381591319292,0.010509178366703363,0.9734883642421618,0.010795381591319292,,"{""colsample_bytree"": 1.0, ""learning_rate"": 0.01, ""max_depth"": 1, ""min_child_weight"": 3, ""n_estimators"": 1200, ""subsample"": 0.7}",32.66,
+exymax_tw2,SVR,0.0074343683500841745,0.004806222418653051,0.6464869901958292,0.0074343683500841745,,"{""svr__C"": 1578.3879853890564, ""svr__epsilon"": 0.002061045404501547, ""svr__gamma"": 0.003800674800490907}",18.66,
+exymax_tw2,GradientBoosting,0.008905631385286099,0.008209564382824648,0.9218396795973957,0.008905631385286099,,"{""learning_rate"": 0.03095578839842604, ""max_depth"": 3, ""max_features"": 0.7985866814386867, ""n_estimators"": 1289, ""subsample"": 0.7648119964568747}",42.64,
+exymax_tw2,FlexibleMLP,0.008952722063855102,0.009142550582241303,1.0212034414820714,0.008952722063855102,,"{""mlp__activation"": ""tanh"", ""mlp__alpha"": 3.278043287004692e-05, ""mlp__learning_rate_init"": 0.001570703295827246, ""mlp__n_layers"": 4, ""mlp__n_neurons"": 298}",293.09,
+exymax_tw2,GaussianProcess,0.00993961934007177,0.005790548068277946,0.5825724175304419,0.00993961934007177,,"{""gpr__amplitude"": 15.783879853890564, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 1.2778531518898433, ""gpr__n_restarts_optimizer"": 7, ""gpr__noise"": 1.7014397704982245e-05, ""gpr__rq_alpha"": 7.38115974661544}",27.01,"1.59**2 * Matern(length_scale=3.64, nu=1.5) + WhiteKernel(noise_level=1.7e-05)"
+exymax_tw2,RandomForest,0.011853450685807647,0.014190565021059023,1.1971674238329304,0.011853450685807647,,"{""max_depth"": 3, ""max_features"": 0.9696848688013859, ""min_samples_leaf"": 2, ""min_samples_split"": 3, ""n_estimators"": 680}",49.74,
+exymax_tw2,XGBoost,0.014017723306074738,0.015356355068290214,1.0954956616696354,0.014017723306074738,,"{""colsample_bytree"": 0.7850364898178779, ""learning_rate"": 0.014280526798824952, ""max_depth"": 3, ""min_child_weight"": 5, ""n_estimators"": 266, ""subsample"": 0.7011683788047537}",26.57,
+exymax_tw3,GradientBoosting,0.006215295360582448,0.007255618692319494,1.1673811575126103,0.006215295360582448,,"{""learning_rate"": 0.07999999999999999, ""max_depth"": 1, ""max_features"": 1.0, ""n_estimators"": 200, ""subsample"": 0.9}",37.7,
+exymax_tw3,SVR,0.006866803384362145,0.005579658066440473,0.8125553848166233,0.006866803384362145,,"{""svr__C"": 5535.810253232563, ""svr__epsilon"": 0.0001, ""svr__gamma"": 0.0016100194468603758}",19.23,
+exymax_tw3,GaussianProcess,0.00988507902808412,0.00784538748727503,0.7936595615458207,0.00988507902808412,,"{""gpr__amplitude"": 21.801034672053195, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 1.0353558840868569, ""gpr__n_restarts_optimizer"": 7, ""gpr__noise"": 4.868712960124535e-10, ""gpr__rq_alpha"": 72.3265102682836}",30.32,"2.35**2 * Matern(length_scale=6.54, nu=1.5) + WhiteKernel(noise_level=4.87e-10)"
+exymax_tw3,RandomForest,0.0118828453288412,0.009887495429239017,0.8320814717028074,0.0118828453288412,,"{""max_depth"": 4, ""max_features"": 1.0, ""min_samples_leaf"": 2, ""min_samples_split"": 2, ""n_estimators"": 800}",46.68,
+exymax_tw3,XGBoost,0.012508393807823957,0.012129338915953802,0.969695957954805,0.012508393807823957,,"{""colsample_bytree"": 0.7, ""learning_rate"": 0.01, ""max_depth"": 4, ""min_child_weight"": 3, ""n_estimators"": 1200, ""subsample"": 0.765008548037887}",24.8,
+exymax_tw3,FlexibleMLP,0.012812137651591084,0.008407871565191323,0.6562426812630432,0.012812137651591084,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.005160135756716969, ""mlp__learning_rate_init"": 0.01, ""mlp__n_layers"": 2, ""mlp__n_neurons"": 512}",350.14,
+tfmmax_frame,SVR,2.5401682807293624,1.9339303643963996,0.7613394667856836,2.5401682807293624,,"{""svr__C"": 2392.4696144711884, ""svr__epsilon"": 0.1, ""svr__gamma"": 0.07938429797160905}",24.79,
+tfmmax_frame,GaussianProcess,3.290103702830376,3.064705291431367,0.9314920039738852,3.290103702830376,,"{""gpr__amplitude"": 0.01108476210136268, ""gpr__kernel_type"": ""RQ"", ""gpr__length_scale"": 29.047432587351736, ""gpr__n_restarts_optimizer"": 3, ""gpr__noise"": 1.1374107298776648e-05, ""gpr__rq_alpha"": 0.3351721071240817}",39.44,"1.39**2 * RationalQuadratic(alpha=1e+03, length_scale=2.62) + WhiteKernel(noise_level=0.000243)"
+tfmmax_frame,FlexibleMLP,8.293164384685781,6.848956148966488,0.8258555879603474,8.293164384685781,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 1e-05, ""mlp__learning_rate_init"": 0.004845805447602877, ""mlp__n_layers"": 3, ""mlp__n_neurons"": 512}",851.3,
+tfmmax_frame,GradientBoosting,10.149793444979426,7.941774573933256,0.7824567679120247,10.149793444979426,,"{""learning_rate"": 0.01, ""max_depth"": 2, ""max_features"": 1.0, ""n_estimators"": 1436, ""subsample"": 0.7}",46.98,
+tfmmax_frame,XGBoost,11.909477101898194,10.321340620821681,0.8666493526551735,11.909477101898194,,"{""colsample_bytree"": 0.7038318977036059, ""learning_rate"": 0.19789970002066382, ""max_depth"": 1, ""min_child_weight"": 3, ""n_estimators"": 1200, ""subsample"": 0.95}",28.28,
+tfmmax_frame,RandomForest,11.995015586073901,8.374664413872166,0.6981787021264959,11.995015586073901,,"{""max_depth"": 4, ""max_features"": 1.0, ""min_samples_leaf"": 2, ""min_samples_split"": 2, ""n_estimators"": 200}",51.1,
+tfmmax_tw1,GaussianProcess,9.626528576791392,12.59742504746399,1.3086155561658162,9.626528576791392,,"{""gpr__amplitude"": 14.23439735511954, ""gpr__kernel_type"": ""Matern32"", ""gpr__length_scale"": 1.3324202691254317, ""gpr__n_restarts_optimizer"": 7, ""gpr__noise"": 3.930566415820431e-05, ""gpr__rq_alpha"": 7.487145894090875}",39.28,"1.98**2 * Matern(length_scale=4.66, nu=1.5) + WhiteKernel(noise_level=6.6e-10)"
+tfmmax_tw1,FlexibleMLP,10.0635704605035,10.744361421080878,1.067649047944691,10.0635704605035,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.00013301295828297907, ""mlp__learning_rate_init"": 0.00025137486913624557, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 296}",554.03,
+tfmmax_tw1,GradientBoosting,10.174849755796814,10.732300056415774,1.0547870793179397,10.174849755796814,,"{""learning_rate"": 0.07999999999999999, ""max_depth"": 3, ""max_features"": 0.9039407276230857, ""n_estimators"": 1498, ""subsample"": 0.7}",45.78,
+tfmmax_tw1,SVR,10.424559775421898,10.20992219313162,0.9794103936363499,10.424559775421898,,"{""svr__C"": 2810.82923959197, ""svr__epsilon"": 0.0001, ""svr__gamma"": 0.06531008775025517}",27.8,
+tfmmax_tw1,RandomForest,17.93885405526413,17.049424612269636,0.9504188260713625,17.93885405526413,,"{""max_depth"": 3, ""max_features"": 1.0, ""min_samples_leaf"": 2, ""min_samples_split"": 3, ""n_estimators"": 800}",57.07,
+tfmmax_tw1,XGBoost,21.216950493896007,23.0348171312776,1.0856799207739387,21.216950493896007,,"{""colsample_bytree"": 0.9205996170623962, ""learning_rate"": 0.01364888047412422, ""max_depth"": 1, ""min_child_weight"": 4, ""n_estimators"": 416, ""subsample"": 0.7567999303843258}",33.86,
+tfmmax_tw2,SVR,6.756727839118981,6.676538341560321,0.9881319035681159,6.756727839118981,,"{""svr__C"": 10000.0, ""svr__epsilon"": 0.0001, ""svr__gamma"": 0.05217529822341832}",26.05,
+tfmmax_tw2,GaussianProcess,7.603540683928204,7.080372403752352,0.931194123642728,7.603540683928204,,"{""gpr__amplitude"": 3.774672668226507, ""gpr__kernel_type"": ""RBF"", ""gpr__length_scale"": 1.1774847267003916, ""gpr__n_restarts_optimizer"": 8, ""gpr__noise"": 7.4362827567973694e-09, ""gpr__rq_alpha"": 0.011804465944974302}",35.69,1.91**2 * RBF(length_scale=2.5) + WhiteKernel(noise_level=7.44e-09)
+tfmmax_tw2,FlexibleMLP,8.087966887956073,7.634582836376119,0.9439433843064942,8.087966887956073,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 1e-05, ""mlp__learning_rate_init"": 0.0007405874332616178, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 512}",1472.57,
+tfmmax_tw2,GradientBoosting,9.717010582825283,10.597194502567135,1.090581759918793,9.717010582825283,,"{""learning_rate"": 0.01, ""max_depth"": 1, ""max_features"": 1.0, ""n_estimators"": 1084, ""subsample"": 0.7}",43.03,
+tfmmax_tw2,RandomForest,13.232088236123507,16.22612642400946,1.2262710264969554,13.232088236123507,,"{""max_depth"": 4, ""max_features"": 1.0, ""min_samples_leaf"": 2, ""min_samples_split"": 2, ""n_estimators"": 200}",52.0,
+tfmmax_tw2,XGBoost,14.919652176666261,12.959112652737284,0.8685934832317885,14.919652176666261,,"{""colsample_bytree"": 1.0, ""learning_rate"": 0.2, ""max_depth"": 4, ""min_child_weight"": 3, ""n_estimators"": 200, ""subsample"": 0.95}",34.13,
+tfmmax_tw3,SVR,4.670361578439981,5.422009365315906,1.1609399559866613,4.670361578439981,,"{""svr__C"": 8424.551575529082, ""svr__epsilon"": 0.1, ""svr__gamma"": 0.035969539073433233}",26.14,
+tfmmax_tw3,GaussianProcess,6.376907789529663,6.921580345617358,1.0854132714576807,6.376907789529663,,"{""gpr__amplitude"": 100.0, ""gpr__kernel_type"": ""RQ"", ""gpr__length_scale"": 0.01, ""gpr__n_restarts_optimizer"": 5, ""gpr__noise"": 0.0001, ""gpr__rq_alpha"": 0.44672126693118624}",36.65,"2.47**2 * RationalQuadratic(alpha=1e+03, length_scale=3.84) + WhiteKernel(noise_level=0.00044)"
+tfmmax_tw3,GradientBoosting,8.779461099318883,9.519607163223423,1.0843042705618868,8.779461099318883,,"{""learning_rate"": 0.01, ""max_depth"": 1, ""max_features"": 1.0, ""n_estimators"": 1500, ""subsample"": 0.758923798656051}",47.55,
+tfmmax_tw3,FlexibleMLP,9.862308877726177,9.042706144380263,0.91689545080088,9.862308877726177,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.00011774574559119063, ""mlp__learning_rate_init"": 0.00014572882138056477, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 475}",1224.44,
+tfmmax_tw3,RandomForest,17.046907741542668,17.766336867856367,1.0422029107695865,17.046907741542668,,"{""max_depth"": 4, ""max_features"": 1.0, ""min_samples_leaf"": 2, ""min_samples_split"": 2, ""n_estimators"": 200}",51.34,
+tfmmax_tw3,XGBoost,20.811745992279054,21.539345469580947,1.0349610012332375,20.811745992279054,,"{""colsample_bytree"": 0.7, ""learning_rate"": 0.2, ""max_depth"": 4, ""min_child_weight"": 3, ""n_estimators"": 200, ""subsample"": 0.95}",31.14,

Code/models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/cv_summary_per_output_B29_H45.csv

 output,best_model,cv_rmse,std_rmse,cv_rmse_dispersion,cv_mae,cv_r2,BEST_PARAMS,model_path,train_time_sec,gpr_kernel,selected_by
-exymax_tw1,SVR,0.006585985009931142,0.0064982791163432545,0.9866829497097801,0.006585985009931142,,"{""svr__C"": 1245.9545122809943, ""svr__epsilon"": 0.0012027645559314329, ""svr__gamma"": 0.005025785483194675}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_exymax_tw1.joblib,395.13,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
-exymax_tw3,SVR,0.006837992343800181,0.004553580884860971,0.6659236594480216,0.006837992343800181,,"{""svr__C"": 448.7880306471758, ""svr__epsilon"": 0.00013403355081127117, ""svr__gamma"": 0.005847007418075073}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_exymax_tw3.joblib,271.87,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
-exymax_tw2,SVR,0.0074343683500841745,0.004806222418653051,0.6464869901958292,0.0074343683500841745,,"{""svr__C"": 1578.3879853890564, ""svr__epsilon"": 0.002061045404501547, ""svr__gamma"": 0.003800674800490907}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_exymax_tw2.joblib,357.86,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
-tfmmax_frame,SVR,2.619870006205032,2.179555686783801,0.8319327606414179,2.619870006205032,,"{""svr__C"": 5387.889822738673, ""svr__epsilon"": 0.00023269836732785153, ""svr__gamma"": 0.07097468738318204}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_frame.joblib,379.78,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
-tfmmax_tw3,SVR,5.238414599827629,6.046768431344908,1.1543126868086917,5.238414599827629,,"{""svr__C"": 5330.328717919815, ""svr__epsilon"": 0.05582902448723831, ""svr__gamma"": 0.04143240634749279}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_tw3.joblib,503.18,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
-tfmmax_tw2,GaussianProcess,7.603551373657773,7.080381390450335,0.9311939963975333,7.603551373657773,,"{""gpr__amplitude"": 17.76576664980768, ""gpr__kernel_type"": ""RBF"", ""gpr__length_scale"": 2.467108843522573, ""gpr__n_restarts_optimizer"": 4, ""gpr__noise"": 2.3000512910597964e-12, ""gpr__rq_alpha"": 0.024337729656929853}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_tw2.joblib,593.72,1.91**2 * RBF(length_scale=2.5) + WhiteKernel(noise_level=2.3e-12),lowest_cv_rmse_dispersion_within_5pct_rmse_band
-tfmmax_tw1,FlexibleMLP,9.347145297566009,11.923942423962007,1.275677444221071,9.347145297566009,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 1.9379088771032204e-05, ""mlp__learning_rate_init"": 0.0004224115725246676, ""mlp__n_layers"": 4, ""mlp__n_neurons"": 489}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_tw1.joblib,1210.15,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+exymax_tw1,GradientBoosting,0.00603072181438803,0.0070969169841838655,1.176793956446825,0.00603072181438803,,"{""learning_rate"": 0.04253596650346244, ""max_depth"": 3, ""max_features"": 0.7976937056494365, ""n_estimators"": 415, ""subsample"": 0.7}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_exymax_tw1.joblib,558.4,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+exymax_tw3,GradientBoosting,0.006215295360582448,0.007255618692319494,1.1673811575126103,0.006215295360582448,,"{""learning_rate"": 0.07999999999999999, ""max_depth"": 1, ""max_features"": 1.0, ""n_estimators"": 200, ""subsample"": 0.9}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_exymax_tw3.joblib,508.89,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+exymax_tw2,SVR,0.0074343683500841745,0.004806222418653051,0.6464869901958292,0.0074343683500841745,,"{""svr__C"": 1578.3879853890564, ""svr__epsilon"": 0.002061045404501547, ""svr__gamma"": 0.003800674800490907}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_exymax_tw2.joblib,457.72,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+tfmmax_frame,SVR,2.5401682807293624,1.9339303643963996,0.7613394667856836,2.5401682807293624,,"{""svr__C"": 2392.4696144711884, ""svr__epsilon"": 0.1, ""svr__gamma"": 0.07938429797160905}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_frame.joblib,1041.89,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+tfmmax_tw3,SVR,4.670361578439981,5.422009365315906,1.1609399559866613,4.670361578439981,,"{""svr__C"": 8424.551575529082, ""svr__epsilon"": 0.1, ""svr__gamma"": 0.035969539073433233}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_tw3.joblib,1417.27,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+tfmmax_tw2,SVR,6.756727839118981,6.676538341560321,0.9881319035681159,6.756727839118981,,"{""svr__C"": 10000.0, ""svr__epsilon"": 0.0001, ""svr__gamma"": 0.05217529822341832}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_tw2.joblib,1663.47,,lowest_cv_rmse_dispersion_within_5pct_rmse_band
+tfmmax_tw1,FlexibleMLP,10.0635704605035,10.744361421080878,1.067649047944691,10.0635704605035,,"{""mlp__activation"": ""relu"", ""mlp__alpha"": 0.00013301295828297907, ""mlp__learning_rate_init"": 0.00025137486913624557, ""mlp__n_layers"": 5, ""mlp__n_neurons"": 296}",../../models/width_optimization/3W/ml_models/per_output_models_B29_H45/it0/best_model_tfmmax_tw1.joblib,757.86,,lowest_cv_rmse_dispersion_within_5pct_rmse_band

Code/reports/width_optimization/3W/ml_optimization/it0/optimization_results_3W_B29_H45_TFD_W90.csv

@@ -4,14 +4,14 @@ Configuration_B,29.0
 Configuration_H,45.0
 Configuration_TFD_W,90.0
 Iteration,0.0
-tw1_optimal,5.525014015514342
-tw2_optimal,8.471896290298302
-tw3_optimal,9.333863643334162
-Objective_score,-1.1926307429844958e-06
-Exy_tw1,0.062156306095972946
-Exy_tw2,0.04245576123008932
-Exy_tw3,0.04710451673468341
-TFM_tw1,90.00000000228151
-TFM_tw2,90.00000000327702
-TFM_tw3,90.00000000384767
-TFM_frame,70.8310257119319
+tw1_optimal,5.962169735306373
+tw2_optimal,8.231458736581402
+tw3_optimal,9.46671346480481
+Objective_score,-1.1396992006835905e-06
+Exy_tw1,0.051639785547767394
+Exy_tw2,0.04546536916285704
+Exy_tw3,0.04887117548120472
+TFM_tw1,90.00000001877713
+TFM_tw2,90.00000003753132
+TFM_tw3,90.00000000166554
+TFM_frame,70.99551612604436

Manuscript/RESILINK_surrogate_optimization_manuscript.tex

@@ -217,28 +217,26 @@ The number of samples in all cases is defined as a power of two. This choice fac
 
 \subsection{Supervised ML surrogate models}\label{subsec:ml_models}
 
-This work compares the predictive accuracy and robustness of several surrogate modelling techniques of different nature for the estimation of structural response quantities related to damage and distortion. The supervised surrogate set includes tree-based methods, namely Random Forest (RF), Gradient Boosting Regression (GBR), and XGBoost; kernel-based approaches, including Support Vector Regression (SVR) and Gaussian Process Regression (GPR); and neural-network-based models represented by Multilayer Perceptron (MLP).
+This work compares several supervised surrogate models for predicting FEM-derived damage and distortion indicators. The considered models cover three families: tree-based methods, including Random Forest (RF) \cite{Breiman2001}, Gradient Boosting Regression (GBR) \cite{Friedman2001}, and XGBoost \cite{Chen2016}; kernel-based methods, including Support Vector Regression (SVR) \cite{Drucker1996} and Gaussian Process Regression (GPR) \cite{Williams1995}; and neural-network models, represented by the Multilayer Perceptron (MLP) \cite{Rosenblatt1958,Rumelhart1986}. RF relies on bootstrap aggregation of decision trees, GBR and XGBoost are sequential boosting approaches, SVR and GPR exploit kernel functions to model nonlinear relationships, and MLP approximates nonlinear input--output mappings through interconnected layers.
 
-The objective is the integration of FEM-calibrated damage and distortion indicators into a surrogate-assisted optimization framework capable of evaluating the structural performance of different geometrical configurations. The proposed methodology seeks to minimize local damage in both the dissipative windows and the surrounding frame while promoting a balanced contribution of all windows to the energy dissipation process. Following optimization, an adaptive FEM validation loop is incorporated to verify the predicted optimal geometries before acceptance.
+For each geometry family, an independent regression model is trained for every target output. The input vector contains the window thicknesses of the corresponding device, while the outputs are the local distortion indicators $\varepsilon_{xy,i}$, the window damage indicators $\TFD_i$, and the frame damage indicator $\TFD_f$. Therefore, for a device with $W$ windows, $2W+1$ surrogate models are trained: $W$ for distortion, $W$ for window damage, and one for frame damage. For each output, all candidate algorithms are evaluated and the selected model is stored.
 
-For each geometry family, an independent supervised regression model is trained for every target output variable. The input vector contains only the window thicknesses associated with the corresponding device configuration. The predicted outputs include the local distortion indicators $\varepsilon_{xy,i}$, the window damage indicators $\TFD_i$, and the frame damage indicator $\TFD_f$. Consequently, for each geometry family, a total of $2W+1$ surrogate models are trained, where $W$ denotes the number of windows: $W$ models associated with distortion indicators, $W$ models associated with window damage indicators, and one additional model corresponding to the frame damage indicator. For every target output, all candidate algorithms are evaluated independently, and the selected surrogate is stored as a serialized \texttt{joblib} file.
+Hyperparameters are optimized using Bayesian optimization \cite{Snoek2012}, with 40 evaluations per model. The first 10 evaluations are randomly sampled to explore the search space, while the remaining 30 are guided by the Bayesian surrogate model. This strategy provides a more efficient alternative to exhaustive grid search, particularly given the number of geometry families, target outputs, and candidate algorithms considered.
 
-For each geometry family and target output, hyperparameter optimization is performed independently for every ML model using Bayesian optimization. This iterative search strategy constructs a probabilistic surrogate of the objective function, typically based on Gaussian processes as adopted in the present study, to guide the exploration of the hyperparameter space \cite{Snoek2012}. Compared with conventional grid-search strategies, Bayesian optimization is significantly more efficient because it concentrates the search on regions that are more likely to improve model performance while avoiding less promising combinations.
+The cross-validation strategy is adapted to the dataset size. Leave-One-Out validation is used for $N\leq20$, repeated five-fold cross-validation with five repetitions for $21\leq N\leq80$, and shuffled five-fold cross-validation for larger datasets. For small datasets, the search spaces of tree-based models are restricted to reduce overfitting.
 
-A total of 40 optimization iterations are performed for each ML model. The first 10 iterations are devoted to random exploration of the hyperparameter space, whereas the remaining 30 iterations are guided by the probabilistic surrogate model. This strategy considerably reduces the number of hyperparameter evaluations required compared with exhaustive grid-search procedures, particularly considering the large number of geometry families, target outputs, and candidate surrogate models analysed in this work. The optimal hyperparameter configuration is selected as the one minimizing the mean absolute error (MAE) during cross-validation.
+Model selection is performed in two stages. First, for each candidate algorithm, Bayesian hyperparameter optimization is carried out using cross-validated RMSE as the refit criterion. The best hyperparameter configuration is therefore the one with the lowest mean RMSE. Then, the best configurations from all candidate algorithms are compared. The model with the lowest mean RMSE defines a competitive threshold, and all models with an RMSE within 5\% of this value are retained. Among these competitive models, the final selection is based on the lowest relative RMSE dispersion, computed as the standard deviation of the fold-wise RMSE divided by the mean RMSE. If two models have the same dispersion, the one with the lower mean RMSE is preferred.
 
-Model performance is evaluated primarily through the root mean squared error (RMSE). Since the available dataset size varies between geometry families and adaptive iterations, the cross-validation strategy is selected automatically. Leave-One-Out validation is employed for datasets with $N\leq20$, repeated five-fold cross-validation with five repetitions is adopted for $21\leq N\leq80$, and standard shuffled five-fold cross-validation is used for larger datasets. For small datasets, the hyperparameter search spaces of tree-based models are additionally restricted to mitigate overfitting.
+This procedure favours surrogates that are both accurate and stable, which is important because small prediction errors near active damage constraints may alter the optimized geometry.
 
-The surrogate selection process is not based exclusively on the minimum average RMSE. Models whose RMSE falls within a 5\% tolerance band of the best-performing candidate are considered competitive. When fold-level dispersion metrics are available for all competitive candidates, the model with the lowest coefficient of variation of the RMSE is selected. This criterion favours not only accurate but also stable surrogates, which is particularly important in optimization problems where small local prediction errors near active damage constraints may alter the final accepted geometry.
-
-\begin{figure}[ht!]
-\centering
-\fbox{\parbox[c][0.24\textheight][c]{0.85\textwidth}{\centering Placeholder for supervised surrogate workflow: FEM dataset generation, adaptive cross-validation, Bayesian hyperparameter optimization, model selection, and surrogate persistence.}}
-\caption{Workflow of the supervised surrogate training strategy. A separate surrogate model is trained for each output variable, and the selected algorithm may differ depending on the predicted quantity.}
-\label{fig:ml_workflow}
+\begin{figure}[!ht]
+  \centering
+  \includegraphics[width=1.0\textwidth]{./Figures/BayesianSearchCV.png}
+  \caption{Workflow of the supervised surrogate training and selection strategy. For each output variable, the cross-validation strategy is adapted to the dataset size, Bayesian optimization is used to tune each candidate model, and the final surrogate is selected according to RMSE accuracy and fold-wise RMSE dispersion.}
+  \label{fig:BayesianSearchCV}
 \end{figure}
 
-Preliminary executions of the proposed workflow indicate that SVR and GPR consistently provide the highest predictive accuracy for the considered datasets. Since both approaches rely on radial kernel functions, this observation motivated the additional assessment of Radial Basis Function (RBF) interpolation as a simpler and computationally efficient surrogate strategy, particularly suitable for low-dimensional and moderately sampled design spaces.
+Preliminary executions of the proposed workflow show that SVR and GPR frequently provide the highest, or second-highest, predictive accuracy for the considered datasets. Since both methods rely on radial kernel functions, this result motivates the assessment of Radial Basis Function (RBF) interpolation \cite{Gutmann2001} as a simpler and computationally efficient surrogate alternative. Although the cost of surrogate evaluation is negligible compared with FEM simulations, training and repeatedly evaluating complex supervised models can still become relevant within optimization loops involving many candidate geometries. RBF interpolation offers a non-parametric and fast-to-train alternative that can capture nonlinear response surfaces with limited hyperparameter tuning, making it attractive for low-dimensional and moderately sampled design spaces.
 
 \subsection{RBF surrogate models}\label{subsec:rbf_models}
 
@@ -259,6 +257,8 @@ This validation strategy is particularly suitable for the present application be
 
 \subsection{Optimization algorithm}\label{subsec:de}
 
+The proposed methodology seeks to minimize local damage in both the dissipative windows and the surrounding frame while promoting a balanced contribution of all windows to the energy dissipation process. Following optimization, an adaptive FEM validation loop is incorporated to verify the predicted optimal geometries before acceptance.
+
 The geometric optimization is carried out with Differential Evolution. DE is a population-based global optimizer that does not require gradient information and is therefore suitable for nonlinear, non-convex surrogate response surfaces. In the current implementation, the DE algorithm is run with a maximum of 500 iterations, a population size factor of 25, a convergence tolerance of $10^{-6}$ and a fixed random seed equal to 42 for reproducibility.
 
 For each candidate geometry $\mathbf{x}$, the trained surrogate models predict the window distortions $\hat{\Exy}_i$, the window damage indicators $\widehat{\TFD}_i$ and the frame damage indicator $\widehat{\TFD}_f$. These predictions are then combined into a scalar objective function to be minimized.

Manuscript/wileyNJD-AMA.bib

@@ -594,8 +594,8 @@ steel from coupon test results available. First, the theory of metal plasticity
   year      = {2012},
   address   = {Red Hook, NY, USA},
   month     = dec,
+  organization = {Curran Associates Inc.},
   pages     = {2951--2959},
-  publisher = {Curran Associates Inc.},
   series    = {{NIPS}'12},
   volume    = {2},
   abstract  = {The use of machine learning algorithms frequently involves careful tuning of learning parameters and model hyperparameters. Unfortunately, this tuning is often a "black art" requiring expert experience, rules of thumb, or sometimes brute-force search. There is therefore great appeal for automatic approaches that can optimize the performance of any given learning algorithm to the problem at hand. In this work, we consider this problem through the framework of Bayesian optimization, in which a learning algorithm's generalization performance is modeled as a sample from a Gaussian process (GP). We show that certain choices for the nature of the GP, such as the type of kernel and the treatment of its hyperparameters, can play a crucial role in obtaining a good optimizer that can achieve expertlevel performance. We describe new algorithms that take into account the variable cost (duration) of learning algorithm experiments and that can leverage the presence of multiple cores for parallel experimentation. We show that these proposed algorithms improve on previous automatic procedures and can reach or surpass human expert-level optimization for many algorithms including latent Dirichlet allocation, structured SVMs and convolutional neural networks.},
@@ -603,4 +603,128 @@ steel from coupon test results available. First, the theory of metal plasticity
   urldate   = {2026-04-21},
 }
 
+@Article{Friedman2001,
+  author   = {Friedman, J.H.},
+  journal  = {Annals of Statistics},
+  title    = {Greedy function approximation: a gradient boosting machine},
+  year     = {2001},
+  pages    = {1189 -- 1232},
+  keywords = {62-02, 62-07, 62-08, 62G08, 62H30, 68T10, boosting, decision trees, Function estimation, robust nonparametric regression},
+}
+
+@Article{Breiman2001,
+  author   = {Breiman, Leo},
+  journal  = {Machine Learning},
+  title    = {Random {Forests}},
+  year     = {2001},
+  issn     = {1573-0565},
+  month    = oct,
+  number   = {1},
+  pages    = {5--32},
+  volume   = {45},
+  abstract = {Random forests are a combination of tree predictors such that each tree depends on the values of a random vector sampled independently and with the same distribution for all trees in the forest. The generalization error for forests converges a.s. to a limit as the number of trees in the forest becomes large. The generalization error of a forest of tree classifiers depends on the strength of the individual trees in the forest and the correlation between them. Using a random selection of features to split each node yields error rates that compare favorably to Adaboost (Y. Freund \& R. Schapire, Machine Learning: Proceedings of the Thirteenth International conference, ***, 148–156), but are more robust with respect to noise. Internal estimates monitor error, strength, and correlation and these are used to show the response to increasing the number of features used in the splitting. Internal estimates are also used to measure variable importance. These ideas are also applicable to regression.},
+  doi      = {10.1023/A:1010933404324},
+  keywords = {classification, ensemble, regression},
+  language = {en},
+  urldate  = {2026-05-07},
+}
+
+@Article{Chen2016,
+  author     = {Chen, Tianqi and Guestrin, Carlos},
+  journal    = {Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining},
+  title      = {{XGBoost}: {A} {Scalable} {Tree} {Boosting} {System}},
+  year       = {2016},
+  month      = aug,
+  note       = {Conference Name: KDD '16: The 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining ISBN: 9781450342322},
+  pages      = {785--794},
+  abstract   = {Tree boosting is a highly effective and widely used machine learning method. In this paper, we describe a scalable end-to-end tree boosting system called XGBoost, which is used widely by data scientists to achieve state-of-the-art results on many machine learning challenges. We propose a novel sparsity-aware algorithm for sparse data and weighted quantile sketch for approximate tree learning. More importantly, we provide insights on cache access patterns, data compression and sharding to build a scalable tree boosting system. By combining these insights, XGBoost scales beyond billions of examples using far fewer resources than existing systems.},
+  address    = {San Francisco California USA},
+  doi        = {10.1145/2939672.2939785},
+  language   = {en},
+  publisher  = {ACM},
+  shorttitle = {{XGBoost}},
+  urldate    = {2026-05-07},
+}
+
+@InProceedings{Drucker1996,
+  author    = {Drucker, Harris and Burges, Chris J. C. and Kaufman, Linda and Smola, Alex and Vapnik, Vladimir},
+  booktitle = {Proceedings of the 10th {International} {Conference} on {Neural} {Information} {Processing} {Systems}},
+  title     = {Support vector regression machines},
+  year      = {1996},
+  address   = {Cambridge, MA, USA},
+  month     = dec,
+  organization = {MIT Press},
+  pages     = {155--161},
+  series    = {{NIPS}'96},
+  abstract  = {A new regression technique based on Vapnik's concept of support vectors is introduced. We compare support vector regression (SVR) with a committee regression technique (bagging) based on regression trees and ridge regression done in feature space. On the basis of these experiments, it is expected that SVR will have advantages in high dimensionality space because SVR optimization does not depend on the dimensionality of the input space.},
+  urldate   = {2026-05-07},
+}
+
+@InProceedings{Williams1995,
+  author    = {Williams, Christopher K. I. and Rasmussen, Carl Edward},
+  booktitle = {Proceedings of the 9th {International} {Conference} on {Neural} {Information} {Processing} {Systems}},
+  title     = {Gaussian processes for regression},
+  year      = {1995},
+  address   = {Cambridge, MA, USA},
+  month     = nov,
+  organization = {MIT Press},
+  pages     = {514--520},
+  series    = {{NIPS}'95},
+  abstract  = {The Bayesian analysis of neural networks is difficult because a simple prior over weights implies a complex prior distribution over functions. In this paper we investigate the use of Gaussian process priors over functions, which permit the predictive Bayesian analysis for fixed values of hyperparameters to be carried out exactly using matrix operations. Two methods, using optimization and averaging (via Hybrid Monte Carlo) over hyperparameters have been tested on a number of challenging problems and have produced excellent results.},
+  urldate   = {2026-05-07},
+}
+
+@Article{Rosenblatt1958,
+  author     = {Rosenblatt, F.},
+  journal    = {Psychological Review},
+  title      = {The perceptron: {A} probabilistic model for information storage and organization in the brain},
+  year       = {1958},
+  issn       = {1939-1471},
+  number     = {6},
+  pages      = {386--408},
+  volume     = {65},
+  abstract   = {To answer the questions of how information about the physical world is sensed, in what form is information remembered, and how does information retained in memory influence recognition and behavior, a theory is developed for a hypothetical nervous system called a perceptron. The theory serves as a bridge between biophysics and psychology. It is possible to predict learning curves from neurological variables and vice versa. The quantitative statistical approach is fruitful in the understanding of the organization of cognitive systems. 18 references. (PsycInfo Database Record (c) 2025 APA, all rights reserved)},
+  address    = {US},
+  doi        = {10.1037/h0042519},
+  keywords   = {Brain, Cognition, Memory, Nervous System},
+  publisher  = {American Psychological Association},
+  shorttitle = {The perceptron},
+}
+
+@Article{Rumelhart1986,
+  author    = {Rumelhart, David E. and Hinton, Geoffrey E. and Williams, Ronald J.},
+  journal   = {Nature},
+  title     = {Learning representations by back-propagating errors},
+  year      = {1986},
+  issn      = {1476-4687},
+  month     = oct,
+  number    = {6088},
+  pages     = {533--536},
+  volume    = {323},
+  abstract  = {We describe a new learning procedure, back-propagation, for networks of neurone-like units. The procedure repeatedly adjusts the weights of the connections in the network so as to minimize a measure of the difference between the actual output vector of the net and the desired output vector. As a result of the weight adjustments, internal ‘hidden’ units which are not part of the input or output come to represent important features of the task domain, and the regularities in the task are captured by the interactions of these units. The ability to create useful new features distinguishes back-propagation from earlier, simpler methods such as the perceptron-convergence procedure1.},
+  copyright = {1986 Springer Nature Limited},
+  doi       = {10.1038/323533a0},
+  keywords  = {Humanities and Social Sciences, multidisciplinary, Science},
+  language  = {en},
+  publisher = {Nature Publishing Group},
+  urldate   = {2026-05-07},
+}
+
+@Article{Gutmann2001,
+  author   = {Gutmann, H.-M.},
+  journal  = {Journal of Global Optimization},
+  title    = {A {Radial} {Basis} {Function} {Method} for {Global} {Optimization}},
+  year     = {2001},
+  issn     = {1573-2916},
+  month    = mar,
+  number   = {3},
+  pages    = {201--227},
+  volume   = {19},
+  abstract = {We introduce a method that aims to find the global minimum of a continuous nonconvex function on a compact subset of \$\${\textbackslash}mathbb\{R\}{\textasciicircum}d \$\$. It is assumed that function evaluations are expensive and that no additional information is available. Radial basis function interpolation is used to define a utility function. The maximizer of this function is the next point where the objective function is evaluated. We show that, for most types of radial basis functions that are considered in this paper, convergence can be achieved without further assumptions on the objective function. Besides, it turns out that our method is closely related to a statistical global optimization method, the P-algorithm. A general framework for both methods is presented. Finally, a few numerical examples show that on the set of Dixon-Szegö test functions our method yields favourable results in comparison to other global optimization methods.},
+  doi      = {10.1023/A:1011255519438},
+  keywords = {Global optimization, interpolation, P-algorithm, radial basis functions},
+  language = {en},
+  urldate  = {2026-05-07},
+}
+
 @Comment{jabref-meta: databaseType:bibtex;}