--- title: "xgboost: Multiclass Classification" vignette: > %\VignetteEncoding{UTF-8} %\VignetteIndexEntry{xgboost: Multiclass Classification} %\VignetteEngine{quarto::html} editor_options: chunk_output_type: console execute: eval: false collapse: true comment: "#>" --- ```{r setup} # nolint start library(mlexperiments) library(mllrnrs) ``` See [https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R](https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R) for implementation details. # Preprocessing ## Import and Prepare Data ```{r} library(mlbench) data("DNA") dataset <- DNA |> data.table::as.data.table() |> na.omit() feature_cols <- colnames(dataset)[160:180] target_col <- "Class" ``` ## General Configurations ```{r} seed <- 123 if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) { # on cran ncores <- 2L } else { ncores <- ifelse( test = parallel::detectCores() > 4, yes = 4L, no = ifelse( test = parallel::detectCores() < 2L, yes = 1L, no = parallel::detectCores() ) ) } options("mlexperiments.bayesian.max_init" = 4L) options("mlexperiments.optim.xgb.nrounds" = 20L) options("mlexperiments.optim.xgb.early_stopping_rounds" = 5L) ``` ## Generate Training- and Test Data ```{r} data_split <- splitTools::partition( y = dataset[, get(target_col)], p = c(train = 0.7, test = 0.3), type = "stratified", seed = seed ) train_x <- model.matrix( ~ -1 + ., dataset[data_split$train, .SD, .SDcols = feature_cols] ) train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L test_x <- model.matrix( ~ -1 + ., dataset[data_split$test, .SD, .SDcols = feature_cols] ) test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1L ``` ## Generate Training Data Folds ```{r} fold_list <- splitTools::create_folds( y = train_y, k = 3, type = "stratified", seed = seed ) ``` # Experiments ## Prepare Experiments ```{r} # required learner arguments, not optimized learner_args <- list( objective = "multi:softprob", eval_metric = "mlogloss", num_class = 3 ) # set arguments for predict function and performance metric, # required for mlexperiments::MLCrossValidation and # mlexperiments::MLNestedCV predict_args <- list(reshape = TRUE) performance_metric <- metric("ACC") performance_metric_args <- NULL return_models <- FALSE # required for grid search and initialization of bayesian optimization parameter_grid <- expand.grid( subsample = seq(0.6, 1, .2), colsample_bytree = seq(0.6, 1, .2), min_child_weight = seq(1, 5, 4), learning_rate = seq(0.1, 0.2, 0.1), max_depth = seq(1, 5, 4) ) # reduce to a maximum of 10 rows if (nrow(parameter_grid) > 10) { set.seed(123) sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE) parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows) } # required for bayesian optimization parameter_bounds <- list( subsample = c(0.2, 1), colsample_bytree = c(0.2, 1), min_child_weight = c(1L, 10L), learning_rate = c(0.1, 0.2), max_depth = c(1L, 10L) ) optim_args <- list( n_iter = ncores, kappa = 3.5, acq = "ucb" ) ``` ## Hyperparameter Tuning ### Grid Search ```{r} tuner <- mlexperiments::MLTuneParameters$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), strategy = "grid", ncores = ncores, seed = seed ) tuner$parameter_grid <- parameter_grid tuner$learner_args <- learner_args tuner$split_type <- "stratified" tuner$set_data( x = train_x, y = train_y ) tuner_results_grid <- tuner$execute(k = 3) #> #> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%) #> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%) #> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%) #> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%) #> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%) #> Parameter settings [============================================================================>-------------------] 8/10 ( 80%) #> Parameter settings [=====================================================================================>----------] 9/10 ( 90%) #> Parameter settings [===============================================================================================] 10/10 (100%) head(tuner_results_grid) #> setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective #> #> 1: 1 1.0106675 38 0.6 0.8 5 0.2 1 multi:softprob #> 2: 2 0.9828797 37 1.0 0.8 5 0.1 5 multi:softprob #> 3: 3 1.0102800 76 0.8 0.8 5 0.1 1 multi:softprob #> 4: 4 0.9867769 20 0.6 0.8 5 0.2 5 multi:softprob #> 5: 5 0.9815158 32 1.0 0.8 1 0.1 5 multi:softprob #> 6: 6 0.9741743 50 0.8 0.8 5 0.1 5 multi:softprob #> eval_metric num_class #> #> 1: mlogloss 3 #> 2: mlogloss 3 #> 3: mlogloss 3 #> 4: mlogloss 3 #> 5: mlogloss 3 #> 6: mlogloss 3 ``` ### Bayesian Optimization ```{r} tuner <- mlexperiments::MLTuneParameters$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), strategy = "bayesian", ncores = ncores, seed = seed ) tuner$parameter_grid <- parameter_grid tuner$parameter_bounds <- parameter_bounds tuner$learner_args <- learner_args tuner$optim_args <- optim_args tuner$split_type <- "stratified" tuner$set_data( x = train_x, y = train_y ) tuner_results_bayesian <- tuner$execute(k = 3) #> #> Registering parallel backend using 4 cores. head(tuner_results_bayesian) #> Epoch setting_id subsample colsample_bytree min_child_weight learning_rate max_depth gpUtility acqOptimum inBounds Elapsed #> #> 1: 0 1 0.6 0.8 5 0.2 1 NA FALSE TRUE 0.995 #> 2: 0 2 1.0 0.8 5 0.1 5 NA FALSE TRUE 1.057 #> 3: 0 3 0.8 0.8 5 0.1 1 NA FALSE TRUE 1.088 #> 4: 0 4 0.6 0.8 5 0.2 5 NA FALSE TRUE 1.035 #> 5: 0 5 1.0 0.8 1 0.1 5 NA FALSE TRUE 0.282 #> 6: 0 6 0.8 0.8 5 0.1 5 NA FALSE TRUE 0.314 #> Score metric_optim_mean nrounds errorMessage objective eval_metric num_class #> #> 1: -1.0106675 1.0106675 38 NA multi:softprob mlogloss 3 #> 2: -0.9828797 0.9828797 37 NA multi:softprob mlogloss 3 #> 3: -1.0102800 1.0102800 76 NA multi:softprob mlogloss 3 #> 4: -0.9867769 0.9867769 20 NA multi:softprob mlogloss 3 #> 5: -0.9815158 0.9815158 32 NA multi:softprob mlogloss 3 #> 6: -0.9741743 0.9741743 50 NA multi:softprob mlogloss 3 ``` ## k-Fold Cross Validation ```{r} validator <- mlexperiments::MLCrossValidation$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), fold_list = fold_list, ncores = ncores, seed = seed ) validator$learner_args <- tuner$results$best.setting[-1] validator$predict_args <- predict_args validator$performance_metric <- performance_metric validator$performance_metric_args <- performance_metric_args validator$return_models <- return_models validator$set_data( x = train_x, y = train_y ) validator_results <- validator$execute() #> #> CV fold: Fold1 #> #> CV fold: Fold2 #> #> CV fold: Fold3 #> CV progress [========================================================================================================] 3/3 (100%) #> head(validator_results) #> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth nrounds objective eval_metric #> #> 1: Fold1 0.5685484 0.7220588 0.8330246 1 0.1099728 10 19 multi:softprob mlogloss #> 2: Fold2 0.5587045 0.7220588 0.8330246 1 0.1099728 10 19 multi:softprob mlogloss #> 3: Fold3 0.5483871 0.7220588 0.8330246 1 0.1099728 10 19 multi:softprob mlogloss #> num_class #> #> 1: 3 #> 2: 3 #> 3: 3 ``` ## Nested Cross Validation ### Inner Grid Search ```{r} validator <- mlexperiments::MLNestedCV$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), strategy = "grid", fold_list = fold_list, k_tuning = 3L, ncores = ncores, seed = seed ) validator$parameter_grid <- parameter_grid validator$learner_args <- learner_args validator$split_type <- "stratified" validator$predict_args <- predict_args validator$performance_metric <- performance_metric validator$performance_metric_args <- performance_metric_args validator$return_models <- return_models validator$set_data( x = train_x, y = train_y ) validator_results <- validator$execute() #> #> CV fold: Fold1 #> #> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%) #> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%) #> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%) #> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%) #> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%) #> Parameter settings [============================================================================>-------------------] 8/10 ( 80%) #> Parameter settings [=====================================================================================>----------] 9/10 ( 90%) #> Parameter settings [===============================================================================================] 10/10 (100%) #> CV fold: Fold2 #> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%) #> #> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%) #> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%) #> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%) #> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%) #> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%) #> Parameter settings [============================================================================>-------------------] 8/10 ( 80%) #> Parameter settings [=====================================================================================>----------] 9/10 ( 90%) #> Parameter settings [===============================================================================================] 10/10 (100%) #> CV fold: Fold3 #> CV progress [========================================================================================================] 3/3 (100%) #> #> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%) #> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%) #> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%) #> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%) #> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%) #> Parameter settings [============================================================================>-------------------] 8/10 ( 80%) #> Parameter settings [=====================================================================================>----------] 9/10 ( 90%) #> Parameter settings [===============================================================================================] 10/10 (100%) head(validator_results) #> fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric #> #> 1: Fold1 0.5591398 32 0.6 1.0 1 0.1 5 multi:softprob mlogloss #> 2: Fold2 0.5344130 34 0.8 0.8 5 0.1 5 multi:softprob mlogloss #> 3: Fold3 0.5510753 24 0.6 1.0 1 0.1 5 multi:softprob mlogloss #> num_class #> #> 1: 3 #> 2: 3 #> 3: 3 ``` ### Inner Bayesian Optimization ```{r} validator <- mlexperiments::MLNestedCV$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), strategy = "bayesian", fold_list = fold_list, k_tuning = 3L, ncores = ncores, seed = seed ) validator$parameter_grid <- parameter_grid validator$learner_args <- learner_args validator$split_type <- "stratified" validator$parameter_bounds <- parameter_bounds validator$optim_args <- optim_args validator$predict_args <- predict_args validator$performance_metric <- performance_metric validator$performance_metric_args <- performance_metric_args validator$return_models <- TRUE validator$set_data( x = train_x, y = train_y ) validator_results <- validator$execute() #> #> CV fold: Fold1 #> #> Registering parallel backend using 4 cores. #> #> CV fold: Fold2 #> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%) #> #> Registering parallel backend using 4 cores. #> #> CV fold: Fold3 #> CV progress [========================================================================================================] 3/3 (100%) #> #> Registering parallel backend using 4 cores. head(validator_results) #> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth nrounds objective eval_metric #> #> 1: Fold1 0.5591398 0.6000000 1.0000000 1 0.1000000 5 32 multi:softprob mlogloss #> 2: Fold2 0.5641026 0.6936177 0.7695365 1 0.1099728 10 20 multi:softprob mlogloss #> 3: Fold3 0.5537634 0.5955781 0.8688622 1 0.1099728 10 19 multi:softprob mlogloss #> num_class #> #> 1: 3 #> 2: 3 #> 3: 3 ``` ## Holdout Test Dataset Performance ### Predict Outcome in Holdout Test Dataset ```{r} preds_xgboost <- mlexperiments::predictions( object = validator, newdata = test_x ) ``` ### Evaluate Performance on Holdout Test Dataset ```{r} perf_xgboost <- mlexperiments::performance( object = validator, prediction_results = preds_xgboost, y_ground_truth = test_y ) perf_xgboost #> model performance #> #> 1: Fold1 0.5590387 #> 2: Fold2 0.5579937 #> 3: Fold3 0.5611285 ``` ## Appendix I: Grid-Search with Target Weigths Here, `xgboost`'s [`weight`-argument](https://rdrr.io/cran/xgboost/man/xgb.DMatrix.html) is used to rescale the case-weights during the training. ```{r} # define the target weights y_weights <- ifelse(train_y == 1, 0.8, ifelse(train_y == 2, 1.2, 1)) head(y_weights) #> [1] 1.2 1.2 0.0 0.8 0.8 0.0 ``` ```{r} tuner_w_weights <- mlexperiments::MLTuneParameters$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), strategy = "grid", ncores = ncores, seed = seed ) tuner_w_weights$parameter_grid <- parameter_grid tuner_w_weights$learner_args <- c( learner_args, list(case_weights = y_weights) ) tuner_w_weights$split_type <- "stratified" tuner_w_weights$set_data( x = train_x, y = train_y ) tuner_results_grid <- tuner_w_weights$execute(k = 3) #> #> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%) #> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%) #> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%) #> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%) #> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%) #> Parameter settings [============================================================================>-------------------] 8/10 ( 80%) #> Parameter settings [=====================================================================================>----------] 9/10 ( 90%) #> Parameter settings [===============================================================================================] 10/10 (100%) head(tuner_results_grid) #> setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective #> #> 1: 1 0.9442324 50 0.6 0.8 5 0.2 1 multi:softprob #> 2: 2 0.9217258 35 1.0 0.8 5 0.1 5 multi:softprob #> 3: 3 0.9443002 93 0.8 0.8 5 0.1 1 multi:softprob #> 4: 4 0.9245540 20 0.6 0.8 5 0.2 5 multi:softprob #> 5: 5 0.9212009 26 1.0 0.8 1 0.1 5 multi:softprob #> 6: 6 0.9145242 39 0.8 0.8 5 0.1 5 multi:softprob #> eval_metric num_class #> #> 1: mlogloss 3 #> 2: mlogloss 3 #> 3: mlogloss 3 #> 4: mlogloss 3 #> 5: mlogloss 3 #> 6: mlogloss 3 ``` ## Appendix II: k-Fold Cross Validation with Target Weigths ```{r} validator_w_weights <- mlexperiments::MLCrossValidation$new( learner = mllrnrs::LearnerXgboost$new( metric_optimization_higher_better = FALSE ), fold_list = fold_list, ncores = ncores, seed = seed ) # append the optimized setting from above with the newly created weights validator_w_weights$learner_args <- c( tuner_w_weights$results$best.setting[-1] ) validator_w_weights$predict_args <- predict_args validator_w_weights$performance_metric <- performance_metric validator_w_weights$performance_metric_args <- performance_metric_args validator_w_weights$return_models <- return_models validator_w_weights$set_data( x = train_x, y = train_y ) validator_results <- validator_w_weights$execute() #> #> CV fold: Fold1 #> #> CV fold: Fold2 #> #> CV fold: Fold3 #> CV progress [========================================================================================================] 3/3 (100%) #> head(validator_results) #> fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric #> #> 1: Fold1 0.5658602 39 0.8 0.8 5 0.1 5 multi:softprob mlogloss #> 2: Fold2 0.5222672 39 0.8 0.8 5 0.1 5 multi:softprob mlogloss #> 3: Fold3 0.5577957 39 0.8 0.8 5 0.1 5 multi:softprob mlogloss #> num_class #> #> 1: 3 #> 2: 3 #> 3: 3 ``` ```{r include=FALSE} # nolint end ```