⭐ 1. Introduction & Overview¶
Predicting students test scores
🔹 2. Import Libraries & Set Up¶
# !/usr/bin/env python3
# -*- coding: utf-8 -*-
# =====================
# General utilities
# =====================
import json
import os
import pickle
import time
from collections import Counter
# =====================
# Data handling & processing
# =====================
import numpy as np
import pandas as pd
from tqdm import tqdm
import category_encoders as ce
# =====================
# Visualization
# =====================
import matplotlib.pyplot as plt
import seaborn as sns
# =====================
# Machine Learning - Core scikit-learn
# =====================
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif
from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge
from sklearn.metrics import (
accuracy_score, f1_score, precision_score, recall_score,
mean_absolute_error, mean_squared_error, r2_score,
root_mean_squared_error, roc_auc_score
)
from sklearn.model_selection import train_test_split, GridSearchCV, KFold, cross_val_score
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.svm import SVC, SVR
# =====================
# Machine Learning - Tree Boosting & advanced
# =====================
import xgboost as xg
import lightgbm as lgb
import catboost
# =====================
# Deep Learning - TensorFlow / Keras
# =====================
import tensorflow as tf
from tensorflow.keras import regularizers
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
# =====================
# Deep Learning - PyTorch
# =====================
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
# =====================
# Imbalanced data handling
# =====================
from imblearn.over_sampling import SMOTE
# =====================
# Optimization / AutoML
# =====================
import optuna
# =====================
# Feature importance & explainability
# =====================
import shap
# =====================
# Settings & reproducibility
# =====================
import warnings
warnings.filterwarnings("ignore")
SEED = 42
np.random.seed(SEED)
print("Libraries successfully loaded. Ready to go!")
Libraries successfully loaded. Ready to go!
train = pd.read_csv('../data/train.csv')
test = pd.read_csv('../data/test.csv')
🔹 3. Data Exploration¶
train.head()
| id | age | gender | course | study_hours | class_attendance | internet_access | sleep_hours | sleep_quality | study_method | facility_rating | exam_difficulty | exam_score | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0 | 21 | female | b.sc | 7.91 | 98.8 | no | 4.9 | average | online videos | low | easy | 78.3 |
| 1 | 1 | 18 | other | diploma | 4.95 | 94.8 | yes | 4.7 | poor | self-study | medium | moderate | 46.7 |
| 2 | 2 | 20 | female | b.sc | 4.68 | 92.6 | yes | 5.8 | poor | coaching | high | moderate | 99.0 |
| 3 | 3 | 19 | male | b.sc | 2.00 | 49.5 | yes | 8.3 | average | group study | high | moderate | 63.9 |
| 4 | 4 | 23 | male | bca | 7.65 | 86.9 | yes | 9.6 | good | self-study | high | easy | 100.0 |
from utils import *
# heatmap_nums(train)
# plot_cats(train)
# plot_nums(train)
🔹 4. Feature Engineering¶
train.head(1)
| id | age | gender | course | study_hours | class_attendance | internet_access | sleep_hours | sleep_quality | study_method | facility_rating | exam_difficulty | exam_score | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0 | 21 | female | b.sc | 7.91 | 98.8 | no | 4.9 | average | online videos | low | easy | 78.3 |
🔹 5. Model Testing and Submission¶
CATS = train.select_dtypes(include=['object']).columns.tolist()
NUMS = train.select_dtypes(include=['int64', 'float64']).columns.tolist()
y = train['exam_score']
train = train.drop(columns=['id', 'exam_score'])
test = test.drop(columns=['id'])
ohe = ce.OneHotEncoder(cols=CATS, use_cat_names=True)
train = ohe.fit_transform(train)
test = ohe.transform(test)
# 4. Assign AFTER encoding
X = train
X_test = test
n_splits = 10
kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)
y_preds = np.zeros(len(X_test))
oof_preds = np.zeros(len(X))
models = []
for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):
print(f"\nTraining fold {fold + 1}/{n_splits} >>>")
X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]
X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]
model = catboost.CatBoostRegressor(
task_type='GPU',
devices='0',
iterations=1000,
learning_rate=0.1,
eval_metric='RMSE',
early_stopping_rounds=100,
verbose=500
)
model.fit(
X_train,
y_train,
eval_set=(X_val, y_val),
)
oof_preds[val_idx] = model.predict(X_val)
y_preds += model.predict(X_test) / n_splits
models.append(model)
cv_rmse = np.sqrt(mean_squared_error(y, oof_preds))
print(f"\nCV RMSE: {cv_rmse:.4f}")
list_of_results = {}
list_of_results['CatBoost'] = cv_rmse
Training fold 1/10 >>> 0: learn: 17.6761085 test: 17.6097277 best: 17.6097277 (0) total: 5.08ms remaining: 5.07s 500: learn: 8.7546817 test: 8.7503048 best: 8.7503048 (500) total: 2.26s remaining: 2.25s 999: learn: 8.7037060 test: 8.7357812 best: 8.7357812 (999) total: 4.51s remaining: 0us bestTest = 8.735781191 bestIteration = 999 Training fold 2/10 >>> 0: learn: 17.6743531 test: 17.6177994 best: 17.6177994 (0) total: 5.08ms remaining: 5.08s 500: learn: 8.7514856 test: 8.8097772 best: 8.8097759 (499) total: 2.23s remaining: 2.22s 999: learn: 8.7014066 test: 8.7997655 best: 8.7997254 (998) total: 4.5s remaining: 0us bestTest = 8.799725375 bestIteration = 998 Shrink model to first 999 iterations. Training fold 3/10 >>> 0: learn: 17.6705904 test: 17.6540890 best: 17.6540890 (0) total: 5.13ms remaining: 5.13s 500: learn: 8.7554822 test: 8.7606697 best: 8.7606697 (500) total: 2.62s remaining: 2.61s 999: learn: 8.7051336 test: 8.7454954 best: 8.7454954 (999) total: 4.98s remaining: 0us bestTest = 8.745495439 bestIteration = 999 Training fold 4/10 >>> 0: learn: 17.6710718 test: 17.6582639 best: 17.6582639 (0) total: 4.81ms remaining: 4.81s 500: learn: 8.7484408 test: 8.8164259 best: 8.8164259 (500) total: 2.92s remaining: 2.91s 999: learn: 8.6986634 test: 8.8023644 best: 8.8023504 (998) total: 5.9s remaining: 0us bestTest = 8.802350426 bestIteration = 998 Shrink model to first 999 iterations. Training fold 5/10 >>> 0: learn: 17.6693886 test: 17.6697050 best: 17.6697050 (0) total: 6.97ms remaining: 6.96s 500: learn: 8.7546857 test: 8.7674466 best: 8.7674466 (500) total: 3.36s remaining: 3.35s 999: learn: 8.7043929 test: 8.7542457 best: 8.7541772 (996) total: 6.07s remaining: 0us bestTest = 8.754177234 bestIteration = 996 Shrink model to first 997 iterations. Training fold 6/10 >>> 0: learn: 17.6711309 test: 17.6500704 best: 17.6500704 (0) total: 5.56ms remaining: 5.56s 500: learn: 8.7520422 test: 8.7876008 best: 8.7876008 (500) total: 2.45s remaining: 2.44s 999: learn: 8.7020053 test: 8.7750404 best: 8.7749974 (995) total: 4.75s remaining: 0us bestTest = 8.7749974 bestIteration = 995 Shrink model to first 996 iterations. Training fold 7/10 >>> 0: learn: 17.6628483 test: 17.7328294 best: 17.7328294 (0) total: 4.43ms remaining: 4.43s 500: learn: 8.7503413 test: 8.8119134 best: 8.8119134 (500) total: 2.43s remaining: 2.42s 999: learn: 8.6997103 test: 8.7998940 best: 8.7998819 (998) total: 5.41s remaining: 0us bestTest = 8.799881853 bestIteration = 998 Shrink model to first 999 iterations. Training fold 8/10 >>> 0: learn: 17.6663612 test: 17.7034361 best: 17.7034361 (0) total: 7.76ms remaining: 7.75s 500: learn: 8.7519975 test: 8.7790479 best: 8.7790479 (500) total: 2.38s remaining: 2.37s 999: learn: 8.7016672 test: 8.7677774 best: 8.7677774 (999) total: 5.83s remaining: 0us bestTest = 8.767777406 bestIteration = 999 Training fold 9/10 >>> 0: learn: 17.6696338 test: 17.6755306 best: 17.6755306 (0) total: 6.3ms remaining: 6.29s 500: learn: 8.7501551 test: 8.7997055 best: 8.7996731 (498) total: 2.42s remaining: 2.41s 999: learn: 8.6991978 test: 8.7882836 best: 8.7882836 (999) total: 4.67s remaining: 0us bestTest = 8.788283596 bestIteration = 999 Training fold 10/10 >>> 0: learn: 17.6634794 test: 17.7254028 best: 17.7254028 (0) total: 4.51ms remaining: 4.51s 500: learn: 8.7467626 test: 8.8260905 best: 8.8260707 (499) total: 2.27s remaining: 2.26s 999: learn: 8.6965995 test: 8.8152501 best: 8.8151079 (994) total: 4.99s remaining: 0us bestTest = 8.815107883 bestIteration = 994 Shrink model to first 995 iterations. CV RMSE: 8.7784
def objective(trial):
params = {
'n_estimators': 5000, # Reduced from 20000
'learning_rate': trial.suggest_float('learning_rate', 0.02, 0.1, log=True),
'num_leaves': trial.suggest_int('num_leaves', 31, 127), # Narrower range
'max_depth': trial.suggest_int('max_depth', 4, 10), # Narrower range
'min_child_samples': trial.suggest_int('min_child_samples', 10, 50),
'subsample': trial.suggest_float('subsample', 0.6, 0.9),
'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 0.9),
'reg_alpha': trial.suggest_float('reg_alpha', 0.0, 5.0),
'reg_lambda': trial.suggest_float('reg_lambda', 0.0, 5.0),
'random_state': 42,
'verbosity': -1,
'device': 'gpu',
}
n_splits = 3 # Reduced from 5
kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)
oof_preds = np.zeros(len(X))
for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):
X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]
X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]
model = lgb.LGBMRegressor(**params)
model.fit(
X_train,
y_train,
eval_set=[(X_val, y_val)],
callbacks=[
lgb.early_stopping(50, verbose=False), # Reduced from 100
]
)
oof_preds[val_idx] = model.predict(X_val)
return np.sqrt(mean_squared_error(y, oof_preds))
study = optuna.create_study(
direction='minimize',
sampler=optuna.samplers.TPESampler(seed=42)
)
study.optimize(objective, n_trials=25, show_progress_bar=True) # Reduced from 50
print(f"\nBest CV RMSE: {study.best_value:.6f}")
print(f"\nBest hyperparameters:")
for key, value in study.best_params.items():
print(f" {key}: {value}")
best_params_lgb = study.best_params
[I 2026-01-25 18:22:03,002] A new study created in memory with name: no-name-4ecd6595-a6c6-4213-99a1-fdff40388995 Best trial: 0. Best value: 8.75709: 4%|▍ | 1/25 [01:42<40:59, 102.47s/it]
[I 2026-01-25 18:23:45,467] Trial 0 finished with value: 8.757093627237396 and parameters: {'learning_rate': 0.03654452355213248, 'num_leaves': 123, 'max_depth': 9, 'min_child_samples': 34, 'subsample': 0.6468055921327309, 'colsample_bytree': 0.562397808134481, 'reg_alpha': 0.2904180608409973, 'reg_lambda': 4.330880728874676}. Best is trial 0 with value: 8.757093627237396.
Best trial: 1. Best value: 8.75455: 8%|▊ | 2/25 [03:13<36:47, 95.96s/it]
[I 2026-01-25 18:25:16,869] Trial 1 finished with value: 8.754553273365604 and parameters: {'learning_rate': 0.05262490902114903, 'num_leaves': 99, 'max_depth': 4, 'min_child_samples': 49, 'subsample': 0.8497327922401265, 'colsample_bytree': 0.5849356442713105, 'reg_alpha': 0.9091248360355031, 'reg_lambda': 0.9170225492671691}. Best is trial 1 with value: 8.754553273365604.
Best trial: 1. Best value: 8.75455: 12%|█▏ | 3/25 [04:46<34:35, 94.36s/it]
[I 2026-01-25 18:26:49,329] Trial 2 finished with value: 8.756516886086983 and parameters: {'learning_rate': 0.03263519391284685, 'num_leaves': 81, 'max_depth': 7, 'min_child_samples': 21, 'subsample': 0.7835558684167139, 'colsample_bytree': 0.5557975442608167, 'reg_alpha': 1.4607232426760908, 'reg_lambda': 1.8318092164684585}. Best is trial 1 with value: 8.754553273365604.
Best trial: 1. Best value: 8.75455: 16%|█▌ | 4/25 [06:09<31:29, 89.97s/it]
[I 2026-01-25 18:28:12,572] Trial 3 finished with value: 8.755330919095686 and parameters: {'learning_rate': 0.04166863122305895, 'num_leaves': 107, 'max_depth': 5, 'min_child_samples': 31, 'subsample': 0.7777243706586128, 'colsample_bytree': 0.5185801650879991, 'reg_alpha': 3.0377242595071916, 'reg_lambda': 0.8526206184364576}. Best is trial 1 with value: 8.754553273365604.
Best trial: 1. Best value: 8.75455: 20%|██ | 5/25 [08:10<33:45, 101.26s/it]
[I 2026-01-25 18:30:13,836] Trial 4 finished with value: 8.754766646325614 and parameters: {'learning_rate': 0.022207471217033644, 'num_leaves': 123, 'max_depth': 10, 'min_child_samples': 43, 'subsample': 0.6913841307520112, 'colsample_bytree': 0.5390688456025535, 'reg_alpha': 3.4211651325607844, 'reg_lambda': 2.2007624686980067}. Best is trial 1 with value: 8.754553273365604.
Best trial: 5. Best value: 8.75408: 24%|██▍ | 6/25 [10:14<34:27, 108.81s/it]
[I 2026-01-25 18:32:17,308] Trial 5 finished with value: 8.754084118767887 and parameters: {'learning_rate': 0.02434058776747756, 'num_leaves': 79, 'max_depth': 4, 'min_child_samples': 47, 'subsample': 0.677633994480005, 'colsample_bytree': 0.7650089137415927, 'reg_alpha': 1.5585553804470549, 'reg_lambda': 2.600340105889054}. Best is trial 5 with value: 8.754084118767887.
Best trial: 5. Best value: 8.75408: 28%|██▊ | 7/25 [11:12<27:41, 92.29s/it]
[I 2026-01-25 18:33:15,585] Trial 6 finished with value: 8.761185264612369 and parameters: {'learning_rate': 0.04821299180434324, 'num_leaves': 48, 'max_depth': 10, 'min_child_samples': 41, 'subsample': 0.8818496824692568, 'colsample_bytree': 0.8579309401710595, 'reg_alpha': 2.9894998940554256, 'reg_lambda': 4.609371175115584}. Best is trial 5 with value: 8.754084118767887.
Best trial: 5. Best value: 8.75408: 32%|███▏ | 8/25 [13:16<29:01, 102.46s/it]
[I 2026-01-25 18:35:19,819] Trial 7 finished with value: 8.756884013109495 and parameters: {'learning_rate': 0.02306129016195514, 'num_leaves': 50, 'max_depth': 4, 'min_child_samples': 23, 'subsample': 0.7166031869068445, 'colsample_bytree': 0.6085396127095584, 'reg_alpha': 4.143687545759647, 'reg_lambda': 1.7837666334679465}. Best is trial 5 with value: 8.754084118767887.
Best trial: 8. Best value: 8.75219: 36%|███▌ | 9/25 [15:02<27:35, 103.48s/it]
[I 2026-01-25 18:37:05,555] Trial 8 finished with value: 8.75218673700281 and parameters: {'learning_rate': 0.03143364840330303, 'num_leaves': 83, 'max_depth': 4, 'min_child_samples': 42, 'subsample': 0.6223651931039312, 'colsample_bytree': 0.8947547746402069, 'reg_alpha': 3.861223846483287, 'reg_lambda': 0.993578407670862}. Best is trial 8 with value: 8.75218673700281.
Best trial: 8. Best value: 8.75219: 40%|████ | 10/25 [17:40<30:05, 120.34s/it]
[I 2026-01-25 18:39:43,654] Trial 9 finished with value: 8.754147685385568 and parameters: {'learning_rate': 0.020178542315724052, 'num_leaves': 110, 'max_depth': 8, 'min_child_samples': 39, 'subsample': 0.8313811040057837, 'colsample_bytree': 0.5296178606936361, 'reg_alpha': 1.7923286427213632, 'reg_lambda': 0.5793452976256486}. Best is trial 8 with value: 8.75218673700281.
Best trial: 8. Best value: 8.75219: 44%|████▍ | 11/25 [18:08<21:29, 92.14s/it]
[I 2026-01-25 18:40:11,830] Trial 10 finished with value: 8.765240160802971 and parameters: {'learning_rate': 0.09154868069571301, 'num_leaves': 31, 'max_depth': 6, 'min_child_samples': 10, 'subsample': 0.6043701023501858, 'colsample_bytree': 0.8630659181130071, 'reg_alpha': 4.828999576156499, 'reg_lambda': 3.3820892378437533}. Best is trial 8 with value: 8.75218673700281.
Best trial: 8. Best value: 8.75219: 48%|████▊ | 12/25 [20:04<21:30, 99.24s/it]
[I 2026-01-25 18:42:07,332] Trial 11 finished with value: 8.75230111696203 and parameters: {'learning_rate': 0.027942101634242596, 'num_leaves': 77, 'max_depth': 5, 'min_child_samples': 48, 'subsample': 0.65772147881177, 'colsample_bytree': 0.7675622840286896, 'reg_alpha': 1.9960702008242055, 'reg_lambda': 3.0976697156679958}. Best is trial 8 with value: 8.75218673700281.
Best trial: 8. Best value: 8.75219: 52%|█████▏ | 13/25 [21:46<20:02, 100.20s/it]
[I 2026-01-25 18:43:49,746] Trial 12 finished with value: 8.752796836600588 and parameters: {'learning_rate': 0.03112800822689242, 'num_leaves': 77, 'max_depth': 6, 'min_child_samples': 49, 'subsample': 0.6161933598319791, 'colsample_bytree': 0.7758146404482568, 'reg_alpha': 2.343526094439211, 'reg_lambda': 3.277425013185314}. Best is trial 8 with value: 8.75218673700281.
Best trial: 8. Best value: 8.75219: 56%|█████▌ | 14/25 [22:41<15:50, 86.41s/it]
[I 2026-01-25 18:44:44,277] Trial 13 finished with value: 8.754147864369646 and parameters: {'learning_rate': 0.06289596012254178, 'num_leaves': 64, 'max_depth': 5, 'min_child_samples': 37, 'subsample': 0.6513611695432825, 'colsample_bytree': 0.6826173426603852, 'reg_alpha': 3.999612751508559, 'reg_lambda': 3.330344500311682}. Best is trial 8 with value: 8.75218673700281.
Best trial: 14. Best value: 8.75202: 60%|██████ | 15/25 [24:21<15:05, 90.54s/it]
[I 2026-01-25 18:46:24,399] Trial 14 finished with value: 8.752023481139867 and parameters: {'learning_rate': 0.028346021123157483, 'num_leaves': 92, 'max_depth': 5, 'min_child_samples': 43, 'subsample': 0.7201474224873063, 'colsample_bytree': 0.8045436922164887, 'reg_alpha': 2.31057379200564, 'reg_lambda': 0.25437172295521027}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 64%|██████▍ | 16/25 [25:24<12:21, 82.34s/it]
[I 2026-01-25 18:47:27,689] Trial 15 finished with value: 8.756657108573442 and parameters: {'learning_rate': 0.03890398156078323, 'num_leaves': 94, 'max_depth': 6, 'min_child_samples': 26, 'subsample': 0.7356978938632825, 'colsample_bytree': 0.8969687269188596, 'reg_alpha': 4.982503512317817, 'reg_lambda': 0.014216691624587247}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 68%|██████▊ | 17/25 [27:08<11:50, 88.83s/it]
[I 2026-01-25 18:49:11,610] Trial 16 finished with value: 8.754597997743403 and parameters: {'learning_rate': 0.027752121443207392, 'num_leaves': 90, 'max_depth': 7, 'min_child_samples': 44, 'subsample': 0.772388514453492, 'colsample_bytree': 0.8144757159353142, 'reg_alpha': 3.7877381489304853, 'reg_lambda': 0.050570056912904504}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 72%|███████▏ | 18/25 [28:00<09:03, 77.58s/it]
[I 2026-01-25 18:50:03,015] Trial 17 finished with value: 8.754174940185681 and parameters: {'learning_rate': 0.06390829428406782, 'num_leaves': 64, 'max_depth': 5, 'min_child_samples': 35, 'subsample': 0.7151660082003752, 'colsample_bytree': 0.6892681265068146, 'reg_alpha': 2.5520101045470245, 'reg_lambda': 1.3742647449763135}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 76%|███████▌ | 19/25 [30:01<09:04, 90.73s/it]
[I 2026-01-25 18:52:04,358] Trial 18 finished with value: 8.752630230174022 and parameters: {'learning_rate': 0.028605097323580003, 'num_leaves': 110, 'max_depth': 4, 'min_child_samples': 29, 'subsample': 0.8211218874400065, 'colsample_bytree': 0.8255974822311326, 'reg_alpha': 4.421058216715657, 'reg_lambda': 1.2906218036301278}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 80%|████████ | 20/25 [31:20<07:16, 87.35s/it]
[I 2026-01-25 18:53:23,853] Trial 19 finished with value: 8.761422531818905 and parameters: {'learning_rate': 0.03617189967745581, 'num_leaves': 65, 'max_depth': 7, 'min_child_samples': 11, 'subsample': 0.6254161042608545, 'colsample_bytree': 0.8958295303740914, 'reg_alpha': 3.264423346529435, 'reg_lambda': 0.4508702647371394}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 84%|████████▍ | 21/25 [32:25<05:21, 80.41s/it]
[I 2026-01-25 18:54:28,067] Trial 20 finished with value: 8.753689129071741 and parameters: {'learning_rate': 0.05415400108137362, 'num_leaves': 89, 'max_depth': 5, 'min_child_samples': 44, 'subsample': 0.6903091479975801, 'colsample_bytree': 0.7271395673780533, 'reg_alpha': 2.5694057876825567, 'reg_lambda': 1.207917130883944}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 88%|████████▊ | 22/25 [34:39<04:49, 96.54s/it]
[I 2026-01-25 18:56:42,236] Trial 21 finished with value: 8.752125569322361 and parameters: {'learning_rate': 0.026573736989523825, 'num_leaves': 73, 'max_depth': 5, 'min_child_samples': 50, 'subsample': 0.6587536683623662, 'colsample_bytree': 0.7817027743696668, 'reg_alpha': 2.0208437711808127, 'reg_lambda': 2.464092995102434}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 92%|█████████▏| 23/25 [36:39<03:27, 103.73s/it]
[I 2026-01-25 18:58:42,728] Trial 22 finished with value: 8.752473569755438 and parameters: {'learning_rate': 0.02500495174812093, 'num_leaves': 70, 'max_depth': 6, 'min_child_samples': 45, 'subsample': 0.6372851846141122, 'colsample_bytree': 0.8119787336647469, 'reg_alpha': 2.154653181289028, 'reg_lambda': 4.084471294411268}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 96%|█████████▌| 24/25 [38:11<01:40, 100.25s/it]
[I 2026-01-25 19:00:14,859] Trial 23 finished with value: 8.75285351444547 and parameters: {'learning_rate': 0.032915096126697234, 'num_leaves': 100, 'max_depth': 4, 'min_child_samples': 40, 'subsample': 0.6764022442000822, 'colsample_bytree': 0.7315363157092678, 'reg_alpha': 0.8103426473682367, 'reg_lambda': 2.6010687028828423}. Best is trial 14 with value: 8.752023481139867.
Best trial: 14. Best value: 8.75202: 100%|██████████| 25/25 [39:38<00:00, 95.16s/it]
[I 2026-01-25 19:01:41,928] Trial 24 finished with value: 8.753208596385173 and parameters: {'learning_rate': 0.026460652559166072, 'num_leaves': 54, 'max_depth': 5, 'min_child_samples': 50, 'subsample': 0.7459852619483623, 'colsample_bytree': 0.8505344753059966, 'reg_alpha': 1.1299774540530483, 'reg_lambda': 0.421664068722281}. Best is trial 14 with value: 8.752023481139867.
Best CV RMSE: 8.752023
Best hyperparameters:
learning_rate: 0.028346021123157483
num_leaves: 92
max_depth: 5
min_child_samples: 43
subsample: 0.7201474224873063
colsample_bytree: 0.8045436922164887
reg_alpha: 2.31057379200564
reg_lambda: 0.25437172295521027
Best hyperparameters: learning_rate: 0.028346021123157483 num_leaves: 92 max_depth: 5 min_child_samples: 43 subsample: 0.7201474224873063 colsample_bytree: 0.8045436922164887 reg_alpha: 2.31057379200564 reg_lambda: 0.25437172295521027
n_splits = 20
kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)
y_preds = np.zeros(len(X_test))
oof_preds = np.zeros(len(X))
models = []
for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):
print(f"\nTraining fold {fold + 1}/{n_splits} >>>")
X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]
X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]
model = lgb.LGBMRegressor(
**best_params_lgb,
EarlyStopping_rounds=100,
random_state=42,
verbosity=-1,
metric='rmse',
device="gpu",
)
model.fit(
X_train,
y_train,
eval_set=[(X_val, y_val)],
callbacks=[
lgb.early_stopping(100),
lgb.log_evaluation(500)
]
)
oof_preds[val_idx] = model.predict(X_val)
y_preds += model.predict(X_test) / n_splits
models.append(model)
cv_rmse = np.sqrt(mean_squared_error(y, oof_preds))
print(f"\nCV RMSE: {cv_rmse:.4f}")
list_of_results = {}
list_of_results['LightGBM-2Ofolds-study'] = cv_rmse
Training fold 1/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.16016 Training fold 2/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.25149 Training fold 3/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.21651 Training fold 4/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.30162 Training fold 5/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.20525 Training fold 6/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.21936 Training fold 7/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.29118 Training fold 8/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.25117 Training fold 9/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.22323 Training fold 10/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.24203 Training fold 11/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.21689 Training fold 12/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.26325 Training fold 13/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.24219 Training fold 14/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.30379 Training fold 15/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.23178 Training fold 16/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.25148 Training fold 17/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.22516 Training fold 18/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.3019 Training fold 19/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.26307 Training fold 20/20 >>> Training until validation scores don't improve for 100 rounds Did not meet early stopping. Best iteration is: [100] valid_0's rmse: 9.32474 CV RMSE: 9.2494
submission = pd.read_csv('../data/sample_submission.csv')
submission['exam_score'] = y_preds
submission.to_csv('../submissions/lightgbm_submission-2.csv', index=False)
submission.head()
| id | exam_score | |
|---|---|---|
| 0 | 630000 | 71.417941 |
| 1 | 630001 | 69.313881 |
| 2 | 630002 | 83.645040 |
| 3 | 630003 | 54.469451 |
| 4 | 630004 | 48.456100 |
import pandas as pd
sub1 = pd.read_csv('../submissions/lightgbm_submission.csv')
sub2 = pd.read_csv('../submissions/best_public.csv')
# Try different weight combinations
best_weights = []
for w1 in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
w2 = 1.0 - w1
ensemble = sub1.copy()
ensemble.iloc[:, 1:] = w1 * sub1.iloc[:, 1:] + w2 * sub2.iloc[:, 1:]
ensemble.to_csv(f'ensemble_w{w1:.1f}.csv', index=False)
print(f"Created ensemble with weights: {w1:.1f} / {w2:.1f}")
Created ensemble with weights: 0.0 / 1.0 Created ensemble with weights: 0.1 / 0.9 Created ensemble with weights: 0.2 / 0.8 Created ensemble with weights: 0.3 / 0.7 Created ensemble with weights: 0.4 / 0.6 Created ensemble with weights: 0.5 / 0.5 Created ensemble with weights: 0.6 / 0.4 Created ensemble with weights: 0.7 / 0.3 Created ensemble with weights: 0.8 / 0.2 Created ensemble with weights: 0.9 / 0.1 Created ensemble with weights: 1.0 / 0.0