Your Goal: Predict the likelihood of customer churn.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import minimize
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif
from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge, Lasso
from sklearn.metrics import (
accuracy_score, f1_score, precision_score, recall_score,
mean_absolute_error, mean_squared_error, r2_score,
root_mean_squared_error, roc_auc_score
)
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import LabelEncoder, OrdinalEncoder, StandardScaler
from sklearn.svm import SVC, SVR
import xgboost
import lightgbm
import catboost
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from imblearn.over_sampling import SMOTE
import optuna
import shap
import warnings
warnings.filterwarnings("ignore")
SEED = 42
np.random.seed(SEED)
c:\Users\user\miniconda3\Lib\site-packages\tqdm\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html from .autonotebook import tqdm as notebook_tqdm
sample_submission = pd.read_csv('sample_submission.csv')
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
best_public_ensemble = pd.read_csv("best-public-ensemble.csv")
best_public_lb = pd.read_csv("best-public-LB.csv")
# Analyze data
print(f"Train shape: {train.shape}")
print(f"Test shape: {test.shape}")
print(f"\nTarget distribution:")
print(train['Irrigation_Need'].value_counts())
print(f"\nTarget distribution (%):")
print(train['Irrigation_Need'].value_counts(normalize=True) * 100)
# Compare submissions
print(f"\n--- Submission Comparison ---")
print(f"Ensemble predictions:\n{best_public_ensemble['Irrigation_Need'].value_counts()}")
print(f"\nLB predictions:\n{best_public_lb['Irrigation_Need'].value_counts()}")
# Find disagreements
disagreements = best_public_ensemble[best_public_ensemble['Irrigation_Need'] != best_public_lb['Irrigation_Need']]
print(f"\nDisagreements between ensemble and LB: {len(disagreements)}")
Train shape: (630000, 21) Test shape: (270000, 20) Target distribution: Irrigation_Need Low 369917 Medium 239074 High 21009 Name: count, dtype: int64 Target distribution (%): Irrigation_Need Low 58.716984 Medium 37.948254 High 3.334762 Name: proportion, dtype: float64 --- Submission Comparison --- Ensemble predictions: Irrigation_Need Low 159436 Medium 100488 High 10076 Name: count, dtype: int64 LB predictions: Irrigation_Need Low 159491 Medium 100433 High 10076 Name: count, dtype: int64 Disagreements between ensemble and LB: 55
# Detailed feature analysis
print("=== Feature Types ===")
print(train.dtypes)
print("\n=== Missing Values ===")
print(train.isnull().sum().sum(), "total missing in train")
print(test.isnull().sum().sum(), "total missing in test")
# Analyze High class patterns
high_cases = train[train['Irrigation_Need'] == 'High']
low_cases = train[train['Irrigation_Need'] == 'Low']
medium_cases = train[train['Irrigation_Need'] == 'Medium']
print("\n=== Feature Means by Class ===")
numeric_cols = train.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
if col != 'id':
print(f"{col}: High={high_cases[col].mean():.2f}, Medium={medium_cases[col].mean():.2f}, Low={low_cases[col].mean():.2f}")
=== Feature Types === id int64 Soil_Type object Soil_pH float64 Soil_Moisture float64 Organic_Carbon float64 Electrical_Conductivity float64 Temperature_C float64 Humidity float64 Rainfall_mm float64 Sunlight_Hours float64 Wind_Speed_kmh float64 Crop_Type object Crop_Growth_Stage object Season object Irrigation_Type object Water_Source object Field_Area_hectare float64 Mulching_Used object Previous_Irrigation_mm float64 Region object Irrigation_Need object dtype: object === Missing Values === 0 total missing in train 0 total missing in test === Feature Means by Class === Soil_pH: High=6.58, Medium=6.47, Low=6.49 Soil_Moisture: High=17.67, Medium=29.74, Low=43.31 Organic_Carbon: High=0.92, Medium=0.93, Low=0.92 Electrical_Conductivity: High=1.69, Medium=1.77, Low=1.73 Temperature_C: High=34.57, Medium=28.89, Low=25.35 Humidity: High=61.12, Medium=61.00, Low=61.95 Rainfall_mm: High=989.16, Medium=1444.48, Low=1500.53 Sunlight_Hours: High=7.46, Medium=7.52, Low=7.51 Wind_Speed_kmh: High=14.64, Medium=11.79, Low=9.22 Field_Area_hectare: High=7.53, Medium=7.63, Low=7.45 Previous_Irrigation_mm: High=63.05, Medium=63.18, Low=61.72
# Feature preparation: native LightGBM categoricals + inverse frequency class weights
import lightgbm as lgb
cat_cols = ['Soil_Type', 'Crop_Type', 'Crop_Growth_Stage', 'Season',
'Irrigation_Type', 'Water_Source', 'Mulching_Used', 'Region']
num_cols = ['Soil_pH', 'Soil_Moisture', 'Organic_Carbon', 'Electrical_Conductivity',
'Temperature_C', 'Humidity', 'Rainfall_mm', 'Sunlight_Hours',
'Wind_Speed_kmh', 'Field_Area_hectare', 'Previous_Irrigation_mm']
# Encode target
le = LabelEncoder()
y = le.fit_transform(train['Irrigation_Need'])
print(f"Classes: {le.classes_}") # ['High', 'Low', 'Medium']
# Inverse frequency class weights — replaces hand-tuned {High:3, Low:1, Med:1.5}
class_counts = np.bincount(y)
class_weights_dict = {i: len(y) / (3 * class_counts[i]) for i in range(3)}
print(f"Inverse frequency weights: {class_weights_dict}")
# High ~10x, Low ~0.57x, Medium ~0.88x
sample_weights = np.array([class_weights_dict[c] for c in y])
# LightGBM path: DataFrame with native category dtype (strictly better than ordinal for trees)
train_lgb = train[num_cols + cat_cols].copy()
test_lgb = test[num_cols + cat_cols].copy()
for col in cat_cols:
train_lgb[col] = train_lgb[col].astype('category')
test_lgb[col] = test_lgb[col].astype('category')
# XGBoost / CatBoost path: ordinal encoding, no scaling needed for tree models
oe = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)
train_cat = oe.fit_transform(train[cat_cols])
test_cat = oe.transform(test[cat_cols])
X = np.hstack([train[num_cols].values, train_cat])
X_test = np.hstack([test[num_cols].values, test_cat])
n_splits = 5
skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)
print(f"train_lgb shape : {train_lgb.shape}")
print(f"X shape (XGB/CB): {X.shape}")
Classes: ['High' 'Low' 'Medium']
Inverse frequency weights: {0: np.float64(9.995716121662145), 1: np.float64(0.5676949153458749), 2: np.float64(0.8783891180136694)}
train_lgb shape : (630000, 19)
X shape (XGB/CB): (630000, 19)
# LightGBM base: native categoricals, tuned hyperparameters, inverse frequency weights
lgb_params = {
'objective': 'multiclass',
'num_class': 3,
'metric': 'multi_logloss',
'boosting_type': 'gbdt',
'learning_rate': 0.02, # was 0.05
'num_leaves': 127, # was 63
'max_depth': -1,
'min_child_samples': 20, # was 50 — smaller leaf size helps sparse High class
'feature_fraction': 0.8,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'lambda_l1': 0.1,
'lambda_l2': 0.1,
'verbose': -1,
'seed': SEED,
'n_jobs': -1
}
oof_preds = np.zeros((len(train_lgb), 3))
test_preds = np.zeros((len(test_lgb), 3))
cv_scores = []
for fold, (train_idx, val_idx) in enumerate(skf.split(train_lgb, y)):
print(f"Fold {fold + 1}/{n_splits}")
X_tr, X_vl = train_lgb.iloc[train_idx], train_lgb.iloc[val_idx]
y_tr, y_vl = y[train_idx], y[val_idx]
train_data = lgb.Dataset(X_tr, label=y_tr, weight=sample_weights[train_idx])
val_data = lgb.Dataset(X_vl, label=y_vl, reference=train_data)
model = lgb.train(
lgb_params,
train_data,
num_boost_round=2000, # was 1000
valid_sets=[val_data],
callbacks=[lgb.early_stopping(100), lgb.log_evaluation(200)]
)
oof_preds[val_idx] = model.predict(X_vl)
test_preds += model.predict(test_lgb) / n_splits
fold_f1 = f1_score(y_vl, np.argmax(oof_preds[val_idx], axis=1), average='macro')
cv_scores.append(fold_f1)
print(f"Fold {fold + 1} Macro F1: {fold_f1:.5f}\n")
print(f"CV Mean Macro F1: {np.mean(cv_scores):.5f} (+/- {np.std(cv_scores):.5f})")
Fold 1/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0868831 [400] valid_0's multi_logloss: 0.0658711 [600] valid_0's multi_logloss: 0.0614606 [800] valid_0's multi_logloss: 0.059702 [1000] valid_0's multi_logloss: 0.0588304 [1200] valid_0's multi_logloss: 0.0585541 Early stopping, best iteration is: [1216] valid_0's multi_logloss: 0.0585438 Fold 1 Macro F1: 0.96873 Fold 2/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0882623 [400] valid_0's multi_logloss: 0.0672058 [600] valid_0's multi_logloss: 0.0628341 [800] valid_0's multi_logloss: 0.0610142 [1000] valid_0's multi_logloss: 0.0602539 [1200] valid_0's multi_logloss: 0.0600171 Early stopping, best iteration is: [1250] valid_0's multi_logloss: 0.0599898 Fold 2 Macro F1: 0.96876 Fold 3/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0875167 [400] valid_0's multi_logloss: 0.0662144 [600] valid_0's multi_logloss: 0.0618011 [800] valid_0's multi_logloss: 0.0599564 [1000] valid_0's multi_logloss: 0.059137 [1200] valid_0's multi_logloss: 0.0589157 Early stopping, best iteration is: [1174] valid_0's multi_logloss: 0.0589035 Fold 3 Macro F1: 0.97032 Fold 4/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0874361 [400] valid_0's multi_logloss: 0.0658403 [600] valid_0's multi_logloss: 0.0612771 [800] valid_0's multi_logloss: 0.0595312 [1000] valid_0's multi_logloss: 0.058684 [1200] valid_0's multi_logloss: 0.0584779 Early stopping, best iteration is: [1266] valid_0's multi_logloss: 0.0584436 Fold 4 Macro F1: 0.96807 Fold 5/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0863632 [400] valid_0's multi_logloss: 0.0646446 [600] valid_0's multi_logloss: 0.0601181 [800] valid_0's multi_logloss: 0.0582912 [1000] valid_0's multi_logloss: 0.0575154 [1200] valid_0's multi_logloss: 0.0572361 Early stopping, best iteration is: [1253] valid_0's multi_logloss: 0.0571585 Fold 5 Macro F1: 0.96966 CV Mean Macro F1: 0.96911 (+/- 0.00079)
# Evaluate OOF predictions
oof_pred_labels = np.argmax(oof_preds, axis=1)
oof_f1_macro = f1_score(y, oof_pred_labels, average='macro')
oof_f1_weighted = f1_score(y, oof_pred_labels, average='weighted')
print(f"OOF Macro F1: {oof_f1_macro:.5f}")
print(f"OOF Weighted F1: {oof_f1_weighted:.5f}")
print(f"\nOOF Prediction distribution:")
print(pd.Series(le.inverse_transform(oof_pred_labels)).value_counts())
OOF Macro F1: 0.96911 OOF Weighted F1: 0.98418 OOF Prediction distribution: Low 373199 Medium 236184 High 20617 Name: count, dtype: int64
# Threshold optimization: tune per-class decision boundaries to maximize macro F1
#
# Default argmax requires P(High) > P(Low) AND P(High) > P(Medium), which is too strict
# for a 3.3%-prevalence class. Dividing by class-specific thresholds shifts the boundaries.
def threshold_predict(probas, thresholds):
adjusted = probas / np.clip(np.array(thresholds), 1e-9, None)
return np.argmax(adjusted, axis=1)
def neg_macro_f1(thresholds, probas, labels):
return -f1_score(labels, threshold_predict(probas, thresholds), average='macro')
result = minimize(
neg_macro_f1,
x0=[1.0, 1.0, 1.0],
args=(oof_preds, y),
method='Nelder-Mead',
options={'xatol': 1e-6, 'fatol': 1e-6, 'maxiter': 20000}
)
best_thresholds = result.x
print(f"Optimal thresholds (High, Low, Medium): {best_thresholds}")
print(f"Default OOF Macro F1: {oof_f1_macro:.5f}")
print(f"Tuned OOF Macro F1: {-result.fun:.5f}")
tuned_labels = le.inverse_transform(threshold_predict(oof_preds, best_thresholds))
print(f"\nTuned OOF prediction distribution:")
print(pd.Series(tuned_labels).value_counts())
Optimal thresholds (High, Low, Medium): [1.58865222 0.62150828 0.72547884] Default OOF Macro F1: 0.96911 Tuned OOF Macro F1: 0.97005 Tuned OOF prediction distribution: Low 373331 Medium 236509 High 20160 Name: count, dtype: int64
# Feature engineering for both LGB (DataFrame) and XGB/CB (numpy) paths
def add_features(df):
df = df.copy()
df['Moisture_Temp_Ratio'] = df['Soil_Moisture'] / (df['Temperature_C'] + 1)
df['Rainfall_Wind_Ratio'] = df['Rainfall_mm'] / (df['Wind_Speed_kmh'] + 1)
df['Temp_Humidity_Product']= df['Temperature_C'] * df['Humidity'] / 100
df['Soil_Quality_Index'] = df['Soil_pH'] * df['Organic_Carbon'] / 10
df['Water_Stress_Index'] = df['Temperature_C'] / (df['Soil_Moisture'] + 1) * df['Wind_Speed_kmh']
df['Drought_Indicator'] = (df['Rainfall_mm'] < 500).astype(int) * (df['Soil_Moisture'] < 20).astype(int)
df['Heat_Stress_Indicator']= (df['Temperature_C'] > 35).astype(int) * (df['Humidity'] < 50).astype(int)
df['Soil_Moisture_sq'] = df['Soil_Moisture'] ** 2
df['Temperature_C_sq'] = df['Temperature_C'] ** 2
df['Rainfall_mm_sq'] = df['Rainfall_mm'] ** 2
return df
# LGB path: DataFrame preserving category dtypes
train_fe_lgb = add_features(train_lgb)
test_fe_lgb = add_features(test_lgb)
for col in cat_cols:
train_fe_lgb[col] = train_fe_lgb[col].astype('category')
test_fe_lgb[col] = test_fe_lgb[col].astype('category')
# Numeric column names in FE set (used to index X_fe and for cb_cat_indices)
new_num_cols = [c for c in train_fe_lgb.columns if c not in cat_cols]
# XGB / CB path: ordinal-encode categoricals, no scaling
train_fe_raw = add_features(train[num_cols + cat_cols])
test_fe_raw = add_features(test[num_cols + cat_cols])
oe_fe = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)
train_cat_fe = oe_fe.fit_transform(train_fe_raw[cat_cols])
test_cat_fe = oe_fe.transform(test_fe_raw[cat_cols])
X_fe = np.hstack([train_fe_raw[new_num_cols].values, train_cat_fe])
X_test_fe = np.hstack([test_fe_raw[new_num_cols].values, test_cat_fe])
print(f"train_fe_lgb shape: {train_fe_lgb.shape}")
print(f"X_fe shape (XGB/CB): {X_fe.shape}")
print(f"New features added: {len(new_num_cols) - len(num_cols)}")
train_fe_lgb shape: (630000, 29) X_fe shape (XGB/CB): (630000, 29) New features added: 10
# LightGBM with engineered features (native categoricals + tuned params)
print("Training LightGBM with engineered features...")
oof_preds_fe = np.zeros((len(train_fe_lgb), 3))
test_preds_fe = np.zeros((len(test_fe_lgb), 3))
cv_scores_fe = []
for fold, (train_idx, val_idx) in enumerate(skf.split(train_fe_lgb, y)):
print(f"Fold {fold + 1}/{n_splits}")
X_tr, X_vl = train_fe_lgb.iloc[train_idx], train_fe_lgb.iloc[val_idx]
y_tr, y_vl = y[train_idx], y[val_idx]
train_data = lgb.Dataset(X_tr, label=y_tr, weight=sample_weights[train_idx])
val_data = lgb.Dataset(X_vl, label=y_vl, reference=train_data)
model = lgb.train(
lgb_params,
train_data,
num_boost_round=2000,
valid_sets=[val_data],
callbacks=[lgb.early_stopping(100), lgb.log_evaluation(200)]
)
oof_preds_fe[val_idx] = model.predict(X_vl)
test_preds_fe += model.predict(test_fe_lgb) / n_splits
fold_f1 = f1_score(y_vl, np.argmax(oof_preds_fe[val_idx], axis=1), average='macro')
cv_scores_fe.append(fold_f1)
print(f"Fold {fold + 1} Macro F1: {fold_f1:.5f}\n")
print(f"CV Mean Macro F1 (LGB FE): {np.mean(cv_scores_fe):.5f} (+/- {np.std(cv_scores_fe):.5f})")
Training LightGBM with engineered features... Fold 1/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0789514 [400] valid_0's multi_logloss: 0.0645872 [600] valid_0's multi_logloss: 0.0611905 [800] valid_0's multi_logloss: 0.0597502 [1000] valid_0's multi_logloss: 0.0591013 [1200] valid_0's multi_logloss: 0.0590087 Early stopping, best iteration is: [1124] valid_0's multi_logloss: 0.0589665 Fold 1 Macro F1: 0.96874 Fold 2/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0804748 [400] valid_0's multi_logloss: 0.0659368 [600] valid_0's multi_logloss: 0.0624369 [800] valid_0's multi_logloss: 0.0609968 [1000] valid_0's multi_logloss: 0.0604903 Early stopping, best iteration is: [1055] valid_0's multi_logloss: 0.0604235 Fold 2 Macro F1: 0.96831 Fold 3/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0798092 [400] valid_0's multi_logloss: 0.0650985 [600] valid_0's multi_logloss: 0.0616846 [800] valid_0's multi_logloss: 0.0603285 [1000] valid_0's multi_logloss: 0.0596958 Early stopping, best iteration is: [1049] valid_0's multi_logloss: 0.0596347 Fold 3 Macro F1: 0.97019 Fold 4/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0792085 [400] valid_0's multi_logloss: 0.0644532 [600] valid_0's multi_logloss: 0.061203 [800] valid_0's multi_logloss: 0.0598357 [1000] valid_0's multi_logloss: 0.0593048 [1200] valid_0's multi_logloss: 0.0592468 Early stopping, best iteration is: [1158] valid_0's multi_logloss: 0.0592283 Fold 4 Macro F1: 0.96859 Fold 5/5 Training until validation scores don't improve for 100 rounds [200] valid_0's multi_logloss: 0.0781335 [400] valid_0's multi_logloss: 0.0633705 [600] valid_0's multi_logloss: 0.059859 [800] valid_0's multi_logloss: 0.0584043 [1000] valid_0's multi_logloss: 0.0578193 [1200] valid_0's multi_logloss: 0.0577286 Early stopping, best iteration is: [1145] valid_0's multi_logloss: 0.0577042 Fold 5 Macro F1: 0.96942 CV Mean Macro F1 (LGB FE): 0.96905 (+/- 0.00068)
# XGBoost training
print("Training XGBoost...")
import xgboost as xgb
xgb_params = {
'objective': 'multi:softprob',
'num_class': 3,
'eval_metric': 'mlogloss',
'learning_rate': 0.02, # was 0.05
'max_depth': 8,
'min_child_weight': 3, # was 5
'subsample': 0.8,
'colsample_bytree': 0.8,
'reg_alpha': 0.1,
'reg_lambda': 0.1,
'seed': SEED,
'tree_method': 'hist',
'verbosity': 0
}
oof_preds_xgb = np.zeros((len(X_fe), 3))
test_preds_xgb = np.zeros((len(X_test_fe), 3))
for fold, (train_idx, val_idx) in enumerate(skf.split(X_fe, y)):
print(f"XGB Fold {fold + 1}/{n_splits}")
X_tr, X_vl = X_fe[train_idx], X_fe[val_idx]
y_tr, y_vl = y[train_idx], y[val_idx]
dtrain = xgb.DMatrix(X_tr, label=y_tr, weight=sample_weights[train_idx])
dval = xgb.DMatrix(X_vl, label=y_vl)
dtest = xgb.DMatrix(X_test_fe)
model = xgb.train(
xgb_params,
dtrain,
num_boost_round=2000, # was 1000
evals=[(dval, 'val')],
early_stopping_rounds=100, # was 50
verbose_eval=200
)
oof_preds_xgb[val_idx] = model.predict(dval)
test_preds_xgb += model.predict(dtest) / n_splits
xgb_f1 = f1_score(y, np.argmax(oof_preds_xgb, axis=1), average='macro')
print(f"XGBoost OOF Macro F1: {xgb_f1:.5f}")
Training XGBoost... XGB Fold 1/5 [0] val-mlogloss:1.07595 [200] val-mlogloss:0.10205 [400] val-mlogloss:0.07073 [600] val-mlogloss:0.06378 [800] val-mlogloss:0.06066 [1000] val-mlogloss:0.05886 [1200] val-mlogloss:0.05780 [1400] val-mlogloss:0.05720 [1600] val-mlogloss:0.05682 [1800] val-mlogloss:0.05665 [1999] val-mlogloss:0.05666 XGB Fold 2/5 [0] val-mlogloss:1.07592 [200] val-mlogloss:0.10336 [400] val-mlogloss:0.07218 [600] val-mlogloss:0.06531 [800] val-mlogloss:0.06211 [1000] val-mlogloss:0.06029 [1200] val-mlogloss:0.05920
# CatBoost training with native categorical features (DataFrame path, string categoricals)
print("Training CatBoost...")
import catboost as cb
# CatBoost requires string or int categoricals — use train_fe_lgb and convert category→str
def to_cb_df(df):
out = df.copy()
for col in cat_cols:
out[col] = out[col].astype(str)
return out
train_fe_cb = to_cb_df(train_fe_lgb)
test_fe_cb = to_cb_df(test_fe_lgb)
oof_preds_cb = np.zeros((len(train_fe_cb), 3))
test_preds_cb = np.zeros((len(test_fe_cb), 3))
for fold, (train_idx, val_idx) in enumerate(skf.split(train_fe_cb, y)):
print(f"CB Fold {fold + 1}/{n_splits}")
X_tr, X_vl = train_fe_cb.iloc[train_idx], train_fe_cb.iloc[val_idx]
y_tr, y_vl = y[train_idx], y[val_idx]
train_pool = cb.Pool(X_tr, y_tr, weight=sample_weights[train_idx], cat_features=cat_cols)
val_pool = cb.Pool(X_vl, y_vl, cat_features=cat_cols)
test_pool = cb.Pool(test_fe_cb, cat_features=cat_cols)
model = cb.CatBoostClassifier(
iterations=2000,
learning_rate=0.02,
depth=8,
loss_function='MultiClass',
eval_metric='TotalF1:average=Macro',
random_seed=SEED,
early_stopping_rounds=100,
verbose=200
)
model.fit(train_pool, eval_set=val_pool, use_best_model=True)
oof_preds_cb[val_idx] = model.predict_proba(val_pool)
test_preds_cb += model.predict_proba(test_pool) / n_splits
cb_f1 = f1_score(y, np.argmax(oof_preds_cb, axis=1), average='macro')
print(f"CatBoost OOF Macro F1: {cb_f1:.5f}")
# Final ensemble: weighted blend + threshold optimization
print("=== Creating Final Ensemble ===")
weights = {'lgb_fe': 0.40, 'xgb': 0.35, 'cb': 0.25}
ensemble_oof = (weights['lgb_fe'] * oof_preds_fe +
weights['xgb'] * oof_preds_xgb +
weights['cb'] * oof_preds_cb)
ensemble_test = (weights['lgb_fe'] * test_preds_fe +
weights['xgb'] * test_preds_xgb +
weights['cb'] * test_preds_cb)
ensemble_f1 = f1_score(y, np.argmax(ensemble_oof, axis=1), average='macro')
print(f"Ensemble OOF Macro F1 (argmax): {ensemble_f1:.5f}")
# Optimize per-class thresholds on ensemble OOF predictions
result_ens = minimize(
neg_macro_f1,
x0=[1.0, 1.0, 1.0],
args=(ensemble_oof, y),
method='Nelder-Mead',
options={'xatol': 1e-6, 'fatol': 1e-6, 'maxiter': 20000}
)
ensemble_thresholds = result_ens.x
ensemble_f1_tuned = -result_ens.fun
print(f"Ensemble OOF Macro F1 (tuned thresholds): {ensemble_f1_tuned:.5f}")
print(f"Optimal thresholds (High, Low, Medium): {ensemble_thresholds}")
argmax_labels = le.inverse_transform(np.argmax(ensemble_test, axis=1))
tuned_labels = le.inverse_transform(threshold_predict(ensemble_test, ensemble_thresholds))
print(f"\nArgmax distribution: {pd.Series(argmax_labels).value_counts().to_dict()}")
print(f"Tuned distribution: {pd.Series(tuned_labels).value_counts().to_dict()}")
print(f"Target (Best LB): {best_public_lb['Irrigation_Need'].value_counts().to_dict()}")
# Final submission using threshold-optimized ensemble predictions
print("=== Final Submission ===")
final_labels = le.inverse_transform(threshold_predict(ensemble_test, ensemble_thresholds))
submission = pd.DataFrame({
'id': test['id'],
'Irrigation_Need': final_labels
})
submission.to_csv('improved_submission.csv', index=False)
dist = submission['Irrigation_Need'].value_counts()
print(f"Distribution:\n{dist}")
print(f"\nSaved to improved_submission.csv")
print(f"\n=== Comparison ===")
print(f"{'Submission':<25} {'Low':>8} {'Medium':>8} {'High':>8}")
print("-" * 50)
print(f"{'Best Public Ensemble':<25} {159436:>8} {100488:>8} {10076:>8}")
print(f"{'Best Public LB':<25} {159491:>8} {100433:>8} {10076:>8}")
print(f"{'Our Model':<25} {dist.get('Low',0):>8} {dist.get('Medium',0):>8} {dist.get('High',0):>8}")
print("=" * 60)
print("SUMMARY: Irrigation Need Prediction Improvements")
print("=" * 60)
print(f"""
CHANGES MADE:
1. Inverse frequency class weights
High: {len(y) / (3 * class_counts[0]):.1f}x (was 3.0x)
Low: {len(y) / (3 * class_counts[1]):.2f}x (was 1.0x)
Med: {len(y) / (3 * class_counts[2]):.2f}x (was 1.5x)
2. LightGBM native categorical encoding (was OrdinalEncoder + StandardScaler)
3. Tuned hyperparameters: LR=0.02, leaves=127, min_child_samples=20
4. More training rounds: 2000 + early_stopping=100 (was 1000 + ES=50)
5. CatBoost cat_features indices passed to Pool (was raw ordinal floats)
6. Threshold optimization on ensemble OOF predictions (was ad-hoc oracle blending)
MODEL OOF MACRO F1:
LGB FE: {np.mean(cv_scores_fe):.5f}
XGBoost: {xgb_f1:.5f}
CatBoost: {cb_f1:.5f}
Ensemble (argmax): {ensemble_f1:.5f}
Ensemble (tuned thresholds):{ensemble_f1_tuned:.5f}
""")