import json
import os
import pickle
import time
from collections import Counter

import numpy as np
import pandas as pd
from tqdm import tqdm
import category_encoders as ce

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif
from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge, Lasso
from sklearn.metrics import (
    accuracy_score, f1_score, precision_score, recall_score,
    mean_absolute_error, mean_squared_error, r2_score,
    root_mean_squared_error, roc_auc_score
)
from sklearn.model_selection import StratifiedKFold
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.svm import SVC, SVR

import xgboost as xg
import lightgbm as lg
import catboost as cb

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from imblearn.over_sampling import SMOTE

import optuna
import shap
import warnings
warnings.filterwarnings("ignore")

SEED = 42
np.random.seed(SEED)

c:\Users\user\miniconda3\Lib\site-packages\tqdm\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

train = pd.read_csv('data/data.csv')
test = pd.read_csv('data/test.csv')

train.head()

train = train.drop(columns=['customerID'])
test = test.drop(columns=['id'])

train['TotalCharges'] = pd.to_numeric(train['TotalCharges'], errors='coerce')
test['TotalCharges'] = pd.to_numeric(test['TotalCharges'], errors='coerce')

train['TotalCharges'] = train['TotalCharges'].fillna(0)
test['TotalCharges'] = test['TotalCharges'].fillna(0)

def engineer_features(df):
    df = df.copy()

    # Tenure groups
    df['tenure_group'] = pd.cut(
        df['tenure'],
        bins=[-1, 12, 24, 36, 48, 60, 72],
        labels=['0-12', '12-24', '24-36', '36-48', '48-60', '60-72']
    ).astype(str)

    # Charge ratios
    df['monthly_to_total_ratio'] = df['MonthlyCharges'] / (df['TotalCharges'] + 1)
    df['avg_monthly_charge'] = df['TotalCharges'] / (df['tenure'] + 1)
    df['charge_difference'] = df['MonthlyCharges'] - df['avg_monthly_charge']

    # Service counts
    service_cols = [
        'PhoneService', 'MultipleLines', 'InternetService',
        'OnlineSecurity', 'OnlineBackup', 'DeviceProtection',
        'TechSupport', 'StreamingTV', 'StreamingMovies'
    ]
    df['num_services'] = df[service_cols].apply(
        lambda row: sum(1 for v in row if v not in ['No', 'No phone service', 'No internet service']),
        axis=1
    )

    protection_cols = ['OnlineSecurity', 'OnlineBackup', 'DeviceProtection', 'TechSupport']
    df['num_protection_services'] = df[protection_cols].apply(
        lambda row: sum(1 for v in row if v == 'Yes'), axis=1
    )

    streaming_cols = ['StreamingTV', 'StreamingMovies']
    df['num_streaming_services'] = df[streaming_cols].apply(
        lambda row: sum(1 for v in row if v == 'Yes'), axis=1
    )

    # Interaction features
    df['contract_billing'] = df['Contract'] + '_' + df['PaperlessBilling']
    df['contract_payment'] = df['Contract'] + '_' + df['PaymentMethod']
    df['tenure_x_monthly'] = df['tenure'] * df['MonthlyCharges']

    # Flags
    df['is_new_customer'] = (df['tenure'] <= 6).astype(int)

    return df

# Apply on combined data so thresholds are consistent
n_train = len(train)
combined = pd.concat([train, test], axis=0, ignore_index=True)
combined = engineer_features(combined)
train = combined.iloc[:n_train].reset_index(drop=True)
test = combined.iloc[n_train:].reset_index(drop=True)

X = train.drop(columns=['Churn'])
X_test = test.copy()
y = train["Churn"].map({'No': 0, 'Yes': 1})

cat_cols = X.select_dtypes(include=['object', 'category']).columns.tolist()

# Fill NaN in categorical columns (CatBoost can't handle NaN in cat features)
for col in cat_cols:
    X[col] = X[col].astype(str).replace('nan', 'Missing')
    X_test[col] = X_test[col].astype(str).replace('nan', 'Missing')

# Label-encoded versions for XGBoost / LightGBM
le_dict = {}
X_encoded = X.copy()
X_test_encoded = X_test.copy()

for col in cat_cols:
    le = LabelEncoder()
    le.fit(pd.concat([X[col], X_test[col]], axis=0).astype(str))
    X_encoded[col] = le.transform(X[col].astype(str))
    X_test_encoded[col] = le.transform(X_test[col].astype(str))
    le_dict[col] = le

cat_col_indices = [X_encoded.columns.get_loc(c) for c in cat_cols]

N_FOLDS = 5
skf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)

def objective_catboost(trial):
    params = {
        'iterations': trial.suggest_int('iterations', 300, 2000),
        'depth': trial.suggest_int('depth', 4, 10),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-2, 10.0, log=True),
        'border_count': trial.suggest_int('border_count', 32, 255),
        'bagging_temperature': trial.suggest_float('bagging_temperature', 0.0, 10.0),
        'random_strength': trial.suggest_float('random_strength', 0.0, 10.0),
        'cat_features': cat_cols,
        'verbose': 0,
        'random_seed': SEED,
        'eval_metric': 'Logloss',
        'task_type': 'CPU'
    }

    scores = []
    for train_idx, val_idx in skf.split(X, y):
        X_tr, X_vl = X.iloc[train_idx], X.iloc[val_idx]
        y_tr, y_vl = y.iloc[train_idx], y.iloc[val_idx]

        model = cb.CatBoostClassifier(**params)
        model.fit(X_tr, y_tr, eval_set=(X_vl, y_vl), early_stopping_rounds=50, verbose=0)
        preds = model.predict_proba(X_vl)[:, 1]
        scores.append(roc_auc_score(y_vl, preds))

    return np.mean(scores)

study_cb = optuna.create_study(direction='maximize', study_name='catboost')
study_cb.optimize(objective_catboost, n_trials=50, show_progress_bar=True)
best_params_cb = study_cb.best_params
print(f"Best CatBoost AUC: {study_cb.best_value:.4f}")

[I 2026-03-04 21:57:57,896] A new study created in memory with name: catboost
Best trial: 0. Best value: 0.848946:   2%|▏         | 1/50 [01:41<1:23:03, 101.71s/it]

[I 2026-03-04 21:59:39,605] Trial 0 finished with value: 0.8489455650962283 and parameters: {'iterations': 444, 'depth': 6, 'learning_rate': 0.08404855668901212, 'l2_leaf_reg': 9.57449874385346, 'border_count': 252, 'bagging_temperature': 1.8383396754674342, 'random_strength': 4.125046077004781}. Best is trial 0 with value: 0.8489455650962283.

Best trial: 0. Best value: 0.848946:   4%|▍         | 2/50 [03:51<1:34:30, 118.13s/it]

[I 2026-03-04 22:01:49,232] Trial 1 finished with value: 0.8463328154014429 and parameters: {'iterations': 1703, 'depth': 10, 'learning_rate': 0.040988234045945066, 'l2_leaf_reg': 0.05449828816010084, 'border_count': 155, 'bagging_temperature': 1.7039817371742083, 'random_strength': 7.0454781899357855}. Best is trial 0 with value: 0.8489455650962283.

Best trial: 0. Best value: 0.848946:   6%|▌         | 3/50 [04:59<1:14:37, 95.27s/it]

[I 2026-03-04 22:02:57,291] Trial 2 finished with value: 0.8461782558149228 and parameters: {'iterations': 932, 'depth': 7, 'learning_rate': 0.07389637057842925, 'l2_leaf_reg': 1.8617594600732394, 'border_count': 85, 'bagging_temperature': 0.3473719715580392, 'random_strength': 0.8984233727929414}. Best is trial 0 with value: 0.8489455650962283.

Best trial: 0. Best value: 0.848946:   8%|▊         | 4/50 [09:24<2:04:29, 162.38s/it]

[I 2026-03-04 22:07:22,563] Trial 3 finished with value: 0.8482215108411066 and parameters: {'iterations': 1105, 'depth': 7, 'learning_rate': 0.022349535387804795, 'l2_leaf_reg': 4.355590471717159, 'border_count': 171, 'bagging_temperature': 9.236302027112433, 'random_strength': 5.437755237617701}. Best is trial 0 with value: 0.8489455650962283.

Best trial: 0. Best value: 0.848946:  10%|█         | 5/50 [11:44<1:55:34, 154.10s/it]

[I 2026-03-04 22:09:41,985] Trial 4 finished with value: 0.8480968756622105 and parameters: {'iterations': 1368, 'depth': 7, 'learning_rate': 0.04633951061043291, 'l2_leaf_reg': 0.9613071694927249, 'border_count': 211, 'bagging_temperature': 7.7280721777151715, 'random_strength': 6.478324778773031}. Best is trial 0 with value: 0.8489455650962283.

def objective_xgboost(trial):
    params = {
        'n_estimators': trial.suggest_int('n_estimators', 300, 2000),
        'max_depth': trial.suggest_int('max_depth', 3, 10),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'subsample': trial.suggest_float('subsample', 0.5, 1.0),
        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),
        'reg_alpha': trial.suggest_float('reg_alpha', 1e-3, 10.0, log=True),
        'reg_lambda': trial.suggest_float('reg_lambda', 1e-3, 10.0, log=True),
        'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),
        'gamma': trial.suggest_float('gamma', 0.0, 5.0),
        'objective': 'binary:logistic',
        'eval_metric': 'auc',
        'random_state': SEED,
        'verbosity': 0,
        'early_stopping_rounds': 50,
        'device': 'cuda', 
        'tree_method': 'hist'
    }

    scores = []
    for train_idx, val_idx in skf.split(X_encoded, y):
        X_tr, X_vl = X_encoded.iloc[train_idx], X_encoded.iloc[val_idx]
        y_tr, y_vl = y.iloc[train_idx], y.iloc[val_idx]

        model = xg.XGBClassifier(**params)
        model.fit(X_tr, y_tr, eval_set=[(X_vl, y_vl)], verbose=0)
        preds = model.predict_proba(X_vl)[:, 1]
        scores.append(roc_auc_score(y_vl, preds))

    return np.mean(scores)

study_xgb = optuna.create_study(direction='maximize', study_name='xgboost')
study_xgb.optimize(objective_xgboost, n_trials=50, show_progress_bar=True)
best_params_xgb = study_xgb.best_params
print(f"Best XGBoost AUC: {study_xgb.best_value:.4f}")

[I 2026-03-04 07:17:45,935] A new study created in memory with name: xgboost
Best trial: 0. Best value: 0.846737:   2%|▏         | 1/50 [00:04<03:48,  4.66s/it]

[I 2026-03-04 07:17:50,591] Trial 0 finished with value: 0.8467374296718375 and parameters: {'n_estimators': 496, 'max_depth': 8, 'learning_rate': 0.012864236628113318, 'subsample': 0.8166073433907668, 'colsample_bytree': 0.9164448864190609, 'reg_alpha': 1.1136455170278383, 'reg_lambda': 0.006332629757280146, 'min_child_weight': 8, 'gamma': 2.476069982657094}. Best is trial 0 with value: 0.8467374296718375.

Best trial: 0. Best value: 0.846737:   4%|▍         | 2/50 [00:05<01:58,  2.47s/it]

[I 2026-03-04 07:17:51,535] Trial 1 finished with value: 0.8460137827428271 and parameters: {'n_estimators': 804, 'max_depth': 3, 'learning_rate': 0.21786850304276598, 'subsample': 0.7430994966279258, 'colsample_bytree': 0.5955168805660132, 'reg_alpha': 6.845603351720182, 'reg_lambda': 0.6157215993031069, 'min_child_weight': 8, 'gamma': 2.1196598347299473}. Best is trial 0 with value: 0.8467374296718375.

Best trial: 0. Best value: 0.846737:   6%|▌         | 3/50 [00:06<01:22,  1.75s/it]

[I 2026-03-04 07:17:52,429] Trial 2 finished with value: 0.8461151562618505 and parameters: {'n_estimators': 954, 'max_depth': 3, 'learning_rate': 0.20327643550706168, 'subsample': 0.971779400483129, 'colsample_bytree': 0.701464984158042, 'reg_alpha': 2.2315035716374605, 'reg_lambda': 0.025587923275197315, 'min_child_weight': 2, 'gamma': 3.8662433388378616}. Best is trial 0 with value: 0.8467374296718375.

Best trial: 3. Best value: 0.846963:   8%|▊         | 4/50 [00:07<01:04,  1.41s/it]

[I 2026-03-04 07:17:53,305] Trial 3 finished with value: 0.8469627886803497 and parameters: {'n_estimators': 1962, 'max_depth': 9, 'learning_rate': 0.2581203382786872, 'subsample': 0.9168085300535684, 'colsample_bytree': 0.6515597464911167, 'reg_alpha': 3.97477465842831, 'reg_lambda': 0.00857636116611166, 'min_child_weight': 8, 'gamma': 1.8979391905726695}. Best is trial 3 with value: 0.8469627886803497.

Best trial: 4. Best value: 0.847674:  10%|█         | 5/50 [00:15<02:44,  3.67s/it]

[I 2026-03-04 07:18:00,977] Trial 4 finished with value: 0.8476739548644814 and parameters: {'n_estimators': 1991, 'max_depth': 10, 'learning_rate': 0.011136841569737084, 'subsample': 0.5991030107312154, 'colsample_bytree': 0.6914706532714088, 'reg_alpha': 5.597986647935248, 'reg_lambda': 0.08267237775303585, 'min_child_weight': 7, 'gamma': 1.19864822360089}. Best is trial 4 with value: 0.8476739548644814.

def objective_lightgbm(trial):
    params = {
        'n_estimators': trial.suggest_int('n_estimators', 300, 2000),
        'max_depth': trial.suggest_int('max_depth', 3, 12),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'subsample': trial.suggest_float('subsample', 0.5, 1.0),
        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),
        'reg_alpha': trial.suggest_float('reg_alpha', 1e-3, 10.0, log=True),
        'reg_lambda': trial.suggest_float('reg_lambda', 1e-3, 10.0, log=True),
        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
        'num_leaves': trial.suggest_int('num_leaves', 20, 300),
        'objective': 'binary',
        'metric': 'auc',
        'device': 'gpu',
        'verbose': -1,
    }

    scores = []
    for train_idx, val_idx in skf.split(X_encoded, y):
        X_tr, X_vl = X_encoded.iloc[train_idx], X_encoded.iloc[val_idx]
        y_tr, y_vl = y.iloc[train_idx], y.iloc[val_idx]

        model = lg.LGBMClassifier(**params)
        model.fit(
            X_tr, y_tr, eval_set=[(X_vl, y_vl)],
            callbacks=[lg.early_stopping(50, verbose=False), lg.log_evaluation(0)]
        )
        preds = model.predict_proba(X_vl)[:, 1]
        scores.append(roc_auc_score(y_vl, preds))

    return np.mean(scores)

study_lgb = optuna.create_study(direction='maximize', study_name='lightgbm')
study_lgb.optimize(objective_lightgbm, n_trials=50, show_progress_bar=True)
best_params_lgb = study_lgb.best_params
print(f"Best LightGBM AUC: {study_lgb.best_value:.4f}")

[I 2026-03-04 07:25:16,555] A new study created in memory with name: lightgbm
Best trial: 0. Best value: 0.845435:   2%|▏         | 1/50 [00:13<10:38, 13.03s/it]

[I 2026-03-04 07:25:29,584] Trial 0 finished with value: 0.8454349762331457 and parameters: {'n_estimators': 1381, 'max_depth': 6, 'learning_rate': 0.011222464214799964, 'subsample': 0.8772834496241411, 'colsample_bytree': 0.708310275436183, 'reg_alpha': 0.010407514137537011, 'reg_lambda': 1.5420869539685067, 'min_child_samples': 46, 'num_leaves': 155}. Best is trial 0 with value: 0.8454349762331457.

Best trial: 1. Best value: 0.845691:   4%|▍         | 2/50 [00:22<08:52, 11.09s/it]

[I 2026-03-04 07:25:39,316] Trial 1 finished with value: 0.8456907875257349 and parameters: {'n_estimators': 1193, 'max_depth': 8, 'learning_rate': 0.012868601132743338, 'subsample': 0.8781307207190829, 'colsample_bytree': 0.7323612776124275, 'reg_alpha': 0.08006607298391034, 'reg_lambda': 0.025654364174249832, 'min_child_samples': 93, 'num_leaves': 273}. Best is trial 1 with value: 0.8456907875257349.

Best trial: 2. Best value: 0.846572:   6%|▌         | 3/50 [00:25<05:34,  7.11s/it]

[I 2026-03-04 07:25:41,692] Trial 2 finished with value: 0.8465722000156306 and parameters: {'n_estimators': 446, 'max_depth': 4, 'learning_rate': 0.1351987791843374, 'subsample': 0.6814885272390994, 'colsample_bytree': 0.9250478445840618, 'reg_alpha': 0.05286841192062867, 'reg_lambda': 0.0030147534792867203, 'min_child_samples': 19, 'num_leaves': 273}. Best is trial 2 with value: 0.8465722000156306.

Best trial: 2. Best value: 0.846572:   8%|▊         | 4/50 [00:30<04:52,  6.37s/it]

[I 2026-03-04 07:25:46,916] Trial 3 finished with value: 0.8440630336733761 and parameters: {'n_estimators': 474, 'max_depth': 7, 'learning_rate': 0.09976895249716199, 'subsample': 0.9559742099608501, 'colsample_bytree': 0.8250517061648519, 'reg_alpha': 0.0015669692700304662, 'reg_lambda': 3.2077954258900045, 'min_child_samples': 7, 'num_leaves': 270}. Best is trial 2 with value: 0.8465722000156306.

Best trial: 2. Best value: 0.846572:  10%|█         | 5/50 [00:48<08:05, 10.79s/it]

[I 2026-03-04 07:26:05,549] Trial 4 finished with value: 0.8425125571436428 and parameters: {'n_estimators': 1229, 'max_depth': 8, 'learning_rate': 0.010704197032531556, 'subsample': 0.5495612781214141, 'colsample_bytree': 0.7895915691559032, 'reg_alpha': 0.47491603685144956, 'reg_lambda': 0.07843597555419612, 'min_child_samples': 8, 'num_leaves': 104}. Best is trial 2 with value: 0.8465722000156306.

def oof_predictions_proba(model_class, model_params, X_train, y_train, X_test,
                          skf, is_catboost=False):
    oof_preds = np.zeros(len(X_train))
    test_preds = np.zeros(len(X_test))

    for fold, (train_idx, val_idx) in enumerate(skf.split(X_train, y_train)):
        X_tr, X_vl = X_train.iloc[train_idx], X_train.iloc[val_idx]
        y_tr, y_vl = y_train.iloc[train_idx], y_train.iloc[val_idx]

        model = model_class(**model_params)

        if is_catboost:
            model.fit(X_tr, y_tr, eval_set=(X_vl, y_vl),
                      early_stopping_rounds=50, verbose=0)
        else:
            model.fit(X_tr, y_tr, eval_set=[(X_vl, y_vl)], verbose=0)

        oof_preds[val_idx] = model.predict_proba(X_vl)[:, 1]
        test_preds += model.predict_proba(X_test)[:, 1] / skf.n_splits

        print(f"  Fold {fold+1} AUC: {roc_auc_score(y_vl, oof_preds[val_idx]):.4f}")

    print(f"  Overall OOF AUC: {roc_auc_score(y_train, oof_preds):.4f}")
    return oof_preds, test_preds

# CatBoost OOF
cb_params = {**best_params_cb, 'cat_features': cat_cols, 'verbose': 0,
             'random_seed': SEED, 'eval_metric': 'Logloss'}
print("=== CatBoost OOF ===")
oof_cb, test_cb = oof_predictions_proba(
    cb.CatBoostClassifier, cb_params, X, y, X_test, skf, is_catboost=True
)

# XGBoost OOF
xgb_params = {**best_params_xgb, 'objective': 'binary:logistic',
              'eval_metric': 'auc', 'random_state': SEED, 'verbosity': 0,
              'early_stopping_rounds': 50}
print("\n=== XGBoost OOF ===")
oof_xgb, test_xgb = oof_predictions_proba(
    xg.XGBClassifier, xgb_params, X_encoded, y, X_test_encoded, skf
)

# LightGBM OOF
lgb_params = {**best_params_lgb, 'objective': 'binary', 'metric': 'auc',
              'random_state': SEED, 'verbose': -1}
print("\n=== LightGBM OOF ===")
oof_lgb, test_lgb = oof_predictions_proba(
    lg.LGBMClassifier, lgb_params, X_encoded, y, X_test_encoded, skf
)

=== CatBoost OOF ===

=== XGBoost OOF ===

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\Users\user\miniconda3\Lib\site-packages\xgboost\data.py:407, in pandas_feature_info(data, meta, feature_names, feature_types, enable_categorical)
    406 try:
--> 407     new_feature_types.append(_pandas_dtype_mapper[dtype.name])
    408 except KeyError:

KeyError: 'object'

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
Cell In[157], line 45
     41 xgb_params = {**best_params_xgb, 'objective': 'binary:logistic',
     42               'eval_metric': 'auc', 'random_state': SEED, 'verbosity': 0,
     43               'early_stopping_rounds': 50}
     44 print("\n=== XGBoost OOF ===")
---> 45 oof_xgb, test_xgb = oof_predictions_proba(
     46     xg.XGBClassifier, xgb_params, X_encoded, y, X_test_encoded, skf
     47 )
     49 # LightGBM OOF
     50 lgb_params = {**best_params_lgb, 'objective': 'binary', 'metric': 'auc',
     51               'random_state': SEED, 'verbose': -1}

Cell In[157], line 19, in oof_predictions_proba(model_class, model_params, X_train, y_train, X_test, skf, is_catboost)
     16         model.fit(X_tr, y_tr, eval_set=[(X_vl, y_vl)], verbose=0)
     18     oof_preds[val_idx] = model.predict_proba(X_vl)[:, 1]
---> 19     test_preds += model.predict_proba(X_test)[:, 1] / skf.n_splits
     21     print(f"  Fold {fold+1} AUC: {roc_auc_score(y_vl, oof_preds[val_idx]):.4f}")
     23 print(f"  Overall OOF AUC: {roc_auc_score(y_train, oof_preds):.4f}")

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\sklearn.py:1923, in XGBClassifier.predict_proba(self, X, validate_features, base_margin, iteration_range)
   1921     class_prob = softmax(raw_predt, axis=1)
   1922     return class_prob
-> 1923 class_probs = super().predict(
   1924     X=X,
   1925     validate_features=validate_features,
   1926     base_margin=base_margin,
   1927     iteration_range=iteration_range,
   1928 )
   1929 return _cls_predict_proba(self.n_classes_, class_probs, np.vstack)

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\core.py:774, in require_keyword_args.<locals>.throw_if.<locals>.inner_f(*args, **kwargs)
    772 for k, arg in zip(sig.parameters, args):
    773     kwargs[k] = arg
--> 774 return func(**kwargs)

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\sklearn.py:1448, in XGBModel.predict(self, X, output_margin, validate_features, base_margin, iteration_range)
   1446 if self._can_use_inplace_predict():
   1447     try:
-> 1448         predts = self.get_booster().inplace_predict(
   1449             data=X,
   1450             iteration_range=iteration_range,
   1451             predict_type="margin" if output_margin else "value",
   1452             missing=self.missing,
   1453             base_margin=base_margin,
   1454             validate_features=validate_features,
   1455         )
   1456         if _is_cupy_alike(predts):
   1457             cp = import_cupy()

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\core.py:774, in require_keyword_args.<locals>.throw_if.<locals>.inner_f(*args, **kwargs)
    772 for k, arg in zip(sig.parameters, args):
    773     kwargs[k] = arg
--> 774 return func(**kwargs)

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\core.py:2852, in Booster.inplace_predict(self, data, iteration_range, predict_type, missing, validate_features, base_margin, strict_shape)
   2850     data = pd.DataFrame(data)
   2851 if _is_pandas_df(data):
-> 2852     data, fns, _ = _transform_pandas_df(data, enable_categorical)
   2853     if validate_features:
   2854         self._validate_features(fns)

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\data.py:662, in _transform_pandas_df(data, enable_categorical, feature_names, feature_types, meta)
    659     raise ValueError(f"DataFrame for {meta} cannot have multiple columns")
    661 feature_types, ref_categories = get_ref_categories(feature_types)
--> 662 feature_names, feature_types = pandas_feature_info(
    663     data, meta, feature_names, feature_types, enable_categorical
    664 )
    666 arrays = pandas_transform_data(data)
    667 return (
    668     PandasTransformed(arrays, ref_categories=ref_categories),
    669     feature_names,
    670     feature_types,
    671 )

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\data.py:409, in pandas_feature_info(data, meta, feature_names, feature_types, enable_categorical)
    407             new_feature_types.append(_pandas_dtype_mapper[dtype.name])
    408         except KeyError:
--> 409             _invalid_dataframe_dtype(data)
    411 if feature_types is None and meta is None:
    412     feature_types = new_feature_types

File c:\Users\user\miniconda3\Lib\site-packages\xgboost\data.py:372, in _invalid_dataframe_dtype(data)
    370 type_err = "DataFrame.dtypes for data must be int, float, bool or category."
    371 msg = f"""{type_err} {_ENABLE_CAT_ERR} {err}"""
--> 372 raise ValueError(msg)

ValueError: DataFrame.dtypes for data must be int, float, bool or category. When categorical type is supplied, the experimental DMatrix parameter`enable_categorical` must be set to `True`.  Invalid columns:Churn: object

from scipy.optimize import minimize

# Approach A: Optimized weighted blending
def blend_objective(weights):
    w = np.array(weights)
    w = w / w.sum()
    blended = w[0] * oof_cb + w[1] * oof_xgb + w[2] * oof_lgb
    return -roc_auc_score(y, blended)

result = minimize(
    blend_objective,
    x0=[1/3, 1/3, 1/3],
    method='L-BFGS-B',
    bounds=[(0.0, 1.0), (0.0, 1.0), (0.0, 1.0)]
)

best_weights = np.array(result.x)
best_weights = best_weights / best_weights.sum()
print(f"Optimal blend weights: CB={best_weights[0]:.3f}, XGB={best_weights[1]:.3f}, LGB={best_weights[2]:.3f}")

blended_oof = best_weights[0] * oof_cb + best_weights[1] * oof_xgb + best_weights[2] * oof_lgb
blended_test = best_weights[0] * test_cb + best_weights[1] * test_xgb + best_weights[2] * test_lgb
blend_auc = roc_auc_score(y, blended_oof)
print(f"Blended OOF AUC: {blend_auc:.4f}")

# Approach B: Logistic regression stacking
meta_train = np.column_stack([oof_cb, oof_xgb, oof_lgb])
meta_test = np.column_stack([test_cb, test_xgb, test_lgb])

meta_oof = np.zeros(len(y))
meta_test_preds = np.zeros(len(X_test))
meta_model = LogisticRegression(random_state=SEED, C=1.0)

for fold, (train_idx, val_idx) in enumerate(skf.split(meta_train, y)):
    meta_model.fit(meta_train[train_idx], y.iloc[train_idx])
    meta_oof[val_idx] = meta_model.predict_proba(meta_train[val_idx])[:, 1]
    meta_test_preds += meta_model.predict_proba(meta_test)[:, 1] / N_FOLDS

stack_auc = roc_auc_score(y, meta_oof)
print(f"Stacked OOF AUC: {stack_auc:.4f}")

# Pick the better approach
if blend_auc >= stack_auc:
    final_preds = blended_test
    print(f"\nUsing BLENDING (AUC: {blend_auc:.4f})")
else:
    meta_model.fit(meta_train, y)
    final_preds = meta_model.predict_proba(meta_test)[:, 1]
    print(f"\nUsing STACKING (AUC: {stack_auc:.4f})")

sub = pd.read_csv("data/sample_submission.csv")
sub['Churn'] = final_preds
sub.to_csv("ensemble-tuned-001.csv", index=False)
print(f"Submission saved. Shape: {sub.shape}")
sub.head()

sub1 = pd.read_csv('catboost-001.csv')
sub2 = pd.read_csv('best-public.csv')

# Try different weight combinations
best_weights = []
for w1 in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
    w2 = 1.0 - w1
    ensemble = sub1.copy()
    ensemble.iloc[:, 1:] = w1 * sub1.iloc[:, 1:] + w2 * sub2.iloc[:, 1:]
    ensemble.to_csv(f'ensemble_w{w1:.1f}.csv', index=False)
    print(f"Created ensemble with weights: {w1:.1f} / {w2:.1f}")

ensemble = sub1.copy()
for col in sub1.columns[1:]:  
    rank1 = sub1[col].rank(pct=True)
    rank2 = sub2[col].rank(pct=True)
    ensemble[col] = (rank1 + rank2) / 2

ensemble.to_csv('ensemble_rank_avg.csv', index=False)

Created ensemble with weights: 0.0 / 1.0
Created ensemble with weights: 0.1 / 0.9
Created ensemble with weights: 0.2 / 0.8
Created ensemble with weights: 0.3 / 0.7
Created ensemble with weights: 0.4 / 0.6
Created ensemble with weights: 0.5 / 0.5
Created ensemble with weights: 0.6 / 0.4
Created ensemble with weights: 0.7 / 0.3
Created ensemble with weights: 0.8 / 0.2
Created ensemble with weights: 0.9 / 0.1
Created ensemble with weights: 1.0 / 0.0

	customerID	gender	Partner	Dependents	tenure	PhoneService	MultipleLines	InternetService	OnlineSecurity	...	DeviceProtection	TechSupport	StreamingTV	StreamingMovies	Contract	PaperlessBilling	PaymentMethod	MonthlyCharges	TotalCharges	Churn
0	7590-VHVEG	Female	Yes	No	1	No	No phone service	DSL	No	...	No	No	No	No	Month-to-month	Yes	Electronic check	29.85	29.85	No
1	5575-GNVDE	Male	No	No	34	Yes	No	DSL	Yes	...	Yes	No	No	No	One year	No	Mailed check	56.95	1889.5	No
2	3668-QPYBK	Male	No	No	2	Yes	No	DSL	Yes	...	No	No	No	No	Month-to-month	Yes	Mailed check	53.85	108.15	Yes
3	7795-CFOCW	Male	No	No	45	No	No phone service	DSL	Yes	...	Yes	Yes	No	No	One year	No	Bank transfer (automatic)	42.30	1840.75	No
4	9237-HQITU	Female	No	No	2	Yes	No	Fiber optic	No	...	No	No	No	No	Month-to-month	Yes	Electronic check	70.70	151.65	Yes