← Back to projects

#4

Kaggle competition: [link]

Entry by Robin R.P.M. Kras

⭐ 1. Introduction & Overview

Your Goal: Your goal is to predict rainfall for each day of the year.

🔹 2. Import Libraries & Set Up

# General
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Machine Learning
import xgboost as xg

from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.metrics import accuracy_score, mean_absolute_error, mean_squared_error, r2_score, root_mean_squared_error, roc_auc_score, cross_val_score
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.optimizers import Adam

from imblearn.over_sampling import SMOTE

# Feature Importance & Explainability
import shap

# Settings
import warnings
warnings.filterwarnings("ignore")

# Set random seed for reproducibility
SEED = 42
np.random.seed(SEED)

print("Libraries loaded. Ready to go!")

Libraries loaded. Ready to go!

🔹 3. Load & Explore Data

train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

train.head()

	id	day	pressure	maxtemp	temparature	mintemp	dewpoint	humidity	cloud	sunshine	winddirection	windspeed	rainfall
0	0	1	1017.4	21.2	20.6	19.9	19.4	87.0	88.0	1.1	60.0	17.2	1
1	1	2	1019.5	16.2	16.9	15.8	15.4	95.0	91.0	0.0	50.0	21.9	1
2	2	3	1024.1	19.4	16.1	14.6	9.3	75.0	47.0	8.3	70.0	18.1	1
3	3	4	1013.4	18.1	17.8	16.9	16.8	95.0	95.0	0.0	60.0	35.6	1
4	4	5	1021.8	21.3	18.4	15.2	9.6	52.0	45.0	3.6	40.0	24.8	0

train.shape

(2190, 13)

train.isnull().sum()

id               0
day              0
pressure         0
maxtemp          0
temparature      0
mintemp          0
dewpoint         0
humidity         0
cloud            0
sunshine         0
winddirection    0
windspeed        0
rainfall         0
dtype: int64

# Quick summary of dataset
train.describe()
train.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2190 entries, 0 to 2189
Data columns (total 13 columns):
 #   Column         Non-Null Count  Dtype  
---  ------         --------------  -----  
 0   id             2190 non-null   int64  
 1   day            2190 non-null   int64  
 2   pressure       2190 non-null   float64
 3   maxtemp        2190 non-null   float64
 4   temparature    2190 non-null   float64
 5   mintemp        2190 non-null   float64
 6   dewpoint       2190 non-null   float64
 7   humidity       2190 non-null   float64
 8   cloud          2190 non-null   float64
 9   sunshine       2190 non-null   float64
 10  winddirection  2190 non-null   float64
 11  windspeed      2190 non-null   float64
 12  rainfall       2190 non-null   int64  
dtypes: float64(10), int64(3)
memory usage: 222.5 KB

🔹 4. Data Visualization & EDA

float_cols = [col for col in train.columns if train[col].dtype == "float64"]

cols_per_row = 3
num_plots = len(float_cols)
rows = (num_plots // cols_per_row) + (num_plots % cols_per_row > 0) 

fig, axes = plt.subplots(rows, cols_per_row, figsize=(15, 5 * rows)) 
axes = axes.flatten()  

for idx, col in enumerate(float_cols):
    sns.histplot(train[col], bins=50, kde=True, ax=axes[idx])
    axes[idx].set_title(f"Distribution of {col}")

for i in range(idx + 1, len(axes)):
    fig.delaxes(axes[i])

plt.tight_layout()
plt.show()

heatmap_train = train.select_dtypes(include=["float64", "int64"])

corr_matrix = heatmap_train.corr()

threshold = 0.8

high_corr_pairs = (
    corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool)) 
    .stack()  
    .reset_index()
)

high_corr_pairs.columns = ["Feature 1", "Feature 2", "Correlation"]
high_corr_pairs = high_corr_pairs[high_corr_pairs["Correlation"].abs() > threshold]  

plt.figure(figsize=(30, 12))
sns.heatmap(corr_matrix, annot=True, cmap="coolwarm")
plt.title("Feature Correlation Matrix")
plt.show()

print("Highly correlated feature pairs (above threshold):")
print(high_corr_pairs)

Highly correlated feature pairs (above threshold):
      Feature 1    Feature 2  Correlation
23     pressure      maxtemp    -0.800499
24     pressure  temparature    -0.816531
25     pressure      mintemp    -0.814453
26     pressure     dewpoint    -0.817008
33      maxtemp  temparature     0.982932
34      maxtemp      mintemp     0.965529
35      maxtemp     dewpoint     0.906703
42  temparature      mintemp     0.987150
43  temparature     dewpoint     0.933617
50      mintemp     dewpoint     0.941342
68        cloud     sunshine    -0.805128

l1 = high_corr_pairs['Feature 1'].tolist()
l2 = high_corr_pairs['Feature 2'].tolist()
interesting_features = list(set(l1+l2))

print(interesting_features)

['pressure', 'maxtemp', 'mintemp', 'temparature', 'dewpoint', 'sunshine', 'cloud']

🔹 5. Feature Engineering

train['humidity_cloud_interaction'] = train['humidity'] * train['cloud']
train['humidity_sunshine_interaction'] = train['humidity'] * train['sunshine']
train['cloud_sunshine_ratio'] = train['cloud'] / (train['sunshine'] + 1e-5)
train['relative_dryness'] = 100 - train['humidity']
train['sunshine_percentage'] = train['sunshine'] / (train['sunshine'] + train['cloud'] + 1e-5)
train['weather_index'] = (0.4 * train['humidity']) + (0.3 * train['cloud']) - (0.3 * train['sunshine'])

test['humidity_cloud_interaction'] = test['humidity'] * test['cloud']
test['humidity_sunshine_interaction'] = test['humidity'] * test['sunshine']
test['cloud_sunshine_ratio'] = test['cloud'] / (test['sunshine'] + 1e-5)
test['relative_dryness'] = 100 - test['humidity']
test['sunshine_percentage'] = test['sunshine'] / (test['sunshine'] + test['cloud'] + 1e-5)
test['weather_index'] = (0.4 * test['humidity']) + (0.3 * test['cloud']) - (0.3 * test['sunshine'])

# Test set contains an instance of null
test['winddirection'].fillna(test['winddirection'].median(), inplace=True)

test.head()

	id	day	pressure	maxtemp	temparature	mintemp	dewpoint	humidity	cloud	sunshine	winddirection	windspeed	humidity_cloud_interaction	humidity_sunshine_interaction	cloud_sunshine_ratio	relative_dryness	sunshine_percentage	weather_index
0	2190	1	1019.5	17.5	15.8	12.7	14.9	96.0	99.0	0.0	50.0	24.3	9504.0	0.0	9.900000e+06	4.0	0.000000	68.10
1	2191	2	1016.5	17.5	16.5	15.8	15.1	97.0	99.0	0.0	50.0	35.3	9603.0	0.0	9.900000e+06	3.0	0.000000	68.50
2	2192	3	1023.9	11.2	10.4	9.4	8.9	86.0	96.0	0.0	40.0	16.9	8256.0	0.0	9.600000e+06	14.0	0.000000	63.20
3	2193	4	1022.9	20.6	17.3	15.2	9.5	75.0	45.0	7.1	20.0	50.6	3375.0	532.5	6.338019e+00	25.0	0.136276	41.37
4	2194	5	1022.2	16.1	13.8	6.4	4.3	68.0	49.0	9.2	20.0	19.4	3332.0	625.6	5.326081e+00	32.0	0.158076	39.14

Experiment

X = train.drop(columns=['rainfall'], errors='ignore')
y = train['rainfall']

from sklearn.model_selection import KFold
from sklearn.metrics import root_mean_squared_error
import numpy as np

kf = KFold(n_splits=5, shuffle=True, random_state=42)
oof_predictions = np.zeros(len(train))

for train_idx, val_idx in kf.split(train):
    X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
    y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]

    model = xg.XGBRegressor()
    model.fit(X_train, y_train)
    
    y_pred = model.predict(X_val)
    oof_predictions[val_idx] = y_pred
    
    print(f"Fold RMSE: {root_mean_squared_error(y_val, y_pred)}")

final_rmse = root_mean_squared_error(y, oof_predictions)
print(f"Final Cross-Validation RMSE: {final_rmse}")

Fold RMSE: 0.3665955066680908
Fold RMSE: 0.34317660331726074
Fold RMSE: 0.3477436602115631
Fold RMSE: 0.3161795735359192
Fold RMSE: 0.34899771213531494
Final Cross-Validation RMSE: 0.34492231409811286

🔹 6. Model Selection

X = train.drop(columns=['rainfall'], errors='ignore')
X_test = test

y = train['rainfall']

model = Ridge()
model.fit(X, y)
predictions = model.predict(X_test)

output = pd.DataFrame({'id': test.id, 'rainfall': predictions})
output.to_csv('submission_ridge.csv', index=False)
print("Your submission was successfully saved!")

Your submission was successfully saved!

🔹 7. Keras!

X_train = train.drop(columns=['day','rainfall'], errors='ignore')
X_test = test.drop(columns=['day'])

y_train = train['rainfall']

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

from tensorflow.keras.callbacks import EarlyStopping
early_stopping = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True)

model = Sequential([
    Dense(128, activation='relu', kernel_initializer='he_normal', input_shape=(X_train_scaled.shape[1],)),
    Dropout(0.3),
    Dense(64, activation='relu', kernel_initializer='he_normal', input_shape=(X_train_scaled.shape[1],)),
    Dropout(0.3),
    Dense(32, activation='relu', kernel_initializer='he_normal'),
    Dropout(0.2),
    Dense(16, activation='relu', kernel_initializer='he_normal'),
    Dense(1, activation='sigmoid') 
])

optimizer = Adam(learning_rate=0.001)
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

history = model.fit(X_train_scaled, y_train, epochs=200, batch_size=32, validation_split=0.2, 
                    callbacks=[early_stopping], verbose=1)

Epoch 1/200
55/55 [==============================] - 1s 7ms/step - loss: 0.5314 - accuracy: 0.7614 - val_loss: 0.3429 - val_accuracy: 0.8539
Epoch 2/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3997 - accuracy: 0.8196 - val_loss: 0.3200 - val_accuracy: 0.8767
Epoch 3/200
55/55 [==============================] - 0s 5ms/step - loss: 0.4051 - accuracy: 0.8225 - val_loss: 0.3204 - val_accuracy: 0.8721
Epoch 4/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3675 - accuracy: 0.8579 - val_loss: 0.3196 - val_accuracy: 0.8767
Epoch 5/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3791 - accuracy: 0.8442 - val_loss: 0.3152 - val_accuracy: 0.8744
Epoch 6/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3720 - accuracy: 0.8527 - val_loss: 0.3197 - val_accuracy: 0.8744
Epoch 7/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3747 - accuracy: 0.8436 - val_loss: 0.3201 - val_accuracy: 0.8836
Epoch 8/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3674 - accuracy: 0.8533 - val_loss: 0.3248 - val_accuracy: 0.8767
Epoch 9/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3637 - accuracy: 0.8522 - val_loss: 0.3256 - val_accuracy: 0.8699
Epoch 10/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3623 - accuracy: 0.8624 - val_loss: 0.3242 - val_accuracy: 0.8721
Epoch 11/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3610 - accuracy: 0.8590 - val_loss: 0.3239 - val_accuracy: 0.8699
Epoch 12/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3498 - accuracy: 0.8670 - val_loss: 0.3245 - val_accuracy: 0.8744
Epoch 13/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3462 - accuracy: 0.8584 - val_loss: 0.3259 - val_accuracy: 0.8767
Epoch 14/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3440 - accuracy: 0.8636 - val_loss: 0.3254 - val_accuracy: 0.8699
Epoch 15/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3452 - accuracy: 0.8693 - val_loss: 0.3213 - val_accuracy: 0.8699
Epoch 16/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3473 - accuracy: 0.8636 - val_loss: 0.3199 - val_accuracy: 0.8790
Epoch 17/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3469 - accuracy: 0.8619 - val_loss: 0.3219 - val_accuracy: 0.8676
Epoch 18/200
55/55 [==============================] - 0s 4ms/step - loss: 0.3422 - accuracy: 0.8602 - val_loss: 0.3217 - val_accuracy: 0.8721
Epoch 19/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3439 - accuracy: 0.8699 - val_loss: 0.3231 - val_accuracy: 0.8699
Epoch 20/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3441 - accuracy: 0.8624 - val_loss: 0.3219 - val_accuracy: 0.8699
Epoch 21/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3404 - accuracy: 0.8647 - val_loss: 0.3200 - val_accuracy: 0.8790
Epoch 22/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3356 - accuracy: 0.8693 - val_loss: 0.3195 - val_accuracy: 0.8721
Epoch 23/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3384 - accuracy: 0.8687 - val_loss: 0.3181 - val_accuracy: 0.8721
Epoch 24/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3393 - accuracy: 0.8659 - val_loss: 0.3226 - val_accuracy: 0.8676
Epoch 25/200
55/55 [==============================] - 0s 5ms/step - loss: 0.3417 - accuracy: 0.8664 - val_loss: 0.3288 - val_accuracy: 0.8721

predictions_keras = model.predict(X_test_scaled).flatten()

output = pd.DataFrame({'id': test.id, 'rainfall': predictions_keras})
output.to_csv('submission_keras.csv', index=False)
print("Your submission was successfully saved!")

23/23 [==============================] - 0s 1ms/step
Your submission was successfully saved!

output.head()

	id	rainfall
0	2190	0.990231
1	2191	0.994316
2	2192	0.973429
3	2193	0.146203
4	2194	0.146294

🔹 8. kNN KFolds

RMV = ['rainfall','id']
FEATURES = [c for c in train.columns if not c in RMV]

from sklearn.model_selection import KFold
from sklearn.neighbors import KNeighborsClassifier
from xgboost import XGBRegressor, XGBClassifier
import xgboost
print("Using XGBoost version",xgboost.__version__)

Using XGBoost version 2.1.4

%%time
FOLDS = 5
kf = KFold(n_splits=FOLDS, shuffle=True, random_state=777)
    
oof_knn = np.zeros(len(train))
pred_knn = np.zeros(len(test))

for i, (train_index, test_index) in enumerate(kf.split(train)):

    print("#"*25)
    print(f"### Fold {i+1}")
    print("#"*25)
    
    x_train = train.loc[train_index,FEATURES].copy()
    y_train = train.loc[train_index,"rainfall"]    
    x_valid = train.loc[test_index,FEATURES].copy()
    y_valid = train.loc[test_index,"rainfall"]
    x_test = test[FEATURES].copy()

    for c in FEATURES:
        m = x_train[c].mean()
        s = x_train[c].std()
        x_train[c] = (x_train[c]-m)/s
        x_valid[c] = (x_valid[c]-m)/s
        x_test[c] = (x_test[c]-m)/s
        x_test[c] = x_test[c].fillna(0)
        x_train[c] = x_train[c].fillna(0)

    model = KNeighborsClassifier(n_neighbors=101, p=1)
    model.fit(x_train.values, y_train.values)

    val_probs = model.predict_proba(x_valid)[:, 1]
    val_auc = roc_auc_score(y_val, val_probs)
    print("auc: ", val_auc)

    # INFER OOF
    oof_knn[test_index] = model.predict_proba(x_valid.values)[:,1]
    # INFER TEST
    pred_knn += model.predict_proba(x_test.values)[:,1]

# COMPUTE AVERAGE TEST PREDS
pred_knn /= FOLDS

final_auc = roc_auc_score(train["rainfall"], oof_knn)
print(f"Final Cross-Validation ROC AUC: {final_auc:.4f}")

#########################
### Fold 1
#########################
auc:  0.498921306759811
#########################
### Fold 2
#########################
auc:  0.5191082802547771
#########################
### Fold 3
#########################
auc:  0.5050724265461269
#########################
### Fold 4
#########################
auc:  0.5059841791658106
#########################
### Fold 5
#########################
auc:  0.5340302034107253
Final Cross-Validation ROC AUC: 0.8898
CPU times: total: 2.44 s
Wall time: 493 ms

best_public = pd.read_csv("best_public.csv")
display(best_public.head())
best_public = best_public.rainfall.values

	id	rainfall
0	2190	0.960959
1	2191	0.946575
2	2192	0.994521
3	2193	0.089041
4	2194	0.020548

from scipy.stats import rankdata

sub = pd.read_csv("sample_submission.csv")
sub.rainfall = -0.067 * rankdata(pred_knn) + 1.067 * rankdata(best_public)
sub.rainfall = rankdata( sub.rainfall ) / len(sub)
print(sub.shape)
sub.to_csv(f"submission_knn.csv",index=False)
sub.head()

(730, 2)

	id	rainfall
0	2190	0.962329
1	2191	0.943151
2	2192	0.993151
3	2193	0.097260
4	2194	0.020548