[python] LightGBM (regression problem) template

!sw_vers

!Python -V

import sys
sys.executable

%matplotlib inline
%config InlineBackend.figure_format = 'svg'

import time
import json
import random

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import japanize_matplotlib
import snap

import lightgbm as lgb

from sklearn import datasets
from sklearn import metrics

from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score, recall_score

from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split

from sklearn.datasets import fetch_california_housing
import pandas as pd

import seaborn as sns
sns.set_style('darkgrid')
sns.set(font='IPAexGothic')

import warnings
warnings.filterwarnings('ignore')

california_housing = fetch_california_housing(as_frame=True)
df = pd.DataFrame(california_housing['data'], columns=california_housing.feature_names)
df_target = pd.Series(California_housing['target'])

# template
# Set the explanatory variables
X_data = df
# set the target variable
Y_data = df_target

X_data.shape, Y_data.shape

X_data.head()

Y_data.head()

# const
seed = 123
random_state = 123
n_splits=5
test_size=0.2

def set_seed(seed=seed):
  os.environ["PYTHONHASHSEED"] = str(seed)
  np.random.seed(seed)
  random.seed(seed)
  torch.manual_seed(seed)
  torch.cuda.manual_seed(seed)
  # Comment out if it degrades GPU performance
  torch.backends.cudnn.deterministic = True
  torch.use_deterministic_algorithms = True

X_train, X_test, Y_train, Y_test = train_test_split(X_data, Y_data, test_size=test_size, random_state=random_state)

X_train.head()

X_test.head()

Y_train.head()

Y_test.head()

%%time

params = {
  'random_state': random_state,
  'objective': 'regression',
  'boosting_type': 'gbdt',
  'metric': {'rmse'},
  'verbosity': -1,
  'bagging_freq': 1,
  'feature_fraction': 0.8,
  'max_depth': 8,
  'min_data_in_leaf': 25,
  'num_leaves': 256,
  'learning_rate': 0.07,
  'lambda_l1': 0.2,
  'lambda_l2': 0.5,
}

model_list = [].
kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)

for _index, (_train_index, _val_index) in enumerate(kf.split(X_train, Y_train)):

   _X_train = X_train.iloc[_train_index].
   _Y_train = Y_train.iloc[_train_index].

   _X_val = X_train.iloc[_val_index]
   _Y_val = Y_train.iloc[_val_index]

   lgb_train = lgb.Dataset(_X_train, _Y_train)
   lgb_val = lgb.Dataset(_X_val, _Y_val, reference=lgb_train)

   lgb_results = {}

   model = lgb.train(
     params,
     train_set=lgb_train,
     valid_sets=[lgb_train, lgb_val],
     verbose_eval=-1,
     num_boost_round=1000,
     early_stopping_rounds=100,
     valid_names=['Train', 'Val'],
     evals_result=lgb_results
   )

  # Save each model in CV
   model_list.append(model)

   # Transition of the loss function
   loss_train = lgb_results['Train']['rmse']
   loss_val = lgb_results['Val']['rmse']
   best_iteration = model.best_iteration

   plt.figure()
   plt.xlabel('Iteration')
   plt.ylabel('rmse')
   plt.plot(loss_train, label='train loss')
   plt.plot(loss_val, label='valid loss')
   plt.title('kFold : {} \n RMSE'.format(_index))
   plt.legend()
   plt.show()

  # scatter plot
   plt.figure(figsize=(4,4))
   y_val = model.predict(_X_val, num_iteration=model.best_iteration)
   plt.plot(y_val, y_val, color = 'red', label = '$y=x$')
   plt.scatter(y_val,_Y_val, s=1)
   plt.xlabel('Predicted value')
   plt.ylabel('True value')
   plt.title('kFold : {} \n predicted value vs true value'.format(_index))
   plt.legend()
   plt.show()

Feature_importances = pd.DataFrame()

for fold, model in enumerate(model_list):

    DataFrame() tmp = pd.
    tmp['feature'] = model.feature_name()
    tmp['importance'] = model.feature_importance(importance_type='gain')
    tmp['fold'] = fold

    feature_importances = feature_importances.append(tmp)

order = list(feature_importances.groupby("feature")["importance"].mean().sort_values(ascending=False).index)

plt.figure(figsize=(4, 4))
sns.barplot(x='importance', y='feature', data=feature_importances, order=order)
plt.title('LGBM importance')
plt.tight_layout()
plt.show()

_test_score_array = np.zeros(len(X_test))

for model in model_list:
  y_pred = model.predict(X_test, num_iteration=model.best_iteration)

  _test_score_array += y_pred / n_splits

_test_score_array

### Prepare a dataframe to store the feature importance.
# Extract from https://www.sairablog.com/article/lightgbm-sklearn-kaggle-classification.html
top_feature_num = 10
feature_importances = pd.DataFrame({
  'feature' : model.feature_name(),
  'importance': model.feature_importance(importance_type='gain'),
})

order = list(feature_importances.groupby("feature")["importance"].mean().sort_values(ascending=False).index)[:top_feature_num])

plt.figure(figsize=(4, 6))
sns.barplot(x='importance', y='feature', data=feature_importances, order=order)
plt.title('LGBM importance')
plt.tight_layout()
plt.show()

import shap

shap.initjs()
Explainer(model, _X_train) explainer = shap.
shap_values = explainer(_X_train, check_additivity=False)

shap.summary_plot(shap_values, _X_train)

shap.summary_plot(shap_values, _X_train, plot_type="bar")

shap.plots.waterfall(shap_values[0], max_display=20)

shap_values.shape

Y_val = model.predict(_X_train, num_iteration=model.best_iteration)
df_result = pd.DataFrame({
  'index': _X_train.index.tolist(),
  'train_score': Y_val,
})

top = df_result.sort_values(by='train_score', ascending=False).index.tolist()
bottom = df_result.sort_values(by='train_score').index.tolist()

for i in top[:5]:
  shap.plots.waterfall(shap_values[i], max_display=20)

for i in bottom[:5]:
  shap.plots.waterfall(shap_values[i], max_display=20)

Feature_importances = pd.DataFrame({
  'feature' : model.feature_name(),
  DataFrame({ 'feature' : model.feature_name(), 'importance': model.feature_importance(importance_type='gain'),
})
feature_list = list(feature_importances.groupby("feature")["importance"].mean().sort_values(ascending=False).index)
for feature in feature_list:
  shap.dependence_plot(feature, shap_values.values, _X_train)