def train_propensity_to_convert_model(session: Session) -> str:
import lightgbm as lgb
from imblearn.over_sampling import SMOTENC
import dill
import pandas as pd
import numpy as np
# Prepare data
user_features = session.table('FIRST_TOUCH_USER_FEATURES')
df = pd.DataFrame(user_features.collect())
df.columns = map(str.lower, df.columns)
ref_cols = ['refr_urlhost', 'refr_medium']
mkt_cols = ['mkt_medium', 'mkt_source', 'mkt_term', 'mkt_campaign']
geo_cols = ['geo_country', 'geo_region', 'br_lang']
dev_cols = ['device_family', 'os_family']
url_cols = ['first_page_title']
engagement_cols = ['engaged_time_in_s', 'absolute_time_in_s', 'vertical_percentage_scrolled']
discrete_col = ref_cols + mkt_cols + geo_cols + dev_cols + url_cols
continues_col = engagement_cols
all_features = discrete_col + continues_col
# Input missing data
for col in discrete_col:
df[col].fillna('N/A', inplace=True)
for col in continues_col:
df[col].fillna(df[col].mean(), inplace=True)
# Reduce number of categorical features so `SMOTENC` doesn't run out of memory.
class TakeTopK(BaseEstimator, TransformerMixin):
def __init__(self, k=20):
self.largest_cat = {}
self.k = k
def fit(self, X, y=None):
for col in discrete_col:
self.largest_cat[col] = df[col].value_counts().nlargest(self.k).index
return self
def transform(self, X, y=None):
Xt = pd.DataFrame()
for col in discrete_col:
Xt[col] = pd.Series(np.where(X[col].isin(self.largest_cat[col]), X[col], 'Other'), dtype='category')
Xt[continues_col] = X[continues_col].astype(float)
return Xt
# Create train and test data sets
cat_index = [ pd.Index(all_features).get_loc(col) for col in discrete_col ]
df_train, df_test = df.iloc[:df.shape[0]//10*8,:], df.iloc[df.shape[0]//10*8:,:]
smote_nc = SMOTENC(categorical_features=cat_index, k_neighbors=5, random_state=0)
topk = TakeTopK(50)
X_res, y_res = smote_nc.fit_resample(topk.fit_transform(df_train[all_features]), df_train.converted_user)
# Train model
lgbm = Pipeline([('top_20', TakeTopK(50)), ('clf', lgb.LGBMClassifier())])
search_space = {
'clf__max_depth': [3, 4, 5, 6, 7],
'clf__objective': ['binary'],
'clf__metric': ['binary_logloss'],
'clf__n_estimators': [10, 50, 100, 200],
'clf__learning_rate': [0.0001, 0.001, 0.01, 0.1, 1.0],
'clf__num_leaves': [20, 30],
'clf__min_child_samples': [5, 10, 15]
}
scorer = make_scorer(fbeta_score, beta=2, pos_label=True)
gscv = GridSearchCV(estimator=lgbm, param_grid=search_space, cv=5, scoring=scorer, n_jobs=-1)
gscv.fit(X_res, y_res.astype(int))
# log the best model information
model = gscv.best_estimator_
# Upload trained model to a stage
model_file = os.path.join('/tmp', 'model.joblib')
with open(model_file, "wb") as dill_file:
dill.dump(model, dill_file)
session.file.put(model_file, "@ml_models", overwrite=True)
# Return model F2 score on train and test data and show best model parameters
y_pred = np.where(model.predict(df_test[all_features]) > 0.5, 1, 0)
return {
'F2 score on train': gscv.best_score_,
'F2 score on test': fbeta_score(df_test['converted_user'].fillna(False), y_pred, beta=2),
'Best parameters': gscv.best_params_
}
