CaptainAshis

#!/usr/bin/env python
# coding: utf-8


import matplotlib.pyplot as plt
from sklearn.metrics import classification_report, confusion_matrix, plot_confusion_matrix, plot_precision_recall_curve, plot_roc_curve
import logging
import pickle
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.neighbors import KNeighborsClassifier

CaptainAshis

CaptainAshis

m = md.get_learner(emb_szs, len(df.columns)-len(cat_vars),
                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range, 
                   tmp_name=f"{PATH_WRITE}tmp", models_name=f"{PATH_WRITE}models")

m.lr_find()

m.sched.plot(100)

CaptainAshis

emb_szs = [(c, min(50, (c+1)//2)) for _,c in cat_sz]
emb_szs

CaptainAshis

cat_sz = [(c, len(joined_samp[c].cat.categories)+1) for c in cat_vars]
cat_sz

CaptainAshis

# Approach 1
# One approach is to take the last 25% of rows (sorted by date) as our validation set.

train_ratio = 0.75
# train_ratio = 0.9
train_size = int(samp_size * train_ratio); train_size
val_idx = list(range(train_size, len(df)))

# Approach 2:- Just the last 2 weeks of data

CaptainAshis

# Step 14
df, y, nas, mapper = proc_df(joined_samp, 'Sales', do_scale=True)
yl = np.log(y)



joined_test = joined_test.set_index("Date")

df_test, _, nas, mapper = proc_df(joined_test, 'Sales', do_scale=True, skip_flds=['Id'],
                                  mapper=mapper, na_dict=nas)

CaptainAshis

# Step 13
for v in cat_vars: joined[v] = joined[v].astype('category').cat.as_ordered()
for v in contin_vars:
    joined[v] = joined[v].fillna(0).astype('float32')
    joined_test[v] = joined_test[v].fillna(0).astype('float32')
    
dep = 'Sales'
joined = joined[cat_vars+contin_vars+[dep, 'Date']].copy()

CaptainAshis

# Step 12
joined = pd.read_feather(f'{PATH_WRITE}joined')
joined_test = pd.read_feather(f'{PATH_WRITE}joined_test')

joined.head().T.head(40)

# Defining the categorical and continious variables
cat_vars = ['Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'StateHoliday', 'CompetitionMonthsOpen',
    'Promo2Weeks', 'StoreType', 'Assortment', 'PromoInterval', 'CompetitionOpenSinceYear', 'Promo2SinceYear',
    'State', 'Week', 'Events', 'Promo_fw', 'Promo_bw', 'StateHoliday_fw', 'StateHoliday_bw',

CaptainAshis

md = ColumnarModelData.from_data_frame(PATH, val_idx, df, yl.astype(np.float32), cat_flds=cat_vars, bs=128,
                                       test_df=df_test, is_reg=True)