import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns
plt.style.use('ggplot')

from tqdm import tqdm_notebook
from ipywidgets import IntProgress

import pickle
from itertools import product
import gc
import time

from sklearn.preprocessing import MinMaxScaler, LabelEncoder

from sklearn.preprocessing import LabelEncoder
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import train_test_split, cross_val_score, KFold
from sklearn.linear_model import ElasticNet, Lasso, Ridge, RidgeCV, LassoCV, ElasticNetCV, BayesianRidge, LinearRegression
from sklearn.tree import DecisionTreeRegressor, ExtraTreeClassifier
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.model_selection import GridSearchCV, ShuffleSplit
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error

from lightgbm import LGBMRegressor
import lightgbm as lgb
import xgboost as xgb
from xgboost import XGBRegressor

np.random.seed(42)

sales_train = pd.read_csv('./competitive-data-science-predict-future-sales/sales_train.csv', parse_dates=['date'])
sales_test = pd.read_csv('./competitive-data-science-predict-future-sales/test.csv')
sample_submission = pd.read_csv('./competitive-data-science-predict-future-sales/sample_submission.csv')
items = pd.read_csv('./competitive-data-science-predict-future-sales/items.csv')
item_categories = pd.read_csv('./competitive-data-science-predict-future-sales/item_categories.csv')
shops = pd.read_csv('./competitive-data-science-predict-future-sales/shops.csv')

sample_submission.head(3)

print("Sales Train")
sales_train.head()

print("Sales Train set shape", sales_train.shape)

sales_train.info()

print("Null records")
sales_train.isnull().sum()

print('Description')
sales_train.describe()

print("Sales Test")
sales_test.head()

print("Sales Test set shape", sales_test.shape)

sales_test.info()

print("Items")
items.head()

print("Items shape", items.shape)

print("Null records")
items.isnull().sum()

items.info()

print("Item Categories")
item_categories.head()

print("Item categories shape", item_categories.shape)

item_categories.info()

print("Null records")
item_categories.isnull().sum()

print("Shops")
shops.head()

print("Shop shape", shops.shape)

print("Null records")
shops.isnull().sum()

shops.info()

# merge datset and drop duplicates keys
train = sales_train.join(items, on='item_id', rsuffix='_') \
                   .join(shops, on='shop_id', rsuffix='_') \
                   .join(item_categories, on='item_category_id', rsuffix='_') \
                   .drop(['item_id_', 'shop_id_', 'item_category_id_'], axis=1)
print("Join dataset shape:",train.shape)
train.head()

# isolate date from October 2015
bench_oct2015 = train.loc[train['date_block_num'] == 33]

# display the subset
bench_oct2015.head()

# the sales are aggregated by shop_id, item_id
test_oct2015 = bench_oct2015.groupby(["shop_id", "item_id"])

# aggregate item_cnt by sum
test_oct2015 = test_oct2015.aggregate({"item_cnt_day":np.sum}).fillna(0)

# melt indexes
test_oct2015 = test_oct2015.reset_index(level=["shop_id", "item_id"])

# adjust column names to match submission
test_oct2015 = test_oct2015.rename(columns={ test_oct2015.columns[2]: "item_cnt_month" })

sales_test['item_id'].value_counts()

test_oct2015['item_id'].value_counts()

# the store+item without any sale are not listed in our table.
test_submission = pd.merge(test_oct2015, sales_test,
                           how='right', left_on=['shop_id','item_id'],
                           right_on = ['shop_id','item_id']).fillna(0)
test_submission.drop(['shop_id', 'item_id'], axis=1)
test_submission = test_submission[['ID','item_cnt_month']]

# clip predictions
test_submission['item_cnt_month'] = test_submission['item_cnt_month'].clip(0,20)
test_submission.describe()

test_submission.to_csv('previous_value_benchmark.csv', index=False)

# set predictions equal to 0
test_submission['item_cnt_month'] = 0.0
test_submission.to_csv('zero_value.csv', index=False)

# set predictions equal to 1.0
test_submission['item_cnt_month'] = 1.0
test_submission.to_csv('one_value.csv', index=False)

mean_test = (1.25011**2 + 1.0 - 1.41241**2) / 2.0
print("Mean of test set: {:.8f}".format(mean_test))

train.info()

train.describe()

def change_data_type(df):
    float_cols = [col for col in df if df[col].dtype == 'float64']
    int_cols = [col for col in df if df[col].dtype == 'int64']

    df[float_cols] = df[float_cols].astype(np.float32)
    df[int_cols] = df[int_cols].astype(np.int16)
    return df


train = change_data_type(train)
train.info()

# plot boxplot
f, ax = plt.subplots(figsize=(12, 4))
sns.boxplot(train['item_price'])

train[train['item_price'] > 150000]

train[train['item_id'] == 6066]

# remove the single outlier for large price
train = train[train['item_price'] < 150000]

train['item_price'].value_counts().sort_index().head()

# remove the single outlier for negative price
train = train[train['item_price'] >= 0]

# plot boxplot
f, ax = plt.subplots(figsize=(12, 4))
ax.set_xlim(0.01, 1e5)
ax.set_xscale("log")
sns.boxplot(train['item_price'])

# plot boxplot
f, ax = plt.subplots(figsize=(12, 4))
sns.boxplot(train['item_cnt_day'])

train[train['item_cnt_day'] > 1500]

# remove the outlier for item_cnt_day
train = train[train['item_cnt_day'] < 1500]

fig, ax = plt.subplots(figsize=(12, 12))
sns.scatterplot(x='item_cnt_day', y='item_price', data=train, alpha=0.3)

shops['shop_name'].value_counts().sort_values(ascending=False).head()

shops['shop_name'].str.strip('!').sort_values()

shops['shop_name'].loc[[0, 57, 1, 58, 10, 11]]

# (0, 57) -> Якутск Орджоникидзе, 56
train.loc[train['shop_id'] == 0, 'shop_id'] = 57
sales_test.loc[sales_test['shop_id'] == 0, 'shop_id'] = 57

# (1, 58) -> Якутск ТЦ "Центральный"
train.loc[train['shop_id'] == 1, 'shop_id'] = 58
sales_test.loc[sales_test['shop_id'] == 1, 'shop_id'] = 58

# (10, 11) -> Жуковский ул. Чкалова 39м²
train.loc[train['shop_id'] == 10, 'shop_id'] = 11
sales_test.loc[sales_test['shop_id'] == 10, 'shop_id'] = 11

# add revenue to train set
train['revenue'] = train['item_price'] * train['item_cnt_day']

train.columns

# replace one faulty city name (Сергиев Посад)
shops.loc[shops['shop_name']=='Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'

# split shop name (and remove the leading !)
shops['city'] = shops['shop_name'].str.strip('!').str.split(' ').map(lambda x: x[0])
shops['city'].value_counts().sort_index()

shops['city_id'] = LabelEncoder().fit_transform(shops['city'])

shops = shops[['shop_id', 'city_id']]

item_categories.head(10)

# split `item_category_name`
item_categories['split'] = item_categories['item_category_name'].str.split(' - ')

# isolate `type_name` and 'subtype_name'
item_categories['type_name'] = item_categories['split'].apply(lambda x: x[0].strip())
item_categories['subtype_name'] = item_categories['split'].apply(lambda x: x[1].strip() if len(x)>1 else x[0].strip())

# the type and subtype can be encoded
item_categories['type_id'] = LabelEncoder().fit_transform(item_categories['type_name'])
item_categories['subtype_id'] = LabelEncoder().fit_transform(item_categories['subtype_name'])

# filter out text features
item_categories = item_categories[['item_category_id', 'type_id', 'subtype_id']]

items = items[['item_id', 'item_category_id']]

print("=====  ITEMS  =====")
print("There are {} unique items in the train set.".format(len(train['item_id'].unique())))
print("There are {} unique items in the test set.".format(len(sales_test['item_id'].unique())))
print("{} items are in the test but not in the train set.".format(len(set(sales_test['item_id'].unique()) - set(train['item_id'].unique()))))

print("\n===== SHOPS  =====")
print("There are {} unique shops in the train set.".format(len(train['shop_id'].unique())))
print("There are {} unique shops in the test set.".format(len(sales_test['shop_id'].unique())))
print("{} shop are in the test but not in the train set.".format(len(set(sales_test['shop_id'].unique()) - set(train['shop_id'].unique()))))

print("\n===== TEST  =====")
print("The test set contains {} pairs.".format(sales_test.shape[0]))
print("There are {} possible unique pairs using the test data.".format(len(sales_test['item_id'].unique()) * len(sales_test['shop_id'].unique())))

from itertools import product

full_data = []
cols = ['date_block_num','shop_id','item_id']

for i in range(34):
    
    # isolate sales made on ith month
    monthly_sales = train[train['date_block_num']==i]
    
    # create pairs of id, shops, items
    full_data.append(np.array(list(product([i],
                                        monthly_sales['shop_id'].unique(),
                                        monthly_sales['item_id'].unique())
                               ), dtype='int16'))
    
# create dataframe    
full_data = pd.DataFrame(np.vstack(full_data), columns=cols)

# convert data to optimize memory
full_data['date_block_num'] = full_data['date_block_num'].astype(np.int8)
full_data['shop_id'] = full_data['shop_id'].astype(np.int8)
full_data['item_id'] = full_data['item_id'].astype(np.int16)

# sort values
full_data =  full_data.sort_values(cols)

full_data.shape

full_data.head()

# aggregate item_cnt_day
monthly_train = train.groupby(['date_block_num','shop_id','item_id']).agg({'item_cnt_day':'sum'})

# reset index and columns
monthly_train = monthly_train.reset_index()

# combine monthly train and pairs
full_data = pd.merge(left=full_data, right=monthly_train, on=cols, how='left')

# clip data, fill nulls with 0, and downsize the datatype
full_data['item_cnt_day'] = full_data['item_cnt_day'].fillna(0).clip(0,20).astype(np.float32)

# create month number
sales_test['date_block_num'] = 34

# downsize features
sales_test['shop_id'] = sales_test['shop_id'].astype(np.int16)
sales_test['item_id'] = sales_test['item_id'].astype(np.int16)
sales_test['date_block_num'] = sales_test['date_block_num'].astype(np.int16)

sales_test = sales_test.set_index('ID')

# concat test and full_data
full_data = pd.concat([full_data, sales_test], ignore_index=True, keys=cols, sort=False)

# fill test values to 0
full_data = full_data.fillna(0)

# shops
full_data = pd.merge(left=full_data, right=shops, how='left', on='shop_id')

# items
full_data = pd.merge(left=full_data, right=items, how='left', on='item_id')

# item_categories
full_data = pd.merge(left=full_data, right=item_categories, how='left', on='item_category_id')

# downsize the in64
full_data['city_id'] = full_data['city_id'].astype(np.int16)
full_data['item_category_id'] = full_data['item_category_id'].astype(np.int16)
full_data['type_id'] = full_data['type_id'].astype(np.int16)
full_data['subtype_id'] = full_data['subtype_id'].astype(np.int16)

full_data.head()

# generate ticks for monthly plots
x_tick_vals = pd.Series(pd.date_range('2013', freq='M', periods=34))
x_tick_vals = [str(x.month) + '/' + str(x.year) for x in x_tick_vals]

groups = full_data.groupby(['date_block_num'])['item_cnt_day'].sum()

fig, ax = plt.subplots(figsize=(14,6))
ax.set_title('Total item sold')
ax.set_ylim(0,groups.max())
ax.set_xlabel('Date')
ax.set_xticks(range(34))
ax.set_xticklabels(x_tick_vals, rotation=60)
ax.plot(groups[:-1]);

train['month'] = train['date'].dt.month
train['year'] = train['date'].dt.year

groups = train.groupby(['year','month'])['item_cnt_day'].sum()
groups = groups.reset_index()
groups = groups.groupby('month')['item_cnt_day'].mean()

fig, ax = plt.subplots(figsize=(14,6))
ax.set_title('Total items sold per month')
ax.set_ylim(0,1.1*groups.max())
ax.set_xlabel('Month')
ax.set_xticks(range(1,13))
ax.set_xticklabels(x_tick_vals, rotation=60)
ax.plot(groups);

gp_category_mean = full_data.groupby(['item_category_id'], as_index=False)['item_cnt_day'].mean()
gp_category_sum = full_data.groupby(['item_category_id'], as_index=False)['item_cnt_day'].sum()

f, axes = plt.subplots(2, 1, figsize=(16, 10), sharex=True)
sns.barplot(x="item_category_id", y="item_cnt_day", data=gp_category_mean, ax=axes[0], palette="deep").set_title("Average number of item sold daily per category")
sns.barplot(x="item_category_id", y="item_cnt_day", data=gp_category_sum, ax=axes[1], palette="deep").set_title("Average number of item sold per category")
axes[0].set_xlabel('')
axes[1].set_xticklabels(axes[1].get_xticklabels(), rotation=60)
axes[1].tick_params(labelsize=8)
plt.show()

gp_shops_sum = full_data.groupby(['shop_id'], as_index=False)['item_cnt_day'].sum()

f, ax = plt.subplots(figsize=(16, 8))
sns.barplot(x="shop_id", y="item_cnt_day", data=gp_shops_sum, ax=ax, palette="deep").set_title("Total number of item sold per store")
ax.set_xlabel('')
ax.set_xticklabels(axes[1].get_xticklabels(), rotation=60)
ax.tick_params(labelsize=8)
plt.show()

gp_cat_rev = train.groupby(['year', 'item_category_id'], as_index=False)['revenue'].sum()

sns.catplot(x="item_category_id", y="revenue",
            data=gp_cat_rev, row="year",
            palette="deep", kind="bar",
            height=4, aspect=3.5)
plt.show()

return_df = train.copy()
return_df['return'] = return_df['item_cnt_day'].apply(lambda x: -min(x, 0))
return_df['sales'] = return_df['item_cnt_day'].apply(lambda x: max(x, 0))

return_cats = return_df.groupby(['item_category_id'], as_index=False)['return', 'sales'].sum()
return_cats['percent_return'] = return_cats['return'] / return_cats['sales'] * 100.

f, ax = plt.subplots(figsize=(16, 8))
sns.barplot(x="item_category_id", y="percent_return", data=return_cats, ax=ax, palette="deep").set_title("Percentage of return per category")
ax.set_xlabel('')
ax.set_xticklabels(axes[1].get_xticklabels(), rotation=60)
ax.tick_params(labelsize=8)
plt.show()

return_stores = return_df.groupby(['shop_id'], as_index=False)['return', 'sales'].sum()
return_stores['percent_return'] = return_stores['return'] / return_stores['sales'] * 100.

f, ax = plt.subplots(figsize=(16, 8))
sns.barplot(x="shop_id", y="percent_return", data=return_stores, ax=ax, palette="deep").set_title("Return rate per store")
ax.set_xlabel('')
ax.set_xticklabels(axes[1].get_xticklabels(), rotation=60)
ax.tick_params(labelsize=8)
plt.show()

return_df['date'] = return_df['month'].astype('str').str.cat(return_df['year'].astype('str'),sep="-")

return_month = return_df.groupby(['date'], as_index=False)['return', 'sales'].sum()
return_month['percent_return'] = return_month['return'] / return_month['sales'] * 100.

f, ax = plt.subplots(figsize=(16, 8))
sns.barplot(x="date", y="percent_return", data=return_month, ax=ax, palette="deep").set_title("Return rate per store")
ax.set_xlabel('')
ax.set_xticklabels(ax.get_xticklabels(), rotation=60)
ax.tick_params(labelsize=8)
plt.show()

del ax, axes
del bench_oct2015, test_oct2015, test_submission
del f, fig, x_tick_vals
del i
del gp_cat_rev, gp_category_mean, gp_category_sum, gp_shops_sum, groups
del return_cats, return_stores, monthly_train, monthly_sales

print('Datasize before downcast:')
print(full_data.info())

def downcast_dtypes(df):
    '''
        Changes column types in the dataframe: 
                
                `float64` type to `float16`
                `int64`   type to `int16`http://localhost:8888/notebooks/Google%20Drive/2-Coding/1-Coursera/Advanced%20ML/2-Kaggle/Final%20Project/Final%20Project.ipynb#
    '''
    
    # Select columns to downcast
    float_cols = [c for c in df if df[c].dtype == "float64" or df[c].dtype == 'float32']
    int_cols =   [c for c in df if (df[c].dtype == "int64" or df[c].dtype == 'int32'  or df[c].dtype == 'int16') and (c!='item_id')]
    
    # Downcast
    df[float_cols] = df[float_cols].astype(np.float16)
    df[int_cols]   = df[int_cols].astype(np.int8)
    
    return df

full_data = downcast_dtypes(full_data)

print('Datasize after downcast:')
print(full_data.info())

# create return features
return_df = return_df[['date_block_num', 'shop_id', 'item_id', 'item_category_id', 'return']]

# ===========================
# create groups
# ===========================

# items
group_item = full_data.groupby(['date_block_num','item_id'], as_index=False)

# store + item
group_store_item = full_data.groupby(['date_block_num','item_id', 'shop_id'], as_index=False)

# store
group_store = full_data.groupby(['date_block_num', 'shop_id'])

# category
group_cat = full_data.groupby(['date_block_num', 'item_category_id'])

# category + store
group_cat_store = full_data.groupby(['date_block_num', 'shop_id', 'item_category_id'])

# type code
group_type = full_data.groupby(['date_block_num', 'type_id'])

# type + store
group_type_shop = full_data.groupby(['date_block_num', 'shop_id', 'type_id'])

# subtype
group_subtype = full_data.groupby(['date_block_num', 'subtype_id'])

# subtype + store
group_subtype_shop = full_data.groupby(['date_block_num', 'shop_id', 'subtype_id'])

# city
group_city = full_data.groupby(['date_block_num', 'city_id'])

# city + item
group_city_item = full_data.groupby(['date_block_num', 'item_id', 'city_id'])

# return per shop + item
return_shop_item = return_df.groupby(['date_block_num', 'shop_id', 'item_id'], as_index=False).agg(np.sum)

# return per shop
return_shop = return_df.groupby(['date_block_num', 'shop_id'], as_index=False)['return'].agg(np.sum)

# return per item
retun_item = return_df.groupby(['date_block_num', 'item_id'], as_index=False)['return'].agg(np.sum)

# ===========================
# create encodings
# ===========================

# PRICE
# min, max, average of price per items per month
monthly_sales_group = train.groupby(['date_block_num','item_id'])
min_max_avg_prices = monthly_sales_group['item_price'].agg(['min', 'max', np.mean]).reset_index()
min_max_avg_prices = min_max_avg_prices.rename(columns={"min": "min_item_price",
                                                        "max": "max_item_price",
                                                        "mean": "avg_item_price"})

# min, max, average of price per item per item per store per month
monthly_sales_item_store_group = train.groupby(['date_block_num','item_id', 'shop_id'])
min_max_avg_item_store = monthly_sales_item_store_group['item_price'].agg(['min', 'max', np.mean]).reset_index()
min_max_avg_item_store = min_max_avg_item_store.rename(columns={"min": "min_item_shop_price",
                                                                "max": "max_item_shop_price",
                                                                "mean": "avg_item_shop_price"})

# RETURNS
# sum returns per store
return_shop = return_df.groupby(['date_block_num', 'shop_id'], as_index=False)['return'].agg(np.sum).rename(columns={'return': "shop_return"})

# sum returns per item
return_item = return_df.groupby(['date_block_num', 'item_id'], as_index=False)['return'].agg(np.sum).rename(columns={'return': "item_return"})

# sum returns per category
return_cat = return_df.groupby(['date_block_num', 'item_category_id'], as_index=False)['return'].agg(np.sum).rename(columns={'return': "cat_return"})

# sum returns per store + item
return_item_store = return_df.groupby(['date_block_num', 'item_id', 'shop_id'], as_index=False)['return'].agg(np.sum).rename(columns={'return': "item_shop_return"})

# sum returns per store per category
return_store_cat = return_df.groupby(['date_block_num', 'shop_id', 'item_category_id'], as_index=False)['return'].agg(np.sum).rename(columns={'return': "cat_shop_return"})

# SALES
# number of stores selling the item per month
store_count = full_data.groupby(['date_block_num','item_id'])['shop_id'].count().reset_index()
store_count = store_count.rename(columns={'shop_id': "shop_count"})

# target item count
target_item = full_data.groupby(['date_block_num', 'item_id'], as_index=False)['item_cnt_day'].sum()
target_item = target_item.rename(columns={"item_cnt_day": "item_sold"})

# number of unique items sold in store per month
unique_items = group_store['item_id'].nunique().reset_index()
unique_items = unique_items.rename(columns={'item_id': "item_avail"})

# number of item sold per category
cat_count = group_cat['item_cnt_day'].agg(np.mean).reset_index()
cat_count = cat_count.rename(columns={'item_cnt_day': 'avg_cat_count'})

# nunber of item sold per category per store
cat_store_count = group_cat_store['item_cnt_day'].agg(np.mean).reset_index()
cat_store_count = cat_store_count.rename(columns={'item_cnt_day': 'avg_cat_store_count'})

# TYPES AND SUBTYPES
# types
type_count = group_type['item_cnt_day'].agg(np.mean).reset_index()
type_count = type_count.rename(columns={'item_cnt_day':'avg_type'})

type_store_count = group_type_shop['item_cnt_day'].agg(np.mean).reset_index()
type_store_count = type_store_count.rename(columns={'item_cnt_day':'avg_type_store'})

# subtype
subtype_count = group_subtype['item_cnt_day'].agg(np.mean).reset_index()
subtype_count = subtype_count.rename(columns={'item_cnt_day':'avg_subtype'})

subtype_store_count = group_subtype_shop['item_cnt_day'].agg(np.mean).reset_index()
subtype_store_count = subtype_store_count.rename(columns={'item_cnt_day':'avg_subtype_store'})

# CITY
city_count = group_city['item_cnt_day'].agg(np.mean).reset_index().rename(columns={'item_cnt_day':'avg_city'})
city_count_item = group_city_item['item_cnt_day'].agg(np.mean).reset_index().rename(columns={'item_cnt_day':'avg_city_item'})

def streak(df, col):
    """
    Return a new feature corresponding to the streak of the sales for an item.
    """

    clone = df[['item_id', 'shop_id', 'date_block_num', col]].copy()

    # sort clone (item_id, shop_id, data_block_num)
    clone = clone.sort_values(['item_id', 'shop_id', 'date_block_num'])

    # create new sold feature
    clone['sold'] = (clone[col] > 0).astype(int)

    # create streak reset condition
    reset = (clone['sold']!=clone['sold'].shift()) | \
            (clone['item_id']!=clone['item_id'].shift()) | \
            (clone['shop_id']!=clone['shop_id'].shift())

    # create streak
    clone['streak'] = clone['sold'].groupby((reset).cumsum()).cumsum()

    # add total sales (10, 11, 0, 0, 5, 1, 0) => (10, 21, 21, 21, 26, 27, 27)
    cum_sales_df = clone.groupby(
        by=['item_id', 'shop_id', 'date_block_num'])[col].sum().groupby(
            level=[0, 1]).cumsum().reset_index().rename(
                columns={col: 'cum_sales'})
    clone = pd.merge(left=clone,
                     right=cum_sales_df,
                     on=['item_id', 'shop_id', 'date_block_num'],
                     how='left')

    # delete column
    del clone[col]

    # merge streak with original df
    return_df = pd.merge(left=df,
                         right=clone,
                         on=['item_id', 'shop_id', 'date_block_num'],
                         how='left')

    # add sale streak in term of number (10, 11, 0, 0, 5, 1, 0) => (10, 21, 0, 0, 5, 6, 0)
    return_df['cum_if_sales'] = return_df['cum_sales']

    # remove values of cum_if_sales for months without sales
    return_df['cum_if_sales'] = return_df['cum_if_sales'] * return_df['sold']

    # delete column
    del return_df['sold']

    return return_df

# combine all new features with original dataframe and create sale streaks
key_item_store = ['item_id', 'shop_id', 'date_block_num']
key_item = ['item_id', 'date_block_num']
key_store = ['shop_id', 'date_block_num']
key_store_cat = ['shop_id', 'shop_id', 'item_category_id', 'date_block_num']
key_cat = ['item_category_id', 'date_block_num']
key_type = ['type_id', 'date_block_num']
key_type_shop = ['type_id', 'shop_id', 'date_block_num']
key_subype = ['subtype_id', 'date_block_num']
key_subtype_shop = ['subtype_id', 'shop_id', 'date_block_num']
key_city = ['city_id', 'date_block_num']
key_city_item = ['city_id', 'item_id', 'date_block_num']

filename = 'data'
with open(filename, 'wb') as outfile:
    pickle.dump(full_data, outfile)

# PRICE
full_data = pd.merge(left=full_data, right=min_max_avg_prices, on=key_item, how='left')
full_data = pd.merge(left=full_data, right=min_max_avg_item_store, on=key_item_store, how='left')

# RETURNS
full_data = pd.merge(left=full_data, right=return_shop, on=key_store, how='left')
full_data = pd.merge(left=full_data, right=return_item, on=key_item, how='left')
#full_data = pd.merge(left=full_data, right=return_cat, on=key_cat, how='left')
full_data = pd.merge(left=full_data, right=return_item_store, on=key_item_store, how='left')
#full_data = pd.merge(left=full_data, right=return_store_cat, on=key_store_cat, how='left')

# SALES
full_data = pd.merge(left=full_data, right=store_count, on=key_item, how='left')
full_data = pd.merge(left=full_data, right=target_item, on=key_item, how='left')
#full_data = pd.merge(left=full_data, right=unique_items, on=key_store, how='left')
full_data = pd.merge(left=full_data, right=cat_count, on=key_cat, how='left')
full_data = pd.merge(left=full_data, right=cat_store_count, on=key_store_cat, how='left')

# TYPES AND SUBTYPES
full_data = pd.merge(left=full_data, right=type_count, on=key_type, how='left')
full_data = pd.merge(left=full_data, right=type_store_count, on=key_type_shop, how='left')
full_data = pd.merge(left=full_data, right=subtype_count, on=key_subype, how='left')
full_data = pd.merge(left=full_data, right=subtype_store_count, on=key_subtype_shop, how='left')

# CITY
#full_data = pd.merge(left=full_data, right=city_count, on=key_city, how='left')
#full_data = pd.merge(left=full_data, right=city_count_item, on=key_city_item, how='left')

# fill_na
columns = ['min_item_price', 'max_item_price', 'avg_item_price',
           'min_item_shop_price', 'max_item_shop_price', 'avg_item_shop_price', 
           'shop_return', 'item_return', 'item_shop_return']

full_data.loc[:, columns] = full_data[columns].fillna(0)

# create data skreak
full_data = streak(full_data, 'item_cnt_day')

del min_max_avg_prices, min_max_avg_item_store, return_shop, return_item, return_cat, return_item_store
del return_store_cat, store_count, target_item, unique_items, cat_count, cat_store_count
del type_count, type_store_count, subtype_count, subtype_store_count
del city_count, city_count_item

del return_df, return_shop_item, return_month

filename = 'data'
with open(filename, 'wb') as outfile:
    pickle.dump(full_data, outfile)
#with open(filename, 'rb') as infile:
#    full_data = pickle.load(infile)

# List of columns that we will use to create lags
index_cols = ['shop_id', 'item_id', 'date_block_num']
no_shift = ['city_id', 'item_category_id', 'subtype_id', 'type_id']
cols_to_rename = list(full_data.columns.difference(index_cols).difference(no_shift))

def lag_feature(df,lags, cols):
    '''
    Add new lag columns to the dataframe (df).
    Inputs:
        df: input dataframe containing time-series data.
        lags: list of integer corresponding to the desired lags.
    '''
    for month_shift in tqdm_notebook(lags):
     
        # clone df
        df_shift = df[index_cols + cols].copy()

        # shift date_block_num
        df_shift['date_block_num'] = df_shift['date_block_num'] + month_shift

        # dummy rename function
        foo = lambda x: '{}_lag_{}'.format(x, month_shift) if x in cols else x

        # rename columns
        df_shift = df_shift.rename(columns=foo)

        # downcast
        df_shift = downcast_dtypes(df_shift)
        
        # merge original data with shift
        df = pd.merge(df, df_shift, on=index_cols, how='left').fillna(0)

    return df

# create lags
full_data = lag_feature(full_data, [1], ['avg_cat_count', 'avg_cat_store_count',
                                         'avg_subtype', 'avg_subtype_store',
                                         'avg_type', 'avg_type_store',
                                         'item_return', 'item_shop_return',
                                         'item_sold', 'shop_count', 'shop_return'])

full_data = lag_feature(full_data, [1,2,3,4], ['min_item_price', 'avg_item_price',
                                               'cum_if_sales', 'cum_sales',
                                               'max_item_price'])

full_data = lag_feature(full_data, [1,2,3,4,5,6], ['max_item_shop_price', 'min_item_shop_price',
                                                   'avg_item_shop_price'])

full_data = lag_feature(full_data, [1,2,3,4,5,6,12], ['item_cnt_day', 'streak'])

# Extract time based features.
full_data['year'] = full_data['date_block_num'].apply(lambda x: ((x//12) + 2013))
full_data['month'] = full_data['date_block_num'].apply(lambda x: (x % 12))

# metrics related to the sales of current month (to be deleted)
to_delete = [
    "min_item_price", "max_item_price", "avg_item_price",
    "min_item_shop_price", "max_item_shop_price", "avg_item_shop_price",
    "shop_return", "item_return", "item_shop_return", "shop_count",
    "item_sold", "item_avail", "avg_cat_count", "avg_cat_store_count",
    "avg_type", "avg_type_store", "avg_subtype", "avg_subtype_store", "streak",
    "cum_sales", "cum_if_sales"
]

# purge full set of columns to be deleted
full_data = full_data[full_data.columns.difference(to_delete)]

# remove first 12 months (null lag and old data)
full_data = full_data[(full_data['date_block_num']>=12)]

filename = 'data_before_split'
with open(filename, 'wb') as outfile:
    pickle.dump(full_data, outfile)
#with open(filename, 'rb') as infile:
#    full_data = pickle.load(infile)

# train set
X_train = full_data[full_data['date_block_num'] < 28].drop(['item_cnt_day'],
                                                           axis=1)
X_train_dates = full_data[full_data['date_block_num'] < 28]['date_block_num']
Y_train = full_data[full_data['date_block_num'] < 28]['item_cnt_day']

# validation set for first-layer model
X_valid = full_data[(full_data['date_block_num'] >= 28)
                    & (full_data['date_block_num'] < 33)].drop(
                        ['item_cnt_day'], axis=1)
X_valid_dates = full_data[(full_data['date_block_num'] >= 28)
                    & (full_data['date_block_num'] < 33)]['date_block_num']
Y_valid = full_data[(full_data['date_block_num'] >= 28)
                    & (full_data['date_block_num'] < 33)]['item_cnt_day']

# validation set for meta-model
X_valid_meta = full_data[full_data['date_block_num'] == 33].drop(['item_cnt_day'],axis=1)
X_valid_meta_dates = full_data[full_data['date_block_num'] == 33]['date_block_num']
Y_valid_meta = full_data[full_data['date_block_num'] == 33]['item_cnt_day']

# test set (predictions)
X_test = full_data[full_data['date_block_num'] == 34].drop(['item_cnt_day'],
                                                           axis=1)

# train
pickle.dump(X_train, open('X_train.pickle', 'wb'))
pickle.dump(Y_train, open('Y_train.pickle', 'wb'))
pickle.dump(X_train_dates, open('X_train_dates.pickle', 'wb'))

# validation
pickle.dump(X_valid, open('X_valid.pickle', 'wb'))
pickle.dump(Y_valid, open('Y_valid.pickle', 'wb'))
pickle.dump(X_valid_dates, open('X_valid_dates.pickle', 'wb'))

# validation meta
pickle.dump(X_valid_meta, open('X_valid_meta.pickle', 'wb'))
pickle.dump(Y_valid_meta, open('Y_valid_meta.pickle', 'wb'))
pickle.dump(X_valid_meta_dates, open('X_valid_meta_dates.pickle', 'wb'))

# test
pickle.dump(X_test, open('X_test.pickle', 'wb'))

# define number of total records
n_rows = X_train.shape[0] + X_valid.shape[0] + X_test.shape[0] + X_valid_meta.shape[0]

# print fractions
print('Train set records:', X_train.shape[0])
print('Validation set records:', X_valid.shape[0])
print('Meta validation set records:', X_valid_meta.shape[0])
print('Test set records:', X_test.shape[0])

print('Train set records: %s (%.f%% of complete data)' % (X_train.shape[0], ((X_train.shape[0]/n_rows)*100)))
print('Validation set records: %s (%.f%% of complete data)' % (X_valid.shape[0], ((X_valid.shape[0]/n_rows)*100)))
print('Meta validation set records: %s (%.f%% of complete data)' % (X_valid_meta.shape[0], ((X_valid_meta.shape[0]/n_rows)*100)))

print('mean for whole train set: {0}'.format(
    np.mean(full_data.loc[full_data['date_block_num'] < 28, 'item_cnt_day'].astype(
        np.float32))))
print('mean for validation train set: {0}'.format(
    np.mean(full_data.loc[(full_data['date_block_num'] < 33) & (full_data['date_block_num'] >= 28), 'item_cnt_day'].astype(
        np.float32))))

item_in_test_only = set(X_test['item_id'].unique()).difference(set(X_train['item_id']))
print('Items in test and not in train: {0}'.format(len(item_in_test_only)))
item_in_train_only = set(X_train['item_id'].unique()).difference(set(X_test['item_id']))
print('Items in train and not in test: {0}'.format(len(item_in_train_only)))

del full_data

import pickle
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import xgboost
from xgboost import XGBRegressor
import gc

gc.collect()
# reload
X_train = pd.read_pickle('X_train.pickle')
X_train_dates = pd.read_pickle('X_train_dates.pickle')
Y_train = pd.read_pickle('Y_train.pickle')

X_valid = pd.read_pickle('X_valid.pickle')
X_valid_dates = pd.read_pickle('X_valid_dates.pickle')
Y_valid = pd.read_pickle('Y_valid.pickle')

X_valid_meta = pd.read_pickle('X_valid_meta.pickle')
X_valid_meta_dates = pd.read_pickle('X_valid_meta_dates.pickle')
Y_valid_meta = pd.read_pickle('Y_valid_meta.pickle')

X_test = pd.read_pickle('X_test.pickle')

# create simple random forest
forest = ExtraTreesRegressor(n_estimators=50,
                              random_state=0,
                              verbose=1,
                              n_jobs=-1)

# train model on training set
forest.fit(X_train, Y_train)

# extract feature importances
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

pd.options.display.max_rows = 999
feature_importance = pd.DataFrame(data = {'feature':X_train.columns,'score_mean':importances,'score_std':std})
feature_importance.sort_values('score_mean', ascending=False)