BDC-team-1/2_Datasets_Generation.py

# Purpose of the script : Construction of training and test datasets for modelling by company
# Input : KPI construction function and clean databases in the 0_Input folder
# Output : Train and test datasets by compagnies 

# Packages
import pandas as pd
import numpy as np
import os
import s3fs
import re
import warnings
from datetime import date, timedelta, datetime
from sklearn.model_selection import train_test_split

# Create filesystem object
S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})

# Import KPI construction functions
exec(open('utils_features_construction.py').read())

# Ignore warning
warnings.filterwarnings('ignore')


def dataset_construction(min_date, end_features_date, max_date, directory_path):
    
    # Import of cleaned and merged datasets
    df_customerplus_clean_0 = display_input_databases(directory_path, file_name = "customerplus_cleaned")
    df_campaigns_information = display_input_databases(directory_path, file_name = "campaigns_information", datetime_col = ['opened_at', 'sent_at', 'campaign_sent_at'])
    df_products_purchased_reduced = display_input_databases(directory_path, file_name = "products_purchased_reduced", datetime_col = ['purchase_date'])
    df_target_information = display_input_databases(directory_path, file_name = "target_information")
    
    # Dates in datetime format
    max_date =  pd.to_datetime(max_date, utc = True, format = 'ISO8601') 
    end_features_date = pd.to_datetime(end_features_date, utc = True, format = 'ISO8601')
    min_date = pd.to_datetime(min_date, utc = True, format = 'ISO8601')

    # Filter for database df_campaigns_information
    df_campaigns_information = df_campaigns_information[(df_campaigns_information['sent_at'] < end_features_date) & (df_campaigns_information['sent_at'] >= min_date)]
    df_campaigns_information['opened_at'][df_campaigns_information['opened_at'] >= end_features_date] = np.datetime64('NaT')
    
    # Filter for database df_products_purchased_reduced
    df_products_purchased_features = df_products_purchased_reduced[(df_products_purchased_reduced['purchase_date'] < end_features_date) & (df_products_purchased_reduced['purchase_date'] >= min_date)]

    print("Data filtering : SUCCESS")
    
    # Building and merging features 

    # Campaigns features
    df_campaigns_kpi = campaigns_kpi_function(campaigns_information = df_campaigns_information, max_date = end_features_date) 

    # Purchasing behavior features
    df_tickets_kpi = tickets_kpi_function(tickets_information = df_products_purchased_features)

    # Socio-demographic features
    df_customerplus_clean = customerplus_kpi_function(customerplus_clean = df_customerplus_clean_0)

    # Targets features
    df_targets_kpi = targets_KPI(df_target = df_target_information)
    
    print("KPIs construction : SUCCESS")
    
    # Merge - campaigns features
    df_customer = pd.merge(df_customerplus_clean, df_campaigns_kpi, on = 'customer_id', how = 'left')
    
    # Fill NaN values
    df_customer[['nb_campaigns', 'nb_campaigns_opened', 'taux_ouverture_mail']] = df_customer[['nb_campaigns', 'nb_campaigns_opened', 'taux_ouverture_mail']].fillna(0)
    df_customer['time_to_open'] = df_customer['time_to_open'].fillna(df_customer['time_to_open'].mean())
    
    # Merge - targets features
    df_customer = pd.merge(df_customer, df_targets_kpi, on = 'customer_id', how = 'left')
    
    # Fill NaN values
    targets_columns = list(df_targets_kpi.columns)
    targets_columns.remove('customer_id')

    df_customer[targets_columns] = df_customer[targets_columns].fillna(0)

    # We standardise the number of targets closely linked to the company's operations
    df_customer['nb_targets'] = (df_customer['nb_targets'] - (df_customer['nb_targets'].mean())) / (df_customer['nb_targets'].std())
    
    # Merge - purchasing behavior features
    df_customer_product = pd.merge(df_customer, df_tickets_kpi, on = 'customer_id', how = 'left')
    
    # Fill NaN values
    special_fill_nan = ['customer_id', 'purchase_date_min', 'purchase_date_max', 'time_between_purchase']    
    simple_fill_nan = [column for column in list(df_tickets_kpi.columns) if column not in special_fill_nan]

    df_customer_product[simple_fill_nan] = df_customer_product[simple_fill_nan].fillna(0)

    max_interval = (end_features_date - min_date) / np.timedelta64(1, 'D') + 1
    df_customer_product[['purchase_date_max', 'purchase_date_min']] = df_customer_product[['purchase_date_max', 'purchase_date_min']].fillna(max_interval)
    df_customer_product[['time_between_purchase']] = df_customer_product[['time_between_purchase']].fillna(-1)

    # Customers who have neither received an e-mail nor made a purchase during the feature estimation period are removed
    df_customer_product = df_customer_product[(df_customer_product['nb_purchases'] > 0) | (df_customer_product['nb_campaigns'] > 0)]

    print("Explanatory variable construction : SUCCESS")

    # 2. Construction of the explained variable 
    df_products_purchased_to_predict = df_products_purchased_reduced[(df_products_purchased_reduced['purchase_date'] < max_date) & (df_products_purchased_reduced['purchase_date'] >= end_features_date)]

    # Construction of the dependant variable
    df_products_purchased_to_predict['y_has_purchased'] = 1

    y = df_products_purchased_to_predict[['customer_id', 'y_has_purchased']].drop_duplicates()

    print("Explained variable construction : SUCCESS")
    
    # 3. Merge between explained and explanatory variables
    dataset = pd.merge(df_customer_product, y, on = ['customer_id'], how = 'left')

    # 0 if there is no purchase
    dataset[['y_has_purchased']] = dataset[['y_has_purchased']].fillna(0)

    # add id_company prefix to customer_id
    dataset['customer_id'] = directory_path + '_' + dataset['customer_id'].astype('str')
    
    return dataset

## Exportation
# Sectors
companies = {'musee' : ['1', '2', '3', '4'], # , '101'
            'sport': ['5', '6', '7', '8', '9'],
            'musique' : ['10', '11', '12', '13', '14']}

# Choosed sector
type_of_comp = input('Choisissez le type de compagnie : sport ? musique ? musee ?')
list_of_comp = companies[type_of_comp] 

# Export folder
BUCKET_OUT = f'projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}'

# Dates used for the construction of features and the dependant variable
start_date = "2021-05-01"
end_of_features = "2022-11-01"
final_date = "2023-11-01"

# Anonymous customer to be deleted from the datasets
anonymous_customer = {'1' : '1_1', '2' : '2_12184', '3' : '3_1', '4' : '4_2', '101' : '101_1',
                      '5' : '5_191835', '6' : '6_591412', '7' : '7_49632', '8' : '8_1942', '9' : '9_19683',
                     '10' : '10_19521', '11' : '11_36', '12' : '12_1706757', '13' : '13_8422', '14' : '14_6354'}

for company in list_of_comp:
    dataset = dataset_construction(min_date = start_date, end_features_date = end_of_features,
                                        max_date = final_date, directory_path = company)    
    
    # Deletion of the anonymous customer
    dataset = dataset[dataset['customer_id'] != anonymous_customer[company]]

    # Split between train and test
    dataset_train, dataset_test = train_test_split(dataset, test_size=0.3, random_state=42)
    
    # Dataset Test
    # Export
    FILE_KEY_OUT_S3 = "dataset_test" + company +  ".csv"
    FILE_PATH_OUT_S3 = BUCKET_OUT + "/Test_set/" + FILE_KEY_OUT_S3
    
    with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
        dataset_test.to_csv(file_out, index = False)
    
    print("Export of dataset test : SUCCESS")

    # Dataset train
    # Export
    FILE_KEY_OUT_S3 = "dataset_train" + company + ".csv" 
    FILE_PATH_OUT_S3 = BUCKET_OUT + "/Train_set/" + FILE_KEY_OUT_S3
    
    with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
        dataset_train.to_csv(file_out, index = False)
        
    print("Export of dataset train : SUCCESS")


print("End of dataset generation for ", type_of_comp," compagnies : SUCCESS")