From 62c5d6747627cbba52f05f1fdb537ea5a4614948 Mon Sep 17 00:00:00 2001 From: ajoubrel-ensae Date: Tue, 13 Feb 2024 18:32:32 +0000 Subject: [PATCH] Ajout dummies pays et genre --- 0_Cleaning_and_merge.py | 51 +++++++++++++++++++++++++------ 0_Cleaning_and_merge_functions.py | 2 +- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/0_Cleaning_and_merge.py b/0_Cleaning_and_merge.py index 55fd043..939d4be 100644 --- a/0_Cleaning_and_merge.py +++ b/0_Cleaning_and_merge.py @@ -39,16 +39,18 @@ for i in range(len(liste_database)) : # Cleaning customerplus df1_customerplus_clean = preprocessing_customerplus(df1_customersplus) -# Cleaning ticket area -df1_ticket_information = preprocessing_tickets_area(tickets = df1_tickets, purchases = df1_purchases, suppliers = df1_suppliers, type_ofs = df1_type_ofs) - # Cleaning target area df1_target_information = preprocessing_target_area(targets = df1_targets, target_types = df1_target_types, customer_target_mappings = df1_customer_target_mappings) # Cleaning campaign area df1_campaigns_information = preprocessing_campaigns_area(campaign_stats = df1_campaign_stats, campaigns = df1_campaigns) -# Cleaning product area +## Cleaning product area + +# Cleaning ticket area +df1_ticket_information = preprocessing_tickets_area(tickets = df1_tickets, purchases = df1_purchases, suppliers = df1_suppliers, type_ofs = df1_type_ofs) + + BUCKET = "bdc2324-data" directory_path = '1' @@ -63,6 +65,14 @@ df1_products_purchased = pd.merge(df1_ticket_information, products_global, left_ # Selection des variables d'intérêts df1_products_purchased_reduced = df1_products_purchased[['ticket_id', 'customer_id', 'purchase_id' ,'event_type_id', 'supplier_name', 'purchase_date', 'type_of_ticket_name', 'amount', 'children', 'is_full_price', 'name_event_types', 'name_facilities', 'name_categories', 'name_events', 'name_seasons']] +#Exportation +BUCKET_OUT = "projet-bdc2324-team1" +FILE_KEY_OUT_S3 = "0_Temp/Company 1 - Purchases.csv" +FILE_PATH_OUT_S3 = BUCKET_OUT + "/" + FILE_KEY_OUT_S3 + +with fs.open(FILE_PATH_OUT_S3, 'w') as file_out: + df1_products_purchased_reduced.to_csv(file_out, index = False) + ## 2 - Construction of KPIs on a given period def explanatory_variables(min_date = "2021-09-01", max_date = "2023-09-01", df_campaigns_information = df1_campaigns_information, df_products_purchased_reduced = df1_products_purchased_reduced, df_customerplus_clean = df1_customerplus_clean): @@ -81,10 +91,29 @@ def explanatory_variables(min_date = "2021-09-01", max_date = "2023-09-01", df_c print("Data filtering : SUCCESS") # Fusion de l'ensemble et creation des KPI + + # KPI sur les campagnes publicitaires df_campaigns_kpi = campaigns_kpi_function(campaigns_information = df_campaigns_information) + + # KPI sur le comportement d'achat df_tickets_kpi = tickets_kpi_function(tickets_information = df_products_purchased_reduced) + # KPI sur les données socio-demographique + + ## Le genre + df_customerplus_clean["gender_label"] = df_customerplus_clean["gender"].map({ + 0: 'female', + 1: 'male', + 2: 'other' + }) + gender_dummies = pd.get_dummies(df_customerplus_clean["gender_label"], prefix='gender').astype(int) + df_customerplus_clean = pd.concat([df_customerplus_clean, gender_dummies], axis=1) + + ## Indicatrice si individue vit en France + df_customerplus_clean["country_fr"] = df_customerplus_clean["country"].apply(lambda x : int(x=="fr") if pd.notna(x) else np.nan) + print("KPIs construction : SUCCESS") + # Fusion avec KPI liés au customer df_customer = pd.merge(df_customerplus_clean, df_campaigns_kpi, on = 'customer_id', how = 'left') @@ -97,6 +126,8 @@ def explanatory_variables(min_date = "2021-09-01", max_date = "2023-09-01", df_c # Fill NaN values df_customer_product[['nb_tickets', 'nb_purchases', 'total_amount', 'nb_suppliers', 'vente_internet_max', 'nb_tickets_internet']] = df_customer_product[['nb_tickets', 'nb_purchases', 'total_amount', 'nb_suppliers', 'vente_internet_max', 'nb_tickets_internet']].fillna(0) + print("Explanatory variable construction : SUCCESS") + return df_customer_product # Fonction pour créer les variables expliquée @@ -110,6 +141,8 @@ def explained_variable(min_date = "2023-08-01", max_date = "2023-11-01", df_prod y = df_products_purchased_reduced[['customer_id', 'event_type_id', 'y_has_purchased']].drop_duplicates() + print("Explained variable construction : SUCCESS") + return y ## Exportation @@ -117,12 +150,14 @@ def explained_variable(min_date = "2023-08-01", max_date = "2023-11-01", df_prod # Dossier d'exportation BUCKET_OUT = "projet-bdc2324-team1/1_Output/Logistique Regression databases - First approach" +# Dataset test X_test = explanatory_variables(min_date = "2021-08-01", max_date = "2023-08-01", df_campaigns_information = df1_campaigns_information, df_products_purchased_reduced = df1_products_purchased_reduced, df_customerplus_clean = df1_customerplus_clean) y_test = explained_variable(min_date = "2023-08-01", max_date = "2023-11-01", df_products_purchased_reduced = df1_products_purchased_reduced) dataset_test = pd.merge(X_test, y_test, on = ['customer_id', 'event_type_id'], how = 'left') +# Exportation FILE_KEY_OUT_S3 = "dataset_test.csv" FILE_PATH_OUT_S3 = BUCKET_OUT + "/" + FILE_KEY_OUT_S3 @@ -131,12 +166,14 @@ with fs.open(FILE_PATH_OUT_S3, 'w') as file_out: print("Exportation dataset test : SUCCESS") +# Dataset train X_train = explanatory_variables(min_date = "2021-05-01", max_date = "2023-05-01", df_campaigns_information = df1_campaigns_information, df_products_purchased_reduced = df1_products_purchased_reduced, df_customerplus_clean = df1_customerplus_clean) y_train = explained_variable(min_date = "2023-05-01", max_date = "2023-08-01", df_products_purchased_reduced = df1_products_purchased_reduced) dataset_train = pd.merge(X_train, y_train, on = ['customer_id', 'event_type_id'], how = 'left') +# Exportation FILE_KEY_OUT_S3 = "dataset_train.csv" FILE_PATH_OUT_S3 = BUCKET_OUT + "/" + FILE_KEY_OUT_S3 @@ -146,8 +183,4 @@ with fs.open(FILE_PATH_OUT_S3, 'w') as file_out: print("Exportation dataset train : SUCCESS") - -# # Exportation vers 'projet-bdc2324-team1' - - -print("Exportation base de la base X d'entraînement : SUCCESS") +print("FIN DE LA GENERATION DES DATASETS : SUCCESS") diff --git a/0_Cleaning_and_merge_functions.py b/0_Cleaning_and_merge_functions.py index 15a24dc..042c60e 100644 --- a/0_Cleaning_and_merge_functions.py +++ b/0_Cleaning_and_merge_functions.py @@ -27,7 +27,7 @@ def preprocessing_customerplus(customerplus = None): cleaning_date(customerplus_copy, 'last_visiting_date') # Selection des variables - customerplus_copy.drop(['lastname', 'firstname', 'email', 'civility', 'note', 'created_at', 'updated_at', 'deleted_at', 'extra', 'reference', 'extra_field', 'identifier', 'need_reload', 'preferred_category', 'preferred_supplier', 'preferred_formula', 'zipcode', 'last_visiting_date'], axis = 1, inplace=True) + customerplus_copy.drop(['lastname', 'firstname', 'birthdate', 'profession', 'language', 'age', 'email', 'civility', 'note', 'created_at', 'updated_at', 'deleted_at', 'extra', 'reference', 'extra_field', 'identifier', 'need_reload', 'preferred_category', 'preferred_supplier', 'preferred_formula', 'zipcode', 'last_visiting_date'], axis = 1, inplace=True) customerplus_copy.rename(columns = {'id' : 'customer_id'}, inplace = True) return customerplus_copy