import pandas as pd import os import s3fs import warnings from datetime import date, timedelta, datetime import numpy as np import matplotlib.pyplot as plt import matplotlib.dates as mdates import seaborn as sns def load_files(nb_compagnie): customer = pd.DataFrame() campaigns_brut = pd.DataFrame() campaigns_kpi = pd.DataFrame() products = pd.DataFrame() tickets = pd.DataFrame() # début de la boucle permettant de générer des datasets agrégés pour les 5 compagnies de spectacle for directory_path in nb_compagnie: df_customerplus_clean_0 = display_databases(directory_path, file_name = "customerplus_cleaned") df_campaigns_brut = display_databases(directory_path, file_name = "campaigns_information", datetime_col = ['opened_at', 'sent_at', 'campaign_sent_at']) df_products_purchased_reduced = display_databases(directory_path, file_name = "products_purchased_reduced", datetime_col = ['purchase_date']) df_target_information = display_databases(directory_path, file_name = "target_information") df_campaigns_kpi = campaigns_kpi_function(campaigns_information = df_campaigns_brut) df_tickets_kpi = tickets_kpi_function(tickets_information = df_products_purchased_reduced) df_customerplus_clean = customerplus_kpi_function(customerplus_clean = df_customerplus_clean_0) # creation de la colonne Number compagnie, qui permettra d'agréger les résultats df_tickets_kpi["number_company"]=int(directory_path) df_campaigns_brut["number_company"]=int(directory_path) df_campaigns_kpi["number_company"]=int(directory_path) df_customerplus_clean["number_company"]=int(directory_path) df_target_information["number_company"]=int(directory_path) # Traitement des index df_tickets_kpi["customer_id"]= directory_path + '_' + df_tickets_kpi['customer_id'].astype('str') df_campaigns_brut["customer_id"]= directory_path + '_' + df_campaigns_brut['customer_id'].astype('str') df_campaigns_kpi["customer_id"]= directory_path + '_' + df_campaigns_kpi['customer_id'].astype('str') df_customerplus_clean["customer_id"]= directory_path + '_' + df_customerplus_clean['customer_id'].astype('str') df_products_purchased_reduced["customer_id"]= directory_path + '_' + df_products_purchased_reduced['customer_id'].astype('str') # Concaténation customer = pd.concat([customer, df_customerplus_clean], ignore_index=True) campaigns_kpi = pd.concat([campaigns_kpi, df_campaigns_kpi], ignore_index=True) campaigns_brut = pd.concat([campaigns_brut, df_campaigns_brut], ignore_index=True) tickets = pd.concat([tickets, df_tickets_kpi], ignore_index=True) products = pd.concat([products, df_products_purchased_reduced], ignore_index=True) return customer, campaigns_kpi, campaigns_brut, tickets, products def save_file_s3(File_name, type_of_activity): FILE_PATH = f"projet-bdc2324-team1/stat_desc/{type_of_activity}/" FILE_PATH_OUT_S3 = FILE_PATH + File_name + type_of_activity + '.png' with fs.open(FILE_PATH_OUT_S3, 'wb') as file_out: plt.savefig(file_out) def outlier_detection(tickets, company_list, show_diagram=False): outlier_list = list() for company in company_list: total_amount_share = tickets[tickets['number_company']==int(company)].groupby('customer_id')['total_amount'].sum().reset_index() total_amount_share['CA'] = total_amount_share['total_amount'].sum() total_amount_share['share_total_amount'] = total_amount_share['total_amount']/total_amount_share['CA'] total_amount_share_index = total_amount_share.set_index('customer_id') df_circulaire = total_amount_share_index['total_amount'].sort_values(axis = 0, ascending = False) print('df circulaire : ', df_circulaire.head()) top = df_circulaire[:1] print('top : ', top) outlier_list.append(top.index[0]) rest = df_circulaire[1:] rest_sum = rest.sum() new_series = pd.concat([top, pd.Series([rest_sum], index=['Autre'])]) if show_diagram: plt.figure(figsize=(3, 3)) plt.pie(new_series, labels=new_series.index, autopct='%1.1f%%', startangle=140, pctdistance=0.5) plt.axis('equal') plt.title(f'Répartition des montants totaux pour la compagnie {company}') plt.show() return outlier_list def valid_customer_detection(products): products_valid = products[products['purchase_date']>="2021-05-01"] consumer_valid = products_valid['customer_id'].to_list() return consumer_valid def remove_elements(lst, elements_to_remove): return ''.join([x for x in lst if x not in elements_to_remove]) def keep_elements(lst, elements_to_remove): return ''.join([x for x in lst if x in elements_to_remove]) def compute_nb_clients(customer, type_of_activity): company_nb_clients = customer[customer["purchase_count"]>0].groupby("number_company")["customer_id"].count().reset_index() plt.bar(company_nb_clients["number_company"], company_nb_clients["customer_id"]/1000) plt.xlabel('Company') plt.ylabel("Number of clients (thousands)") plt.title(f"Number of clients for {type_of_activity}") plt.show() save_file_s3("nb_clients_", type_of_activity) def maximum_price_paid(customer, type_of_activity): company_max_price = customer.groupby("number_company")["max_price"].max().reset_index() plt.bar(company_max_price["number_company"], company_max_price["max_price"]) plt.xlabel('Company') plt.ylabel("Maximal price of a ticket Prix") plt.title(f"Maximal price of a ticket for {type_of_activity}") plt.show() save_file_s3("Maximal_price_", type_of_activity) def mailing_consent(customer, type_of_activity): mailing_consent = customer.groupby("number_company")["opt_in"].mean().reset_index() plt.bar(mailing_consent["number_company"], mailing_consent["opt_in"]) plt.xlabel('Company') plt.ylabel('Consent of mailing (%)') plt.title(f'Consent of mailing for {type_of_activity}') plt.show() save_file_s3("mailing_consent_", type_of_activity) def gender_bar(customer, type_of_activity): company_genders = customer.groupby("number_company")[["gender_male", "gender_female", "gender_other"]].mean().reset_index() plt.bar(company_genders["number_company"], company_genders["gender_male"], label = "Homme") plt.bar(company_genders["number_company"], company_genders["gender_female"], bottom = company_genders["gender_male"], label = "Femme") plt.bar(company_genders["number_company"], company_genders["gender_other"], bottom = company_genders["gender_male"] + company_genders["gender_female"], label = "Inconnu") plt.xlabel('Company') plt.ylabel("Gender") plt.title(f"Gender of Customer for {type_of_activity}") plt.legend() plt.xticks(company_genders["number_company"], ["{}".format(i) for i in company_genders["number_company"]]) plt.show() save_file_s3("gender_bar_", type_of_activity) def country_bar(customer, type_of_activity): company_country_fr = customer.groupby("number_company")["country_fr"].mean().reset_index() plt.bar(company_country_fr["number_company"], company_country_fr["country_fr"]) plt.xlabel('Company') plt.ylabel("Share of French Customer") plt.title(f"Share of French Customer for {type_of_activity}") plt.show() save_file_s3("country_bar_", type_of_activity) def lazy_customer_plot(campaigns_kpi, type_of_activity): company_lazy_customers = campaigns_kpi.groupby("number_company")["nb_campaigns_opened"].mean().reset_index() plt.bar(company_lazy_customers["number_company"], company_lazy_customers["nb_campaigns_opened"]) plt.xlabel('Company') plt.ylabel("Share of Customers who did not open mail") plt.title(f"Share of Customers who did not open mail for {type_of_activity}") plt.show() save_file_s3("lazy_customer_", type_of_activity) def campaigns_effectiveness(customer, type_of_activity): campaigns_effectiveness = customer.groupby("number_company")["opt_in"].mean().reset_index() plt.bar(campaigns_effectiveness["number_company"], campaigns_effectiveness["opt_in"]) plt.xlabel('Company') plt.ylabel("Number of Customers (thousands)") plt.title(f"Number of Customers of have bought or have received mails for {type_of_activity}") plt.legend() plt.show() save_file_s3("campaigns_effectiveness_", type_of_activity) def sale_dynamics(products, campaigns_brut, type_of_activity): purchase_min = products.groupby(['customer_id'])['purchase_date'].min().reset_index() purchase_min.rename(columns = {'purchase_date' : 'first_purchase_event'}, inplace = True) purchase_min['first_purchase_event'] = pd.to_datetime(purchase_min['first_purchase_event']) purchase_min['first_purchase_month'] = pd.to_datetime(purchase_min['first_purchase_event'].dt.strftime('%Y-%m')) first_mail_received = campaigns_brut.groupby('customer_id')['sent_at'].min().reset_index() first_mail_received.rename(columns = {'sent_at' : 'first_email_reception'}, inplace = True) first_mail_received['first_email_reception'] = pd.to_datetime(first_mail_received['first_email_reception']) first_mail_received['first_email_month'] = pd.to_datetime(first_mail_received['first_email_reception'].dt.strftime('%Y-%m')) known_customer = pd.merge(purchase_min[['customer_id', 'first_purchase_month']], first_mail_received[['customer_id', 'first_email_month']], on = 'customer_id', how = 'outer') known_customer['known_date'] = pd.to_datetime(known_customer[['first_email_month', 'first_purchase_month']].min(axis = 1), utc = True, format = 'ISO8601') purchases_count = pd.merge(products[['customer_id', 'purchase_id', 'purchase_date']].drop_duplicates(), known_customer[['customer_id', 'known_date']], on = ['customer_id'], how = 'inner') purchases_count['is_customer_known'] = purchases_count['purchase_date'] > purchases_count['known_date'] + pd.DateOffset(months=1) purchases_count['purchase_date_month'] = pd.to_datetime(purchases_count['purchase_date'].dt.strftime('%Y-%m')) purchases_count = purchases_count[purchases_count['customer_id'] != 1] nb_purchases_graph = purchases_count.groupby(['purchase_date_month', 'is_customer_known'])['purchase_id'].count().reset_index() nb_purchases_graph.rename(columns = {'purchase_id' : 'nb_purchases'}, inplace = True) nb_purchases_graph_2 = purchases_count.groupby(['purchase_date_month', 'is_customer_known'])['customer_id'].nunique().reset_index() nb_purchases_graph_2.rename(columns = {'customer_id' : 'nb_new_customer'}, inplace = True) purchases_graph = nb_purchases_graph purchases_graph_used = purchases_graph[purchases_graph["purchase_date_month"] >= datetime(2021,3,1)] purchases_graph_used_0 = purchases_graph_used[purchases_graph_used["is_customer_known"]==False] purchases_graph_used_1 = purchases_graph_used[purchases_graph_used["is_customer_known"]==True] plt.bar(purchases_graph_used_0["purchase_date_month"], purchases_graph_used_0["nb_purchases"], width=12, label = "Nouveau client") plt.bar(purchases_graph_used_0["purchase_date_month"], purchases_graph_used_1["nb_purchases"], bottom = purchases_graph_used_0["nb_purchases"], width=12, label = "Ancien client") plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b%y')) plt.xlabel('Month') plt.ylabel("Number of Sales") plt.title(f"Number of Sales for {type_of_activity}") plt.legend() plt.show() save_file_s3("sale_dynamics_", type_of_activity) def tickets_internet(tickets, type_of_activity): nb_tickets_internet = tickets.groupby("number_company")[["nb_tickets", "nb_tickets_internet"]].sum().reset_index() nb_tickets_internet["Share_ticket_internet"] = nb_tickets_internet["nb_tickets_internet"]*100 / nb_tickets_internet["nb_tickets"] plt.bar(nb_tickets_internet["number_company"], nb_tickets_internet["Share_ticket_internet"]) plt.xlabel('Company') plt.ylabel("Share of Tickets Bought Online") plt.title(f"Share of Tickets Bought Online for {type_of_activity}") plt.show() save_file_s3("tickets_internet_", type_of_activity) def box_plot_price_tickets(tickets, type_of_activity): price_tickets = tickets[(tickets['total_amount'] > 0)] sns.boxplot(data=price_tickets, y="total_amount", x="number_company", showfliers=False, showmeans=True) plt.title(f"Box plot of price tickets for {type_of_activity}") plt.show() save_file_s3("box_plot_price_tickets_", type_of_activity)