ajoubrel-ensae/BDC-team-1
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

generalization #7

Merged
arevelle-ensae merged 9 commits from generalization into main 2024-03-07 13:58:03 +01:00
Conversation 0 Commits 9 Files Changed 4 +2949 -1324
5 changed files with 2949 additions and 1324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit de342a2f77 - Show all commits

2
0_KPI_functions.py
View File

@ -107,7 +107,7 @@ def customerplus_kpi_function(customerplus_clean = None):
customerplus_clean["country_fr"] = customerplus_clean["country"].apply(lambda x : int(x=="fr") if pd.notna(x) else np.nan)
# Dummy if the customer has a structure id (tags)
customerplus_clean['has_tags'] = customerplus_clean['structure_id'].apply(lambda x: 1 if not pd.isna(x) else 0)
# customerplus_clean['has_tags'] = customerplus_clean['structure_id'].apply(lambda x: 1 if not pd.isna(x) else 0)
return customerplus_clean

2464
1_Descriptive_Statistics_Museum.ipynb
View File

File diff suppressed because one or more lines are too long

22
Exploration_billet_AJ.ipynb
View File

@ -96,9 +96,7 @@
{
"cell_type": "markdown",
"id": "ca2c8b6a-4965-422e-ba7c-66423a464fc1",
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"metadata": {},
"source": [
"## Base communes au types Musée"
]
@ -475,9 +473,7 @@
{
"cell_type": "markdown",
"id": "605cced5-052f-4a99-ac26-020c5d2ab633",
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"metadata": {},
"source": [
"## KPI sur tags"
]
@ -1877,14 +1873,6 @@
"nb_tickets_per_events"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6a38ff60-b608-43a0-9e18-4cd2bcbcea70",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "1ede9eaa-7f0a-4856-9349-b2747d6a4901",
@ -2878,9 +2866,11 @@
{
"cell_type": "markdown",
"id": "80ae4c42",
"metadata": {},
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"source": [
"## Exploration variables"
"## Supplier"
]
},
{

1325
Spectacle/Stat_desc.ipynb
View File

File diff suppressed because one or more lines are too long

460
code_base_train_test.ipynb Normal file
View File

@ -0,0 +1,460 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "bf34b03c-536f-4f93-93a5-e452552653aa",
"metadata": {},
"outputs": [
{
"name": "stdin",
"output_type": "stream",
"text": [
"Choisissez le type de compagnie : sport ? musique ? musee ? musique\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"File path : projet-bdc2324-team1/0_Input/Company_10/products_purchased_reduced.csv\n",
"Couverture Company 10 : 2016-03-07 - 2023-09-25\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/products_purchased_reduced.csv\n",
"Couverture Company 11 : 2015-06-26 - 2023-11-08\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/products_purchased_reduced.csv\n",
"Couverture Company 12 : 2016-06-14 - 2023-11-08\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/products_purchased_reduced.csv\n",
"Couverture Company 13 : 2010-07-31 - 2023-11-08\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/products_purchased_reduced.csv\n",
"Couverture Company 14 : 1901-01-01 - 2023-11-08\n",
"File path : projet-bdc2324-team1/0_Input/Company_10/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_10/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_10/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset test : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_10/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_10/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_10/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset train : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset test : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_11/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset train : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset test : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_12/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset train : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset test : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_13/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset train : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset test : SUCCESS\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/customerplus_cleaned.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/campaigns_information.csv\n",
"File path : projet-bdc2324-team1/0_Input/Company_14/products_purchased_reduced.csv\n",
"Data filtering : SUCCESS\n",
"KPIs construction : SUCCESS\n",
"Explanatory variable construction : SUCCESS\n",
"Explained variable construction : SUCCESS\n",
"Exportation dataset train : SUCCESS\n",
"FIN DE LA GENERATION DES DATASETS : SUCCESS\n"
]
}
],
"source": [
"# Business Data Challenge - Team 1\n",
"\n",
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"import s3fs\n",
"import re\n",
"import warnings\n",
"from datetime import date, timedelta, datetime\n",
"\n",
"# Create filesystem object\n",
"S3_ENDPOINT_URL = \"https://\" + os.environ[\"AWS_S3_ENDPOINT\"]\n",
"fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})\n",
"\n",
"\n",
"# Import KPI construction functions\n",
"exec(open('0_KPI_functions.py').read())\n",
"\n",
"# Ignore warning\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"\n",
"def display_covering_time(df, company, datecover):\n",
" \"\"\"\n",
" This function draws the time coverage of each company\n",
" \"\"\"\n",
" min_date = df['purchase_date'].min().strftime(\"%Y-%m-%d\")\n",
" max_date = df['purchase_date'].max().strftime(\"%Y-%m-%d\")\n",
" datecover[company] = [datetime.strptime(min_date, \"%Y-%m-%d\") + timedelta(days=x) for x in range((datetime.strptime(max_date, \"%Y-%m-%d\") - datetime.strptime(min_date, \"%Y-%m-%d\")).days)]\n",
" print(f'Couverture Company {company} : {min_date} - {max_date}')\n",
" return datecover\n",
"\n",
"\n",
"def compute_time_intersection(datecover):\n",
" \"\"\"\n",
" This function returns the time coverage for all companies\n",
" \"\"\"\n",
" timestamps_sets = [set(timestamps) for timestamps in datecover.values()]\n",
" intersection = set.intersection(*timestamps_sets)\n",
" intersection_list = list(intersection)\n",
" formated_dates = [dt.strftime(\"%Y-%m-%d\") for dt in intersection_list]\n",
" return sorted(formated_dates)\n",
"\n",
"\n",
"def df_coverage_modelization(sport, coverage_train = 0.7):\n",
" \"\"\"\n",
" This function returns start_date, end_of_features and final dates\n",
" that help to construct train and test datasets\n",
" \"\"\"\n",
" datecover = {}\n",
" for company in sport:\n",
" df_products_purchased_reduced = display_databases(company, file_name = \"products_purchased_reduced\",\n",
" datetime_col = ['purchase_date'])\n",
" datecover = display_covering_time(df_products_purchased_reduced, company, datecover)\n",
" #print(datecover.keys())\n",
" dt_coverage = compute_time_intersection(datecover)\n",
" start_date = dt_coverage[0]\n",
" end_of_features = dt_coverage[int(0.7 * len(dt_coverage))]\n",
" final_date = dt_coverage[-1]\n",
" return start_date, end_of_features, final_date\n",
" \n",
"\n",
"def dataset_construction(min_date, end_features_date, max_date, directory_path):\n",
" \n",
" # Import customerplus\n",
" df_customerplus_clean_0 = display_databases(directory_path, file_name = \"customerplus_cleaned\")\n",
" df_campaigns_information = display_databases(directory_path, file_name = \"campaigns_information\", datetime_col = ['opened_at', 'sent_at', 'campaign_sent_at'])\n",
" df_products_purchased_reduced = display_databases(directory_path, file_name = \"products_purchased_reduced\", datetime_col = ['purchase_date'])\n",
" \n",
" # Filtre de cohérence pour la mise en pratique de notre méthode\n",
" max_date = pd.to_datetime(max_date, utc = True, format = 'ISO8601') \n",
" end_features_date = pd.to_datetime(end_features_date, utc = True, format = 'ISO8601')\n",
" min_date = pd.to_datetime(min_date, utc = True, format = 'ISO8601')\n",
"\n",
" #Filtre de la base df_campaigns_information\n",
" df_campaigns_information = df_campaigns_information[(df_campaigns_information['sent_at'] <= end_features_date) & (df_campaigns_information['sent_at'] >= min_date)]\n",
" df_campaigns_information['opened_at'][df_campaigns_information['opened_at'] >= end_features_date] = np.datetime64('NaT')\n",
" \n",
" #Filtre de la base df_products_purchased_reduced\n",
" df_products_purchased_reduced = df_products_purchased_reduced[(df_products_purchased_reduced['purchase_date'] <= end_features_date) & (df_products_purchased_reduced['purchase_date'] >= min_date)]\n",
"\n",
" print(\"Data filtering : SUCCESS\")\n",
" \n",
" # Fusion de l'ensemble et creation des KPI\n",
"\n",
" # KPI sur les campagnes publicitaires\n",
" df_campaigns_kpi = campaigns_kpi_function(campaigns_information = df_campaigns_information) \n",
"\n",
" # KPI sur le comportement d'achat\n",
" df_tickets_kpi = tickets_kpi_function(tickets_information = df_products_purchased_reduced)\n",
"\n",
" # KPI sur les données socio-démographiques\n",
" df_customerplus_clean = customerplus_kpi_function(customerplus_clean = df_customerplus_clean_0)\n",
" \n",
" print(\"KPIs construction : SUCCESS\")\n",
" \n",
" # Fusion avec KPI liés au customer\n",
" df_customer = pd.merge(df_customerplus_clean, df_campaigns_kpi, on = 'customer_id', how = 'left')\n",
" \n",
" # Fill NaN values\n",
" df_customer[['nb_campaigns', 'nb_campaigns_opened']] = df_customer[['nb_campaigns', 'nb_campaigns_opened']].fillna(0)\n",
" \n",
" # Fusion avec KPI liés au comportement d'achat\n",
" df_customer_product = pd.merge(df_tickets_kpi, df_customer, on = 'customer_id', how = 'outer')\n",
" \n",
" # Fill NaN values\n",
" df_customer_product[['nb_tickets', 'nb_purchases', 'total_amount', 'nb_suppliers', 'vente_internet_max', 'nb_tickets_internet']] = df_customer_product[['nb_tickets', 'nb_purchases', 'total_amount', 'nb_suppliers', 'vente_internet_max', 'nb_tickets_internet']].fillna(0)\n",
"\n",
" print(\"Explanatory variable construction : SUCCESS\")\n",
"\n",
" # 2. Construction of the explained variable \n",
" df_products_purchased_to_predict = df_products_purchased_reduced[(df_products_purchased_reduced['purchase_date'] <= max_date) & (df_products_purchased_reduced['purchase_date'] > end_features_date)]\n",
"\n",
" # Indicatrice d'achat\n",
" df_products_purchased_to_predict['y_has_purchased'] = 1\n",
"\n",
" y = df_products_purchased_to_predict[['customer_id', 'y_has_purchased']].drop_duplicates()\n",
"\n",
" print(\"Explained variable construction : SUCCESS\")\n",
" \n",
" # 3. Merge between explained and explanatory variables\n",
" dataset = pd.merge(df_customer_product, y, on = ['customer_id'], how = 'left')\n",
"\n",
" # 0 if there is no purchase\n",
" dataset[['y_has_purchased']].fillna(0)\n",
"\n",
" # add id_company prefix to customer_id\n",
" dataset['customer_id'] = directory_path + '_' + dataset['customer_id'].astype('str')\n",
" \n",
" return dataset\n",
"\n",
"## Exportation\n",
"\n",
"companies = {'musee' : ['1', '2', '3', '4', '101'],\n",
" 'sport': ['5', '6', '7', '8', '9'],\n",
" 'musique' : ['10', '11', '12', '13', '14']}\n",
"\n",
"type_of_comp = input('Choisissez le type de compagnie : sport ? musique ? musee ?')\n",
"list_of_comp = companies[type_of_comp] \n",
"# Dossier d'exportation\n",
"BUCKET_OUT = f'projet-bdc2324-team1/Generalization/{type_of_comp}'\n",
"\n",
"# Create test dataset and train dataset for sport companies\n",
"\n",
"start_date, end_of_features, final_date = df_coverage_modelization(list_of_comp, coverage_train = 0.7)\n",
"\n",
"for company in list_of_comp:\n",
" dataset_test = dataset_construction(min_date = start_date, end_features_date = end_of_features,\n",
" max_date = final_date, directory_path = company) \n",
"\n",
" # Exportation\n",
" FILE_KEY_OUT_S3 = \"dataset_test\" + company + \".csv\"\n",
" FILE_PATH_OUT_S3 = BUCKET_OUT + \"/Test_set/\" + FILE_KEY_OUT_S3\n",
" \n",
" with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:\n",
" dataset_test.to_csv(file_out, index = False)\n",
" \n",
" print(\"Exportation dataset test : SUCCESS\")\n",
"\n",
"# Dataset train\n",
" dataset_train = dataset_construction(min_date = start_date, end_features_date = end_of_features,\n",
" max_date = final_date, directory_path = company)\n",
" # Export\n",
" FILE_KEY_OUT_S3 = \"dataset_train\" + company + \".csv\" \n",
" FILE_PATH_OUT_S3 = BUCKET_OUT + \"/Train_test/\" + FILE_KEY_OUT_S3\n",
" \n",
" with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:\n",
" dataset_train.to_csv(file_out, index = False)\n",
" \n",
" print(\"Exportation dataset train : SUCCESS\")\n",
"\n",
"\n",
"print(\"FIN DE LA GENERATION DES DATASETS : SUCCESS\")\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3721427e-5957-4556-b278-2e7ffca892f4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'projet-bdc2324-team1/Generalization/musique/Train_test/dataset_train14.csv'"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"FILE_PATH_OUT_S3"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "f8546992-f425-4d1e-ad75-ad26a8052a18",
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'projet' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mprojet\u001b[49m\u001b[38;5;241m-\u001b[39mbdc2324\u001b[38;5;241m-\u001b[39mteam1\u001b[38;5;241m/\u001b[39mGeneralization\u001b[38;5;241m/\u001b[39mmusique\u001b[38;5;241m/\u001b[39mTrain_test\n",
"\u001b[0;31mNameError\u001b[0m: name 'projet' is not defined"
]
}
],
"source": [
"projet-bdc2324-team1/Generalization/musique/Train_test"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "0dd34710-6da2-4438-9e1d-0ac092c1d28c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(343126, 41)"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset_train.shape"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "a3bfeeb6-2db0-4f1d-866c-8721343e97c5",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"customer_id 0.000000\n",
"nb_tickets 0.000000\n",
"nb_purchases 0.000000\n",
"total_amount 0.000000\n",
"nb_suppliers 0.000000\n",
"vente_internet_max 0.000000\n",
"purchase_date_min 0.858950\n",
"purchase_date_max 0.858950\n",
"time_between_purchase 0.858950\n",
"nb_tickets_internet 0.000000\n",
"street_id 0.000000\n",
"structure_id 0.869838\n",
"mcp_contact_id 0.276677\n",
"fidelity 0.000000\n",
"tenant_id 0.000000\n",
"is_partner 0.000000\n",
"deleted_at 1.000000\n",
"gender 0.000000\n",
"is_email_true 0.000000\n",
"opt_in 0.000000\n",
"last_buying_date 0.709626\n",
"max_price 0.709626\n",
"ticket_sum 0.000000\n",
"average_price 0.709626\n",
"average_purchase_delay 0.709731\n",
"average_price_basket 0.709731\n",
"average_ticket_basket 0.709731\n",
"total_price 0.000000\n",
"purchase_count 0.000000\n",
"first_buying_date 0.709626\n",
"country 0.152090\n",
"gender_label 0.000000\n",
"gender_female 0.000000\n",
"gender_male 0.000000\n",
"gender_other 0.000000\n",
"country_fr 0.152090\n",
"has_tags 0.000000\n",
"nb_campaigns 0.000000\n",
"nb_campaigns_opened 0.000000\n",
"time_to_open 0.848079\n",
"y_has_purchased 1.000000\n",
"dtype: float64"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
" dataset_train.isna().sum()/dataset_train.shape[0]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "75f9a672-641f-49a2-a8d6-7673845506f5",
"metadata": {},
"outputs": [],
"source": [
"#Creation de la variable dependante fictive: 1 si l'individu a effectué un achat au cours de la periode de train et 0 sinon\n",
"\n",
"dataset_train_modif=dataset_train\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c121c1e2-d8e4-4b93-a882-9385581b63c9",
"metadata": {},
"outputs": [],
"source": [
"dataset_train_modif[\""
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}