2194 lines
60 KiB
Plaintext
2194 lines
60 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "5bf5c226",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Business Data Challenge - Team 1"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "b1a5b9d3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import numpy as np\n",
|
||
"import os\n",
|
||
"import s3fs\n",
|
||
"import re\n",
|
||
"import warnings"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "ecfa2219",
|
||
"metadata": {},
|
||
"source": [
|
||
"Configuration de l'accès aux données"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "1a094277",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Create filesystem object\n",
|
||
"S3_ENDPOINT_URL = \"https://\" + os.environ[\"AWS_S3_ENDPOINT\"]\n",
|
||
"fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "30d77451-2df6-4c07-8b15-66e0e990ff03",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Import cleaning and merge functions\n",
|
||
"\n",
|
||
"exec(open('0_Cleaning_and_merge_functions.py').read())\n",
|
||
"\n",
|
||
"exec(open('0_KPI_functions.py').read())\n",
|
||
"\n",
|
||
"# Ignore warning\n",
|
||
"warnings.filterwarnings('ignore')\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "f1b44d3e-76bb-4860-b9db-a2840db7cf39",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def load_dataset_2(directory_path, file_name):\n",
|
||
" \"\"\"\n",
|
||
" This function loads csv file\n",
|
||
" \"\"\"\n",
|
||
" file_path = \"bdc2324-data\" + \"/\" + directory_path + \"/\" + directory_path + file_name + \".csv\"\n",
|
||
" with fs.open(file_path, mode=\"rb\") as file_in:\n",
|
||
" df = pd.read_csv(file_in, sep=\",\")\n",
|
||
"\n",
|
||
" # drop na :\n",
|
||
" #df = df.dropna(axis=1, thresh=len(df))\n",
|
||
" # if identifier in table : delete it\n",
|
||
" if 'identifier' in df.columns:\n",
|
||
" df = df.drop(columns = 'identifier')\n",
|
||
" return df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "31ab76f0-fbb1-46f6-b359-97228620c207",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def export_in_temporary(df, output_name):\n",
|
||
" print('Export of dataset :', output_name)\n",
|
||
" FILE_PATH_OUT_S3 = \"ajoubrel-ensae/Temporary\" + \"/\" + output_name + '.csv'\n",
|
||
" with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:\n",
|
||
" df.to_csv(file_out, index = False)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "ccf597b0-b459-4ea5-baf0-5ba8c90915e4",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Cleaning target area and tags"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"id": "fd88e294-e038-4cec-ad94-2bbbc10a4059",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def concatenate_names(names):\n",
|
||
" return ', '.join(names)\n",
|
||
"\n",
|
||
"def targets_KPI(df_target = None):\n",
|
||
" \n",
|
||
" df_target['target_name'] = df_target['target_name'].fillna('').str.lower()\n",
|
||
"\n",
|
||
" # Target name cotegory musees / \n",
|
||
" df_target['target_jeune'] = df_target['target_name'].str.contains('|'.join(['jeune', 'pass_culture', 'etudiant', '12-25 ans', 'student', 'jeunesse']), case=False).astype(int)\n",
|
||
" df_target['target_optin'] = df_target['target_name'].str.contains('|'.join(['optin' ,'opt-in']), case=False).astype(int)\n",
|
||
" df_target['target_optout'] = df_target['target_name'].str.contains('|'.join(['optout', 'unsubscribed']), case=False).astype(int)\n",
|
||
" df_target['target_scolaire'] = df_target['target_name'].str.contains('|'.join(['scolaire' , 'enseignant', 'chercheur', 'schulen', 'école']), case=False).astype(int)\n",
|
||
" df_target['target_entreprise'] = df_target['target_name'].str.contains('|'.join(['b2b', 'btob', 'cse']), case=False).astype(int)\n",
|
||
" df_target['target_famille'] = df_target['target_name'].str.contains('|'.join(['famille', 'enfants', 'family']), case=False).astype(int)\n",
|
||
" df_target['target_newsletter'] = df_target['target_name'].str.contains('|'.join(['nl', 'newsletter']), case=False).astype(int)\n",
|
||
" \n",
|
||
" # Target name category for sport compagnies\n",
|
||
" df_target['target_abonne'] = ((\n",
|
||
" df_target['target_name']\n",
|
||
" .str.contains('|'.join(['abo', 'adh']), case=False)\n",
|
||
" & ~df_target['target_name'].str.contains('|'.join(['hors abo', 'anciens abo']), case=False)\n",
|
||
" ).astype(int))\n",
|
||
" \n",
|
||
" df_target_categorie = df_target.groupby('customer_id')[['target_jeune', 'target_optin', 'target_optout', 'target_scolaire', 'target_entreprise', 'target_famille', 'target_newsletter', 'target_abonne']].max()\n",
|
||
" \n",
|
||
" target_agg = df_target.groupby('customer_id').agg(\n",
|
||
" nb_targets=('target_name', 'nunique') # Utilisation de tuples pour spécifier les noms de colonnes\n",
|
||
" # all_targets=('target_name', concatenate_names),\n",
|
||
" # all_target_types=('target_type_name', concatenate_names)\n",
|
||
" ).reset_index()\n",
|
||
"\n",
|
||
" target_agg['nb_targets'] = (target_agg['nb_targets'] - (target_agg['nb_targets'].mean())) / (target_agg['nb_targets'].std())\n",
|
||
" \n",
|
||
" target_agg = pd.merge(target_agg, df_target_categorie, how='left', on='customer_id')\n",
|
||
" \n",
|
||
" return target_agg"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 15,
|
||
"id": "1b124018-9637-463e-b512-15743ec9480b",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"File path : projet-bdc2324-team1/0_Input/Company_5/target_information.csv\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>customer_id</th>\n",
|
||
" <th>nb_targets</th>\n",
|
||
" <th>target_jeune</th>\n",
|
||
" <th>target_optin</th>\n",
|
||
" <th>target_optout</th>\n",
|
||
" <th>target_scolaire</th>\n",
|
||
" <th>target_entreprise</th>\n",
|
||
" <th>target_famille</th>\n",
|
||
" <th>target_newsletter</th>\n",
|
||
" <th>target_abonne</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>160516</td>\n",
|
||
" <td>6.938264</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>160517</td>\n",
|
||
" <td>10.357387</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>160518</td>\n",
|
||
" <td>5.228703</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>160519</td>\n",
|
||
" <td>6.083483</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>160520</td>\n",
|
||
" <td>2.949288</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>471205</th>\n",
|
||
" <td>6405875</td>\n",
|
||
" <td>-0.754762</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>471206</th>\n",
|
||
" <td>6405905</td>\n",
|
||
" <td>-0.469835</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>471207</th>\n",
|
||
" <td>6405909</td>\n",
|
||
" <td>-0.754762</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>471208</th>\n",
|
||
" <td>6405917</td>\n",
|
||
" <td>-0.754762</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>471209</th>\n",
|
||
" <td>6405963</td>\n",
|
||
" <td>-0.754762</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" <td>0</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>471210 rows × 10 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" customer_id nb_targets target_jeune target_optin target_optout \\\n",
|
||
"0 160516 6.938264 0 1 0 \n",
|
||
"1 160517 10.357387 0 1 1 \n",
|
||
"2 160518 5.228703 0 1 1 \n",
|
||
"3 160519 6.083483 0 1 1 \n",
|
||
"4 160520 2.949288 0 1 0 \n",
|
||
"... ... ... ... ... ... \n",
|
||
"471205 6405875 -0.754762 0 0 1 \n",
|
||
"471206 6405905 -0.469835 0 0 1 \n",
|
||
"471207 6405909 -0.754762 0 0 1 \n",
|
||
"471208 6405917 -0.754762 0 0 1 \n",
|
||
"471209 6405963 -0.754762 0 0 1 \n",
|
||
"\n",
|
||
" target_scolaire target_entreprise target_famille target_newsletter \\\n",
|
||
"0 0 1 0 0 \n",
|
||
"1 0 0 0 0 \n",
|
||
"2 0 0 0 0 \n",
|
||
"3 0 0 1 0 \n",
|
||
"4 0 0 0 0 \n",
|
||
"... ... ... ... ... \n",
|
||
"471205 0 0 0 0 \n",
|
||
"471206 0 0 0 0 \n",
|
||
"471207 0 0 0 0 \n",
|
||
"471208 0 0 0 0 \n",
|
||
"471209 0 0 0 0 \n",
|
||
"\n",
|
||
" target_abonne \n",
|
||
"0 1 \n",
|
||
"1 1 \n",
|
||
"2 1 \n",
|
||
"3 1 \n",
|
||
"4 1 \n",
|
||
"... ... \n",
|
||
"471205 0 \n",
|
||
"471206 0 \n",
|
||
"471207 0 \n",
|
||
"471208 0 \n",
|
||
"471209 0 \n",
|
||
"\n",
|
||
"[471210 rows x 10 columns]"
|
||
]
|
||
},
|
||
"execution_count": 15,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"targets_KPI(display_input_databases('5', file_name = \"target_information\"))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c75efea3-b5e8-4a7a-bed4-dd64ae9ff9f2",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"#export_in_temporary(target_agg, 'Target_kpi_concatenate')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "5d91263e-8a97-4cb1-8d94-db8ab0b77cdf",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"# Brouillon"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c5e864b1-adad-4267-b956-3f7ef371d677",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n",
|
||
"def display_covering_time(df, company, datecover):\n",
|
||
" \"\"\"\n",
|
||
" This function draws the time coverage of each company\n",
|
||
" \"\"\"\n",
|
||
" min_date = df['purchase_date'].min().strftime(\"%Y-%m-%d\")\n",
|
||
" max_date = df['purchase_date'].max().strftime(\"%Y-%m-%d\")\n",
|
||
" datecover[company] = [datetime.strptime(min_date, \"%Y-%m-%d\") + timedelta(days=x) for x in range((datetime.strptime(max_date, \"%Y-%m-%d\") - datetime.strptime(min_date, \"%Y-%m-%d\")).days)]\n",
|
||
" print(f'Couverture Company {company} : {min_date} - {max_date}')\n",
|
||
" return datecover\n",
|
||
"\n",
|
||
"\n",
|
||
"def compute_time_intersection(datecover):\n",
|
||
" \"\"\"\n",
|
||
" This function returns the time coverage for all companies\n",
|
||
" \"\"\"\n",
|
||
" timestamps_sets = [set(timestamps) for timestamps in datecover.values()]\n",
|
||
" intersection = set.intersection(*timestamps_sets)\n",
|
||
" intersection_list = list(intersection)\n",
|
||
" formated_dates = [dt.strftime(\"%Y-%m-%d\") for dt in intersection_list]\n",
|
||
" return sorted(formated_dates)\n",
|
||
"\n",
|
||
"\n",
|
||
"def df_coverage_modelization(sport, coverage_features = 0.7):\n",
|
||
" \"\"\"\n",
|
||
" This function returns start_date, end_of_features and final dates\n",
|
||
" that help to construct train and test datasets\n",
|
||
" \"\"\"\n",
|
||
" datecover = {}\n",
|
||
" for company in sport:\n",
|
||
" df_products_purchased_reduced = display_input_databases(company, file_name = \"products_purchased_reduced\",\n",
|
||
" datetime_col = ['purchase_date'])\n",
|
||
" datecover = display_covering_time(df_products_purchased_reduced, company, datecover)\n",
|
||
" #print(datecover.keys())\n",
|
||
" dt_coverage = compute_time_intersection(datecover)\n",
|
||
" start_date = dt_coverage[0]\n",
|
||
" end_of_features = dt_coverage[int(0.7 * len(dt_coverage))]\n",
|
||
" final_date = dt_coverage[-1]\n",
|
||
" return start_date, end_of_features, final_date\n",
|
||
" "
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "2435097a-95a5-43e1-84d0-7f6b701441ba",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"# Bases non communes : mise à plat"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "f8f988fb-5aab-4b57-80d1-e242f7e5b384",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"companies = {'musee' : ['1', '2', '3', '4'],\n",
|
||
" 'sport': ['5', '6', '7', '8', '9'],\n",
|
||
" 'musique' : ['10', '11', '12', '13', '14']}\n",
|
||
"\n",
|
||
"all_companies = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14']"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "35ac004f-c191-4f45-a4b1-6d993d9ec38c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"companies_databases = pd.DataFrame()\n",
|
||
"\n",
|
||
"for i in all_companies:\n",
|
||
" company_databases = pd.DataFrame({'company_number' : [i]})\n",
|
||
"\n",
|
||
" BUCKET = \"bdc2324-data/\"+i\n",
|
||
" for base in fs.ls(BUCKET):\n",
|
||
" match = re.search(r'\\/(\\d+)\\/(\\d+)([a-zA-Z_]+)\\.csv$', base)\n",
|
||
" if match:\n",
|
||
" nom_base = match.group(3)\n",
|
||
" company_databases[nom_base] = 1\n",
|
||
"\n",
|
||
" companies_databases = pd.concat([companies_databases, company_databases])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8986e477-e6c5-4d6c-83b2-2c90c134b599",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.set_option(\"display.max_columns\", None)\n",
|
||
"companies_databases\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8fecc3bb-4c03-4144-97c5-615224d9729e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.reset_option(\"display.max_columns\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "0294ce71-840e-458b-8ffa-cadabbc6da21",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Debut Travail 25/02"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "ca2c8b6a-4965-422e-ba7c-66423a464fc1",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## Base communes au types Musée"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "dbce1124-9a22-4502-a47a-fc3d0e2db70b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"companies['musee']"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "5080f66e-f779-410a-876d-b4fe2795e17e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"for i in companies['musique']:\n",
|
||
" BUCKET = \"bdc2324-data/\"+i\n",
|
||
" liste_base = []\n",
|
||
" for base in fs.ls(BUCKET):\n",
|
||
" match = re.search(r'\\/(\\d+)\\/(\\d+)([a-zA-Z_]+)\\.csv$', base)\n",
|
||
" if match:\n",
|
||
" nom_base = match.group(3)\n",
|
||
" liste_base.append(nom_base)\n",
|
||
" globals()['base_'+i] = liste_base\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "abd477e1-7479-4c88-a5aa-f987af3f5b79",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Trouver l'intersection entre les cinq listes\n",
|
||
"intersection = set(base_1).intersection(base_2, base_3, base_4, base_101)\n",
|
||
"\n",
|
||
"# Convertir le résultat en liste si nécessaire\n",
|
||
"intersection_liste = list(intersection)\n",
|
||
"\n",
|
||
"print(intersection_liste)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8d93888f-a511-4ee5-8bc3-d5173a7f119e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Trouver l'intersection entre les cinq listes\n",
|
||
"intersection = set(base_10).intersection(base_12, base_13, base_14, base_11)\n",
|
||
"\n",
|
||
"# Convertir le résultat en liste si nécessaire\n",
|
||
"intersection_liste = list(intersection)\n",
|
||
"\n",
|
||
"print(intersection_liste)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "10e89669-42bb-4652-a4bc-1a3d1caf4d1a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"len(intersection_liste)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7d058b21-a538-4f59-aefb-ef7966f73fdc",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_tags = load_dataset_2(\"1\", \"tags\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "aa441f99-733c-4675-8676-bed4682d3324",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_structure_tag_mappings = load_dataset_2(\"1\", 'structure_tag_mappings')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6767a750-14a4-4c05-903e-d2f07170825b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_customersplus = load_dataset_2(\"1\", \"customersplus\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "125e9145-a815-46fd-bdf4-07589508b259",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_customersplus.groupby('structure_id')['id'].count().reset_index().sort_values('id', ascending=False).head(20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c17a6976-792f-474d-bcff-c89396eddb3f",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_customersplus['structure_id'].isna().sum() / len(df1_customersplus['structure_id'])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "ecfc155a-cb42-46ec-8da5-33fdcd087355",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"len(df1_structure_tag_mappings)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "071410b8-950d-4fcc-b2b9-57415253c286",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_structure_tag_mappings.groupby('tag_id')['structure_id'].count().reset_index().sort_values('structure_id', ascending=False).head(20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "f48d27a9-14e4-4bb9-a60a-73e9438b58fc",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"?np.sort_values()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "14eaa0ea-02cc-430b-ab9b-38e6637810c3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def info_colonnes_dataframe(df):\n",
|
||
" # Créer une liste pour stocker les informations sur chaque colonne\n",
|
||
" infos_colonnes = []\n",
|
||
"\n",
|
||
" # Parcourir les colonnes du DataFrame\n",
|
||
" for nom_colonne, serie in df.items(): # Utiliser items() au lieu de iteritems()\n",
|
||
" # Calculer le taux de valeurs manquantes\n",
|
||
" taux_na = serie.isna().mean() * 100\n",
|
||
"\n",
|
||
" # Ajouter les informations à la liste\n",
|
||
" infos_colonnes.append({\n",
|
||
" 'Nom_colonne': nom_colonne,\n",
|
||
" 'Type_colonne': str(serie.dtype),\n",
|
||
" 'Taux_NA': taux_na\n",
|
||
" })\n",
|
||
"\n",
|
||
" # Créer une nouvelle DataFrame à partir de la liste d'informations\n",
|
||
" df_infos_colonnes = pd.DataFrame(infos_colonnes)\n",
|
||
"\n",
|
||
" return df_infos_colonnes"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6b031c32-d4c8-42a5-9a71-a7810f9bf8d8",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"info_colonnes_dataframe(df1_tags)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "e1a87f27-c4d4-4832-ac20-0c3c54aa4980",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"info_colonnes_dataframe(df1_structure_tag_mappings)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "fa5c65a8-2f74-4f3f-85fc-9ac91e0bb361",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.set_option('display.max_colwidth', None)\n",
|
||
"\n",
|
||
"print(df1_tags['name'])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a59bf932-5b54-4600-81f5-c55ac93ae510",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.set_option('display.max_rows', None)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a4ab298e-2cae-4865-9f00-4caff5f75ea1",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print(df1_tags['name'])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "76bffba1-5f7e-4308-9224-437ca66148f8",
|
||
"metadata": {},
|
||
"source": [
|
||
"## KPI sur target_type"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "f6daf22e-6583-4431-a467-660a1dd4e5a4",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d91d5895",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.set_option('display.max_colwidth', None)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "c58b17d3",
|
||
"metadata": {},
|
||
"source": [
|
||
"Raisonnement : on prends les target_type qui représente 90% des clients et on fait des catégories dessus."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6930bff5",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def print_main_target(tenant_id, nb_print = 40):\n",
|
||
" df_target = display_input_databases(tenant_id, \"target_information\")\n",
|
||
"\n",
|
||
" print('Nombre de ciblage : ', len(df_target))\n",
|
||
" nb_customers = df_target['customer_id'].nunique()\n",
|
||
" print('Nombre de client avec étiquette target : ', nb_customers) \n",
|
||
"\n",
|
||
" nb_custumers_per_target = df_target.groupby(\"target_name\")['customer_id'].count().reset_index().sort_values('customer_id', ascending=False)\n",
|
||
" nb_custumers_per_target['cumulative_customers'] = nb_custumers_per_target['customer_id'].cumsum()/len(df_target)\n",
|
||
" nb_custumers_per_target['customer_id'] = nb_custumers_per_target['customer_id']/nb_customers\n",
|
||
"\n",
|
||
" return nb_custumers_per_target.head(nb_print)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1e7ee1a0",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.set_option(\"max_colwidth\", None)\n",
|
||
"print_main_target('1', 60)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "19f3a2dd-ba3d-4dec-8e10-fed544ab6a53",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.reset_option('display.max_rows')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b57a28ac",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('2', 25)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9a65991f",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('3', 70)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "5f34b8bf",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('4', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "52b24d66-92ad-4421-a62b-5cba837f1893",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"pd.set_option('display.max_rows', None)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "40fe3676",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n",
|
||
"\n",
|
||
"print_main_target('5', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "820d3600-379b-4245-a977-f1f1fa1f1839",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('6', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "86f64a1b-763a-4e43-9601-a38c80392d47",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('7', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "fbf2ea42-515a-4cdf-a4c1-50f99c379ed9",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('8', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9684045c-4e25-4952-b099-a559baa5d749",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('9', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "cf8f7816-e7f3-4b7a-a987-8350a76eb140",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('10', 100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "76c818a5-3c52-4d97-ac81-b7f3f89092bd",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('11', 100)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "603b11e4-5d76-4699-a1b2-e795929edc04",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('12', 100)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "fa93aecd-d117-481e-8507-15e49937ce14",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('13', 100)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a115ebcf-4488-47f3-9d7e-75a1fca52f0f",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"print_main_target('14', 100)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "605cced5-052f-4a99-ac26-020c5d2ab633",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## KPI sur tags"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "916c3e2b-04d3-4877-b894-8f26f10d926e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"customersplus = load_dataset_2(\"4\", \"customersplus\")[['id', 'structure_id']]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "46847b24-15a4-464e-969f-f16ed3653f1f",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"structure_tag_mappings = load_dataset_2('4', \"structure_tag_mappings\")[['structure_id', 'tag_id']]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "3c10c69d-735f-453e-96bf-750697d965d0",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"customersplus[customersplus['structure_id'].notna()]['structure_id'].nunique()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9b0e77b3-5f16-4484-9564-7d3826583418",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"len(customersplus[customersplus['structure_id'].notna()])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "dfa27722-37f9-435a-8221-8aa6f9a4a107",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"structure_tag_mappings['structure_id'].nunique()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "2daabdd5-31e3-4918-9856-9bbc30cde602",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def tags_information(tenant_id, first_tags):\n",
|
||
"\n",
|
||
" customersplus = load_dataset_2(tenant_id, \"customersplus\")[['id', 'structure_id']]\n",
|
||
" customersplus.rename(columns = {'id' : 'customer_id'}, inplace = True)\n",
|
||
" tags = load_dataset_2(tenant_id, \"tags\")[['id', 'name']]\n",
|
||
" tags.rename(columns = {'id' : 'tag_id', 'name' : 'tag_name'}, inplace = True)\n",
|
||
" structure_tag_mappings = load_dataset_2(tenant_id, \"structure_tag_mappings\")[['structure_id', 'tag_id']]\n",
|
||
" \n",
|
||
" customer_tags = pd.merge(customersplus, structure_tag_mappings, on = 'structure_id', how = 'left')\n",
|
||
" customer_tags = pd.merge(customer_tags, tags, on = 'tag_id', how = 'inner')\n",
|
||
" \n",
|
||
" nb_customers_with_tag = customer_tags['customer_id'].nunique()\n",
|
||
" \n",
|
||
" print('Nombre de client avec tag : ', nb_customers_with_tag)\n",
|
||
" print('Proportion de clients avec tags : ', nb_customers_with_tag/len(customersplus))\n",
|
||
" print('Moyenne de tags par client : ', len(customer_tags)/nb_customers_with_tag)\n",
|
||
" \n",
|
||
" info = customer_tags.groupby(['tag_id', 'tag_name'])['customer_id'].count().reset_index().sort_values('customer_id', ascending = False).head(first_tags)\n",
|
||
"\n",
|
||
" return info"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "0b9f5f71-a927-4cc8-bb0c-9538e28d3553",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"tags_information(\"1\", 20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "bd5bef41-1774-4601-86b5-b7c1aea8f1d2",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"tags_information(\"2\", 20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7c2dc3e6-1418-44db-a8c0-4a9d59ec5232",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"load_dataset_2(\"2\", \"tags\")[['id', 'name']]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c7b2c670-7122-4f67-b1aa-8c80a10f16d8",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"tags_information(\"3\", 20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "76639995-252d-4a58-83d8-c0c00900c3a9",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"tags_information(\"4\", 20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "07e91791-d4d4-42b1-ac18-22d3b0b9f7bd",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"tags_information(\"101\", 20)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "87d131cd-ead0-4ef4-a8ee-b09022d08ffa",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## KPI product"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "26582be9-cfd1-48ea-a0a7-31101fdeb9d1",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"tenant_id = \"1\"\n",
|
||
"\n",
|
||
"df_product = display_databases(tenant_id, file_name = \"products_purchased_reduced\", datetime_col = ['purchase_date'])\n",
|
||
"\n",
|
||
"df_product.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "533bf499-dd56-4d29-b261-ca1e4928c9c7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"nb_tickets_per_events = df_product.groupby(['name_event_types', 'name_events'])['ticket_id'].count().reset_index().sort_values('ticket_id', ascending = False)\n",
|
||
"nb_tickets_per_events['prop_tickets'] = round(nb_tickets_per_events['ticket_id']/len(df_product), 3)\n",
|
||
"nb_tickets_per_events"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "1ede9eaa-7f0a-4856-9349-b2747d6a4901",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"# Fin travail 25/02"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "c437eaec",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"# Exemple sur Company 1"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "a1c1fc39",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Chargement données"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "66f8c17b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"BUCKET = \"bdc2324-data/1\"\n",
|
||
"liste_database = fs.ls(BUCKET)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c08e6798",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"liste_database_select = ['suppliers', 'ticket', 'purchase', 'consumption', 'type_ofs']\n",
|
||
"\n",
|
||
"# Filtrer la liste pour les éléments contenant au moins un élément de la liste à tester\n",
|
||
"liste_database_filtered = [element for element in liste_database if any(element_part in element for element_part in liste_database_select)]\n",
|
||
"\n",
|
||
"# Afficher le résultat\n",
|
||
"print(liste_database_filtered)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "675f518d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# loop to create dataframes from liste\n",
|
||
"files_path = liste_database\n",
|
||
"\n",
|
||
"client_number = files_path[0].split(\"/\")[1]\n",
|
||
"df_prefix = \"df\" + str(client_number) + \"_\"\n",
|
||
"\n",
|
||
"for i in range(len(files_path)) :\n",
|
||
" current_path = files_path[i]\n",
|
||
" with fs.open(current_path, mode=\"rb\") as file_in:\n",
|
||
" df = pd.read_csv(file_in)\n",
|
||
" # the pattern of the name is df1xxx\n",
|
||
" nom_dataframe = df_prefix + re.search(r'\\/(\\d+)\\/(\\d+)([a-zA-Z_]+)\\.csv$', current_path).group(3)\n",
|
||
" globals()[nom_dataframe] = df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "e855f403",
|
||
"metadata": {},
|
||
"source": [
|
||
"## customersplus.csv"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "91a8f8c4",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"a = pd.DataFrame(df1_customersplus.info())"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "2fda171d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def info_colonnes_dataframe(df):\n",
|
||
" # Créer une liste pour stocker les informations sur chaque colonne\n",
|
||
" infos_colonnes = []\n",
|
||
"\n",
|
||
" # Parcourir les colonnes du DataFrame\n",
|
||
" for nom_colonne, serie in df.items(): # Utiliser items() au lieu de iteritems()\n",
|
||
" # Calculer le taux de valeurs manquantes\n",
|
||
" taux_na = serie.isna().mean() * 100\n",
|
||
"\n",
|
||
" # Ajouter les informations à la liste\n",
|
||
" infos_colonnes.append({\n",
|
||
" 'Nom_colonne': nom_colonne,\n",
|
||
" 'Type_colonne': str(serie.dtype),\n",
|
||
" 'Taux_NA': taux_na\n",
|
||
" })\n",
|
||
"\n",
|
||
" # Créer une nouvelle DataFrame à partir de la liste d'informations\n",
|
||
" df_infos_colonnes = pd.DataFrame(infos_colonnes)\n",
|
||
"\n",
|
||
" return df_infos_colonnes"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "205eeeab",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def cleaning_date(df, column_name):\n",
|
||
" \"\"\"\n",
|
||
" Nettoie la colonne spécifiée du DataFrame en convertissant les valeurs en datetime avec le format ISO8601.\n",
|
||
"\n",
|
||
" Parameters:\n",
|
||
" - df: DataFrame\n",
|
||
" Le DataFrame contenant la colonne à nettoyer.\n",
|
||
" - column_name: str\n",
|
||
" Le nom de la colonne à nettoyer.\n",
|
||
"\n",
|
||
" Returns:\n",
|
||
" - DataFrame\n",
|
||
" Le DataFrame modifié avec la colonne nettoyée.\n",
|
||
" \"\"\"\n",
|
||
" df[column_name] = pd.to_datetime(df[column_name], utc = True, format = 'ISO8601')\n",
|
||
" return df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "634282c5",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"a = info_colonnes_dataframe(df1_customersplus)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "0e8d4133",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"a"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1268ad5a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"a = pd.DataFrame(df1_customersplus.isna().sum()/len(df1_customersplus)*100)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "bd41dc80",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Selection des variables\n",
|
||
"df1_customersplus_clean = df1_customersplus.copy()\n",
|
||
"\n",
|
||
"cleaning_date(df1_customersplus_clean, 'first_buying_date')\n",
|
||
"cleaning_date(df1_customersplus_clean, 'last_visiting_date')\n",
|
||
"\n",
|
||
"df1_customersplus_clean.drop(['lastname', 'firstname', 'email', 'civility', 'note', 'created_at', 'updated_at', 'deleted_at', 'extra', 'reference', 'extra_field', 'identifier', 'need_reload', 'preferred_category', 'preferred_supplier', 'preferred_formula', 'zipcode', 'last_visiting_date'], axis = 1, inplace=True)\n",
|
||
"df1_customersplus_clean.rename(columns = {'id' : 'customer_id'}, inplace = True)\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "64d0f76b",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## tickets.csv"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7e683711",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_tickets"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "e7b9a52e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_tickets.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "568280e8",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_tickets.isna().sum()/len(df1_tickets)*100"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "29ecec90",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Selection des variables\n",
|
||
"df1_tickets_clean = df1_tickets.drop(['lastname', 'firstname', 'email', 'created_at', 'updated_at', 'extra', 'reference', 'extra_field', 'identifier', 'need_reload', 'preferred_category', 'preferred_supplier', 'preferred_formula', 'zipcode'], axis = 1, inplace=True)\n",
|
||
"df1_tickets_clean.rename(columns = {'id' : 'customer_id'}, inplace = True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "22bb5de4",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## suppliers.csv"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6a9a91f4",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_suppliers"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "bab4758a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_suppliers.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b5fff251",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_suppliers.isna().sum()/len(df1_suppliers)*100"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "8b09e2a3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Selection des variables\n",
|
||
"df1_suppliers_clean = df1_suppliers[['id', 'name']]\n",
|
||
"df1_suppliers_clean.rename(columns = {'name' : 'supplier_name'}, inplace = True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "ecee7cdc",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_suppliers_clean"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "c8e6e69b",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## type_ofs.csv"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1a6cff1f",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_type_ofs"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "93630b41",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_type_ofs.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "4f94481a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Selection des variables\n",
|
||
"df1_type_ofs_clean = df1_type_ofs[['id', 'name', 'children']]\n",
|
||
"df1_type_ofs_clean.rename(columns = {'name' : 'type_of_ticket_name'}, inplace = True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "1b2811e2",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## purchases.csv"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "2455d2e1",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_purchases"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "5f9a159d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_purchases.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "db201bf7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Nettoyage purchase_date\n",
|
||
"df1_purchases['purchase_date'] = pd.to_datetime(df1_purchases['purchase_date'], utc = True)\n",
|
||
"df1_purchases['purchase_date'] = pd.to_datetime(df1_purchases['purchase_date'], format = 'ISO8601')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "bd436fca",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_purchases.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "83435862",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Selection des variables\n",
|
||
"df1_purchases_clean = df1_purchases[['id', 'purchase_date', 'customer_id']]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f210e730",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## Fusion de l'ensemble des données billétiques"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1f8b3aa7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Fusion avec fournisseurs\n",
|
||
"df1_ticket_information = pd.merge(df1_tickets_clean, df1_suppliers_clean, left_on = 'supplier_id', right_on = 'id', how = 'inner')\n",
|
||
"df1_ticket_information.drop(['supplier_id', 'id'], axis = 1, inplace=True)\n",
|
||
"\n",
|
||
"# Fusion avec type de tickets\n",
|
||
"df1_ticket_information = pd.merge(df1_ticket_information, df1_type_ofs_clean, left_on = 'type_of', right_on = 'id', how = 'inner')\n",
|
||
"df1_ticket_information.drop(['type_of', 'id'], axis = 1, inplace=True)\n",
|
||
"\n",
|
||
"# Fusion avec achats\n",
|
||
"df1_ticket_information = pd.merge(df1_ticket_information, df1_purchases_clean, left_on = 'purchase_id', right_on = 'id', how = 'inner')\n",
|
||
"df1_ticket_information.drop(['purchase_id', 'id'], axis = 1, inplace=True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "83a4d021",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_ticket_information"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "56e6ebd1",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"# Utilisation de fonctions"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "88fcde4b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Créer un DataFrame exemple\n",
|
||
"df_not_clean = df1_campaign_stats[['opened_at']].head(20)\n",
|
||
"\n",
|
||
"# Appliquer la fonction pour nettoyer la colonne 'purchase_date' de manière vectorisée\n",
|
||
"df_clean = cleaning_date(df_not_clean, 'opened_at')\n",
|
||
"df_clean.rename(columns = {'opened_at' : 'opened_at_clean'}, inplace = True)\n",
|
||
"\n",
|
||
"test = pd.concat([df1_campaign_stats[['opened_at']].head(20), df_clean], axis=1)\n",
|
||
"\n",
|
||
"test.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "818f69db",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Nettoyage, selection et fusion"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c9654eda",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_ticket_information"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7f2b620c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_ticket_information.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "637bdb72",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Customer information"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "14c52894",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Target area"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d83abfbf",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Target.csv cleaning\n",
|
||
"df1_targets_clean = df1_targets[[\"id\", \"target_type_id\", \"name\"]]\n",
|
||
"df1_targets_clean.rename(columns = {'id' : 'target_id' , 'name' : 'target_name'}, inplace = True)\n",
|
||
"\n",
|
||
"# target_type cleaning\n",
|
||
"df1_target_types_clean = df1_target_types[[\"id\",\"is_import\",\"name\"]].add_prefix(\"target_type_\")\n",
|
||
"\n",
|
||
"#customer_target_mappings cleaning\n",
|
||
"df1_customer_target_mappings_clean = df1_customer_target_mappings[[\"id\", \"customer_id\", \"target_id\"]]\n",
|
||
"\n",
|
||
"# Merge target et target_type\n",
|
||
"df1_targets_full = pd.merge(df1_targets_clean, df1_target_types_clean, left_on='target_type_id', right_on='target_type_id', how='inner')\n",
|
||
"df1_targets_full.drop(['target_type_id'], axis = 1, inplace=True)\n",
|
||
"\n",
|
||
"# Merge\n",
|
||
"df1_targets_full = pd.merge(df1_customer_target_mappings_clean, df1_targets_full, left_on='target_id', right_on='target_id', how='inner')\n",
|
||
"df1_targets_full.drop(['target_id'], axis = 1, inplace=True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "90d71b2c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_targets_test = df1_targets_full[['id', 'customer_id']].groupby(['customer_id']).count()\n",
|
||
"len(df1_targets_test[df1_targets_test['id'] > 1]) / len(df1_targets_test)\n",
|
||
"\n",
|
||
"# 99,6% des 151 000 client visés sont catégorisés plusieurs fois et en moyenne 5 fois... \n",
|
||
"df1_targets_test.mean()\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "2301de1e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_targets_full.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "75fbc2f7",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Catégorisation des target_name\n",
|
||
"import pandas as pd\n",
|
||
"import nltk\n",
|
||
"from nltk.tokenize import word_tokenize\n",
|
||
"from nltk.corpus import stopwords\n",
|
||
"from nltk.stem import WordNetLemmatizer\n",
|
||
"from nltk.probability import FreqDist\n",
|
||
"\n",
|
||
"# Téléchargement des ressources nécessaires\n",
|
||
"nltk.download('punkt')\n",
|
||
"nltk.download('stopwords')\n",
|
||
"nltk.download('wordnet')\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "55cddf92",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Définition des fonctions de tokenisation, suppression des mots vides et lemmatisation\n",
|
||
"def preprocess_text(texte):\n",
|
||
" # Concaténation des éléments de la liste en une seule chaîne de caractères\n",
|
||
" texte_concat = ' '.join(texte)\n",
|
||
" \n",
|
||
" # Tokenisation des mots\n",
|
||
" tokens = word_tokenize(texte_concat.lower())\n",
|
||
" \n",
|
||
" # Suppression des mots vides (stopwords)\n",
|
||
" stop_words = set(stopwords.words('french'))\n",
|
||
" filtered_tokens = [word for word in tokens if word not in stop_words]\n",
|
||
" \n",
|
||
" # Lemmatisation des mots\n",
|
||
" lemmatizer = WordNetLemmatizer()\n",
|
||
" lemmatized_tokens = [lemmatizer.lemmatize(word) for word in filtered_tokens]\n",
|
||
" \n",
|
||
" return lemmatized_tokens\n",
|
||
"\n",
|
||
"\n",
|
||
"# Appliquer le prétraitement à la colonne de texte\n",
|
||
"df1_targets_full['target_name_tokened'] = df1_targets_full['target_name'].apply(preprocess_text)\n",
|
||
"\n",
|
||
"# Concaténer les listes de mots pour obtenir une liste de tous les mots dans le corpus\n",
|
||
"all_words = [word for tokens in df1_targets_full['target_name_tokened'] for word in tokens]\n",
|
||
"\n",
|
||
"# Calculer la fréquence des mots\n",
|
||
"freq_dist = FreqDist(all_words)\n",
|
||
"\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7fd98a85",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Affichage des mots les plus fréquents\n",
|
||
"print(\"Mots les plus fréquents:\")\n",
|
||
"for mot, freq in freq_dist.most_common(15):\n",
|
||
" print(f\"{mot}: {freq}\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "cf94bb1d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import nltk\n",
|
||
"from nltk.tokenize import word_tokenize\n",
|
||
"from nltk.corpus import stopwords\n",
|
||
"from nltk.stem import WordNetLemmatizer\n",
|
||
"\n",
|
||
"# Téléchargement des ressources nécessaires\n",
|
||
"nltk.download('punkt')\n",
|
||
"nltk.download('stopwords')\n",
|
||
"nltk.download('wordnet')\n",
|
||
"\n",
|
||
"# Création de la DataFrame d'exemple\n",
|
||
"data = {'texte': [\"Le chat noir mange une souris.\", \"Le chien blanc aboie.\"]}\n",
|
||
"df = pd.DataFrame(data)\n",
|
||
"\n",
|
||
"# Fonction pour prétraiter le texte\n",
|
||
"def preprocess_text(texte):\n",
|
||
" # Concaténation des éléments de la liste en une seule chaîne de caractères\n",
|
||
" texte_concat = ' '.join(texte)\n",
|
||
" \n",
|
||
" # Tokenisation des mots\n",
|
||
" tokens = word_tokenize(texte_concat.lower())\n",
|
||
" \n",
|
||
" # Suppression des mots vides (stopwords)\n",
|
||
" stop_words = set(stopwords.words('french'))\n",
|
||
" filtered_tokens = [word for word in tokens if word not in stop_words]\n",
|
||
" \n",
|
||
" # Lemmatisation des mots\n",
|
||
" lemmatizer = WordNetLemmatizer()\n",
|
||
" lemmatized_tokens = [lemmatizer.lemmatize(word) for word in filtered_tokens]\n",
|
||
" \n",
|
||
" return lemmatized_tokens\n",
|
||
"\n",
|
||
"# Appliquer la fonction de prétraitement à la colonne de texte\n",
|
||
"df['texte_preprocessed'] = df['texte'].apply(preprocess_text)\n",
|
||
"\n",
|
||
"# Afficher le résultat\n",
|
||
"print(df)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "711d3884",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## Campaign area"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c25b5295",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# campaign_stats cleaning \n",
|
||
"df1_campaign_stats_clean = df1_campaign_stats[[\"id\", \"campaign_id\", \"customer_id\", \"opened_at\", \"sent_at\", \"delivered_at\"]]\n",
|
||
"cleaning_date(df1_campaign_stats_clean, 'opened_at')\n",
|
||
"cleaning_date(df1_campaign_stats_clean, 'sent_at')\n",
|
||
"cleaning_date(df1_campaign_stats_clean, 'delivered_at')\n",
|
||
"\n",
|
||
"# campaigns cleaning\n",
|
||
"df1_campaigns_clean = df1_campaigns[[\"id\", \"name\", \"service_id\", \"sent_at\"]].add_prefix(\"campaign_\")\n",
|
||
"cleaning_date(df1_campaigns_clean, 'campaign_sent_at')\n",
|
||
"\n",
|
||
"# Merge \n",
|
||
"df1_campaigns_full = pd.merge(df1_campaign_stats_clean, df1_campaigns_clean, on = \"campaign_id\", how = \"left\")\n",
|
||
"df1_campaigns_full.drop(['campaign_id'], axis = 1, inplace=True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "2a3de6a5",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_campaigns_full.info()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "3fc1f446",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_campaigns_information"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "20e69ee3",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## Link area"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d9cbdbce",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_campaigns"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c07459f0",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_link_stats"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "80ae4c42",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true
|
||
},
|
||
"source": [
|
||
"## Supplier"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b50b8f95",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Fonction d'exploration pour suppliers.csv = label itr et commission inconnues\n",
|
||
"def suppliers_exploration(suppliers = None) : \n",
|
||
" \n",
|
||
" # Taux de NaN pour ces colonnes\n",
|
||
" label_na = suppliers['label'].isna().sum()/len(suppliers)*100\n",
|
||
" itr_na = suppliers['itr'].isna().sum()/len(suppliers)*100\n",
|
||
" commission_na = suppliers['commission'].isna().sum()/len(suppliers)*100\n",
|
||
"\n",
|
||
" suppliers_desc = pd.DataFrame({'nb_suppliers' : [suppliers['name'].nunique()],\n",
|
||
" 'label_na' : [label_na],\n",
|
||
" 'itr_na' : [itr_na],\n",
|
||
" 'commission_na' : [commission_na]})\n",
|
||
"\n",
|
||
" return suppliers_desc"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7e292935",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_suppliers_desc = suppliers_exploration(suppliers = df1_suppliers)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "05b6f2b0",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df1_suppliers_desc"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c9324d80",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"BUCKET = \"bdc2324-data\"\n",
|
||
"liste_folders = fs.ls(BUCKET)\n",
|
||
"\n",
|
||
"liste_files = []\n",
|
||
"for company_folder in liste_folders : \n",
|
||
" liste_files.extend(fs.ls(company_folder))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "10304058",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"liste_database_select = ['suppliers']\n",
|
||
"\n",
|
||
"# Filtrer la liste pour les éléments contenant au moins un élément de la liste à tester\n",
|
||
"liste_suppliers = [element for element in liste_files if any(element_part in element for element_part in liste_database_select)]\n",
|
||
"\n",
|
||
"# Afficher le résultat\n",
|
||
"print(liste_suppliers)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "ffa423e5",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# loop to create dataframes from file 2\n",
|
||
"def database_loading(database_name = None):\n",
|
||
" files_path = database_name\n",
|
||
" \n",
|
||
" client_number = files_path.split(\"/\")[1]\n",
|
||
" df_prefix = \"df\" + str(client_number) + \"_\"\n",
|
||
" \n",
|
||
" current_path = files_path\n",
|
||
" with fs.open(current_path, mode=\"rb\") as file_in:\n",
|
||
" df = pd.read_csv(file_in)\n",
|
||
"\n",
|
||
" return df, client_number"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "70bdc88d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "6a0f567d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df_all = pd.DataFrame()\n",
|
||
"\n",
|
||
"for link in liste_suppliers:\n",
|
||
" \n",
|
||
" df_supplier, tenant_id = database_loading(link)\n",
|
||
" \n",
|
||
" df_supplier['tenant_id'] = int(tenant_id)\n",
|
||
"\n",
|
||
" df_all = pd.concat([df_all, df_supplier], axis = 0)\n",
|
||
" "
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "1522d8cd",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# df_all[df_all['tenant_id'] == 101]['name'].unique()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b0e42a61",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"liste_mots = ['en ligne', 'internet', 'web', 'net', 'vad', 'online'] \n",
|
||
"# vad = vente à distance\n",
|
||
"df_all['name'] = df_all['name'].fillna('')\n",
|
||
"\n",
|
||
"df_all['canal_vente_internet'] = df_all['name'].str.contains('|'.join(liste_mots), case=False).astype(int)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "d299ae91",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df_all.groupby('tenant_id')['canal_vente_internet'].max()"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.6"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|