{ "cells": [ { "cell_type": "markdown", "id": "5bf5c226", "metadata": {}, "source": [ "# Business Data Challenge - Team 1" ] }, { "cell_type": "code", "execution_count": 1, "id": "b1a5b9d3", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import os\n", "import s3fs\n", "import re\n", "import warnings" ] }, { "cell_type": "markdown", "id": "ecfa2219", "metadata": {}, "source": [ "Configuration de l'accès aux données" ] }, { "cell_type": "code", "execution_count": 2, "id": "1a094277", "metadata": {}, "outputs": [], "source": [ "# Create filesystem object\n", "S3_ENDPOINT_URL = \"https://\" + os.environ[\"AWS_S3_ENDPOINT\"]\n", "fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})" ] }, { "cell_type": "code", "execution_count": 3, "id": "30d77451-2df6-4c07-8b15-66e0e990ff03", "metadata": {}, "outputs": [], "source": [ "# Import cleaning and merge functions\n", "\n", "exec(open('0_Cleaning_and_merge_functions.py').read())\n", "\n", "exec(open('0_KPI_functions.py').read())\n", "\n", "# Ignore warning\n", "warnings.filterwarnings('ignore')\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "f1b44d3e-76bb-4860-b9db-a2840db7cf39", "metadata": {}, "outputs": [], "source": [ "def load_dataset_2(directory_path, file_name):\n", " \"\"\"\n", " This function loads csv file\n", " \"\"\"\n", " file_path = \"bdc2324-data\" + \"/\" + directory_path + \"/\" + directory_path + file_name + \".csv\"\n", " with fs.open(file_path, mode=\"rb\") as file_in:\n", " df = pd.read_csv(file_in, sep=\",\")\n", "\n", " # drop na :\n", " #df = df.dropna(axis=1, thresh=len(df))\n", " # if identifier in table : delete it\n", " if 'identifier' in df.columns:\n", " df = df.drop(columns = 'identifier')\n", " return df" ] }, { "cell_type": "code", "execution_count": 5, "id": "31ab76f0-fbb1-46f6-b359-97228620c207", "metadata": {}, "outputs": [], "source": [ "def export_in_temporary(df, output_name):\n", " print('Export of dataset :', output_name)\n", " FILE_PATH_OUT_S3 = \"ajoubrel-ensae/Temporary\" + \"/\" + output_name + '.csv'\n", " with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:\n", " df.to_csv(file_out, index = False)" ] }, { "cell_type": "markdown", "id": "ccf597b0-b459-4ea5-baf0-5ba8c90915e4", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# Cleaning target area and tags" ] }, { "cell_type": "code", "execution_count": 14, "id": "fd88e294-e038-4cec-ad94-2bbbc10a4059", "metadata": {}, "outputs": [], "source": [ "def concatenate_names(names):\n", " return ', '.join(names)\n", "\n", "def targets_KPI(df_target = None):\n", " \n", " df_target['target_name'] = df_target['target_name'].fillna('').str.lower()\n", "\n", " # Target name cotegory musees / \n", " df_target['target_jeune'] = df_target['target_name'].str.contains('|'.join(['jeune', 'pass_culture', 'etudiant', '12-25 ans', 'student', 'jeunesse']), case=False).astype(int)\n", " df_target['target_optin'] = df_target['target_name'].str.contains('|'.join(['optin' ,'opt-in']), case=False).astype(int)\n", " df_target['target_optout'] = df_target['target_name'].str.contains('|'.join(['optout', 'unsubscribed']), case=False).astype(int)\n", " df_target['target_scolaire'] = df_target['target_name'].str.contains('|'.join(['scolaire' , 'enseignant', 'chercheur', 'schulen', 'école']), case=False).astype(int)\n", " df_target['target_entreprise'] = df_target['target_name'].str.contains('|'.join(['b2b', 'btob', 'cse']), case=False).astype(int)\n", " df_target['target_famille'] = df_target['target_name'].str.contains('|'.join(['famille', 'enfants', 'family']), case=False).astype(int)\n", " df_target['target_newsletter'] = df_target['target_name'].str.contains('|'.join(['nl', 'newsletter']), case=False).astype(int)\n", " \n", " # Target name category for sport compagnies\n", " df_target['target_abonne'] = ((\n", " df_target['target_name']\n", " .str.contains('|'.join(['abo', 'adh']), case=False)\n", " & ~df_target['target_name'].str.contains('|'.join(['hors abo', 'anciens abo']), case=False)\n", " ).astype(int))\n", " \n", " df_target_categorie = df_target.groupby('customer_id')[['target_jeune', 'target_optin', 'target_optout', 'target_scolaire', 'target_entreprise', 'target_famille', 'target_newsletter', 'target_abonne']].max()\n", " \n", " target_agg = df_target.groupby('customer_id').agg(\n", " nb_targets=('target_name', 'nunique') # Utilisation de tuples pour spécifier les noms de colonnes\n", " # all_targets=('target_name', concatenate_names),\n", " # all_target_types=('target_type_name', concatenate_names)\n", " ).reset_index()\n", "\n", " target_agg['nb_targets'] = (target_agg['nb_targets'] - (target_agg['nb_targets'].mean())) / (target_agg['nb_targets'].std())\n", " \n", " target_agg = pd.merge(target_agg, df_target_categorie, how='left', on='customer_id')\n", " \n", " return target_agg" ] }, { "cell_type": "code", "execution_count": 15, "id": "1b124018-9637-463e-b512-15743ec9480b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "File path : projet-bdc2324-team1/0_Input/Company_5/target_information.csv\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
customer_idnb_targetstarget_jeunetarget_optintarget_optouttarget_scolairetarget_entreprisetarget_familletarget_newslettertarget_abonne
01605166.93826401001001
116051710.35738701100001
21605185.22870301100001
31605196.08348301100101
41605202.94928801000001
.................................
4712056405875-0.75476200100000
4712066405905-0.46983500100000
4712076405909-0.75476200100000
4712086405917-0.75476200100000
4712096405963-0.75476200100000
\n", "

471210 rows × 10 columns

\n", "
" ], "text/plain": [ " customer_id nb_targets target_jeune target_optin target_optout \\\n", "0 160516 6.938264 0 1 0 \n", "1 160517 10.357387 0 1 1 \n", "2 160518 5.228703 0 1 1 \n", "3 160519 6.083483 0 1 1 \n", "4 160520 2.949288 0 1 0 \n", "... ... ... ... ... ... \n", "471205 6405875 -0.754762 0 0 1 \n", "471206 6405905 -0.469835 0 0 1 \n", "471207 6405909 -0.754762 0 0 1 \n", "471208 6405917 -0.754762 0 0 1 \n", "471209 6405963 -0.754762 0 0 1 \n", "\n", " target_scolaire target_entreprise target_famille target_newsletter \\\n", "0 0 1 0 0 \n", "1 0 0 0 0 \n", "2 0 0 0 0 \n", "3 0 0 1 0 \n", "4 0 0 0 0 \n", "... ... ... ... ... \n", "471205 0 0 0 0 \n", "471206 0 0 0 0 \n", "471207 0 0 0 0 \n", "471208 0 0 0 0 \n", "471209 0 0 0 0 \n", "\n", " target_abonne \n", "0 1 \n", "1 1 \n", "2 1 \n", "3 1 \n", "4 1 \n", "... ... \n", "471205 0 \n", "471206 0 \n", "471207 0 \n", "471208 0 \n", "471209 0 \n", "\n", "[471210 rows x 10 columns]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "targets_KPI(display_input_databases('5', file_name = \"target_information\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "c75efea3-b5e8-4a7a-bed4-dd64ae9ff9f2", "metadata": {}, "outputs": [], "source": [ "#export_in_temporary(target_agg, 'Target_kpi_concatenate')" ] }, { "cell_type": "markdown", "id": "5d91263e-8a97-4cb1-8d94-db8ab0b77cdf", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# Brouillon" ] }, { "cell_type": "code", "execution_count": null, "id": "c5e864b1-adad-4267-b956-3f7ef371d677", "metadata": {}, "outputs": [], "source": [ "\n", "def display_covering_time(df, company, datecover):\n", " \"\"\"\n", " This function draws the time coverage of each company\n", " \"\"\"\n", " min_date = df['purchase_date'].min().strftime(\"%Y-%m-%d\")\n", " max_date = df['purchase_date'].max().strftime(\"%Y-%m-%d\")\n", " datecover[company] = [datetime.strptime(min_date, \"%Y-%m-%d\") + timedelta(days=x) for x in range((datetime.strptime(max_date, \"%Y-%m-%d\") - datetime.strptime(min_date, \"%Y-%m-%d\")).days)]\n", " print(f'Couverture Company {company} : {min_date} - {max_date}')\n", " return datecover\n", "\n", "\n", "def compute_time_intersection(datecover):\n", " \"\"\"\n", " This function returns the time coverage for all companies\n", " \"\"\"\n", " timestamps_sets = [set(timestamps) for timestamps in datecover.values()]\n", " intersection = set.intersection(*timestamps_sets)\n", " intersection_list = list(intersection)\n", " formated_dates = [dt.strftime(\"%Y-%m-%d\") for dt in intersection_list]\n", " return sorted(formated_dates)\n", "\n", "\n", "def df_coverage_modelization(sport, coverage_features = 0.7):\n", " \"\"\"\n", " This function returns start_date, end_of_features and final dates\n", " that help to construct train and test datasets\n", " \"\"\"\n", " datecover = {}\n", " for company in sport:\n", " df_products_purchased_reduced = display_input_databases(company, file_name = \"products_purchased_reduced\",\n", " datetime_col = ['purchase_date'])\n", " datecover = display_covering_time(df_products_purchased_reduced, company, datecover)\n", " #print(datecover.keys())\n", " dt_coverage = compute_time_intersection(datecover)\n", " start_date = dt_coverage[0]\n", " end_of_features = dt_coverage[int(0.7 * len(dt_coverage))]\n", " final_date = dt_coverage[-1]\n", " return start_date, end_of_features, final_date\n", " " ] }, { "cell_type": "markdown", "id": "2435097a-95a5-43e1-84d0-7f6b701441ba", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# Bases non communes : mise à plat" ] }, { "cell_type": "code", "execution_count": null, "id": "f8f988fb-5aab-4b57-80d1-e242f7e5b384", "metadata": {}, "outputs": [], "source": [ "companies = {'musee' : ['1', '2', '3', '4'],\n", " 'sport': ['5', '6', '7', '8', '9'],\n", " 'musique' : ['10', '11', '12', '13', '14']}\n", "\n", "all_companies = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14']" ] }, { "cell_type": "code", "execution_count": null, "id": "35ac004f-c191-4f45-a4b1-6d993d9ec38c", "metadata": {}, "outputs": [], "source": [ "companies_databases = pd.DataFrame()\n", "\n", "for i in all_companies:\n", " company_databases = pd.DataFrame({'company_number' : [i]})\n", "\n", " BUCKET = \"bdc2324-data/\"+i\n", " for base in fs.ls(BUCKET):\n", " match = re.search(r'\\/(\\d+)\\/(\\d+)([a-zA-Z_]+)\\.csv$', base)\n", " if match:\n", " nom_base = match.group(3)\n", " company_databases[nom_base] = 1\n", "\n", " companies_databases = pd.concat([companies_databases, company_databases])" ] }, { "cell_type": "code", "execution_count": null, "id": "8986e477-e6c5-4d6c-83b2-2c90c134b599", "metadata": {}, "outputs": [], "source": [ "pd.set_option(\"display.max_columns\", None)\n", "companies_databases\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8fecc3bb-4c03-4144-97c5-615224d9729e", "metadata": {}, "outputs": [], "source": [ "pd.reset_option(\"display.max_columns\")" ] }, { "cell_type": "markdown", "id": "0294ce71-840e-458b-8ffa-cadabbc6da21", "metadata": {}, "source": [ "# Debut Travail 25/02" ] }, { "cell_type": "markdown", "id": "ca2c8b6a-4965-422e-ba7c-66423a464fc1", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## Base communes au types Musée" ] }, { "cell_type": "code", "execution_count": null, "id": "5080f66e-f779-410a-876d-b4fe2795e17e", "metadata": {}, "outputs": [], "source": [ "for i in companies['musique']:\n", " BUCKET = \"bdc2324-data/\"+i\n", " liste_base = []\n", " for base in fs.ls(BUCKET):\n", " match = re.search(r'\\/(\\d+)\\/(\\d+)([a-zA-Z_]+)\\.csv$', base)\n", " if match:\n", " nom_base = match.group(3)\n", " liste_base.append(nom_base)\n", " globals()['base_'+i] = liste_base\n" ] }, { "cell_type": "code", "execution_count": null, "id": "abd477e1-7479-4c88-a5aa-f987af3f5b79", "metadata": {}, "outputs": [], "source": [ "# Trouver l'intersection entre les cinq listes\n", "intersection = set(base_1).intersection(base_2, base_3, base_4, base_101)\n", "\n", "# Convertir le résultat en liste si nécessaire\n", "intersection_liste = list(intersection)\n", "\n", "print(intersection_liste)" ] }, { "cell_type": "code", "execution_count": null, "id": "8d93888f-a511-4ee5-8bc3-d5173a7f119e", "metadata": {}, "outputs": [], "source": [ "# Trouver l'intersection entre les cinq listes\n", "intersection = set(base_10).intersection(base_12, base_13, base_14, base_11)\n", "\n", "# Convertir le résultat en liste si nécessaire\n", "intersection_liste = list(intersection)\n", "\n", "print(intersection_liste)" ] }, { "cell_type": "code", "execution_count": null, "id": "10e89669-42bb-4652-a4bc-1a3d1caf4d1a", "metadata": {}, "outputs": [], "source": [ "len(intersection_liste)" ] }, { "cell_type": "code", "execution_count": null, "id": "7d058b21-a538-4f59-aefb-ef7966f73fdc", "metadata": {}, "outputs": [], "source": [ "df1_tags = load_dataset_2(\"1\", \"tags\")" ] }, { "cell_type": "code", "execution_count": null, "id": "aa441f99-733c-4675-8676-bed4682d3324", "metadata": {}, "outputs": [], "source": [ "df1_structure_tag_mappings = load_dataset_2(\"1\", 'structure_tag_mappings')" ] }, { "cell_type": "code", "execution_count": null, "id": "6767a750-14a4-4c05-903e-d2f07170825b", "metadata": {}, "outputs": [], "source": [ "df1_customersplus = load_dataset_2(\"1\", \"customersplus\")" ] }, { "cell_type": "code", "execution_count": null, "id": "125e9145-a815-46fd-bdf4-07589508b259", "metadata": {}, "outputs": [], "source": [ "df1_customersplus.groupby('structure_id')['id'].count().reset_index().sort_values('id', ascending=False).head(20)" ] }, { "cell_type": "code", "execution_count": null, "id": "c17a6976-792f-474d-bcff-c89396eddb3f", "metadata": {}, "outputs": [], "source": [ "df1_customersplus['structure_id'].isna().sum() / len(df1_customersplus['structure_id'])" ] }, { "cell_type": "code", "execution_count": null, "id": "ecfc155a-cb42-46ec-8da5-33fdcd087355", "metadata": {}, "outputs": [], "source": [ "len(df1_structure_tag_mappings)" ] }, { "cell_type": "code", "execution_count": null, "id": "071410b8-950d-4fcc-b2b9-57415253c286", "metadata": {}, "outputs": [], "source": [ "df1_structure_tag_mappings.groupby('tag_id')['structure_id'].count().reset_index().sort_values('structure_id', ascending=False).head(20)" ] }, { "cell_type": "code", "execution_count": null, "id": "f48d27a9-14e4-4bb9-a60a-73e9438b58fc", "metadata": {}, "outputs": [], "source": [ "?np.sort_values()" ] }, { "cell_type": "code", "execution_count": null, "id": "14eaa0ea-02cc-430b-ab9b-38e6637810c3", "metadata": {}, "outputs": [], "source": [ "def info_colonnes_dataframe(df):\n", " # Créer une liste pour stocker les informations sur chaque colonne\n", " infos_colonnes = []\n", "\n", " # Parcourir les colonnes du DataFrame\n", " for nom_colonne, serie in df.items(): # Utiliser items() au lieu de iteritems()\n", " # Calculer le taux de valeurs manquantes\n", " taux_na = serie.isna().mean() * 100\n", "\n", " # Ajouter les informations à la liste\n", " infos_colonnes.append({\n", " 'Nom_colonne': nom_colonne,\n", " 'Type_colonne': str(serie.dtype),\n", " 'Taux_NA': taux_na\n", " })\n", "\n", " # Créer une nouvelle DataFrame à partir de la liste d'informations\n", " df_infos_colonnes = pd.DataFrame(infos_colonnes)\n", "\n", " return df_infos_colonnes" ] }, { "cell_type": "code", "execution_count": null, "id": "6b031c32-d4c8-42a5-9a71-a7810f9bf8d8", "metadata": { "scrolled": true }, "outputs": [], "source": [ "info_colonnes_dataframe(df1_tags)" ] }, { "cell_type": "code", "execution_count": null, "id": "e1a87f27-c4d4-4832-ac20-0c3c54aa4980", "metadata": {}, "outputs": [], "source": [ "info_colonnes_dataframe(df1_structure_tag_mappings)" ] }, { "cell_type": "code", "execution_count": null, "id": "fa5c65a8-2f74-4f3f-85fc-9ac91e0bb361", "metadata": {}, "outputs": [], "source": [ "pd.set_option('display.max_colwidth', None)\n", "\n", "print(df1_tags['name'])" ] }, { "cell_type": "code", "execution_count": null, "id": "a59bf932-5b54-4600-81f5-c55ac93ae510", "metadata": {}, "outputs": [], "source": [ "pd.set_option('display.max_rows', None)" ] }, { "cell_type": "code", "execution_count": null, "id": "a4ab298e-2cae-4865-9f00-4caff5f75ea1", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print(df1_tags['name'])" ] }, { "cell_type": "markdown", "id": "76bffba1-5f7e-4308-9224-437ca66148f8", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## KPI sur target_type" ] }, { "cell_type": "code", "execution_count": null, "id": "f6daf22e-6583-4431-a467-660a1dd4e5a4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d91d5895", "metadata": {}, "outputs": [], "source": [ "pd.set_option('display.max_colwidth', None)\n" ] }, { "cell_type": "markdown", "id": "c58b17d3", "metadata": {}, "source": [ "Raisonnement : on prends les target_type qui représente 90% des clients et on fait des catégories dessus." ] }, { "cell_type": "code", "execution_count": null, "id": "6930bff5", "metadata": {}, "outputs": [], "source": [ "def print_main_target(tenant_id, nb_print = 40):\n", " df_target = display_input_databases(tenant_id, \"target_information\")\n", "\n", " print('Nombre de ciblage : ', len(df_target))\n", " nb_customers = df_target['customer_id'].nunique()\n", " print('Nombre de client avec étiquette target : ', nb_customers) \n", "\n", " nb_custumers_per_target = df_target.groupby(\"target_name\")['customer_id'].count().reset_index().sort_values('customer_id', ascending=False)\n", " nb_custumers_per_target['cumulative_customers'] = nb_custumers_per_target['customer_id'].cumsum()/len(df_target)\n", " nb_custumers_per_target['customer_id'] = nb_custumers_per_target['customer_id']/nb_customers\n", "\n", " return nb_custumers_per_target.head(nb_print)" ] }, { "cell_type": "code", "execution_count": null, "id": "1e7ee1a0", "metadata": { "scrolled": true }, "outputs": [], "source": [ "pd.set_option(\"max_colwidth\", None)\n", "print_main_target('1', 60)" ] }, { "cell_type": "code", "execution_count": null, "id": "19f3a2dd-ba3d-4dec-8e10-fed544ab6a53", "metadata": {}, "outputs": [], "source": [ "pd.reset_option('display.max_rows')" ] }, { "cell_type": "code", "execution_count": null, "id": "b57a28ac", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('2', 25)" ] }, { "cell_type": "code", "execution_count": null, "id": "9a65991f", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('3', 70)" ] }, { "cell_type": "code", "execution_count": null, "id": "5f34b8bf", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('4', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "52b24d66-92ad-4421-a62b-5cba837f1893", "metadata": {}, "outputs": [], "source": [ "pd.set_option('display.max_rows', None)" ] }, { "cell_type": "code", "execution_count": null, "id": "40fe3676", "metadata": { "scrolled": true }, "outputs": [], "source": [ "\n", "\n", "print_main_target('5', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "820d3600-379b-4245-a977-f1f1fa1f1839", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('6', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "86f64a1b-763a-4e43-9601-a38c80392d47", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('7', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "fbf2ea42-515a-4cdf-a4c1-50f99c379ed9", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('8', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "9684045c-4e25-4952-b099-a559baa5d749", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('9', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "cf8f7816-e7f3-4b7a-a987-8350a76eb140", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('10', 100)" ] }, { "cell_type": "code", "execution_count": null, "id": "76c818a5-3c52-4d97-ac81-b7f3f89092bd", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('11', 100)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "603b11e4-5d76-4699-a1b2-e795929edc04", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('12', 100)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "fa93aecd-d117-481e-8507-15e49937ce14", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('13', 100)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a115ebcf-4488-47f3-9d7e-75a1fca52f0f", "metadata": { "scrolled": true }, "outputs": [], "source": [ "print_main_target('14', 100)\n" ] }, { "cell_type": "markdown", "id": "605cced5-052f-4a99-ac26-020c5d2ab633", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## KPI sur tags" ] }, { "cell_type": "code", "execution_count": null, "id": "916c3e2b-04d3-4877-b894-8f26f10d926e", "metadata": {}, "outputs": [], "source": [ "customersplus = load_dataset_2(\"4\", \"customersplus\")[['id', 'structure_id']]" ] }, { "cell_type": "code", "execution_count": null, "id": "46847b24-15a4-464e-969f-f16ed3653f1f", "metadata": {}, "outputs": [], "source": [ "structure_tag_mappings = load_dataset_2('4', \"structure_tag_mappings\")[['structure_id', 'tag_id']]" ] }, { "cell_type": "code", "execution_count": null, "id": "3c10c69d-735f-453e-96bf-750697d965d0", "metadata": {}, "outputs": [], "source": [ "customersplus[customersplus['structure_id'].notna()]['structure_id'].nunique()" ] }, { "cell_type": "code", "execution_count": null, "id": "9b0e77b3-5f16-4484-9564-7d3826583418", "metadata": {}, "outputs": [], "source": [ "len(customersplus[customersplus['structure_id'].notna()])" ] }, { "cell_type": "code", "execution_count": null, "id": "dfa27722-37f9-435a-8221-8aa6f9a4a107", "metadata": {}, "outputs": [], "source": [ "structure_tag_mappings['structure_id'].nunique()" ] }, { "cell_type": "code", "execution_count": null, "id": "2daabdd5-31e3-4918-9856-9bbc30cde602", "metadata": {}, "outputs": [], "source": [ "def tags_information(tenant_id, first_tags):\n", "\n", " customersplus = load_dataset_2(tenant_id, \"customersplus\")[['id', 'structure_id']]\n", " customersplus.rename(columns = {'id' : 'customer_id'}, inplace = True)\n", " tags = load_dataset_2(tenant_id, \"tags\")[['id', 'name']]\n", " tags.rename(columns = {'id' : 'tag_id', 'name' : 'tag_name'}, inplace = True)\n", " structure_tag_mappings = load_dataset_2(tenant_id, \"structure_tag_mappings\")[['structure_id', 'tag_id']]\n", " \n", " customer_tags = pd.merge(customersplus, structure_tag_mappings, on = 'structure_id', how = 'left')\n", " customer_tags = pd.merge(customer_tags, tags, on = 'tag_id', how = 'inner')\n", " \n", " nb_customers_with_tag = customer_tags['customer_id'].nunique()\n", " \n", " print('Nombre de client avec tag : ', nb_customers_with_tag)\n", " print('Proportion de clients avec tags : ', nb_customers_with_tag/len(customersplus))\n", " print('Moyenne de tags par client : ', len(customer_tags)/nb_customers_with_tag)\n", " \n", " info = customer_tags.groupby(['tag_id', 'tag_name'])['customer_id'].count().reset_index().sort_values('customer_id', ascending = False).head(first_tags)\n", "\n", " return info" ] }, { "cell_type": "code", "execution_count": null, "id": "0b9f5f71-a927-4cc8-bb0c-9538e28d3553", "metadata": {}, "outputs": [], "source": [ "tags_information(\"1\", 20)" ] }, { "cell_type": "code", "execution_count": null, "id": "bd5bef41-1774-4601-86b5-b7c1aea8f1d2", "metadata": {}, "outputs": [], "source": [ "tags_information(\"2\", 20)" ] }, { "cell_type": "code", "execution_count": null, "id": "7c2dc3e6-1418-44db-a8c0-4a9d59ec5232", "metadata": {}, "outputs": [], "source": [ "load_dataset_2(\"2\", \"tags\")[['id', 'name']]" ] }, { "cell_type": "code", "execution_count": null, "id": "c7b2c670-7122-4f67-b1aa-8c80a10f16d8", "metadata": {}, "outputs": [], "source": [ "tags_information(\"3\", 20)" ] }, { "cell_type": "code", "execution_count": null, "id": "76639995-252d-4a58-83d8-c0c00900c3a9", "metadata": {}, "outputs": [], "source": [ "tags_information(\"4\", 20)" ] }, { "cell_type": "code", "execution_count": null, "id": "07e91791-d4d4-42b1-ac18-22d3b0b9f7bd", "metadata": {}, "outputs": [], "source": [ "tags_information(\"101\", 20)" ] }, { "cell_type": "markdown", "id": "87d131cd-ead0-4ef4-a8ee-b09022d08ffa", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## KPI product" ] }, { "cell_type": "code", "execution_count": null, "id": "26582be9-cfd1-48ea-a0a7-31101fdeb9d1", "metadata": {}, "outputs": [], "source": [ "tenant_id = \"1\"\n", "\n", "df_product = display_databases(tenant_id, file_name = \"products_purchased_reduced\", datetime_col = ['purchase_date'])\n", "\n", "df_product.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "533bf499-dd56-4d29-b261-ca1e4928c9c7", "metadata": {}, "outputs": [], "source": [ "nb_tickets_per_events = df_product.groupby(['name_event_types', 'name_events'])['ticket_id'].count().reset_index().sort_values('ticket_id', ascending = False)\n", "nb_tickets_per_events['prop_tickets'] = round(nb_tickets_per_events['ticket_id']/len(df_product), 3)\n", "nb_tickets_per_events" ] }, { "cell_type": "markdown", "id": "1ede9eaa-7f0a-4856-9349-b2747d6a4901", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "# Fin travail 25/02" ] }, { "cell_type": "markdown", "id": "c437eaec", "metadata": {}, "source": [ "# Exemple sur Company 1" ] }, { "cell_type": "markdown", "id": "e855f403", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## customersplus.csv" ] }, { "cell_type": "code", "execution_count": null, "id": "91a8f8c4", "metadata": {}, "outputs": [], "source": [ "a = pd.DataFrame(df1_customersplus.info())" ] }, { "cell_type": "code", "execution_count": null, "id": "2fda171d", "metadata": {}, "outputs": [], "source": [ "def info_colonnes_dataframe(df):\n", " # Créer une liste pour stocker les informations sur chaque colonne\n", " infos_colonnes = []\n", "\n", " # Parcourir les colonnes du DataFrame\n", " for nom_colonne, serie in df.items(): # Utiliser items() au lieu de iteritems()\n", " # Calculer le taux de valeurs manquantes\n", " taux_na = serie.isna().mean() * 100\n", "\n", " # Ajouter les informations à la liste\n", " infos_colonnes.append({\n", " 'Nom_colonne': nom_colonne,\n", " 'Type_colonne': str(serie.dtype),\n", " 'Taux_NA': taux_na\n", " })\n", "\n", " # Créer une nouvelle DataFrame à partir de la liste d'informations\n", " df_infos_colonnes = pd.DataFrame(infos_colonnes)\n", "\n", " return df_infos_colonnes" ] }, { "cell_type": "code", "execution_count": null, "id": "205eeeab", "metadata": {}, "outputs": [], "source": [ "def cleaning_date(df, column_name):\n", " \"\"\"\n", " Nettoie la colonne spécifiée du DataFrame en convertissant les valeurs en datetime avec le format ISO8601.\n", "\n", " Parameters:\n", " - df: DataFrame\n", " Le DataFrame contenant la colonne à nettoyer.\n", " - column_name: str\n", " Le nom de la colonne à nettoyer.\n", "\n", " Returns:\n", " - DataFrame\n", " Le DataFrame modifié avec la colonne nettoyée.\n", " \"\"\"\n", " df[column_name] = pd.to_datetime(df[column_name], utc = True, format = 'ISO8601')\n", " return df" ] }, { "cell_type": "code", "execution_count": null, "id": "634282c5", "metadata": {}, "outputs": [], "source": [ "a = info_colonnes_dataframe(df1_customersplus)" ] }, { "cell_type": "code", "execution_count": null, "id": "0e8d4133", "metadata": {}, "outputs": [], "source": [ "a" ] }, { "cell_type": "code", "execution_count": null, "id": "1268ad5a", "metadata": {}, "outputs": [], "source": [ "a = pd.DataFrame(df1_customersplus.isna().sum()/len(df1_customersplus)*100)" ] }, { "cell_type": "code", "execution_count": null, "id": "bd41dc80", "metadata": {}, "outputs": [], "source": [ "# Selection des variables\n", "df1_customersplus_clean = df1_customersplus.copy()\n", "\n", "cleaning_date(df1_customersplus_clean, 'first_buying_date')\n", "cleaning_date(df1_customersplus_clean, 'last_visiting_date')\n", "\n", "df1_customersplus_clean.drop(['lastname', 'firstname', 'email', 'civility', 'note', 'created_at', 'updated_at', 'deleted_at', 'extra', 'reference', 'extra_field', 'identifier', 'need_reload', 'preferred_category', 'preferred_supplier', 'preferred_formula', 'zipcode', 'last_visiting_date'], axis = 1, inplace=True)\n", "df1_customersplus_clean.rename(columns = {'id' : 'customer_id'}, inplace = True)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2455d2e1", "metadata": { "scrolled": true }, "outputs": [], "source": [ "df1_purchases" ] }, { "cell_type": "code", "execution_count": null, "id": "5f9a159d", "metadata": {}, "outputs": [], "source": [ "df1_purchases.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "db201bf7", "metadata": {}, "outputs": [], "source": [ "# Nettoyage purchase_date\n", "df1_purchases['purchase_date'] = pd.to_datetime(df1_purchases['purchase_date'], utc = True)\n", "df1_purchases['purchase_date'] = pd.to_datetime(df1_purchases['purchase_date'], format = 'ISO8601')" ] }, { "cell_type": "code", "execution_count": null, "id": "bd436fca", "metadata": {}, "outputs": [], "source": [ "df1_purchases.info()" ] }, { "cell_type": "code", "execution_count": null, "id": "83435862", "metadata": {}, "outputs": [], "source": [ "# Selection des variables\n", "df1_purchases_clean = df1_purchases[['id', 'purchase_date', 'customer_id']]" ] }, { "cell_type": "markdown", "id": "637bdb72", "metadata": {}, "source": [ "# Customer information" ] }, { "cell_type": "markdown", "id": "14c52894", "metadata": { "jp-MarkdownHeadingCollapsed": true }, "source": [ "## Target area - NLP" ] }, { "cell_type": "code", "execution_count": null, "id": "d83abfbf", "metadata": {}, "outputs": [], "source": [ "# Target.csv cleaning\n", "df1_targets_clean = df1_targets[[\"id\", \"target_type_id\", \"name\"]]\n", "df1_targets_clean.rename(columns = {'id' : 'target_id' , 'name' : 'target_name'}, inplace = True)\n", "\n", "# target_type cleaning\n", "df1_target_types_clean = df1_target_types[[\"id\",\"is_import\",\"name\"]].add_prefix(\"target_type_\")\n", "\n", "#customer_target_mappings cleaning\n", "df1_customer_target_mappings_clean = df1_customer_target_mappings[[\"id\", \"customer_id\", \"target_id\"]]\n", "\n", "# Merge target et target_type\n", "df1_targets_full = pd.merge(df1_targets_clean, df1_target_types_clean, left_on='target_type_id', right_on='target_type_id', how='inner')\n", "df1_targets_full.drop(['target_type_id'], axis = 1, inplace=True)\n", "\n", "# Merge\n", "df1_targets_full = pd.merge(df1_customer_target_mappings_clean, df1_targets_full, left_on='target_id', right_on='target_id', how='inner')\n", "df1_targets_full.drop(['target_id'], axis = 1, inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "90d71b2c", "metadata": {}, "outputs": [], "source": [ "df1_targets_test = df1_targets_full[['id', 'customer_id']].groupby(['customer_id']).count()\n", "len(df1_targets_test[df1_targets_test['id'] > 1]) / len(df1_targets_test)\n", "\n", "# 99,6% des 151 000 client visés sont catégorisés plusieurs fois et en moyenne 5 fois... \n", "df1_targets_test.mean()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2301de1e", "metadata": {}, "outputs": [], "source": [ "df1_targets_full.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "75fbc2f7", "metadata": {}, "outputs": [], "source": [ "# Catégorisation des target_name\n", "import pandas as pd\n", "import nltk\n", "from nltk.tokenize import word_tokenize\n", "from nltk.corpus import stopwords\n", "from nltk.stem import WordNetLemmatizer\n", "from nltk.probability import FreqDist\n", "\n", "# Téléchargement des ressources nécessaires\n", "nltk.download('punkt')\n", "nltk.download('stopwords')\n", "nltk.download('wordnet')\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "55cddf92", "metadata": {}, "outputs": [], "source": [ "# Définition des fonctions de tokenisation, suppression des mots vides et lemmatisation\n", "def preprocess_text(texte):\n", " # Concaténation des éléments de la liste en une seule chaîne de caractères\n", " texte_concat = ' '.join(texte)\n", " \n", " # Tokenisation des mots\n", " tokens = word_tokenize(texte_concat.lower())\n", " \n", " # Suppression des mots vides (stopwords)\n", " stop_words = set(stopwords.words('french'))\n", " filtered_tokens = [word for word in tokens if word not in stop_words]\n", " \n", " # Lemmatisation des mots\n", " lemmatizer = WordNetLemmatizer()\n", " lemmatized_tokens = [lemmatizer.lemmatize(word) for word in filtered_tokens]\n", " \n", " return lemmatized_tokens\n", "\n", "\n", "# Appliquer le prétraitement à la colonne de texte\n", "df1_targets_full['target_name_tokened'] = df1_targets_full['target_name'].apply(preprocess_text)\n", "\n", "# Concaténer les listes de mots pour obtenir une liste de tous les mots dans le corpus\n", "all_words = [word for tokens in df1_targets_full['target_name_tokened'] for word in tokens]\n", "\n", "# Calculer la fréquence des mots\n", "freq_dist = FreqDist(all_words)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7fd98a85", "metadata": {}, "outputs": [], "source": [ "# Affichage des mots les plus fréquents\n", "print(\"Mots les plus fréquents:\")\n", "for mot, freq in freq_dist.most_common(15):\n", " print(f\"{mot}: {freq}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "cf94bb1d", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import nltk\n", "from nltk.tokenize import word_tokenize\n", "from nltk.corpus import stopwords\n", "from nltk.stem import WordNetLemmatizer\n", "\n", "# Téléchargement des ressources nécessaires\n", "nltk.download('punkt')\n", "nltk.download('stopwords')\n", "nltk.download('wordnet')\n", "\n", "# Création de la DataFrame d'exemple\n", "data = {'texte': [\"Le chat noir mange une souris.\", \"Le chien blanc aboie.\"]}\n", "df = pd.DataFrame(data)\n", "\n", "# Fonction pour prétraiter le texte\n", "def preprocess_text(texte):\n", " # Concaténation des éléments de la liste en une seule chaîne de caractères\n", " texte_concat = ' '.join(texte)\n", " \n", " # Tokenisation des mots\n", " tokens = word_tokenize(texte_concat.lower())\n", " \n", " # Suppression des mots vides (stopwords)\n", " stop_words = set(stopwords.words('french'))\n", " filtered_tokens = [word for word in tokens if word not in stop_words]\n", " \n", " # Lemmatisation des mots\n", " lemmatizer = WordNetLemmatizer()\n", " lemmatized_tokens = [lemmatizer.lemmatize(word) for word in filtered_tokens]\n", " \n", " return lemmatized_tokens\n", "\n", "# Appliquer la fonction de prétraitement à la colonne de texte\n", "df['texte_preprocessed'] = df['texte'].apply(preprocess_text)\n", "\n", "# Afficher le résultat\n", "print(df)\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.6" } }, "nbformat": 4, "nbformat_minor": 5 }