ajoubrel-ensae/BDC-team-1
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modification access key

Browse Source
This commit is contained in:
Antoine JOUBREL 2024-02-10 22:19:35 +00:00
parent 23487bea92
commit 9f433d2c12
2 changed files with 56 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
0_Cleaning_and_merge.ipynb
View File

@ -19,7 +19,8 @@
"import numpy as np\n",
"import os\n",
"import s3fs\n",
"import re"
"import re\n",
"import warnings"
]
},
{
@ -38,7 +39,19 @@
"outputs": [],
"source": [
"# Create filesystem object\n",
"fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': 'https://'+'minio-simple.lab.groupe-genes.fr'},key ='WKTGH4YGUBAT3TR0OSUR', secret = 'g8ozi6ZUrBy8DzaAip4F7zOizbr4DKf4RgYNseqU', token = 'eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJXS1RHSDRZR1VCQVQzVFIwT1NVUiIsImFjciI6IjAiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9vbnl4aWEubGFiLmdyb3VwZS1nZW5lcy5mciJdLCJhdWQiOlsibWluaW8iLCJhY2NvdW50Il0sImF1dGhfdGltZSI6MTcwNzU4NjUwMCwiYXpwIjoib255eGlhLW1pbmlvIiwiZW1haWwiOiJhbnRvaW5lLmpvdWJyZWxAZW5zYWUuZnIiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwiZXhwIjoxNzA3NjczMDQ3LCJmYW1pbHlfbmFtZSI6IkpPVUJSRUwiLCJnaXZlbl9uYW1lIjoiQW50b2luZSIsImdyb3VwcyI6WyJiZGMyMzI0LXRlYW0xIl0sImlhdCI6MTcwNzU4NjY0NywiaXNzIjoiaHR0cHM6Ly9hdXRoLmdyb3VwZS1nZW5lcy5mci9yZWFsbXMvZ2VuZXMiLCJqdGkiOiI1MjQ2MDZmMS1lYWM3LTQxZDgtYTEzMy04MGZjMDk0MGVlNzEiLCJuYW1lIjoiQW50b2luZSBKT1VCUkVMIiwicG9saWN5Ijoic3Rzb25seSIsInByZWZlcnJlZF91c2VybmFtZSI6ImFqb3VicmVsLWVuc2FlIiwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbIm9mZmxpbmVfYWNjZXNzIiwiZGVmYXVsdC1yb2xlcy1nZW5lcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsInNlc3Npb25fc3RhdGUiOiI1OTk2MWNkYy0xNmFiLTQ4MTAtYWE4Zi1iZGUyMjkwNjhiNzUiLCJzaWQiOiI1OTk2MWNkYy0xNmFiLTQ4MTAtYWE4Zi1iZGUyMjkwNjhiNzUiLCJzdWIiOiIwNWYwZDk3Mi1jNWM4LTQyNmYtODAwZC00NmQ0OGU4NjkwMzUiLCJ0eXAiOiJCZWFyZXIifQ.-imw-N4bk1uCcQGobkxhsRoeBAqxC9rT7PifElbC7ODOStnwIulc7HRR2fmtiqI2PdyrfnVvzfmIPK1g056HbA')"
"S3_ENDPOINT_URL = \"https://\" + os.environ[\"AWS_S3_ENDPOINT\"]\n",
"fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a9b84234-d5df-4c43-a9cd-80cfe2f1e34d",
"metadata": {},
"outputs": [],
"source": [
"# Ignore warning\n",
"warnings.filterwarnings('ignore')"
]
},
{
@ -73,18 +86,10 @@
"execution_count": 4,
"id": "dd6a3518-b752-4a1e-b77b-9e03e853c3ed",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42764/4081512283.py:10: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
" df = pd.read_csv(file_in)\n"
]
}
],
"outputs": [],
"source": [
"# loop to create dataframes from liste\n",
"\n",
"files_path = liste_database\n",
"\n",
"client_number = files_path[0].split(\"/\")[1]\n",
@ -237,35 +242,7 @@
"execution_count": 9,
"id": "3e1d2ba7-ff4f-48eb-93a8-2bb648c70396",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42764/3092893564.py:5: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" tickets.rename(columns = {'id' : 'ticket_id'}, inplace = True)\n",
"/tmp/ipykernel_42764/3092893564.py:9: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" suppliers.rename(columns = {'name' : 'supplier_name'}, inplace = True)\n",
"/tmp/ipykernel_42764/3092893564.py:10: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" suppliers['supplier_name'] = suppliers['supplier_name'].fillna('')\n",
"/tmp/ipykernel_42764/3092893564.py:14: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" type_ofs.rename(columns = {'name' : 'type_of_ticket_name'}, inplace = True)\n"
]
}
],
"outputs": [],
"source": [
"df1_ticket_information = preprocessing_tickets_area(tickets = df1_tickets, purchases = df1_purchases, suppliers = df1_suppliers, type_ofs = df1_type_ofs)"
]
@ -434,19 +411,7 @@
"execution_count": 12,
"id": "5fbfd88b-b94c-489c-9201-670e96e453e7",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42764/3848597476.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" targets.rename(columns = {'id' : 'target_id' , 'name' : 'target_name'}, inplace = True)\n"
]
}
],
"outputs": [],
"source": [
"df1_target_information = preprocessing_target_area(targets = df1_targets, target_types = df1_target_types, customer_target_mappings = df1_customer_target_mappings)"
]
@ -489,32 +454,7 @@
"execution_count": 14,
"id": "c8552dd6-52c5-4431-b43d-3cd6c578fd9f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42764/1967867975.py:15: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" df[column_name] = pd.to_datetime(df[column_name], utc = True, format = 'ISO8601')\n",
"/tmp/ipykernel_42764/1967867975.py:15: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" df[column_name] = pd.to_datetime(df[column_name], utc = True, format = 'ISO8601')\n",
"/tmp/ipykernel_42764/1967867975.py:15: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" df[column_name] = pd.to_datetime(df[column_name], utc = True, format = 'ISO8601')\n"
]
}
],
"outputs": [],
"source": [
"df1_campaigns_information = preprocessing_campaigns_area(campaign_stats = df1_campaign_stats, campaigns = df1_campaigns)"
]
@ -1613,7 +1553,7 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 25,
"id": "e2c88552-b863-47a2-be23-8d2898fb28bc",
"metadata": {},
"outputs": [],
@ -1647,29 +1587,17 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 26,
"id": "24537647-bc29-4777-9848-ac4120a4aa60",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_42764/3700263836.py:11: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" opened_campaign.dropna(subset=['opened_at'], inplace=True)\n"
]
}
],
"outputs": [],
"source": [
"df1_campaigns_kpi = campaigns_kpi_function(campaigns_information = df1_campaigns_information) "
]
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": 27,
"id": "6be2a9a6-056b-4e19-8c26-a18ba3df36b3",
"metadata": {},
"outputs": [
@ -1749,7 +1677,7 @@
"4 6 20 0.0 NaT"
]
},
"execution_count": 30,
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
@ -1768,7 +1696,7 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": 28,
"id": "b913a69e-3146-4919-b5f6-a6108532bffa",
"metadata": {},
"outputs": [
@ -1779,7 +1707,7 @@
" 'offre muséale groupe'], dtype=object)"
]
},
"execution_count": 31,
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
@ -1790,7 +1718,7 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": 29,
"id": "2bda0b97-b28b-4070-a57d-aeab0e2f7dfe",
"metadata": {},
"outputs": [],
@ -1801,7 +1729,7 @@
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": 30,
"id": "043303fe-e90f-4689-a2a9-5d690555a045",
"metadata": {},
"outputs": [],
@ -1847,7 +1775,7 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": 31,
"id": "5882234a-1ed5-4269-87a6-0d75613476e3",
"metadata": {},
"outputs": [],
@ -1865,7 +1793,7 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": 32,
"id": "a4a2311d-8a72-4030-afd5-218004d5d2a5",
"metadata": {},
"outputs": [],
@ -1881,7 +1809,7 @@
},
{
"cell_type": "code",
"execution_count": 36,
"execution_count": 33,
"id": "a7a452a6-cd5e-4c8b-b250-8a7d26e48fad",
"metadata": {},
"outputs": [
@ -2011,7 +1939,7 @@
"5032 1049 days 18:46:12 13497.0 "
]
},
"execution_count": 36,
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
@ -2030,7 +1958,7 @@
},
{
"cell_type": "code",
"execution_count": 37,
"execution_count": 34,
"id": "4ab1c0d2-0097-4669-b984-b6822c976740",
"metadata": {},
"outputs": [
@ -2092,7 +2020,7 @@
"3 6 6.439463"
]
},
"execution_count": 37,
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
@ -2107,7 +2035,7 @@
},
{
"cell_type": "code",
"execution_count": 38,
"execution_count": 35,
"id": "a9c62b39-389e-4dac-89a6-ac8a59fea58a",
"metadata": {},
"outputs": [
@ -2187,7 +2115,7 @@
"4 2 2 143 6.150659"
]
},
"execution_count": 38,
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
@ -2202,7 +2130,7 @@
},
{
"cell_type": "code",
"execution_count": 39,
"execution_count": 36,
"id": "8710611c-7eb8-45ca-bdcc-009f4081f9e2",
"metadata": {},
"outputs": [
@ -2408,7 +2336,7 @@
"[5 rows x 28 columns]"
]
},
"execution_count": 39,
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
@ -2421,7 +2349,7 @@
},
{
"cell_type": "code",
"execution_count": 40,
"execution_count": 37,
"id": "a89fad43-ee68-4081-9384-3e9f08ec6a59",
"metadata": {},
"outputs": [
@ -2634,7 +2562,7 @@
"[5 rows x 31 columns]"
]
},
"execution_count": 40,
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
@ -2647,7 +2575,7 @@
},
{
"cell_type": "code",
"execution_count": 41,
"execution_count": 38,
"id": "a19fec00-4ece-400c-937c-ce5cd8daccfd",
"metadata": {},
"outputs": [],
@ -2665,7 +2593,7 @@
},
{
"cell_type": "code",
"execution_count": 42,
"execution_count": 39,
"id": "46de1912-4a66-46e5-8b9e-7768b2d2723b",
"metadata": {},
"outputs": [],
@ -2676,7 +2604,7 @@
},
{
"cell_type": "code",
"execution_count": 43,
"execution_count": 40,
"id": "1e42a790-b215-4107-a969-85005da06ebd",
"metadata": {},
"outputs": [],
@ -2687,7 +2615,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 41,
"id": "d950f24d-a5d1-4f1e-aeaa-ca826470365f",
"metadata": {},
"outputs": [],

21
1_Descriptive_Statistics.ipynb
View File

@ -10,7 +10,7 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 1,
"id": "abfaf341-7b35-4407-9133-d21336c04027",
"metadata": {},
"outputs": [],
@ -30,8 +30,9 @@
"metadata": {},
"outputs": [],
"source": [
"# Access Key to Minio\n",
"fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': 'https://'+'minio-simple.lab.groupe-genes.fr'},key ='WKTGH4YGUBAT3TR0OSUR', secret = 'g8ozi6ZUrBy8DzaAip4F7zOizbr4DKf4RgYNseqU', token = 'eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJXS1RHSDRZR1VCQVQzVFIwT1NVUiIsImFjciI6IjAiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9vbnl4aWEubGFiLmdyb3VwZS1nZW5lcy5mciJdLCJhdWQiOlsibWluaW8iLCJhY2NvdW50Il0sImF1dGhfdGltZSI6MTcwNzU4NjUwMCwiYXpwIjoib255eGlhLW1pbmlvIiwiZW1haWwiOiJhbnRvaW5lLmpvdWJyZWxAZW5zYWUuZnIiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwiZXhwIjoxNzA3NjczMDQ3LCJmYW1pbHlfbmFtZSI6IkpPVUJSRUwiLCJnaXZlbl9uYW1lIjoiQW50b2luZSIsImdyb3VwcyI6WyJiZGMyMzI0LXRlYW0xIl0sImlhdCI6MTcwNzU4NjY0NywiaXNzIjoiaHR0cHM6Ly9hdXRoLmdyb3VwZS1nZW5lcy5mci9yZWFsbXMvZ2VuZXMiLCJqdGkiOiI1MjQ2MDZmMS1lYWM3LTQxZDgtYTEzMy04MGZjMDk0MGVlNzEiLCJuYW1lIjoiQW50b2luZSBKT1VCUkVMIiwicG9saWN5Ijoic3Rzb25seSIsInByZWZlcnJlZF91c2VybmFtZSI6ImFqb3VicmVsLWVuc2FlIiwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbIm9mZmxpbmVfYWNjZXNzIiwiZGVmYXVsdC1yb2xlcy1nZW5lcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsInNlc3Npb25fc3RhdGUiOiI1OTk2MWNkYy0xNmFiLTQ4MTAtYWE4Zi1iZGUyMjkwNjhiNzUiLCJzaWQiOiI1OTk2MWNkYy0xNmFiLTQ4MTAtYWE4Zi1iZGUyMjkwNjhiNzUiLCJzdWIiOiIwNWYwZDk3Mi1jNWM4LTQyNmYtODAwZC00NmQ0OGU4NjkwMzUiLCJ0eXAiOiJCZWFyZXIifQ.-imw-N4bk1uCcQGobkxhsRoeBAqxC9rT7PifElbC7ODOStnwIulc7HRR2fmtiqI2PdyrfnVvzfmIPK1g056HbA')"
"# Create filesystem object\n",
"S3_ENDPOINT_URL = \"https://\" + os.environ[\"AWS_S3_ENDPOINT\"]\n",
"fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})"
]
},
{
@ -201,7 +202,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 5,
"id": "0e5d3b2e-1a75-4d46-80e6-c306e9f8de84",
"metadata": {},
"outputs": [
@ -214,7 +215,7 @@
" dtype='object')"
]
},
"execution_count": 8,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
@ -225,7 +226,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 6,
"id": "7667e8eb-9a1e-4216-96f4-bf987c6e30b5",
"metadata": {},
"outputs": [
@ -435,7 +436,7 @@
"36376 2022-08-25 13:08:38+00:00 447 days 00:47:59 5750.0 "
]
},
"execution_count": 12,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@ -447,7 +448,7 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": 7,
"id": "9b2e27f2-703d-465b-a0f9-76e996de617c",
"metadata": {},
"outputs": [],
@ -463,7 +464,7 @@
},
{
"cell_type": "code",
"execution_count": 57,
"execution_count": 8,
"id": "36141803-8865-4210-bd39-0a980301fd0c",
"metadata": {},
"outputs": [
@ -501,7 +502,7 @@
},
{
"cell_type": "code",
"execution_count": 58,
"execution_count": 9,
"id": "94cf1a25-9ded-48f2-b1b2-75225bdaf49d",
"metadata": {},
"outputs": [],