diff --git a/0_Input_cleaning.py b/0_Input_cleaning.py new file mode 100644 index 0000000..151c41a --- /dev/null +++ b/0_Input_cleaning.py @@ -0,0 +1,56 @@ +# Business Data Challenge - Team 1 + +import pandas as pd +import numpy as np +import os +import s3fs +import re +import warnings + +# Create filesystem object +S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"] +fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL}) + +# Import cleaning and merge functions +exec(open('0_Cleaning_and_merge_functions.py').read()) + +# Output folder +BUCKET_OUT = "projet-bdc2324-team1" + +# Ignore warning +warnings.filterwarnings('ignore') + + +def export_dataset(df, output_name): + print('Exportation of temporary dataset :', output_name) + FILE_PATH_OUT_S3 = BUCKET_OUT + "/" + output_name + with fs.open(FILE_PATH_OUT_S3, 'w') as file_out: + df.to_csv(file_out, index = False) + +## 1 - Cleaning of the datasets + +# Cleaning customerplus +df1_customerplus_clean = preprocessing_customerplus(directory_path = "1") + +## Exportation +export_dataset(df = df1_customerplus_clean, output_name = "0_Input/Company_1/customerplus_cleaned.csv") + +# Cleaning target area +df1_target_information = preprocessing_target_area(directory_path = "1") +## Exportation +export_dataset(df = df1_campaigns_information, output_name = "0_Input/Company_1/Campaigns dataset clean.csv") + +# Cleaning campaign area +df1_campaigns_information = preprocessing_campaigns_area(directory_path = "1") +## Exportation +export_dataset(df = df1_campaigns_information, output_name = "0_Input/Company_1/Campaigns dataset clean.csv") + +## Exportation +export_dataset(df = df1_campaigns_information, output_name = "0_Temp/Company 1 - Campaigns dataset clean.csv") + +# Cleaning product area +df1_products_purchased_reduced = uniform_product_df(directory_path = "1") +## Exportation +export_dataset(df = df1_campaigns_information, output_name = "0_Input/Company_1/Campaigns dataset clean.csv") +#Exportation +export_dataset(df = df1_products_purchased_reduced, output_name = "0_Temp/Company 1 - Purchases.csv")