58 lines
2.4 KiB
Python
58 lines
2.4 KiB
Python
# Business Data Challenge - Team 1
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
import os
|
|
import s3fs
|
|
import re
|
|
import warnings
|
|
|
|
# Create filesystem object
|
|
S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
|
|
fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})
|
|
|
|
# Import cleaning and merge functions
|
|
exec(open('0_Cleaning_and_merge_functions.py').read())
|
|
|
|
# Output folder
|
|
BUCKET_OUT = "projet-bdc2324-team1"
|
|
|
|
# Ignore warning
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
def export_dataset(df, output_name):
|
|
print('Exportation of dataset :', output_name)
|
|
FILE_PATH_OUT_S3 = BUCKET_OUT + "/" + output_name
|
|
with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
|
|
df.to_csv(file_out, index = False)
|
|
|
|
## 1 - Cleaning of the datasets
|
|
for tenant_id in ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "101"):
|
|
# Cleaning customerplus
|
|
df1_customerplus_clean = preprocessing_customerplus(directory_path = tenant_id)
|
|
|
|
## Exportation
|
|
export_dataset(df = df1_customerplus_clean, output_name = "0_Input/Company_"+ tenant_id +"/customerplus_cleaned.csv")
|
|
|
|
# Cleaning target area
|
|
df1_target_information = preprocessing_target_area(directory_path = tenant_id)
|
|
## Exportation
|
|
export_dataset(df = df1_target_information, output_name = "0_Input/Company_"+ tenant_id +"/target_information.csv")
|
|
|
|
# Cleaning campaign area
|
|
df1_campaigns_information = preprocessing_campaigns_area(directory_path = tenant_id)
|
|
## Exportation
|
|
export_dataset(df = df1_campaigns_information, output_name = "0_Input/Company_"+ tenant_id +"/campaigns_information.csv")
|
|
|
|
## Exportation
|
|
# export_dataset(df = df1_campaigns_information, output_name = "0_Temp/Company 1 - Campaigns dataset clean.csv")
|
|
|
|
# Cleaning product area
|
|
df1_products_purchased_reduced = uniform_product_df(directory_path = tenant_id)
|
|
## Exportation
|
|
export_dataset(df = df1_products_purchased_reduced, output_name = "0_Input/Company_"+ tenant_id +"/products_purchased_reduced.csv")
|
|
#Exportation
|
|
# export_dataset(df = df1_products_purchased_reduced, output_name = "0_Temp/Company 1 - Purchases.csv")
|
|
|
|
print("\n ------------------------------------------------------------------ \n --------------------- END CLEANING COMPANY " + tenant_id + " --------------------- \n ------------------------------------------------------------------") |