Changement architecture p1

1_Input_Cleaning.py

@@ -13,7 +13,7 @@ S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
 fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})
 
 # Import cleaning and merge functions
-exec(open('0_Cleaning_and_merge_functions.py').read())
+exec(open('utils_cleaning_and_merge.py').read())
 
 # Output folder 
 BUCKET_OUT = "projet-bdc2324-team1"
@@ -51,9 +51,6 @@ for tenant_id in ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12",
     ## Exportation 
     export_dataset(df = df1_campaigns_information, output_name = "0_Input/Company_"+ tenant_id +"/campaigns_information.csv")
     
-    # Exportation 
-    export_dataset(df = df1_campaigns_information, output_name = "1_Temp/Company 1 - Campaigns dataset clean.csv")
-
     if tenant_id == "101":
         # Cleaning product area
         products_purchased_reduced, products_purchased_reduced_1 = uniform_product_df(directory_path = tenant_id)

2_Datasets_Generation.py

@@ -17,7 +17,7 @@ S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
 fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})
 
 # Import KPI construction functions
-exec(open('0_KPI_functions.py').read())
+exec(open('utils_features_construction.py').read())
 
 # Ignore warning
 warnings.filterwarnings('ignore')
@@ -130,7 +130,7 @@ type_of_comp = input('Choisissez le type de compagnie : sport ? musique ? musee
 list_of_comp = companies[type_of_comp] 
 
 # Export folder
-BUCKET_OUT = f'projet-bdc2324-team1/Generalization_v2/{type_of_comp}'
+BUCKET_OUT = f'projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}'
 
 # Dates used for the construction of features and the dependant variable
 start_date = "2021-05-01"

3_Modelling_Datasets.py

@@ -21,7 +21,7 @@ warnings.filterwarnings('ignore')
 
 # functions
 def generate_test_set(type_of_comp):
-    file_path_list = fs.ls(f"projet-bdc2324-team1/Generalization_v2/{type_of_comp}/Test_set")
+    file_path_list = fs.ls(f"projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}/Test_set")
     test_set = pd.DataFrame()
     for file in file_path_list:
         print(file)
@@ -32,7 +32,7 @@ def generate_test_set(type_of_comp):
 
 
 def generate_train_set(type_of_comp):
-    file_path_list = fs.ls(f"projet-bdc2324-team1/Generalization_v2/{type_of_comp}/Train_set")
+    file_path_list = fs.ls(f"projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}/Train_set")
     train_set = pd.DataFrame()
     for file in file_path_list:
         print(file)
@@ -43,7 +43,7 @@ def generate_train_set(type_of_comp):
 
 
 type_of_comp = input('Choisissez le type de compagnie : sport ? musique ? musee ?')
-BUCKET_OUT = f'projet-bdc2324-team1/Generalization_v2/{type_of_comp}/'
+BUCKET_OUT = f'projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}/'
 
 # create test and train datasets
 test_set = generate_test_set(type_of_comp)

4_Descriptive_Statistics.py

@@ -9,7 +9,7 @@ import warnings
 # Ignore warning
 warnings.filterwarnings('ignore')
 
-exec(open('0_KPI_functions.py').read())
+exec(open('utils_features_construction.py').read())
 exec(open('utils_stat_desc.py').read())
 
 # Create filesystem object