0_3_General_modelization_dataset.py

@@ -0,0 +1,68 @@
+# Business Data Challenge - Team 1
+
+import pandas as pd
+import numpy as np
+import os
+import s3fs
+import re
+import warnings
+from datetime import date, timedelta, datetime
+
+# Create filesystem object
+S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
+fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})
+
+
+# Import KPI construction functions
+exec(open('0_KPI_functions.py').read())
+
+# Ignore warning
+warnings.filterwarnings('ignore')
+
+# functions
+def generate_test_set():
+    file_path_list = fs.ls("projet-bdc2324-team1/Generalization/sport/Test_set")
+    test_set = pd.DataFrame()
+    for file in file_path_list:
+        print(file)
+        with fs.open(file, mode="rb") as file_in:
+            df = pd.read_csv(file_in, sep=",")
+        test_set = pd.concat([test_set, df], ignore_index = True)
+    return test_set
+
+
+def generate_train_set():
+    file_path_list = fs.ls("projet-bdc2324-team1/Generalization/sport/Train_set")
+    train_set = pd.DataFrame()
+    for file in file_path_list:
+        print(file)
+        with fs.open(file, mode="rb") as file_in:
+            df = pd.read_csv(file_in, sep=",")
+        train_set = pd.concat([test_set, df], ignore_index = True)
+    return train_set
+
+
+type_of_comp = input('Choisissez le type de compagnie : sport ? musique ? musee ?')
+BUCKET_OUT = f'projet-bdc2324-team1/Generalization/{type_of_comp}/'
+
+# create test and train datasets
+test_set = generate_test_set()
+train_set = generate_train_set()
+
+# Exportation test set
+FILE_KEY_OUT_S3 = "Test_set.csv"
+FILE_PATH_OUT_S3 = BUCKET_OUT + FILE_KEY_OUT_S3
+
+with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
+    test_set.to_csv(file_out, index = False)
+
+print("Exportation dataset test : SUCCESS")
+
+# Exportation train set
+FILE_KEY_OUT_S3 = "Train_set.csv"
+FILE_PATH_OUT_S3 = BUCKET_OUT +  FILE_KEY_OUT_S3
+
+with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
+    train_set.to_csv(file_out, index = False)
+
+print("Exportation dataset train : SUCCESS")