# Business Data Challenge - Team 1

import pandas as pd
import numpy as np
import os
import s3fs
import re
import warnings
from datetime import date, timedelta, datetime

# Create filesystem object
S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': S3_ENDPOINT_URL})


# Import KPI construction functions
exec(open('utils_features_construction.py').read())

# Ignore warning
warnings.filterwarnings('ignore')

# functions
def generate_test_set(type_of_comp):
    file_path_list = fs.ls(f"projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}/Test_set")
    test_set = pd.DataFrame()
    for file in file_path_list:
        print(file)
        with fs.open(file, mode="rb") as file_in:
            df = pd.read_csv(file_in, sep=",")
        test_set = pd.concat([test_set, df], ignore_index = True)
    return test_set


def generate_train_set(type_of_comp):
    file_path_list = fs.ls(f"projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}/Train_set")
    train_set = pd.DataFrame()
    for file in file_path_list:
        print(file)
        with fs.open(file, mode="rb") as file_in:
            df = pd.read_csv(file_in, sep=",")
        train_set = pd.concat([train_set, df], ignore_index = True)
    return train_set


type_of_comp = input('Choisissez le type de compagnie : sport ? musique ? musee ?')
BUCKET_OUT = f'projet-bdc2324-team1/1_Temp/1_0_Modelling_Datasets/{type_of_comp}/'

# create test and train datasets
test_set = generate_test_set(type_of_comp)
train_set = generate_train_set(type_of_comp)

# Exportation test set
FILE_KEY_OUT_S3 = "Test_set.csv"
FILE_PATH_OUT_S3 = BUCKET_OUT + FILE_KEY_OUT_S3

with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
    test_set.to_csv(file_out, index = False)

print("Exportation dataset test : SUCCESS")

# Exportation train set
FILE_KEY_OUT_S3 = "Train_set.csv"
FILE_PATH_OUT_S3 = BUCKET_OUT +  FILE_KEY_OUT_S3

with fs.open(FILE_PATH_OUT_S3, 'w') as file_out:
    train_set.to_csv(file_out, index = False)

print("Exportation dataset train : SUCCESS")
