ajoubrel-ensae/BDC-team-1
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Correction erreur de construction des datasets de modelisation

Browse Source
This commit is contained in:
Antoine JOUBREL 2024-03-10 15:30:23 +00:00
parent 27ef78a486
commit f40ae6ead0
2 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
0_2_Dataset_construction.py
View File

@ -7,6 +7,8 @@ import s3fs
import re
import warnings
from datetime import date, timedelta, datetime
from sklearn.model_selection import train_test_split
# Create filesystem object
S3_ENDPOINT_URL = "https://" + os.environ["AWS_S3_ENDPOINT"]
@ -172,14 +174,16 @@ for company in list_of_comp:
dataset = dataset[dataset['customer_id'] != anonymous_customer[company]]
#train test set
np.random.seed(42)
# #train test set
# np.random.seed(42)
split_ratio = 0.7
split_index = int(len(dataset) * split_ratio)
dataset = dataset.sample(frac=1).reset_index(drop=True)
dataset_train = dataset.iloc[:split_index]
dataset_test = dataset.iloc[split_index:]
# split_ratio = 0.7
# split_index = int(len(dataset) * split_ratio)
# dataset = dataset.sample(frac=1).reset_index(drop=True)
# dataset_train = dataset.iloc[:split_index]
# dataset_test = dataset.iloc[split_index:]
dataset_train, dataset_test = train_test_split(dataset, test_size=0.3, random_state=42)
# Dataset Test
# Exportation

2
0_3_General_modelization_dataset.py
View File

@ -38,7 +38,7 @@ def generate_train_set(type_of_comp):
print(file)
with fs.open(file, mode="rb") as file_in:
df = pd.read_csv(file_in, sep=",")
train_set = pd.concat([test_set, df], ignore_index = True)
train_set = pd.concat([train_set, df], ignore_index = True)
return train_set