{ "cells": [ { "cell_type": "markdown", "id": "bd938e6e", "metadata": {}, "source": [ "**Short notebook to test connectivity with S3 services and explore the data**" ] }, { "cell_type": "code", "execution_count": 1, "id": "127753ac", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 2, "id": "ae3c64fe", "metadata": {}, "outputs": [], "source": [ "import os\n", "import s3fs\n", "fs = s3fs.S3FileSystem(\n", " client_kwargs={'endpoint_url': 'https://'+'minio-simple.lab.groupe-genes.fr'},\n", " key = os.environ[\"AWS_ACCESS_KEY_ID\"], \n", " secret = os.environ[\"AWS_SECRET_ACCESS_KEY\"], \n", " token = os.environ[\"AWS_SESSION_TOKEN\"])" ] }, { "cell_type": "code", "execution_count": 3, "id": "84b9ac42", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "\n", "def sample_by_blocks(df, block_size=10, num_blocks=10, random_state=None):\n", " \"\"\"Sample num_blocks blocks of block_size consecutive rows (no overlapping blocks).\"\"\"\n", " n = len(df)\n", " max_start = n - block_size\n", " if max_start < 0:\n", " raise ValueError(f\"DataFrame has {n} rows, need at least {block_size}\")\n", " if max_start + 1 < num_blocks:\n", " raise ValueError(f\"Not enough room for {num_blocks} non-overlapping blocks (need at least {num_blocks * block_size} rows)\")\n", " rng = np.random.default_rng(random_state)\n", " chosen_starts = rng.choice(max_start + 1, size=num_blocks, replace=False)\n", " chosen_starts.sort() # blocks in order of position in original df\n", " indices = np.concatenate([np.arange(s, s + block_size) for s in chosen_starts])\n", " return df.iloc[indices].reset_index(drop=True)\n", "\n", "# sample_df = sample_by_blocks(df, block_size=10, num_blocks=10, random_state=42)" ] }, { "cell_type": "markdown", "id": "7f7d45bb", "metadata": {}, "source": [ "### OG AUM" ] }, { "cell_type": "code", "execution_count": 11, "id": "83472648", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_149812/3855858483.py:2: DtypeWarning: Columns (0,1,2,3) have mixed types. Specify dtype option on import or set low_memory=False.\n", " df = pd.read_csv(f, sep =\";\")\n" ] } ], "source": [ "with fs.open('s3://projet-bdc-data/carmignac/AUM ENSAE V2 -20251105.csv', 'rb') as f:\n", " df = pd.read_csv(f, sep =\";\")\n", "\n", "sample_df = sample_by_blocks(df, block_size=10, num_blocks=10, random_state=42)" ] }, { "cell_type": "code", "execution_count": 13, "id": "79af063e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | Agreement - Code | \n", "Company - Id | \n", "Company - Ultimate Parent Id | \n", "Registrar Account - ID | \n", "Registrar Account - Region | \n", "RegistrarAccount - Country | \n", "Product - Asset Type | \n", "Product - Strategy | \n", "Product - Legal Status | \n", "Product - Is Dedie ? | \n", "Product - Fund | \n", "Product - Shareclass Type | \n", "Product - Shareclass Currency | \n", "Product - Isin | \n", "Centralisation Date | \n", "Quantity - AUM | \n", "Value - AUM CCY | \n", "Value - AUM € | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "088 | \n", "82.0 | \n", "82.0 | \n", "406321 | \n", "France | \n", "France | \n", "Diversified | \n", "Multi Expertise | \n", "FCP | \n", "NO | \n", "Carmignac Profil Réactif 100 | \n", "A | \n", "EUR | \n", "FR0010149211 | \n", "2020-02-29 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 1 | \n", "088 | \n", "82.0 | \n", "82.0 | \n", "406321 | \n", "France | \n", "France | \n", "Diversified | \n", "Multi Expertise | \n", "FCP | \n", "NO | \n", "Carmignac Profil Réactif 100 | \n", "A | \n", "EUR | \n", "FR0010149211 | \n", "2021-01-31 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 2 | \n", "088 | \n", "82.0 | \n", "82.0 | \n", "406321 | \n", "France | \n", "France | \n", "Diversified | \n", "Multi Expertise | \n", "FCP | \n", "NO | \n", "Carmignac Profil Réactif 100 | \n", "A | \n", "EUR | \n", "FR0010149211 | \n", "2021-05-31 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 3 | \n", "088 | \n", "82.0 | \n", "82.0 | \n", "406321 | \n", "France | \n", "France | \n", "Equity | \n", "Large Cap Emerging Markets Strategy | \n", "FCP | \n", "NO | \n", "Carmignac Emergents | \n", "A | \n", "EUR | \n", "FR0010149302 | \n", "2015-03-31 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 4 | \n", "088 | \n", "82.0 | \n", "82.0 | \n", "403909 | \n", "France | \n", "France | \n", "Equity | \n", "Investissement | \n", "FCP | \n", "NO | \n", "Carmignac Investissement | \n", "A | \n", "EUR | \n", "FR0010148981 | \n", "2020-09-30 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 95 | \n", "L652 | \n", "101.0 | \n", "33676.0 | \n", "364535 | \n", "Luxembourg | \n", "Luxembourg | \n", "Equity | \n", "Investissement | \n", "FCP | \n", "NO | \n", "Carmignac Investissement | \n", "A | \n", "EUR | \n", "FR0010148981 | \n", "2017-03-31 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 96 | \n", "L652 | \n", "101.0 | \n", "33676.0 | \n", "364535 | \n", "Luxembourg | \n", "Luxembourg | \n", "Equity | \n", "Investissement | \n", "FCP | \n", "NO | \n", "Carmignac Investissement | \n", "A | \n", "EUR | \n", "FR0010148981 | \n", "2017-11-30 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 97 | \n", "L652 | \n", "101.0 | \n", "33676.0 | \n", "364535 | \n", "Luxembourg | \n", "Luxembourg | \n", "Equity | \n", "Investissement | \n", "FCP | \n", "NO | \n", "Carmignac Investissement | \n", "A | \n", "EUR | \n", "FR0010148981 | \n", "2018-02-28 | \n", "0.0 | \n", "0.00 | \n", "0.0000 | \n", "
| 98 | \n", "L503_16 | \n", "9390.0 | \n", "19105.0 | \n", "200037779 | \n", "Belgium | \n", "Belgium | \n", "Diversified | \n", "Emerging Patrimoine | \n", "SICAV | \n", "NO | \n", "Carmignac Portfolio Emerging Patrimoine | \n", "A | \n", "USD | \n", "LU0592699259 | \n", "2017-04-30 | \n", "65.0 | \n", "6820.45 | \n", "6263.3270 | \n", "
| 99 | \n", "L503_16 | \n", "9390.0 | \n", "19105.0 | \n", "200037779 | \n", "Belgium | \n", "Belgium | \n", "Diversified | \n", "Emerging Patrimoine | \n", "SICAV | \n", "NO | \n", "Carmignac Portfolio Emerging Patrimoine | \n", "A | \n", "USD | \n", "LU0592699259 | \n", "2017-09-30 | \n", "65.0 | \n", "6857.50 | \n", "5800.6259 | \n", "
100 rows × 18 columns
\n", "| \n", " | Unnamed: 0.1 | \n", "Unnamed: 0 | \n", "Agreement - Code | \n", "Company - Id | \n", "Company - Ultimate Parent Id | \n", "Registrar Account - ID | \n", "Registrar Account - Region | \n", "RegistrarAccount - Country | \n", "Product - Asset Type | \n", "Product - Strategy | \n", "... | \n", "Product - Is Dedie ? | \n", "Product - Fund | \n", "Product - Shareclass Type | \n", "Product - Shareclass Currency | \n", "Product - Isin | \n", "Centralisation Date | \n", "Quantity - AUM | \n", "Value - AUM CCY | \n", "Value - AUM € | \n", "repair_flag | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "474086 | \n", "474032 | \n", "089 | \n", "151.0 | \n", "877.0 | \n", "406474 | \n", "FRANCE | \n", "FRANCE | \n", "DIVERSIFIED | \n", "MULTI EXPERTISE | \n", "... | \n", "NO | \n", "CARMIGNAC MULTI EXPERTISE | \n", "A | \n", "EUR | \n", "FR0010149203 | \n", "2015-04-30 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 1 | \n", "474087 | \n", "474033 | \n", "089 | \n", "151.0 | \n", "877.0 | \n", "406474 | \n", "FRANCE | \n", "FRANCE | \n", "DIVERSIFIED | \n", "MULTI EXPERTISE | \n", "... | \n", "NO | \n", "CARMIGNAC MULTI EXPERTISE | \n", "A | \n", "EUR | \n", "FR0010149203 | \n", "2015-10-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 2 | \n", "474088 | \n", "474034 | \n", "089 | \n", "151.0 | \n", "877.0 | \n", "406474 | \n", "FRANCE | \n", "FRANCE | \n", "DIVERSIFIED | \n", "MULTI EXPERTISE | \n", "... | \n", "NO | \n", "CARMIGNAC MULTI EXPERTISE | \n", "A | \n", "EUR | \n", "FR0010149203 | \n", "2016-03-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 3 | \n", "474089 | \n", "474035 | \n", "089 | \n", "151.0 | \n", "877.0 | \n", "406474 | \n", "FRANCE | \n", "FRANCE | \n", "DIVERSIFIED | \n", "MULTI EXPERTISE | \n", "... | \n", "NO | \n", "CARMIGNAC MULTI EXPERTISE | \n", "A | \n", "EUR | \n", "FR0010149203 | \n", "2016-11-30 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 4 | \n", "474090 | \n", "474036 | \n", "089 | \n", "151.0 | \n", "877.0 | \n", "406474 | \n", "FRANCE | \n", "FRANCE | \n", "DIVERSIFIED | \n", "MULTI EXPERTISE | \n", "... | \n", "NO | \n", "CARMIGNAC MULTI EXPERTISE | \n", "A | \n", "EUR | \n", "FR0010149203 | \n", "2017-01-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 95 | \n", "4736134 | \n", "4230498 | \n", "L694_A | \n", "15004.0 | \n", "15004.0 | \n", "420029 | \n", "SWITZERLAND | \n", "SWITZERLAND | \n", "DIVERSIFIED | \n", "EMERGING PATRIMOINE | \n", "... | \n", "NO | \n", "CARMIGNAC PORTFOLIO EMERGING PATRIMOINE | \n", "A | \n", "EUR | \n", "LU0592698954 | \n", "2019-09-30 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 96 | \n", "4736135 | \n", "4230499 | \n", "L694_A | \n", "15004.0 | \n", "15004.0 | \n", "420029 | \n", "SWITZERLAND | \n", "SWITZERLAND | \n", "DIVERSIFIED | \n", "EMERGING PATRIMOINE | \n", "... | \n", "NO | \n", "CARMIGNAC PORTFOLIO EMERGING PATRIMOINE | \n", "A | \n", "EUR | \n", "LU0592698954 | \n", "2020-05-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 97 | \n", "4736136 | \n", "4230500 | \n", "L694_A | \n", "15004.0 | \n", "15004.0 | \n", "420029 | \n", "SWITZERLAND | \n", "SWITZERLAND | \n", "DIVERSIFIED | \n", "PATRIMOINE | \n", "... | \n", "NO | \n", "CARMIGNAC PATRIMOINE | \n", "A | \n", "EUR | \n", "FR0010135103 | \n", "2015-08-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 98 | \n", "4736137 | \n", "4230501 | \n", "L694_A | \n", "15004.0 | \n", "15004.0 | \n", "420029 | \n", "SWITZERLAND | \n", "SWITZERLAND | \n", "DIVERSIFIED | \n", "PATRIMOINE | \n", "... | \n", "NO | \n", "CARMIGNAC PATRIMOINE | \n", "A | \n", "EUR | \n", "FR0010135103 | \n", "2016-07-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
| 99 | \n", "4736138 | \n", "4230502 | \n", "L694_A | \n", "15004.0 | \n", "15004.0 | \n", "420029 | \n", "SWITZERLAND | \n", "SWITZERLAND | \n", "DIVERSIFIED | \n", "PATRIMOINE | \n", "... | \n", "NO | \n", "CARMIGNAC PATRIMOINE | \n", "A | \n", "EUR | \n", "FR0010135103 | \n", "2016-12-31 | \n", "0.0 | \n", "NaN | \n", "NaN | \n", "False | \n", "
100 rows × 21 columns
\n", "