{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2e8cf88b-cecf-409f-9c2d-c3762b233f05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting openpyxl\n",
      "  Downloading openpyxl-3.1.5-py2.py3-none-any.whl.metadata (2.5 kB)\n",
      "Collecting et-xmlfile (from openpyxl)\n",
      "  Downloading et_xmlfile-2.0.0-py3-none-any.whl.metadata (2.7 kB)\n",
      "Downloading openpyxl-3.1.5-py2.py3-none-any.whl (250 kB)\n",
      "Downloading et_xmlfile-2.0.0-py3-none-any.whl (18 kB)\n",
      "Installing collected packages: et-xmlfile, openpyxl\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2/2\u001b[0m [openpyxl]1/2\u001b[0m [openpyxl]\n",
      "\u001b[1A\u001b[2KSuccessfully installed et-xmlfile-2.0.0 openpyxl-3.1.5\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install openpyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "126c8a80-d9ad-4816-84f0-0c3d580f62c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ff2261fb-9516-4410-b42d-3acc8dc1a460",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import s3fs\n",
    "os.environ[\"AWS_ACCESS_KEY_ID\"] = 'N1DBJCHI7YTK9AVMG6XT'\n",
    "os.environ[\"AWS_SECRET_ACCESS_KEY\"] = 'SRCPMh8a1eQxX6Z09GeDxZoD55MBpnkJzyBctLII'\n",
    "os.environ[\"AWS_SESSION_TOKEN\"] = 'eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJOMURCSkNISTdZVEs5QVZNRzZYVCIsImFjciI6IjAiLCJhbGxvd2VkLW9yaWdpbnMiOlsiKiJdLCJhdWQiOlsibWluaW8iLCJhY2NvdW50Il0sImF1dGhfdGltZSI6MTc2MzEzMTgzNiwiYXpwIjoib255eGlhLW1pbmlvIiwiZW1haWwiOiJzYXJhaC50aG91bXlyZUBlbnNhZS5mciIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE3NjQzNDE0MzksImZhbWlseV9uYW1lIjoiVEhPVU1ZUkUiLCJnaXZlbl9uYW1lIjoiU2FyYWgiLCJncm91cHMiOlsiYmRjLWRhdGEiLCJiZGMtY2FybWlnbmFjLWczIl0sImlhdCI6MTc2MzEzMTgzOCwiaXNzIjoiaHR0cHM6Ly9hdXRoLmdyb3VwZS1nZW5lcy5mci9yZWFsbXMvZ2VuZXMiLCJqdGkiOiJkY2I2MWJiZi1lZjU4LTRhMTItOGYyZS1jYTI0ZmUyNTA2YzEiLCJuYW1lIjoiU2FyYWggVEhPVU1ZUkUiLCJwb2xpY3kiOiJzdHNvbmx5IiwicHJlZmVycmVkX3VzZXJuYW1lIjoic3Rob3VteXJlLWVuc2FlIiwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbIm9mZmxpbmVfYWNjZXNzIiwiZGVmYXVsdC1yb2xlcy1nZW5lcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCIsInNpZCI6ImQxMDI0NGVlLWE3ZDMtNDA5MC04ZDA3LWNlOWY3YjM5MDRkNCIsInN1YiI6ImVhYWVkN2QyLWM4MjYtNGIxNC05MzczLTYwYjNhODhlMWFiNiIsInR5cCI6IkJlYXJlciJ9.sLXOE8w930_dXU0yNWroaDvaTvcUUCONMcbgbKeMEduQebXQjOS7gEQxo-I7Q2oqLFb_dhg1zBlwx5VpNjyTMA'\n",
    "os.environ[\"AWS_DEFAULT_REGION\"] = 'us-east-1'\n",
    "fs = s3fs.S3FileSystem(\n",
    "    client_kwargs={'endpoint_url': 'https://'+'minio-simple.lab.groupe-genes.fr'},\n",
    "    key = os.environ[\"AWS_ACCESS_KEY_ID\"], \n",
    "    secret = os.environ[\"AWS_SECRET_ACCESS_KEY\"], \n",
    "    token = os.environ[\"AWS_SESSION_TOKEN\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d36f3f0-bd40-4a83-96d1-b46d75f5a4c5",
   "metadata": {},
   "source": [
    "# data exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaf5c5a0-eb1c-4242-b893-7600e6def109",
   "metadata": {},
   "source": [
    "Fonctions utiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60e2035c-c2f0-4c51-97df-102e67ba96ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_account(account_id, isin=None):\n",
    "    \"\"\"\n",
    "    Plots the stock (Quantity - AUM) evolution for a given Registrar Account.\n",
    "    Optionally, only for one ISIN.\n",
    "    \"\"\"\n",
    "\n",
    "    df = merged.copy()\n",
    "\n",
    "    # Filter by account\n",
    "    df = df[df[\"Registrar Account - ID\"] == account_id]\n",
    "\n",
    "    if isin is not None:\n",
    "        df = df[df[\"Product - Isin\"] == isin]\n",
    "\n",
    "    if df.empty:\n",
    "        print(f\"No data found for account {account_id}\")\n",
    "        return\n",
    "\n",
    "    df_plot = df.groupby(\"Centralisation Date\")[\"Quantity - AUM\"].sum().reset_index()\n",
    "\n",
    "    df_plot = df_plot.sort_values(\"Centralisation Date\")\n",
    "\n",
    "    # Plot\n",
    "    plt.figure(figsize=(12, 4))\n",
    "    plt.plot(df_plot[\"Centralisation Date\"], df_plot[\"Quantity - AUM\"], marker='o')\n",
    "    plt.title(f\"Stock Evolution for Account {account_id}\", fontsize=14)\n",
    "    plt.xlabel(\"Date\")\n",
    "    plt.ylabel(\"Total AUM\")\n",
    "    plt.grid(True)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "37e008b1-32d4-44be-9d23-1b90a5a26f89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================================================================\n",
      "DATASET : STOCKS\n",
      "================================================================================\n",
      "\n",
      "Shape : (4880297, 18)\n",
      "\n",
      "Columns : ['Agreement - Code', 'Company - Id', 'Company - Ultimate Parent Id', 'Registrar Account - ID', 'Registrar Account - Region', 'RegistrarAccount - Country', 'Product - Asset Type', 'Product - Strategy', 'Product - Legal Status', 'Product - Is Dedie ?', 'Product - Fund', 'Product - Shareclass Type', 'Product - Shareclass Currency', 'Product - Isin', 'Centralisation Date', 'Quantity - AUM', 'Value - AUM CCY', 'Value - AUM €']\n",
      "\n",
      "Dtypes :\n",
      " Agreement - Code                  object\n",
      "Company - Id                      object\n",
      "Company - Ultimate Parent Id      object\n",
      "Registrar Account - ID            object\n",
      "Registrar Account - Region        object\n",
      "RegistrarAccount - Country        object\n",
      "Product - Asset Type              object\n",
      "Product - Strategy                object\n",
      "Product - Legal Status            object\n",
      "Product - Is Dedie ?              object\n",
      "Product - Fund                    object\n",
      "Product - Shareclass Type         object\n",
      "Product - Shareclass Currency     object\n",
      "Product - Isin                    object\n",
      "Centralisation Date               object\n",
      "Quantity - AUM                   float64\n",
      "Value - AUM CCY                  float64\n",
      "Value - AUM €                    float64\n",
      "dtype: object\n",
      "\n",
      "Missing values (%) :\n",
      " Product - Asset Type             6.471553\n",
      "Company - Id                     2.330801\n",
      "Company - Ultimate Parent Id     2.330801\n",
      "Product - Strategy               0.001537\n",
      "Product - Shareclass Type        0.000717\n",
      "Agreement - Code                 0.000000\n",
      "RegistrarAccount - Country       0.000000\n",
      "Registrar Account - Region       0.000000\n",
      "Product - Legal Status           0.000000\n",
      "Registrar Account - ID           0.000000\n",
      "Product - Is Dedie ?             0.000000\n",
      "Product - Fund                   0.000000\n",
      "Product - Shareclass Currency    0.000000\n",
      "Product - Isin                   0.000000\n",
      "Centralisation Date              0.000000\n",
      "Quantity - AUM                   0.000000\n",
      "Value - AUM CCY                  0.000000\n",
      "Value - AUM €                    0.000000\n",
      "dtype: float64\n",
      "\n",
      "Sample rows:\n",
      "   Agreement - Code Company - Id Company - Ultimate Parent Id  \\\n",
      "0                3        166.0                        166.0   \n",
      "1                3        166.0                        166.0   \n",
      "2                3        166.0                        166.0   \n",
      "3                3        166.0                        166.0   \n",
      "4                3        166.0                        166.0   \n",
      "\n",
      "  Registrar Account - ID Registrar Account - Region  \\\n",
      "0              200000647                     France   \n",
      "1              200000647                     France   \n",
      "2              200000647                     France   \n",
      "3              200000647                     France   \n",
      "4              200000647                     France   \n",
      "\n",
      "  RegistrarAccount - Country Product - Asset Type Product - Strategy  \\\n",
      "0                     France          Diversified         Patrimoine   \n",
      "1                     France          Diversified         Patrimoine   \n",
      "2                     France          Diversified         Patrimoine   \n",
      "3                     France          Diversified         Patrimoine   \n",
      "4                     France          Diversified         Patrimoine   \n",
      "\n",
      "  Product - Legal Status Product - Is Dedie ?        Product - Fund  \\\n",
      "0                    FCP                   NO  Carmignac Patrimoine   \n",
      "1                    FCP                   NO  Carmignac Patrimoine   \n",
      "2                    FCP                   NO  Carmignac Patrimoine   \n",
      "3                    FCP                   NO  Carmignac Patrimoine   \n",
      "4                    FCP                   NO  Carmignac Patrimoine   \n",
      "\n",
      "  Product - Shareclass Type Product - Shareclass Currency Product - Isin  \\\n",
      "0                         A                           EUR   FR0010135103   \n",
      "1                         A                           EUR   FR0010135103   \n",
      "2                         A                           EUR   FR0010135103   \n",
      "3                         A                           EUR   FR0010135103   \n",
      "4                         A                           EUR   FR0010135103   \n",
      "\n",
      "  Centralisation Date  Quantity - AUM  Value - AUM CCY  Value - AUM €  \n",
      "0          2015-03-31          35.368       24648.6666     24648.6666  \n",
      "1          2015-11-30          35.368       22413.0553     22413.0553  \n",
      "2          2015-12-31          35.368       22051.2406     22051.2406  \n",
      "3          2016-03-31          35.368       21626.1173     21626.1173  \n",
      "4          2016-11-30          35.368       22489.4502     22489.4502  \n",
      "\n",
      "Unique values per column:\n",
      " Value - AUM €                    1697923\n",
      "Value - AUM CCY                  1689620\n",
      "Quantity - AUM                    554404\n",
      "Registrar Account - ID             15532\n",
      "Agreement - Code                    2521\n",
      "Company - Id                        1970\n",
      "Company - Ultimate Parent Id        1392\n",
      "Product - Isin                       491\n",
      "Centralisation Date                  130\n",
      "Product - Fund                        74\n",
      "Product - Strategy                    52\n",
      "RegistrarAccount - Country            39\n",
      "Registrar Account - Region            15\n",
      "Product - Shareclass Type             11\n",
      "Product - Legal Status                 6\n",
      "Product - Shareclass Currency          6\n",
      "Product - Asset Type                   5\n",
      "Product - Is Dedie ?                   2\n",
      "dtype: int64\n",
      "\n",
      "================================================================================\n",
      "DATASET : FLOWS\n",
      "================================================================================\n",
      "\n",
      "Shape : (2574461, 24)\n",
      "\n",
      "Columns : ['Agreement - Code', 'Company - Id', 'Company - Ultimate Parent Id', 'Registrar Account - ID', 'Registrar Account - Region', 'RegistrarAccount - Country', 'Product - Asset Type', 'Product - Strategy', 'Product - Legal Status', 'Product - Is Dedie ?', 'Product - Fund', 'Product - Shareclass Type', 'Product - Shareclass Currency', 'Product - Isin', 'Centralisation Date', 'Quantity - Subscription', 'Quantity - Redemption', 'Quantity - NetFlows', 'Value Ccy - Subscription', 'Value Ccy - Redemption', 'Value Ccy - NetFlows', 'Value € - Subscription', 'Value € - Redemption', 'Value € - NetFlows']\n",
      "\n",
      "Dtypes :\n",
      " Agreement - Code                  object\n",
      "Company - Id                      object\n",
      "Company - Ultimate Parent Id      object\n",
      "Registrar Account - ID            object\n",
      "Registrar Account - Region        object\n",
      "RegistrarAccount - Country        object\n",
      "Product - Asset Type              object\n",
      "Product - Strategy                object\n",
      "Product - Legal Status            object\n",
      "Product - Is Dedie ?              object\n",
      "Product - Fund                    object\n",
      "Product - Shareclass Type         object\n",
      "Product - Shareclass Currency     object\n",
      "Product - Isin                    object\n",
      "Centralisation Date               object\n",
      "Quantity - Subscription          float64\n",
      "Quantity - Redemption            float64\n",
      "Quantity - NetFlows              float64\n",
      "Value Ccy - Subscription         float64\n",
      "Value Ccy - Redemption           float64\n",
      "Value Ccy - NetFlows             float64\n",
      "Value € - Subscription           float64\n",
      "Value € - Redemption             float64\n",
      "Value € - NetFlows               float64\n",
      "dtype: object\n",
      "\n",
      "Missing values (%) :\n",
      " Product - Asset Type             0.079589\n",
      "Company - Id                     0.059818\n",
      "Company - Ultimate Parent Id     0.059818\n",
      "Product - Strategy               0.000233\n",
      "Product - Shareclass Type        0.000078\n",
      "Registrar Account - ID           0.000000\n",
      "RegistrarAccount - Country       0.000000\n",
      "Agreement - Code                 0.000000\n",
      "Registrar Account - Region       0.000000\n",
      "Product - Legal Status           0.000000\n",
      "Product - Is Dedie ?             0.000000\n",
      "Product - Fund                   0.000000\n",
      "Product - Shareclass Currency    0.000000\n",
      "Product - Isin                   0.000000\n",
      "Centralisation Date              0.000000\n",
      "Quantity - Subscription          0.000000\n",
      "Quantity - Redemption            0.000000\n",
      "Quantity - NetFlows              0.000000\n",
      "Value Ccy - Subscription         0.000000\n",
      "Value Ccy - Redemption           0.000000\n",
      "Value Ccy - NetFlows             0.000000\n",
      "Value € - Subscription           0.000000\n",
      "Value € - Redemption             0.000000\n",
      "Value € - NetFlows               0.000000\n",
      "dtype: float64\n",
      "\n",
      "Sample rows:\n",
      "   Agreement - Code Company - Id Company - Ultimate Parent Id  \\\n",
      "0              003          166                          166   \n",
      "1              003          166                          166   \n",
      "2              003          166                          166   \n",
      "3              003          166                          166   \n",
      "4              003          166                          166   \n",
      "\n",
      "  Registrar Account - ID Registrar Account - Region  \\\n",
      "0              200127202                     France   \n",
      "1                 406533                     France   \n",
      "2                 406533                     France   \n",
      "3                 406533                     France   \n",
      "4                 406533                     France   \n",
      "\n",
      "  RegistrarAccount - Country Product - Asset Type Product - Strategy  \\\n",
      "0                     France               Equity     Investissement   \n",
      "1                     France          Diversified         Patrimoine   \n",
      "2                     France               Equity     Investissement   \n",
      "3                     France               Equity     Investissement   \n",
      "4                     France               Equity     Investissement   \n",
      "\n",
      "  Product - Legal Status Product - Is Dedie ?  ... Centralisation Date  \\\n",
      "0                  SICAV                   NO  ...          2020-11-05   \n",
      "1                    FCP                   NO  ...          2015-03-09   \n",
      "2                    FCP                   NO  ...          2016-10-26   \n",
      "3                    FCP                   NO  ...          2018-10-18   \n",
      "4                    FCP                   NO  ...          2019-04-08   \n",
      "\n",
      "  Quantity - Subscription Quantity - Redemption Quantity - NetFlows  \\\n",
      "0                 1636.00                 0.000            1636.000   \n",
      "1                  144.69                 0.000             144.690   \n",
      "2                    0.00                -8.321              -8.321   \n",
      "3                    0.00               -22.083             -22.083   \n",
      "4                    0.00              -465.992            -465.992   \n",
      "\n",
      "  Value Ccy - Subscription  Value Ccy - Redemption  Value Ccy - NetFlows  \\\n",
      "0                280983.00                    0.00             280983.00   \n",
      "1                 99985.13                    0.00              99985.13   \n",
      "2                     0.00                -9384.76              -9384.76   \n",
      "3                     0.00               -25227.40             -25227.40   \n",
      "4                     0.00              -563775.76            -563775.76   \n",
      "\n",
      "   Value € - Subscription  Value € - Redemption  Value € - NetFlows  \n",
      "0               280983.00                  0.00           280983.00  \n",
      "1                99985.13                  0.00            99985.13  \n",
      "2                    0.00              -9384.76            -9384.76  \n",
      "3                    0.00             -25227.40           -25227.40  \n",
      "4                    0.00            -563775.76          -563775.76  \n",
      "\n",
      "[5 rows x 24 columns]\n",
      "\n",
      "Unique values per column:\n",
      " Value € - NetFlows               2018916\n",
      "Value Ccy - NetFlows             1972319\n",
      "Value € - Redemption             1323531\n",
      "Value Ccy - Redemption           1296468\n",
      "Value € - Subscription            955890\n",
      "Value Ccy - Subscription          926633\n",
      "Quantity - NetFlows               667586\n",
      "Quantity - Redemption             374378\n",
      "Quantity - Subscription           359661\n",
      "Registrar Account - ID              9805\n",
      "Centralisation Date                 2780\n",
      "Company - Id                        1929\n",
      "Agreement - Code                    1626\n",
      "Company - Ultimate Parent Id        1283\n",
      "Product - Isin                       474\n",
      "Product - Fund                        70\n",
      "Product - Strategy                    49\n",
      "RegistrarAccount - Country            34\n",
      "Registrar Account - Region            15\n",
      "Product - Shareclass Type             10\n",
      "Product - Shareclass Currency          6\n",
      "Product - Legal Status                 6\n",
      "Product - Asset Type                   5\n",
      "Product - Is Dedie ?                   2\n",
      "dtype: int64\n",
      "\n",
      "================================================================================\n",
      "DATASET : NAV/PRICES\n",
      "================================================================================\n",
      "\n",
      "Shape : (30333, 13)\n",
      "\n",
      "Columns : ['NavDate', 'LegalForm', 'Cod', 'PortfolioName', 'PTFCurrency', 'PortfolioAum_Eur', 'ShareClassIsin', 'ShareClassName', 'ShareClassCurrency', 'ShareClassPrice', 'NumberOfShares', 'ShareClassAumLocalCur', 'ShareClassAum_EUR']\n",
      "\n",
      "Dtypes :\n",
      " 0\n",
      "NavDate                  object\n",
      "LegalForm                object\n",
      "Cod                      object\n",
      "PortfolioName            object\n",
      "PTFCurrency              object\n",
      "PortfolioAum_Eur         object\n",
      "ShareClassIsin           object\n",
      "ShareClassName           object\n",
      "ShareClassCurrency       object\n",
      "ShareClassPrice          object\n",
      "NumberOfShares           object\n",
      "ShareClassAumLocalCur    object\n",
      "ShareClassAum_EUR        object\n",
      "dtype: object\n",
      "\n",
      "Missing values (%) :\n",
      " 0\n",
      "NavDate                  0.0\n",
      "LegalForm                0.0\n",
      "Cod                      0.0\n",
      "PortfolioName            0.0\n",
      "PTFCurrency              0.0\n",
      "PortfolioAum_Eur         0.0\n",
      "ShareClassIsin           0.0\n",
      "ShareClassName           0.0\n",
      "ShareClassCurrency       0.0\n",
      "ShareClassPrice          0.0\n",
      "NumberOfShares           0.0\n",
      "ShareClassAumLocalCur    0.0\n",
      "ShareClassAum_EUR        0.0\n",
      "dtype: float64\n",
      "\n",
      "Sample rows:\n",
      " 0     NavDate LegalForm  Cod                           PortfolioName  \\\n",
      "0  31/12/2009     SICAV   CC  Carmignac Portfolio Climate Transition   \n",
      "1  31/12/2009     SICAV  CFB       Carmignac Portfolio Flexible Bond   \n",
      "2  31/12/2009       FCP  CCT                   Carmignac Court Terme   \n",
      "3  31/12/2009       FCP   CE                     Carmignac Emergents   \n",
      "4  31/12/2009     SICAV  CAD      Carmignac Portfolio Asia Discovery   \n",
      "\n",
      "0 PTFCurrency PortfolioAum_Eur ShareClassIsin ShareClassName  \\\n",
      "0         EUR        941059600   LU0164455502      A EUR ACC   \n",
      "1         EUR      57063272.31   LU0336084032      A EUR ACC   \n",
      "2         EUR      788828666.5   FR0010149161      A EUR ACC   \n",
      "3         EUR       1508087050   FR0010149302      A EUR ACC   \n",
      "4         EUR      149490224.2   LU0336083810      A EUR ACC   \n",
      "\n",
      "0 ShareClassCurrency ShareClassPrice NumberOfShares ShareClassAumLocalCur  \\\n",
      "0                EUR          287.21     3276555.83             941059600   \n",
      "1                EUR        1016.833    56118.62745           57063272.31   \n",
      "2                EUR         3687.84    213899.9161           788828666.5   \n",
      "3                EUR          559.82    2693878.478            1508087050   \n",
      "4                EUR           884.9    168934.5962           149490224.2   \n",
      "\n",
      "0 ShareClassAum_EUR  \n",
      "0         941059600  \n",
      "1       57063272.31  \n",
      "2       788828666.5  \n",
      "3        1508087050  \n",
      "4       149490224.2  \n",
      "\n",
      "Unique values per column:\n",
      " 0\n",
      "ShareClassAum_EUR        30211\n",
      "ShareClassAumLocalCur    30032\n",
      "NumberOfShares           28910\n",
      "ShareClassPrice          14747\n",
      "PortfolioAum_Eur          5505\n",
      "ShareClassIsin             416\n",
      "NavDate                    210\n",
      "ShareClassName              90\n",
      "Cod                         55\n",
      "PortfolioName               55\n",
      "LegalForm                    6\n",
      "ShareClassCurrency           6\n",
      "PTFCurrency                  2\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#   2. BASIC INSPECTION\n",
    "\n",
    "def quick_info(df, name):\n",
    "    print(\"\\n\" + \"=\"*80)\n",
    "    print(f\"DATASET : {name}\")\n",
    "    print(\"=\"*80)\n",
    "    print(\"\\nShape :\", df.shape)\n",
    "    print(\"\\nColumns :\", df.columns.tolist())\n",
    "    print(\"\\nDtypes :\\n\", df.dtypes)\n",
    "    print(\"\\nMissing values (%) :\\n\", df.isna().mean().sort_values(ascending=False)*100)\n",
    "    print(\"\\nSample rows:\\n\", df.head(5))\n",
    "    print(\"\\nUnique values per column:\\n\", df.nunique().sort_values(ascending=False))\n",
    "\n",
    "quick_info(stocks, \"STOCKS\")\n",
    "quick_info(flows,  \"FLOWS\")\n",
    "quick_info(nav,    \"NAV/PRICES\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e67a99ea-ddf4-4627-8f48-ec183c671acb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1042/1864850766.py:2: DtypeWarning: Columns (0,1,2,3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  flows = pd.read_csv(f, sep=\";\")\n",
      "/tmp/ipykernel_1042/1864850766.py:5: DtypeWarning: Columns (0,1,2,3) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  stocks = pd.read_csv(f, sep=\";\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NavDate</th>\n",
       "      <th>LegalForm</th>\n",
       "      <th>Cod</th>\n",
       "      <th>PortfolioName</th>\n",
       "      <th>PTFCurrency</th>\n",
       "      <th>PortfolioAum_Eur</th>\n",
       "      <th>ShareClassIsin</th>\n",
       "      <th>ShareClassName</th>\n",
       "      <th>ShareClassCurrency</th>\n",
       "      <th>ShareClassPrice</th>\n",
       "      <th>NumberOfShares</th>\n",
       "      <th>ShareClassAumLocalCur</th>\n",
       "      <th>ShareClassAum_EUR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31/12/2009</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CC</td>\n",
       "      <td>Carmignac Portfolio Climate Transition</td>\n",
       "      <td>EUR</td>\n",
       "      <td>941059600</td>\n",
       "      <td>LU0164455502</td>\n",
       "      <td>A EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>287.21</td>\n",
       "      <td>3276555.83</td>\n",
       "      <td>941059600</td>\n",
       "      <td>941059600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31/12/2009</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CFB</td>\n",
       "      <td>Carmignac Portfolio Flexible Bond</td>\n",
       "      <td>EUR</td>\n",
       "      <td>57063272.31</td>\n",
       "      <td>LU0336084032</td>\n",
       "      <td>A EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>1016.833</td>\n",
       "      <td>56118.62745</td>\n",
       "      <td>57063272.31</td>\n",
       "      <td>57063272.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31/12/2009</td>\n",
       "      <td>FCP</td>\n",
       "      <td>CCT</td>\n",
       "      <td>Carmignac Court Terme</td>\n",
       "      <td>EUR</td>\n",
       "      <td>788828666.5</td>\n",
       "      <td>FR0010149161</td>\n",
       "      <td>A EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>3687.84</td>\n",
       "      <td>213899.9161</td>\n",
       "      <td>788828666.5</td>\n",
       "      <td>788828666.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31/12/2009</td>\n",
       "      <td>FCP</td>\n",
       "      <td>CE</td>\n",
       "      <td>Carmignac Emergents</td>\n",
       "      <td>EUR</td>\n",
       "      <td>1508087050</td>\n",
       "      <td>FR0010149302</td>\n",
       "      <td>A EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>559.82</td>\n",
       "      <td>2693878.478</td>\n",
       "      <td>1508087050</td>\n",
       "      <td>1508087050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31/12/2009</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CAD</td>\n",
       "      <td>Carmignac Portfolio Asia Discovery</td>\n",
       "      <td>EUR</td>\n",
       "      <td>149490224.2</td>\n",
       "      <td>LU0336083810</td>\n",
       "      <td>A EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>884.9</td>\n",
       "      <td>168934.5962</td>\n",
       "      <td>149490224.2</td>\n",
       "      <td>149490224.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30328</th>\n",
       "      <td>18/11/2025</td>\n",
       "      <td>FCPE</td>\n",
       "      <td>CEAM</td>\n",
       "      <td>Carmignac Epargne Actions Monde ISR</td>\n",
       "      <td>EUR</td>\n",
       "      <td>8758111.45</td>\n",
       "      <td>FR001400XC60</td>\n",
       "      <td>FW EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>97.94</td>\n",
       "      <td>89423.2331</td>\n",
       "      <td>8758111.45</td>\n",
       "      <td>8758111.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30329</th>\n",
       "      <td>18/11/2025</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CAD</td>\n",
       "      <td>Carmignac Portfolio Asia Discovery</td>\n",
       "      <td>EUR</td>\n",
       "      <td>112323819.3</td>\n",
       "      <td>LU0336083810</td>\n",
       "      <td>A EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>2206.68</td>\n",
       "      <td>29628.26787</td>\n",
       "      <td>65380106.15</td>\n",
       "      <td>65380106.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30330</th>\n",
       "      <td>18/11/2025</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CAD</td>\n",
       "      <td>Carmignac Portfolio Asia Discovery</td>\n",
       "      <td>EUR</td>\n",
       "      <td>112323819.3</td>\n",
       "      <td>LU0992629740</td>\n",
       "      <td>F EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>226.23</td>\n",
       "      <td>64439.56235</td>\n",
       "      <td>14578162.19</td>\n",
       "      <td>14578162.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30331</th>\n",
       "      <td>18/11/2025</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CAD</td>\n",
       "      <td>Carmignac Portfolio Asia Discovery</td>\n",
       "      <td>EUR</td>\n",
       "      <td>112323819.3</td>\n",
       "      <td>LU1623762256</td>\n",
       "      <td>FW EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>165.7</td>\n",
       "      <td>4299.866445</td>\n",
       "      <td>712487.87</td>\n",
       "      <td>712487.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30332</th>\n",
       "      <td>18/11/2025</td>\n",
       "      <td>SICAV</td>\n",
       "      <td>CAD</td>\n",
       "      <td>Carmignac Portfolio Asia Discovery</td>\n",
       "      <td>EUR</td>\n",
       "      <td>112323819.3</td>\n",
       "      <td>LU2420651155</td>\n",
       "      <td>I EUR ACC</td>\n",
       "      <td>EUR</td>\n",
       "      <td>119.22</td>\n",
       "      <td>159415.2581</td>\n",
       "      <td>19005487.07</td>\n",
       "      <td>19005487.07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30333 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0         NavDate LegalForm   Cod                           PortfolioName  \\\n",
       "0      31/12/2009     SICAV    CC  Carmignac Portfolio Climate Transition   \n",
       "1      31/12/2009     SICAV   CFB       Carmignac Portfolio Flexible Bond   \n",
       "2      31/12/2009       FCP   CCT                   Carmignac Court Terme   \n",
       "3      31/12/2009       FCP    CE                     Carmignac Emergents   \n",
       "4      31/12/2009     SICAV   CAD      Carmignac Portfolio Asia Discovery   \n",
       "...           ...       ...   ...                                     ...   \n",
       "30328  18/11/2025      FCPE  CEAM     Carmignac Epargne Actions Monde ISR   \n",
       "30329  18/11/2025     SICAV   CAD      Carmignac Portfolio Asia Discovery   \n",
       "30330  18/11/2025     SICAV   CAD      Carmignac Portfolio Asia Discovery   \n",
       "30331  18/11/2025     SICAV   CAD      Carmignac Portfolio Asia Discovery   \n",
       "30332  18/11/2025     SICAV   CAD      Carmignac Portfolio Asia Discovery   \n",
       "\n",
       "0     PTFCurrency PortfolioAum_Eur ShareClassIsin ShareClassName  \\\n",
       "0             EUR        941059600   LU0164455502      A EUR ACC   \n",
       "1             EUR      57063272.31   LU0336084032      A EUR ACC   \n",
       "2             EUR      788828666.5   FR0010149161      A EUR ACC   \n",
       "3             EUR       1508087050   FR0010149302      A EUR ACC   \n",
       "4             EUR      149490224.2   LU0336083810      A EUR ACC   \n",
       "...           ...              ...            ...            ...   \n",
       "30328         EUR       8758111.45   FR001400XC60     FW EUR ACC   \n",
       "30329         EUR      112323819.3   LU0336083810      A EUR ACC   \n",
       "30330         EUR      112323819.3   LU0992629740      F EUR ACC   \n",
       "30331         EUR      112323819.3   LU1623762256     FW EUR ACC   \n",
       "30332         EUR      112323819.3   LU2420651155      I EUR ACC   \n",
       "\n",
       "0     ShareClassCurrency ShareClassPrice NumberOfShares ShareClassAumLocalCur  \\\n",
       "0                    EUR          287.21     3276555.83             941059600   \n",
       "1                    EUR        1016.833    56118.62745           57063272.31   \n",
       "2                    EUR         3687.84    213899.9161           788828666.5   \n",
       "3                    EUR          559.82    2693878.478            1508087050   \n",
       "4                    EUR           884.9    168934.5962           149490224.2   \n",
       "...                  ...             ...            ...                   ...   \n",
       "30328                EUR           97.94     89423.2331            8758111.45   \n",
       "30329                EUR         2206.68    29628.26787           65380106.15   \n",
       "30330                EUR          226.23    64439.56235           14578162.19   \n",
       "30331                EUR           165.7    4299.866445             712487.87   \n",
       "30332                EUR          119.22    159415.2581           19005487.07   \n",
       "\n",
       "0     ShareClassAum_EUR  \n",
       "0             941059600  \n",
       "1           57063272.31  \n",
       "2           788828666.5  \n",
       "3            1508087050  \n",
       "4           149490224.2  \n",
       "...                 ...  \n",
       "30328        8758111.45  \n",
       "30329       65380106.15  \n",
       "30330       14578162.19  \n",
       "30331         712487.87  \n",
       "30332       19005487.07  \n",
       "\n",
       "[30333 rows x 13 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with fs.open('projet-bdc-data//carmignac/Flows ENSAE V2 -20251105.csv', 'rb') as f:\n",
    "    flows = pd.read_csv(f, sep=\";\")\n",
    "\n",
    "with fs.open('projet-bdc-data//carmignac/AUM ENSAE V2 -20251105.csv', 'rb') as f:\n",
    "    stocks = pd.read_csv(f, sep=\";\")\n",
    "\n",
    "with fs.open('projet-bdc-data/carmignac/Monthly AUM and NAV since 2010.xlsx', 'rb') as f:\n",
    "    nav_raw = pd.read_excel(f, header=None, engine=\"openpyxl\")\n",
    "nav = nav_raw[0].str.split(\",\", expand=True)\n",
    "nav.columns = nav.iloc[0]\n",
    "nav = nav[1:].reset_index(drop=True)\n",
    "\n",
    "nav"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9bc92c9f-216c-475e-bfb8-edc1a4e839f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Date conversion done.\n",
      "NAV numeric conversion done.\n",
      "String normalization done.\n",
      "\n",
      "ISIN missing in FLOWS but present in STOCKS : 17\n",
      "\n",
      "ISIN missing in STOCKS but present in FLOWS : 0\n",
      "\n",
      "ISIN missing in NAV but present in FLOWS : 67\n",
      "\n",
      "ISIN missing in NAV but present in STOCKS : 76\n",
      "\n",
      "Accounts in STOCKS but NEVER in FLOWS : 5777\n",
      "\n",
      "Accounts in FLOWS but NEVER in STOCKS : 118\n",
      "\n",
      "CLIENT BEHAVIOR (first 5 rows):\n",
      "   Registrar Account - ID  n_days  n_transactions  total_netflows   mean_flow  \\\n",
      "0              100000028       3               3        -109.238  -36.412667   \n",
      "1              100000042       1               1        -660.115 -660.115000   \n",
      "2              100000065       1               1        -174.646 -174.646000   \n",
      "3              100000069      65              73       -7479.755 -102.462397   \n",
      "4              100000073       1               1        -133.402 -133.402000   \n",
      "\n",
      "      std_flow  total_subscription  total_redemption   churn_ratio  \n",
      "0    49.280511               0.000          -109.238 -1.092380e+11  \n",
      "1          NaN               0.000          -660.115 -6.601150e+11  \n",
      "2          NaN               0.000          -174.646 -1.746460e+11  \n",
      "3  2168.971331           33320.402        -40800.157 -1.224480e+00  \n",
      "4          NaN               0.000          -133.402 -1.334020e+11  \n",
      "\n",
      "FUND BEHAVIOR (first 5 rows):\n",
      "   Product - Isin  n_accounts  n_days  total_netflows    vol_flows\n",
      "0   FR0010135103        2690    2723   -2.571327e+07  2622.609244\n",
      "1   FR0010147603         733    2719   -2.562187e+06  1206.248205\n",
      "2   FR0010148981        1841    2722   -3.609440e+06  1051.069183\n",
      "3   FR0010148999         454    2306   -7.130297e+05  1265.364138\n",
      "4   FR0010149112         934    2000   -9.438901e+05  1834.961721\n",
      "\n",
      "EXPLORATION STEP 2 COMPLETED.\n"
     ]
    }
   ],
   "source": [
    "# 1. CLEAN DATES (formats différents)\n",
    "\n",
    "stocks[\"Centralisation Date\"] = pd.to_datetime(stocks[\"Centralisation Date\"], errors=\"coerce\")\n",
    "flows[\"Centralisation Date\"]  = pd.to_datetime(flows[\"Centralisation Date\"], errors=\"coerce\")\n",
    "nav[\"NavDate\"] = pd.to_datetime(nav[\"NavDate\"], format=\"%d/%m/%Y\", errors=\"coerce\")\n",
    "\n",
    "print(\"Date conversion done.\")\n",
    "\n",
    "# 2. CLEAN NUMERIC COLUMNS FOR NAV FILE\n",
    "\n",
    "num_cols = [\"PortfolioAum_Eur\",\"ShareClassPrice\",\"NumberOfShares\",\n",
    "            \"ShareClassAumLocalCur\",\"ShareClassAum_EUR\"]\n",
    "\n",
    "for col in num_cols:\n",
    "    nav[col] = (\n",
    "        nav[col]\n",
    "        .astype(str)\n",
    "        .str.replace(\",\", \".\", regex=False)\n",
    "        .str.replace(\" \", \"\")\n",
    "        .astype(float)\n",
    "    )\n",
    "\n",
    "print(\"NAV numeric conversion done.\")\n",
    "\n",
    "# 3. STANDARDIZE STRINGS FOR JOIN KEYS\n",
    "\n",
    "def norm(df):\n",
    "    for col in df.columns:\n",
    "        if df[col].dtype == \"object\":\n",
    "            df[col] = df[col].astype(str).str.strip().str.upper()\n",
    "    return df\n",
    "\n",
    "stocks = norm(stocks)\n",
    "flows = norm(flows)\n",
    "nav = norm(nav)\n",
    "\n",
    "print(\"String normalization done.\")\n",
    "\n",
    "\n",
    "# 4. ANALYSE RELATIONS ACROSS FILES\n",
    "\n",
    "# Unique sets\n",
    "isin_stocks = set(stocks[\"Product - Isin\"].unique())\n",
    "isin_flows  = set(flows[\"Product - Isin\"].unique())\n",
    "isin_nav    = set(nav[\"ShareClassIsin\"].unique())\n",
    "\n",
    "print(\"\\nISIN missing in FLOWS but present in STOCKS :\", len(isin_stocks - isin_flows))\n",
    "print(\"\\nISIN missing in STOCKS but present in FLOWS :\", len(isin_flows - isin_stocks))\n",
    "print(\"\\nISIN missing in NAV but present in FLOWS :\", len(isin_flows - isin_nav))\n",
    "print(\"\\nISIN missing in NAV but present in STOCKS :\", len(isin_stocks - isin_nav))\n",
    "\n",
    "\n",
    "# 5. CLIENTS: STOCKS VS FLOWS\n",
    "\n",
    "acc_stocks = set(stocks[\"Registrar Account - ID\"].unique())\n",
    "acc_flows  = set(flows[\"Registrar Account - ID\"].unique())\n",
    "\n",
    "print(\"\\nAccounts in STOCKS but NEVER in FLOWS :\", len(acc_stocks - acc_flows))\n",
    "print(\"\\nAccounts in FLOWS but NEVER in STOCKS :\", len(acc_flows - acc_stocks))\n",
    "\n",
    "\n",
    "# 6. CLIENT ACTIVITY METRICS (DETAILED)\n",
    "\n",
    "client_behavior = flows.groupby(\"Registrar Account - ID\").agg(\n",
    "    n_days=(\"Centralisation Date\", lambda x: x.nunique()),\n",
    "    n_transactions=(\"Quantity - NetFlows\", \"count\"),\n",
    "    total_netflows=(\"Quantity - NetFlows\", \"sum\"),\n",
    "    mean_flow=(\"Quantity - NetFlows\", \"mean\"),\n",
    "    std_flow=(\"Quantity - NetFlows\", \"std\"),\n",
    "    total_subscription=(\"Quantity - Subscription\", \"sum\"),\n",
    "    total_redemption=(\"Quantity - Redemption\", \"sum\")\n",
    ").reset_index()\n",
    "\n",
    "# Add churn metric\n",
    "client_behavior[\"churn_ratio\"] = (\n",
    "    client_behavior[\"total_redemption\"] /\n",
    "    (client_behavior[\"total_subscription\"] + 1e-9)\n",
    ")\n",
    "\n",
    "print(\"\\nCLIENT BEHAVIOR (first 5 rows):\\n\", client_behavior.head())\n",
    "\n",
    "\n",
    "# 7. FUNDS ACTIVITY METRICS\n",
    "\n",
    "fund_behavior = flows.groupby(\"Product - Isin\").agg(\n",
    "    n_accounts=(\"Registrar Account - ID\", \"nunique\"),\n",
    "    n_days=(\"Centralisation Date\", lambda x: x.nunique()),\n",
    "    total_netflows=(\"Quantity - NetFlows\", \"sum\"),\n",
    "    vol_flows=(\"Quantity - NetFlows\", \"std\")\n",
    ").reset_index()\n",
    "\n",
    "print(\"\\nFUND BEHAVIOR (first 5 rows):\\n\", fund_behavior.head())\n",
    "\n",
    "\n",
    "# 8. SAVE INTERMEDIATE\n",
    "\n",
    "client_behavior.to_csv(\"client_behavior.csv\", index=False)\n",
    "fund_behavior.to_csv(\"fund_behavior.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "afb51598-3a7b-41f2-8d25-5b4b8bfb1c8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL usable ISIN : 407\n",
      "Stocks only ISIN : 17\n",
      "Flows only ISIN : 0\n",
      "Missing NAV : 76\n",
      "All ISIN groups saved into 4 separate files.\n"
     ]
    }
   ],
   "source": [
    "valid_full = isin_stocks & isin_flows & isin_nav\n",
    "stocks_only = isin_stocks - isin_flows\n",
    "flows_only = isin_flows - isin_stocks\n",
    "missing_nav = (isin_stocks | isin_flows) - isin_nav\n",
    "\n",
    "print(\"FULL usable ISIN :\", len(valid_full))\n",
    "print(\"Stocks only ISIN :\", len(stocks_only))\n",
    "print(\"Flows only ISIN :\", len(flows_only))\n",
    "print(\"Missing NAV :\", len(missing_nav))\n",
    "\n",
    "pd.DataFrame({\"isin\": list(valid_full)}).to_csv(\"isin_full.csv\", index=False)\n",
    "pd.DataFrame({\"isin\": list(stocks_only)}).to_csv(\"isin_stocks_only.csv\", index=False)\n",
    "pd.DataFrame({\"isin\": list(flows_only)}).to_csv(\"isin_flows_only.csv\", index=False)\n",
    "pd.DataFrame({\"isin\": list(missing_nav)}).to_csv(\"isin_missing_nav.csv\", index=False)\n",
    "\n",
    "print(\"All ISIN groups saved into 4 separate files.\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "61e0c71a-a1c6-4ed8-ba15-b7a9badc4d4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Registrar Account - ID  n_days  n_transactions  total_netflows   mean_flow  \\\n",
      "0              100000028       3               3        -109.238  -36.412667   \n",
      "1              100000042       1               1        -660.115 -660.115000   \n",
      "2              100000065       1               1        -174.646 -174.646000   \n",
      "3              100000069      65              73       -7479.755 -102.462397   \n",
      "4              100000073       1               1        -133.402 -133.402000   \n",
      "\n",
      "      std_flow  total_subscription  total_redemption   churn_ratio  \\\n",
      "0    49.280511               0.000          -109.238 -1.092380e+08   \n",
      "1          NaN               0.000          -660.115 -6.601150e+08   \n",
      "2          NaN               0.000          -174.646 -1.746460e+08   \n",
      "3  2168.971331           33320.402        -40800.157 -1.224480e+00   \n",
      "4          NaN               0.000          -133.402 -1.334020e+08   \n",
      "\n",
      "   churn_flag  activity_score  flow_volatility  inertia_ratio  \n",
      "0           0        1.386294        49.280511       0.998921  \n",
      "1           0        0.693147         0.000000       0.999640  \n",
      "2           0        0.693147         0.000000       0.999640  \n",
      "3           0        4.304065      2168.971331       0.976619  \n",
      "4           0        0.693147         0.000000       0.999640  \n"
     ]
    }
   ],
   "source": [
    "eps = 1e-6\n",
    "\n",
    "client_behavior[\"churn_ratio\"] = (\n",
    "    client_behavior[\"total_redemption\"] /\n",
    "    (client_behavior[\"total_subscription\"] + eps)\n",
    ")\n",
    "\n",
    "client_behavior[\"churn_flag\"] = (\n",
    "    client_behavior[\"total_redemption\"] > client_behavior[\"total_subscription\"]\n",
    ").astype(int)\n",
    "\n",
    "client_behavior[\"activity_score\"] = np.log1p(client_behavior[\"n_transactions\"])\n",
    "\n",
    "client_behavior[\"flow_volatility\"] = client_behavior[\"std_flow\"].fillna(0)\n",
    "\n",
    "client_behavior[\"inertia_ratio\"] = (\n",
    "    1 - client_behavior[\"n_days\"] / flows[\"Centralisation Date\"].nunique()\n",
    ")\n",
    "\n",
    "print(client_behavior.head())\n",
    "\n",
    "client_behavior.to_csv(\"client_behavior_clean.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "8ee7e911-eb73-4846-b545-661140411c1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1042/1645623303.py:17: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  .apply(lambda x: x[\"Value - AUM €\"].max() / x[\"Value - AUM €\"].sum()) \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Registrar Account - ID  n_isin_held  n_funds_held  n_asset_types  \\\n",
      "0              100000014            1             1              1   \n",
      "1              100000016            2             2              2   \n",
      "2              100000028            1             1              1   \n",
      "3              100000038            3             3              2   \n",
      "4              100000042            1             1              1   \n",
      "\n",
      "   n_strategies    total_aum  median_aum  concentration_ratio  \n",
      "0             1       0.0000         0.0                  NaN  \n",
      "1             2       0.0000         0.0                  NaN  \n",
      "2             1  126236.2184         0.0                  1.0  \n",
      "3             3       0.0000         0.0                  NaN  \n",
      "4             1  446362.9015         0.0                  1.0  \n",
      "        n_isin_held  n_funds_held  n_asset_types  n_strategies     total_aum  \\\n",
      "count  12501.000000  12501.000000   12501.000000  12501.000000  1.250100e+04   \n",
      "mean       5.514759      4.408367       2.082473      4.109271  4.218474e+08   \n",
      "std       10.434698      5.472756       1.254048      4.714800  5.618341e+09   \n",
      "min        1.000000      1.000000       1.000000      1.000000 -2.586805e+08   \n",
      "25%        1.000000      1.000000       1.000000      1.000000  0.000000e+00   \n",
      "50%        2.000000      2.000000       2.000000      2.000000  2.587605e+05   \n",
      "75%        6.000000      5.000000       3.000000      5.000000  8.817014e+06   \n",
      "max      469.000000     67.000000       6.000000     48.000000  4.780234e+11   \n",
      "\n",
      "         median_aum  concentration_ratio  \n",
      "count  1.250100e+04          7708.000000  \n",
      "mean   2.573991e+05             0.790503  \n",
      "std    3.487976e+06             0.261535  \n",
      "min   -2.317333e+06            -2.591840  \n",
      "25%    0.000000e+00             0.576503  \n",
      "50%    0.000000e+00             0.972159  \n",
      "75%    1.474502e+02             1.000000  \n",
      "max    2.215373e+08             2.983529  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1042/1645623303.py:17: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  .apply(lambda x: x[\"Value - AUM €\"].max() / x[\"Value - AUM €\"].sum()) \\\n"
     ]
    }
   ],
   "source": [
    "# Diversification per account\n",
    "account_div = stocks.groupby(\"Registrar Account - ID\").agg(\n",
    "    n_isin_held=(\"Product - Isin\", \"nunique\"),\n",
    "    n_funds_held=(\"Product - Fund\", \"nunique\"),\n",
    "    n_asset_types=(\"Product - Asset Type\", \"nunique\"),\n",
    "    n_strategies=(\"Product - Strategy\", \"nunique\"),\n",
    "    total_aum=(\"Value - AUM €\", \"sum\"),\n",
    "    median_aum=(\"Value - AUM €\", \"median\")\n",
    ").reset_index()\n",
    "\n",
    "# Concentration ratio per account\n",
    "aum_by_account_fund = stocks.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Fund\"]\n",
    ")[\"Value - AUM €\"].sum().reset_index()\n",
    "\n",
    "concentration = aum_by_account_fund.groupby(\"Registrar Account - ID\") \\\n",
    "    .apply(lambda x: x[\"Value - AUM €\"].max() / x[\"Value - AUM €\"].sum()) \\\n",
    "    .reset_index(name=\"concentration_ratio\")\n",
    "\n",
    "# Merge diversification + concentration\n",
    "account_static = account_div.merge(concentration, on=\"Registrar Account - ID\", how=\"left\")\n",
    "\n",
    "print(account_static.head())\n",
    "print(account_static.describe())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "76f6fa0d-9d7a-4145-af1c-986d83947f91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Registrar Account - ID         country          region\n",
      "0              100000014     SWITZERLAND     SWITZERLAND\n",
      "1              100000016  UNITED KINGDOM  UNITED KINGDOM\n",
      "2              100000028  UNITED KINGDOM  UNITED KINGDOM\n",
      "3              100000038     SWITZERLAND     SWITZERLAND\n",
      "4              100000042  UNITED KINGDOM  UNITED KINGDOM\n"
     ]
    }
   ],
   "source": [
    "# Geographic info per account\n",
    "geo = stocks.groupby(\"Registrar Account - ID\").agg(\n",
    "    country=(\"RegistrarAccount - Country\", lambda x: x.mode()[0]),\n",
    "    region=(\"Registrar Account - Region\", lambda x: x.mode()[0])\n",
    ").reset_index()\n",
    "\n",
    "print(geo.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e9bb67ab-9029-4ace-b960-b3d6e0b8683c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Registrar Account - ID  n_days  n_transactions  total_netflows   mean_flow  \\\n",
      "0              100000028       3               3        -109.238  -36.412667   \n",
      "1              100000042       1               1        -660.115 -660.115000   \n",
      "2              100000065       1               1        -174.646 -174.646000   \n",
      "3              100000069      65              73       -7479.755 -102.462397   \n",
      "4              100000073       1               1        -133.402 -133.402000   \n",
      "\n",
      "      std_flow  total_subscription  total_redemption   churn_ratio  \\\n",
      "0    49.280511               0.000          -109.238 -1.092380e+08   \n",
      "1          NaN               0.000          -660.115 -6.601150e+08   \n",
      "2          NaN               0.000          -174.646 -1.746460e+08   \n",
      "3  2168.971331           33320.402        -40800.157 -1.224480e+00   \n",
      "4          NaN               0.000          -133.402 -1.334020e+08   \n",
      "\n",
      "   churn_flag  ...  n_funds_held  n_asset_types  n_strategies    total_aum  \\\n",
      "0           0  ...           1.0            1.0           1.0  126236.2184   \n",
      "1           0  ...           1.0            1.0           1.0  446362.9015   \n",
      "2           0  ...           1.0            1.0           1.0  488743.4240   \n",
      "3           0  ...           0.0            0.0           0.0          NaN   \n",
      "4           0  ...           2.0            2.0           2.0  373322.8948   \n",
      "\n",
      "   median_aum  concentration_ratio         country          region  \\\n",
      "0         0.0                  1.0  UNITED KINGDOM  UNITED KINGDOM   \n",
      "1         0.0                  1.0  UNITED KINGDOM  UNITED KINGDOM   \n",
      "2         0.0                  1.0  UNITED KINGDOM  UNITED KINGDOM   \n",
      "3         NaN                  NaN         UNKNOWN         UNKNOWN   \n",
      "4         0.0                  1.0  UNITED KINGDOM  UNITED KINGDOM   \n",
      "\n",
      "   log_total_aum  log_median_aum  \n",
      "0      11.745918             0.0  \n",
      "1      13.008890             0.0  \n",
      "2      13.099595             0.0  \n",
      "3            NaN             NaN  \n",
      "4      12.830202             0.0  \n",
      "\n",
      "[5 rows x 24 columns]\n",
      "       Registrar Account - ID       n_days  n_transactions  total_netflows  \\\n",
      "count                    6842  6842.000000     6842.000000    6.842000e+03   \n",
      "unique                   6842          NaN             NaN             NaN   \n",
      "top                 100000028          NaN             NaN             NaN   \n",
      "freq                        1          NaN             NaN             NaN   \n",
      "mean                      NaN   122.731804      376.273166    3.426362e+04   \n",
      "std                       NaN   363.726141     1972.817028    1.644676e+06   \n",
      "min                       NaN     1.000000        1.000000   -9.982301e+06   \n",
      "25%                       NaN     2.000000        2.000000   -3.448661e+02   \n",
      "50%                       NaN     5.000000        6.000000   -1.116000e+00   \n",
      "75%                       NaN    27.000000       42.000000    4.220087e+01   \n",
      "max                       NaN  2715.000000    53314.000000    1.319043e+08   \n",
      "\n",
      "            mean_flow      std_flow  total_subscription  total_redemption  \\\n",
      "count     6842.000000  5.696000e+03        6.842000e+03      6.842000e+03   \n",
      "unique            NaN           NaN                 NaN               NaN   \n",
      "top               NaN           NaN                 NaN               NaN   \n",
      "freq              NaN           NaN                 NaN               NaN   \n",
      "mean       420.503483  6.035653e+03        1.565831e+05     -1.223195e+05   \n",
      "std      15548.555778  7.652710e+04        4.204641e+06      2.600731e+06   \n",
      "min    -333474.890000  0.000000e+00       -3.931320e+02     -2.069900e+08   \n",
      "25%        -35.658375  5.643245e+01        0.000000e+00     -6.968600e+03   \n",
      "50%         -0.067287  2.479222e+02        3.393685e+02     -4.493603e+02   \n",
      "75%          2.235534  9.869604e+02        6.000000e+03     -7.275400e+01   \n",
      "max     871531.706418  4.697263e+06        3.388942e+08      0.000000e+00   \n",
      "\n",
      "         churn_ratio   churn_flag  ...  n_funds_held  n_asset_types  \\\n",
      "count   6.842000e+03  6842.000000  ...   6842.000000    6842.000000   \n",
      "unique           NaN          NaN  ...           NaN            NaN   \n",
      "top              NaN          NaN  ...           NaN            NaN   \n",
      "freq             NaN          NaN  ...           NaN            NaN   \n",
      "mean   -7.445486e+08     0.000146  ...      5.054224       2.114294   \n",
      "std     1.163193e+10     0.012090  ...      6.457703       1.342230   \n",
      "min    -5.212597e+11     0.000000  ...      0.000000       0.000000   \n",
      "25%    -2.279500e+06     0.000000  ...      1.000000       1.000000   \n",
      "50%    -1.048786e+00     0.000000  ...      2.000000       2.000000   \n",
      "75%    -9.333542e-01     0.000000  ...      7.000000       3.000000   \n",
      "max     3.225589e+01     1.000000  ...     67.000000       6.000000   \n",
      "\n",
      "        n_strategies     total_aum    median_aum  concentration_ratio  \\\n",
      "count    6842.000000  6.724000e+03  6.724000e+03          6586.000000   \n",
      "unique           NaN           NaN           NaN                  NaN   \n",
      "top              NaN           NaN           NaN                  NaN   \n",
      "freq             NaN           NaN           NaN                  NaN   \n",
      "mean        4.631102  7.136776e+08  4.051506e+05             0.782903   \n",
      "std         5.493014  7.438256e+09  4.121668e+06             0.267426   \n",
      "min         0.000000 -9.151116e+07 -2.317333e+06            -2.591840   \n",
      "25%         1.000000  5.107611e+05  0.000000e+00             0.561855   \n",
      "50%         2.000000  4.221523e+06  0.000000e+00             0.965881   \n",
      "75%         6.000000  3.987106e+07  2.531545e+04             1.000000   \n",
      "max        48.000000  4.780234e+11  2.215373e+08             2.983529   \n",
      "\n",
      "        country  region  log_total_aum  log_median_aum  \n",
      "count      6842    6842    6724.000000     6724.000000  \n",
      "unique       34      16            NaN             NaN  \n",
      "top      FRANCE  FRANCE            NaN             NaN  \n",
      "freq       2631    2643            NaN             NaN  \n",
      "mean        NaN     NaN      15.046065        4.392450  \n",
      "std         NaN     NaN       4.320148        5.462132  \n",
      "min         NaN     NaN       0.000000        0.000000  \n",
      "25%         NaN     NaN      13.143657        0.000000  \n",
      "50%         NaN     NaN      15.255707        0.000000  \n",
      "75%         NaN     NaN      17.501160       10.139210  \n",
      "max         NaN     NaN      26.892926       19.216101  \n",
      "\n",
      "[11 rows x 24 columns]\n"
     ]
    }
   ],
   "source": [
    "# 1. Merge behavior (flows) with static diversification (stocks)\n",
    "client_master = client_behavior.merge(\n",
    "    account_static,\n",
    "    on=\"Registrar Account - ID\",\n",
    "    how=\"left\"\n",
    ")\n",
    "\n",
    "# 2. Add geographic info\n",
    "client_master = client_master.merge(\n",
    "    geo,\n",
    "    on=\"Registrar Account - ID\",\n",
    "    how=\"left\"\n",
    ")\n",
    "\n",
    "# 3. Create additional engineered features\n",
    "client_master[\"log_total_aum\"] = np.log1p(client_master[\"total_aum\"].clip(lower=0))\n",
    "client_master[\"log_median_aum\"] = np.log1p(client_master[\"median_aum\"].clip(lower=0))\n",
    "\n",
    "\n",
    "# 4. Replace NaN flow volatility with 0 (inactive accounts)\n",
    "client_master[\"flow_volatility\"] = client_master[\"flow_volatility\"].fillna(0)\n",
    "\n",
    "# 5. Fill missing diversification metrics with 0 (for accounts without stocks)\n",
    "client_master[[\"n_isin_held\",\"n_funds_held\",\"n_asset_types\",\"n_strategies\"]] = \\\n",
    "    client_master[[\"n_isin_held\",\"n_funds_held\",\"n_asset_types\",\"n_strategies\"]].fillna(0)\n",
    "\n",
    "# 6. Fill missing geography as “UNKNOWN”\n",
    "client_master[\"country\"] = client_master[\"country\"].fillna(\"UNKNOWN\")\n",
    "client_master[\"region\"] = client_master[\"region\"].fillna(\"UNKNOWN\")\n",
    "\n",
    "# 7. Export\n",
    "client_master.to_csv(\"client_master.csv\", index=False)\n",
    "\n",
    "print(client_master.head())\n",
    "print(client_master.describe(include='all'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb1e98a5-6ab4-4371-ba45-6558ff38c839",
   "metadata": {},
   "source": [
    "Détection des ruptures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "6bdd8077-c8e0-451d-a7b8-15a2705ad196",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- 1. PREPARE STOCKS ---\n",
    "stocks_clean = stocks[[\n",
    "    \"Registrar Account - ID\", \"Product - Isin\", \n",
    "    \"Centralisation Date\", \"Quantity - AUM\"\n",
    "]].copy()\n",
    "\n",
    "stocks_clean[\"Centralisation Date\"] = pd.to_datetime(stocks_clean[\"Centralisation Date\"])\n",
    "stocks_clean = stocks_clean.sort_values([\"Registrar Account - ID\", \"Product - Isin\", \"Centralisation Date\"])\n",
    "\n",
    "# --- 2. PREPARE FLOWS ---\n",
    "flows_clean = flows[[\n",
    "    \"Registrar Account - ID\", \"Product - Isin\", \n",
    "    \"Centralisation Date\", \"Quantity - NetFlows\"\n",
    "]].copy()\n",
    "\n",
    "flows_clean[\"Centralisation Date\"] = pd.to_datetime(flows_clean[\"Centralisation Date\"])\n",
    "\n",
    "# Aggregate flows per day to avoid duplicates\n",
    "flows_clean = flows_clean.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\", \"Centralisation Date\"]\n",
    ")[\"Quantity - NetFlows\"].sum().reset_index()\n",
    "\n",
    "# --- 3. MERGE STOCKS WITH FLOWS ---\n",
    "merged = stocks_clean.merge(\n",
    "    flows_clean,\n",
    "    on=[\"Registrar Account - ID\", \"Product - Isin\", \"Centralisation Date\"],\n",
    "    how=\"left\"\n",
    ")\n",
    "\n",
    "merged[\"Quantity - NetFlows\"] = merged[\"Quantity - NetFlows\"].fillna(0)\n",
    "\n",
    "# --- 4. SHIFT STOCKS TO COMPARE t vs t+1 ---\n",
    "merged[\"prev_stock\"] = merged.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\"]\n",
    ")[\"Quantity - AUM\"].shift(1)\n",
    "\n",
    "# SHIFT NET FLOWS FROM PREVIOUS DATE\n",
    "merged[\"prev_netflows\"] = merged.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\"]\n",
    ")[\"Quantity - NetFlows\"].shift(1).fillna(0)\n",
    "\n",
    "# Expected stock\n",
    "merged[\"expected_stock\"] = merged[\"prev_stock\"] + merged[\"prev_netflows\"]\n",
    "\n",
    "# --- 5. COMPUTE GAP BETWEEN EXPECTED AND REAL ---\n",
    "merged[\"gap\"] = merged[\"Quantity - AUM\"] - merged[\"expected_stock\"]\n",
    "\n",
    "# tolerance for numerical noise\n",
    "TOL = 1e-6\n",
    "merged[\"rupture_flag\"] = (merged[\"prev_stock\"].notna()) & (merged[\"gap\"].abs() > TOL)\n",
    "\n",
    "# --- 6. AGGREGATE BY CLIENT TO DETECT BIG ISSUES ---\n",
    "rupture_summary = merged.groupby(\"Registrar Account - ID\").agg(\n",
    "    n_ruptures=(\"rupture_flag\", \"sum\"),\n",
    "    total_obs=(\"rupture_flag\", \"count\"),\n",
    "    rupture_ratio=(\"rupture_flag\", \"mean\"),\n",
    "    max_gap=(\"gap\", lambda x: x.abs().max())\n",
    ").reset_index()\n",
    "\n",
    "# Sort by biggest anomalies\n",
    "rupture_summary = rupture_summary.sort_values(\"rupture_ratio\", ascending=False)\n",
    "\n",
    "rupture_summary.head(10)\n",
    "\n",
    "rupture_summary_asc = rupture_summary.sort_values(\"rupture_ratio\", ascending=True)\n",
    "rupture_summary_asc.to_csv('rupture.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "72332a7e-0ab0-474b-aac7-b52ebbba7a8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 44300 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43200 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 45752 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 46356 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 43796 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_account('200001928')\n",
    "plot_account('366351')\n",
    "plot_account('365966')\n",
    "plot_account('365568')\n",
    "plot_account('200129601')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "31407450-a833-4fce-8b0b-dba1b1de585f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Registrar Account - ID</th>\n",
       "      <th>Product - Isin</th>\n",
       "      <th>n_ruptures</th>\n",
       "      <th>obs</th>\n",
       "      <th>rupture_ratio</th>\n",
       "      <th>max_gap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17027</th>\n",
       "      <td>200127410</td>\n",
       "      <td>FR0010135103</td>\n",
       "      <td>434</td>\n",
       "      <td>436</td>\n",
       "      <td>0.995413</td>\n",
       "      <td>295985.420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17029</th>\n",
       "      <td>200127410</td>\n",
       "      <td>FR0010148981</td>\n",
       "      <td>317</td>\n",
       "      <td>319</td>\n",
       "      <td>0.993730</td>\n",
       "      <td>67134.706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68901</th>\n",
       "      <td>PRIVATE CLIENT</td>\n",
       "      <td>LU0992630599</td>\n",
       "      <td>154</td>\n",
       "      <td>155</td>\n",
       "      <td>0.993548</td>\n",
       "      <td>529752.634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39099</th>\n",
       "      <td>366441</td>\n",
       "      <td>FR0010135103</td>\n",
       "      <td>142</td>\n",
       "      <td>143</td>\n",
       "      <td>0.993007</td>\n",
       "      <td>439160.588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39101</th>\n",
       "      <td>366441</td>\n",
       "      <td>FR0010148981</td>\n",
       "      <td>142</td>\n",
       "      <td>143</td>\n",
       "      <td>0.993007</td>\n",
       "      <td>86246.897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3083</th>\n",
       "      <td>200001928</td>\n",
       "      <td>LU0992624949</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>110779.418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2080</th>\n",
       "      <td>200001349</td>\n",
       "      <td>FR0010149120</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>23881.992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3084</th>\n",
       "      <td>200001928</td>\n",
       "      <td>LU0992625839</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>12675.630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31744</th>\n",
       "      <td>365095</td>\n",
       "      <td>FR0010149120</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>22990.942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2085</th>\n",
       "      <td>200001349</td>\n",
       "      <td>FR0010149302</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>412.499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65735</th>\n",
       "      <td>422329</td>\n",
       "      <td>FR0010306142</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>18289.694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3129</th>\n",
       "      <td>200001939</td>\n",
       "      <td>LU0592698954</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>57179.957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090</th>\n",
       "      <td>200001349</td>\n",
       "      <td>FR0011269083</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>29403.491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2086</th>\n",
       "      <td>200001349</td>\n",
       "      <td>FR0010306142</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>9417.847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65730</th>\n",
       "      <td>422329</td>\n",
       "      <td>FR0010149120</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>20342.726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2108</th>\n",
       "      <td>200001349</td>\n",
       "      <td>LU0336083497</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>592.246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65618</th>\n",
       "      <td>422310</td>\n",
       "      <td>FR0010149120</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>6006.071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65613</th>\n",
       "      <td>422310</td>\n",
       "      <td>FR0010135103</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>4258.656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31804</th>\n",
       "      <td>365096</td>\n",
       "      <td>FR0010149120</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>844.519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31800</th>\n",
       "      <td>365096</td>\n",
       "      <td>FR0010148981</td>\n",
       "      <td>129</td>\n",
       "      <td>130</td>\n",
       "      <td>0.992308</td>\n",
       "      <td>962.783</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Registrar Account - ID Product - Isin  n_ruptures  obs  rupture_ratio  \\\n",
       "17027              200127410   FR0010135103         434  436       0.995413   \n",
       "17029              200127410   FR0010148981         317  319       0.993730   \n",
       "68901         PRIVATE CLIENT   LU0992630599         154  155       0.993548   \n",
       "39099                 366441   FR0010135103         142  143       0.993007   \n",
       "39101                 366441   FR0010148981         142  143       0.993007   \n",
       "3083               200001928   LU0992624949         129  130       0.992308   \n",
       "2080               200001349   FR0010149120         129  130       0.992308   \n",
       "3084               200001928   LU0992625839         129  130       0.992308   \n",
       "31744                 365095   FR0010149120         129  130       0.992308   \n",
       "2085               200001349   FR0010149302         129  130       0.992308   \n",
       "65735                 422329   FR0010306142         129  130       0.992308   \n",
       "3129               200001939   LU0592698954         129  130       0.992308   \n",
       "2090               200001349   FR0011269083         129  130       0.992308   \n",
       "2086               200001349   FR0010306142         129  130       0.992308   \n",
       "65730                 422329   FR0010149120         129  130       0.992308   \n",
       "2108               200001349   LU0336083497         129  130       0.992308   \n",
       "65618                 422310   FR0010149120         129  130       0.992308   \n",
       "65613                 422310   FR0010135103         129  130       0.992308   \n",
       "31804                 365096   FR0010149120         129  130       0.992308   \n",
       "31800                 365096   FR0010148981         129  130       0.992308   \n",
       "\n",
       "          max_gap  \n",
       "17027  295985.420  \n",
       "17029   67134.706  \n",
       "68901  529752.634  \n",
       "39099  439160.588  \n",
       "39101   86246.897  \n",
       "3083   110779.418  \n",
       "2080    23881.992  \n",
       "3084    12675.630  \n",
       "31744   22990.942  \n",
       "2085      412.499  \n",
       "65735   18289.694  \n",
       "3129    57179.957  \n",
       "2090    29403.491  \n",
       "2086     9417.847  \n",
       "65730   20342.726  \n",
       "2108      592.246  \n",
       "65618    6006.071  \n",
       "65613    4258.656  \n",
       "31804     844.519  \n",
       "31800     962.783  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. Prepare stock dataset ISIN-by-ISIN\n",
    "stocks_isin = stocks[[\n",
    "    \"Registrar Account - ID\", \"Product - Isin\",\n",
    "    \"Centralisation Date\", \"Quantity - AUM\"\n",
    "]].copy()\n",
    "\n",
    "stocks_isin[\"Centralisation Date\"] = pd.to_datetime(stocks_isin[\"Centralisation Date\"])\n",
    "stocks_isin = stocks_isin.sort_values(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\", \"Centralisation Date\"]\n",
    ")\n",
    "\n",
    "# 2. Prepare flows dataset ISIN-by-ISIN\n",
    "flows_isin = flows[[\n",
    "    \"Registrar Account - ID\", \"Product - Isin\",\n",
    "    \"Centralisation Date\", \"Quantity - NetFlows\"\n",
    "]].copy()\n",
    "\n",
    "flows_isin[\"Centralisation Date\"] = pd.to_datetime(flows_isin[\"Centralisation Date\"])\n",
    "\n",
    "flows_isin = flows_isin.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\", \"Centralisation Date\"]\n",
    ")[\"Quantity - NetFlows\"].sum().reset_index()\n",
    "\n",
    "# 3. Merge stocks & flows ISIN-by-ISIN\n",
    "merged_isin = stocks_isin.merge(\n",
    "    flows_isin,\n",
    "    on=[\"Registrar Account - ID\", \"Product - Isin\", \"Centralisation Date\"],\n",
    "    how=\"left\"\n",
    ")\n",
    "\n",
    "merged_isin[\"Quantity - NetFlows\"] = merged_isin[\"Quantity - NetFlows\"].fillna(0)\n",
    "\n",
    "# 4. Compute expected stock per ISIN for each account\n",
    "merged_isin[\"prev_stock\"] = merged_isin.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\"]\n",
    ")[\"Quantity - AUM\"].shift(1)\n",
    "\n",
    "merged_isin[\"prev_netflows\"] = merged_isin.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\"]\n",
    ")[\"Quantity - NetFlows\"].shift(1).fillna(0)\n",
    "\n",
    "merged_isin[\"expected_stock\"] = merged_isin[\"prev_stock\"] + merged_isin[\"prev_netflows\"]\n",
    "\n",
    "# 5. Detect ruptures ISIN-by-ISIN (no aggregation)\n",
    "TOL = 1e-6\n",
    "merged_isin[\"gap\"] = merged_isin[\"Quantity - AUM\"] - merged_isin[\"expected_stock\"]\n",
    "merged_isin[\"rupture_flag\"] = (\n",
    "    merged_isin[\"prev_stock\"].notna()\n",
    "    & (merged_isin[\"gap\"].abs() > TOL)\n",
    ")\n",
    "\n",
    "# 6. Summarize ruptures per (Account, ISIN)\n",
    "rupture_isin_summary = merged_isin.groupby(\n",
    "    [\"Registrar Account - ID\", \"Product - Isin\"]\n",
    ").agg(\n",
    "    n_ruptures=(\"rupture_flag\", \"sum\"),\n",
    "    obs=(\"rupture_flag\", \"count\"),\n",
    "    rupture_ratio=(\"rupture_flag\", \"mean\"),\n",
    "    max_gap=(\"gap\", lambda x: x.abs().max())\n",
    ").reset_index()\n",
    "\n",
    "# Sort by worst ISIN trajectories\n",
    "rupture_isin_summary = rupture_isin_summary.sort_values(\n",
    "    \"rupture_ratio\", ascending=False\n",
    ")\n",
    "\n",
    "rupture_isin_summary.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "baa4b6cd-887d-45a6-af27-253a9aa8710f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Strong ruptures (ratio > 0.5 OR huge max_gap)\n",
    "strong = rupture_isin_summary[\n",
    "    (rupture_isin_summary[\"rupture_ratio\"] > 0.5)\n",
    "    | (rupture_isin_summary[\"max_gap\"] > 50000)\n",
    "]\n",
    "\n",
    "def find_successors(account_id, isin, window_days=15):\n",
    "    # Extract rupture dates for (account_id, isin)\n",
    "    ruptures = merged_isin[\n",
    "        (merged_isin[\"Registrar Account - ID\"] == account_id)\n",
    "        & (merged_isin[\"Product - Isin\"] == isin)\n",
    "        & (merged_isin[\"rupture_flag\"])\n",
    "    ][\"Centralisation Date\"].unique()\n",
    "\n",
    "    if len(ruptures) == 0:\n",
    "        return []\n",
    "\n",
    "    candidates = []\n",
    "\n",
    "    for rupture_date in ruptures:\n",
    "        start = rupture_date - pd.Timedelta(days=window_days)\n",
    "        end = rupture_date + pd.Timedelta(days=window_days)\n",
    "\n",
    "        # Look for accounts with strong positive jump at the same time\n",
    "        window_df = merged_isin[\n",
    "            (merged_isin[\"Centralisation Date\"] >= start)\n",
    "            & (merged_isin[\"Centralisation Date\"] <= end)\n",
    "            & (merged_isin[\"Product - Isin\"] == isin)\n",
    "        ]\n",
    "\n",
    "        # Look for positive gap (jump)\n",
    "        pos_jumps = window_df[window_df[\"gap\"] > 0]\n",
    "\n",
    "        candidates.extend(pos_jumps[\"Registrar Account - ID\"].unique())\n",
    "\n",
    "    # Remove self\n",
    "    candidates = [c for c in candidates if c != account_id]\n",
    "\n",
    "    return list(set(candidates))\n",
    "\n",
    "find_successors(\"200129601\", \"FR0010135103\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0b834da2-f781-476d-84a6-aebb38fb8dac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 213636 characters]",
      "text/plain": [
       "<Figure size 1400x700 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df = merged_isin.copy()\n",
    "\n",
    "# Ajouter année / mois\n",
    "df[\"year\"] = df[\"Centralisation Date\"].dt.year\n",
    "df[\"month\"] = df[\"Centralisation Date\"].dt.month\n",
    "\n",
    "# 1. Nombre total de lignes par mois\n",
    "total = df.groupby([\"year\", \"month\"]).size().reset_index(name=\"total_lines\")\n",
    "\n",
    "# 2. Nombre de ruptures par mois\n",
    "ruptures = df[df[\"rupture_flag\"]].groupby([\"year\", \"month\"]).size().reset_index(name=\"n_ruptures\")\n",
    "\n",
    "# 3. Merge pour obtenir total + ruptures\n",
    "ratio = total.merge(ruptures, on=[\"year\",\"month\"], how=\"left\")\n",
    "ratio[\"n_ruptures\"] = ratio[\"n_ruptures\"].fillna(0)\n",
    "\n",
    "# 4. Proportion (en %)\n",
    "ratio[\"rupture_ratio\"] = ratio[\"n_ruptures\"] / ratio[\"total_lines\"]\n",
    "\n",
    "# 5. Pivot pour heatmap\n",
    "heatmap_ratio = ratio.pivot(index=\"year\", columns=\"month\", values=\"rupture_ratio\").fillna(0)\n",
    "\n",
    "# 6. Plot\n",
    "plt.figure(figsize=(14, 7))\n",
    "sns.heatmap(\n",
    "    heatmap_ratio, \n",
    "    cmap=\"Reds\",\n",
    "    linewidths=.3,\n",
    "    linecolor=\"grey\",\n",
    "    annot=True,\n",
    "    fmt=\".2%\",\n",
    "    cbar_kws={'label': 'Proportion de ruptures'}\n",
    ")\n",
    "\n",
    "plt.title(\"Heatmap de la proportion de ruptures (par année et mois)\", fontsize=16)\n",
    "plt.xlabel(\"Mois\")\n",
    "plt.ylabel(\"Année\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "86d2a91c-d8d8-416c-8dc4-dc3f4ae7ca90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: plotly in /opt/python/lib/python3.13/site-packages (6.3.1)\n",
      "Requirement already satisfied: narwhals>=1.15.1 in /opt/python/lib/python3.13/site-packages (from plotly) (2.8.0)\n",
      "Requirement already satisfied: packaging in /opt/python/lib/python3.13/site-packages (from plotly) (25.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "customdata": {
          "bdata": "xU7sxE5sWEAAAAAAAMBvQAAAAAAAQHBA198e2OhdR0AAAAAAABB9QAAAAAAAGI9AlH+9dRU2RUAAAAAAAJjBQAAAAACAvNRAHFtG3vbHQ0AAAAAAAICOQAAAAAAARqNA0F5CewntOUAAAAAAAAA8QAAAAAAAAFtANJzUfXlsOUAAAAAAAAAuQAAAAAAAgE1A4SpTwHwgOUAAAAAAAMBYQAAAAAAAoHhAZluhtRLZN0AAAAAAYMHrQAAAAACgGA1BXn0WV0YbNkAAAAAAuMQCQQAAAACYOSVBPxKbCsIRNkAAAAAAYM3zQAAAAACMbhZBzafGIeilM0AAAAAAQEn3QAAAAAAIoR1By3uP6ZWYMUAAAAAA4MPqQAAAAAB8AxNBNeF+QPqVLkAAAAAAwBLaQAAAAADATwVBb8JdUw+UKkAAAAAAwKHWQAAAAACwSQVBjhDbtOLeKUAAAAAAAOBxQAAAAAAARqFARQMgZFR6KEAAAAAAAMCMQAAAAAAAXb1ATK4gJleQJ0AAAAAAALCbQAAAAAAAYM1A05E9A1HpJkAAAAAAAKBmQAAAAAAAsJhAKTDp1DzIJUAAAAAAAAA2QAAAAAAAQGlAj8DhOzKPJEAAAAAA0FELQQAAAAA1nEBBIPw39b67I0AAAAAAALagQAAAAADAK9VA1sr0lOakIUAAAAAAALCAQAAAAAAApbdAL/vifHPjIEAAAAAAAPixQAAAAABgmepATufRh/2UHUAAAAAAAABXQAAAAAAAcJNAyEIWspCFHEAAAAAAAIBEQAAAAAAA+IFALDDozo3/EUAAAAAAAHCbQAAAAABADuNAYQb4iZOHDkAAAAAAAIBIQAAAAAAAEJRAawqzdyc0DkAAAAAAAFiiQAAAAADgXe5AfscrRvyXDUAAAAAAAIBOQAAAAAAAxJlAeeJY/KtzBEAAAAAAAIBFQAAAAAAASJpAyp2ihTMgBEAAAAAAAABcQAAAAAAAZLFAAAAAAADAAkAAAAAAAGBoQAAAAAAAQMBAC0fmBDyf/j8AAAAAAAAQQAAAAAAAIGpAmb5KGJsF9T8AAAAAAAAiQAAAAAAAaIVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRA",
          "dtype": "f8",
          "shape": "37, 3"
         },
         "hovertemplate": "Pays=%{x}<br>% de ruptures=%{customdata[0]:.2f}<br>Nb de ruptures=%{customdata[1]}<br>Nb d'observations=%{customdata[2]}<extra></extra>",
         "legendgroup": "",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "",
         "orientation": "v",
         "showlegend": false,
         "textposition": "auto",
         "type": "bar",
         "x": [
          "JAPAN",
          "TAIWAN",
          "SWEDEN",
          "DENMARK",
          "SOUTH AFRICA",
          "LITHUANIA",
          "FINLAND",
          "GERMANY",
          "ITALY",
          "LUXEMBOURG",
          "SWITZERLAND",
          "SPAIN",
          "BELGIUM",
          "UNITED KINGDOM",
          "MAURITIUS",
          "PORTUGAL",
          "AUSTRIA",
          "UNITED ARAB EMIRATES",
          "CZECH REPUBLIC",
          "FRANCE",
          "MONACO",
          "ISRAEL",
          "NETHERLANDS",
          "GREECE",
          "CANADA",
          "LATAM",
          "INTERNATIONAL",
          "US OFFSHORE",
          "MALTA",
          "IRELAND",
          "UNKNOWN",
          "SINGAPORE",
          "POLAND",
          "HONG KONG",
          "NORWAY",
          "SOUTH KOREA",
          "UNITED STATES"
         ],
         "xaxis": "x",
         "y": {
          "bdata": "L/RCL/RC7z/W1t8e2OjdP6kyY96BJts/gO/U1N9R2T97Ce0ltJfQP5zUfXlsRdA/ryWHuMoU0D/+dCBZfobOP+ndyiffS8w/y1RqsbA/zD/yMlDZRybJP2DwmJvehcY/S+xvhS+Twz89SR0rmQLBP624d5KljsA/WGCP5uZUvz/S/YVPZSm+P3SHbeWaU70/uetni5Xhuz+iUrAj5FC6P83wKBs8Qrk/3w1YBqKVtj+OGOV2BJ61P2UjU7227rI/V8T5cQBBsj++60dGqwmnP8NLYQb4iaM/OsmlI5RUoz8T5pZ0ofCiP8ScFfGoLZo/AqHkTtHCmT8AAAAAAACYP2N/GCwcmZM/40X5mXToij8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
          "dtype": "f8"
         },
         "yaxis": "y"
        }
       ],
       "layout": {
        "bargap": 0.2,
        "barmode": "relative",
        "legend": {
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermap": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermap"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Proportion de ruptures par pays (avec volumes au survol)"
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "tickangle": -45,
         "title": {
          "text": "Pays"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "tickformat": ".1%",
         "title": {
          "text": "Proportion de ruptures"
         }
        }
       }
      },
      "image/png": "[encoded data, 113876 characters]"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly.express as px\n",
    "\n",
    "# On ajoute une colonne en % pour l’affichage\n",
    "country_stats_plot = country_stats.copy()\n",
    "country_stats_plot[\"rupture_pct\"] = country_stats_plot[\"rupture_ratio\"] * 100\n",
    "\n",
    "# Tri décroissant par proportion de ruptures\n",
    "country_stats_plot = country_stats_plot.sort_values(\"rupture_ratio\", ascending=False)\n",
    "\n",
    "fig = px.bar(\n",
    "    country_stats_plot,\n",
    "    x=\"country\",\n",
    "    y=\"rupture_ratio\",\n",
    "    hover_data={\n",
    "        \"rupture_pct\": ':.2f',\n",
    "        \"ruptures\": True,\n",
    "        \"total_obs\": True,\n",
    "        \"rupture_ratio\": False,  # on cache la version décimale\n",
    "    },\n",
    "    labels={\n",
    "        \"country\": \"Pays\",\n",
    "        \"rupture_ratio\": \"Proportion de ruptures\",\n",
    "        \"rupture_pct\": \"% de ruptures\",\n",
    "        \"ruptures\": \"Nb de ruptures\",\n",
    "        \"total_obs\": \"Nb d'observations\"\n",
    "    },\n",
    "    title=\"Proportion de ruptures par pays (avec volumes au survol)\"\n",
    ")\n",
    "\n",
    "# Format en %\n",
    "fig.update_yaxes(tickformat=\".1%\")\n",
    "\n",
    "fig.update_layout(\n",
    "    xaxis_tickangle=-45,\n",
    "    bargap=0.2\n",
    ")\n",
    "\n",
    "fig.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "95bc353d-e883-4989-aaca-1b3c9b51ee5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== BASIC NUMERIC STATS ===\n",
      "count    12501.000000\n",
      "mean         0.069464\n",
      "std          0.137360\n",
      "min          0.000000\n",
      "1%           0.000000\n",
      "5%           0.000000\n",
      "10%          0.000000\n",
      "25%          0.000000\n",
      "50%          0.013699\n",
      "75%          0.069767\n",
      "90%          0.213497\n",
      "95%          0.333333\n",
      "99%          0.720000\n",
      "max          0.992308\n",
      "Name: rupture_ratio, dtype: float64\n",
      "rupture_bucket\n",
      "0–0.1%       12\n",
      "0.1–1%      619\n",
      "1–5%       3078\n",
      "5–10%      1189\n",
      "10–25%     1463\n",
      "25–50%      703\n",
      "50–100%     311\n",
      "0%         5126\n",
      "Name: count, dtype: int64\n",
      "\n",
      "=== DISTRIBUTION (PERCENT) ===\n",
      "rupture_bucket\n",
      "0–0.1%      0.10\n",
      "0.1–1%      4.95\n",
      "1–5%       24.62\n",
      "5–10%       9.51\n",
      "10–25%     11.70\n",
      "25–50%      5.62\n",
      "50–100%     2.49\n",
      "0%         41.00\n",
      "Name: count, dtype: float64\n",
      "\n",
      "Comptes avec 0 rupture = 5126 (41.00%)\n",
      "Comptes avec rupture_ratio > 75% = 99 (0.79%)\n",
      "Comptes avec rupture_ratio > 10% = 2477 (19.81%)\n"
     ]
    }
   ],
   "source": [
    "rs = rupture_summary.copy()\n",
    "\n",
    "# 1. Stats numériques classiques\n",
    "print(\"\\n=== BASIC NUMERIC STATS ===\")\n",
    "print(rs[\"rupture_ratio\"].describe(percentiles=[0.01, 0.05, 0.10, 0.25, 0.5, 0.75, 0.90, 0.95, 0.99]))\n",
    "\n",
    "\n",
    "# 2. Distribution par classes (bins)\n",
    "\n",
    "rs[\"rupture_bucket\"] = pd.cut(\n",
    "    rs[\"rupture_ratio\"],\n",
    "    bins=[0, 0.001, 0.01, 0.05, 0.10, 0.25, 0.50, 1.01],\n",
    "    labels=[\n",
    "        \"0–0.1%\",\n",
    "        \"0.1–1%\",\n",
    "        \"1–5%\",\n",
    "        \"5–10%\",\n",
    "        \"10–25%\",\n",
    "        \"25–50%\",\n",
    "        \"50–100%\"\n",
    "    ],\n",
    "    include_lowest=True\n",
    ")\n",
    "\n",
    "# Ajouter la catégorie \"0%\"\n",
    "rs[\"rupture_bucket\"] = rs[\"rupture_bucket\"].cat.add_categories(\"0%\")\n",
    "\n",
    "# Remplacer les 0% exacts\n",
    "rs.loc[rs[\"rupture_ratio\"] == 0, \"rupture_bucket\"] = \"0%\"\n",
    "\n",
    "bucket_counts = rs[\"rupture_bucket\"].value_counts().sort_index()\n",
    "print(bucket_counts)\n",
    "\n",
    "\n",
    "# 3. Pourcentages\n",
    "bucket_percent = (bucket_counts / len(rs) * 100).round(2)\n",
    "\n",
    "print(\"\\n=== DISTRIBUTION (PERCENT) ===\")\n",
    "print(bucket_percent)\n",
    "\n",
    "\n",
    "# 4. Nombre de comptes totalement propres\n",
    "no_rupture = (rs[\"n_ruptures\"] == 0).sum()\n",
    "print(f\"\\nComptes avec 0 rupture = {no_rupture} ({no_rupture/len(rs)*100:.2f}%)\")\n",
    "\n",
    "# 5. Comptes extrêmement problématiques\n",
    "severe = (rs[\"rupture_ratio\"] > 0.75).sum()\n",
    "print(f\"Comptes avec rupture_ratio > 75% = {severe} ({severe/len(rs)*100:.2f}%)\")\n",
    "\n",
    "medium = (rs[\"rupture_ratio\"] > 0.10).sum()\n",
    "print(f\"Comptes avec rupture_ratio > 10% = {medium} ({medium/len(rs)*100:.2f}%)\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "425b36d0-c92a-4405-be28-35b1fc292fec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nombre total de ruptures en 2021 : 71090\n",
      "\n",
      "=== RUPTURES 2021 — POSITIVES vs NEGATIVES ===\n",
      "gap_type\n",
      "negative    38092\n",
      "positive    32998\n",
      "Name: count, dtype: int64\n",
      "\n",
      "(%)\n",
      "gap_type\n",
      "negative    53.58%\n",
      "positive    46.42%\n",
      "Name: proportion, dtype: object\n",
      "\n",
      "=== STATISTIQUES DES GAPS ===\n",
      "            count         mean            std           min        25%  \\\n",
      "gap_type                                                                 \n",
      "negative  38092.0 -7150.538497   99000.868201 -5.257530e+06 -438.04975   \n",
      "positive  32998.0  9865.849003  139488.218647  1.000000e-05   22.76075   \n",
      "\n",
      "               50%         75%           max  \n",
      "gap_type                                      \n",
      "negative  -57.9725    -2.74375 -6.000000e-06  \n",
      "positive  159.3720  1007.14050  1.814960e+07  \n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41860 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# --- 1. Filtres de base ---\n",
    "merged[\"year\"] = merged[\"Centralisation Date\"].dt.year\n",
    "\n",
    "# Filtrer uniquement l'année 2021\n",
    "ruptures_2021 = merged[(merged[\"year\"] == 2021) & (merged[\"rupture_flag\"] == True)].copy()\n",
    "\n",
    "print(\"Nombre total de ruptures en 2021 :\", len(ruptures_2021))\n",
    "\n",
    "# --- 2. Classification du type de gap ---\n",
    "ruptures_2021[\"gap_type\"] = np.where(ruptures_2021[\"gap\"] > 0, \"positive\", \"negative\")\n",
    "\n",
    "# --- 3. Statistiques globales ---\n",
    "gap_counts = ruptures_2021[\"gap_type\"].value_counts()\n",
    "gap_percent = ruptures_2021[\"gap_type\"].value_counts(normalize=True) * 100\n",
    "\n",
    "print(\"\\n=== RUPTURES 2021 — POSITIVES vs NEGATIVES ===\")\n",
    "print(gap_counts)\n",
    "print(\"\\n(%)\")\n",
    "print(gap_percent.map(lambda x: f\"{x:.2f}%\"))\n",
    "\n",
    "# --- 4. Intensité des écarts ---\n",
    "intensity_stats = ruptures_2021.groupby(\"gap_type\")[\"gap\"].describe()\n",
    "print(\"\\n=== STATISTIQUES DES GAPS ===\")\n",
    "print(intensity_stats)\n",
    "\n",
    "# --- 5. Visualisation rapide ---\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(10,5))\n",
    "sns.histplot(data=ruptures_2021, x=\"gap\", hue=\"gap_type\", bins=80, kde=True)\n",
    "plt.xlim(-merged[\"gap\"].abs().max(), merged[\"gap\"].abs().max())\n",
    "plt.title(\"Distribution des gaps de rupture en 2021\")\n",
    "plt.xlabel(\"Gap (AUM_{t} − Expected AUM_{t})\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "df9e0005-93f2-4885-baef-2e54921a42f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== RUPTURE SUMMARY (in %) ===\n",
      "             total_obs  rupture_count  rupture_ratio\n",
      "period                                              \n",
      "after_2021      935390         242411          25.92\n",
      "before_2021    3414943         428083          12.54\n",
      "during_2021     529964          71090          13.41\n",
      "\n",
      "=== GAP POSITIVE / NEGATIVE DISTRIBUTION (in %) ===\n",
      "period       period       gap_type\n",
      "after_2021   after_2021   negative    60.163524\n",
      "                          positive    39.836476\n",
      "before_2021  before_2021  negative    62.273905\n",
      "                          positive    37.726095\n",
      "during_2021  during_2021  negative    53.582782\n",
      "                          positive    46.417218\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# --- 1. ADD YEAR ---\n",
    "merged[\"year\"] = merged[\"Centralisation Date\"].dt.year\n",
    "\n",
    "# --- 2. DEFINE PERIODS ---\n",
    "conditions = [\n",
    "    merged[\"year\"] < 2021,\n",
    "    merged[\"year\"] == 2021,\n",
    "    merged[\"year\"] > 2021\n",
    "]\n",
    "\n",
    "period_labels = [\"before_2021\", \"during_2021\", \"after_2021\"]\n",
    "\n",
    "merged[\"period\"] = np.select(\n",
    "    conditions,\n",
    "    period_labels,\n",
    "    default=\"unknown\"\n",
    ")\n",
    "\n",
    "# --- 3. CREATE GAP TYPE & FILTER ONLY RUPTURES ---\n",
    "merged[\"gap_type\"] = np.where(\n",
    "    merged[\"gap\"] > 0, \"positive\",\n",
    "    np.where(merged[\"gap\"] < 0, \"negative\", \"zero\")\n",
    ")\n",
    "\n",
    "ruptures = merged[merged[\"rupture_flag\"] == True].copy()\n",
    "\n",
    "# --- 4. TOTAL OBS PER PERIOD ---\n",
    "total_obs = merged.groupby(\"period\").size().rename(\"total_obs\")\n",
    "\n",
    "# --- 5. TOTAL RUPTURES PER PERIOD ---\n",
    "rupture_counts = ruptures.groupby(\"period\").size().rename(\"rupture_count\")\n",
    "\n",
    "# --- 6. PROPORTION OF RUPTURES ---\n",
    "rupture_ratio = (rupture_counts / total_obs).rename(\"rupture_ratio\")\n",
    "\n",
    "# --- 7. POSITIVE / NEGATIVE GAPS (% among ruptures) ---\n",
    "gap_dist = (\n",
    "    ruptures.groupby([\"period\", \"gap_type\"])\n",
    "    .size()\n",
    "    .groupby(level=0)\n",
    "    .apply(lambda x: (x / x.sum()) * 100)   # % par période\n",
    ")\n",
    "\n",
    "\n",
    "# --- 8. MERGE AND DISPLAY ---\n",
    "summary = pd.concat([total_obs, rupture_counts, rupture_ratio], axis=1)\n",
    "summary[\"rupture_ratio\"] = (summary[\"rupture_ratio\"] * 100).round(2)\n",
    "\n",
    "print(\"\\n=== RUPTURE SUMMARY (in %) ===\")\n",
    "print(summary)\n",
    "\n",
    "print(\"\\n=== GAP POSITIVE / NEGATIVE DISTRIBUTION (in %) ===\")\n",
    "print(gap_dist)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45a2a748-014d-44a0-87a9-a26bcbc10410",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
