new weekly data for the traders

data/unknown_daily_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1db631b6cc5b6ff1aadd6ce3285dc032fe79c83cd14bb2c1cb1fa7b7917e61b0
-size 25139
+oid sha256:b1ca416d2d0eeab63185e637d4acbbd0beb6e476a8a8fd33d77bb1b6b5a25e4a
+size 55580

data/unknown_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9be58c1de361e7c9df25ae05c54b77f6a6417e58e19d5d6ef8bd37516da1f70e
-size 198407
+oid sha256:5c237838416f8339b145719e5b370df1d9763099f9e5fa3e75d4d69053e5d311
+size 200722

data/weekly_mech_calls.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e321e63d58f312fe2769880d9ec5ec9fba24229e427a514a3a9567936edbab5
-size 50009
+oid sha256:fcaf2cf4955d8bf73727d2e83aa959fe69b6ac99be801008d724b4810eea9c5f
+size 50599

notebooks/daily_data.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -20,7 +20,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -31,11 +31,11 @@
        "       'trade_fee_amount', 'outcomes_tokens_traded', 'current_answer',\n",
        "       'is_invalid', 'winning_trade', 'earnings', 'redeemed',\n",
        "       'redeemed_amount', 'num_mech_calls', 'mech_fee_amount', 'net_earnings',\n",
-       "       'roi', 'staking', 'nr_mech_calls'],\n",
+       "       'roi', 'staking'],\n",
        "      dtype='object')"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -44,6 +44,13 @@
     "all_trades.columns"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 32,
@@ -624,7 +631,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -633,7 +640,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -641,33 +648,33 @@
      "output_type": "stream",
      "text": [
       "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 9686 entries, 0 to 9685\n",
+      "RangeIndex: 7104 entries, 0 to 7103\n",
       "Data columns (total 21 columns):\n",
       " #   Column                  Non-Null Count  Dtype              \n",
       "---  ------                  --------------  -----              \n",
-      " 0   trader_address          9686 non-null   object             \n",
-      " 1   market_creator          9686 non-null   object             \n",
-      " 2   trade_id                9686 non-null   object             \n",
-      " 3   creation_timestamp      9686 non-null   datetime64[ns, UTC]\n",
-      " 4   title                   9686 non-null   object             \n",
-      " 5   market_status           9686 non-null   object             \n",
-      " 6   collateral_amount       9686 non-null   float64            \n",
-      " 7   outcome_index           9686 non-null   object             \n",
-      " 8   trade_fee_amount        9686 non-null   float64            \n",
-      " 9   outcomes_tokens_traded  9686 non-null   float64            \n",
-      " 10  current_answer          671 non-null    float64            \n",
-      " 11  is_invalid              9686 non-null   bool               \n",
-      " 12  winning_trade           671 non-null    object             \n",
-      " 13  earnings                9686 non-null   float64            \n",
-      " 14  redeemed                9686 non-null   bool               \n",
-      " 15  redeemed_amount         9686 non-null   int64              \n",
-      " 16  num_mech_calls          9686 non-null   int64              \n",
-      " 17  mech_fee_amount         9686 non-null   float64            \n",
-      " 18  net_earnings            9686 non-null   float64            \n",
-      " 19  roi                     9686 non-null   float64            \n",
-      " 20  staking                 9686 non-null   object             \n",
-      "dtypes: bool(2), datetime64[ns, UTC](1), float64(8), int64(2), object(8)\n",
-      "memory usage: 1.4+ MB\n"
+      " 0   trader_address          7104 non-null   object             \n",
+      " 1   market_creator          7104 non-null   object             \n",
+      " 2   trade_id                7104 non-null   object             \n",
+      " 3   creation_timestamp      7104 non-null   datetime64[ns, UTC]\n",
+      " 4   title                   7104 non-null   object             \n",
+      " 5   market_status           7104 non-null   object             \n",
+      " 6   collateral_amount       7104 non-null   float64            \n",
+      " 7   outcome_index           7104 non-null   object             \n",
+      " 8   trade_fee_amount        7104 non-null   float64            \n",
+      " 9   outcomes_tokens_traded  7104 non-null   float64            \n",
+      " 10  current_answer          0 non-null      object             \n",
+      " 11  is_invalid              7104 non-null   bool               \n",
+      " 12  winning_trade           0 non-null      object             \n",
+      " 13  earnings                7104 non-null   float64            \n",
+      " 14  redeemed                7104 non-null   bool               \n",
+      " 15  redeemed_amount         7104 non-null   float64            \n",
+      " 16  num_mech_calls          7104 non-null   int64              \n",
+      " 17  mech_fee_amount         7104 non-null   float64            \n",
+      " 18  net_earnings            7104 non-null   float64            \n",
+      " 19  roi                     7104 non-null   float64            \n",
+      " 20  staking                 7104 non-null   object             \n",
+      "dtypes: bool(2), datetime64[ns, UTC](1), float64(8), int64(1), object(9)\n",
+      "memory usage: 1.0+ MB\n"
      ]
     }
    ],
@@ -703,22 +710,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Timestamp('2024-12-02 17:30:05+0000', tz='UTC')"
+       "Timestamp('2025-01-07 09:01:05+0000', tz='UTC')"
       ]
      },
-     "execution_count": 28,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "max(all_trades_before.creation_timestamp)"
+    "min(all_trades_before.creation_timestamp)"
    ]
   },
   {

notebooks/divergence.ipynb

@@ -9,7 +9,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/cyberosa/.pyenv/versions/hf_dashboards/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "/Users/cyberosa/.pyenv/versions/3.12.2/envs/hf_dashboards/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
       "  from .autonotebook import tqdm as notebook_tqdm\n"
      ]
     }
@@ -26,13 +26,167 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
     "div_data = pd.read_parquet(\"../data/closed_markets_div.parquet\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>currentAnswer</th>\n",
+       "      <th>id</th>\n",
+       "      <th>openingTimestamp</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>opening_datetime</th>\n",
+       "      <th>first_outcome_prob</th>\n",
+       "      <th>second_outcome_prob</th>\n",
+       "      <th>kl_divergence</th>\n",
+       "      <th>off_by_perc</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0x5cae41dcc8a30d76a31b126c6f3b358b8d0131ca</td>\n",
+       "      <td>1731542400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-11-14 01:00:00</td>\n",
+       "      <td>0.3679</td>\n",
+       "      <td>0.6321</td>\n",
+       "      <td>0.999944</td>\n",
+       "      <td>63.21</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0xf8442bd26cd80d8447bb6203ededc44a77cd2a12</td>\n",
+       "      <td>1731542400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-11-14 01:00:00</td>\n",
+       "      <td>0.3368</td>\n",
+       "      <td>0.6632</td>\n",
+       "      <td>1.088266</td>\n",
+       "      <td>66.32</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0xde3a4b0d527013165b1b6b8aae051d223f8b770e</td>\n",
+       "      <td>1731542400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-11-14 01:00:00</td>\n",
+       "      <td>0.4468</td>\n",
+       "      <td>0.5532</td>\n",
+       "      <td>0.805644</td>\n",
+       "      <td>55.32</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0xccadef7757659ce271b209d647c2a51fabd88c77</td>\n",
+       "      <td>1731542400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-11-14 01:00:00</td>\n",
+       "      <td>0.6804</td>\n",
+       "      <td>0.3196</td>\n",
+       "      <td>0.385074</td>\n",
+       "      <td>31.96</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x78d0fc5884e74d87b0529e40da2b9490db60e731</td>\n",
+       "      <td>1731628800</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-11-15 01:00:00</td>\n",
+       "      <td>0.2358</td>\n",
+       "      <td>0.7642</td>\n",
+       "      <td>0.268926</td>\n",
+       "      <td>23.58</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  currentAnswer                                          id openingTimestamp  \\\n",
+       "0           yes  0x5cae41dcc8a30d76a31b126c6f3b358b8d0131ca       1731542400   \n",
+       "1           yes  0xf8442bd26cd80d8447bb6203ededc44a77cd2a12       1731542400   \n",
+       "2           yes  0xde3a4b0d527013165b1b6b8aae051d223f8b770e       1731542400   \n",
+       "3           yes  0xccadef7757659ce271b209d647c2a51fabd88c77       1731542400   \n",
+       "4            no  0x78d0fc5884e74d87b0529e40da2b9490db60e731       1731628800   \n",
+       "\n",
+       "  market_creator    opening_datetime  first_outcome_prob  second_outcome_prob  \\\n",
+       "0     quickstart 2024-11-14 01:00:00              0.3679               0.6321   \n",
+       "1     quickstart 2024-11-14 01:00:00              0.3368               0.6632   \n",
+       "2     quickstart 2024-11-14 01:00:00              0.4468               0.5532   \n",
+       "3     quickstart 2024-11-14 01:00:00              0.6804               0.3196   \n",
+       "4     quickstart 2024-11-15 01:00:00              0.2358               0.7642   \n",
+       "\n",
+       "   kl_divergence  off_by_perc  \n",
+       "0       0.999944        63.21  \n",
+       "1       1.088266        66.32  \n",
+       "2       0.805644        55.32  \n",
+       "3       0.385074        31.96  \n",
+       "4       0.268926        23.58  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "div_data.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2024-12-28 01:00:00')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "max(div_data.opening_datetime)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 8,