new weekly data

data/closed_markets_div.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06e904950a630f45605438542ade8c8b0d0cad96fbe197745339b6bd189e8513
-size 60416
+oid sha256:8df376959d26e13d47dcbe3c25acc82f028e35f33ef7928a869eed102d8c7e34
+size 64570

notebooks/outliers.ipynb

@@ -0,0 +1,607 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import sys\n",
+    "sys.path.append('..')\n",
+    "from scripts.metrics import compute_weekly_metrics_by_market_creator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_trades = pd.read_parquet('../data/all_trades_profitability.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "num_mech_calls\n",
+       "1     5231\n",
+       "2     4590\n",
+       "0     4555\n",
+       "4     4457\n",
+       "3     4387\n",
+       "      ... \n",
+       "63       1\n",
+       "59       1\n",
+       "37       1\n",
+       "65       1\n",
+       "53       1\n",
+       "Name: count, Length: 67, dtype: int64"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_trades.num_mech_calls.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_trades[\"creation_date\"] = all_trades[\"creation_timestamp\"].dt.date\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/gp/02mb1d514ng739czlxw1lhh00000gn/T/ipykernel_15029/1825242321.py:6: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
+      "  all_trades[\"creation_timestamp\"].dt.to_period(\"W\").dt.strftime(\"%b-%d\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "all_trades = all_trades.sort_values(\n",
+    "    by=\"creation_timestamp\", ascending=True\n",
+    ")\n",
+    "\n",
+    "all_trades[\"month_year_week\"] = (\n",
+    "    all_trades[\"creation_timestamp\"].dt.to_period(\"W\").dt.strftime(\"%b-%d\")\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Sep-15 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 38/38 [00:00<00:00, 858.56metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Sep-22 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 95/95 [00:00<00:00, 726.25metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Sep-29 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 119/119 [00:00<00:00, 724.34metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Oct-06 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 95/95 [00:00<00:00, 662.54metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Oct-13 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 117/117 [00:00<00:00, 665.98metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Oct-20 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 129/129 [00:00<00:00, 819.97metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Oct-27 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 205/205 [00:00<00:00, 679.75metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Nov-03 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 361/361 [00:00<00:00, 754.52metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Nov-10 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 357/357 [00:00<00:00, 723.25metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Computing weekly metrics for week =Nov-17 by market creator\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trader' metrics: 100%|██████████| 411/411 [00:00<00:00, 714.79metrics/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "End computing all weekly metrics by market creator\n"
+     ]
+    }
+   ],
+   "source": [
+    "weekly_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(\n",
+    "    all_trades\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "weekly_metrics_by_market_creator_pearl = weekly_metrics_by_market_creator.loc[weekly_metrics_by_market_creator[\"market_creator\"]==\"pearl\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "nr_mech_calls\n",
+       "0        191\n",
+       "1        152\n",
+       "2        105\n",
+       "3         63\n",
+       "4         41\n",
+       "        ... \n",
+       "62         1\n",
+       "13429      1\n",
+       "1099       1\n",
+       "154        1\n",
+       "254        1\n",
+       "Name: count, Length: 88, dtype: int64"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weekly_metrics_by_market_creator_pearl.nr_mech_calls.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>trader_address</th>\n",
+       "      <th>net_earnings</th>\n",
+       "      <th>earnings</th>\n",
+       "      <th>bet_amount</th>\n",
+       "      <th>nr_mech_calls</th>\n",
+       "      <th>nr_trades</th>\n",
+       "      <th>roi</th>\n",
+       "      <th>month_year_week</th>\n",
+       "      <th>market_creator</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1998</th>\n",
+       "      <td>0x87f0fcfe810502555f8d1439793155cbfa2eb583</td>\n",
+       "      <td>-135.245314</td>\n",
+       "      <td>1.014186</td>\n",
+       "      <td>1.95</td>\n",
+       "      <td>13429</td>\n",
+       "      <td>78</td>\n",
+       "      <td>-0.499927</td>\n",
+       "      <td>Nov-03</td>\n",
+       "      <td>pearl</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                  trader_address  net_earnings  earnings  \\\n",
+       "1998  0x87f0fcfe810502555f8d1439793155cbfa2eb583   -135.245314  1.014186   \n",
+       "\n",
+       "      bet_amount  nr_mech_calls  nr_trades       roi month_year_week  \\\n",
+       "1998        1.95          13429         78 -0.499927          Nov-03   \n",
+       "\n",
+       "     market_creator  \n",
+       "1998          pearl  "
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weekly_metrics_by_market_creator_pearl.loc[weekly_metrics_by_market_creator_pearl[\"nr_mech_calls\"]==13429]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trader = \"0x87f0fcfe810502555f8d1439793155cbfa2eb583\"\n",
+    "selected_week = \"Nov-03\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trader_data = all_trades.loc[(all_trades[\"trader_address\"]==trader)&(all_trades[\"month_year_week\"]==selected_week)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "Index: 78 entries, 26553 to 31970\n",
+      "Data columns (total 23 columns):\n",
+      " #   Column                  Non-Null Count  Dtype              \n",
+      "---  ------                  --------------  -----              \n",
+      " 0   trader_address          78 non-null     object             \n",
+      " 1   market_creator          78 non-null     object             \n",
+      " 2   trade_id                78 non-null     object             \n",
+      " 3   creation_timestamp      78 non-null     datetime64[ns, UTC]\n",
+      " 4   title                   78 non-null     object             \n",
+      " 5   market_status           78 non-null     object             \n",
+      " 6   collateral_amount       78 non-null     float64            \n",
+      " 7   outcome_index           78 non-null     object             \n",
+      " 8   trade_fee_amount        78 non-null     float64            \n",
+      " 9   outcomes_tokens_traded  78 non-null     float64            \n",
+      " 10  current_answer          78 non-null     int64              \n",
+      " 11  is_invalid              78 non-null     bool               \n",
+      " 12  winning_trade           78 non-null     bool               \n",
+      " 13  earnings                78 non-null     float64            \n",
+      " 14  redeemed                78 non-null     bool               \n",
+      " 15  redeemed_amount         78 non-null     float64            \n",
+      " 16  num_mech_calls          78 non-null     int64              \n",
+      " 17  mech_fee_amount         78 non-null     float64            \n",
+      " 18  net_earnings            78 non-null     float64            \n",
+      " 19  roi                     78 non-null     float64            \n",
+      " 20  staking                 78 non-null     object             \n",
+      " 21  creation_date           78 non-null     object             \n",
+      " 22  month_year_week         78 non-null     object             \n",
+      "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(9)\n",
+      "memory usage: 13.0+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "trader_data.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "count     78.000000\n",
+       "mean     172.166667\n",
+       "std       73.238698\n",
+       "min        1.000000\n",
+       "25%      206.000000\n",
+       "50%      206.000000\n",
+       "75%      206.000000\n",
+       "max      206.000000\n",
+       "Name: num_mech_calls, dtype: float64"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trader_data.num_mech_calls.describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trader_data_selected = trader_data.loc[trader_data[\"num_mech_calls\"]>200]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "Index: 64 entries, 26553 to 26582\n",
+      "Data columns (total 23 columns):\n",
+      " #   Column                  Non-Null Count  Dtype              \n",
+      "---  ------                  --------------  -----              \n",
+      " 0   trader_address          64 non-null     object             \n",
+      " 1   market_creator          64 non-null     object             \n",
+      " 2   trade_id                64 non-null     object             \n",
+      " 3   creation_timestamp      64 non-null     datetime64[ns, UTC]\n",
+      " 4   title                   64 non-null     object             \n",
+      " 5   market_status           64 non-null     object             \n",
+      " 6   collateral_amount       64 non-null     float64            \n",
+      " 7   outcome_index           64 non-null     object             \n",
+      " 8   trade_fee_amount        64 non-null     float64            \n",
+      " 9   outcomes_tokens_traded  64 non-null     float64            \n",
+      " 10  current_answer          64 non-null     int64              \n",
+      " 11  is_invalid              64 non-null     bool               \n",
+      " 12  winning_trade           64 non-null     bool               \n",
+      " 13  earnings                64 non-null     float64            \n",
+      " 14  redeemed                64 non-null     bool               \n",
+      " 15  redeemed_amount         64 non-null     float64            \n",
+      " 16  num_mech_calls          64 non-null     int64              \n",
+      " 17  mech_fee_amount         64 non-null     float64            \n",
+      " 18  net_earnings            64 non-null     float64            \n",
+      " 19  roi                     64 non-null     float64            \n",
+      " 20  staking                 64 non-null     object             \n",
+      " 21  creation_date           64 non-null     object             \n",
+      " 22  month_year_week         64 non-null     object             \n",
+      "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(9)\n",
+      "memory usage: 10.7+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "trader_data_selected.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "title\n",
+       "Will the U.S. Congress hold a hearing to discuss the security threats faced by former U.S. Presidents before November 1, 2024?    64\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trader_data_selected.title.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "creation_date\n",
+       "2024-10-29    32\n",
+       "2024-10-30    29\n",
+       "2024-10-28     3\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trader_data_selected.creation_date.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_dashboards",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}