corrected version of wow retention

app.py

@@ -93,14 +93,22 @@ def get_all_data():
     FROM read_parquet('./data/unknown_traders.parquet')
     """
     df4 = con.execute(query4).fetchdf()
-    con.close()
 
-    return df1, df2, df3, df4
+    # Query to fetch retention activity data
+    query5 = f"""
+    SELECT *
+    FROM read_parquet('./data/retention_activity.parquet')
+    """
+    df5 = con.execute(query5).fetchdf()
+    con.close()
+    return df1, df2, df3, df4, df5
 
 
 def prepare_data():
 
-    all_trades, closed_markets, daily_info, unknown_traders = get_all_data()
+    all_trades, closed_markets, daily_info, unknown_traders, retention_df = (
+        get_all_data()
+    )
 
     all_trades["creation_date"] = all_trades["creation_timestamp"].dt.date
 
@@ -135,12 +143,12 @@ def prepare_data():
     closed_markets["month_year_week"] = (
         closed_markets["opening_datetime"].dt.to_period("W").dt.strftime("%b-%d-%Y")
     )
-    return traders_data, closed_markets, daily_info, unknown_traders
+    return traders_data, closed_markets, daily_info, unknown_traders, retention_df
 
 
-traders_data, closed_markets, daily_info, unknown_traders = prepare_data()
+traders_data, closed_markets, daily_info, unknown_traders, retention_df = prepare_data()
 retention_df = prepare_retention_dataset(
-    traders_df=traders_data, unknown_df=unknown_traders
+    retention_df=retention_df, unknown_df=unknown_traders
 )
 
 demo = gr.Blocks()
@@ -406,17 +414,37 @@ with demo:
             with gr.Row():
                 gr.Markdown("# Wow retention by trader type")
             with gr.Row():
-                wow_retention = calculate_wow_retention_by_type(retention_df)
-                wow_retention_plot = plot_wow_retention_by_type(
-                    wow_retention=wow_retention
-                )
-            with gr.Row():
-                gr.Markdown("# Cohort retention")
-            with gr.Row():
-                cohort_retention = calculate_cohort_retention(df=retention_df)
-                cohort_retention_plot = plot_cohort_retention_heatmap(
-                    retention_matrix=cohort_retention
-                )
+                with gr.Column(scale=1):
+                    gr.Markdown("## Wow retention in Pearl markets")
+                    wow_retention = calculate_wow_retention_by_type(
+                        retention_df, market_creator="pearl"
+                    )
+                    wow_retention_plot = plot_wow_retention_by_type(
+                        wow_retention=wow_retention
+                    )
+                with gr.Column(scale=1):
+                    gr.Markdown("## Wow retention in Quickstart markets")
+                    wow_retention = calculate_wow_retention_by_type(
+                        retention_df, market_creator="quickstart"
+                    )
+                    wow_retention_plot = plot_wow_retention_by_type(
+                        wow_retention=wow_retention
+                    )
+
+            # with gr.Row():
+            #     gr.Markdown("# Cohort retention in pearl traders")
+            # with gr.Row():
+            #     cohort_retention = calculate_cohort_retention(df=retention_df)
+            #     cohort_retention_plot = plot_cohort_retention_heatmap(
+            #         retention_matrix=cohort_retention
+            #     )
+            # with gr.Row():
+            #     gr.Markdown("# Cohort retention in qs traders")
+            # with gr.Row():
+            #     cohort_retention = calculate_cohort_retention(df=retention_df)
+            #     cohort_retention_plot = plot_cohort_retention_heatmap(
+            #         retention_matrix=cohort_retention
+            #     )
 
         with gr.TabItem("⚙️ Active traders"):
             with gr.Row():

notebooks/retention_metrics.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -12,6 +12,178 @@
     "import gc"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Get all activity info from tools.parquet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retention_df = pd.read_parquet(\"../data/retention_activity.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['trader_address', 'request_time', 'market_creator', 'request_date',\n",
+       "       'staking', 'month_year_week'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "retention_df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>trader_address</th>\n",
+       "      <th>request_time</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>request_date</th>\n",
+       "      <th>staking</th>\n",
+       "      <th>month_year_week</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0x721de88cee9be146c8f0c7ef1a4188bee36494d6</td>\n",
+       "      <td>2024-10-25 00:00:20+00:00</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-10-25</td>\n",
+       "      <td>non_staking</td>\n",
+       "      <td>Oct-25-2024</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0x8a1d5f22b5a3bea34697b85e7b4ad894bf9ee36a</td>\n",
+       "      <td>2024-10-25 00:00:25+00:00</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-10-25</td>\n",
+       "      <td>non_staking</td>\n",
+       "      <td>Oct-25-2024</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0xf839eaf4b42eadd917b46d7b6da0dd0e1fd6f684</td>\n",
+       "      <td>2024-10-25 00:00:55+00:00</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-10-25</td>\n",
+       "      <td>non_staking</td>\n",
+       "      <td>Oct-25-2024</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0x01274796ce41aa8e8312e05a427ffb4b0d2148f6</td>\n",
+       "      <td>2024-10-25 00:00:55+00:00</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-10-25</td>\n",
+       "      <td>non_staking</td>\n",
+       "      <td>Oct-25-2024</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0xc20678890f94d0162593c46fe5da67d9a4b7a6fb</td>\n",
+       "      <td>2024-10-25 00:01:05+00:00</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-10-25</td>\n",
+       "      <td>non_staking</td>\n",
+       "      <td>Oct-25-2024</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               trader_address              request_time  \\\n",
+       "0  0x721de88cee9be146c8f0c7ef1a4188bee36494d6 2024-10-25 00:00:20+00:00   \n",
+       "1  0x8a1d5f22b5a3bea34697b85e7b4ad894bf9ee36a 2024-10-25 00:00:25+00:00   \n",
+       "2  0xf839eaf4b42eadd917b46d7b6da0dd0e1fd6f684 2024-10-25 00:00:55+00:00   \n",
+       "3  0x01274796ce41aa8e8312e05a427ffb4b0d2148f6 2024-10-25 00:00:55+00:00   \n",
+       "4  0xc20678890f94d0162593c46fe5da67d9a4b7a6fb 2024-10-25 00:01:05+00:00   \n",
+       "\n",
+       "  market_creator request_date      staking month_year_week  \n",
+       "0     quickstart   2024-10-25  non_staking     Oct-25-2024  \n",
+       "1     quickstart   2024-10-25  non_staking     Oct-25-2024  \n",
+       "2     quickstart   2024-10-25  non_staking     Oct-25-2024  \n",
+       "3     quickstart   2024-10-25  non_staking     Oct-25-2024  \n",
+       "4     quickstart   2024-10-25  non_staking     Oct-25-2024  "
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "retention_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "staking\n",
+       "non_Olas       764956\n",
+       "non_staking    275246\n",
+       "pearl           56487\n",
+       "quickstart      48511\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "retention_df.staking.value_counts()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

scripts/retention_metrics.py

@@ -4,10 +4,13 @@ from scripts.utils import DATA_DIR
 
 
 # Basic Week over Week Retention
-def calculate_wow_retention_by_type(df: pd.DataFrame) -> pd.DataFrame:
+def calculate_wow_retention_by_type(
+    df: pd.DataFrame, market_creator: str
+) -> pd.DataFrame:
+    filtered_df = df.loc[df["market_creator"] == market_creator]
     # Get unique traders per week and type
     weekly_traders = (
-        df.groupby(["month_year_week", "trader_type"])["trader_address"]
+        filtered_df.groupby(["month_year_week", "trader_type"])["trader_address"]
         .nunique()
         .reset_index()
     )
@@ -26,16 +29,16 @@ def calculate_wow_retention_by_type(df: pd.DataFrame) -> pd.DataFrame:
 
             # Get traders in both weeks for this type
             current_traders = set(
-                df[
-                    (df["month_year_week"] == current_week)
-                    & (df["trader_type"] == trader_type)
+                filtered_df[
+                    (filtered_df["month_year_week"] == current_week)
+                    & (filtered_df["trader_type"] == trader_type)
                 ]["trader_address"]
             )
 
             previous_traders = set(
-                df[
-                    (df["month_year_week"] == previous_week)
-                    & (df["trader_type"] == trader_type)
+                filtered_df[
+                    (filtered_df["month_year_week"] == previous_week)
+                    & (filtered_df["trader_type"] == trader_type)
                 ]["trader_address"]
             )
 
@@ -60,10 +63,13 @@ def calculate_wow_retention_by_type(df: pd.DataFrame) -> pd.DataFrame:
 
 
 # Cohort Retention
-def calculate_cohort_retention(df, max_weeks=12) -> pd.DataFrame:
+def calculate_cohort_retention(
+    df: pd.DataFrame, trader_type: str, max_weeks=12
+) -> pd.DataFrame:
+    df_filtered = df.loc[df["trader_type"] == trader_type]
     # Get first week for each trader
     first_trades = (
-        df.groupby("trader_address")
+        df_filtered.groupby("trader_address")
         .agg({"creation_timestamp": "min", "month_year_week": "first"})
         .reset_index()
     )
@@ -111,7 +117,7 @@ def calculate_cohort_retention(df, max_weeks=12) -> pd.DataFrame:
     return retention_matrix.round(2)
 
 
-def prepare_retention_dataset(
+def merge_retention_dataset(
     traders_df: pd.DataFrame, unknown_df: pd.DataFrame
 ) -> pd.DataFrame:
 
@@ -131,6 +137,33 @@ def prepare_retention_dataset(
     return all_traders
 
 
+def prepare_retention_dataset(
+    retention_df: pd.DataFrame, unknown_df: pd.DataFrame
+) -> pd.DataFrame:
+
+    retention_df["trader_type"] = retention_df["staking"].apply(
+        lambda x: "non_Olas" if x == "non_Olas" else "Olas"
+    )
+    retention_df.rename(columns={"request_time": "creation_timestamp"}, inplace=True)
+    retention_df = retention_df[
+        ["trader_type", "market_creator", "trader_address", "creation_timestamp"]
+    ]
+    unknown_df["trader_type"] = "unclassified"
+    unknown_df = unknown_df[
+        ["trader_type", "market_creator", "trader_address", "creation_timestamp"]
+    ]
+    all_traders = pd.concat([retention_df, unknown_df], ignore_index=True)
+
+    all_traders["creation_timestamp"] = pd.to_datetime(
+        all_traders["creation_timestamp"]
+    )
+    all_traders = all_traders.sort_values(by="creation_timestamp", ascending=True)
+    all_traders["month_year_week"] = (
+        all_traders["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+    )
+    return all_traders
+
+
 if __name__ == "__main__":
     # read all datasets
     traders_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")

tabs/retention_plots.py

@@ -53,7 +53,8 @@ def plot_wow_retention_by_type(wow_retention):
     )
 
 
-def plot_cohort_retention_heatmap(retention_matrix):
+def plot_cohort_retention_heatmap(retention_matrix: pd.DataFrame):
+
     # Create a copy of the matrix to avoid modifying the original
     retention_matrix = retention_matrix.copy()