removing data folder and adding dataset import

app.py

@@ -1,7 +1,11 @@
 import gradio as gr
 import pandas as pd
 import duckdb
+import gzip
+import shutil
+import os
 import logging
+from huggingface_hub import hf_hub_download
 
 
 from scripts.metrics import (
@@ -58,6 +62,82 @@ def get_logger():
 logger = get_logger()
 
 
+def load_all_data():
+
+    # all trades profitability
+    # Download the compressed file
+    gz_file_path = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="all_trades_profitability.parquet.gz",
+        repo_type="dataset",
+    )
+
+    parquet_file_path = gz_file_path.replace(".gz", "")
+
+    with gzip.open(gz_file_path, "rb") as f_in:
+        with open(parquet_file_path, "wb") as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+    # Now read the decompressed parquet file
+    df1 = pd.read_parquet(parquet_file_path)
+    os.remove(parquet_file_path)
+
+    # closed_markets_div
+    closed_markets_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="closed_markets_div.parquet",
+        repo_type="dataset",
+    )
+    df2 = pd.read_parquet(closed_markets_df)
+
+    # daily_info
+    daily_info_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="daily_info.parquet",
+        repo_type="dataset",
+    )
+    df3 = pd.read_parquet(daily_info_df)
+
+    # unknown traders
+    unknown_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="unknown_traders.parquet",
+        repo_type="dataset",
+    )
+    df4 = pd.read_parquet(unknown_df)
+
+    # retention activity
+    gz_file_path = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="retention_activity.parquet.gz",
+        repo_type="dataset",
+    )
+    parquet_file_path = gz_file_path.replace(".gz", "")
+
+    with gzip.open(gz_file_path, "rb") as f_in:
+        with open(parquet_file_path, "wb") as f_out:
+            shutil.copyfileobj(f_in, f_out)
+    df5 = pd.read_parquet(parquet_file_path)
+    os.remove(parquet_file_path)
+
+    # active_traders.parquet
+    active_traders_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="active_traders.parquet",
+        repo_type="dataset",
+    )
+    df6 = pd.read_parquet(active_traders_df)
+
+    # weekly_mech_calls.parquet
+    all_mech_calls_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="weekly_mech_calls.parquet",
+        repo_type="dataset",
+    )
+    df7 = pd.read_parquet(all_mech_calls_df)
+    return df1, df2, df3, df4, df5, df6, df7
+
+
 def get_all_data():
     """
     Get parquet files from weekly stats and new generated
@@ -120,7 +200,8 @@ def prepare_data():
         unknown_traders,
         retention_df,
         active_traders,
-    ) = get_all_data()
+        all_mech_calls,
+    ) = load_all_data()
 
     all_trades["creation_date"] = all_trades["creation_timestamp"].dt.date
 
@@ -168,6 +249,7 @@ def prepare_data():
         unknown_traders,
         retention_df,
         active_traders,
+        all_mech_calls,
     )
 
 
@@ -178,6 +260,7 @@ def prepare_data():
     unknown_traders,
     raw_retention_df,
     active_traders,
+    all_mech_calls,
 ) = prepare_data()
 retention_df = prepare_retention_dataset(
     retention_df=raw_retention_df, unknown_df=unknown_traders
@@ -188,21 +271,23 @@ print(max(retention_df.creation_timestamp))
 demo = gr.Blocks()
 # get weekly metrics by market creator: qs, pearl or all.
 weekly_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
-    traders_data
+    traders_data, all_mech_calls
 )
 weekly_o_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
-    traders_data, trader_filter="Olas"
+    traders_data, all_mech_calls, trader_filter="Olas"
 )
 weekly_non_olas_metrics_by_market_creator = pd.DataFrame()
 if len(traders_data.loc[traders_data["staking"] == "non_Olas"]) > 0:
     weekly_non_olas_metrics_by_market_creator = (
-        compute_weekly_metrics_by_market_creator(traders_data, trader_filter="non_Olas")
+        compute_weekly_metrics_by_market_creator(
+            traders_data, all_mech_calls, trader_filter="non_Olas"
+        )
     )
 weekly_unknown_trader_metrics_by_market_creator = None
 if len(unknown_traders) > 0:
     weekly_unknown_trader_metrics_by_market_creator = (
         compute_weekly_metrics_by_market_creator(
-            unknown_traders, trader_filter=None, unknown_trader=True
+            unknown_traders, all_mech_calls, trader_filter=None, unknown_trader=True
         )
     )
 

data/active_traders.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4e3553505541a2a8c6173b6bf2eb6d4375e667103d3c9e6157e40b8fee1a24c6
-size 73479

data/all_trades_profitability.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3f35e07f90679e6ef7241b3ddaffb3222e04eba8768c06d221358e5cec9a2dc7
-size 7968232

data/closed_markets_div.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:200149ef5bf001c3b3c10777b03887387074273dba0237333fae14421b699b88
-size 66133

data/daily_info.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a12927dc3b935cb3208aea26426063a569cab6c27afb588765f751af9037e7c1
-size 896490

data/retention_activity.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:162ffffff8161f2fabc1ccb32ba2c7d51c402fa373d998698b1a9e868ecce8c6
-size 10795901

data/unknown_daily_traders.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fe6aaf7bc0c124ed833a9f59327685c5c4686d3114945afb3face7259780731f
-size 56402

data/unknown_traders.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4cf113d76828b906deb1604877cdd9009074cfc7f5730d86a442329966269dd3
-size 274887

data/weekly_mech_calls.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:813e94540515d33c6dcef25d56bf80a8d48c97af19a4e434ba1dbc0e2144270f
-size 53556

requirements.txt

@@ -10,5 +10,4 @@ pydantic
 pydantic_core
 nbformat
 pytz
-duckdb
 ipfshttpclient

scripts/closed_markets_divergence.py

@@ -1,269 +0,0 @@
-import os
-import pandas as pd
-import numpy as np
-from typing import Any, Union
-from string import Template
-import requests
-import pickle
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from tqdm import tqdm
-import time
-from datetime import datetime
-from utils import DATA_DIR, TMP_DIR
-
-NUM_WORKERS = 10
-IPFS_POLL_INTERVAL = 0.2
-INVALID_ANSWER_HEX = (
-    "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
-)
-INVALID_ANSWER = -1
-SUBGRAPH_API_KEY = os.environ.get("SUBGRAPH_API_KEY", None)
-OMEN_SUBGRAPH_URL = Template(
-    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
-)
-get_token_amounts_query = Template(
-    """
-    {
-  
-      fpmmLiquidities(
-        where: {
-                fpmm_: {
-                        creator: "${fpmm_creator}",
-                        id: "${fpmm_id}",
-                    },
-                id_gt: ""
-                }
-                orderBy: creationTimestamp
-                orderDirection: asc
-        )
-      {
-        id
-        outcomeTokenAmounts
-        creationTimestamp
-        additionalLiquidityParameter
-      }
-    }
-"""
-)
-CREATOR = "0x89c5cc945dd550BcFfb72Fe42BfF002429F46Fec"
-PEARL_CREATOR = "0xFfc8029154ECD55ABED15BD428bA596E7D23f557"
-market_creators_map = {"quickstart": CREATOR, "pearl": PEARL_CREATOR}
-headers = {
-    "Accept": "application/json, multipart/mixed",
-    "Content-Type": "application/json",
-}
-
-
-def _to_content(q: str) -> dict[str, Any]:
-    """Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes."""
-    finalized_query = {
-        "query": q,
-        "variables": None,
-        "extensions": {"headers": None},
-    }
-    return finalized_query
-
-
-def collect_liquidity_info(
-    index: int, fpmm_id: str, market_creator: str
-) -> dict[str, Any]:
-    omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
-    market_creator_id = market_creators_map[market_creator]
-    query = get_token_amounts_query.substitute(
-        fpmm_creator=market_creator_id.lower(),
-        fpmm_id=fpmm_id,
-    )
-    content_json = _to_content(query)
-    # print(f"Executing liquidity query {query}")
-    res = requests.post(omen_subgraph, headers=headers, json=content_json)
-    result_json = res.json()
-    tokens_info = result_json.get("data", {}).get("fpmmLiquidities", [])
-    if not tokens_info:
-        return None
-
-    # the last item is the final information of the market
-    last_info = tokens_info[-1]
-    token_amounts = [int(x) for x in last_info["outcomeTokenAmounts"]]
-    time.sleep(IPFS_POLL_INTERVAL)
-    return {fpmm_id: token_amounts}
-
-
-def convert_hex_to_int(x: Union[str, float]) -> Union[int, float]:
-    """Convert hex to int"""
-    if isinstance(x, float):
-        return np.nan
-    if isinstance(x, str):
-        if x == INVALID_ANSWER_HEX:
-            return "invalid"
-        return "yes" if int(x, 16) == 0 else "no"
-
-
-def get_closed_markets():
-    print("Reading parquet file with closed markets data from trades")
-    try:
-        markets = pd.read_parquet(TMP_DIR / "fpmmTrades.parquet")
-    except Exception:
-        print("Error reading the parquet file")
-
-    columns_of_interest = [
-        "fpmm.currentAnswer",
-        "fpmm.id",
-        "fpmm.openingTimestamp",
-        "market_creator",
-    ]
-    markets = markets[columns_of_interest]
-    markets.rename(
-        columns={
-            "fpmm.currentAnswer": "currentAnswer",
-            "fpmm.openingTimestamp": "openingTimestamp",
-            "fpmm.id": "id",
-        },
-        inplace=True,
-    )
-    markets = markets.drop_duplicates(subset=["id"], keep="last")
-    # remove invalid answers
-    markets = markets.loc[markets["currentAnswer"] != INVALID_ANSWER_HEX]
-    markets["currentAnswer"] = markets["currentAnswer"].apply(
-        lambda x: convert_hex_to_int(x)
-    )
-    markets.dropna(inplace=True)
-    markets["opening_datetime"] = markets["openingTimestamp"].apply(
-        lambda x: datetime.fromtimestamp(int(x))
-    )
-    markets = markets.sort_values(by="opening_datetime", ascending=True)
-    return markets
-
-
-def kl_divergence(P, Q):
-    """
-    Compute KL divergence for a single sample with two prob distributions.
-
-    :param P: True distribution)
-    :param Q: Approximating distribution)
-    :return: KL divergence value
-    """
-    # Review edge cases
-    if P[0] == Q[0]:
-        return 0.0
-    # If P is complete opposite of Q, divergence is some max value.
-    # Here set to 20--allows for Q [\mu, 1-\mu] or Q[1-\mu, \mu] where \mu = 10^-8
-    if P[0] == Q[1]:
-        return 20
-
-    nonzero = P > 0.0
-    # Compute KL divergence
-    kl_div = np.sum(P[nonzero] * np.log(P[nonzero] / Q[nonzero]))
-
-    return kl_div
-
-
-def market_KL_divergence(market_row: pd.DataFrame) -> float:
-    """Function to compute the divergence based on the formula
-    Formula in https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence"""
-    current_answer = market_row.currentAnswer  # "yes", "no"
-    approx_prob = market_row.first_outcome_prob
-    true_prob = 1.0  # for yes outcome
-    if current_answer == "no":
-        true_prob = 0.0  # = 0% for yes outcome and 100% for no
-
-    # we have only one sample, the final probability based on tokens
-    # Ensure probabilities sum to 1
-    P = np.array([true_prob, 1 - true_prob])
-    Q = np.array([approx_prob, 1 - approx_prob])
-    return kl_divergence(P, Q)
-
-
-def off_by_values(market_row: pd.DataFrame) -> float:
-    current_answer = market_row.currentAnswer  # "yes", "no"
-    approx_prob = market_row.first_outcome_prob
-    true_prob = 1.0  # for yes outcome
-    if current_answer == "no":
-        true_prob = 0.0  # = 0% for yes outcome and 100% for no
-
-    # we have only one sample, the final probability based on tokens
-    # Ensure probabilities sum to 1
-    P = np.array([true_prob, 1 - true_prob])
-    Q = np.array([approx_prob, 1 - approx_prob])
-    return abs(P[0] - Q[0]) * 100.0
-
-
-def compute_tokens_prob(token_amounts: list) -> list:
-    first_token_amounts = token_amounts[0]
-    second_token_amounts = token_amounts[1]
-    total_tokens = first_token_amounts + second_token_amounts
-    first_token_prob = 1 - round((first_token_amounts / total_tokens), 4)
-    return [first_token_prob, 1 - first_token_prob]
-
-
-def prepare_closed_markets_data():
-    closed_markets = get_closed_markets()
-    closed_markets["first_outcome_prob"] = -1.0
-    closed_markets["second_outcome_prob"] = -1.0
-    total_markets = len(closed_markets)
-    markets_no_info = []
-    no_info = 0
-    with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
-        futures = []
-        for i in range(total_markets):
-            futures.append(
-                executor.submit(
-                    collect_liquidity_info,
-                    i,
-                    closed_markets.iloc[i].id,
-                    closed_markets.iloc[i].market_creator,
-                )
-            )
-        markets_with_info = 0
-        for future in tqdm(
-            as_completed(futures),
-            total=len(futures),
-            desc=f"Fetching Market liquidity info",
-        ):
-            token_amounts_dict = future.result()
-            if token_amounts_dict:
-                fpmm_id, token_amounts = token_amounts_dict.popitem()
-                if token_amounts:
-                    tokens_prob = compute_tokens_prob(token_amounts)
-                    closed_markets.loc[
-                        closed_markets["id"] == fpmm_id, "first_outcome_prob"
-                    ] = tokens_prob[0]
-                    closed_markets.loc[
-                        closed_markets["id"] == fpmm_id, "second_outcome_prob"
-                    ] = tokens_prob[1]
-                    markets_with_info += 1
-                else:
-                    tqdm.write(f"Skipping market with no liquidity info")
-                    markets_no_info.append(i)
-            else:
-                tqdm.write(f"Skipping market with no liquidity info")
-                no_info += 1
-
-    print(f"Markets with info = {markets_with_info}")
-    # Removing markets with no liq info
-    closed_markets = closed_markets.loc[closed_markets["first_outcome_prob"] != -1.0]
-    print(
-        f"Finished computing all markets liquidity info. Final length = {len(closed_markets)}"
-    )
-    if len(markets_no_info) > 0:
-        print(
-            f"There were {len(markets_no_info)} markets with no liquidity info. Printing some index of the dataframe"
-        )
-        with open("no_liq_info.pickle", "wb") as file:
-            pickle.dump(markets_no_info, file)
-        print(markets_no_info[:1])
-    print(closed_markets.head())
-    # Add the Kullback–Leibler divergence values
-    print("Computing Kullback–Leibler (KL) divergence")
-    closed_markets["kl_divergence"] = closed_markets.apply(
-        lambda x: market_KL_divergence(x), axis=1
-    )
-    closed_markets["off_by_perc"] = closed_markets.apply(
-        lambda x: off_by_values(x), axis=1
-    )
-    closed_markets.to_parquet(DATA_DIR / "closed_markets_div.parquet", index=False)
-    print("Finished preparing final dataset for visualization")
-    print(closed_markets.head())
-
-
-if __name__ == "__main__":
-    prepare_closed_markets_data()

scripts/metrics.py

@@ -1,16 +1,38 @@
 import pandas as pd
 from tqdm import tqdm
-from scripts.num_mech_calls import (
-    get_weekly_total_mech_calls,
-)
 from scripts.utils import get_next_week
 
 DEFAULT_MECH_FEE = 0.01  # xDAI
 
 
+def get_weekly_total_mech_calls(
+    trader_data: pd.DataFrame, all_mech_calls_df: pd.DataFrame
+) -> int:
+    """Function to compute the total weekly number of mech calls for all markets
+    that the trader bet upon"""
+
+    trading_weeks = trader_data.month_year_week.unique()
+    trader_address = trader_data.trader_address.unique()[0]
+    if len(trading_weeks) > 1:
+        raise ValueError("The trader data should contain only one week information")
+    trading_week = trading_weeks[0]
+    try:
+        return all_mech_calls_df.loc[
+            (all_mech_calls_df["trader_address"] == trader_address)
+            & (all_mech_calls_df["month_year_week"] == trading_week),
+            "total_mech_calls",
+        ].iloc[0]
+    except Exception as e:
+        print(
+            f"Error getting the number of mech calls for the trader {trader_address} and week {trading_week}"
+        )
+        return 280  # average number 40 mech calls in 7 days
+
+
 def compute_metrics(
     trader_address: str,
     trader_data: pd.DataFrame,
+    all_mech_calls: pd.DataFrame,
     live_metrics: bool = False,
     unknown_trader: bool = False,
 ) -> dict:
@@ -29,7 +51,9 @@ def compute_metrics(
         # num of mech calls is always zero
         total_nr_mech_calls_all_markets = 0
     else:
-        total_nr_mech_calls_all_markets = get_weekly_total_mech_calls(trader_data)
+        total_nr_mech_calls_all_markets = get_weekly_total_mech_calls(
+            trader_data=trader_data, all_mech_calls_df=all_mech_calls
+        )
 
     agg_metrics["bet_amount"] = total_bet_amounts
     agg_metrics["nr_mech_calls"] = total_nr_mech_calls_all_markets
@@ -56,6 +80,7 @@ def compute_metrics(
 def compute_trader_metrics_by_market_creator(
     trader_address: str,
     traders_data: pd.DataFrame,
+    all_mech_calls: pd.DataFrame,
     market_creator: str = "all",
     live_metrics: bool = False,
     unknown_trader: bool = False,
@@ -78,19 +103,28 @@ def compute_trader_metrics_by_market_creator(
             return {}  # No Data
 
     metrics = compute_metrics(
-        trader_address, filtered_traders_data, live_metrics, unknown_trader
+        trader_address,
+        filtered_traders_data,
+        all_mech_calls,
+        live_metrics,
+        unknown_trader,
     )
     return metrics
 
 
 def merge_trader_weekly_metrics(
-    trader: str, weekly_data: pd.DataFrame, week: str, unknown_trader: bool = False
+    trader: str,
+    weekly_data: pd.DataFrame,
+    all_mech_calls: pd.DataFrame,
+    week: str,
+    unknown_trader: bool = False,
 ) -> pd.DataFrame:
     trader_metrics = []
     # computation as specification 1 for all types of markets
     weekly_metrics_all = compute_trader_metrics_by_market_creator(
         trader,
         weekly_data,
+        all_mech_calls=all_mech_calls,
         market_creator="all",
         live_metrics=False,
         unknown_trader=unknown_trader,
@@ -103,6 +137,7 @@ def merge_trader_weekly_metrics(
     weekly_metrics_qs = compute_trader_metrics_by_market_creator(
         trader,
         weekly_data,
+        all_mech_calls=all_mech_calls,
         market_creator="quickstart",
         live_metrics=False,
         unknown_trader=unknown_trader,
@@ -115,6 +150,7 @@ def merge_trader_weekly_metrics(
     weekly_metrics_pearl = compute_trader_metrics_by_market_creator(
         trader,
         weekly_data,
+        all_mech_calls=all_mech_calls,
         market_creator="pearl",
         live_metrics=False,
         unknown_trader=unknown_trader,
@@ -128,12 +164,19 @@ def merge_trader_weekly_metrics(
 
 
 def merge_trader_daily_metrics(
-    trader: str, daily_data: pd.DataFrame, day: str, live_metrics: bool = False
+    trader: str,
+    daily_data: pd.DataFrame,
+    day: str,
+    live_metrics: bool = False,
 ) -> pd.DataFrame:
     trader_metrics = []
     # computation as specification 1 for all types of markets
     daily_metrics_all = compute_trader_metrics_by_market_creator(
-        trader, daily_data, market_creator="all", live_metrics=live_metrics
+        trader,
+        daily_data,
+        all_mech_calls=None,
+        market_creator="all",
+        live_metrics=live_metrics,
     )
     daily_metrics_all["creation_date"] = day
     # staking label is at the trader level
@@ -142,7 +185,11 @@ def merge_trader_daily_metrics(
 
     # computation as specification 1 for quickstart markets
     daily_metrics_qs = compute_trader_metrics_by_market_creator(
-        trader, daily_data, market_creator="quickstart", live_metrics=live_metrics
+        trader,
+        daily_data,
+        all_mech_calls=None,
+        market_creator="quickstart",
+        live_metrics=live_metrics,
     )
     if len(daily_metrics_qs) > 0:
         daily_metrics_qs["creation_date"] = day
@@ -150,7 +197,11 @@ def merge_trader_daily_metrics(
         trader_metrics.append(daily_metrics_qs)
     # computation as specification 1 for pearl markets
     daily_metrics_pearl = compute_trader_metrics_by_market_creator(
-        trader, daily_data, market_creator="pearl", live_metrics=live_metrics
+        trader,
+        daily_data,
+        all_mech_calls=None,
+        market_creator="pearl",
+        live_metrics=live_metrics,
     )
     if len(daily_metrics_pearl) > 0:
         daily_metrics_pearl["creation_date"] = day
@@ -178,7 +229,10 @@ def win_metrics_trader_level(weekly_data):
 
 
 def compute_weekly_metrics_by_market_creator(
-    traders_data: pd.DataFrame, trader_filter: str = None, unknown_trader: bool = False
+    traders_data: pd.DataFrame,
+    all_mech_calls: pd.DataFrame,
+    trader_filter: str = None,
+    unknown_trader: bool = False,
 ) -> pd.DataFrame:
     """Function to compute the metrics at the trader level per week
     and with different categories by market creator"""
@@ -198,18 +252,22 @@ def compute_weekly_metrics_by_market_creator(
             if trader_filter is None:
                 contents.append(
                     merge_trader_weekly_metrics(
-                        trader, weekly_data, week, unknown_trader
+                        trader, weekly_data, all_mech_calls, week, unknown_trader
                     )
                 )
             elif trader_filter == "Olas":
                 filtered_data = weekly_data.loc[weekly_data["staking"] != "non_Olas"]
                 contents.append(
-                    merge_trader_weekly_metrics(trader, filtered_data, week)
+                    merge_trader_weekly_metrics(
+                        trader, filtered_data, all_mech_calls, week
+                    )
                 )
             else:  # non_Olas traders
                 filtered_data = weekly_data.loc[weekly_data["staking"] == "non_Olas"]
                 contents.append(
-                    merge_trader_weekly_metrics(trader, filtered_data, week)
+                    merge_trader_weekly_metrics(
+                        trader, filtered_data, all_mech_calls, week
+                    )
                 )
 
     print("End computing all weekly metrics by market creator")

scripts/num_mech_calls.py

@@ -1,120 +0,0 @@
-import pandas as pd
-
-try:
-    from utils import DATA_DIR, TMP_DIR
-except ImportError:
-    from scripts.utils import DATA_DIR, TMP_DIR
-
-from datetime import datetime, timezone
-from tqdm import tqdm
-
-
-def transform_to_datetime(x):
-    return datetime.fromtimestamp(int(x), tz=timezone.utc)
-
-
-def get_weekly_total_mech_calls(trader_data: pd.DataFrame) -> int:
-    """Function to compute the total weekly number of mech calls for all markets
-    that the trader bet upon"""
-    try:
-        all_mech_calls_df = pd.read_parquet(DATA_DIR / "weekly_mech_calls.parquet")
-    except Exception:
-        print("Error reading the weekly_mech_calls file")
-
-    trading_weeks = trader_data.month_year_week.unique()
-    trader_address = trader_data.trader_address.unique()[0]
-    if len(trading_weeks) > 1:
-        raise ValueError("The trader data should contain only one week information")
-    trading_week = trading_weeks[0]
-    try:
-        return all_mech_calls_df.loc[
-            (all_mech_calls_df["trader_address"] == trader_address)
-            & (all_mech_calls_df["month_year_week"] == trading_week),
-            "total_mech_calls",
-        ].iloc[0]
-    except Exception as e:
-        print(
-            f"Error getting the number of mech calls for the trader {trader_address} and week {trading_week}"
-        )
-        return 280  # average number 40 mech calls in 7 days
-
-
-def compute_weekly_total_mech_calls(
-    trader: str, week: str, weekly_trades: pd.DataFrame, weekly_tools: pd.DataFrame
-) -> dict:
-    weekly_total_mech_calls_dict = {}
-    weekly_total_mech_calls_dict["trader_address"] = trader
-    weekly_total_mech_calls_dict["month_year_week"] = week
-    weekly_total_mech_calls_dict["total_trades"] = len(weekly_trades)
-    weekly_total_mech_calls_dict["total_mech_calls"] = len(weekly_tools)
-    return weekly_total_mech_calls_dict
-
-
-def compute_total_mech_calls():
-    """Function to compute the total number of mech calls for all traders and all markets
-    at a weekly level"""
-    try:
-        print("Reading tools file")
-        tools = pd.read_parquet(TMP_DIR / "tools.parquet")
-        tools["request_time"] = pd.to_datetime(tools["request_time"])
-        tools["request_date"] = tools["request_time"].dt.date
-        tools = tools.sort_values(by="request_time", ascending=True)
-        tools["month_year_week"] = (
-            tools["request_time"]
-            .dt.to_period("W")
-            .dt.start_time.dt.strftime("%b-%d-%Y")
-        )
-
-    except Exception as e:
-        print(f"Error updating the invalid trades parquet {e}")
-
-    print("Reading trades weekly info file")
-    fpmmTrades = pd.read_parquet(TMP_DIR / "fpmmTrades.parquet")
-    try:
-        fpmmTrades["creationTimestamp"] = fpmmTrades["creationTimestamp"].apply(
-            lambda x: transform_to_datetime(x)
-        )
-    except Exception as e:
-        print(f"Transformation not needed")
-
-    fpmmTrades["creation_timestamp"] = pd.to_datetime(fpmmTrades["creationTimestamp"])
-    fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
-    fpmmTrades = fpmmTrades.sort_values(by="creation_timestamp", ascending=True)
-    fpmmTrades["month_year_week"] = (
-        fpmmTrades["creation_timestamp"]
-        .dt.to_period("W")
-        .dt.start_time.dt.strftime("%b-%d-%Y")
-    )
-
-    nr_traders = len(fpmmTrades["trader_address"].unique())
-    all_mech_calls = []
-    for trader in tqdm(
-        fpmmTrades["trader_address"].unique(),
-        total=nr_traders,
-        desc="creating weekly mech calls dataframe",
-    ):
-        # compute the mech calls estimations for each trader
-        all_trades = fpmmTrades[fpmmTrades["trader_address"] == trader]
-        all_tools = tools[tools["trader_address"] == trader]
-        weeks = fpmmTrades.month_year_week.unique()
-
-        for week in weeks:
-            weekly_trades = all_trades.loc[all_trades["month_year_week"] == week]
-            weekly_tools = all_tools.loc[all_tools["month_year_week"] == week]
-
-            weekly_mech_calls_dict = compute_weekly_total_mech_calls(
-                trader, week, weekly_trades, weekly_tools
-            )
-            all_mech_calls.append(weekly_mech_calls_dict)
-
-    all_mech_calls_df: pd.DataFrame = pd.DataFrame.from_dict(
-        all_mech_calls, orient="columns"
-    )
-    print("Saving weekly_mech_calls.parquet file")
-    print(all_mech_calls_df.total_mech_calls.describe())
-
-    all_mech_calls_df.to_parquet(DATA_DIR / "weekly_mech_calls.parquet", index=False)
-
-
-if __name__ == "__main__":
-    compute_total_mech_calls()