diff --git "a/fin_rl_qlearning_v1.ipynb" "b/fin_rl_qlearning_v1.ipynb" new file mode 100644--- /dev/null +++ "b/fin_rl_qlearning_v1.ipynb" @@ -0,0 +1,1162 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "nwaAZRu1NTiI" + }, + "source": [ + "# Q-learning \n", + "\n", + "#### This version implements q-learning using a custom enviroment \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DDf1gLC2NTiK" + }, + "outputs": [], + "source": [ + "# !pip install -r ./requirements.txt\n", + "\n", + "!pip install yfinance\n", + "!pip install talib-binary\n", + "!pip install huggingface_sb3\n" + ] + }, + { + "cell_type": "code", + "execution_count": 130, + "metadata": { + "id": "LNXxxKojNTiL" + }, + "outputs": [], + "source": [ + "import gym\n", + "from gym import spaces\n", + "from gym.utils import seeding\n", + "from gym_anytrading_stocks import TradingEnv \n", + "import talib as ta\n", + "from tqdm.notebook import tqdm\n", + "\n", + "import yfinance as yf\n", + "import pandas as pd\n", + "import numpy as np\n", + "from matplotlib import pyplot as plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": 151, + "metadata": { + "id": "dmAuEhZZNTiL" + }, + "outputs": [], + "source": [ + "# Get data\n", + "eth_usd = yf.Ticker(\"ETH-USD\")\n", + "eth = eth_usd.history(period=\"max\")\n", + "eth_train = eth[-900:-200]\n", + "eth_test = eth[-200:]" + ] + }, + { + "cell_type": "code", + "execution_count": 153, + "metadata": {}, + "outputs": [], + "source": [ + "def initialize_q_table(state_space, action_space):\n", + " Qtable = np.zeros((state_space, action_space))\n", + " return Qtable" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Policy\n", + "\n", + "def greedy_policy(Qtable, state):\n", + " # Exploitation: take the action with the highest state, action value\n", + " action = np.argmax(Qtable[state])\n", + " \n", + " return action\n", + "\n", + "\n", + "def epsilon_greedy_policy(Qtable, state, epsilon):\n", + " # Randomly generate a number between 0 and 1\n", + " random_num = np.random.uniform(size=1)\n", + " # if random_num > greater than epsilon --> exploitation\n", + " if random_num > epsilon:\n", + " # Take the action with the highest value given a state\n", + " # np.argmax can be useful here\n", + " action = greedy_policy(Qtable, state)\n", + " # else --> exploration\n", + " else:\n", + " # action = np.random.random_integers(4,size=1)[0]\n", + " action = env.action_space.sample()\n", + " \n", + " return action" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "wlC-EdLENTiN" + }, + "outputs": [], + "source": [ + "def train(n_training_episodes, min_epsilon, max_epsilon, decay_rate, env, max_steps, Qtable, learning_rate, gamma):\n", + " for episode in range(n_training_episodes):\n", + " # Reduce epsilon (because we need less and less exploration)\n", + " epsilon = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*episode)\n", + " # Reset the environment\n", + " state = env.reset()\n", + " step = 0\n", + " done = False\n", + "\n", + " # repeat\n", + " for step in range(max_steps):\n", + " # Choose the action At using epsilon greedy policy\n", + " action = epsilon_greedy_policy(Qtable, state, epsilon)\n", + "\n", + " # Take action At and observe Rt+1 and St+1\n", + " # Take the action (a) and observe the outcome state(s') and reward (r)\n", + " new_state, reward, done, info = env.step(action)\n", + "\n", + " # Update Q(s,a):= Q(s,a) + lr [R(s,a) + gamma * max Q(s',a') - Q(s,a)]\n", + " Qtable[state][action] = Qtable[state][action] + learning_rate * (reward + gamma * ( np.max(Qtable[new_state]) ) - Qtable[state][action] )\n", + "\n", + " # If done, finish the episode\n", + " if done:\n", + " break\n", + " \n", + " # Our next state is the new state\n", + " state = new_state\n", + " return Qtable" + ] + }, + { + "cell_type": "code", + "execution_count": 327, + "metadata": {}, + "outputs": [], + "source": [ + "from enum import Enum\n", + "class Actions(Enum):\n", + " Sell = 0\n", + " Buy = 1\n", + " Do_nothing = 2\n", + "\n", + "class CustTradingEnv(gym.Env):\n", + " metadata = {'render.modes': ['human']}\n", + "\n", + " def __init__(self, df, max_steps=0):\n", + " self.seed()\n", + " self.df = df\n", + " self.prices, self.signal_features = self._process_data()\n", + "\n", + " # spaces\n", + " self.action_space = spaces.Discrete(3)\n", + " self.observation_space = spaces.Box(low=0, high=1999, shape=(1,) , dtype=np.float64)\n", + "\n", + " # episode\n", + " self._start_tick = 0\n", + " self._end_tick = 0\n", + " self._done = None\n", + " self._current_tick = None\n", + " self._last_trade_tick = None\n", + " self._position = None\n", + " self._position_history = None\n", + " self._total_reward = None\n", + " self._total_profit = None\n", + " self._first_rendering = None\n", + " self.history = None\n", + " self._max_steps = max_steps\n", + " self._start_episode_tick = None\n", + " self._trade_history = None\n", + "\n", + " def reset(self):\n", + " self._done = False\n", + " self._start_episode_tick = np.random.randint(1,len(self.df)- self._max_steps )\n", + " self._end_tick = self._start_episode_tick + self._max_steps\n", + " self._current_tick = self._start_episode_tick\n", + " self._last_trade_tick = self._current_tick - 1\n", + " self._position = 0\n", + " self._position_history = []\n", + " # self._position_history = (self.window_size * [None]) + [self._position]\n", + " self._total_reward = 0.\n", + " self._total_profit = 0.\n", + " self._trade_history = []\n", + " self.history = {}\n", + " return self._get_observation()\n", + "\n", + "\n", + " def step(self, action):\n", + " self._done = False\n", + " self._current_tick += 1\n", + "\n", + " if self._current_tick == self._end_tick:\n", + " self._done = True\n", + "\n", + " step_reward = self._calculate_reward(action)\n", + " self._total_reward += step_reward\n", + "\n", + " observation = self._get_observation()\n", + " info = dict(\n", + " total_reward = self._total_reward,\n", + " total_profit = self._total_profit,\n", + " position = self._position,\n", + " action = action\n", + " )\n", + " self._update_history(info)\n", + "\n", + " return observation, step_reward, self._done, info\n", + "\n", + " def seed(self, seed=None):\n", + " self.np_random, seed = seeding.np_random(seed)\n", + " return [seed]\n", + " \n", + " def _get_observation(self):\n", + " return self.signal_features[self._current_tick]\n", + "\n", + " def _update_history(self, info):\n", + " if not self.history:\n", + " self.history = {key: [] for key in info.keys()}\n", + "\n", + " for key, value in info.items():\n", + " self.history[key].append(value)\n", + "\n", + "\n", + " def render(self, mode='human'):\n", + " window_ticks = np.arange(len(self._position_history))\n", + " prices = self.prices[self._start_episode_tick:self._end_tick+1]\n", + " plt.plot(prices)\n", + "\n", + " open_buy = []\n", + " close_buy = []\n", + " open_sell = []\n", + " close_sell = []\n", + " do_nothing = []\n", + "\n", + " for i, tick in enumerate(window_ticks):\n", + " if self._position_history[i] == 1:\n", + " open_buy.append(tick)\n", + " elif self._position_history[i] == 2 :\n", + " close_buy.append(tick)\n", + " elif self._position_history[i] == 3 :\n", + " open_sell.append(tick)\n", + " elif self._position_history[i] == 4 :\n", + " close_sell.append(tick)\n", + " elif self._position_history[i] == 0 :\n", + " do_nothing.append(tick)\n", + "\n", + " plt.plot(open_buy, prices[open_buy], 'go', marker=\"^\")\n", + " plt.plot(close_buy, prices[close_buy], 'go', marker=\"v\")\n", + " plt.plot(open_sell, prices[open_sell], 'ro', marker=\"v\")\n", + " plt.plot(close_sell, prices[close_sell], 'ro', marker=\"^\")\n", + " \n", + " plt.plot(do_nothing, prices[do_nothing], 'yo')\n", + "\n", + " plt.suptitle(\n", + " \"Total Reward: %.6f\" % self._total_reward + ' ~ ' +\n", + " \"Total Profit: %.6f\" % self._total_profit\n", + " )\n", + "\n", + " def _calculate_reward(self, action):\n", + " step_reward = 0\n", + "\n", + " current_price = self.prices[self._current_tick]\n", + " last_price = self.prices[self._current_tick - 1]\n", + " price_diff = current_price - last_price\n", + "\n", + " # OPEN BUY - 1\n", + " if action == Actions.Buy.value and self._position == 0:\n", + " self._position = 1\n", + " step_reward += price_diff\n", + " self._last_trade_tick = self._current_tick - 1\n", + " self._position_history.append(1)\n", + "\n", + " elif action == Actions.Buy.value and self._position > 0:\n", + " step_reward += 0\n", + " self._position_history.append(-1)\n", + " # CLOSE SELL - 4\n", + " elif action == Actions.Buy.value and self._position < 0:\n", + " self._position = 0\n", + " step_reward += -1 * (self.prices[self._current_tick -1] - self.prices[self._last_trade_tick]) \n", + " self._total_profit += step_reward\n", + " self._position_history.append(4)\n", + " self._trade_history.append(step_reward)\n", + "\n", + " # OPEN SELL - 3\n", + " elif action == Actions.Sell.value and self._position == 0:\n", + " self._position = -1\n", + " step_reward += -1 * price_diff\n", + " self._last_trade_tick = self._current_tick - 1\n", + " self._position_history.append(3)\n", + " # CLOSE BUY - 2\n", + " elif action == Actions.Sell.value and self._position > 0:\n", + " self._position = 0\n", + " step_reward += self.prices[self._current_tick -1] - self.prices[self._last_trade_tick] \n", + " self._total_profit += step_reward\n", + " self._position_history.append(2)\n", + " self._trade_history.append(step_reward)\n", + " elif action == Actions.Sell.value and self._position < 0:\n", + " step_reward += 0\n", + " self._position_history.append(-1)\n", + "\n", + " # DO NOTHING - 0\n", + " elif action == Actions.Do_nothing.value and self._position > 0:\n", + " step_reward += price_diff\n", + " self._position_history.append(0)\n", + " elif action == Actions.Do_nothing.value and self._position < 0:\n", + " step_reward += -1 * price_diff\n", + " self._position_history.append(0)\n", + " elif action == Actions.Do_nothing.value and self._position == 0:\n", + " step_reward += -1 * abs(price_diff)\n", + " self._position_history.append(0)\n", + "\n", + " return step_reward\n", + "\n", + " def _do_bin(self,df):\n", + " df = pd.cut(df,bins=[0,10,20,30,40,50,60,70,80,90,100],labels=False, include_lowest=True)\n", + " return df\n", + " # Our state will be encode with 4 features MFI and Stochastic(only D line), ADX and DI+DI-\n", + " # the values of each feature will be binned in 10 bins, ex:\n", + " # MFI goes from 0-100, if we get 25 will put on the second bin \n", + " # DI+DI- if DI+ is over DI- set (1 otherwise 0) \n", + " # \n", + " # that will give a state space of 10(MFI) * 10(STOCH) * 10(ADX) * 2(DI) = 2000 states\n", + " # encoded as bins of DI MFI STOCH ADX = 1 45.2 25.4 90.1 , binned = 1 4 2 9 state = 1429 \n", + " def _process_data(self):\n", + " timeperiod = 14\n", + " self.df = self.df.copy()\n", + " \n", + " self.df['mfi_r'] = ta.MFI(self.df['High'], self.df['Low'], self.df['Close'],self.df['Volume'], timeperiod=timeperiod)\n", + " _, self.df['stock_d_r'] = ta.STOCH(self.df['High'], self.df['Low'], self.df['Close'], fastk_period=5, slowk_period=3, slowk_matype=0, slowd_period=3, slowd_matype=0)\n", + " self.df['adx_r'] = ta.ADX(self.df['High'], self.df['Low'], self.df['Close'], timeperiod=timeperiod)\n", + " self.df['p_di'] = ta.PLUS_DI(self.df['High'], self.df['Low'], self.df['Close'], timeperiod=timeperiod)\n", + " self.df['m_di'] = ta.MINUS_DI(self.df['High'], self.df['Low'], self.df['Close'], timeperiod=timeperiod)\n", + " self.df['di'] = np.where( self.df['p_di'] > self.df['m_di'], 1, 0)\n", + "\n", + " self.df = self.df.dropna()\n", + " self.df['mfi'] = self._do_bin(self.df['mfi_r'])\n", + " self.df['stock_d'] = self._do_bin(self.df['stock_d_r'])\n", + " self.df['adx'] = self._do_bin(self.df['adx_r'])\n", + " self.df['state'] = self.df['di']*1000+ self.df['mfi']*100 + self.df['stock_d']*10 + self.df['adx']\n", + "\n", + " prices = self.df.loc[:, 'Close'].to_numpy()\n", + " # print(self.df.head(30))\n", + "\n", + " signal_features = self.df.loc[:, 'state'].to_numpy()\n", + "\n", + " return prices, signal_features" + ] + }, + { + "cell_type": "code", + "execution_count": 330, + "metadata": {}, + "outputs": [], + "source": [ + "# Training parameters\n", + "n_training_episodes = 10000 # Total training episodes\n", + "learning_rate = 0.5 # Learning rate\n", + "\n", + "# Environment parameters\n", + "max_steps = 20 # Max steps per episode\n", + "gamma = 0.95 # Discounting rate\n", + "\n", + "# Exploration parameters\n", + "max_epsilon = 1.0 # Exploration probability at start\n", + "min_epsilon = 0.05 # Minimum exploration probability \n", + "decay_rate = 0.0005 # Exponential decay rate for exploration prob" + ] + }, + { + "cell_type": "code", + "execution_count": 331, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "REhmfLkYNTiN", + "outputId": "cf676f6d-83df-43f5-89fe-3258e0041d9d" + }, + "outputs": [], + "source": [ + "# create env\n", + "env = CustTradingEnv(df=eth_train, max_steps=max_steps)" + ] + }, + { + "cell_type": "code", + "execution_count": 332, + "metadata": {}, + "outputs": [], + "source": [ + "# create q-table\n", + "\n", + "action_space = env.action_space.n # buy sell do_nothing\n", + "state_space = 2000\n", + "\n", + "Qtable_trading = initialize_q_table(state_space, action_space)" + ] + }, + { + "cell_type": "code", + "execution_count": 333, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "690" + ] + }, + "execution_count": 333, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Qtable_trading = train(n_training_episodes, min_epsilon, max_epsilon, \n", + " decay_rate, env, max_steps, Qtable_trading, learning_rate, gamma )\n", + "len(np.where( Qtable_trading > 0 )[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 334, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 417 + }, + "id": "FIQ0OqtsO3jo", + "outputId": "f98374ad-c7de-4dc4-80b1-25f018ad96eb" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3MAAAGQCAYAAAAEBjl/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xV9f3H8dcnO2GFEVbCnrJHQMGBqwq4Z3HUXVer1rZq/bXWOutoa8W6bRWtFXErgooLQREIe8gOI5CwwwqZ9/v745xgiAESSHJyk/fz8bgPbr7n3nM+J5xA3vc7jjnnEBERERERkfASEXQBIiIiIiIiUnEKcyIiIiIiImFIYU5ERERERCQMKcyJiIiIiIiEIYU5ERERERGRMKQwJyIiIiIiEoYU5kSk1jAzZ2adg67jcJnZiWaWEXQdUrOY2VVmNrWajnWTmW00s91m1tT/s2N1HFtERCpOYU5Eqpz/C2HxI2Rme0t8fdkB3lOpwcbMvjazXP+YW8zsXTNrVVn7D4KZPWBmC8ys0Mz+Usb2S81sjZntMbP3zaxJiW3JZvaBmW0zswwzu7HUe18ws6X+39dVh6jjsPdlZqP8bTvMbJOZjTGzhiW2/9fMMs1sp5ktM7PrSmzrYWZpZrbdf3xuZj1KbDcze9TMtvqPx8zMDvmNLSczu6zEdbzXP7991/pB3veKmT1YSTW09z/EKD7uajP7w2HuKxr4B3Cac66+c26r/+eqw6nbzNqW+tnf7df6O3/7/5XaVvw9bOZvv9jMvjOzHDP7+hDHOrH099/Mriyx/TEzW+dfR2vM7I8ltnX1r9/N/jX8qZl1O8BxvvTPIapEW+lzLDKzp/xtMWb2tv/34szsxPJ+/0REykNhTkSqnP8LYX3nXH1gLXBWibbXq7GUX/s1dAbqA3+rxmPvp+Qvg0dgBXAn8HEZ++8JPA/8AmgB5ADPlHjJf4F0f9sZwMNmdlKJ7fOAm4HZ5ajjSPb1LXCsc64R0BGIAkoGhr8C7Z1zDYGzgQfNbKC/bQNwIdAEaAZ8CIwt8d7rgXOBvkAf4EzghnKcT7k4514vcV2PADaUutarU6J/zEuAP5vZ8NIvKMc11wKIAxZVRkHOubWlvh+9gRDwjr/94VLbHwW+ds5t8XexDfgn8Eg5D7nf9985N6bEtn8D3f3raChwqZmd729LxLt2uuF9D2YAH5TeuXkfPP3ke1jqHFoAe4G3SrxkKnA5kFXO8xARKTeFOREJjJnFmtk/zWyD//in31YPmAi0LvFpd2szG2xm08ws2++t+ZeZxVT0uM65bOB9oF+JWrqb2ST/k/mlZnax397BP16E//VLZrapxPv+a2a/8Z9fbWY/mNkuM1tlZjeUeN2J5vVa3WVmWcDLZhbv93ZsN7PFwKAKnscY59xEYFcZmy8DPnLOfeOc2w3cA5xvZg3MrD5wIvCQc67AOTcPeBu4psS+n3bOfQHkHqyGI92Xc25diV/eAYrwwnbx9kXOubziL/1HJ39btnNutXPOAVb6vcCVwN+dcxnOufXA34GrDnY+lcXMjjKvNzjbzBaZ2dl++/V4fzd3+tf1R377H8xspX/tLDaz8w7nuM65aXhhrNcBrrkD/cx1BZb6u8k2sy/9upyZdT5Q3RV0BfCNc2516Q1mZngfPOwLYM65z51z4/BC+xFxzi11zu0p0RTCv1acczOcc/92zm1zzhUATwDdzKxpifoaAffifXhyMBcCm4Ap/r7znXP/dM5Nxbs+RUQqlcKciATpj8AxeKGqLzAY+JP/S1fpno4NeL8M3Y7XCzMEOAWvx6dC/F/Szsfr2cIPj5OA/wHN8Xo3njGzns65dGAn0N9/+/HAbjM7yv/6BGCy/3wTXu9PQ+Bq4AkzG1Di0C3xepHa4fUa3YsXTDoBp+OFj5J1PmNmJXvTKqInXo8YAM65lUA+0BUv+FDiz+LnvQ7jOEe8LzM7zsx24IXSC/B6Y0puf8bMcoAlQCYwodT2bLyg+BTwcIlN+30P/Oc9y1tXqWOUe3imecMVPwI+w7uebgFeN7NuzrkXgNeBx/zr+iz/bSvxrq1GwH3Af62Cw4DNcyzeOc7xm0tfcwf6mVvGj9+bROfcySX3faC6K3iNXkGJsFbK8Xi9Wu+Uc19laW7efL90M3vC/7nexw/Mu4EMoB7ez3tZTgCynHNbS7Q9DDzLoXvXrgRe9T9gEBGpcgpzIhKky4D7nXObnHOb8X6J/cWBXuycm+Wc+945V+h/uv88MKwCxxvth4YteIHwFr/9TGC1c+5lf9+z8X6pvNDfPhkYZmYt/a/f9r/ugBfc5vn1feycW+k8k/F+mT++xPFDwL3OuTzn3F7gYrwerW3OuXXA6FLne7NzrsJh1Vcf2FGqbQfQwDm3C2944z1mFucHzguAhIoepDL25Zyb6g+zTAEeB1aX2n4z0ADve/kukFdqeyJeCPo1P4YY+On3YAdQ/0DBzO9ZnecHgtfMbJiZNTezq4Hbyns+eGGpPvCI3zPzJTAe70OCMjnn3nLObXDOhZxzbwLL8YJWeW3BG5b4EvAHvycUfnrNVehn7lDKe42aWXFYe/sAL7kSeNvvRT4cS/ACaivgZGAg3vy/krU+gncdDQBe46c/H5hZCvA08NsSbanAsXgfFhyQmbXF+/foQIFVRKTSKcyJSJBaA2tKfL3GbyuTeQsVjDezLDPbifdpebMKHO9WPzT0ARrjhQfwei2O9ofEZfs9PZfh9WqAF+ZOxPvE/hvga7xf2oYBU5xzIb++EWb2vT9UMxsYWaq+zc65kkMNWwPrSp1/ZdmNFzRLasiPQzIvAzr4x38Wr9flcBecqZR9+UMhP2H/eW/F24r8oWopwE1lbN8DPAe8ambN/ebS34OGwO6D9JqcAQwHuuAF1CfwgvqJ/jmVV2tgXfF14VsDJB/oDWZ2hZnNLXH99aJi13Yz51xj59xRzrmSHwqUdc2V+2euEl0JvFNWWDOzeOAijiAEOeeynHOL/TCcjjcc8sIyXuecc3Pw5rXdV6qOJLwPYJ5xzr3ht0XgzTW9zTlXeIgyrgCm+scXEakWCnMiEqQNeEGqWFt+nB9T1i/cz+J9At/FeQsZ/B/7D+8rF+fcArxFNp72e2nWAZOdc4klHvWdc8WhYTJer9CJ/vOpeJ/UD/O/xsxi8Xrz/ga08HuLJpSqr/Q5ZQJtSnzdtqLnchCL8IbR4dfXEYgFlgE459Y45850ziU5544GmuIt/FBhlbkvvAUmOh3m9gi8HsHi0LTf98B/frDFPS52zmU653Y6555zzg1wzrVyzl3p92KV1wagjR8EirUF1vvP97sOzKwd8CJez2JT/9pZyGFc22Uofc0d7Geuovsql3KEtfPxehW/Ppz9H0DxPMoD2e86MrPGeEHuQ+fcQyVe1xBIBd705x3O9Nsz/N7Gkg42jFREpEoozIlIkN4A/mRmSeYtR/5nvJURATYCTf2FB4o1wJu/ttvMulNGD00FjMGbz3Q23hC4rmb2CzOL9h+DiufFOeeW432SfzneAg47/fou4Mf5cjF4YWkzUGhmI4DTDlHDOOBuM2vsD++65RCv349fZxzev+VR/jDHSH/z68BZZna8P3fofuBdf1hk8QIdDcxbOv1yv9Z/lNh3jL9vA6L9fZf5f8aR7Mu85f3b+vO92gEPAV/425qbd+uC+mYWaWan4w1VLF6c42dm1t/f1tA/5nbgB//QrwK/Ne/WCa2B3wGvHOj7Waon7UhMB/bgLRYSbd5y9GfxY4/jRryVO4vVwwsfm/3zuprDm79YHgf7mTuU0nWX13lANvDVAbaXOc/M/3uNwwteEf51E13WDsxb7KX4OmqDtwLmB/62CDO7wf85MzMbDPyKH6+zhsCnwLfOudK3ddiB13PZz3+M9NsH4v09Fx9/KN6HCG+Ven/xQk9x/pcx/nlU2i0yRKRuU5gTkSA9CKQB84EFeEvXPwjgnFuC94vnKn/oWWvg98CleEMFXwTePNwDO+fy8eao3eMHnNOAUXi9FFl4y6THlnjLZGCrc25tia8Nf46Wv49b8QLadr/ODw9Rxn14w9zS8XoFXiu50cyeM7PnDvL+F/FC5iV4C1vsxZ//5JxbBNyIF+o24QXhknObTgdW+bXeCAwv1fv0mb+/ocAL/vMT/LouM7NFlbEvoAfwHd6QyG/xVlT8pb/N4QX2DH/ffwN+45wrXjY+Ee8a2YG3gEhn/9jFwwqfx1uIZAFeT9fHfluV8q+ts/EW8dmCN0zvCv+aBm+Z/B7+df2+c24x3kqb0/ACU2+870VVOODPXDnsVzeU6xqFgywKYmbJeHPcXi3jfb/Au1aexesZ34t3zRe/d3eJ3rEBeN+/PXjX00K8n8di5+FdI7vwwutT/DgH7jy8lWSvtv3vF9fWH5aZVfzAD9zARv/vueQ57vuwpJSlfu3JeKFxL/v3joqIHDY78NQBERERERERqanUMyciIiIiIhKGFOZERERERETCkMKciIiIiIhIGFKYExERERERCUMKcyIiIiIiImFIYU5ERERERCQMKcyJiIiIiIiEIYU5ERERERGRMKQwJyIiIiIiEoYU5kRERERERMKQwpyIiIiIiEgYUpgTEREREREJQwpzIiIiIiIiYUhhTkREREREJAwpzImIiIiIiIQhhTkREREREZEwpDAnIiIiIiIShhTmREREREREwpDCnIiIiIiISBhSmBMREREREQlDCnMiIiIiIiJhSGFOREREREQkDCnMiYiIiIiIhCGFORERERERkTCkMCciIiIiIhKGFOZERERERETCkMKciIiIiIhIGFKYExERERERCUMKcyIiIiIiImFIYU5ERERERCQMKcyJiIiIiIiEIYU5ERERERGRMBQVdAGH0qxZM9e+ffugyxAREREREQnErFmztjjnkkq31/gw1759e9LS0oIuQ0REREREJBBmtqasdg2zFBERERERCUMKcyIiIiIiImFIYU5ERERERCQMKcyJiIiIiIiEIYU5ERERERGRMHTIMGdmcWY2w8zmmdkiM7uvxLZbzGyp3/5Yifa7zWyFv+30Eu0DzWyBv220mVnln5KIiIiIiEjtV55bE+QBJzvndptZNDDVzCYC8cA5QB/nXJ6ZNQcwsx7AKKAn0Br43My6OueKgGeB64HvgQnAcGBiZZ+UiIiIiIhIbXfInjnn2e1/Ge0/HHAT8IhzLs9/3Sb/NecAY51zec65dGAFMNjMWgENnXPTnHMOeBU4t3JPR0REREREpG4o15w5M4s0s7nAJmCSc2460BU43symm9lkMxvkvzwZWFfi7Rl+W7L/vHR7Wce73szSzCxt8+bNFTsjERERERGROqBcYc45V+Sc6wek4PWy9cIbotkYOAa4Axjnz4Erax6cO0h7Wcd7wTmX6pxLTUpKKk+JIiIiIiIidUqFVrN0zmUDX+PNdcsA3vWHYc4AQkAzv71NibelABv89pQy2kVERERERKSCyrOaZZKZJfrP44FTgSXA+8DJfntXIAbYAnwIjDKzWDPrAHQBZjjnMoFdZnaM34N3BfBBFZyTiIiIiFRAKOTYviefVZt3M3vtdqat3MrO3IKgyxKRQyjPapatgDFmFokX/sY558abWQzwHzNbCOQDV/oLmywys3HAYqAQ+JW/kiV4i6a8grcS5kS0kqWIiIhIpdqbX8T2nHy25+SzI6eA7TkFbM/JJzsnv8Tz/f/csbcAV2ryS4TBUa0aMrhDE47u0IRB7ZvQtH5sMCclImUyV/ont4ZJTU11aWlpQZchIiIiUq2KQo4de38MYtl+MMv2g9q+53v2D2Z5haED7jMhJpLGCTEkJkTv92fjhGgSE2JoXM/7M8KMOWu3MyN9G7PXbie3wNtn5+b1GdTeC3eDOzShdWJ8dX07ROo0M5vlnEst3V6enjkRERERqWSZO/by3pz1bNmV/9OAllPAztyf9pYVi4wwEuOj94WxlMYJ9E6OpnG9EkEt/seAVhzcYqMiy13fsK7eInT5hSEWbtjBjPRtzEjfxvj5G3hjxloAUhrH7+u5G9yhKe2bJuDNphGR6qCeOREREZFqVFgUYsy0Nfzjs6XsyS+iXkzkfqGrUXzZvWUl2xrERhEREUxoKgo5lmTt3BfuZqRvY+uefACSGsQyuL3Xaze4QxO6tWgQWJ0itcmBeuYU5kRERESqydx12fzxvQUs2rCTE7slcf/ZvWjbNCHoso6Ic45VW/bsC3bTV21lw45cABrGRe0LdoM7NKVn64ZER1ZoMfUaJye/kA3Ze1mfncv67XvZkL2XnbkF3HZKF80plCqjYZYiIiIiAdmZW8DfPl3Ka9+voXmDWJ65bAAjerWsFUMSzYxOSfXplFSfSwa3BSBje85+PXef/7AJgPjoSAa2a7wv4PVrk0hcdPmHflY15xxb9+R7YW37XtZne48N/p/rt+9le87+q3xGRhjOOTbvyuOZywbUir9TCR8KcyIiIiJVxDnHR/MzeWD8YrbuzuPKIe353WldaRAXHXRpVSqlcQIpjRM4f4B3i+FNu3JJW+0tqDI9fRtPfL4M5yAmMoK+bRrt67kb2K4x9WOr7tfTgqIQWTtyydj+Y0DbF9T858WLvRRLiIkkOTGe1onx9ElJJDkxft/XyY3jadEglhemrOKxT5Yyfn4mZ/VtXWX1i5SmYZYiIiIiVWDN1j386f2FTFm+hd7JjXj4vN70TmkUdFk1wo6cAtLWbNsX7hau30FhyBFh0LN1o309d4PaN6FJvZhy73dXbgEbsnNZn53j96zl/hjYtu9l467cnywq06x+LMmJcSQ3jqd1Iy+gtfYDW0rjeBrFRx+yt62wKMQFz01j7dY9fHb7MJIaaLilVC7NmRMRERGpBnmFRbwweRX/+moF0ZER3HF6Ny4/ph2RWgjkgHLyC5mzNpvp6duYkb6VOWuz991ioWuL+vuCXe/kRmTvLdgXzop71TL2zV0r3G+/0ZFGq0b796SllHjeqlFcpQ3zXL5xF2eMnspJ3ZN47vKBGm4plUphTkRERKSKTVu5lT+9v4CVm/dwRu9W/PmsHrRoGBd0WWEnr7CIBRk7/HC3jVlrtrM7r/Anr2sYF0Vrvwetdanhj8mJ8STVj63W1TSf/Xolj36yhCdH9eOcfsnVdlyp/RTmRERERKrI1t15PDxhCe/MzqBNk3juP6cXJ3VrHnRZtUZhUYglWbtYnLmTZvVj9gW3mjb3sLAoxIXPTWP11j18dvsJNG+gIC+VQ2FOREREpJKFQo63Zq3jrxOXsCevkOtP6MivT+pCfEzNWaFRqteKTbsZOXoKJ3ZN4vlfaLilVA7dmkBERESkEi3buIs/vreAmau3M7h9Ex46rxddWjQIuiwJWOfm9fndz7ry14lL+HDeBg23lCqlMCciIiJSAXvzixj95XJe/GYVDeKieOzCPlw0MEU9MLLPdcd35JNFWdz74SKGdGqq4ZZSZSKCLkBEREQkXHy1ZBM/e2Iyz369kvP6J/PF707k4tQ2CnKyn8gI4/EL+5KTX8Qf31tITZ/WVG369weznz769w+6srClMCciIiJyCFk7crnpv7O4+pWZxEVH8ub1x/D4RX0rdA80qVs6N6/P70/ryqTFG/lg7oagy6kZhgyBmFI/MzExMHRoMPXUAloARUREROQACotCvDptDX//bCmFIcetp3Thl8d3JCZKn4fLoRWFHBc99x0rN+9h0u0n0Lyu36YiM5OiDh2JzMv9sS0+HlatgpYtg6srDBxoART9SyQiIiJShnnrsjn3mW+5f/xiUts3YdLtw/jVSZ0V5KTcIiOMxy/qS25BEf/33oI6P9xyZVQD3u59CvmR3rIdeZFRfD30DFZE1Au4svClf41EREREStiZW8C9Hyzk3Ge+ZdPOPJ6+dACvXD2Itk0Tgi5NwlCnpPrccXo3Pv9hE+/NWR90OYHJzsnnujFp/Oeky4mK9sJcRGQk9/Y9n9Oe+IbfvzWPjO05AVcZfhTmRERERADnHOPnb+DUv0/m1e/XcMUx7fj8d8M4o08rLXAiR+TqYzuQ2q4xf/lwERt35h76DbVMQVGIm1+fzfrte3no5tOIuPpqiIgg+rpree/+C7jm2A58OG8DJ/9tMn/5cBFbducFXXLY0Jw5ERERqfPWbs3hng8WMnnZZnolN+Th83rTJyUx6LKkFlm1eTcjnpzCcZ2b8dKVqXXmAwLnHP/33kLemLGWf1zcl/MHpEBmJowaBW++uW+u3IbsvYz+YjlvzcogNiqCa4/rwC9P6EjDuOiAz6BmONCcOYU5ERERqbPyC0O8OGUVo79YTnRkBL87rStXDGlPZETd+EVbqtdLU1bx4Mc/8PeL+nLBwJSgy6kW/5mazv3jF3PziZ24c3j3Q75+1ebd/GPSMsbPzyQxIZqbhnXiyqHtiYuOrIZqay6FOREREZESpq/ayh/fX8iKTbsZ0asl957Vk5aN6vhqg1KlikKOnz8/jWUbdzHpt8NoUctXt/xq6SaufWUmP+vRgmcvG0hEBT4kWbh+B3/7bClfL91Mi4ax3HJyF34+qA3RkXVzlpjCnIiIiAiwbU8+D0/4gbdnZZDSOJ77z+nJyd1bBF2W1BHpW/Yw4slvGNqpGf+uxcMtl23cxfnPfEfbJgm8fdMQEmKiDms/M9K38dgnS0hbs512TRP47c+6claf1hUKhrWBbk0gIiIidUv//mD2k8emzj14f856bjqxE5NuH6YgJ9WqQ7N63HF6d75csom3Z2UEXU6V2Lo7j2vHzCQ+JpJ/X5V62EEOYHCHJrx14xD+c1Uq8dGR3DZ2LiNHT+GLHzbW+Vs9gMKciIiI1FZDhkBMzH5NeZFRrO7ah49vPZ67hncnPqZuz8ORYFw9tD2D2zfh/vGLydpRu1a3zCss4sb/zmLTzjxeuiKVVo3ij3ifZsbJ3Vsw4dbjeXJUP/YWFHHtmDQufG4a01dtrYSqw5eGWYqIiEjtlJkJHTtC7o+/LBfGxhGxahURrVsFWJgIrN6yh+FPfsMxHZvy8lWDasVwS+ccv39rPu/MzuBfl/bnzD6tq+Q4BUUhxqWtY/QXy9m4M49hXZO44/Ru9EpuVCXHqwk0zFJERETqllat2HXJ5eRFekO8XEwMUddeoyAnNUL7ZvW4a3h3vl66mbdqyXDL5yav4p3ZGfzm1C5VFuQAoiMjuOzodky+4yTuHtGdeRnZnPnUVH71v9ms3Ly7yo5bEynMiYiISK1UWBTiN13OxJn3645FRsI99wRclciPrhzSnsEdmvDAR4vJ3LE36HKOyKeLsnjs0yWc1bc1t53SpVqOGRcdyQ3DOvHNnSdxy8md+WrJJk574hvuens+G7LD+/tZXgpzIiIiUis9+cVyvtgRReZ5oyAiAq6+et8NikVqgogI4/EL+1AYcvzhnQVhu6DHog07+M3YufRJSeTxC/tU+5DRhnHR/O60bnxz50lcMaQd781Zz4l/+5oHxi9m6+68aq2luinMiYiISK0zbeVW/vXVCi4amEKHJx+B445Tr5zUSO2a1uOu4d2YvGwzb6WF33DLTbtyuW5MGokJ0bz4i4GB3ty7Wf1Y7j2rJ1/+fhjn9G3Ny9+mc8JjX/HEpGXsyi0IrK6qdMgwZ2ZxZjbDzOaZ2SIzu89v/4uZrTezuf5jZIn33G1mK8xsqZmdXqJ9oJkt8LeNttow01NERERqlO178rn9zbl0aFqPv5zdE1q1gsmT1SsnNdYVQ9pzdIcmPDB+cVgND8wtKOKXr84iO6eAl65MpXkNuQl6SuMEHr+oL5/dfgIndE3iyS+Wc8JjX/HSlFXkFhQFXV6lKk/PXB5wsnOuL9APGG5mx/jbnnDO9fMfEwDMrAcwCugJDAeeMbPiiP4scD3QxX8Mr7xTERERkbrOOcdd78xn6548Rl/Sn3qxh39/K5Hq4g237OsNt3w3PIZbOue44+35zM/I5p+j+tGzdc1bSbJz8wY8e/lAPvz1sfRKbsSDH//ASX/7mrEz1lJYFAq6vEpxyDDnPMXLwkT7j4NdYecAY51zec65dGAFMNjMWgENnXPTnHeFvgqce2Tli4iIiPzo9elr+WzxRu4a3r1WL1MutU/bpgncPbI73yzbzJsz1wVdziGN/mIFH83bwJ2nd+f0njW717tPSiKvXXs0//vl0bRoGMcf3l3Az574ho/mbSAUqvnB+WDKNWfOzCLNbC6wCZjknJvub/q1mc03s/+YWWO/LRkoeQVm+G3J/vPS7WUd73ozSzOztM2bN1fgdERERKSuWrZxFw+MX8wJXZO45tgOQZcjUmGXH92OYzo24cGPf2B9DR5uOX7+Bp74fBnnD0jmxmEdgy6n3IZ2asZ7Nw/lxStSiYmM4JY35nDWv6by1dJNYdEbWpZyhTnnXJFzrh+QgtfL1gtvyGQnvKGXmcDf/ZeXNQ/OHaS9rOO94JxLdc6lJiUlladEERERqcNyC4q45X9zaBAXxd8v6ktEhKblS/gpHm4Zco4/vDO/RgaMeeuy+d24eQxq35i/nt877G52bmb8rEcLJtx2PE/8vC87cwu4+uWZ/Pz570lbvS3o8iqsQqtZOueyga+B4c65jX7ICwEvAoP9l2UAbUq8LQXY4LenlNEuIiIickQenvADSzfu4m8X9SWpQWzQ5YgctjZNErh7RHemLN/C2Bo23DJzx15++WoaSQ1iee7ygcRGBbdy5ZGKjDDO65/CF789kQfO6Un61j1c+Nw0Xv42PejSKqQ8q1kmmVmi/zweOBVY4s+BK3YesNB//iEwysxizawD3kInM5xzmcAuMzvGX8XyCuCDSjwXERERqYMmLd7Iq9PWcN1xHTixW/OgyxE5Ypcd3Y4hHZvyUA0abpmTX8h1Y9LIyS/iP1cNomn92vGhSUxUBL8Y0p7Jd5zIncO71fj5f6WVp2euFfCVmc0HZuLNmRsPPObfZmA+cBJwO4BzbhEwDlgMfAL8yjlXvAboTcBLeIuirAQmVubJiIiISN2StSOXO9+eR8/WDbljeLegyxGpFBERxmMX9qkxwy1DIcdv35zHD5k7eerS/nRt0SDQeqpCQkwUN5/YmdaJ8UGXUiGHXK/XOTcf6F9G+y8O8gC56ZcAACAASURBVJ6HgIfKaE8DelWwRhEREZGfKAo5bn9zLrkFIUZf0j+sh3yJlNamSQJ3jzyKe95fyBsz1nHp0W0Dq+Vvny3lk0VZ3HNmD05S73eNUqE5cyIiIiI1xXOTVzJt1VbuO6cnnZLqB12OSKW7bHBbju3clIc+XkzG9pxAanh3dgbPfL2SSwa35Zpj2wdSgxyYwpyIiIiEnTlrt/OPScs4s08rLhqYcug3iIShiAjj0Qv6AHBXAMMt01Zv4w/vLGBop6bcf07PsFu5si5QmBMREZGwsjO3gFvHzqFlwzgeOi/8lkYXqYiUxgn83xlH8e2Krbw+fW21HXfdthxueG0WyY3jeeayAURHKjbURPpbERERkbDhnOOe9xeyITuX0Zf0o1F8dNAliVS5Swe35bjOzfjrhB9Yt63qh1vuyi3gujFpFBSFeOnKVBITYqr8mHJ4FOZEREQkbLw7ez0fzN3Ab07pwsB2TYIuR6RamBmPXOD1Qt/1znxCoaobblkUctw2di4rNu/mmcsGaj5qDacwJyIiImEhfcse/vzBQo7u0ISbT+ocdDki1SqlcQL/N/Iovlu5lddnVN1wy79O+IEvl2zivrN7clyXZlV2HKkcCnMiIiJS4+UXhrj1jTlERUbwxM/7ERmheXJS91wyuA3Hd6m64ZZjZ6zlpanpXDW0PZcf067S9y+VT2FOREREary/f7aUBet38OgFfcLupr4ilcUbbtmHCDPufLtyh1tOW7mVP72/kBO6JvGnM46qtP1K1VKYExERkRptyvLNPP/NKi49ui3De7UMuhyRQCUnxvOnM45i2qqtvD59TaXsc/WWPdz0+izaN6vHvy7tT5RWrgwb+psSERGRGmvL7jx+O24eXZrX554zegRdjkiN8PNBbTihaxIPT1jC2q1HNtxyx94CrhkzEwP+fWUqDeO0Qmw4UZgTERGRGsk5xx1vzWPH3gKeurQ/8TGRQZckUiOYGY+c35uoCOOOt+cd9nDLwqIQv/7fbNZty+H5X6TSrmm9Sq5UqprCnIiIiNRIL3+7mq+WbuaPI4+ie8uGQZcjUqO0ToznT2cexfT0bbz2/eENt7zvo8VMWb6Fh8/rzeAOutVHOFKYExERkRpn0YYdPDJxCace1ZwrhmhVPZGyXJzahmFdk3hk4hLWbN1Tofe+Om01r32/hhtO6MhFqW2qpkCpcgpzIiIiUqPk5Bdy6xtzSEyI5rEL+2Km2xCIlKX4ZuLecMvyr275zbLN3PfRYk49qgV3Du9exVVKVVKYExERkRrl/o8Ws2rLHv758340qRcTdDkiNVqrRvHcc2YPZqRv49Vpqw/5+hWbdvGr/82mS/P6PDlK92wMdwpzIiIiUmNMWJDJ2JnruHFYJ4Z2bhZ0OSJh4aLUFE7slsSjnyw96HDL7XvyuXZMGrFRkfz7qkHUi42qxiqlKijMiYiISI2wPnsvf3hnPn3bJPLbn3UNuhyRsGFm/PX83kRFHni4ZX5hiBv+O4vMHbm8cMVAkhPjA6hUKpvCnIiIiASusCjEb8bOIeRg9Kh+ROumxSIV0qpRPH/2h1uOmbZ6v23OOf70/gJmpG/j8Qv7MKBt40BqlMqnfylFREQkcE99uYKZq7fz4Lm9dK8rkcN04cAUTu7enEc/WUL6lh+HW740JZ1xaRncenJnzumXHGCFUtkU5kRERCRQM9K38dSXyzm/fzLn9tcvmiKHy8x4+LzeREdGcOfb8wit38D2QUN44a3vGNm7Jb85VcOXaxuFOREREQnMjpwCfjN2Dm2aJHD/ub2CLkck7LVsFMe9Z/Vk5urtzLzudhrNms5f5r7D3y/qR4RWrqx1FOZEREQkEM45/vDufDbtymP0qP7U18p6IpXiggHJnNcygr6fv0+Ec4xM+5T4bZuDLkuqgMKciIiIBGLszHVMXJjF70/vRt82iUGXI1JrmBkPLXyPSLxVLS1UBA88EHBVUhUU5kRERKTardi0i/s+WsRxnZtx/fEdgy5HpHbJzCTh9deILizwvs7Ph5dfhqysYOuSSqfxDCIiImHAOcfKzbuZunwLU1dsZeuePM7s05pz+7Wmaf3YoMurkNyCIm55Yy4JMVH84+K+mscjUtkeeABCof3bivzeuaefDqYmqRIKcyIiIjXUpp25TF2xhakrtvDtii1s3JkHQLumCdSPjeKB8Yt5ZOIPnHpUCy5ObcPxXZoRFQb3Z3tk4hJ+yNzJf65KpXnDuKDLEal9pk3zeuNKys+H774Lph6pMgpzIiIiNcTuvEKmr9q6L7wt27gbgMYJ0Qzt3Izj/EebJgkALM3axVtp63hvznomLsyieYNYLhiYwkUDU+iYVD/IUzmgL5ds5JXvVnPV0Pac3L1F0OWI1E5z5gRdgVQTc84FXcNBpaamurS0tKDLEBERqXQFRSHmrcv2et+Wb2HuumwKQ47YqAgGd2jCcZ2bcWznZvRo1fCgQxHzC0N8uWQTb6Wt46ulmwg5GNS+MReltuGM3q2oV0NWidy0M5fhT06hRcM43rt5KHHRkUGXJCISFsxslnMu9SftCnMiIiLVwznH8k3evLdvV2zh+1Vb2ZNfhBn0SW7EsX7P24B2jQ876Gzamcu7c9YzLm0dqzbvISEmkjN6t+LiQW1IbdcYs2Dmp4VCjiv+M4O0NdsYf8txdG7eIJA6RETC0YHC3CE/qjOzOOAbINZ//dvOuXtLbP898DiQ5Jzb4rfdDVwLFAG3Ouc+9dsHAq8A8cAE4DZX09OkiIjIEcjakcu3/ry3qSu2sHmXN++tQ7N6nDcgmeM6N+OYjk1JTIiplOM1bxjHjcM6ccMJHZm9djvjZmYwfv4G3pqVQYdm9bhwYAoXDEihZaPqnav2wpRVTF2xhYfP660gJyJSSQ7ZM2feR3j1nHO7zSwamIoXwr43szbAS0B3YKBzbouZ9QDeAAYDrYHPga7OuSIzmwHcBnyPF+ZGO+cmHuz46pkTEZFwsjO3gOmrtu0LcCs2efPemtaL8ee9NeXYzs1IaZxQbTXtyStk4sIsxqWtY0b6NiIMhnVN4uLUNpxyVAtioqp20ZR567K54Nnv+FmPFjxz2YDAegdFRMLVYffM+T1nu/0vo/1HcQJ8ArgT+KDEW84Bxjrn8oB0M1sBDDaz1UBD59w0v6BXgXOBg4Y5ERGRmiy/MMSctdv3hbd5GTsoCjnioiM4ukNTfp7ahmM7N6N7ywaBLcFfLzaKCwemcOHAFNK37OHtWet4e1YGN70+myb1Yji3XzIXpaZwVKuGlX7s3XmF3Dp2Ds0bxPLI+X0U5EREKlG5ZkSbWSQwC+gMPO2cm25mZwPrnXPzSv3DnIzX81Ysw28r8J+Xbi/reNcD1wO0bdu2fGciIiJSWfr3h7lzf9rerx9u9myWbty1b97b9PRt5OQXEWHQJyWRm4Z14rguzejfNpHYqJq3wEeHZvW44/Tu/PZn3fhm+WbeTsvgte9X859v0+md3IiLU1M4u28yjRKiK+V4f/5gIeu25TD2+iGVtk8REfGUK8w554qAfmaWCLxnZn2APwKnlfHysj5ycwdpL+t4LwAvgDfMsjw1ioiIVJohQ2Dx4v3u01QUHcN3SV24/aEv2LLbm/fWMcmbg3asP++tUXz4hJXICOOkbs05qVtztu3J54O563lz5jru+WARD3z8A8N7tuSi1BSO7dTssHsU35+znndnr+fWU7owuEOTSj4DERGp0FrFzrlsM/sabyhlB6C4Vy4FmG1mg/F63NqUeFsKsMFvTymjXUREpGa55x54+eX9mvIdPNj/Ao7t3HTfLQNaJ8YHVGDlalIvhquP7cBVQ9uzaMNO3kpbx/tzN/DhvA0kJ8bvu3dd8f3tymPt1hz+9P5CUts15taTO1dh9SIidVd5FkBJAgr8IBcPfAY86pwbX+I1q4FUfwGUnsD/+HEBlC+ALv4CKDOBW4DpeAugPOWcm3Cw42sBFBERqW5FIcfcsy6l16dvE1tUSGF0NLsuvYLEl1+sM3O+cguKmLR4I+PS1jF1xRacgyEdm3LxoBSG92xFfMyBh5AWFIW48LlprNq8m4m3HV+ti72IiNRGh70ACtAKGOPPm4sAxpUMcqU55xaZ2ThgMVAI/MofpglwEz/emmAiWvxERERqmD15hdw2di7zOozku8j3oKiQqKgoGj/yINSRIAcQFx3JWX1bc1bf1qzP3su7szJ4a1YGt785jz/HLuKsfq25OLUNfVMa/RhwMzNh1CheuOFB5q3bydOXDlCQExGpQrppuIiIiG999l6uG5PGso27uPesHlzx2mPw/PNw443w9NNBlxe4UMgxPX0bb6WtY8LCTHILQnRpXp+LU9twbv9kku66Hffc87zWbziL/u+vPHphn6BLFhGpFQ7UM6cwJyIiAsxZu51fvjqLvIIi/nXZAIZ1TdrX08Sbb0LLlkGXWKPszC3g4/mZjEtbx5y12bTK2c7k564jpiCP3OhYQitWkNA25dA7EhGRQzqSYZYiIiK12ofzNnDHW/No0TCON355NF1aNPA2tGoFkycHW1wN1TAumksGt+WSwW1ZvnEXO6+5HkLerIoYc0Q8+lf1ZoqIVLGIoAsQEREJinOOf36+jFvfmEPflETe/9WxPwY5Kbcuod0M/PJ9YooKAYjIz/dWA83KCrgyEZHaTWFORETqpNyCIm4dO5d/fr6cCwak8Np1g2lSLybossLTAw9AKLR/W1GR1y4iIlVGwyxFRKTO2bQrl+tfncW8jGz+MKI7N5zQsc7ccqBKTJu23w3WAe/r774Lph4RkTpCYU5EROqUxRt2ct2YmWzPKeC5ywdyek8tbHLE5swJugIRkTpJYU5EROqMSYs3ctvYOTSKj+atG4fQK7lR0CWJiIgcNoU5ERGp9ZxzvPDNKh75ZAl9khvx4hWpNG8YF3RZIiIiR0RhTkREarX8whB/en8B49IyOKNPK/5+UV/ioiODLktEROSIKcyJiEittX1PPjf+dxbT07dx6yld+M0pXYiI0EInIiJSOyjMiYhIrbRi026uHTOTzB25PDmqH+f0Sw66JBERkUqlMCciIrXOlOWbufn12cRGRTD2+mMY0LZx0CWJiIhUOoU5ERGpVV6btpq/fLSYLs3r89KVqaQ0Tgi6JBERkSqhMCciIrVCYVGIBz/+gVe+W80p3Zvz5CX9qR+r/+ZERKT20v9yIiIS9nbmFvDr/83hm2Wb+eXxHfjDiKOI1EInIiJSyynMiYhIWFu7NYdrxsxk9ZY9PHpBb34+qG3QJYmIiFQLhTkREQlbM9K3ccNraTjgtWuPZkinpkGXJCIiUm0U5kREJCy9PSuDu9+dT5vGCfz7qkF0aFYv6JJERESqlcKciIiElVDI8dinS3lu8kqO69yMpy8dQKOE6KDLEhERqXYKcyIiEjZy8gv5zdi5fLZ4I5cd3Za/nN2T6MiIoMsSEREJhMKciIiEhcwde7n2lTSWZO3k3rN6cNXQ9phpxUoREam7FOZERKTGm7cum+teTWNvfhH/vmoQJ3VrHnRJIiIigVOYExGRGm38/A38btw8khrE8vp1R9O1RYOgSxIREakRFOZERKRGcs7x1Jcr+MekZaS2a8zzvxhI0/qxQZclIiJSYyjMiYhIjZNbUMRd78zng7kbOH9AMn89vzexUZFBlyUiIlKjKMyJiEiNsnlXHte/lsactdnccXo3bj6xkxY6ERERKYPCnIiIBKb/8/2ZmzX3J+2xriPvXz6V4b1aBVCViIhIeNDNeUREJDBDUoYQExmzX5sRxbk9TlKQExEROQSFORERCcw9J9xDhO3/X1FsVDT/HPlgQBWJiIiEj0OGOTOLM7MZZjbPzBaZ2X1++wNmNt/M5prZZ2bWusR77jazFWa21MxOL9E+0MwW+NtGmyZBiIjUaa0atOLqfldjRAMQExnDNf2upmX9lgFXJiIiUvOVp2cuDzjZOdcX6AcMN7NjgMedc32cc/2A8cCfAcysBzAK6AkMB54xs+IlyJ4Frge6+I/hlXkyIiISfq7u/Ttw3md7kRbJPcPuCbgiERGR8HDIMOc8u/0vo/2Hc87tLPGyeoDzn58DjHXO5Tnn0oEVwGAzawU0dM5Nc8454FXg3Mo6ERERCU+z0qFe0SkYEVytXjkREZFyK9dqln7P2iygM/C0c2663/4QcAWwAzjJf3ky8H2Jt2f4bQX+89LtZR3verwePNq2bVvOUxERkXA0cUEmQ5OuJ7fhHvXKiYiIVEC5FkBxzhX5wylT8HrZevntf3TOtQFeB37tv7yseXDuIO1lHe8F51yqcy41KSmpPCWKiEgYytiew7yMHZzbpzeTr5qsXjkREZEKqNBqls65bOBrfjrX7X/ABf7zDKBNiW0pwAa/PaWMdhERqaM+WZgFwIheCnEiIiIVVZ7VLJPMLNF/Hg+cCiwxsy4lXnY2sMR//iEwysxizawD3kInM5xzmcAuMzvGX8XyCuCDSjwXEREJMxMXZtGjVUPaN6sXdCkiIiJhpzxz5loBY/x5cxHAOOfceDN7x8y6ASFgDXAjgHNukZmNAxYDhcCvnHNF/r5uAl4B4oGJ/kNEROqgrB25zFqznd+f1jXoUkRERMLSIcOcc24+0L+M9gvKeHnxtoeAh8poTwN6VbBGERGphT5ZmAnAiN6tAq5EREQkPFVozpyIiEhlmbAwi24tGtApqX7QpYiIiIQlhTkREal2m3blMnP1NoZr4RMREZHDpjAnIiLV7rNFG3EORmqIpYiIyGFTmBMRkWo3cWEmHZPq0bWFhliKiIgcLoU5ERGpVlt35/H9qm2M7NUK7041IiIicjgU5kREpFpNWryRopBjRG/NlxMRETkSCnMiIlKtJizMol3TBHq0ahh0KSIiImFNYU5ERKpNdk4+363YwggNsRQRETliCnMiIlJtJi3eSGHIMUK3JBARETliCnMiIlJtPlmYRXJiPH1SGgVdioiISNhTmBMRkWqxM7eAKcu3MKJXSw2xFBERqQQKcyIiUi2+/GET+UUhRuhG4SIiIpVCYU5ERKrFhAWZtGwYR/82iUGXIiIiUisozImISJXbnVfI18s2M7xXSyIiNMRSRESkMijMiYhIlftqySbyC0OM1BBLERGRSqMwJyIiVe6ThVk0qx/LwHaNgy5FRESk1lCYExGRKrU3v4gvl2xieK8WRGqIpYiISKVRmBMRkSo1edkm9hYUMbKXhliKiIhUJoU5ERGpUhMWZNGkXgyDOzQJuhQREZFaRWFORESqTG5BEV/8sJHTe7YgKlL/5YiIiFQm/c8qIiJVZsryLezJL2KEhliKiIhUOoU5ERGpMhMXZNIoPpohnZoGXYqIiEitozAnIiJVIr8wxKQfNvKzHi2I1hBLERGRSqf/XUVEpEp8u3ILu3ILGdm7ZdCliIiI1EoKcyIiUiUmLsikQWwUx3ZuFnQpIiIitZLCnIiIVLqCohCfLd7IqT1aEBsVGXQ5IiIitZLCnIiIVLrvV20lO6eAEb00xFJERKSqKMyJiEilm7Agi3oxkZzQNSnoUkRERGothTkREalURSHHZ4uyOKl7c+KiNcRSRESkqhwyzJlZnJnNMLN5ZrbIzO7z2x83syVmNt/M3jOzxBLvudvMVpjZUjM7vUT7QDNb4G8bbWZWNaclIiJBmZG+ja178hnZWzcKFxERqUrl6ZnLA052zvUF+gHDzewYYBLQyznXB1gG3A1gZj2AUUBPYDjwjJkVfzT7LHA90MV/DK/EcxERkRpg4sJM4qIjOLGbhliKiIhUpUOGOefZ7X8Z7T+cc+4z51yh3/49kOI/PwcY65zLc86lAyuAwWbWCmjonJvmnHPAq8C5lXkyIiISrFDIMXFhFid1a05CTFTQ5YiIiNRq5ZozZ2aRZjYX2ARMcs5NL/WSa4CJ/vNkYF2JbRl+W7L/vHR7Wce73szSzCxt8+bN5SlRRERqgFlrt7N5Vx4jNMRSRESkypUrzDnnipxz/fB63wabWa/ibWb2R6AQeL24qaxdHKS9rOO94JxLdc6lJiVpmI6ISLiYsCCTmKgITu7ePOhSREREar0KrWbpnMsGvsaf62ZmVwJnApf5QyfB63FrU+JtKcAGvz2ljHYREakFQiHHJwuzGNY1ifqxGmIpIiJS1cqzmmVS8UqVZhYPnAosMbPhwF3A2c65nBJv+RAYZWaxZtYBb6GTGc65TGCXmR3jr2J5BfBBJZ+PiIgEZF5GNpk7cnWjcBERkWpSno9OWwFj/BUpI4BxzrnxZrYCiAUm+XcY+N45d6NzbpGZjQMW4w2//JVzrsjf103AK0A83hy7iYiISK0wcWEW0ZHGKUe1CLoUERGROuGQYc45Nx/oX0Z754O85yHgoTLa04BeP32HiIiEM+ccExZkclznZjSKjw66HBERkTqhQnPmREREyrJw/U4ytu/VKpYiIiLVSGFORESO2ISFmURFGKf10BBLERGR6qIwJyIiR8Q5x8QFmQzp1JTEhJigyxEREakzFOZEROSILMnaxeqtOYzopSGWIiIi1UlhTkREjsjEBZlEGJzWU0MsRUREqpPCnIiIHJEJC7M4ukNTmtWPDboUERGROkVhTkREDtvyjbtYsWk3I3vrRuEiIiLVTWFOREQO24QFWZjB6T0V5kRERKqbwpyIiBy2iQszGdSuCc0bxgVdioiISJ2jMFeLOOeCLkFE6pBVm3ezJGsXIzTEUkREJBAKc7XEdyu3cOwjX/L85JVBlyIidcTEhVkADO+lMCciIhIEhbkw55zj2a9XcvlL08ncmcuzk1eSk18YdFkiUgdMXJhJ/7aJtGoUH3QpIiIidZLCXBjbmVvADa/N4tFPljCidytevmoQ2TkFvDN7fdCliUgtt3ZrDgvX72SkbhQuIiISmKigC5DD80PmTm767ywytu/lnjN7cM2x7QHo2yaR/0xN57LBbYmIsGCLFJFaa+LCTEBDLEVERIKknrkw9O7sDM575lty8ot44/pjuPa4DpgZZsZ1x3UgfcsevliyKegyRaQWm7Awiz4pjWjTJCHoUkREROoshbkwkldYxB/fW8Bvx82jb0oi4289jkHtm+z3mhG9WpKcGM+LU1YFVKWI1Hbrs/cyb102IzTEUkREJFAKc2FiffZeLn7+e16fvpYbhnXk9euOpnmDn97XKSoygquPbc+M9G3Mz8gOoFIRqe0+8VexHKEhliIiIoFSmAsD3yzbzJmjp7By026eu3wgd484iqjIA//V/XxQG+rHRvHSlPRqrFJE6oqJCzI5qlVD2jerF3QpIiIidZrCXA0WCjlGf7GcK1+eQfMGcXz462PLtdhAg7hoRg1qw8cLMlmfvbcaKhWRuiJrRy5pa7YzUr1yIiIigVOYq6Gyc/K5dsxM/jFpGef0bc17vxpKx6T65X7/Vf7qlmO+W101BYpInfTpIn+IZW/NlxMREQmawlwNtHD9Ds58aipTV2zhgXN68sTP+5EQU7G7SKQ0TmBEr5a8MX0tu3ILqqhSEalrJizIpGuL+nRuXv4Pl0RERKRqKMzVMG/OXMv5z35HUcgx7oYh/GJIe8wO735xvzy+I7vyChmXllHJVYpIXbR5Vx4zVm/TKpYiIiI1hMJcDZFbUMSdb8/jrncWMLh9E8bfchz92zY+on32bZPI4PZN+M/UdAqLQpVUqYjUVZ8tzsI5GKkhliIiIjWCwlwNsHZrDhc8+x3j0jL49UmdGXPNYJrWj62UfV97fAfWZ+/l00UbK2V/IlJ3TVyQRcdm9ejaQkMsRUREagKFuYB9uWQjZz41hXXbcvj3lan8/vRuREYc3rDKspx6VAvaN03gxSmrcM5V2n5FpG7Ztiefaau2MqJ3y8Me+i0iIiKVS2EuIEUhxz8+W8o1r6SR0jiB8bcczylHtaj040RGGNcc14G567KZvXZ7pe9fROqGSYuzKAo5zZcTERGpQRTmArBtTz5XvTyD0V+u4KKBKbx781DaNk2osuNdODCFRvHRvPiNbiIuIodnwoIs2jZJoGfrhkGXIiIiIj6FuWo2d102Z46ewvT0bTxyfm8eu7APcdGRVXrMhJgoLju6LZ8uzmLN1j1VeiwRqX125BTw7YotGmIpIiJSwyjMVRPnHK99v4aLnvuOiAjjnRuHMmpw22r7xejKoe2JijBe/nZ1tRxPRGqPST9spDD0/+3dd3xV9f3H8dcnOwHCTCAQRtiyQyIiglInTtQWCypaqz8Ud6vWamurtdrWaofWWbe4cNVRcLUuZEkgYY+wAyFhGJKQnXx/f+RqY8yEJOfe5P18PPLIyfeck/u+h8O995PzPd+v4wx1sRQREfEr9RZzZhZhZkvNLM3M1pjZXb72ab6fK8wsudo+t5lZupltMLPTqrQnmdkq37oHrY38ibewpJyb5qZxx79Wc9zAbrx33URGxnds0QzdoyM4e3RP5i7bycECTSIuIg33/upMenWKZFQLv26JiIhI3RpyZa4YONE5NxoYA0wxs/HAauB84POqG5vZMGA6MByYAjxiZt/0I3wUmAUM8n1NaYon4c+27jvEeY98yVupu/jZyYN5+tKj6RQV5kmWKyb2p6CknJeW7vDk8UUk8OQVlfL5xn1MGaEuliIiIv6m3mLOVcr3/Rjq+3LOuXXOuQ017DIVeMU5V+yc2wqkA+PMLA6Ids4tcpVj5D8PnNs0T8M/fbBmD+c8tIA9uUU885OjueHkQQQ14bQDjTWsZzQTB3bj2YVbKSnTJOIiUr//rs+mpLyCM0b28DqKiIiIVNOge+bMLNjMUoFs4CPn3JI6Nu8F7Kzyc4avrZdvuXp7TY83y8yWmdmyvXv3NiSiXykrr+AP89dx5QspJMS0473rJjJ5SKzXsYDKScSzcov596rdXkcRkQAwb1Um3aPDSezd2esoIiIiUk2DijnnXLlzbgwQT+VVthF1bF7TpSdXa58H0wAAIABJREFUR3tNj/eEcy7ZOZccExPTkIh+Y29eMTOfWsrjn23hwmP68NpVxxLfufmmHWisyYNjGBTbnn9+vlWTiItInQ4Vl/Hphr2cPiLO014FIiIiUrNGjWbpnMsBPqXue90ygN5Vfo4Hdvva42tobzWWbTvAWQ99wfIdX3P/tNHce95IwkOad9qBxjIzLp+YwNrMXBZt2e91HBHxY59syKa4rILTR6iLpYiIiD9qyGiWMWbWybccCZwMrK9jl3eA6WYWbmYJVA50stQ5lwnkmdl43yiWlwBvH/Ez8APOOZ5esJXpTywmIjSYt64+jh8lxde/o0fOTexF13ZhPPmFJhEXkdrNX72Hbu3DSe7XxesoIiIiUoOGXJmLAz4xs5XAV1TeM/eemZ1nZhnAscC/zewDAOfcGmAusBZ4H7jGOVfu+12zgSepHBRlMzC/SZ+NBw4Vl3Hdyyv43XtrmTwkhneunciwntFex6pTRGgwM4/ty3/XZ5Oened1HBHxQ4Ul5XyyPpspI7oTrC6WIiIifimkvg2ccyuBxBra3wLeqmWfe4B7amhfBtR1v51/S0yE1NTvNLUDZsf2Z9irH3DV8QMC5r6SmeP78sinm3lqwTb+cP5Ir+OIiJ/5bONeCkrKOV0ThYuIiPitRt0z1+YdeyyEfXeOuJLgELqccgJXTx4YMIUcQNf24fxwbC/eXJ7B/vxir+OIiJ+ZvzqTzlGhHJOgLpYiIiL+SsVcY9xxBy7ou4csNDSUuPvv9SjQkbl8Yn+KyyqYs1iTiIvI/xSVlvOfddmcNrwHIcF6mxAREfFXepdujLg4ii+6hJLgyt6pLiwM++ll0CMwR3obGNueE4fG8sLibRSVlte/g4i0CQs27SO/uIzTR6qLpYiIiD9TMddIEXffSWhoZTFnwcFwxx0eJzoyV0xMYF9+CW+n7vI6ioj4iXmrM+kYGcqEAV29jiIiIiJ1UDHXWHFx2GWXQVAQXBa4V+W+ceyArgyLi+bJLzSJuIhASVkFH6/N4pRh3QlVF0sRERG/pnfqw3HHHTBxYsBflYPKScSvmJTApux8Ptu41+s4bVtiIph9/yvxe4PJijSbhZv3kVtUponCRUREAoCKucMRFweffRbwV+W+cdaonnSPDtck4l6rYbRUwsJgwgRv8kibNH/VHtqHhzBxUDevo4iIiEg9VMwJYSFBXDqhHwvS97F2d67XcdquO+6o7L5bVSu4L1MCR2l5BR+s3cPJR8USHhLsdRwRERGph4o5AeCicX2JDA3mqQW6OueZuDh2nTedYt9oqRVhYa3ivkwJHEu2HCCnoFSjWIqIiAQIFXMCQMeoUC5IjuedtF1k5RZ5HadNys4t4qfxp4FV/rcsqYCcm271OJW0JfNWZxIVFswJg2O8jiIiIiINoGJOvvXTiQmUVTieX7TN6yhtTkWF4+dz09ge3pHCi2biLIjXR53CdZ9kUl6hUUal+ZVXOD5cs4cTh8YSEaouliIiIoFAxZx8q2/Xdpw2rAdzFu+goKTM6zhtymOfb2ZB+j7uPHs4nf5wNzZpIlF338kXm/bxt483eh1P2oCvth1gX34JZ6iLpYiISMBQMSffccWkBA4WlvJGSobXUdqMlO1f88CHGzlrVBw/Prr3t6Olnnd6EtOS4nnov+l8vDbL65jSys1flUlEaBCTh6iLpYiISKBQMSffkdS3M2N6d+KpBVvVva8FHCws5fqXV9CzUwT3nj8SM/t2nZlx97kjGN4zmp/NTWXbvkMeJpXWrKLCMX/1HiYPjiUqLMTrOCIiItJAKubkO76ZRHzb/gL+s05Xg5qTc45fvrGSrNwiHpyeSHRE6Pe2iQgN5rGLkwgy46o5KRSWlHuQVFq75Tu+JjuvmNNHauRUERGRQKJiTr5nyvAe9OoUqUnEm9lLS3cwf/Uebj5tCIl9Ote6Xe8uUfx9+hg2ZOVx+1urcE5XTKVpzVu1h7CQIE4cGut1FBEREWkEFXPyPSHBQVx2XD+WbjtA2s4cr+O0Shv25PG7d9cyaVA3Zk3qX+/2k4fE8rOTB/PWil28sHh7CySUtsI5x/urMzl+UAwdarg6LCIiIv5LxZzU6MdH96ZDeAhPahLxJldYUs61Ly2nQ0Qof7lgDEFBVv9OwLU/GMiJQ2P53btrSdl+oJlTSluRlnGQ3QeLOENdLEVERAKOijmpUYeIUKaP6828VZnsyin0Ok6r8rv31rIpO5+//ng0MR3CG7xfUJDx1wvG0LNTJFe/uJzsPE3uLkdu/qpMQoONk47q7nUUERERaSQVc1KrnxyXAMCzX+rqXFN5b+VuXl66g9mTBzBpUOOHgO8YFcpjFyeRU1DKtS+toLS8ohlSSlvhnGPe6kyOG9iNjpHqYikiIhJoVMxJrXp1iuTMkXG8snQneUWlXscJeDsPFHDbG6tI7NOJn58y+LB/z7Ce0fzh/JEs3XqA+95f34QJpa1ZszuXnQcKOWOEJgoXEREJRCrmpE5XTEogr7iMV7/a6XWUgFZaXsF1L68AgwenJxIafGT/9c4fG88lx/bln19s5d8rM5sopbQ181ZlEhxknDJMXSxFREQCkYo5qdOo+E6MS+jCM19uo0xd+g7bAx9uJHVnDn88fxS9u0Q1ye/89ZnDSOzTiVteTyM9O69Jfqe0Hc5VThQ+YUBXOrcL8zqOiIiIHAYVc1KvKyYmsCunkPfX7PE6SkD6fONeHvtsMzPG9eHMUU3XnS0sJIhHLhpLVFgws15IUVdYaZQNWXls3XeI09XFUkREJGCpmJN6nXxUd/p1jeKfX2zVhNWNlJ1XxM/npjK4e3t+c9awJv/9cR0jeWjGWLbvL+AXr6/Uv4802LxVewgyOHW4uliKiIgEKhVzUq+gIOPyiQmk7cwhZfvXXscJGBUVjpvmppFXVMY/LhxLZFhwszzOsQO6cuuUIcxfvYd/frGlWR5DWp/5qzIZl9CFbu0bPj2GiIiI+BcVc9IgP0yKp1NUqIqFRnjiiy18sWkfvz17OIO7d2jWx/q/Sf05fUQP/jh/PQs372vWx5LAtykrj03Z+ZwxUl0sRUREApmKOWmQqLAQLjqmDx+uzWLbvkNex/F7K3Z8zf0fbODMkXHMGNe72R/PzPjztNEkdGvHdS+tIPOgJnqX2s1fvQczOG14D6+jiIiIyBFQMScNdumx/QgNCuIZTSJep4OFpVz38gq6R0dw7/kjMbMWedz24SE8PjOJotJyZs9ZTnFZeYs8rgSWzLxM7lw0jRHxFXSPjvA6joiIiByBeos5M4sws6VmlmZma8zsLl97FzP7yMw2+b53rrLPbWaWbmYbzOy0Ku1JZrbKt+5Ba6lPudIkYqMjOGdMT+YuyyCnoMTrOH7JOcftb60i82ARD85IpGNkaIs+/sDYDvx52mhSd+bw+/fWtehjS2C45cPf8HXZSg6Fz/U6ioiIiByhhlyZKwZOdM6NBsYAU8xsPPBL4D/OuUHAf3w/Y2bDgOnAcGAK8IiZfTPyw6PALGCQ72tKEz4XaQGXT0ygsLScl5bu8DqKX3r1q538e2UmN506mKS+nevfoRmcMTKOWcf354XF23kjJcOTDOKfMvMymbv2BTDHoqw32JOv6UZEREQCWb3FnKuU7/sx1PflgKnAc77254BzfctTgVecc8XOua1AOjDOzOKAaOfcIlc5fvrzVfaRAHFUXDSTBnXjuYXbKCnTJOJVbczK48531zBxYDeuOn6Ap1l+cdoQxvfvwu1vrWLN7oOeZhH/cdtHv6W0vLL7bYUr5+7P7vY4kYiIiByJBt0zZ2bBZpYKZAMfOeeWAN2dc5kAvu+xvs17ATur7J7ha+vlW67eXtPjzTKzZWa2bO/evY15PtICLp+YQFZuMe+t3O11FL9RVFrOtS8tp314CH/58WiCgrztQRwSHMRDM8bSKSqU2XOWc7BAE4q3dbtzdzNn1fNgZQCUlJfwTOozujonIiISwBpUzDnnyp1zY4B4Kq+yjahj85o+xbo62mt6vCecc8nOueSYmJiGRJQWdMLgGAbFttck4lXc/d5aNmbl88AFY4jt4B+DSsR0COeRi5LIPFjIz+amUlGhf6u27Cdv3Eq5++6gOOW6OiciIhLQGjWapXMuB/iUynvdsnxdJ/F9z/ZtlgFUHYs9Htjta4+voV0CjJlxxaQE1mXmsmjzfq/jeG7+qkxeXLKDK4/vzwmD/euPD0l9O/Obs4bx3/XZPPTfdK/jiEd25RTy+faF316V+0ZJeQkLMxZ6lEpERESOVENGs4wxs06+5UjgZGA98A5wqW+zS4G3fcvvANPNLNzMEqgc6GSprytmnpmN941ieUmVfSTATB3Ti27tw9r8JOI7DxTwizdWMrp3J246dYjXcWp08fi+nJ/Yi7/9ZyOfbMiufwdpVZxz3Pr6SgZUPMyO6w7hfuu+87XiyhVeRxQREZHD1JArc3HAJ2a2EviKynvm3gP+CJxiZpuAU3w/45xbA8wF1gLvA9c4923fntnAk1QOirIZmN+Ez0VaUERoMDPH9+OTDXtJz87zOo4nSssruOGVFeDgoemJhIX457SNZsY9541kSPcO3PhKKjsPFHgdSVrQnCU7WJC+j9vPPIreXaK8jiMiIiJNqCGjWa50ziU650Y550Y4537na9/vnDvJOTfI9/1AlX3ucc4NcM4Ncc7Nr9K+zPc7BjjnrnW64SqgXTy+D+EhQTy1oG1OIv7XjzayfEcO954/kj5d/ftDcmRYMI/PTKLCOa6ak0JRqSYUbwt27C/gD/PWMWlQNy4c18frOCIiItLE/PNSggSEru3DOX9sPG8s38W+/GKv47SoBZv28ehnm5l+dG/OHt3T6zgN0rdrO/724zGs2Z3Lr/+1WoPXtHIVFY5bXk8j2Iw//XAUlb3bRUREpDVRMSdH5PKJCZSUVTBn8Xavo7SYffnF/GxuKgNi2vPbs4d7HadRTjqqO9efOJDXUzJ4eenO+neQgPXswm0s2XqAO84eRs9OkV7HERERkWagYk6OyMDY9pw0NJYXFm1vE133KiocN81N42BhKf+4MJHIsGCvIzXaDScP5vjBMdz5zhpSd+Z4HUeawZa9+dz3wXpOHBrLtKT4+ncQERGRgKRiTo7Y5ZMS2H+ohH+t2OV1lGb35IItfLZxL3ecNYyhPaK9jnNYgoOMB6ePITY6nKvnpLC/jXWRbe3KKxw3v5ZGeEgwfzh/pLpXioiItGIq5uSIHdu/K8N7RvPkgq2temLq1J053Pf+BqYM78HFxwT2YBKdosJ47OIk9h0q4fpXVlBWXuF1JGkiT36xheU7crjrnOF0j/aPCexFRESkeaiYkyP2zSTi6dn5fLZpr9dxmkVuUSnXv7yC7tERrWYwiRG9OvL7c0fwZfp+Hvhoo9dxpAlsysrjgY82ctrw7kwdExgD84iIiMjhUzEnTeLMkT3pER3Bk61wEnHnHL96azW7cgp5cMYYOkaFeh2pyVyQ3JsZ4/rw6KebeX/1Hq/jyBEoK6/g5tfSaBcWzO/PVfdKERGRtkDFnDSJsJAgLp3Qjy/T97Nm90Gv4zSp15Zl8G7abn5+ymCS+nbxOk6Tu/OcYYyO78jNr6WxZW++13HkMD322WbSMg7y+3NHEtMh3Os4IiIi0gJUzEmTuXBcH6LCglvVJOLp2Xn85p3VTBjQlatOGOB1nGYRHhLMIxcnERYSxFVzUjhUXOZ1JGmkdZm5/P0/mzhzVBxnjorzOo6IiIi0EBVz0mQ6RoVyQXJv3k3bTVZukddxjlhRaTnXvrSCdmEh/PXHYwgOar3d1np1iuShGYmkZ+dz6xsrNaF4ACkpq+CmuWl0jAzl7qkjvI4jIiIiLUjFnDSpnx6XQHmF47mF27yOcsTu+fc61u/J4/4LRreJUQGPG9iNm08bwnsrM3nmy21ex5EGeviTdNZm5nLveSPp0i7M6zgiIiLSglTMSZPq0zWK04b34MUlOygoCdzueu+vzuSFxdv5v0kJ/GBIrNdxWszsEwZw6rDu3DtvHUu3HvA6jtRj9a6DPPxJOucl9uLU4T28jiMiIiItTMWcNLkrJiVwsLCU11MyvI5yWDK+LuAXr69kVHxHbjltqNdxWpSZcf8Fo+ndJYprXlpOdivoLttaFZeV8/O5qXRtH8adZw/3Oo6IiIh4QMWcNLmkvl1I7NOJpxZspTzAJhEvK6/ghldSqXDw0IxEwkLa3n+R6IhQHrs4ifyiMq5+cTmlmlDcL/39401szMrnj+ePalXTZYiIiEjDtb1PqtIirpjYn+37C/h4XZbXURrlbx9vImX719xz3gj6dm3ndRzPDOnRgT/+cCTLtn/NvfPWeR1Hqlmx42se+2wzFyTH84OhbacbsIiIiHyXijlpFqcN705858iAmkR8Yfo+Hv40nWlJ8Uwd08vrOJ6bOqYXlx3Xj2e+3Mbbqbu8jiM+RaXl3PRaGj2iI/j1WcO8jiMiIiIeUjEnzSIkOIjLjkvgq21fk7ozx+s49dqfX8yNr6aS0K0dd03V/UffuP2Mozi6X2d++cYqNuzJ8zqOAA98uIEtew9x349GEx2h7pUiIiJtmYo5aTY/Pro3HcJD/P7qXEWF46bX0sgpLOUfM8YSFRbidSS/ERocxMMXjqV9RAhXzUkht6jU60ht2lfbDvDkgq1cdEwfJg7q5nUcERER8ZiKOWk27cNDmHFMH+av3kPG1wVex6nV019u5dMNe/n1mUcxrGe013H8Tmx0BI9cNJadBwq4aW4aFQE2qE1rUVBSxs2vpRHfOZLbzzjK6zgiIiLiB3QJQprVTyb04+kFW5n850+JjgwlOiKE6MhQOkSEEB3xv+/fa4sM/c5yh/AQgoKsacNlZpJ/3o946pjZnJo0lJnj+zbt729Fju7XhV+deRR3vbuWRz5N59oTB3kdqc257/0NbN9fwCuzxtMuXC/dIiIiomJOmlnPTpH848KxpGXkkFdUSm5hGblFpeQVlZGdm09eUeXPBSXl9f6uDuE1FIK+ArFDRCjRkb7v31n+37rwkODv/L6S395F1NJF3Ew3TvrDa5g1cbHYyvxkQj/SdubwwEcbGdGrI5Pb0GTqXlu4eR/PLtzGTyb0Y3z/rl7HERERET9hzvl3l6nk5GS3bNkyr2NIMystryDfV9jlFpZVFn5FpeQWlZFbWPk9r/q6wjLyiv/XVl/vv/CQoG8Luz7FB3n8txcQXlZCRUQEQVu3Qo8eLfNkA1hhSTnnP7qQ3TmFvHvtRPp0jfI6UquXX1zGlL99TmhwEPOun0RkWHD9O4mIiEirYmYpzrnk6u26Mid+ITQ4iM7twujcLuyw9nfOcaiknNzC0m+v9lVdzvu2KKwsDM/750ME+f6QEVRRAXffDQ8/3JRPqVWKDAvm8YuTOOuhL7hyTgpvzp6g4qKZ3TtvHbtyCnn9qmN1rEVEROQ7dGVO2p7MTOjfH4qK/tcWGQlbtujqXAN9siGbnz77FVNH9+SvPx6jLqrN5PONe7nk6aXMOr6/Bj0RERFpw2q7MqfRLKXtuftuqKj4blt5eWW7NMgPhsTy85MH86/U3Ty7cJvXcVql3KJSbn1jJQNj2/PzUwZ7HUdERET8kIo5aXsWLYKSku+2lZTAwoXe5AlQ1/xgIKcM6849/17Hki37vY7T6tz97lqy84p5YNpoIkLVvVJERES+T8WctD0rVoBz3/9ascLrZAElKMh44ILR9OkSxTUvrWDPwaL6d5IG+c+6LF5LyeCqE/ozuncnr+OIiIiIn1IxJyKHLToilMdnJlFQUsbsF1MoLqt/igmpW05BCbe9uYqhPTpw/Umaz09ERERqp2JORI7IoO4duH/aaFbsyOF37671Ok7Au/OdNRw4VML900Z/b25EERERkarqLebMrLeZfWJm68xsjZnd4GsfbWaLzGyVmb1rZtFV9rnNzNLNbIOZnValPcm3fbqZPWgaAk+kVThjZBxXntCfF5fsYO5XO72OE7DeX72Hf6Xu5toTBzKiV0ev44iIiIifa8iVuTLgJufcUcB44BozGwY8CfzSOTcSeAu4BcC3bjowHJgCPGJm3/x5+VFgFjDI9zWlCZ+LiHjollOHMHFgN3799mpWZuR4HSfg7M8v5ldvrWJ4z2iu+cFAr+OIiIhIAKi3mHPOZTrnlvuW84B1QC9gCPC5b7OPgB/6lqcCrzjnip1zW4F0YJyZxQHRzrlFrnJyu+eBc5v02YiIZ0KCg3hwRiIx7cO56oUU9ucXex0poPzm7TXkFpXywAWjCQ1WD3gRERGpX6M+MZhZPyARWAKsBs7xrZoG9PYt9wKq9rPK8LX18i1Xb6/pcWaZ2TIzW7Z3797GRBQRD3VpF8ZjFyex71AJ17+ygrLyivp3Et5N282/V2Vy48mDGdojuv4dRERERGhEMWdm7YE3gBudc7nAT6nscpkCdAC+mbirpvvgXB3t32907gnnXLJzLjkmJqahEUXED4yM78g9547gy/T9/PnDDV7H8XvZeUXc8fZqRvfuxJXH9/c6joiIiASQkIZsZGahVBZyLzrn3gRwzq0HTvWtHwyc6ds8g/9dpQOIB3b72uNraBeRVmZacm/SMnJ4/LMtjOrViTNHxXkdyS855/jVW6spKCnngWmjCFH3ShEREWmEhoxmacBTwDrn3F+qtMf6vgcBvwYe8616B5huZuFmlkDlQCdLnXOZQJ6Zjff9zkuAt5v02YiI3/jNWcMZ26cTt7yexsasPK/j+KW3Vuzio7VZ3HLqEAbGdvA6joiIiASYhvwZ+DhgJnCimaX6vs4AZpjZRmA9lVfYngFwzq0B5gJrgfeBa5xz38wkPJvKUTDTgc3A/KZ8MiLiP8JCgnj04iSiwkK48oUUcotKvY7kV/YcLOLOd9aQ3LczP52Y4HUcERERCUBWObCk/0pOTnbLli3zOoaIHKalWw9w4T8XM3lILE/MTCIoSNNLOue47NmvWLxlP/NvOJ6Ebu28jiQiIiJ+zMxSnHPJ1dt1g4aINKtxCV349ZlH8fG6LB7+JN3rOH7htWUZfLphL7+cMlSFnIiIiBw2FXMi0uwundCP8xJ78ZePN/LJhmyv43hqV04hv3tvLeP7d+GSY/t5HUdEREQCmIo5EWl2Zsa9541kaI9obnh5Bdv3H/I6kiecc9z6+kqcc/z5R6PV5VRERESOiIo5EWkRkWHBPH5xEmbGlS+kUFBS5nWkFvfikh0sSN/H7WceRe8uUV7HERERkQCnYk5EWkyfrlE8OCORDVl53PbmKvx9AKamtGN/AffOW8ekQd24cFwfr+OIiIhIK6BiTkRa1AmDY7j51CG8nbqbZ77c5nWcFlFR4bjl9TSCzfjTD0dROdWmiIiIyJFRMSciLW72CQM4dVh37pm3jsVb9nsdp9k9t2gbS7Ye4I6zh9GzU6TXcURERKSVUDEnIi0uKMh44ILR9O0SxbUvLSfzYKHXkZrNlr35/On99Zw4NJZpSfFexxEREZFWRMWciHiiQ0QoT1ySRGFJObPnLKe4rNzrSE2uvMJx82tphIcE84fzR6p7pYiIiDQpFXMi4pmBsR144ILRpO7M4a5313odp2llZpI19hh2rt3KXecMp3t0hNeJREREpJVRMScinpoyIo7Zkwfw0pIdvPrVDq/jNJmDt99Bj5XL+PP6t5k6pqfXcURERKQVUjEnIp67+dQhTBzYjTveXkPazhyv4xyR3TmF/O35zwif8wJBznHCl//GsrK8jiUiIiKtkIo5EfFccJDx4IxEYtqHM3tOCvvzi72O1GjrMnP52aupHH/fJ3T7232EUDmHnlWUw913e5xOREREWiMVcyLiF7q0C+PxmUnsP1TCdS+voKy8wutI9XLO8WX6PmY+tYTT//4FH6zZw9VD2nHRuv8SUlZauVFJCTzzDOzZ421YERERaXVUzImI3xjRqyP3nDeShZv3c98HG7yOU6uy8greTt3FWQ8t4KInl7B+Tx63nDaERb88iZ8veRWrqFaIluvqnIiIiDS9EK8DiIhU9aOkeFZm5PDE51sYFd+Rs0b5z+Ahh4rLePWrnTy1YCu7cgoZENOOP/1wJOcm9iI8JLhyo0WLKq/GVVVSAgsXtnxgERERadVUzImI3/n1mcNYszuXX7y+kkGxHRjSo4OnefbmFfPcwm28sHg7BwtLObpfZ+48ZzgnDY0lKKja3HErVngTUkRERNocc855naFOycnJbtmyZV7HEJEWlpVbxFkPLaB9eAj/uuY4OkaGtniGzXvzefKLLbyxfBel5RWcNqwHs07oz9g+nVs8i4iIiLRdZpbinEuu3q4rcyLil7pHR/DIRWOZ8cRibpqbyhMzk79/FayZLNt2gMc/38LH67IIDQ5iWlI8V0zqT0K3di3y+CIiIiINoWJORPzW0f26cMdZw/jtO2t46L/p3HDyoGZ7rIoKx0frsnji8y2kbP+aTlGhXPeDgVwyoR/d2oc32+OKiIiIHC4VcyLi1y45ti9pO3P42382MjI+mhOHdm/S319UWs6by3fx5Bdb2LLvEPGdI7nrnOFMS44nKkwvkSIiIuK/9ElFRPyamXHv+SNZvyePG19J5Z1rJ9KvCbo75hSU8MKi7Ty3aBv78ksY2asjD81I5PQRPQgJ1qwtIiIi4v9UzImI34sIDebxmUmc/Y8FXDUnhTevnnDYV812HijgqQVbmbtsJwUl5UweEsOs4/tzbP+umLXMPXkiIiIiTUHFnIgEhN5donhweiKXPrOUX76xir9PH9Oo4mv1roM8/vkW5q3KxIBzxvRk1vH9GdojuvlCi4iIiDQjFXMiEjCOHxzDzacO4c8fbGB0705cPjGhzu2dc3y+aR+Pf7aZhZv30z48hMsnJnDZcf2I6xjZQqlFREREmoeKOREJKFdPHsDKjBzunbeOYXHRHDug6/e2KS2v4N203Tzx+RbW78mje3T6wQ6FAAAI50lEQVQ4t50+lBnH9CE6ouXnqxMRERFpDpo0XEQCTl5RKec+/CU5BaW8d/3Eb6+y5RWV8srSnTz95VYyDxYxuHt7/m9Sf6aO6UVYiAY1ERERkcBU26ThKuZEJCClZ+dz7sNfMiC2Pf+YkciLS3bw4pLt5BWVMb5/F648fgCTh8RoUBMREREJeLUVc+pmKSIBaWBse+6fNpqr5qQw6b5PCDI4fUQcs47vz+jenbyOJyIiItLs6i3mzKw38DzQA6gAnnDO/d3MxgCPARFAGXC1c26pb5/bgMuBcuB659wHvvYk4FkgEpgH3OD8/dKgiPitKSN6cNc5w9m+v4BLJ/Slb9cjn39OREREJFA05MpcGXCTc265mXUAUszsI+A+4C7n3HwzO8P382QzGwZMB4YDPYGPzWywc64ceBSYBSymspibAsxv8mclIm3GpRP6eR1BRERExBP1jgjgnMt0zi33LecB64BegAO+maCpI7DbtzwVeMU5V+yc2wqkA+PMLA6Ids4t8l2Nex44t0mfjYiIiIiISBvRqHvmzKwfkAgsAW4EPjCz+6ksCif4NutF5ZW3b2T42kp9y9Xba3qcWVRewaNPnz6NiSgiIiIiItImNHisbjNrD7wB3OicywVmAz9zzvUGfgY89c2mNezu6mj/fqNzTzjnkp1zyTExMQ2NKCIiIiIi0mY0qJgzs1AqC7kXnXNv+povBb5Zfg0Y51vOAHpX2T2eyi6YGb7l6u0iIiIiIiLSSPUWc1Y5SdNTwDrn3F+qrNoNnOBbPhHY5Ft+B5huZuFmlgAMApY65zKBPDMb7/udlwBvN9HzEBERERERaVMacs/cccBMYJWZpfrabgf+D/i7mYUARfjucXPOrTGzucBaKkfCvMY3kiVUds18lsqpCeajkSxFREREREQOi/n7NG/Jyclu2bJlXscQERERERHxhJmlOOeSq7c3eAAUERERERER8R8q5kRERERERAKQijkREREREZEApGJOREREREQkAKmYExERERERCUAq5kRERERERAKQ309NYGZ7ge1e56hBN2Cf1yHaKB177+jYe0fH3js69t7RsfeOjr23dPy946/Hvq9zLqZ6o98Xc/7KzJbVNNeDND8de+/o2HtHx947Ovbe0bH3jo69t3T8vRNox17dLEVERERERAKQijkREREREZEApGLu8D3hdYA2TMfeOzr23tGx946OvXd07L2jY+8tHX/vBNSx1z1zIiIiIiIiAUhX5kRERERERAKQijkREREREZEApGKuDmY2xcw2mFm6mf2yhvVmZg/61q80s7Fe5GyNzKy3mX1iZuvMbI2Z3VDDNpPN7KCZpfq+fuNF1tbIzLaZ2SrfcV1Ww3qd+83AzIZUOZ9TzSzXzG6sto3O+yZiZk+bWbaZra7S1sXMPjKzTb7vnWvZt873B6lbLcf+z2a23vea8paZdapl3zpfn6RutRz7O81sV5XXlTNq2Vfn/RGo5di/WuW4bzOz1Fr21Xl/BGr7XNkaXvN1z1wtzCwY2AicAmQAXwEznHNrq2xzBnAdcAZwDPB359wxHsRtdcwsDohzzi03sw5ACnButeM/GbjZOXeWRzFbLTPbBiQ752qcNFPnfvPzvQbtAo5xzm2v0j4ZnfdNwsyOB/KB551zI3xt9wEHnHN/9L1hd3bO3Vptv3rfH6RutRz7U4H/OufKzOxPANWPvW+7bdTx+iR1q+XY3wnkO+fur2M/nfdHqKZjX239A8BB59zvali3DZ33h622z5XATwjw13xdmavdOCDdObfFOVcCvAJMrbbNVCr/Qzrn3GKgk+9kkSPknMt0zi33LecB64Be3qaSKnTuN7+TgM1VCzlpWs65z4ED1ZqnAs/5lp+j8s2+uoa8P0gdajr2zrkPnXNlvh8XA/EtHqwNqOW8bwid90eormNvZgZcALzcoqHaiDo+Vwb8a76Kudr1AnZW+TmD7xcTDdlGjpCZ9QMSgSU1rD7WzNLMbL6ZDW/RYK2bAz40sxQzm1XDep37zW86tb+p67xvPt2dc5lQ+eYPxNawjc7/5vdTYH4t6+p7fZLDc62vi+vTtXQ103nfvCYBWc65TbWs13nfRKp9rgz413wVc7WzGtqq90ltyDZyBMysPfAGcKNzLrfa6uVAX+fcaOAh4F8tna8VO845NxY4HbjG1zWkKp37zcjMwoBzgNdqWK3z3ns6/5uRmf0KKANerGWT+l6fpPEeBQYAY4BM4IEattF537xmUPdVOZ33TaCez5W17lZDm9+c+yrmapcB9K7yczyw+zC2kcNkZqFU/od70Tn3ZvX1zrlc51y+b3keEGpm3Vo4ZqvknNvt+54NvEVlF4OqdO43r9OB5c65rOordN43u6xvugz7vmfXsI3O/2ZiZpcCZwEXuVpu6m/A65M0knMuyzlX7pyrAP5JzcdU530zMbMQ4Hzg1dq20Xl/5Gr5XBnwr/kq5mr3FTDIzBJ8fyWfDrxTbZt3gEus0ngqb1rNbOmgrZGv7/hTwDrn3F9q2aaHbzvMbByV5/P+lkvZOplZO9/NwZhZO+BUYHW1zXTuN69a/0Kr877ZvQNc6lu+FHi7hm0a8v4gjWRmU4BbgXOccwW1bNOQ1ydppGr3PJ9HzcdU533zORlY75zLqGmlzvsjV8fnyoB/zQ/xOoC/8o2mdS3wARAMPO2cW2NmV/nWPwbMo3I0v3SgALjMq7yt0HHATGCV/W+Y3tuBPvDt8f8RMNvMyoBCYHptf8mVRukOvOWrF0KAl5xz7+vcbxlmFkXliFlXVmmreux13jcRM3sZmAx0M7MM4LfAH4G5ZnY5sAOY5tu2J/Ckc+6M2t4fvHgOgaqWY38bEA585Hv9Weycu6rqsaeW1ycPnkLAquXYTzazMVR2HduG7/VH533TqunYO+eeooZ7pHXeN7naPlcG/Gu+piYQEREREREJQOpmKSIiIiIiEoBUzImIiIiIiAQgFXMiIiIiIiIBSMWciIiIiIhIAFIxJyIiIiIiEoBUzImIiIiIiAQgFXMiIiIiIiIB6P8BzUD+p15dfPMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure(figsize=(15,6))\n", + "plt.cla()\n", + "env.render()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 335, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[152.30224609375,\n", + " 209.1220703125,\n", + " 305.837158203125,\n", + " 11.605224609375,\n", + " 92.665771484375]" + ] + }, + "execution_count": 335, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "env._trade_history" + ] + }, + { + "cell_type": "code", + "execution_count": 176, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_agent(env, max_steps, n_eval_episodes, Q):\n", + " \"\"\"\n", + " Evaluate the agent for ``n_eval_episodes`` episodes and returns average reward and std of reward.\n", + " :param env: The evaluation environment\n", + " :param n_eval_episodes: Number of episode to evaluate the agent\n", + " :param Q: The Q-table\n", + " :param seed: The evaluation seed array (for taxi-v3)\n", + " \"\"\"\n", + " episode_rewards = []\n", + " episode_profits = []\n", + " for episode in tqdm(range(n_eval_episodes)):\n", + " state = env.reset()\n", + " step = 0\n", + " done = False\n", + " total_rewards_ep = 0\n", + " total_profit_ep = 0\n", + " \n", + " for step in range(max_steps):\n", + " # Take the action (index) that have the maximum expected future reward given that state\n", + " action = greedy_policy(Q, state)\n", + " new_state, reward, done, info = env.step(action)\n", + " total_rewards_ep += reward\n", + " \n", + " if done:\n", + " break\n", + " state = new_state\n", + "\n", + " episode_rewards.append(total_rewards_ep)\n", + " episode_profits.append(env.history['total_profit'][-1])\n", + " # print(env.history)\n", + " # env.render()\n", + " # assert 0\n", + "\n", + " mean_reward = np.mean(episode_rewards)\n", + " std_reward = np.std(episode_rewards)\n", + " mean_profit = np.mean(episode_profits)\n", + " std_profit = np.std(episode_profits)\n", + "\n", + " return mean_reward, std_reward, mean_profit, std_profit" + ] + }, + { + "cell_type": "code", + "execution_count": 325, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a0c2015a163743448978dd27f700d2e9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/200 [00:00" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.figure(figsize=(15,6))\n", + "plt.cla()\n", + "env_test.render()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.8.13 ('rl2')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "cd60ab8388a66026f336166410d6a8a46ddf65ece2e85ad2d46c8b98d87580d1" + } + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "01a2dbcb714e40148b41c761fcf43147": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "20b0f38ec3234ff28a62a286cd57b933": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "PasswordModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "PasswordModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "PasswordView", + "continuous_update": true, + "description": "Token:", + "description_tooltip": null, + "disabled": false, + "layout": "IPY_MODEL_01a2dbcb714e40148b41c761fcf43147", + "placeholder": "​", + "style": "IPY_MODEL_90c874e91b304ee1a7ef147767ac00ce", + "value": "" + } + }, + "270cbb5d6e9c4b1e9e2f39c8b3b0c15f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "VBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "VBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "VBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a02224a43d8d4af3bd31d326540d25da", + "IPY_MODEL_20b0f38ec3234ff28a62a286cd57b933", + "IPY_MODEL_f6c845330d6743c0b35c2c7ad834de77", + "IPY_MODEL_f1675c09d16a4251b403f9c56255f168", + "IPY_MODEL_c1a82965ae26479a98e4fdbde1e64ec2" + ], + "layout": "IPY_MODEL_3fa248114ac24656ba74923936a94d2d" + } + }, + "2dc5fa9aa3334dfcbdee9c238f2ef60b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3e753b0212644990b558c68853ff2041": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3fa248114ac24656ba74923936a94d2d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": "center", + "align_self": null, + "border": null, + "bottom": null, + "display": "flex", + "flex": null, + "flex_flow": "column", + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "50%" + } + }, + "42d140b838b844819bc127afc1b7bc84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "90c874e91b304ee1a7ef147767ac00ce": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9d847f9a7d47458d8cd57d9b599e47c6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a02224a43d8d4af3bd31d326540d25da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_caef095934ec47bbb8b64eab22049284", + "placeholder": "​", + "style": "IPY_MODEL_2dc5fa9aa3334dfcbdee9c238f2ef60b", + "value": "

Copy a token from your Hugging Face\ntokens page and paste it below.
Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file.
" + } + }, + "a2cfb91cf66447d7899292854bd64a07": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c1a82965ae26479a98e4fdbde1e64ec2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9d847f9a7d47458d8cd57d9b599e47c6", + "placeholder": "​", + "style": "IPY_MODEL_42d140b838b844819bc127afc1b7bc84", + "value": "\nPro Tip: If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. " + } + }, + "caef095934ec47bbb8b64eab22049284": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eaba3f1de4444aabadfea2a3dadb1d80": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ee4a21bedc504171ad09d205d634b528": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ButtonStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ButtonStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "button_color": null, + "font_weight": "" + } + }, + "f1675c09d16a4251b403f9c56255f168": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ButtonModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ButtonModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ButtonView", + "button_style": "", + "description": "Login", + "disabled": false, + "icon": "", + "layout": "IPY_MODEL_a2cfb91cf66447d7899292854bd64a07", + "style": "IPY_MODEL_ee4a21bedc504171ad09d205d634b528", + "tooltip": "" + } + }, + "f6c845330d6743c0b35c2c7ad834de77": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "CheckboxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "CheckboxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "CheckboxView", + "description": "Add token as git credential?", + "description_tooltip": null, + "disabled": false, + "indent": true, + "layout": "IPY_MODEL_3e753b0212644990b558c68853ff2041", + "style": "IPY_MODEL_eaba3f1de4444aabadfea2a3dadb1d80", + "value": true + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}