Add application file

app.py

@@ -0,0 +1,33 @@
+import gradio as gr
+
+from tqdm import tqdm
+from model import Model
+tqdm.pandas()
+
+def get_predict(stoc_name, pred_days, model_name, add_smoothing):
+    '''Эта функцыю делаем прогнозы, выводит график, и информацыю о важности признаков'''
+    model = Model(stoc_name, model_name)
+    data, string = model.predict(pred_days) 
+    p = model.plot_predict(data, add_smoothing)
+    return p, string 
+
+stock_list = ['VKCO', 'SBER', 'CHMF', 'MTSS', 'SMLT', 'AGRO', 'SIBN', 'YNDX'] # Акции для прогноза
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Приложение предсказания цен акций")
+
+    with gr.Row():  
+        with gr.Column(scale=1):  
+            stock_dropdown = gr.Dropdown(choices=stock_list, label="Выберите акцию")  
+            slider = gr.Slider(1, 7, step=1, label="Выберите длину предсказания (дней)")
+            model_dropdown = gr.Dropdown(choices=['LinearRegression', 'NN', 'LGB'], label="Выберите модель")
+            checkbox = gr.Checkbox(label="Добавить сглаженное предсказание", value=True)
+        
+        with gr.Column(scale=2):  
+            output_plot = gr.Plot()  
+            output_md = gr.Markdown() 
+
+    submit_button = gr.Button("Сгенерировать предсказание")
+    submit_button.click(fn=get_predict, inputs=[stock_dropdown, slider, model_dropdown, checkbox], outputs=[output_plot, output_md])
+
+demo.launch()

model.py

@@ -0,0 +1,104 @@
+import pandas as pd
+import tensorflow as tf
+import pickle 
+import plotly.express as px
+import os 
+import numpy as np
+
+from stock_and import GetNewData
+
+class Model:
+    def __init__(self, stock_name, model_name) -> None:
+        """Этот класс делает прогнозирования, выводит графики, рассчитывает важность признаков."""
+        self.stock_name = stock_name # Название 
+        self.features = ['lag_25', 'lag_34','lag_33','lag_26','lag_32','lag_31','lag_30','lag_29','lag_27','sentiment_neutral', 'lag_28',
+        'sentiment_positive','sentiment_negative', 'month','day'] # Фичи для модели
+        self.model_name = model_name # Название модели
+
+        # Загрузка моделей
+        if model_name == 'NN':
+            self.model = tf.keras.models.load_model(f'models/nn_predict_1day_ver2_{stock_name}.h5', custom_objects={'mae': tf.keras.metrics.MeanAbsoluteError()})
+
+        if model_name == 'LinearRegression':
+            with open(os.path.join('models', f'linear_predict_1day_ver2_{stock_name}.pkl'), 'rb') as f:
+                self.model = pickle.load(f)
+
+        if model_name == 'LGB':
+            with open(os.path.join('models', f'lgb_predict_1day_ver2_{stock_name}.pkl'), 'rb') as f:
+                self.model = pickle.load(f)
+
+    def generate_dataset(self, stock_name, num_day):
+        """Парсим датасет, добавляем новые row для прогонза"""
+        merged_df, string = GetNewData(stock_name).get_full_data() # Тут мы получаем датасет с новостями и акциями 
+
+        # Создаем датасет с дополнительными строками 
+        last_date = merged_df['DATE'].max()
+        new_date_rng = pd.date_range(start=last_date + pd.Timedelta(hours=1), periods=24 * num_day , freq='H') 
+
+        new_df = pd.DataFrame(new_date_rng, columns=['DATE'])
+        new_df['month'] = new_df['DATE'].dt.month
+        new_df['day'] = new_df['DATE'].dt.day
+        
+        for c in  self.features:
+            last_value = merged_df[c].values[-24 * num_day :]
+            new_df[c] = last_value
+
+        # Объединяем датасет
+        merged_df_new = pd.concat([merged_df, new_df[self.features+['DATE']]], ignore_index=True)
+        return merged_df_new, string
+
+    def predict(self, num_day):
+        # Создаем прогнозы, важности признаков
+        merged_df, string = self.generate_dataset(self.stock_name, num_day) # Парсим датасет
+
+        if self.model_name == 'NN':
+            importance = np.abs(self.model.layers[0].get_weights()[0])
+            importance = np.mean(importance, axis=1)
+            df_weighted = merged_df[self.features].ffill().bfill()[-num_day*24:] * importance
+
+        if self.model_name == 'LinearRegression':
+            df_weighted = merged_df[self.features].ffill().bfill()[-num_day*24:] * self.model.coef_
+
+        if self.model_name == 'LGB':
+            df_weighted = merged_df[self.features].ffill().bfill()[-num_day*24:] * self.model.feature_importances_
+
+        average_values = df_weighted.mean(axis=0).abs().sort_values(ascending=False)
+        average_values_filtered = average_values.drop('lag_25')
+            
+        total_sum = average_values_filtered.sum()
+        average_values_percentage = (average_values_filtered / total_sum) * 100
+
+        string += '\n Самые полезные признаки для прогнозов: \n'
+        
+        for f, v in zip(average_values_percentage.index, average_values_percentage.values):
+            string += f'- {f}: важность = {v:.2f}%\n'
+
+        if self.model_name in ['LinearRegression', 'LGB']:
+            return pd.DataFrame({
+                'predict': self.model.predict(merged_df[self.features].ffill().bfill().values),
+                'DATE': merged_df['DATE'].values,
+                'CLOSE': merged_df['CLOSE'].values
+            }), string
+
+        else:
+            return pd.DataFrame({
+                'predict': self.model.predict(merged_df[self.features].ffill().bfill().values)[:, 0],
+                'DATE': merged_df['DATE'].values,
+                'CLOSE': merged_df['CLOSE'].values
+            }), string
+
+    def plot_predict(self, predict, add_smoothing):
+        predict = predict[-24*12:]
+
+        scaling_factor = predict['CLOSE'].mean() / predict['predict'].mean()
+        scaled_preds = predict['predict'] * scaling_factor
+
+        fig = px.line(predict, x=predict.DATE, y='CLOSE', labels={'value': 'Цена'}, title='CLOSE')
+        fig.add_scatter(x=predict.DATE, y=scaled_preds, mode='lines', name='Predict', opacity=0.7)
+
+        if add_smoothing:
+            smoothed_preds = pd.Series(scaled_preds).ewm(3).mean()
+            fig.add_scatter(x=predict.DATE, y=smoothed_preds, mode='lines', name='Сглаженные предсказания', opacity=0.7)
+
+        fig.update_layout(xaxis=dict(type='category'))
+        return fig

models/lgb_predict_1day_ver2_AGRO.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12eac4a543adda6d7e50335fd4c742c75bcb974b662916084057ae3b6080d364
+size 111059

models/lgb_predict_1day_ver2_CHMF.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:363d694e123fa2678c57cc9cc7d2cc08aa18bf6cb8cd27f32504ea3a643a86d6
+size 108149

models/lgb_predict_1day_ver2_MTSS.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0efd081b82763fca72d13ef27090611d07204bf91587d04df28ec2ab5dba813b
+size 112768

models/lgb_predict_1day_ver2_SBER.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6b184feb04c371e6f08071de17c51445a28cee988cff9c9f8b7654ab3911b3b
+size 112729

models/lgb_predict_1day_ver2_SIBN.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdecdfd8f57c50eb3cfefcf85c79b4e560bc26c184cb9b1ad74729fcae40edc1
+size 112621

models/lgb_predict_1day_ver2_SMLT.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fed679e9165076c8b13f80740f5f5783a8f991d091e7ef3f037ccbb970286bc9
+size 112694

models/lgb_predict_1day_ver2_VKCO.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ee5fa66fbd15533f9c7cc5441cde36489754ebbd35228228bf18e677ac9fce3
+size 113490

models/lgb_predict_1day_ver2_YNDX.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d72c9985aa69d15b9d374928fde8da467e479142dddbcd7db420ae485ea82efe
+size 111507

models/linear_predict_1day_ver2_AGRO.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:351f3dab393b18467096c8d7b60328d015a95664ea0a6803eb176687ef4630bc
+size 963

models/linear_predict_1day_ver2_CHMF.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d43e65ea82c9a82a55aff31a8912ed96f2130644f0fe699c988ada0445692c
+size 963

models/linear_predict_1day_ver2_MTSS.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:219e726dfdc4c31bc340629a1d8d27511c31770dec679a5a2c3c91ecdc01265c
+size 963

models/linear_predict_1day_ver2_SBER.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d9702e128cf3bcf6f443d0c43edc252ae49c9ed43154359119a73f0c8a9b57d
+size 963

models/linear_predict_1day_ver2_SIBN.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a4f667e472c79b660aa6699fafeb737f57d298e607a56b665fa4fdc0bb2e6cf
+size 963

models/linear_predict_1day_ver2_SMLT.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17f202f365d672df86daa3dc506d8f9f1107c5decba08500cee86896e54c37d5
+size 963

models/linear_predict_1day_ver2_VKCO.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:137d9f838962f785be93d826d7aee964dd12d260d7a7b55e1c5d382057c16b05
+size 963

models/linear_predict_1day_ver2_YNDX.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed581f0da796a8f864ecd7a35d82fb9ab18f374ee1cb2fc441206643d2e801dd
+size 963

models/nn_predict_1day_ver2_AGRO.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79ef5af4053c6f8568071ecb1312360a53e5167782b72b013b5ef9365e57a95e
+size 23920

models/nn_predict_1day_ver2_CHMF.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9eee0d24f5e43527e7b8b4d8ba0c35ecbb3001c285436a659c1b357182effd1
+size 23920

models/nn_predict_1day_ver2_MTSS.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd3ff689591aff665292602995a5b5b74effea3a93d2f96560059e28487b80e
+size 23920

models/nn_predict_1day_ver2_SBER.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bebb94e7996a989ffb680b359df91d27892faecd20f5a056f0e860c50752d8a7
+size 23920

models/nn_predict_1day_ver2_SIBN.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9659fdb354f975c47e3fbc08b7cda9836e11a740b729eb88ccc1661183ff2527
+size 23920

models/nn_predict_1day_ver2_SMLT.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:847847ee5298aab687d3543cf0ae8ddca9b0075fdd0e56d31c37ec59dcbde815
+size 23920

models/nn_predict_1day_ver2_VKCO.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a6b68dcbabf1775b27d57c018e6ab2b730970a460445fe3ae51039944b7ad96
+size 23920

models/nn_predict_1day_ver2_YNDX.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a229e3e67e112bbbf121b048e3a69a86933584b34f9275d9e3679c1064ea66d9
+size 23920

news.py

@@ -0,0 +1,94 @@
+import requests
+import pandas as pd
+import time
+import pymorphy2
+import re 
+
+from datetime import datetime, timedelta
+from transformers import pipeline
+from bs4 import BeautifulSoup
+
+class NewsData:
+    def __init__(self) -> None:
+        """
+        Парсер статей с ru.investing.com.
+        """
+        self.urls = [
+            ("https://ru.investing.com/news/forex-news/", "forex-news"),
+            ("https://ru.investing.com/news/commodities-news/", "commodities-news"),
+        ]
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Referer': 'http://google.com'
+        }
+
+        self.morph = pymorphy2.MorphAnalyzer() # С помощь него мы будем приводить слова в начальную форму
+    
+    def get_data(self):
+        # Парсим сайты за последнии 7 дней
+        date_limit = datetime.now().replace(minute=0, second=0, microsecond=0) - timedelta(days=7*1)
+        articles_data = []
+
+        for base_url, tag in self.urls:
+            page = 1
+            while True:
+                url = f"{base_url}{page}/" 
+                response = requests.get(url, headers=self.headers)
+
+                if response.status_code == 200:
+                    soup = BeautifulSoup(response.content, "html.parser")
+                    articles = soup.find_all("a", attrs={"data-test": "article-title-link"})
+                    
+                    daily_count = 0
+
+                    for article in articles:
+                        title = article.text.strip()
+                        date_tag = article.find_next("time", attrs={"data-test": "article-publish-date"}) 
+                        url = article["href"]
+
+                        if date_tag:
+                            publish_date_str = date_tag["datetime"]
+                            publish_date = datetime.strptime(publish_date_str, "%Y-%m-%d %H:%M:%S")
+                            
+                            articles_data.append([title, publish_date_str, tag, url])
+                            daily_count += 1
+                       
+                    page += 1
+
+                    if publish_date < date_limit:
+                        print(f"Дата публикации: {publish_date_str} меньше лимита. Остановка.")
+                        print(f"Общее количество скачанных статей: {len(articles_data)}")
+                        break
+                else:
+                    if response.status_code == 404:
+                        break
+
+                    print(f"Ошибка при запросе страницы {page}, код ответа: {response.status_code}")
+                    time.sleep(5)
+
+        return self.get_proccesing_news(articles_data)
+    
+    def clean_text(self, text):
+        text = re.sub(r'[^а-яА-Я\s]', '', text)  
+        text = text.lower()  
+        text = ' '.join([word for word in text.split() if len(word) >= 3]) 
+        text = ' '.join([self.morph.parse(word)[0].normal_form for word in text.split()])
+        return text
+    
+    def get_proccesing_news(self, articles_data):
+        # Обрабатываем полученные данные, добавляем сентиментальный анализ.
+        sentiment_model = pipeline("sentiment-analysis", model="mxlcw/rubert-tiny2-russian-financial-sentiment", framework="pt")
+
+        news = pd.DataFrame(articles_data, columns=["title", "DATE", "tags", 'url'])
+        
+        news['title'] = news['title'].ffill().map(self.clean_text)
+        news['sentiment'] = news['title'].progress_apply(lambda x: sentiment_model(x)[0]['label'])
+        news = pd.get_dummies(news, columns=['sentiment'])
+
+        news['DATE'] = pd.to_datetime(news['DATE'])
+        news['day'] = news['DATE'].dt.day
+        news['year'] = news['DATE'].dt.year
+        news['month'] = news['DATE'].dt.month
+        news['hour'] = news['DATE'].dt.hour
+        return news

stock_and.py

@@ -0,0 +1,81 @@
+import pandas as pd
+import requests
+import time
+import apimoex
+
+from datetime import datetime, timedelta
+from news import NewsData
+
+class GetNewData:
+    def __init__(self, stock_name) -> None:
+        """Этот класс парсит данные"""
+        self.news = NewsData() 
+        self.stock_name = stock_name # Название акции
+       
+    def downolad_stock(self):
+        # Исправляем название акции, просто акция поменяла свое названия
+        if self.stock_name == 'YNDX':
+            self.stock_name = 'YDEX'
+        
+        # Тут мы будем делать до 5 итерацыя, если возниканет ошибка с сервером московской биржы, иногда он просто не находит акции
+        for _ in range(5):
+            try:
+                # Дата с которой будет начинатся наш датасет
+                new_date = datetime.now().replace(minute=0, second=0, microsecond=0) - timedelta(days=24*30) 
+                new_date = str(new_date.strftime('%Y-%m-%d %H:%M')) 
+
+                # Запросы к московской бирже, и создания датасета 
+                with requests.Session() as session:
+                    # Получаем данные с биржи
+                    market_data = apimoex.get_board_candles(session, security=self.stock_name, market='shares', columns=['close', 'begin', 'volume'],
+                                                 start=new_date, interval=60)
+                    
+                    df = pd.DataFrame(market_data)
+                    df.columns = ['CLOSE', 'DATE', 'VOL']
+
+                df['DATE'] = pd.to_datetime(df['DATE'])
+                df['day'] = df['DATE'].dt.day
+                df['year'] = df['DATE'].dt.year
+                df['month'] = df['DATE'].dt.month
+                df['hour'] = df['DATE'].dt.hour
+                
+                # Генерим лаги для модели
+                for i in range(1, 10+1):
+                    df[f'lag_{i+24}'] = df['CLOSE'].shift(i)
+
+                df['lag_24'] = df['CLOSE']
+                break
+            except:
+                # Если возникнет ошибка, то мы будем ждать 5 секунд и повторять все действия снова
+                time.sleep(5)
+        return df 
+    
+    def get_full_data(self):
+        """Функция для получения полного датасета"""
+        articles_data = self.news.get_data() # Получаем данные о новостями
+        stock = self.downolad_stock() # Получаем данные о биржи
+
+        # Обеденяем датасеты
+        news_ = articles_data.drop(columns=['DATE']).groupby(['day', 'month', 'year', 'hour']).agg({
+            'sentiment_negative': 'mean',  
+            'sentiment_neutral': 'mean',
+            'sentiment_positive': 'mean',
+            'url': lambda x: list(x),
+            'title': lambda x: list(x)
+        }).reset_index()
+
+        merged_df = pd.merge(stock, news_, how='left', on=['year', 'month', 'day', 'hour'], suffixes=('', '_y')).bfill().ffill()
+        merged_df = merged_df.sort_values(by=['DATE'])
+        
+        # Расчитываем важность статей
+        merged_df['super'] =  merged_df['sentiment_positive'] + merged_df['sentiment_neutral']
+
+        string = ''
+        string += '\n Самые полезные статьи для прогнозов: \n'
+
+        for url, v, title  in merged_df[-72:].sort_values('super', ascending=False)[['url', 'super', 'title']].values[:10]:
+            for u, t in zip(url, title):
+                string += f'- [New: {t}]({u}), важность признака: {v} \n'
+            
+        return merged_df, string
+