{ "cells": [ { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idspeech_contentdateparty
00Meine Damen und Herren! Ich eröffne die 2. Sit...1949-09-12not found
11Der Bundesrat ist versammelt, Herr Präsident.\\n1949-09-12not found
22Ich danke für diese Erklärung. Ich stelle dami...1949-09-12not found
33Ja, ich habe den Wunsch.\\n1949-09-12not found
44Ich erteile dem Herrn Bundespräsidenten das Wo...1949-09-12not found
...............
9309551084268\\n\\nWir sind zwar Kollegen.2022-12-16not found
9309561084269\\n\\nLiebe, sehr geehrte Frau Präsidentin!2022-12-16CDU/CSU
9309571084270\\n\\nVielen Dank.2022-12-16not found
9309581084272\\n\\nDen Abschluss dieser Aktuellen Stunde bild...2022-12-16not found
9309591084273\\n\\nSehr geehrte Frau Präsidentin! Werte Kolle...2022-12-16SPD
\n", "

930960 rows × 4 columns

\n", "
" ], "text/plain": [ " id speech_content date \\\n", "0 0 Meine Damen und Herren! Ich eröffne die 2. Sit... 1949-09-12 \n", "1 1 Der Bundesrat ist versammelt, Herr Präsident.\\n 1949-09-12 \n", "2 2 Ich danke für diese Erklärung. Ich stelle dami... 1949-09-12 \n", "3 3 Ja, ich habe den Wunsch.\\n 1949-09-12 \n", "4 4 Ich erteile dem Herrn Bundespräsidenten das Wo... 1949-09-12 \n", "... ... ... ... \n", "930955 1084268 \\n\\nWir sind zwar Kollegen. 2022-12-16 \n", "930956 1084269 \\n\\nLiebe, sehr geehrte Frau Präsidentin! 2022-12-16 \n", "930957 1084270 \\n\\nVielen Dank. 2022-12-16 \n", "930958 1084272 \\n\\nDen Abschluss dieser Aktuellen Stunde bild... 2022-12-16 \n", "930959 1084273 \\n\\nSehr geehrte Frau Präsidentin! Werte Kolle... 2022-12-16 \n", "\n", " party \n", "0 not found \n", "1 not found \n", "2 not found \n", "3 not found \n", "4 not found \n", "... ... \n", "930955 not found \n", "930956 CDU/CSU \n", "930957 not found \n", "930958 not found \n", "930959 SPD \n", "\n", "[930960 rows x 4 columns]" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Create vectorstore\n", "import pandas as pd\n", "from vectordatabase import load_documents\n", "from langchain_community.embeddings import HuggingFaceEmbeddings\n", "from langchain_community.vectorstores import FAISS\n", "from datetime import datetime\n", "\n", "\n", "df = pd.read_pickle(\"C:\\\\Users\\Tom\\SynologyDrive\\Tom\\Programming\\\\NLP\\Spaces\\PoliticsToYou\\src\\Speeches\\speeches_1949_09_12\")\n", "df['date'] = pd.to_datetime(df['date'])\n", "# Split speeches into documents\n", "df" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "c:\\Python\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n", "c:\\Python\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Sucessfully created vector store for 1. legislature\n", "Sucessfully created vector store for 2. legislature\n", "Sucessfully created vector store for 3. legislature\n", "Sucessfully created vector store for 4. legislature\n", "Sucessfully created vector store for 5. legislature\n", "Sucessfully created vector store for 6. legislature\n", "Sucessfully created vector store for 7. legislature\n", "Sucessfully created vector store for 8. legislature\n", "Sucessfully created vector store for 9. legislature\n", "Sucessfully created vector store for 10. legislature\n", "Sucessfully created vector store for 11. legislature\n", "Sucessfully created vector store for 12. legislature\n", "Sucessfully created vector store for 13. legislature\n", "Sucessfully created vector store for 14. legislature\n", "Sucessfully created vector store for 15. legislature\n", "Sucessfully created vector store for 16. legislature\n", "Sucessfully created vector store for 17. legislature\n", "Sucessfully created vector store for 18. legislature\n", "Sucessfully created vector store for 19. legislature\n", "Sucessfully created vector store for 20. legislature\n" ] } ], "source": [ "\n", "dates = [\"1953-10-06\", \"1957-10-16\", \"1961-10-17\", \"1965-10-19\", \"1969-10-20\", \"1972-12-13\", \"1976-12-14\", \"1980-11-04\", \"1983-03-29\", \"1987-02-18\",\"1990-12-20\", \"1994-11-10\", \"1998-10-26\", \"2002-10-17\", \"2005-10-18\", \"2009-10-27\", \"2013-10-22\",\"2017-10-24\",\"2021-10-26\", None]\n", "embeddings = HuggingFaceEmbeddings(model_name=\"paraphrase-multilingual-MiniLM-L12-v2\")\n", "\n", "# Iterate over all date to split by legislature getting vector stores for each period\n", "\n", "period = 1\n", "previous_date = None\n", "for date in dates:\n", " if previous_date is None:\n", " legislature = df.loc[df['date'] < datetime.strptime(date, \"%Y-%m-%d\")]\n", " elif date is None:\n", " legislature = df.loc[df['date'] >= datetime.strptime(previous_date, \"%Y-%m-%d\")]\n", " else:\n", " legislature = df.loc[(df['date'] >= datetime.strptime(previous_date, \"%Y-%m-%d\")) & (df['date'] < datetime.strptime(date, \"%Y-%m-%d\"))]\n", "\n", " \n", " # Split text into documents\n", " documents = load_documents(legislature)\n", " index_name = f'{period}_legislature'\n", " db = FAISS.from_documents(documents, embeddings)\n", " db.save_local(folder_path=\"FAISS\", index_name=index_name)\n", " print(f\"Sucessfully created vector store for {period}. legislature\")\n", " # Change for next iteration\n", " period += 1\n", " previous_date = date\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 2 }