prepare ben token added

config/config.yaml

@@ -6,3 +6,9 @@ data_ingestion:
   local_data_file : artifacts/data_ingestion/BanSum.zip
   unzip_dir : artifacts/data_ingestion
 
+ban_tokenization:
+  root_dir: artifacts/ban_tokenization
+  source_dir: artifacts/data_ingestion/BanSum.csv
+  save_dir: artifacts/ban_tokenization  
+
+

main.py

@@ -1,5 +1,6 @@
 from src.benglasummarization.logging import logger
 from src.benglasummarization.pipeline.stage01_data_ingestion import DataIngestionPipeline
+from src.benglasummarization.pipeline.stage_02_prepare_ben_tok import BenTokenizationPreparePipeLine
 
 STAGE_NAME = 'Data Ingestion Stage'
 
@@ -8,6 +9,17 @@ try:
    data_ingestion = DataIngestionPipeline()
    data_ingestion.main()
    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+        logger.exception(e)
+        raise e
+     
+STAGE_NAME = 'Prepare Ban Tokeniation Stage'
+
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   Ban_Token = BenTokenizationPreparePipeLine()
+   Ban_Token.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
 except Exception as e:
         logger.exception(e)
         raise e

params.yaml

@@ -1 +1,2 @@
-key : val
+
+output_file: "combined_text.txt"

research/data_ingestion.ipynb

@@ -0,0 +1,204 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'e:\\\\bengla text summarization\\\\train-pegasus-model-on-bengali-text-summarization-using-mlops'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class DataIngestionConfig:\n",
+    "    root_dir : Path\n",
+    "    source_dir : Path\n",
+    "    local_data_file : Path\n",
+    "    unzip_dir : Path\n",
+    "    \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.benglasummarization.constants import *\n",
+    "from src.benglasummarization.utils.common import read_yaml, create_directories\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self, \n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH\n",
+    "    ):\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "        \n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "        \n",
+    "    def get_data_ingestion_config(self) -> DataIngestionConfig:\n",
+    "        config = self.config.data_ingestion\n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        data_ingestion_config = DataIngestionConfig(\n",
+    "            root_dir= config.root_dir,\n",
+    "            source_dir=config.source_dir,\n",
+    "            local_data_file=config.local_data_file,\n",
+    "            unzip_dir= config.unzip_dir\n",
+    "        )\n",
+    "        \n",
+    "        return data_ingestion_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import zipfile\n",
+    "from src.benglasummarization.logging import logger\n",
+    "from tqdm.notebook import tqdm\n",
+    "from dataclasses import replace\n",
+    "\n",
+    "class DataIngestion:\n",
+    "    def __init__(self, config : DataIngestionConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "    def load_file(self):\n",
+    "        if os.path.exists(self.config.source_dir):\n",
+    "            self.config = replace(self.config, local_data_file = self.config.source_dir)\n",
+    "            logger.info(f'File found at: {self.config.local_data_file}')\n",
+    "        else:\n",
+    "            logger.info(f'File not found at: {self.config.source_dir}')\n",
+    "            raise FileNotFoundError(f'No file found at: {self.config.source_dir}')\n",
+    "        \n",
+    "    def extract_zip_file(self):\n",
+    "        unzip_path = self.config.unzip_dir\n",
+    "        os.makedirs(unzip_path, exist_ok=True)\n",
+    "        \n",
+    "        with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:\n",
+    "            total_files = len(zip_ref.infolist())\n",
+    "            for file in tqdm(iterable=zip_ref.infolist(), total=total_files, desc = 'Extracting Files'):\n",
+    "                zip_ref.extract(member = file, path = unzip_path)\n",
+    "                \n",
+    "            logger.info(f\"Extracted {self.config.local_data_file} to {unzip_path}\")\n",
+    "            \n",
+    "            \n",
+    "            \n",
+    "            \n",
+    "    \n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-10-16 02:22:48,187: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
+      "[2024-10-16 02:22:48,189: INFO: common: yaml file: params.yaml loaded successfully]\n",
+      "[2024-10-16 02:22:48,192: INFO: common: created directory at: artifacts]\n",
+      "[2024-10-16 02:22:48,196: INFO: common: created directory at: artifacts/data_ingestion]\n",
+      "[2024-10-16 02:22:48,198: INFO: 2796563959: File found at: E:\\\\bengla text summarization\\BanSum.zip]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e0cd102bb64748cdb7dfe8d840c14a48",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Extracting Files:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-10-16 02:23:01,006: INFO: 2796563959: Extracted E:\\\\bengla text summarization\\BanSum.zip to artifacts/data_ingestion]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    data_ingestion_config = config.get_data_ingestion_config()\n",
+    "    data_ingestion = DataIngestion(config=data_ingestion_config)\n",
+    "    data_ingestion.load_file()\n",
+    "    data_ingestion.extract_zip_file()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/benglasummarization/components/prepare_ben_token.py

@@ -0,0 +1,29 @@
+import pandas as pd
+from pathlib import Path
+from src.benglasummarization.logging import logger
+from tqdm.notebook import tqdm
+from src.benglasummarization.entity.config_entity import BanTokenizationConfig
+class BanTokenization:
+    def __init__(self, config: BanTokenizationConfig):
+        self.config = config
+
+    def combine_text_columns(self, text_columns=['main']):
+        df = pd.read_csv(self.config.source_dir)
+
+        # Ensure save_dir is a Path object
+        save_dir = Path(self.config.save_dir)
+        
+        # Create the directory if it doesn't exist
+        save_dir.mkdir(parents=True, exist_ok=True)
+
+        # Combine save_dir and output_file to form the output path
+        output_txt_file = save_dir / self.config.output_file
+        
+        # Write the combined text data to the output file
+        with open(output_txt_file, 'w', encoding='utf-8') as f:
+            for index, row in tqdm(df.iterrows(), total=len(df)):
+                combined_text = ' '.join(str(row[col]) for col in text_columns)
+                f.write(combined_text + '\n')
+
+        # Log the success of the operation
+        logger.info(f"All text data has been combined into {output_txt_file}")

src/benglasummarization/config/configuration.py

@@ -1,6 +1,7 @@
 from src.benglasummarization.constants import *
 from src.benglasummarization.utils.common import read_yaml, create_directories
 from benglasummarization.entity.config_entity import DataIngestionConfig
+from src.benglasummarization.entity.config_entity import BanTokenizationConfig
 class ConfigurationManager:
     def __init__(
         self, 
@@ -23,4 +24,19 @@ class ConfigurationManager:
             unzip_dir= config.unzip_dir
         )
         
-        return data_ingestion_config
+        return data_ingestion_config
+    
+    
+    def get_ben_tokenization_config(self) -> BanTokenizationConfig:
+        config = self.config.ban_tokenization
+        params = self.params
+        create_directories([config.root_dir])
+
+        ben_tokenization_config = BanTokenizationConfig(
+            root_dir=config.root_dir,
+            source_dir=config.source_dir,
+            save_dir= config.save_dir,
+            output_file= params.output_file
+        )
+ 
+        return ben_tokenization_config

src/benglasummarization/entity/config_entity.py

@@ -7,4 +7,12 @@ class DataIngestionConfig:
     source_dir : Path
     local_data_file : Path
     unzip_dir : Path
+    
+    
+@dataclass(frozen=True)
+class BanTokenizationConfig:
+    root_dir : Path
+    source_dir : Path
+    save_dir : Path
+    output_file : str
     

src/benglasummarization/pipeline/stage01_data_ingestion.py

@@ -1,6 +1,5 @@
 from src.benglasummarization.components.data_ingestion import DataIngestion
 from src.benglasummarization.config.configuration import ConfigurationManager
-from src.benglasummarization.config.configuration import ConfigurationManager
 
 class DataIngestionPipeline:
     def __init__(self):

src/benglasummarization/pipeline/stage_02_prepare_ben_tok.py

@@ -0,0 +1,13 @@
+from src.benglasummarization.components.prepare_ben_token import BanTokenization
+from src.benglasummarization.config.configuration import ConfigurationManager
+
+
+class BenTokenizationPreparePipeLine:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        prepare_ben_tok_config = config.get_ben_tokenization_config()  
+        ben_data_tok = BanTokenization(config=prepare_ben_tok_config)
+        ben_data_tok.combine_text_columns()