train bengla tokenization added

config/config.yaml

@@ -11,4 +11,10 @@ ban_tokenization:
   source_dir: artifacts/data_ingestion/BanSum.csv
   save_dir: artifacts/ban_tokenization  
 
+train_tokenize:
+  root_dir : artifacts/train_tokenization
+  input_file_dir : artifacts/ban_tokenization/combined_text.txt
+  save_file : artifacts/train_tokenization
+
+  
 

main.py

@@ -1,7 +1,7 @@
 from src.benglasummarization.logging import logger
 from src.benglasummarization.pipeline.stage01_data_ingestion import DataIngestionPipeline
 from src.benglasummarization.pipeline.stage_02_prepare_ben_tok import BenTokenizationPreparePipeLine
-
+from src.benglasummarization.pipeline.stage_03_train_ban_token import TrainTokenizePipeLine
 STAGE_NAME = 'Data Ingestion Stage'
 
 try:
@@ -22,4 +22,16 @@ try:
    logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
 except Exception as e:
         logger.exception(e)
-        raise e
+        raise e
+     
+STAGE_NAME = 'Training Bengla Tokenization Stage'
+
+try:
+   logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") 
+   Train_Ban_Token = TrainTokenizePipeLine()
+   Train_Ban_Token.main()
+   logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x")
+except Exception as e:
+   logger.exception(e)
+   raise e
+

params.yaml

@@ -1,2 +1,7 @@
+pre_tokenize:
+  output_file: "combined_text.txt"
 
-output_file: "combined_text.txt"
+train_tokenize:
+  model_prefix : 'cbengali_tokenizer'
+  model_type : 'unigram'
+  vocab_size : 91902

research/prepare_ben_tokenization.ipynb

@@ -0,0 +1,201 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'e:\\\\bengla text summarization\\\\train-pegasus-model-on-bengali-text-summarization-using-mlops'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class BanTokenizationConfig:\n",
+    "    root_dir : Path\n",
+    "    source_dir : Path\n",
+    "    save_dir : Path\n",
+    "    output_file : str\n",
+    "    \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.benglasummarization.constants import *\n",
+    "from src.benglasummarization.utils.common import  create_directories, read_yaml\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH):\n",
+    "\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "\n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "\n",
+    "    def get_ben_tokenization_config(self) -> BanTokenizationConfig:\n",
+    "        config = self.config.ban_tokenization\n",
+    "        params = self.params.pre_tokenize\n",
+    "        create_directories([config.root_dir])\n",
+    "\n",
+    "        ben_tokenization_config = BanTokenizationConfig(\n",
+    "            root_dir=config.root_dir,\n",
+    "            source_dir=config.source_dir,\n",
+    "            save_dir= config.save_dir,\n",
+    "            output_file= params.output_file\n",
+    "        )\n",
+    " \n",
+    "        return ben_tokenization_config\n",
+    "\n",
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "from src.benglasummarization.logging import logger\n",
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "class BanTokenization:\n",
+    "    def __init__(self, config: BanTokenizationConfig):\n",
+    "        self.config = config\n",
+    "\n",
+    "    def combine_text_columns(self, text_columns=['main']):\n",
+    "        df = pd.read_csv(self.config.source_dir)\n",
+    "\n",
+    "        # Ensure save_dir is a Path object\n",
+    "        save_dir = Path(self.config.save_dir)\n",
+    "        \n",
+    "        # Create the directory if it doesn't exist\n",
+    "        save_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "        # Combine save_dir and output_file to form the output path\n",
+    "        output_txt_file = save_dir / self.config.output_file\n",
+    "        \n",
+    "        # Write the combined text data to the output file\n",
+    "        with open(output_txt_file, 'w', encoding='utf-8') as f:\n",
+    "            for index, row in tqdm(df.iterrows(), total=len(df)):\n",
+    "                combined_text = ' '.join(str(row[col]) for col in text_columns)\n",
+    "                f.write(combined_text + '\\n')\n",
+    "\n",
+    "        # Log the success of the operation\n",
+    "        logger.info(f\"All text data has been combined into {output_txt_file}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-10-16 19:09:09,141: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
+      "[2024-10-16 19:09:09,143: INFO: common: yaml file: params.yaml loaded successfully]\n",
+      "[2024-10-16 19:09:09,145: INFO: common: created directory at: artifacts]\n",
+      "[2024-10-16 19:09:09,146: INFO: common: created directory at: artifacts/ban_tokenization]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "46422977ab65463695c98b98ece484c2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/160000 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-10-16 19:10:00,660: INFO: 206824922: All text data has been combined into artifacts\\ban_tokenization\\combined_text.txt]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    prepare_ben_tok_config = config.get_ben_tokenization_config()  \n",
+    "    ben_data_tok = BanTokenization(config=prepare_ben_tok_config)\n",
+    "    ben_data_tok.combine_text_columns()\n",
+    "except Exception as e:\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

research/train_ban_token.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "from pathlib import Path\n",
+    "\n",
+    "@dataclass(frozen=True)\n",
+    "class BanTokenTrainConfig:\n",
+    "    root_dir : Path\n",
+    "    input_file_dir : Path\n",
+    "    save_file : Path\n",
+    "    model_prefix : str\n",
+    "    model_type : str\n",
+    "    vocab_size : int"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.benglasummarization.constants import  *\n",
+    "from src.benglasummarization.utils.common import create_directories, read_yaml\n",
+    "\n",
+    "class ConfigurationManager:\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        config_filepath = CONFIG_FILE_PATH,\n",
+    "        params_filepath = PARAMS_FILE_PATH):\n",
+    "\n",
+    "        self.config = read_yaml(config_filepath)\n",
+    "        self.params = read_yaml(params_filepath)\n",
+    "\n",
+    "        create_directories([self.config.artifacts_root])\n",
+    "\n",
+    "    def get_train_token_config(self) -> BanTokenTrainConfig:\n",
+    "        config = self.config.train_tokenize\n",
+    "        params = self.params.train_tokenize\n",
+    "        create_directories([config.root_dir])\n",
+    "        \n",
+    "        train_token_config = BanTokenTrainConfig(\n",
+    "            root_dir= config.root_dir,\n",
+    "            input_file_dir= config.input_file_dir,\n",
+    "            save_file= config.save_file,\n",
+    "            model_prefix= params.model_prefix,\n",
+    "            model_type= params.model_type,\n",
+    "            vocab_size= params.vocab_size\n",
+    "        )\n",
+    "        return train_token_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sentencepiece as spm\n",
+    "from src.benglasummarization.logging import logger\n",
+    "from tqdm.notebook import tqdm\n",
+    "import os\n",
+    "\n",
+    "class TrainTokenize:\n",
+    "    def __init__(self, config: BanTokenTrainConfig):\n",
+    "        self.config = config\n",
+    "        \n",
+    "    def train_tokenizer(self):\n",
+    "        with open(self.config.input_file_dir, 'r', encoding='utf-8') as f:\n",
+    "            total_lines = sum(1 for line in f)\n",
+    "\n",
+    "        with tqdm(total=total_lines, desc='Preparing Sentence for Training', unit='lines') as pbar:\n",
+    "            with open(self.config.input_file_dir, 'r', encoding='utf-8') as f:\n",
+    "                for _ in f:\n",
+    "                    pbar.update(1)\n",
+    "                    \n",
+    "        # Ensure the save directory exists\n",
+    "        os.makedirs(os.path.dirname(self.config.save_file), exist_ok=True)\n",
+    "        \n",
+    "        # Training Arguments\n",
+    "        train_params = {\n",
+    "            'input': str(self.config.input_file_dir),\n",
+    "            'model_prefix': os.path.join(self.config.save_file, self.config.model_prefix),\n",
+    "            'vocab_size': self.config.vocab_size,\n",
+    "            'model_type': self.config.model_type,\n",
+    "            'character_coverage': 1.0,\n",
+    "            'input_sentence_size': 1000000,\n",
+    "            'shuffle_input_sentence': True\n",
+    "        }\n",
+    "        \n",
+    "        spm.SentencePieceTrainer.train(**train_params)\n",
+    "        logger.info(f'Tokenizer model saved to {train_params[\"model_prefix\"]}.model')\n",
+    "        logger.info(f'Tokenizer vocabulary saved to {train_params[\"model_prefix\"]}.vocab')\n",
+    "    \n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-10-16 20:25:26,476: INFO: common: yaml file: config\\config.yaml loaded successfully]\n",
+      "[2024-10-16 20:25:26,477: INFO: common: yaml file: params.yaml loaded successfully]\n",
+      "[2024-10-16 20:25:26,478: INFO: common: created directory at: artifacts]\n",
+      "[2024-10-16 20:25:26,480: INFO: common: created directory at: artifacts/train_tokenization]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "57e6c332ff144237a7683e64bf137c3c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Preparing Sentence for Training:   0%|          | 0/160000 [00:00<?, ?lines/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2024-10-16 20:26:03,153: INFO: 489807411: Tokenizer model saved to artifacts/train_tokenization\\cbengali_tokenizer.model]\n",
+      "[2024-10-16 20:26:03,154: INFO: 489807411: Tokenizer vocabulary saved to artifacts/train_tokenization\\cbengali_tokenizer.vocab]\n"
+     ]
+    }
+   ],
+   "source": [
+    "try:\n",
+    "    config = ConfigurationManager()\n",
+    "    train_token_config = config.get_train_token_config()\n",
+    "    train_config = TrainTokenize(config=train_token_config)\n",
+    "    train_config.train_tokenizer()\n",
+    "except Exception as e:\n",
+    "    logger.error(f\"An error occurred: {str(e)}\")\n",
+    "    raise e"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/benglasummarization/components/train_bn_token.py

@@ -0,0 +1,37 @@
+import sentencepiece as spm
+from src.benglasummarization.logging import logger
+from tqdm.notebook import tqdm
+import os
+from src.benglasummarization.entity.config_entity import BanTokenTrainConfig
+class TrainTokenize:
+    def __init__(self, config: BanTokenTrainConfig):
+        self.config = config
+        
+    def train_tokenizer(self):
+        with open(self.config.input_file_dir, 'r', encoding='utf-8') as f:
+            total_lines = sum(1 for line in f)
+
+        with tqdm(total=total_lines, desc='Preparing Sentence for Training', unit='lines') as pbar:
+            with open(self.config.input_file_dir, 'r', encoding='utf-8') as f:
+                for _ in f:
+                    pbar.update(1)
+                    
+        # Ensure the save directory exists
+        os.makedirs(os.path.dirname(self.config.save_file), exist_ok=True)
+        
+        # Training Arguments
+        train_params = {
+            'input': str(self.config.input_file_dir),
+            'model_prefix': os.path.join(self.config.save_file, self.config.model_prefix),
+            'vocab_size': self.config.vocab_size,
+            'model_type': self.config.model_type,
+            'character_coverage': 1.0,
+            'input_sentence_size': 1000000,
+            'shuffle_input_sentence': True
+        }
+        
+        spm.SentencePieceTrainer.train(**train_params)
+        logger.info(f'Tokenizer model saved to {train_params["model_prefix"]}.model')
+        logger.info(f'Tokenizer vocabulary saved to {train_params["model_prefix"]}.vocab')
+    
+    

src/benglasummarization/config/configuration.py

@@ -2,6 +2,7 @@ from src.benglasummarization.constants import *
 from src.benglasummarization.utils.common import read_yaml, create_directories
 from benglasummarization.entity.config_entity import DataIngestionConfig
 from src.benglasummarization.entity.config_entity import BanTokenizationConfig
+from src.benglasummarization.entity.config_entity import BanTokenTrainConfig
 class ConfigurationManager:
     def __init__(
         self, 
@@ -29,7 +30,7 @@ class ConfigurationManager:
     
     def get_ben_tokenization_config(self) -> BanTokenizationConfig:
         config = self.config.ban_tokenization
-        params = self.params
+        params = self.params.pre_tokenize
         create_directories([config.root_dir])
 
         ben_tokenization_config = BanTokenizationConfig(
@@ -39,4 +40,20 @@ class ConfigurationManager:
             output_file= params.output_file
         )
  
-        return ben_tokenization_config
+        return ben_tokenization_config
+    
+    
+    def get_train_token_config(self) -> BanTokenTrainConfig:
+        config = self.config.train_tokenize
+        params = self.params.train_tokenize
+        create_directories([config.root_dir])
+        
+        train_token_config = BanTokenTrainConfig(
+            root_dir= config.root_dir,
+            input_file_dir= config.input_file_dir,
+            save_file= config.save_file,
+            model_prefix= params.model_prefix,
+            model_type= params.model_type,
+            vocab_size= params.vocab_size
+        )
+        return train_token_config

src/benglasummarization/entity/config_entity.py

@@ -15,4 +15,13 @@ class BanTokenizationConfig:
     source_dir : Path
     save_dir : Path
     output_file : str
+    
+@dataclass(frozen=True)
+class BanTokenTrainConfig:
+    root_dir : Path
+    input_file_dir : Path
+    save_file : Path
+    model_prefix : str
+    model_type : str
+    vocab_size : int
     

src/benglasummarization/pipeline/stage_03_train_ban_token.py

@@ -0,0 +1,13 @@
+from src.benglasummarization.config.configuration import ConfigurationManager
+from src.benglasummarization.components.train_bn_token import TrainTokenize
+
+class TrainTokenizePipeLine:
+    def __init__(self):
+        pass
+    
+    def main(self):
+        config = ConfigurationManager()
+        train_ban_tok = config.get_train_token_config()
+        train_tok = TrainTokenize(config=train_ban_tok)
+        train_tok.train_tokenizer()
+