add_embeddings.py

import os
from PyPDF2 import PdfReader
from transformers import AutoTokenizer, AutoModel
import torch
import chromadb
from typing import List, Dict
import re
import numpy as np
from pathlib import Path

class LegalDocumentProcessor:
    def __init__(self):
        print("Initializing Legal Document Processor...")
        self.tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
        self.model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
        self.max_chunk_size = 500  # Reduced chunk size
        self.max_context_length = 4000  # Maximum context length for response
        
        # Initialize ChromaDB
        self.pdf_dir = "/home/user/app"
        db_dir = os.path.join(self.pdf_dir, "chroma_db")
        os.makedirs(db_dir, exist_ok=True)
        
        print(f"Initializing ChromaDB at {db_dir}")
        self.chroma_client = chromadb.PersistentClient(path=db_dir)
        
        try:
            self.collection = self.chroma_client.get_collection("indian_legal_docs")
            print("Found existing collection")
        except:
            print("Creating new collection")
            self.collection = self.chroma_client.create_collection(
                name="indian_legal_docs",
                metadata={"description": "Indian Criminal Law Documents"}
            )
    
    def _split_into_chunks(self, text: str) -> List[str]:
        """Split text into smaller chunks while preserving context"""
        # Split on meaningful boundaries
        patterns = [
            r'(?=Chapter \d+)',
            r'(?=Section \d+)',
            r'(?=\n\d+\.\s)',  # Numbered paragraphs
            r'\n\n'
        ]
        
        # Combine patterns
        split_pattern = '|'.join(patterns)
        sections = re.split(split_pattern, text)
        
        chunks = []
        current_chunk = ""
        
        for section in sections:
            section = section.strip()
            if not section:
                continue
                
            # If section is small enough, add to current chunk
            if len(current_chunk) + len(section) < self.max_chunk_size:
                current_chunk += " " + section
            else:
                # If current chunk is not empty, add it to chunks
                if current_chunk:
                    chunks.append(current_chunk.strip())
                # Start new chunk with current section
                current_chunk = section
        
        # Add the last chunk if not empty
        if current_chunk:
            chunks.append(current_chunk.strip())
        
        return chunks
    
    def process_pdf(self, pdf_path: str) -> List[str]:
        """Extract text from PDF and split into chunks"""
        print(f"Processing PDF: {pdf_path}")
        try:
            reader = PdfReader(pdf_path)
            text = ""
            for page in reader.pages:
                text += page.extract_text() + "\n\n"
            
            chunks = self._split_into_chunks(text)
            print(f"Created {len(chunks)} chunks from {pdf_path}")
            return chunks
        except Exception as e:
            print(f"Error processing PDF {pdf_path}: {str(e)}")
            return []
    
    def get_embedding(self, text: str) -> List[float]:
        """Generate embedding for text"""
        inputs = self.tokenizer(text, padding=True, truncation=True, max_length=512, return_tensors='pt')
        with torch.no_grad():
            model_output = self.model(**inputs)
        
        # Mean pooling
        token_embeddings = model_output[0]
        attention_mask = inputs['attention_mask']
        mask = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
        sum_embeddings = torch.sum(token_embeddings * mask, 1)
        sum_mask = torch.clamp(mask.sum(1), min=1e-9)
        return (sum_embeddings / sum_mask).squeeze().tolist()
    
    def process_and_store_documents(self):
        """Process all legal documents and store in ChromaDB"""
        print("Starting document processing...")
        
        # Define the expected PDF paths
        pdf_files = {
            'BNS': os.path.join(self.pdf_dir, 'BNS.pdf'),
            'BNSS': os.path.join(self.pdf_dir, 'BNSS.pdf'),
            'BSA': os.path.join(self.pdf_dir, 'BSA.pdf')
        }
        
        for law_code, pdf_path in pdf_files.items():
            if os.path.exists(pdf_path):
                print(f"Processing {law_code} from {pdf_path}")
                chunks = self.process_pdf(pdf_path)
                
                if not chunks:
                    print(f"No chunks extracted from {pdf_path}")
                    continue
                
                for i, chunk in enumerate(chunks):
                    try:
                        embedding = self.get_embedding(chunk)
                        
                        self.collection.add(
                            documents=[chunk],
                            embeddings=[embedding],
                            metadatas=[{
                                "law_code": law_code,
                                "chunk_id": f"{law_code}_chunk_{i}",
                                "source": os.path.basename(pdf_path)
                            }],
                            ids=[f"{law_code}_chunk_{i}"]
                        )
                    except Exception as e:
                        print(f"Error processing chunk {i} from {law_code}: {str(e)}")
    
    def search_documents(self, query: str, n_results: int = 3) -> Dict:
        """Search for relevant legal information"""
        try:
            query_embedding = self.get_embedding(query)
            results = self.collection.query(
                query_embeddings=[query_embedding],
                n_results=n_results
            )
            
            # Limit context size
            documents = results["documents"][0]
            total_length = 0
            filtered_documents = []
            filtered_metadatas = []
            
            for doc, metadata in zip(documents, results["metadatas"][0]):
                doc_length = len(doc)
                if total_length + doc_length <= self.max_context_length:
                    filtered_documents.append(doc)
                    filtered_metadatas.append(metadata)
                    total_length += doc_length
                else:
                    break
            
            return {
                "documents": filtered_documents,
                "metadatas": filtered_metadatas
            }
        except Exception as e:
            print(f"Error during search: {str(e)}")
            return {
                "documents": ["Sorry, I couldn't search the documents effectively."],
                "metadatas": [{"law_code": "ERROR", "source": "error"}]
            }