Spaces:
Sleeping
Sleeping
File size: 1,270 Bytes
fbfbbd7 afb405a fbfbbd7 f3dfbd4 fbfbbd7 f3dfbd4 fbfbbd7 0881f45 fbfbbd7 f3dfbd4 e0eceb3 f3dfbd4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
from typing import List, Dict
import os
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
class PDFProcessor:
def __init__(self):
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=200,
length_function=len,
separators=["\n\n", "\n", " ", ""]
)
def process_pdf(self, pdf_path: str) -> List[Dict]:
"""
Process a PDF file and return chunks of text with metadata.
Args:
pdf_path (str): Path to the PDF file
Returns:
List[Dict]: List of text chunks with metadata
"""
# Load PDF
loader = PyPDFLoader(pdf_path)
pages = loader.load()
# Split text into chunks
chunks = self.text_splitter.split_documents(pages)
# Format chunks with metadata
processed_chunks = []
for chunk in chunks:
processed_chunks.append({
'text': chunk.page_content,
'metadata': {
'page': chunk.metadata.get('page', 0) + 1
}
})
return processed_chunks
|