import base64 import os import requests from io import BytesIO from openai import OpenAI from pdf2image import convert_from_path from langchain.schema import Document from modules.config.constants import TIMEOUT class GPTParser: """ This class uses OpenAI's GPT-4o mini model to parse PDFs and extract text, images and equations. It is the most advanced parser in the system and is able to handle complex formats and layouts """ def __init__(self): self.client = OpenAI() self.api_key = os.getenv("OPENAI_API_KEY") self.prompt = """ The provided documents are images of PDFs of lecture slides of deep learning material. They contain LaTeX equations, images, and text. The goal is to extract the text, images and equations from the slides and convert everything to markdown format. Some of the equations may be complicated. The markdown should be clean and easy to read, and any math equation should be converted to LaTeX, between $$. For images, give a description and if you can, a source. Separate each page with '---'. Just respond with the markdown. Do not include page numbers or any other metadata. Do not try to provide titles. Strictly the content. """ def parse(self, pdf_path): images = convert_from_path(pdf_path) encoded_images = [self.encode_image(image) for image in images] chunks = [encoded_images[i : i + 5] for i in range(0, len(encoded_images), 5)] headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", } output = "" for chunk_num, chunk in enumerate(chunks): content = [ { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image}"}, } for image in chunk ] content.insert(0, {"type": "text", "text": self.prompt}) payload = { "model": "gpt-4o-mini", "messages": [{"role": "user", "content": content}], } response = requests.post( "https://api.openai.com/v1/chat/completions", headers=headers, json=payload, timeout=TIMEOUT, ) resp = response.json() chunk_output = ( resp["choices"][0]["message"]["content"] .replace("```", "") .replace("markdown", "") .replace("````", "") ) output += chunk_output + "\n---\n" output = output.split("\n---\n") output = [doc for doc in output if doc.strip() != ""] documents = [ Document(page_content=page, metadata={"source": pdf_path, "page": i}) for i, page in enumerate(output) ] return documents def encode_image(self, image): buffered = BytesIO() image.save(buffered, format="JPEG") return base64.b64encode(buffered.getvalue()).decode("utf-8")