feat: melhorias no código e refatorações

_utils/LLMs/LLM_class.py

@@ -0,0 +1,9 @@
+from setup.environment import default_model
+
+
+class LLM:
+    def __init__(self):
+        pass
+
+    # def create_GPT_model(self, model=default_model):
+    #     return ChatOpen()

_utils/chains/Chain_class.py

@@ -0,0 +1,11 @@
+class Chain:
+    def __init__(self, prompt, model):
+        self.prompt = prompt
+        self.model = model
+
+    def create_prompt_model_chain(self):
+        return self.prompt | self.model
+
+    def invoke_prompt_model_chain(self, invoke_params):
+        chain = self.create_prompt_model_chain()
+        return chain.invoke(invoke_params)

_utils/gerar_relatorio_modelo_usuario/DocumentSummarizer_simples.py

@@ -1,18 +1,16 @@
-import os
 from typing import List, Dict, Tuple, Optional
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.document_loaders import PyPDFLoader
-from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_community.vectorstores import Chroma
-from langchain_community.chat_models import ChatOpenAI
-from langchain.prompts import PromptTemplate
-import uuid
+from _utils.splitters.Splitter_class import Splitter
+from setup.easy_imports import (
+    HuggingFaceEmbeddings,
+    Chroma,
+    ChatOpenAI,
+    PromptTemplate,
+)
 import logging
 from cohere import Client
 from _utils.models.gerar_relatorio import (
     DocumentChunk,
 )
-from langchain.schema import Document
 
 
 class DocumentSummarizer:
@@ -29,94 +27,10 @@ class DocumentSummarizer:
         self.openai_api_key = openai_api_key
         self.cohere_client = Client(cohere_api_key)
         self.embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
-        self.text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=chunk_size, chunk_overlap=chunk_overlap
-        )
-        self.chunk_metadata = {}  # Store chunk metadata for tracing
         self.num_k_rerank = num_k_rerank
         self.model_cohere_rerank = model_cohere_rerank
 
-    def load_and_split_document(self, pdf_path: str) -> List[DocumentChunk]:
-        """Load PDF and split into chunks with metadata"""
-        loader = PyPDFLoader(pdf_path)
-        pages = (
-            loader.load()
-        )  # Gera uma lista de objetos Document, sendo cada item da lista referente a UMA PÁGINA inteira do PDF.
-        chunks = []
-        char_count = 0
-
-        for page in pages:
-            text = page.page_content
-            page_chunks = self.text_splitter.split_text(
-                text
-            )  # Quebra o item que é um Document de UMA PÁGINA inteira em um lista onde cada item é referente a um chunk, que são pedaços menores do que uma página.
-
-            for chunk in page_chunks:
-                chunk_id = str(uuid.uuid4())
-                start_char = text.find(
-                    chunk
-                )  # Retorna a posição onde se encontra o chunk dentro da página inteira
-                end_char = start_char + len(chunk)
-
-                doc_chunk = DocumentChunk(  # Gera o objeto do chunk com informações adicionais, como a posição e id do chunk
-                    content=chunk,
-                    page_number=page.metadata.get("page") + 1,  # 1-based page numbering
-                    chunk_id=chunk_id,
-                    start_char=char_count + start_char,
-                    end_char=char_count + end_char,
-                )
-                chunks.append(doc_chunk)
-
-                # Store metadata for later retrieval
-                self.chunk_metadata[chunk_id] = {
-                    "page": doc_chunk.page_number,
-                    "start_char": doc_chunk.start_char,
-                    "end_char": doc_chunk.end_char,
-                }
-
-            char_count += len(text)
-
-        return chunks
-
-    def load_and_split_text(self, text: str) -> List[DocumentChunk]:
-        """Load Text and split into chunks with metadata - Criei essa função apenas para o ragas"""
-        page = Document(page_content=text, metadata={"page": 1})
-        chunks = []
-        char_count = 0
-
-        text = page.page_content
-        page_chunks = self.text_splitter.split_text(
-            text
-        )  # Quebra o item que é um Document de UMA PÁGINA inteira em um lista onde cada item é referente a um chunk, que são pedaços menores do que uma página.
-        print("\n\n\n")
-        print("page_chunks: ", page_chunks)
-
-        for chunk in page_chunks:
-            chunk_id = str(uuid.uuid4())
-            start_char = text.find(
-                chunk
-            )  # Retorna a posição onde se encontra o chunk dentro da página inteira
-            end_char = start_char + len(chunk)
-
-            doc_chunk = DocumentChunk(  # Gera o objeto do chunk com informações adicionais, como a posição e id do chunk
-                content=chunk,
-                page_number=page.metadata.get("page") + 1,  # 1-based page numbering
-                chunk_id=chunk_id,
-                start_char=char_count + start_char,
-                end_char=char_count + end_char,
-            )
-            chunks.append(doc_chunk)
-
-            # Store metadata for later retrieval
-            self.chunk_metadata[chunk_id] = {
-                "page": doc_chunk.page_number,
-                "start_char": doc_chunk.start_char,
-                "end_char": doc_chunk.end_char,
-            }
-
-        char_count += len(text)
-
-        return chunks
+        self.splitter = Splitter(chunk_size, chunk_overlap)
 
     def create_vector_store(
         self, chunks: List[DocumentChunk]
@@ -233,7 +147,7 @@ class DocumentSummarizer:
             temperature=0, model_name="gpt-4o-mini", api_key=self.openai_api_key
         )
 
-        response = llm.predict(prompt.format(context="\n\n".join(contexts)))
+        response = llm.invoke(prompt.format(context="\n\n".join(contexts))).content
 
         # Split the response into paragraphs
         summaries = [p.strip() for p in response.split("\n\n") if p.strip()]

_utils/gerar_relatorio_modelo_usuario/EnhancedDocumentSummarizer.py

@@ -1,22 +1,23 @@
 import os
 from typing import List, Dict, Tuple, Optional
-from langchain_community.vectorstores import Chroma
-from langchain_community.chat_models import ChatOpenAI
-from langchain.chains import create_extraction_chain
-from langchain.prompts import PromptTemplate
-from rank_bm25 import BM25Okapi
+from _utils.vector_stores.Vector_store_class import VectorStore
+from setup.easy_imports import (
+    Chroma,
+    ChatOpenAI,
+    PromptTemplate,
+    BM25Okapi,
+    Response,
+)
 import logging
 import requests
 from _utils.gerar_relatorio_modelo_usuario.DocumentSummarizer_simples import (
     DocumentSummarizer,
 )
 from _utils.models.gerar_relatorio import (
-    ContextualizedChunk,
     RetrievalConfig,
 )
 from modelos_usuarios.serializer import ModeloUsuarioSerializer
 from setup.environment import api_url
-from rest_framework.response import Response
 from _utils.gerar_relatorio_modelo_usuario.contextual_retriever import (
     ContextualRetriever,
 )
@@ -24,6 +25,7 @@ from asgiref.sync import sync_to_async
 
 
 class EnhancedDocumentSummarizer(DocumentSummarizer):
+
     def __init__(
         self,
         openai_api_key: str,
@@ -35,12 +37,12 @@ class EnhancedDocumentSummarizer(DocumentSummarizer):
         num_k_rerank,
         model_cohere_rerank,
         claude_context_model,
-        prompt_relatorio,
+        prompt_auxiliar,
         gpt_model,
         gpt_temperature,
         id_modelo_do_usuario,
-        prompt_modelo,
-        reciprocal_rank_fusion
+        prompt_gerar_documento,
+        reciprocal_rank_fusion,
     ):
         super().__init__(
             openai_api_key,
@@ -56,58 +58,15 @@ class EnhancedDocumentSummarizer(DocumentSummarizer):
             config, claude_api_key, claude_context_model
         )
         self.logger = logging.getLogger(__name__)
-        self.prompt_relatorio = prompt_relatorio
+        self.prompt_auxiliar = prompt_auxiliar
         self.gpt_model = gpt_model
         self.gpt_temperature = gpt_temperature
         self.id_modelo_do_usuario = id_modelo_do_usuario
-        self.prompt_modelo = prompt_modelo
+        self.prompt_gerar_documento = prompt_gerar_documento
         self.reciprocal_rank_fusion = reciprocal_rank_fusion
         self.resumo_gerado = ""
 
-    def create_enhanced_vector_store(
-        self, chunks: List[ContextualizedChunk], is_contextualized_chunk
-    ) -> Tuple[Chroma, BM25Okapi, List[str]]:
-        """Create vector store and BM25 index with contextualized chunks"""
-        try:
-            # Prepare texts with context
-            if is_contextualized_chunk:
-                texts = [f"{chunk.context} {chunk.content}" for chunk in chunks]
-            else:
-                texts = [f"{chunk.content}" for chunk in chunks]
-
-            # Create vector store
-            metadatas = []
-            for chunk in chunks:
-                if is_contextualized_chunk:
-                    context = chunk.context
-                else:
-                    context = ""
-                metadatas.append(
-                    {
-                        "chunk_id": chunk.chunk_id,
-                        "page": chunk.page_number,
-                        "start_char": chunk.start_char,
-                        "end_char": chunk.end_char,
-                        "context": context,
-                    }
-                )
-
-            vector_store = Chroma.from_texts(
-                texts=texts, metadatas=metadatas, embedding=self.embeddings
-            )
-
-            # Create BM25 index
-            tokenized_texts = [text.split() for text in texts]
-            bm25 = BM25Okapi(tokenized_texts)
-
-            # Get chunk IDs in order
-            chunk_ids = [chunk.chunk_id for chunk in chunks]
-
-            return vector_store, bm25, chunk_ids
-
-        except Exception as e:
-            self.logger.error(f"Error creating enhanced vector store: {str(e)}")
-            raise
+        self.vector_store = VectorStore(embedding_model)
 
     def retrieve_with_rank_fusion(
         self, vector_store: Chroma, bm25: BM25Okapi, chunk_ids: List[str], query: str
@@ -254,25 +213,25 @@ class EnhancedDocumentSummarizer(DocumentSummarizer):
             )
 
             prompt_gerar_relatorio = PromptTemplate(
-                template=self.prompt_relatorio, input_variables=["context"]
+                template=self.prompt_auxiliar, input_variables=["context"]
             )
 
-            relatorio_gerado = llm.predict(
+            relatorio_gerado = llm.invoke(
                 prompt_gerar_relatorio.format(context="\n\n".join(contexts))
             )
 
-            self.resumo_gerado = relatorio_gerado
+            self.resumo_gerado = relatorio_gerado.content
 
             prompt_gerar_modelo = PromptTemplate(
-                template=self.prompt_modelo,
+                template=self.prompt_gerar_documento,
                 input_variables=["context", "modelo_usuario"],
             )
 
-            modelo_gerado = llm.predict(
+            modelo_gerado = llm.invoke(
                 prompt_gerar_modelo.format(
                     context=relatorio_gerado, modelo_usuario=serializer.data["modelo"]
                 )
-            )
+            ).content
 
             # Split the response into paragraphs
             summaries = [p.strip() for p in modelo_gerado.split("\n\n") if p.strip()]

_utils/gerar_relatorio_modelo_usuario/contextual_retriever.py

@@ -1,4 +1,11 @@
 import os
+# from _utils.gerar_relatorio_modelo_usuario.prompts import (
+#     prompt_auxiliar_do_contextual_prompt,
+# )
+from _utils.chains.Chain_class import Chain
+from _utils.prompts.Prompt_class import Prompt
+from _utils.splitters.Splitter_class import Splitter
+from setup.easy_imports import PyPDFLoader
 from langchain_openai import ChatOpenAI
 from typing import List, Dict, Tuple, Optional
 from anthropic import Anthropic, AsyncAnthropic
@@ -12,7 +19,7 @@ from dataclasses import dataclass
 from langchain_core.messages import HumanMessage
 from asgiref.sync import sync_to_async
 
-from _utils.gerar_relatorio_modelo_usuario.llm_calls import claude_answer, gpt_answer
+from _utils.gerar_relatorio_modelo_usuario.llm_calls import aclaude_answer, agpt_answer
 from _utils.gerar_relatorio_modelo_usuario.prompts import contextual_prompt
 from _utils.models.gerar_relatorio import (
     ContextualizedChunk,
@@ -39,11 +46,11 @@ class ContextualRetriever:
         try:
             print("COMEÇOU A REQUISIÇÃO")
             prompt = contextual_prompt(full_text, chunk.content)
-            # response = await claude_answer(
+            # response = await aclaude_answer(
             #     self.claude_client, self.claude_context_model, prompt
             # )
 
-            response = await gpt_answer(prompt)
+            response = await agpt_answer(prompt)
             return response
         except Exception as e:
             self.logger.error(
@@ -51,6 +58,13 @@ class ContextualRetriever:
             )
             return ""
 
+    # def gerar_resumo_auxiliar_do_contextual_embedding(self):
+    #     prompt = Prompt().create_prompt_template(
+    #         "", prompt_auxiliar_do_contextual_prompt
+    #     )
+    #     Chain(prompt, ChatOpenAI())
+    #     return
+
     async def create_contextualized_chunk(self, chunk, full_text):
         lista_contador.append(0)
         print("contador: ", len(lista_contador))
@@ -90,3 +104,40 @@ class ContextualRetriever:
         contextualized_chunks = [task.result() for task in tasks]
 
         return contextualized_chunks
+
+
+def get_full_text_and_all_PDFs_chunks(contexto, listaPDFs, splitterObject: Splitter):
+    all_PDFs_chunks = []
+    full_text = ""
+    if contexto:
+        full_text = contexto
+        chunks = splitterObject.load_and_split_text(full_text)
+        all_PDFs_chunks = chunks
+    else:
+        # Load and process document
+        for pdf in listaPDFs:
+            pdf_path = pdf
+            chunks = splitterObject.load_and_split_document(pdf_path)
+            all_PDFs_chunks = all_PDFs_chunks + chunks
+        # Get full text for contextualization
+        loader = PyPDFLoader(pdf_path)
+        pages = loader.load()
+        full_text = " ".join([page.page_content for page in pages])
+
+    return full_text, all_PDFs_chunks, pages
+
+
+async def contextualize_chunk_based_on_serializer(
+    serializer, contextual_retriever: ContextualRetriever, pages, all_PDFs_chunks
+):
+    if serializer["should_have_contextual_chunks"]:
+        contextualized_chunks = await contextual_retriever.contextualize_all_chunks(
+            pages, all_PDFs_chunks
+        )
+        chunks_passados = contextualized_chunks
+        is_contextualized_chunk = True
+    else:
+        chunks_passados = all_PDFs_chunks
+        is_contextualized_chunk = False
+
+    return chunks_passados, is_contextualized_chunk

_utils/gerar_relatorio_modelo_usuario/llm_calls.py

@@ -1,12 +1,11 @@
 import os
+from setup.environment import default_model
 from langchain_core.messages import HumanMessage
 from langchain_openai import ChatOpenAI
 
 
-async def claude_answer(claude_client, claude_context_model, prompt):
-    print("\n")
-    print("Começou uma requisição pelo Claude")
-    print("\n")
+async def aclaude_answer(claude_client, claude_context_model, prompt):
+    print("\n\nComeçou uma requisição pelo Claude")
     response = await claude_client.messages.create(
         model=claude_context_model,
         max_tokens=100,
@@ -17,7 +16,7 @@ async def claude_answer(claude_client, claude_context_model, prompt):
     ].text  # O response.content é uma lista pois é passada uma lista de mensagens, e também retornado uma lista de mensagens, sendo a primeira a mais recente, que é a resposta do model
 
 
-async def gpt_answer(prompt):
+async def agpt_answer(prompt):
     gpt = ChatOpenAI(
         temperature=0,
         model="gpt-4o-mini",
@@ -26,3 +25,23 @@ async def gpt_answer(prompt):
     )
     response = await gpt.ainvoke([HumanMessage(content=prompt)])
     return response.content
+
+
+def gpt_answer(
+    prompt,
+    temperature=0,
+    model=default_model,
+    max_retries=5,
+    shouldReturnFullResponse=False,
+):
+    gpt = ChatOpenAI(
+        temperature=temperature,
+        model=model,
+        api_key=os.environ.get("OPENAI_API_KEY"),
+        max_retries=max_retries,
+    )
+    response = gpt.invoke([HumanMessage(content=prompt)])
+    if shouldReturnFullResponse:
+        return response
+    else:
+        return response.content

_utils/gerar_relatorio_modelo_usuario/prompts.py

@@ -17,12 +17,55 @@ Please return only the succinct context (without displaying your internal reason
 ```
 """
 
-system_prompt_modelo = """
-You are a large language model that must produce a single final sentence in **Portuguese**. To do this, you will follow a private chain of thought and then produce a final answer. The final answer must follow the formatting and stylistic conventions shown in the user-provided model `user's template`. The information to be included in the final sentence is derived from the `context` (a report describing a legal case).
+# Novo nome --> prompt-auxiliar --> Para gerar documentos (é usado como auxiliar no prompt final)
+prompt_auxiliar = """
+You are a language model specialized in producing concise and well-structured legal case summaries in Portuguese. You will receive a variable `context`, which contains information about a legal case. Your task is to read the `context` carefully and produce a summary report in Portuguese, following the specific format provided below. Do not include any additional comments or reasoning steps in your final answer.
+**Instructions**:
+1. **Chain of Thought**: Before producing your final answer, you must think through and plan your summary silently, without showing this reasoning in the final output. The final answer must only contain the required formatted report and nothing else.
+2. **Reading the Context**: Extract the following information from `context`:
+- The name of the defendant (réu).
+- The crime they have been accused of (nome_do_crime).
+- The applicable article and subsection of the Penal Code (artigo_e_inciso_do_crime).
+- The date the accusation was accepted (data_do_recebimento).
+- The ID of the decision document (id_do_documento).
+3. **Prescriptive Details**: If no other interruptive or suspensive causes of prescription are mentioned, confirm that there are none.
+4. **Formatting**: Your final answer must strictly follow the format below, in Portuguese, and replace the placeholders with the appropriate information:
+```
+<relatorio>
+Trata-se de Ação Penal em que o Ministério Público denunciou [nome_do_reu], pela prática do [nome_do_crime] [artigo_e_inciso_do_crime], do Código Penal.
+A denúncia foi recebida em [data_do_recebimento], conforme Decisão [id_do_documento].
+Não há outras causas interruptivas ou suspensivas da prescrição.
+</relatorio>
+```
+5. **Completeness**: If any piece of required information is missing in the `context`, note that explicitly in the final answer within the format.
+**Reminder**:
+- Do not include your chain of thought in the final output.
+- Do not add extra information or commentary beyond the specified format.
+- The final answer must be in Portuguese.
+---
+
+**Contextual Information (provided separately):**
+{context}
+---
+**Example with a given context**:
+- Input:
+`context` = 'Em 10/03/2021, o Ministério Público denunciou João da Silva, imputando-lhe o crime de furto qualificado, previsto no art. 155, §4º, inciso II, do Código Penal. A denúncia foi recebida em 12/03/2021, conforme Decisão nº 20210312-01. Não há menção a qualquer causa interruptiva ou suspensiva da prescrição.'
+- Expected final answer:
+```
+<formato>
+Trata-se de Ação Penal em que o Ministério Público denunciou João da Silva, pela prática do furto qualificado (art. 155, §4º, inciso II do Código Penal).
+A denúncia foi recebida em 12/03/2021, conforme Decisão 20210312-01.
+Não há outras causas interruptivas ou suspensivas da prescrição.
+</formato>
+"""
+
+# Novo nome --> prompt-gerar-documento --> Para gerar documentos
+prompt_gerar_documento = """
+You are a large language model that must produce a single final document in **Portuguese**. To do this, you will follow a private chain of thought and then produce a final answer. The final answer must follow the formatting and stylistic conventions shown in the user-provided model `user's template`. The information to be included in the final document is derived from the `context` (a report describing a legal case).
 **Contextual Information (provided separately):**
 {context}
 **User Model (provided separately):**
-<user's_template>{modelo_usuario}</user's_template>
+<user's_template>PROMPT DO MODELO DO USUÁRIO</user's_template>
 **Instructions:**
 1. **Goal:** Produce one single final sentence in Portuguese that matches the structure, format, and style given by `user's template`.
 2. **Chain of Thought (private to the assistant and not to be shown in the final answer):**
@@ -46,8 +89,7 @@ After composing the sentence, but before presenting it as the final answer, refl
 - Do not show the chain of thought or the reflection step. Only the final formatted sentence should be visible to the user. 
 """
 
-system_prompt_relatorio = """
-You are a language model specialized in producing concise and well-structured legal case summaries in Portuguese. You will receive a variable `context`, which contains information about a legal case. Your task is to read the `context` carefully and produce a summary report in Portuguese, following the specific format provided below. Do not include any additional comments or reasoning steps in your final answer.
+prompt_auxiliar_SEM_CONTEXT = """You are a language model specialized in producing concise and well-structured legal case summaries in Portuguese. You will receive a variable `context`, which contains information about a legal case. Your task is to read the `context` carefully and produce a summary report in Portuguese, following the specific format provided below. Do not include any additional comments or reasoning steps in your final answer.
 **Instructions**:
 1. **Chain of Thought**: Before producing your final answer, you must think through and plan your summary silently, without showing this reasoning in the final output. The final answer must only contain the required formatted report and nothing else.
 2. **Reading the Context**: Extract the following information from `context`:
@@ -59,26 +101,17 @@ You are a language model specialized in producing concise and well-structured le
 3. **Prescriptive Details**: If no other interruptive or suspensive causes of prescription are mentioned, confirm that there are none.
 4. **Formatting**: Your final answer must strictly follow the format below, in Portuguese, and replace the placeholders with the appropriate information:
 ```
-<relatorio>
+<formato>
 Trata-se de Ação Penal em que o Ministério Público denunciou [nome_do_reu], pela prática do [nome_do_crime] [artigo_e_inciso_do_crime], do Código Penal.
 A denúncia foi recebida em [data_do_recebimento], conforme Decisão [id_do_documento].
 Não há outras causas interruptivas ou suspensivas da prescrição.
-</relatorio>
+</formato>
 ```
 5. **Completeness**: If any piece of required information is missing in the `context`, note that explicitly in the final answer within the format.
 **Reminder**:
 - Do not include your chain of thought in the final output.
 - Do not add extra information or commentary beyond the specified format.
 - The final answer must be in Portuguese.
----
-
-**Contextual Information (provided separately):**
-{context}
----
-**Example with a given context**:
-- Input:
-`context` = "Em 10/03/2021, o Ministério Público denunciou João da Silva, imputando-lhe o crime de furto qualificado, previsto no art. 155, §4º, inciso II, do Código Penal. A denúncia foi recebida em 12/03/2021, conforme Decisão nº 20210312-01. Não há menção a qualquer causa interruptiva ou suspensiva da prescrição."
-- Expected final answer:
 ```
 <formato>
 Trata-se de Ação Penal em que o Ministério Público denunciou João da Silva, pela prática do furto qualificado (art. 155, §4º, inciso II do Código Penal).
@@ -86,3 +119,61 @@ A denúncia foi recebida em 12/03/2021, conforme Decisão 20210312-01.
 Não há outras causas interruptivas ou suspensivas da prescrição.
 </formato>
 """
+
+prompt_auxiliar_do_contextual_prompt = """Você é um assistente jurídico especializado em direito brasileiro. Sua tarefa é criar um resumo conciso e informativo de um processo jurídico, de acordo com as leis do Brasil. O resumo deve focar nos momentos cruciais do processo, na última movimentação processual e nas principais movimentações que ocorreram.
+
+Aqui estão as 10 principais peças processuais em ordem cronológica do processo civil brasileiro que você deve priorizar em sua análise:
+1. Petição Inicial
+2. Contestação
+3. Réplica
+4. Decisão de Saneamento
+5. Sentença
+6. Recurso de Apelação
+7. Embargos de Declaração
+8. Cumprimento de Sentença
+9. Embargos à Execução
+10. Agravo de Instrumento
+
+Siga este passo a passo para criar o resumo:
+
+1. Leia atentamente todo o processo jurídico fornecido.
+<processo_juridico>
+{{PROCESSO_JURIDICO}}
+</processo_juridico>
+
+2. Identifique e anote as datas e conteúdos relevantes relacionados às 10 peças processuais listadas acima.
+
+3. Organize cronologicamente as informações coletadas.
+
+4. Destaque a última movimentação processual e seu significado para o andamento do processo.
+
+5. Resuma as principais movimentações, focando em seu impacto no processo.
+
+6. Elabore um texto coeso que apresente o fluxo do processo, destacando os pontos cruciais e as decisões mais importantes.
+
+Após criar o resumo inicial, utilize a técnica socrática de reflexão para garantir a precisão e completude do resumo. Faça a si mesmo as seguintes perguntas:
+
+1. O resumo abrange todas as 10 peças processuais principais?
+2. A última movimentação processual está claramente identificada e explicada?
+3. O texto apresenta uma visão clara do fluxo do processo?
+4. Todas as informações cruciais para o entendimento do caso estão incluídas?
+5. O resumo está livre de opiniões pessoais e se atém aos fatos do processo?
+6. A linguagem utilizada é clara e acessível, mesmo para quem não é especialista em direito?
+
+Revise e ajuste o resumo conforme necessário com base nessa reflexão.
+
+O resumo final deve ter no máximo 2 páginas de extensão (aproximadamente 1000 palavras).
+
+Formate sua resposta da seguinte maneira:
+
+<resumo_processo>
+[Insira aqui o resumo do processo jurídico]
+</resumo_processo>
+
+<reflexao_socratica>
+[Insira aqui suas respostas às perguntas da reflexão socrática]
+</reflexao_socratica>
+
+<resumo_final>
+[Insira aqui o resumo final revisado, se houver alterações após a reflexão]
+</resumo_final>"""

_utils/gerar_relatorio_modelo_usuario/utils.py

@@ -0,0 +1,22 @@
+def gerar_resposta_compilada(serializer):
+    return {
+        "num_chunks_retrieval": serializer["num_chunks_retrieval"],
+        "embedding_weight": serializer["embedding_weight"],
+        "bm25_weight": serializer["bm25_weight"],
+        "context_window": serializer["context_window"],
+        "chunk_overlap": serializer["chunk_overlap"],
+        "num_k_rerank": serializer["num_k_rerank"],
+        "model_cohere_rerank": serializer["model_cohere_rerank"],
+        "more_initial_chunks_for_reranking": serializer[
+            "more_initial_chunks_for_reranking"
+        ],
+        "claude_context_model": serializer["claude_context_model"],
+        "gpt_temperature": serializer["gpt_temperature"],
+        "user_message": serializer["user_message"],
+        "model": serializer["model"],
+        "hf_embedding": serializer["hf_embedding"],
+        "chunk_size": serializer["chunk_size"],
+        "chunk_overlap": serializer["chunk_overlap"],
+        "prompt_auxiliar": serializer["prompt_auxiliar"],
+        "prompt_gerar_documento": serializer["prompt_gerar_documento"],
+    }

_utils/prompts/Prompt_class.py

@@ -0,0 +1,12 @@
+from setup.easy_imports import ChatPromptTemplate
+
+
+class Prompt:
+    def __init__(self):
+        pass
+
+    def create_prompt_template(self, system_prompt, user_prompt):
+        prompt_template = ChatPromptTemplate.from_messages(
+            [("system", system_prompt), ("user", user_prompt)]
+        )
+        return prompt_template

_utils/resumo_completo_cursor.py

@@ -1,9 +1,13 @@
 import os
-from langchain_community.document_loaders import PyPDFLoader
-import json
+from _utils.gerar_relatorio_modelo_usuario.prompts import prompt_auxiliar_SEM_CONTEXT
 from _utils.gerar_relatorio_modelo_usuario.EnhancedDocumentSummarizer import (
     EnhancedDocumentSummarizer,
 )
+from _utils.gerar_relatorio_modelo_usuario.contextual_retriever import (
+    contextualize_chunk_based_on_serializer,
+    get_full_text_and_all_PDFs_chunks,
+)
+from _utils.gerar_relatorio_modelo_usuario.utils import gerar_resposta_compilada
 from _utils.models.gerar_relatorio import (
     RetrievalConfig,
 )
@@ -38,7 +42,6 @@ async def get_llm_summary_answer_by_cursor_complete(
     serializer, listaPDFs=None, contexto=None
 ):
     """Parâmetro "contexto" só deve ser passado quando quiser utilizar o teste com ragas, e assim, não quiser passar PDFs"""
-    allPdfsChunks = []
     # Configuration
     config = RetrievalConfig(
         num_chunks=serializer["num_chunks_retrieval"],
@@ -59,82 +62,31 @@ async def get_llm_summary_answer_by_cursor_complete(
         num_k_rerank=serializer["num_k_rerank"],
         model_cohere_rerank=serializer["model_cohere_rerank"],
         claude_context_model=serializer["claude_context_model"],
-        prompt_relatorio=serializer["prompt_relatorio"],
+        prompt_auxiliar=serializer["prompt_auxiliar"],
         gpt_model=serializer["model"],
         gpt_temperature=serializer["gpt_temperature"],
         id_modelo_do_usuario=serializer["id_modelo_do_usuario"],
-        prompt_modelo=serializer["prompt_modelo"],
+        prompt_gerar_documento=serializer["prompt_gerar_documento"],
         reciprocal_rank_fusion=reciprocal_rank_fusion,
     )
 
-    full_text = ""
-    if contexto:
-        full_text = contexto
-        chunks = summarizer.load_and_split_text(full_text)
-        allPdfsChunks = chunks
-    else:
-        # # Load and process document
-        # pdf_path = "./Im_a_storyteller.pdf"
-        # chunks = summarizer.load_and_split_document(pdf_path)
-
-        # Load and process document
-        for pdf in listaPDFs:
-            pdf_path = pdf
-            chunks = summarizer.load_and_split_document(pdf_path)
-            allPdfsChunks = allPdfsChunks + chunks
-
-        # Get full text for contextualization
-        loader = PyPDFLoader(pdf_path)
-        pages = loader.load()
-        full_text = " ".join([page.page_content for page in pages])
-    # Contextualize chunks
-    if serializer["should_have_contextual_chunks"]:
-        contextualized_chunks = (
-            await summarizer.contextual_retriever.contextualize_all_chunks(
-                pages, allPdfsChunks
-            )
+    full_text, allPdfsChunks, pages = get_full_text_and_all_PDFs_chunks(
+        contexto, listaPDFs, summarizer.splitter
+    )
+
+    chunks_passados, is_contextualized_chunk = (
+        await contextualize_chunk_based_on_serializer(
+            serializer, summarizer.contextual_retriever, pages, allPdfsChunks
         )
-        chunks_passados = contextualized_chunks
-        is_contextualized_chunk = True
-    else:
-        chunks_passados = allPdfsChunks
-        is_contextualized_chunk = False
+    )
 
     # Create enhanced vector store and BM25 index
-    vector_store, bm25, chunk_ids = summarizer.create_enhanced_vector_store(
-        chunks_passados, is_contextualized_chunk
+    vector_store, bm25, chunk_ids = (
+        summarizer.vector_store.create_enhanced_vector_store(
+            chunks_passados, is_contextualized_chunk
+        )
     )
 
-    prompt_resumo_sem_context = """You are a language model specialized in producing concise and well-structured legal case summaries in Portuguese. You will receive a variable `context`, which contains information about a legal case. Your task is to read the `context` carefully and produce a summary report in Portuguese, following the specific format provided below. Do not include any additional comments or reasoning steps in your final answer.
-**Instructions**:
-1. **Chain of Thought**: Before producing your final answer, you must think through and plan your summary silently, without showing this reasoning in the final output. The final answer must only contain the required formatted report and nothing else.
-2. **Reading the Context**: Extract the following information from `context`:
-- The name of the defendant (réu).
-- The crime they have been accused of (nome_do_crime).
-- The applicable article and subsection of the Penal Code (artigo_e_inciso_do_crime).
-- The date the accusation was accepted (data_do_recebimento).
-- The ID of the decision document (id_do_documento).
-3. **Prescriptive Details**: If no other interruptive or suspensive causes of prescription are mentioned, confirm that there are none.
-4. **Formatting**: Your final answer must strictly follow the format below, in Portuguese, and replace the placeholders with the appropriate information:
-```
-<formato>
-Trata-se de Ação Penal em que o Ministério Público denunciou [nome_do_reu], pela prática do [nome_do_crime] [artigo_e_inciso_do_crime], do Código Penal.
-A denúncia foi recebida em [data_do_recebimento], conforme Decisão [id_do_documento].
-Não há outras causas interruptivas ou suspensivas da prescrição.
-</formato>
-```
-5. **Completeness**: If any piece of required information is missing in the `context`, note that explicitly in the final answer within the format.
-**Reminder**:
-- Do not include your chain of thought in the final output.
-- Do not add extra information or commentary beyond the specified format.
-- The final answer must be in Portuguese.
-```
-<formato>
-Trata-se de Ação Penal em que o Ministério Público denunciou João da Silva, pela prática do furto qualificado (art. 155, §4º, inciso II do Código Penal).
-A denúncia foi recebida em 12/03/2021, conforme Decisão 20210312-01.
-Não há outras causas interruptivas ou suspensivas da prescrição.
-</formato>
-"""
     # Generate enhanced summary
     structured_summaries = await summarizer.generate_enhanced_summary(
         vector_store,
@@ -142,7 +94,7 @@ Não há outras causas interruptivas ou suspensivas da prescrição.
         chunk_ids
         # , serializer["user_message"]
         ,
-        prompt_resumo_sem_context,
+        prompt_auxiliar_SEM_CONTEXT,
     )
 
     if not isinstance(structured_summaries, list):
@@ -150,41 +102,15 @@ Não há outras causas interruptivas ou suspensivas da prescrição.
 
         return Response({"erro": structured_summaries})
 
-    # Output results as JSON
-    # json_output = json.dumps(structured_summaries, indent=2)
-    # print("\nStructured Summaries:")
-    # print(json_output)
-    texto_completo = ""
-    print("\n\n\n")
-    print("summarizer.resumo_gerado: ", summarizer.resumo_gerado)
-    texto_completo += summarizer.resumo_gerado
-    texto_completo += "\n\n"
-    print("\n\n\n")
-    print("structured_summaries: ", structured_summaries)
+    texto_completo = summarizer.resumo_gerado + "\n\n"
+
     for x in structured_summaries:
         texto_completo = texto_completo + x["content"] + "\n"
+
+    print("\n\ntexto_completo: ", texto_completo)
+
     return {
         "resultado": structured_summaries,
         "texto_completo": texto_completo,
-        "parametros-utilizados": {
-            "num_chunks_retrieval": serializer["num_chunks_retrieval"],
-            "embedding_weight": serializer["embedding_weight"],
-            "bm25_weight": serializer["bm25_weight"],
-            "context_window": serializer["context_window"],
-            "chunk_overlap": serializer["chunk_overlap"],
-            "num_k_rerank": serializer["num_k_rerank"],
-            "model_cohere_rerank": serializer["model_cohere_rerank"],
-            "more_initial_chunks_for_reranking": serializer[
-                "more_initial_chunks_for_reranking"
-            ],
-            "claude_context_model": serializer["claude_context_model"],
-            "gpt_temperature": serializer["gpt_temperature"],
-            "user_message": serializer["user_message"],
-            "model": serializer["model"],
-            "hf_embedding": serializer["hf_embedding"],
-            "chunk_size": serializer["chunk_size"],
-            "chunk_overlap": serializer["chunk_overlap"],
-            "prompt_relatorio": serializer["prompt_relatorio"],
-            "prompt_modelo": serializer["prompt_modelo"],
-        },
+        "parametros-utilizados": gerar_resposta_compilada(serializer),
     }

_utils/resumo_simples_cursor.py

@@ -1,221 +1,234 @@
 import os
 from typing import List, Dict, Tuple
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.document_loaders import PyPDFLoader
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import Chroma
-from langchain.chat_models import ChatOpenAI
-from langchain.chains import create_extraction_chain
-from langchain.prompts import PromptTemplate
+from setup.easy_imports import (
+    HuggingFaceEmbeddings,
+    PyPDFLoader,
+    Chroma,
+    ChatOpenAI,
+    create_extraction_chain,
+    PromptTemplate,
+    RecursiveCharacterTextSplitter,
+)
 from dataclasses import dataclass
 import uuid
 import json
 from langchain_huggingface import HuggingFaceEndpoint
 from setup.environment import default_model
 
-os.environ["LANGCHAIN_TRACING_V2"]="true"
-os.environ["LANGCHAIN_ENDPOINT"]="https://api.smith.langchain.com"
+os.environ["LANGCHAIN_TRACING_V2"] = "true"
+os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"
 os.environ.get("LANGCHAIN_API_KEY")
-os.environ["LANGCHAIN_PROJECT"]="VELLA"
+os.environ["LANGCHAIN_PROJECT"] = "VELLA"
+
 
 @dataclass
 class DocumentChunk:
-  content: str
-  page_number: int
-  chunk_id: str
-  start_char: int
-  end_char: int
+    content: str
+    page_number: int
+    chunk_id: str
+    start_char: int
+    end_char: int
+
 
 class DocumentSummarizer:
-  def __init__(self, openai_api_key: str, model, embedding, chunk_config, system_prompt):
-      self.model = model
-      self.system_prompt = system_prompt
-      self.openai_api_key = openai_api_key
-      self.embeddings = HuggingFaceEmbeddings(
-          model_name=embedding
-      )
-      self.text_splitter = RecursiveCharacterTextSplitter(
-          chunk_size=chunk_config["size"],
-          chunk_overlap=chunk_config["overlap"]
-      )
-      self.chunk_metadata = {}  # Store chunk metadata for tracing
-  
-  def load_and_split_document(self, pdf_path: str) -> List[DocumentChunk]:
-      """Load PDF and split into chunks with metadata"""
-      loader = PyPDFLoader(pdf_path)
-      pages = loader.load()
-      chunks = []
-      char_count = 0
-      
-      for page in pages:
-          text = page.page_content
-          # Split the page content
-          page_chunks = self.text_splitter.split_text(text)
-          
-          for chunk in page_chunks:
-              chunk_id = str(uuid.uuid4())
-              start_char = text.find(chunk)
-              end_char = start_char + len(chunk)
-              
-              doc_chunk = DocumentChunk(
-                  content=chunk,
-                  page_number=page.metadata.get('page') + 1,  # 1-based page numbering
-                  chunk_id=chunk_id,
-                  start_char=char_count + start_char,
-                  end_char=char_count + end_char
-              )
-              chunks.append(doc_chunk)
-              
-              # Store metadata for later retrieval
-              self.chunk_metadata[chunk_id] = {
-                  'page': doc_chunk.page_number,
-                  'start_char': doc_chunk.start_char,
-                  'end_char': doc_chunk.end_char
-              }
-          
-          char_count += len(text)
-      
-      return chunks
-
-  def create_vector_store(self, chunks: List[DocumentChunk]) -> Chroma:
-      """Create vector store with metadata"""
-      texts = [chunk.content for chunk in chunks]
-      metadatas = [{
-          'chunk_id': chunk.chunk_id,
-          'page': chunk.page_number,
-          'start_char': chunk.start_char,
-          'end_char': chunk.end_char
-      } for chunk in chunks]
-      
-      vector_store = Chroma.from_texts(
-          texts=texts,
-          metadatas=metadatas,
-          embedding=self.embeddings
-      )
-      return vector_store
-
-  def generate_summary_with_sources(
-      self, 
-      vector_store: Chroma, 
-      query: str = "Summarize the main points of this document"
-  ) -> List[Dict]:
-      """Generate summary with source citations, returning structured JSON data"""
-      # Retrieve relevant chunks with metadata
-      relevant_docs = vector_store.similarity_search_with_score(query, k=5)
-      
-      # Prepare context and track sources
-      contexts = []
-      sources = []
-      
-      for doc, score in relevant_docs:
-          chunk_id = doc.metadata['chunk_id']
-          context = doc.page_content
-          contexts.append(context)
-          
-          sources.append({
-              'content': context,
-              'page': doc.metadata['page'],
-              'chunk_id': chunk_id,
-              'relevance_score': score
-          })
-      
-      prompt = PromptTemplate(
-          template=self.system_prompt,
-          input_variables=["context"]
-      )
-      llm = ""
-
-      if (self.model == default_model):
-        llm = ChatOpenAI(
-            temperature=0,
-            model_name="gpt-4o-mini",
-            api_key=self.openai_api_key
+
+    def __init__(
+        self, openai_api_key: str, model, embedding, chunk_config, system_prompt
+    ):
+        self.model = model
+        self.system_prompt = system_prompt
+        self.openai_api_key = openai_api_key
+        self.embeddings = HuggingFaceEmbeddings(model_name=embedding)
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_config["size"], chunk_overlap=chunk_config["overlap"]
         )
-      else:
-        llm = HuggingFaceEndpoint(
-          repo_id=self.model,
-          task="text-generation",
-          max_new_tokens=1100,
-          do_sample=False,
-          huggingfacehub_api_token=os.environ.get("HUGGINGFACEHUB_API_TOKEN")
+        self.chunk_metadata = {}  # Store chunk metadata for tracing
+
+    def load_and_split_document(self, pdf_path: str) -> List[DocumentChunk]:
+        """Load PDF and split into chunks with metadata"""
+        loader = PyPDFLoader(pdf_path)
+        pages = loader.load()
+        chunks = []
+        char_count = 0
+
+        for page in pages:
+            text = page.page_content
+            # Split the page content
+            page_chunks = self.text_splitter.split_text(text)
+
+            for chunk in page_chunks:
+                chunk_id = str(uuid.uuid4())
+                start_char = text.find(chunk)
+                end_char = start_char + len(chunk)
+
+                doc_chunk = DocumentChunk(
+                    content=chunk,
+                    page_number=page.metadata.get("page") + 1,  # 1-based page numbering
+                    chunk_id=chunk_id,
+                    start_char=char_count + start_char,
+                    end_char=char_count + end_char,
+                )
+                chunks.append(doc_chunk)
+
+                # Store metadata for later retrieval
+                self.chunk_metadata[chunk_id] = {
+                    "page": doc_chunk.page_number,
+                    "start_char": doc_chunk.start_char,
+                    "end_char": doc_chunk.end_char,
+                }
+
+            char_count += len(text)
+
+        return chunks
+
+    def create_vector_store(self, chunks: List[DocumentChunk]) -> Chroma:
+        """Create vector store with metadata"""
+        texts = [chunk.content for chunk in chunks]
+        metadatas = [
+            {
+                "chunk_id": chunk.chunk_id,
+                "page": chunk.page_number,
+                "start_char": chunk.start_char,
+                "end_char": chunk.end_char,
+            }
+            for chunk in chunks
+        ]
+
+        vector_store = Chroma.from_texts(
+            texts=texts, metadatas=metadatas, embedding=self.embeddings
         )
+        return vector_store
+
+    def generate_summary_with_sources(
+        self,
+        vector_store: Chroma,
+        query: str = "Summarize the main points of this document",
+    ) -> List[Dict]:
+        """Generate summary with source citations, returning structured JSON data"""
+        # Retrieve relevant chunks with metadata
+        relevant_docs = vector_store.similarity_search_with_score(query, k=5)
+
+        # Prepare context and track sources
+        contexts = []
+        sources = []
+
+        for doc, score in relevant_docs:
+            chunk_id = doc.metadata["chunk_id"]
+            context = doc.page_content
+            contexts.append(context)
+
+            sources.append(
+                {
+                    "content": context,
+                    "page": doc.metadata["page"],
+                    "chunk_id": chunk_id,
+                    "relevance_score": score,
+                }
+            )
+
+        prompt = PromptTemplate(
+            template=self.system_prompt, input_variables=["context"]
+        )
+        llm = ""
+
+        if self.model == default_model:
+            llm = ChatOpenAI(
+                temperature=0, model_name="gpt-4o-mini", api_key=self.openai_api_key
+            )
+        else:
+            llm = HuggingFaceEndpoint(
+                repo_id=self.model,
+                task="text-generation",
+                max_new_tokens=1100,
+                do_sample=False,
+                huggingfacehub_api_token=os.environ.get("HUGGINGFACEHUB_API_TOKEN"),
+            )
+
+        response = llm.invoke(prompt.format(context="\n\n".join(contexts))).content
+
+        # Split the response into paragraphs
+        summaries = [p.strip() for p in response.split("\n\n") if p.strip()]
+
+        # Create structured output
+        structured_output = []
+        for idx, summary in enumerate(summaries):
+            # Associate each summary with the most relevant source
+            structured_output.append(
+                {
+                    "content": summary,
+                    "source": {
+                        "page": sources[min(idx, len(sources) - 1)]["page"],
+                        "text": sources[min(idx, len(sources) - 1)]["content"][:200]
+                        + "...",
+                        "relevance_score": sources[min(idx, len(sources) - 1)][
+                            "relevance_score"
+                        ],
+                    },
+                }
+            )
+
+        return structured_output
+
+    def get_source_context(self, chunk_id: str, window: int = 100) -> Dict:
+        """Get extended context around a specific chunk"""
+        metadata = self.chunk_metadata.get(chunk_id)
+        if not metadata:
+            return None
+
+        return {
+            "page": metadata["page"],
+            "start_char": metadata["start_char"],
+            "end_char": metadata["end_char"],
+        }
 
-      
-      response = llm.predict(prompt.format(context="\n\n".join(contexts)))
-      
-      # Split the response into paragraphs
-      summaries = [p.strip() for p in response.split('\n\n') if p.strip()]
-      
-      # Create structured output
-      structured_output = []
-      for idx, summary in enumerate(summaries):
-          # Associate each summary with the most relevant source
-          structured_output.append({
-              "content": summary,
-              "source": {
-                  "page": sources[min(idx, len(sources)-1)]['page'],
-                  "text": sources[min(idx, len(sources)-1)]['content'][:200] + "...",
-                  "relevance_score": sources[min(idx, len(sources)-1)]['relevance_score']
-              }
-          })
-      
-      return structured_output
-
-  def get_source_context(self, chunk_id: str, window: int = 100) -> Dict:
-      """Get extended context around a specific chunk"""
-      metadata = self.chunk_metadata.get(chunk_id)
-      if not metadata:
-          return None
-      
-      return {
-          'page': metadata['page'],
-          'start_char': metadata['start_char'],
-          'end_char': metadata['end_char']
-      }
 
 def get_llm_summary_answer_by_cursor(serializer, listaPDFs):
-  # By Luan
-  allPdfsChunks = []
-
-  # Initialize summarizer
-  summarizer = DocumentSummarizer(
-    openai_api_key=os.environ.get("OPENAI_API_KEY"),
-    embedding=serializer["hf_embedding"],
-    chunk_config={"size": serializer["chunk_size"], "overlap": serializer["chunk_overlap"]},
-    system_prompt=serializer["system_prompt"],
-    model=serializer["model"]
-  )
-  
-  # Load and process document
-  for pdf in listaPDFs:
-    pdf_path = pdf
-    chunks = summarizer.load_and_split_document(pdf_path)
-    allPdfsChunks = allPdfsChunks + chunks
-
-  vector_store = summarizer.create_vector_store(allPdfsChunks)
-  
-  # Generate structured summary
-  structured_summaries = summarizer.generate_summary_with_sources(vector_store)
-  
-  # Print or return the structured data
-  # print(structured_summaries)
-  json_data = json.dumps(structured_summaries)
-  print("\n\n")
-  print(json_data)
-  return structured_summaries
-  # If you need to send to frontend, you can just return structured_summaries
-  # It will be in the format:
-  # [
-  #     {
-  #         "content": "Summary point 1...",
-  #         "source": {
-  #             "page": 1,
-  #             "text": "Source text...",
-  #             "relevance_score": 0.95
-  #         }
-  #     },
-  #     ...
-  # ]
+    # By Luan
+    allPdfsChunks = []
+
+    # Initialize summarizer
+    summarizer = DocumentSummarizer(
+        openai_api_key=os.environ.get("OPENAI_API_KEY"),
+        embedding=serializer["hf_embedding"],
+        chunk_config={
+            "size": serializer["chunk_size"],
+            "overlap": serializer["chunk_overlap"],
+        },
+        system_prompt=serializer["system_prompt"],
+        model=serializer["model"],
+    )
+
+    # Load and process document
+    for pdf in listaPDFs:
+        pdf_path = pdf
+        chunks = summarizer.load_and_split_document(pdf_path)
+        allPdfsChunks = allPdfsChunks + chunks
+
+    vector_store = summarizer.create_vector_store(allPdfsChunks)
+
+    # Generate structured summary
+    structured_summaries = summarizer.generate_summary_with_sources(vector_store)
+
+    # Print or return the structured data
+    # print(structured_summaries)
+    json_data = json.dumps(structured_summaries)
+    print("\n\n")
+    print(json_data)
+    return structured_summaries
+    # If you need to send to frontend, you can just return structured_summaries
+    # It will be in the format:
+    # [
+    #     {
+    #         "content": "Summary point 1...",
+    #         "source": {
+    #             "page": 1,
+    #             "text": "Source text...",
+    #             "relevance_score": 0.95
+    #         }
+    #     },
+    #     ...
+    # ]
+
 
 if __name__ == "__main__":
     get_llm_summary_answer_by_cursor()

_utils/splitters/Splitter_class.py

@@ -0,0 +1,100 @@
+from setup.easy_imports import PyPDFLoader, RecursiveCharacterTextSplitter, Document
+from typing import List, Dict, Tuple, Optional
+from _utils.models.gerar_relatorio import (
+    DocumentChunk,
+)
+import uuid
+
+
+class Splitter:
+    def __init__(
+        self,
+        chunk_size,
+        chunk_overlap,
+    ):
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size, chunk_overlap=chunk_overlap
+        )
+        self.chunk_metadata = {}  # Store chunk metadata for tracing
+
+    def load_and_split_document(self, pdf_path: str) -> List[DocumentChunk]:
+        """Load PDF and split into chunks with metadata"""
+        loader = PyPDFLoader(pdf_path)
+        pages = (
+            loader.load()
+        )  # Gera uma lista de objetos Document, sendo cada item da lista referente a UMA PÁGINA inteira do PDF.
+        chunks = []
+        char_count = 0
+
+        for page in pages:
+            text = page.page_content
+            page_chunks = self.text_splitter.split_text(
+                text
+            )  # Quebra o item que é um Document de UMA PÁGINA inteira em um lista onde cada item é referente a um chunk, que são pedaços menores do que uma página.
+
+            for chunk in page_chunks:
+                chunk_id = str(uuid.uuid4())
+                start_char = text.find(
+                    chunk
+                )  # Retorna a posição onde se encontra o chunk dentro da página inteira
+                end_char = start_char + len(chunk)
+
+                doc_chunk = DocumentChunk(  # Gera o objeto do chunk com informações adicionais, como a posição e id do chunk
+                    content=chunk,
+                    page_number=page.metadata.get("page") + 1,  # 1-based page numbering
+                    chunk_id=chunk_id,
+                    start_char=char_count + start_char,
+                    end_char=char_count + end_char,
+                )
+                chunks.append(doc_chunk)
+
+                # Store metadata for later retrieval
+                self.chunk_metadata[chunk_id] = {
+                    "page": doc_chunk.page_number,
+                    "start_char": doc_chunk.start_char,
+                    "end_char": doc_chunk.end_char,
+                }
+
+            char_count += len(text)
+
+        return chunks
+
+    def load_and_split_text(self, text: str) -> List[DocumentChunk]:
+        """Load Text and split into chunks with metadata - Criei essa função apenas para o ragas"""
+        page = Document(page_content=text, metadata={"page": 1})
+        chunks = []
+        char_count = 0
+
+        text = page.page_content
+        page_chunks = self.text_splitter.split_text(
+            text
+        )  # Quebra o item que é um Document de UMA PÁGINA inteira em um lista onde cada item é referente a um chunk, que são pedaços menores do que uma página.
+        print("\n\n\n")
+        print("page_chunks: ", page_chunks)
+
+        for chunk in page_chunks:
+            chunk_id = str(uuid.uuid4())
+            start_char = text.find(
+                chunk
+            )  # Retorna a posição onde se encontra o chunk dentro da página inteira
+            end_char = start_char + len(chunk)
+
+            doc_chunk = DocumentChunk(  # Gera o objeto do chunk com informações adicionais, como a posição e id do chunk
+                content=chunk,
+                page_number=page.metadata.get("page") + 1,  # 1-based page numbering
+                chunk_id=chunk_id,
+                start_char=char_count + start_char,
+                end_char=char_count + end_char,
+            )
+            chunks.append(doc_chunk)
+
+            # Store metadata for later retrieval
+            self.chunk_metadata[chunk_id] = {
+                "page": doc_chunk.page_number,
+                "start_char": doc_chunk.start_char,
+                "end_char": doc_chunk.end_char,
+            }
+
+        char_count += len(text)
+
+        return chunks

_utils/vector_stores/Vector_store_class.py

@@ -0,0 +1,58 @@
+from typing import List, Dict, Tuple, Optional
+from _utils.models.gerar_relatorio import (
+    ContextualizedChunk,
+)
+from setup.easy_imports import Chroma, BM25Okapi, HuggingFaceEmbeddings
+import logging
+
+
+class VectorStore:
+    def __init__(self, embedding_model):
+        self.logger = logging.getLogger(__name__)
+        self.embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
+        pass
+
+    def create_enhanced_vector_store(
+        self, chunks: List[ContextualizedChunk], is_contextualized_chunk
+    ) -> Tuple[Chroma, BM25Okapi, List[str]]:
+        """Create vector store and BM25 index with contextualized chunks"""
+        try:
+            # Prepare texts with context
+            if is_contextualized_chunk:
+                texts = [f"{chunk.context} {chunk.content}" for chunk in chunks]
+            else:
+                texts = [f"{chunk.content}" for chunk in chunks]
+
+            # Create vector store
+            metadatas = []
+            for chunk in chunks:
+                if is_contextualized_chunk:
+                    context = chunk.context
+                else:
+                    context = ""
+                metadatas.append(
+                    {
+                        "chunk_id": chunk.chunk_id,
+                        "page": chunk.page_number,
+                        "start_char": chunk.start_char,
+                        "end_char": chunk.end_char,
+                        "context": context,
+                    }
+                )
+
+            vector_store = Chroma.from_texts(
+                texts=texts, metadatas=metadatas, embedding=self.embeddings
+            )
+
+            # Create BM25 index
+            tokenized_texts = [text.split() for text in texts]
+            bm25 = BM25Okapi(tokenized_texts)
+
+            # Get chunk IDs in order
+            chunk_ids = [chunk.chunk_id for chunk in chunks]
+
+            return vector_store, bm25, chunk_ids
+
+        except Exception as e:
+            self.logger.error(f"Error creating enhanced vector store: {str(e)}")
+            raise

gerar_documento/serializer.py

@@ -1,8 +1,8 @@
 from rest_framework import serializers
 from _antigos.resumos.serializer import ResumoCursorSerializer
 from _utils.gerar_relatorio_modelo_usuario.prompts import (
-    system_prompt_modelo,
-    system_prompt_relatorio,
+    prompt_gerar_documento,
+    prompt_auxiliar,
 )
 
 user_message = "What are the main points of this document?"
@@ -10,10 +10,10 @@ user_message = "What are the main points of this document?"
 
 class ResumoCursorCompeltoSerializer(ResumoCursorSerializer):
     system_prompt = None
-    prompt_relatorio = serializers.CharField(
-        required=False, default=system_prompt_relatorio
+    prompt_auxiliar = serializers.CharField(required=False, default=prompt_auxiliar)
+    prompt_gerar_documento = serializers.CharField(
+        required=False, default=prompt_gerar_documento
     )
-    prompt_modelo = serializers.CharField(required=False, default=system_prompt_modelo)
     user_message = serializers.CharField(required=False, default=user_message)
     num_chunks_retrieval = serializers.IntegerField(default=5)
     embedding_weight = serializers.FloatField(default=0.5)

gerar_documento/views.py

@@ -1,7 +1,11 @@
-from rest_framework.views import APIView
-from adrf.views import APIView as AsyncAPIView
-from rest_framework.response import Response
-
+from setup.easy_imports import (
+    Response,
+    AsyncAPIView,
+    APIView,
+    MultiPartParser,
+    extend_schema,
+)
+from datetime import datetime
 from _utils.handle_files import handle_pdf_files_from_serializer, remove_pdf_temp_files
 from _utils.resumo_completo_cursor import (
     get_llm_summary_answer_by_cursor_complete,
@@ -9,9 +13,6 @@ from _utils.resumo_completo_cursor import (
 from .serializer import (
     ResumoCursorCompeltoSerializer,
 )
-from rest_framework.parsers import MultiPartParser
-from drf_spectacular.utils import extend_schema
-from datetime import datetime
 
 
 class ResumoSimplesCursorCompletoView(AsyncAPIView):

setup/easy_imports.py

@@ -0,0 +1,22 @@
+from adrf.views import APIView as AsyncAPIView
+from drf_spectacular.utils import extend_schema
+
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from rest_framework.parsers import MultiPartParser
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+# from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain.prompts import PromptTemplate
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.vectorstores import Chroma
+
+# from langchain_community.chat_models import ChatOpenAI
+from langchain_openai import ChatOpenAI
+from langchain.schema import Document
+from langchain.chains import create_extraction_chain
+
+from rank_bm25 import BM25Okapi