File size: 2,917 Bytes
f7b5341 05bb703 f7b5341 253659d b296805 253659d a92b008 253659d 93f139a 253659d 4c69c75 253659d a109d31 253659d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
import os
import re
import pandas as pd
from pypdf import PdfReader
from typing import List, Dict
from langchain.prompts import PromptTemplate
from langchain_google_genai import GoogleGenerativeAI
api_key = "AIzaSyCYGj5e2eAQbUi9HtuMaW0LDSnDuxLG54U"
class InvoicePipeline:
def __init__(self, paths):
self._paths = paths
self._llm = GoogleGenerativeAI(model="gemini-1.0-pro", google_api_key=api_key)
self._prompt_template = self._get_default_prompt_template()
# This funcition will help in extracting and run the code, and will produce a dataframe for us
def run(self) -> pd.DataFrame:
# We have defined the way the data has to be returned
df = pd.DataFrame({
"Invoice ID": pd.Series(dtype = "int"),
"DESCRIPTION": pd.Series(dtype = "str"),
"Issue Data": pd.Series(dtype = "str"),
"UNIT PRICE": pd.Series(dtype = "str"),
"AMOUNT": pd.Series(dtype = "int"),
"Bill For": pd.Series(dtype = "str"),
"From": pd.Series(dtype ="str"),
"Terms": pd.Series(dtype = "str")}
)
for path in self._paths:
raw_text = self._get_raw_text_from_pdf(path) # This function needs to be created
llm_resp = self._extract_data_from_llm(raw_text) #
data = self._parse_response(llm_resp)
df = pd.concat([df, pd.DataFrame([data])], ignore_index = True)
return df
# The default template that the machine will take
def _get_default_prompt_template(self) -> PromptTemplate:
template = """Extract all the following values: Invoice ID, DESCRIPTION, Issue Data,UNIT PRICE, AMOUNT, Bill for, From and Terms for: {pages}
Expected Outcome: remove any dollar symbols {{"Invoice ID":"12341234", "DESCRIPTION": "UNIT PRICE", "AMOUNT": "3", "Date": "2/1/2021", "AMOUNT": "100", "Bill For": "Dev", "From": "Coca Cola", "Terms" : "Net for 30 days"}}
"""
prompt_template = PromptTemplate(input_variables = ["pages"], template = template)
return prompt_template
# We will try to extract the text from the PDF to a normal variable.
def _get_raw_text_from_pdf(self, path:str) -> str:
text = ""
pdf_reader = PdfReader(path)
for page in pdf_reader.pages:
text += page.extract_text()
return text
def _extract_data_from_llm(self, raw_data:str) -> str:
resp = self._llm(self._prompt_template.format(pages = raw_data))
return resp
def _parse_response(self, response: str) -> Dict[str, str]:
pattern = r'{(.+)}'
re_match = re.search(pattern, response, re.DOTALL)
if re_match:
extracted_text = re_match.group(1)
data = eval('{' + extracted_text + '}')
return data
else:
raise Exception("No match found.")
|