July 18 update

app.py

@@ -144,6 +144,8 @@ if 'present_annotations' not in st.session_state:
     st.session_state['present_annotations'] = None
 if 'missing_annotations' not in st.session_state:
     st.session_state['missing_annotations'] = None
+if 'model_annotations' not in st.session_state:
+    st.session_state['model_annotations'] = None
 if 'date_of_check' not in st.session_state:
     st.session_state['date_of_check'] = None
 
@@ -1016,6 +1018,16 @@ def create_private_file():
         st.write("Leave keys blank if you do not intend to use that service.")
         st.info("Note: You can manually edit these API keys later by opening the /PRIVATE_DATA.yaml file in a plain text editor.")
 
+        st.write("---")
+        st.subheader("Hugging Face  (*Required For Local LLMs*)")
+        st.markdown("VoucherVision relies on LLM models from Hugging Face. Some models are 'gated', meaning that you have to agree to the creator's usage guidelines.")
+        st.markdown("""Create a [Hugging Face account](https://huggingface.co/join). Once your account is created, in your profile settings [navigate to 'Access Tokens'](https://huggingface.co/settings/tokens) and click 'Create new token'. Create a token that has 'Read' privileges. Copy the token into the field below.""")
+
+        hugging_face_token = st.text_input(label = 'Hugging Face token', value = cfg_private['huggingface'].get('hf_token', ''),
+                                                placeholder = 'e.g. hf_GNRLIUBnvfkjvnf....',
+                                                help ="This is your Hugging Face access token. It only needs Read access. Please see https://huggingface.co/settings/tokens",
+                                                type='password')
+
         st.write("---")
         st.subheader("Google Vision  (*Required*) / Google PaLM 2 / Google Gemini")
         st.markdown("VoucherVision currently uses [Google Vision API](https://cloud.google.com/vision/docs/ocr) for OCR. Generating an API key for this is more involved than the others. [Please carefully follow the instructions outlined here to create and setup your account.](https://cloud.google.com/vision/docs/setup) ")
@@ -1170,6 +1182,7 @@ def create_private_file():
         st.button("Set API Keys",type='primary', on_click=save_changes_to_API_keys, 
                     args=[cfg_private,
                         openai_api_key,
+                        hugging_face_token,
                         azure_openai_api_version, azure_openai_api_key, azure_openai_api_base, azure_openai_organization, azure_openai_api_type,
                         google_application_credentials, google_project_location, google_project_id,
                         mistral_API_KEY, 
@@ -1183,12 +1196,15 @@ def create_private_file():
 
 def save_changes_to_API_keys(cfg_private,
                         openai_api_key,
+                        hugging_face_token,
                         azure_openai_api_version, azure_openai_api_key, azure_openai_api_base, azure_openai_organization, azure_openai_api_type,
                         google_application_credentials, google_project_location, google_project_id,
                         mistral_API_KEY, 
                         here_APP_ID, here_API_KEY): 
     
     # Update the configuration dictionary with the new values
+    cfg_private['huggingface']['hf_token'] = hugging_face_token 
+
     cfg_private['openai']['OPENAI_API_KEY'] = openai_api_key 
 
     cfg_private['openai_azure']['OPENAI_API_VERSION'] = azure_openai_api_version
@@ -1269,8 +1285,19 @@ def display_api_key_status(ccol):
             # Convert keys to annotations (similar to what you do in check_api_key_status)
             present_annotations = []
             missing_annotations = []
+            model_annotations = []
             for key in present_keys:
-                if "Valid" in key:
+                if "[MODEL]" in key:
+                    show_text = key.split(']')[1]
+                    show_text = show_text.split('(')[0]
+                    if 'Under Review' in key:
+                        model_annotations.append((show_text, "under review", "#9C0586"))  # Green for valid
+                    elif 'invalid' in key:
+                        model_annotations.append((show_text, "error!", "#870307"))  # Green for valid
+                    else:
+                        model_annotations.append((show_text, "ready!", "#059c1b"))  # Green for valid
+
+                elif "Valid" in key:
                     show_text = key.split('(')[0]
                     present_annotations.append((show_text, "ready!", "#059c1b"))  # Green for valid
                 elif "Invalid" in key:
@@ -1279,6 +1306,7 @@ def display_api_key_status(ccol):
 
             st.session_state['present_annotations'] = present_annotations
             st.session_state['missing_annotations'] = missing_annotations
+            st.session_state['model_annotations'] = model_annotations
             st.session_state['date_of_check'] = date_of_check
             st.session_state['API_checked'] = True
             # print('for')
@@ -1307,6 +1335,14 @@ def display_api_key_status(ccol):
     if 'missing_annotations' in st.session_state and st.session_state['missing_annotations']:
         annotated_text(*st.session_state['missing_annotations'])
     
+    if not st.session_state['is_hf']:
+        st.markdown(f"Access to Hugging Face Models")
+        
+        if 'model_annotations' in st.session_state and st.session_state['model_annotations']:
+            annotated_text(*st.session_state['model_annotations'])
+
+    
+    
 
 
 def check_api_key_status():
@@ -1322,8 +1358,19 @@ def check_api_key_status():
     # Prepare annotations for present keys
     present_annotations = []
     missing_annotations = []
+    model_annotations = []
     for key in present_keys:
-        if "Valid" in key:
+        if "[MODEL]" in key:
+            show_text = key.split(']')[1]
+            show_text = show_text.split('(')[0]
+            if 'Under Review' in key:
+                model_annotations.append((show_text, "under review", "#9C0586"))  # Green for valid
+            elif 'invalid' in key:
+                model_annotations.append((show_text, "error!", "#870307"))  # Green for valid
+            else:
+                model_annotations.append((show_text, "ready!", "#059c1b"))  # Green for valid
+
+        elif "Valid" in key:
             show_text = key.split('(')[0]
             present_annotations.append((show_text, "ready!", "#059c1b"))  # Green for valid
         elif "Invalid" in key:
@@ -1340,6 +1387,7 @@ def check_api_key_status():
 
     st.session_state['present_annotations'] = present_annotations
     st.session_state['missing_annotations'] = missing_annotations
+    st.session_state['model_annotations'] = model_annotations
     st.session_state['date_of_check'] = date_of_check
     
 
@@ -1831,7 +1879,7 @@ def content_ocr_method():
     demo_text_trh = demo_text_h + '\n' + demo_text_tr
     demo_text_trp = demo_text_p + '\n' + demo_text_tr
 
-    options = ["Google Vision Handwritten", "Google Vision Printed", "CRAFT + trOCR","LLaVA"]
+    options = ["Google Vision Handwritten", "Google Vision Printed", "CRAFT + trOCR","LLaVA", "Florence-2"]
     options_llava = ["llava-v1.6-mistral-7b", "llava-v1.6-34b", "llava-v1.6-vicuna-13b", "llava-v1.6-vicuna-7b",]
     options_llava_bit = ["full", "4bit",]
     captions_llava = [
@@ -1882,6 +1930,7 @@ def content_ocr_method():
             "Google Vision Printed": 'normal',
             "CRAFT + trOCR": 'CRAFT',
             "LLaVA": 'LLaVA',
+            "Florence-2": 'Florence-2',
         }
 
         # Map selected options to their corresponding internal representations
@@ -1914,6 +1963,19 @@ def content_ocr_method():
             else:
                 st.session_state.config['leafmachine']['project']['trOCR_model_path'] = user_input_trOCR_model_path
 
+
+    if "Florence-2" in selected_OCR_options:
+        default_florence_model_path = st.session_state.config['leafmachine']['project']['florence_model_path']
+        user_input_florence_model_path = st.text_input("Florence-2 Hugging Face model path. MUST be a Florence-2 version based on 'microsoft/Florence-2-large' or similar.", value=default_florence_model_path)
+
+        if st.session_state.config['leafmachine']['project']['florence_model_path'] != user_input_florence_model_path:
+            is_valid_mp = is_valid_huggingface_model_path(user_input_florence_model_path)
+            if not is_valid_mp:
+                st.error(f"The Hugging Face model path {user_input_florence_model_path} is not valid. Please revise.")
+            else:
+                st.session_state.config['leafmachine']['project']['florence_model_path'] = user_input_florence_model_path
+
+
     if 'LLaVA' in selected_OCR_options:
         OCR_option_llava = st.radio(
             "Select the LLaVA version",

vouchervision/API_validation.py

@@ -1,4 +1,5 @@
 import os, io, openai, vertexai, json, tempfile
+import webbrowser
 from mistralai.client import MistralClient
 from mistralai.models.chat_completion import ChatMessage
 from langchain.schema import HumanMessage
@@ -9,7 +10,7 @@ from google.cloud import vision
 from google.cloud import vision_v1p3beta1 as vision_beta
 # from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain_google_vertexai import VertexAI
-
+from huggingface_hub import HfApi, HfFolder
 
 from datetime import datetime
 # import google.generativeai as genai
@@ -17,6 +18,8 @@ from google.oauth2 import service_account
 # from googleapiclient.discovery import build
 
 
+
+
 class APIvalidation:
 
     def __init__(self, cfg_private, dir_home, is_hf) -> None:
@@ -25,6 +28,13 @@ class APIvalidation:
         self.is_hf = is_hf
         self.formatted_date = self.get_formatted_date()
 
+        self.HF_MODEL_LIST = ['microsoft/Florence-2-large','microsoft/Florence-2-base',
+            'microsoft/trocr-base-handwritten','microsoft/trocr-large-handwritten',
+            'google/gemma-2-9b','google/gemma-2-9b-it','google/gemma-2-27b','google/gemma-2-27b-it',
+            'mistralai/Mistral-7B-Instruct-v0.3','mistralai/Mixtral-8x22B-v0.1','mistralai/Mixtral-8x22B-Instruct-v0.1',
+            'unsloth/mistral-7b-instruct-v0.3-bnb-4bit'
+            ]
+
     def get_formatted_date(self):
         # Get the current date
         current_date = datetime.now()
@@ -59,7 +69,7 @@ class APIvalidation:
             try:
                 # Initialize the Azure OpenAI client
                 model = AzureChatOpenAI(
-                    deployment_name = 'gpt-35-turbo',#'gpt-35-turbo',
+                    deployment_name = 'gpt-4',#'gpt-35-turbo',
                     openai_api_version = self.cfg_private['openai_azure']['OPENAI_API_VERSION'],
                     openai_api_key = self.cfg_private['openai_azure']['OPENAI_API_KEY_AZURE'],
                     azure_endpoint = self.cfg_private['openai_azure']['OPENAI_API_BASE'],
@@ -67,7 +77,7 @@ class APIvalidation:
                 )
                 msg = HumanMessage(content="hello")
                 # self.llm_object.temperature = self.config.get('temperature')
-                response = model([msg])
+                response = model.invoke([msg])
 
                 # Check the response content (you might need to adjust this depending on how your AzureChatOpenAI class handles responses)
                 if response:
@@ -85,7 +95,7 @@ class APIvalidation:
                 azure_organization = os.getenv('AZURE_ORGANIZATION')
                 # Initialize the Azure OpenAI client
                 model = AzureChatOpenAI(
-                    deployment_name = 'gpt-35-turbo',#'gpt-35-turbo',
+                    deployment_name = 'gpt-4',#'gpt-35-turbo',
                     openai_api_version = azure_api_version,
                     openai_api_key = azure_api_key,
                     azure_endpoint = azure_api_base,
@@ -93,7 +103,7 @@ class APIvalidation:
                 )
                 msg = HumanMessage(content="hello")
                 # self.llm_object.temperature = self.config.get('temperature')
-                response = model([msg])
+                response = model.invoke([msg])
 
                 # Check the response content (you might need to adjust this depending on how your AzureChatOpenAI class handles responses)
                 if response:
@@ -223,8 +233,55 @@ class APIvalidation:
 
         return results
 
+    def test_hf_token(self, k_huggingface):
+        if not k_huggingface:
+            print("Hugging Face API token not found in environment variables.")
+            return False
+
+        # Create an instance of the API
+        api = HfApi()
+
+        try:
+            # Try to get details of a known public model
+            model_info = api.model_info("bert-base-uncased", use_auth_token=k_huggingface)
+            if model_info:
+                print("Token is valid. Accessed model details successfully.")
+                return True
+            else:
+                print("Token is valid but failed to access model details.")
+                return True
+        except Exception as e:
+            print(f"Failed to validate token: {e}")
+            return False
+
+    def check_gated_model_access(self, model_id, k_huggingface):
+        api = HfApi()
+        attempts = 0
+        max_attempts = 2
+
+        while attempts < max_attempts:
+            try:
+                model_info = api.model_info(model_id, use_auth_token=k_huggingface)
+                print(f"Access to model '{model_id}' is granted.")
+                return "valid"
+            except Exception as e:
+                error_message = str(e)
+                if 'awaiting a review' in error_message:
+                    print(f"Access to model '{model_id}' is awaiting review. (Under Review)")
+                    return "under_review"
+                print(f"Access to model '{model_id}' is denied. Please accept the terms and conditions.")
+                print(f"Error: {e}")
+                webbrowser.open(f"https://huggingface.co/{model_id}")
+                input("Press Enter after you have accepted the terms and conditions...")
+
+            attempts += 1
+
+        print(f"Failed to access model '{model_id}' after {max_attempts} attempts.")
+        return "invalid"
 
 
+    
+
     def get_google_credentials(self):
         if self.is_hf:
             creds_json_str = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
@@ -251,6 +308,8 @@ class APIvalidation:
             k_google_application_credentials = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
             k_project_id = os.getenv('GOOGLE_PROJECT_ID')
             k_location = os.getenv('GOOGLE_LOCATION')
+
+            k_huggingface = None
             
             k_mistral = os.getenv('MISTRAL_API_KEY')
             k_here = os.getenv('HERE_API_KEY')
@@ -259,6 +318,8 @@ class APIvalidation:
             k_OPENAI_API_KEY = self.cfg_private['openai']['OPENAI_API_KEY']
             k_openai_azure = self.cfg_private['openai_azure']['OPENAI_API_KEY_AZURE']
 
+            k_huggingface = self.cfg_private['huggingface']['hf_token']
+
             k_project_id = self.cfg_private['google']['GOOGLE_PROJECT_ID']
             k_location = self.cfg_private['google']['GOOGLE_LOCATION']
             k_google_application_credentials = self.cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS']
@@ -284,6 +345,29 @@ class APIvalidation:
                 present_keys.append('Google OCR Handwriting (Invalid)')
         else:
             missing_keys.append('Google OCR')
+
+        # present_keys.append('[MODEL] TEST (Under Review)')
+
+        # HF key check
+        if self.has_API_key(k_huggingface):
+            is_valid = self.test_hf_token(k_huggingface)
+            if is_valid:
+                present_keys.append('Hugging Face Local LLMs (Valid)')
+            else:
+                present_keys.append('Hugging Face Local LLMs (Invalid)')
+        else:
+            missing_keys.append('Hugging Face Local LLMs')
+
+        # List of gated models to check access for
+        for model_id in self.HF_MODEL_LIST:
+            access_status = self.check_gated_model_access(model_id, k_huggingface)
+            if access_status == "valid":
+                present_keys.append(f'[MODEL] {model_id} (Valid)')
+            elif access_status == "under_review":
+                present_keys.append(f'[MODEL] {model_id} (Under Review)')
+            else:
+                present_keys.append(f'[MODEL] {model_id} (Invalid)')
+        
         
         
         # OpenAI key check
@@ -297,14 +381,14 @@ class APIvalidation:
             missing_keys.append('OpenAI')
 
         # Azure OpenAI key check
-        if self.has_API_key(k_openai_azure):
-            is_valid = self.check_azure_openai_api_key()
-            if is_valid:
-                present_keys.append('Azure OpenAI (Valid)')
-            else:
-                present_keys.append('Azure OpenAI (Invalid)')
-        else:
-            missing_keys.append('Azure OpenAI')
+        # if self.has_API_key(k_openai_azure):
+        #     is_valid = self.check_azure_openai_api_key()
+        #     if is_valid:
+        #         present_keys.append('Azure OpenAI (Valid)')
+        #     else:
+        #         present_keys.append('Azure OpenAI (Invalid)')
+        # else:
+        #     missing_keys.append('Azure OpenAI')
 
         # Google PALM2/Gemini key check
         if self.has_API_key(k_google_application_credentials) and self.has_API_key(k_project_id) and self.has_API_key(k_location): ##################

vouchervision/LLM_crewAI.py

@@ -0,0 +1,176 @@
+import time, os, json
+import torch
+from crewai import Agent, Task, Crew, Process
+from langchain.prompts import PromptTemplate
+from langchain_openai import ChatOpenAI, OpenAI
+from langchain.schema import HumanMessage
+from langchain_core.output_parsers import JsonOutputParser
+from langchain.output_parsers import RetryWithErrorOutputParser
+
+class VoucherVisionWorkflow:
+    MODEL = 'gpt-4o'
+    SHARED_INSTRUCTIONS = """
+    instructions: 
+    1. Refactor the unstructured OCR text into a dictionary based on the JSON structure outlined below.
+    2. Map the unstructured OCR text to the appropriate JSON key and populate the field given the user-defined rules.
+    3. JSON key values are permitted to remain empty strings if the corresponding information is not found in the unstructured OCR text.
+    4. Duplicate dictionary fields are not allowed.
+    5. Ensure all JSON keys are in camel case.
+    6. Ensure new JSON field values follow sentence case capitalization.
+    7. Ensure all key-value pairs in the JSON dictionary strictly adhere to the format and data types specified in the template.
+    8. Ensure output JSON string is valid JSON format. It should not have trailing commas or unquoted keys.
+    9. Only return a JSON dictionary represented as a string. You should not explain your answer.
+
+    JSON structure: 
+    {"catalogNumber": "", "scientificName": "", "genus": "", "specificEpithet": "", "speciesNameAuthorship": "", "collectedBy": "", "collectorNumber": "", "identifiedBy": "", "verbatimCollectionDate": "", "collectionDate": "", "collectionDateEnd": "", "occurrenceRemarks": "", "habitat": "", "cultivated": "", "country": "", "stateProvince": "", "county": "", "locality": "", "verbatimCoordinates": "", "decimalLatitude": "", "decimalLongitude": "", "minimumElevationInMeters": "", "maximumElevationInMeters": "", "elevationUnits": ""}
+    """
+
+    EXPECTED_OUTPUT_STRUCTURE = """{
+        "JSON_OUTPUT": {
+            "catalogNumber": "", "scientificName": "", "genus": "", "specificEpithet": "", 
+            "speciesNameAuthorship": "", "collectedBy": "", "collectorNumber": "", 
+            "identifiedBy": "", "verbatimCollectionDate": "", "collectionDate": "", 
+            "collectionDateEnd": "", "occurrenceRemarks": "", "habitat": "", "cultivated": "", 
+            "country": "", "stateProvince": "", "county": "", "locality": "", 
+            "verbatimCoordinates": "", "decimalLatitude": "", "decimalLongitude": "", 
+            "minimumElevationInMeters": "", "maximumElevationInMeters": "", "elevationUnits": ""
+        },
+        "explanation": ""
+    }"""
+
+    def __init__(self, api_key, librarian_knowledge_path):
+        self.api_key = api_key
+        os.environ['OPENAI_API_KEY'] = self.api_key
+
+        self.librarian_knowledge = self.load_librarian_knowledge(librarian_knowledge_path)
+        self.worker_agent = self.create_worker_agent()
+        self.supervisor_agent = self.create_supervisor_agent()
+
+    def load_librarian_knowledge(self, path):
+        with open(path) as f:
+            return json.load(f)
+
+    def query_librarian(self, guideline_field):
+        print(f"query_librarian: {guideline_field}")
+        return self.librarian_knowledge.get(guideline_field, "Guideline not found.")
+
+    def create_worker_agent(self):
+        return Agent(
+            role="Transcriber and JSON Formatter",
+            goal="Transcribe product labels accurately and format them into a structured JSON dictionary. Only return a JSON dictionary.",
+            backstory="You're an AI trained to transcribe product labels and format them into JSON.",
+            verbose=True,
+            allow_delegation=False,
+            llm=ChatOpenAI(model=self.MODEL, openai_api_key=self.api_key),
+            prompt_instructions=self.SHARED_INSTRUCTIONS
+        )
+
+    def create_supervisor_agent(self):
+        class SupervisorAgent(Agent):
+            def correct_with_librarian(self, workflow, transcription, json_dict, guideline_field):
+                guideline = workflow.query_librarian(guideline_field)
+                corrected_transcription = self.correct(transcription, guideline)
+                corrected_json = self.correct_json(json_dict, guideline)
+                explanation = f"Corrected {json_dict} based on guideline {guideline_field}: {guideline}"
+                return corrected_transcription, {"JSON_OUTPUT": corrected_json, "explanation": explanation}
+
+        return SupervisorAgent(
+            role="Corrector",
+            goal="Ensure accurate transcriptions and JSON formatting according to specific guidelines. Compare the OCR text to the JSON dictionary and make any required corrections. Given your knowledge, make sure that the values in the JSON object make sense given the cumulative context of the OCR text. If you correct the provided JSON, then state the corrections. Otherwise say that the original worker was correct.",
+            backstory="You're an AI trained to correct transcriptions and JSON formatting, consulting the librarian for guidance.",
+            verbose=True,
+            allow_delegation=False,
+            llm=ChatOpenAI(model=self.MODEL, openai_api_key=self.api_key),
+            prompt_instructions=self.SHARED_INSTRUCTIONS
+        )
+
+    def extract_json_from_string(self, input_string):
+        json_pattern = re.compile(r'\{(?:[^{}]|(?R))*\}')
+        match = json_pattern.search(input_string)
+        if match:
+            return match.group(0)
+        return None
+
+    def extract_json_via_api(self, text):
+        self.api_key = self.api_key
+        extraction_prompt = f"I only need the JSON inside this text. Please return only the JSON object.\n\n{text}"
+        response = openai.ChatCompletion.create(
+            model=self.MODEL,
+            messages=[
+                {"role": "system", "content": extraction_prompt}
+            ]
+        )
+        return self.extract_json_from_string(response['choices'][0]['message']['content'])
+
+
+    def run_workflow(self, ocr_text):
+        openai_model = ChatOpenAI(api_key=self.api_key, model=self.MODEL)
+
+        self.worker_agent.llm = openai_model
+        self.supervisor_agent.llm = openai_model
+
+        transcription_and_formatting_task = Task(
+            description=f"Transcribe product label and format into JSON. OCR text: {ocr_text}", 
+            agent=self.worker_agent,
+            inputs={"ocr_text": ocr_text},
+            expected_output=self.EXPECTED_OUTPUT_STRUCTURE
+        )
+
+        crew = Crew(
+            agents=[self.worker_agent],
+            tasks=[transcription_and_formatting_task],
+            verbose=True,
+            process=Process.sequential,
+        )
+
+        # Run the transcription and formatting task
+        transcription_and_formatting_result = transcription_and_formatting_task.execute()
+        print("Worker Output JSON:", transcription_and_formatting_result)
+
+        # Pass the worker's JSON output to the supervisor for correction
+        correction_task = Task(
+            description=f"Correct transcription and JSON format. OCR text: {ocr_text}", 
+            agent=self.supervisor_agent,
+            inputs={"ocr_text": ocr_text, "json_dict": transcription_and_formatting_result},
+            expected_output=self.EXPECTED_OUTPUT_STRUCTURE,
+            workflow=self  # Pass the workflow instance to the task
+        )
+
+        correction_result = correction_task.execute()
+        
+        try:
+            corrected_json_with_explanation = json.loads(correction_result)
+        except json.JSONDecodeError:
+            # If initial parsing fails, make a call to OpenAI to extract only the JSON
+            corrected_json_string = self.extract_json_via_api(correction_result)
+            if not corrected_json_string:
+                raise ValueError("No JSON found in the supervisor's output.")
+            corrected_json_with_explanation = json.loads(corrected_json_string)
+
+        corrected_json = corrected_json_with_explanation["JSON_OUTPUT"]
+        explanation = corrected_json_with_explanation["explanation"]
+
+        print("Supervisor Corrected JSON:", corrected_json)
+        print("\nCorrection Explanation:", explanation)
+
+        return corrected_json, explanation
+
+if __name__ == "__main__":
+    api_key = ""
+    librarian_knowledge_path = "D:/Dropbox/VoucherVision/vouchervision/librarian_knowledge.json"
+    
+    ocr_text  = "HERBARIUM OF MARYGROVE COLLEGE Name Carex scoparia V. condensa Fernald Locality Interlaken , Ind . Date 7/20/25 No ... ! Gerould Wilhelm & Laura Rericha \" Interlaken , \" was the site for many years of St. Joseph Novitiate , run by the Brothers of the Holy Cross . The buildings were on the west shore of Silver Lake , about 2 miles NE of Rolling Prairie , LaPorte Co. Indiana , ca. 41.688 \u00b0 N , 86.601 \u00b0 W Collector : Sister M. Vincent de Paul McGivney February 1 , 2011 THE UNIVERS Examined for the Flora of the Chicago Region OF 1817 MICH ! Ciscoparia SMVdeP University of Michigan Herbarium 1386297 copyright reserved cm Collector wortet 2010"
+    workflow = VoucherVisionWorkflow(api_key, librarian_knowledge_path)
+    workflow.run_workflow(ocr_text)
+
+    ocr_text  = "CM 1 2 3 QUE : Mt.Jac.Cartier Parc de la Gasp\u00e9sie 13 Aug1988 CA Vogt on Solidago MCZ - ENT OXO Bombus vagans Smith ' det C.A. Vogt 1988 UIUC USDA BBDP 021159 00817079 "
+    workflow = VoucherVisionWorkflow(api_key, librarian_knowledge_path)
+    workflow.run_workflow(ocr_text)
+
+    ocr_text  = "500 200 600 300 dots per inch ( optical ) 700 400 800 500 850 550 850 550 Golden Thread inches centimeters 500 200 600 300 dots per inch ( optical ) 11116 L * 39.12 65.43 49.87 44.26 b * 15.07 18.72 -22.29 22.85 4 -4.34 -13.80 3 13.24 18.11 2 1 5 9 7 11 ( A ) 10 -0.40 48.55 55.56 70.82 63.51 39.92 52.24 97.06 92.02 9.82 -33.43 34.26 11.81 -24.49 -0.35 59.60 -46.07 18.51 8 6 12 13 14 15 87.34 82.14 72.06 62.15 09.0- -0.75 -1.06 -1.19 -1.07 1.13 0.23 0.21 0.43 0.28 0.19 800 500 D50 Illuminant , 2 degree observer Density 0.04 0.09 0.15 0.22 Fam . Saurauiaceae J. G. Agardh Saurauia nepaulensis DC . S. Vietnam , Prov . Kontum . NW slopes of Ngoc Linh mountain system at 1200 m alt . near Ngoc Linh village . Secondary marshland with grasses and shrubs . Tree up to 5 m high . Flowers light rosy - pink . No VH 007 0.36 0.51 23.02.1995 International Botanical Expedition of the U.S.A. National Geographic Society ( grant No 5094-93 ) Participants : L. Averyanov , N.T. Ban , N. Q. Binh , A. Budantzev , L. Budantzev , N.T. Hiep , D.D. Huyen , P.K. Loc , N.X. Tam , G. Yakovlev BOTANICAL RESEARCH INSTITUTE OF TEXAS BRIT610199 Botanical Research Institute of Texas IMAGED 08 JUN 2021 FLORA OF VIETNAM "
+    workflow = VoucherVisionWorkflow(api_key, librarian_knowledge_path)
+    workflow.run_workflow(ocr_text)
+
+    ocr_text  = "Russian - Vietnamese Tropical Centre Styrax argentifolius H.L. Li SOUTHERN VIETNAM Dak Lak prov . , Lak distr . , Bong Krang municip . Chu Yang Sin National Park 10 km S from Krong Kmar village River bank N 12 \u00b0 25 ' 24 \" E 108 \u00b0 21 ' 04 \" elev . 900 m Nuraliev M.S. No 1004 part of MW 0750340 29.05.2014 Materials of complex expedition in spring 2014 BOTANICAL RESEARCH INSTITUTE OF TEXAS ( BRIT ) Styrax benzoides Craib Det . by Peter W. Fritsch , September 2017 0 1 2 3 4 5 6 7 8 9 10 BOTANICAL RESEARCH INSTITUTE OF TEXAS BOTANICAL IMAGED RESEARCH INSTITUTE OF 10 JAN 2013 BRIT402114 copyright reserved cm BOTANICAL RESEARCH INSTITUTE OF TEXAS TM P CameraTrax.com BRIT . TEXAS "
+    workflow = VoucherVisionWorkflow(api_key, librarian_knowledge_path)
+    workflow.run_workflow(ocr_text)

vouchervision/LLM_local_MistralAI.py

@@ -1,11 +1,13 @@
-import json, torch, transformers, gc
+import json, os
+import torch
+import transformers
+import gc
 from transformers import BitsAndBytesConfig
-from langchain.output_parsers import RetryWithErrorOutputParser
+from langchain.output_parsers.retry import RetryOutputParser
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
 from huggingface_hub import hf_hub_download
-from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
-
+from langchain_huggingface import HuggingFacePipeline
 from vouchervision.utils_LLM import SystemLoadMonitor, run_tools, count_tokens, save_individual_prompt, sanitize_prompt
 from vouchervision.utils_LLM_JSON_validation import validate_and_align_JSON_keys_with_template
 
@@ -14,7 +16,7 @@ Local Pipielines:
 https://python.langchain.com/docs/integrations/llms/huggingface_pipelines
 '''
 
-class LocalMistralHandler: 
+class LocalMistralHandler:
     RETRY_DELAY = 2  # Wait 2 seconds before retrying
     MAX_RETRIES = 5  # Maximum number of retries
     STARTING_TEMP = 0.1
@@ -27,29 +29,22 @@ class LocalMistralHandler:
         self.tool_WFO = self.cfg['leafmachine']['project']['tool_WFO']
         self.tool_GEO = self.cfg['leafmachine']['project']['tool_GEO']
         self.tool_wikipedia = self.cfg['leafmachine']['project']['tool_wikipedia']
-
+        
         self.logger = logger
         self.has_GPU = torch.cuda.is_available()
         self.monitor = SystemLoadMonitor(logger)
 
         self.model_name = model_name
         self.model_id = f"mistralai/{self.model_name}"
-        name_parts = self.model_name.split('-')
-
-        self.model_path = hf_hub_download(repo_id=self.model_id, repo_type="model",filename="config.json")
-
+        self.model_path = hf_hub_download(repo_id=self.model_id, repo_type="model", filename="config.json", use_auth_token=os.getenv("HUGGING_FACE_KEY"))
 
         self.JSON_dict_structure = JSON_dict_structure
         self.starting_temp = float(self.STARTING_TEMP)
         self.temp_increment = float(0.2)
-        self.adjust_temp = self.starting_temp 
-
-        system_prompt = "You are a helpful AI assistant who answers queries a JSON dictionary as specified by the user."
-        template = """
-            <s>[INST]{}[/INST]</s>
+        self.adjust_temp = self.starting_temp
 
-            [INST]{}[/INST]
-            """.format(system_prompt, "{query}")
+        system_prompt = "You are a helpful AI assistant who answers queries by returning a JSON dictionary as specified by the user."
+        template = "<s>[INST]{}[/INST]</s>[INST]{}[/INST]".format(system_prompt, "{query}")
 
         # Create a prompt from the template so we can use it with Langchain
         self.prompt = PromptTemplate(template=template, input_variables=["query"])
@@ -59,45 +54,22 @@ class LocalMistralHandler:
 
         self._set_config()
 
-
-    # def _clear_VRAM(self):
-    #     # Clear CUDA cache if it's being used
-    #     if self.has_GPU:
-    #         self.local_model = None
-    #         self.local_model_pipeline = None
-    #         del self.local_model
-    #         del self.local_model_pipeline
-    #         gc.collect()  # Explicitly invoke garbage collector
-    #         torch.cuda.empty_cache()
-    #     else:
-    #         self.local_model_pipeline = None
-    #         self.local_model = None
-    #         del self.local_model_pipeline
-    #         del self.local_model
-    #         gc.collect()  # Explicitly invoke garbage collector
-
-
     def _set_config(self):
-        # self._clear_VRAM()
-        self.config = {'max_new_tokens': 1024,
-                'temperature': self.starting_temp,
-                'seed': 2023,
-                'top_p': 1,
-                'top_k': 40,
-                'do_sample': True,
-                'n_ctx':4096,
-
-                # Activate 4-bit precision base model loading
-                'use_4bit': True,
-                # Compute dtype for 4-bit base models
-                'bnb_4bit_compute_dtype': "float16",
-                # Quantization type (fp4 or nf4)
-                'bnb_4bit_quant_type': "nf4",
-                # Activate nested quantization for 4-bit base models (double quantization)
-                'use_nested_quant': False,
-                }
-        
-        compute_dtype = getattr(torch,self.config.get('bnb_4bit_compute_dtype') )
+        self.config = {
+            'max_new_tokens': 1024,
+            'temperature': self.starting_temp,
+            'seed': 2023,
+            'top_p': 1,
+            'top_k': 40,
+            'do_sample': True,
+            'n_ctx': 4096,
+            'use_4bit': True,
+            'bnb_4bit_compute_dtype': "float16",
+            'bnb_4bit_quant_type': "nf4",
+            'use_nested_quant': False,
+        }
+
+        compute_dtype = getattr(torch, self.config.get('bnb_4bit_compute_dtype'))
 
         self.bnb_config = BitsAndBytesConfig(
             load_in_4bit=self.config.get('use_4bit'),
@@ -106,123 +78,102 @@ class LocalMistralHandler:
             bnb_4bit_use_double_quant=self.config.get('use_nested_quant'),
         )
 
-        # Check GPU compatibility with bfloat16
         if compute_dtype == torch.float16 and self.config.get('use_4bit'):
             major, _ = torch.cuda.get_device_capability()
-            if major >= 8:
-                # print("=" * 80)
-                # print("Your GPU supports bfloat16: accelerate training with bf16=True")
-                # print("=" * 80)
-                self.b_float_opt =  torch.bfloat16
-
-            else:
-                self.b_float_opt =  torch.float16
+            self.b_float_opt = torch.bfloat16 if major >= 8 else torch.float16
+
         self._build_model_chain_parser()
-    
 
     def _adjust_config(self):
         new_temp = self.adjust_temp + self.temp_increment
-        if self.json_report:
-            self.json_report.set_text(text_main=f'Incrementing temperature from {self.adjust_temp} to {new_temp}')
         self.logger.info(f'Incrementing temperature from {self.adjust_temp} to {new_temp}')
         self.adjust_temp += self.temp_increment
 
-    
     def _reset_config(self):
-        if self.json_report:
-            self.json_report.set_text(text_main=f'Resetting temperature from {self.adjust_temp} to {self.starting_temp}')
         self.logger.info(f'Resetting temperature from {self.adjust_temp} to {self.starting_temp}')
         self.adjust_temp = self.starting_temp
-    
 
     def _build_model_chain_parser(self):
-        self.local_model_pipeline = transformers.pipeline("text-generation", 
-                    model=self.model_id,
-                    max_new_tokens=self.config.get('max_new_tokens'),
-                    top_k=self.config.get('top_k'),
-                    top_p=self.config.get('top_p'),
-                    do_sample=self.config.get('do_sample'),
-                    model_kwargs={"torch_dtype": self.b_float_opt, 
-                                "load_in_4bit": True, 
-                                "quantization_config": self.bnb_config})
+        self.local_model_pipeline = transformers.pipeline(
+            "text-generation",
+            model=self.model_id,
+            max_new_tokens=self.config.get('max_new_tokens'),
+            top_k=self.config.get('top_k'),
+            top_p=self.config.get('top_p'),
+            do_sample=self.config.get('do_sample'),
+            model_kwargs={"torch_dtype": self.b_float_opt, "quantization_config": self.bnb_config},
+        )
         self.local_model = HuggingFacePipeline(pipeline=self.local_model_pipeline)
+
         # Set up the retry parser with the runnable
-        self.retry_parser = RetryWithErrorOutputParser.from_llm(parser=self.parser, llm=self.local_model, max_retries=self.MAX_RETRIES)
+        self.retry_parser = RetryOutputParser(parser=self.parser, llm=self.local_model, max_retries=self.MAX_RETRIES)
         # Create an llm chain with LLM and prompt
-        self.chain = self.prompt | self.local_model  # LCEL
-
+        self.chain = self.prompt | self.local_model
 
     def call_llm_local_MistralAI(self, prompt_template, json_report, paths):
-        _____, ____, _, __, ___, json_file_path_wiki, txt_file_path_ind_prompt = paths
+        json_file_path_wiki, txt_file_path_ind_prompt = paths[-2:]
         self.json_report = json_report
         if self.json_report:
             self.json_report.set_text(text_main=f'Sending request to {self.model_name}')
         self.monitor.start_monitoring_usage()
-        
+
         nt_in = 0
         nt_out = 0
 
-        ind = 0
-        while ind < self.MAX_RETRIES:
-            ind += 1
+        for ind in range(self.MAX_RETRIES):
             try:
-                # Dynamically set the temperature for this specific request
                 model_kwargs = {"temperature": self.adjust_temp}
-                
-                # Invoke the chain to generate prompt text
                 results = self.chain.invoke({"query": prompt_template, "model_kwargs": model_kwargs})
 
-                # Use retry_parser to parse the response with retry logic
                 output = self.retry_parser.parse_with_prompt(results, prompt_value=prompt_template)
 
                 if output is None:
                     self.logger.error(f'Failed to extract JSON from:\n{results}')
                     self._adjust_config()
                     del results
-
                 else:
                     nt_in = count_tokens(prompt_template, self.VENDOR, self.TOKENIZER_NAME)
                     nt_out = count_tokens(results, self.VENDOR, self.TOKENIZER_NAME)
 
                     output = validate_and_align_JSON_keys_with_template(output, self.JSON_dict_structure)
-                    
+
                     if output is None:
-                        self.logger.error(f'[Attempt {ind}] Failed to extract JSON from:\n{results}')
+                        self.logger.error(f'[Attempt {ind + 1}] Failed to extract JSON from:\n{results}')
                         self._adjust_config()
                     else:
-                        self.monitor.stop_inference_timer() # Starts tool timer too
-                        
+                        self.monitor.stop_inference_timer()  # Starts tool timer too
+
                         if self.json_report:
                             self.json_report.set_text(text_main=f'Working on WFO, Geolocation, Links')
-                        output_WFO, WFO_record, output_GEO, GEO_record = run_tools(output, self.tool_WFO, self.tool_GEO, self.tool_wikipedia, json_file_path_wiki)
+                        output_WFO, WFO_record, output_GEO, GEO_record = run_tools(
+                            output, self.tool_WFO, self.tool_GEO, self.tool_wikipedia, json_file_path_wiki
+                        )
 
                         save_individual_prompt(sanitize_prompt(prompt_template), txt_file_path_ind_prompt)
 
-                        self.logger.info(f"Formatted JSON:\n{json.dumps(output,indent=4)}")
+                        self.logger.info(f"Formatted JSON:\n{json.dumps(output, indent=4)}")
 
-                        usage_report = self.monitor.stop_monitoring_report_usage()    
+                        usage_report = self.monitor.stop_monitoring_report_usage()
 
-                        if self.adjust_temp != self.starting_temp:            
+                        if self.adjust_temp != self.starting_temp:
                             self._reset_config()
 
                         if self.json_report:
                             self.json_report.set_text(text_main=f'LLM call successful')
                         del results
                         return output, nt_in, nt_out, WFO_record, GEO_record, usage_report
-
             except Exception as e:
                 self.logger.error(f'{e}')
-                self._adjust_config()           
-                
-        self.logger.info(f"Failed to extract valid JSON after [{ind}] attempts")
+                self._adjust_config()
+
+        self.logger.info(f"Failed to extract valid JSON after [{self.MAX_RETRIES}] attempts")
         if self.json_report:
-            self.json_report.set_text(text_main=f'Failed to extract valid JSON after [{ind}] attempts')
+            self.json_report.set_text(text_main=f'Failed to extract valid JSON after [{self.MAX_RETRIES}] attempts')
 
-        self.monitor.stop_inference_timer() # Starts tool timer too
-        usage_report = self.monitor.stop_monitoring_report_usage()                
+        self.monitor.stop_inference_timer()  # Starts tool timer too
+        usage_report = self.monitor.stop_monitoring_report_usage()
         if self.json_report:
             self.json_report.set_text(text_main=f'LLM call failed')
 
         self._reset_config()
-        return None, nt_in, nt_out, None, None, usage_report
-
+        return None, nt_in, nt_out, None, None, usage_report

vouchervision/LLM_local_custom_fine_tune.py

@@ -0,0 +1,358 @@
+import os, re, json, yaml, torch
+from peft import AutoPeftModelForCausalLM
+from transformers import AutoTokenizer
+
+import json, torch, transformers, gc
+from transformers import BitsAndBytesConfig
+from langchain.output_parsers.retry import RetryOutputParser
+from langchain.prompts import PromptTemplate
+from langchain_core.output_parsers import JsonOutputParser
+from huggingface_hub import hf_hub_download
+from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
+
+from vouchervision.utils_LLM import SystemLoadMonitor, run_tools, count_tokens, save_individual_prompt, sanitize_prompt
+from vouchervision.utils_LLM_JSON_validation import validate_and_align_JSON_keys_with_template
+
+# MODEL_NAME = "unsloth/mistral-7b-instruct-v0.2-bnb-4bit"
+# sltp_version = 'HLT_MICH_Angiospermae_SLTPvA_v1-0_medium__OCR-C25-L25-E50-R05'
+# LORA = "phyloforfun/mistral-7b-instruct-v2-bnb-4bit__HLT_MICH_Angiospermae_SLTPvC_v1-0_medium_OCR-C25-L25-E50-R05"
+
+TEXT = "HERBARIUM OF MARCUS W. LYON , JR . Tracaulon sagittatum Indiana : Porter Co. Mincral Springs edge wet subdural woods 1927 TX 11 Flowers pink UNIVERSIT HERBARIUM MICHIGAN MICH University of Michigan Herbarium 1439649 copyright reserved PERSICARIA FEB 26 1965 cm "
+PARENT_MODEL = "unsloth/mistral-7b-instruct-v0.2-bnb-4bit"
+
+class LocalFineTuneHandler: 
+    RETRY_DELAY = 2  # Wait 2 seconds before retrying
+    MAX_RETRIES = 5  # Maximum number of retries
+    STARTING_TEMP = 0.001
+    TOKENIZER_NAME = None
+    VENDOR = 'mistral'
+    MAX_GPU_MONITORING_INTERVAL = 2  # seconds
+
+    
+
+    def __init__(self, cfg, logger, model_name, JSON_dict_structure, config_vals_for_permutation=None):
+        # self.model_id = f"phyloforfun/{self.model_name}"
+        # model_name = LORA #######################################################
+        
+        # self.JSON_dict_structure = JSON_dict_structure
+        # self.JSON_dict_structure_str = json.dumps(self.JSON_dict_structure, sort_keys=False, indent=4)
+        
+        self.JSON_dict_structure_str = """{"catalogNumber": "", "scientificName": "", "genus": "", "specificEpithet": "", "scientificNameAuthorship": "", "collector": "", "recordNumber": "", "identifiedBy": "", "verbatimCollectionDate": "", "collectionDate": "", "occurrenceRemarks": "", "habitat": "", "locality": "", "country": "", "stateProvince": "", "county": "", "municipality": "", "verbatimCoordinates": "", "decimalLatitude": "", "decimalLongitude": "",  "minimumElevationInMeters": "", "maximumElevationInMeters": ""}"""
+
+
+        self.cfg = cfg
+        self.print_output = True
+        self.tool_WFO = self.cfg['leafmachine']['project']['tool_WFO']
+        self.tool_GEO = self.cfg['leafmachine']['project']['tool_GEO']
+        self.tool_wikipedia = self.cfg['leafmachine']['project']['tool_wikipedia']
+
+        self.logger = logger
+
+        self.has_GPU = torch.cuda.is_available()
+        if self.has_GPU:
+            self.device = "cuda"
+        else:
+            self.device = "cpu"
+            
+        self.monitor = SystemLoadMonitor(logger)
+
+        self.model_name = model_name.split("/")[1]
+        self.model_id = model_name
+
+        # self.model_path = hf_hub_download(repo_id=self.model_id, repo_type="model",filename="config.json")
+
+        
+        self.starting_temp = float(self.STARTING_TEMP)
+        self.temp_increment = float(0.2)
+        self.adjust_temp = self.starting_temp 
+
+        self.load_in_4bit = False
+
+        self.parser = JsonOutputParser()
+
+        self._load_model()
+        self._create_prompt()
+        self._set_config()
+        self._build_model_chain_parser()
+
+    def _set_config(self):
+        # self._clear_VRAM()
+        self.config = {'max_new_tokens': 1024,
+                'temperature': self.starting_temp,
+                'seed': 2023,
+                'top_p': 1,
+                # 'top_k': 1,
+                # 'top_k': 40,
+                'do_sample': False,
+                'n_ctx':4096,
+
+                # Activate 4-bit precision base model loading
+                # 'use_4bit': True,
+                # # Compute dtype for 4-bit base models
+                # 'bnb_4bit_compute_dtype': "float16",
+                # # Quantization type (fp4 or nf4)
+                # 'bnb_4bit_quant_type': "nf4",
+                # # Activate nested quantization for 4-bit base models (double quantization)
+                # 'use_nested_quant': False,
+                }
+        
+    def _adjust_config(self):
+        new_temp = self.adjust_temp + self.temp_increment
+        if self.json_report:
+            self.json_report.set_text(text_main=f'Incrementing temperature from {self.adjust_temp} to {new_temp}')
+        self.logger.info(f'Incrementing temperature from {self.adjust_temp} to {new_temp}')
+        self.adjust_temp += self.temp_increment
+
+    
+    def _reset_config(self):
+        if self.json_report:
+            self.json_report.set_text(text_main=f'Resetting temperature from {self.adjust_temp} to {self.starting_temp}')
+        self.logger.info(f'Resetting temperature from {self.adjust_temp} to {self.starting_temp}')
+        self.adjust_temp = self.starting_temp
+
+
+    def _load_model(self):
+        self.model = AutoPeftModelForCausalLM.from_pretrained(
+            pretrained_model_name_or_path=self.model_id, # YOUR MODEL YOU USED FOR TRAINING
+            load_in_4bit = self.load_in_4bit,
+            low_cpu_mem_usage=True,
+            
+        ).to(self.device)
+
+        self.tokenizer = AutoTokenizer.from_pretrained(PARENT_MODEL)
+        self.eos_token_id = self.tokenizer.eos_token_id
+
+
+    # def _build_model_chain_parser(self):
+    #     self.local_model_pipeline = transformers.pipeline("text-generation", 
+    #                 model=self.model_id,
+    #                 max_new_tokens=self.config.get('max_new_tokens'),
+    #                 # top_k=self.config.get('top_k'),
+    #                 top_p=self.config.get('top_p'),
+    #                 do_sample=self.config.get('do_sample'),
+    #                 model_kwargs={"load_in_4bit": self.load_in_4bit})
+    #     self.local_model = HuggingFacePipeline(pipeline=self.local_model_pipeline)
+    #     # Set up the retry parser with the runnable
+    #     # self.retry_parser = RetryWithErrorOutputParser.from_llm(parser=self.parser, llm=self.local_model, max_retries=self.MAX_RETRIES)
+    #     self.retry_parser = RetryOutputParser(parser=self.parser, llm=self.local_model, max_retries=self.MAX_RETRIES)
+
+    #     # Create an llm chain with LLM and prompt
+    #     self.chain = self.prompt | self.local_model  # LCEL
+    def _build_model_chain_parser(self):
+        self.local_model_pipeline = transformers.pipeline(
+            "text-generation", 
+            model=self.model_id,
+            max_new_tokens=self.config.get('max_new_tokens'),
+            top_k=self.config.get('top_k', None),
+            top_p=self.config.get('top_p'),
+            do_sample=self.config.get('do_sample'),
+            model_kwargs={"load_in_4bit": self.load_in_4bit},
+        )
+        self.local_model = HuggingFacePipeline(pipeline=self.local_model_pipeline)
+        self.retry_parser = RetryOutputParser(parser=self.parser, llm=self.local_model, max_retries=self.MAX_RETRIES)
+
+
+
+    def _create_prompt(self):
+        self.alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+### Instruction:
+{}
+
+### Input:
+{}
+
+### Response:
+{}"""   
+        
+        self.template = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+### Instruction:
+{}
+
+### Input:
+{}
+
+### Response:
+{}""".format("{instructions}", "{OCR_text}", "{empty}")
+
+        self.instructions_text = """Refactor the unstructured text into a valid JSON dictionary. The key names follow the Darwin Core Archive Standard. If a key lacks content, then insert an empty string. Fill in the following JSON structure as required: """
+        self.instructions_json = self.JSON_dict_structure_str.replace("\n    ", " ").strip().replace("\n", " ")
+        self.instructions = ''.join([self.instructions_text, self.instructions_json])
+
+
+        # Create a prompt from the template so we can use it with Langchain
+        self.prompt = PromptTemplate(template=self.template, input_variables=["instructions", "OCR_text", "empty"])
+
+        # Set up a parser
+        self.parser = JsonOutputParser()
+    
+
+    def extract_json(self, response_text):
+        # Assuming the response is a list with a single string entry
+        # response_text = response[0]
+
+        response_pattern = re.compile(r'### Response:(.*)', re.DOTALL)
+        response_match = response_pattern.search(response_text)
+        if not response_match:
+            raise ValueError("No '### Response:' section found in the provided text")
+        
+        response_text = response_match.group(1)
+        
+        # Use a regular expression to find JSON objects in the response text
+        json_objects = re.findall(r'\{.*?\}', response_text, re.DOTALL)
+        
+        if json_objects:
+            # Assuming you want the first JSON object if there are multiple
+            json_str = json_objects[0]
+            # Convert the JSON string to a Python dictionary
+            json_dict = json.loads(json_str)
+            return json_str, json_dict
+        else:
+            raise ValueError("No JSON object found in the '### Response:' section")
+
+
+    def call_llm_local_custom_fine_tune(self, OCR_text, json_report, paths):
+        _____, ____, _, __, ___, json_file_path_wiki, txt_file_path_ind_prompt = paths
+        self.json_report = json_report
+        if self.json_report:
+            self.json_report.set_text(text_main=f'Sending request to {self.model_name}')
+        self.monitor.start_monitoring_usage()
+        
+        nt_in = 0
+        nt_out = 0
+
+        self.inputs = self.tokenizer(
+        [
+            self.alpaca_prompt.format(
+                self.instructions, # instruction
+                OCR_text, # input
+                "", # output - leave this blank for generation!
+            )
+        ], return_tensors = "pt").to(self.device)
+
+        ind = 0
+        while ind < self.MAX_RETRIES:
+            ind += 1
+            try:
+                # Fancy
+                # Dynamically set the temperature for this specific request
+                model_kwargs = {"temperature": self.adjust_temp}
+                
+                # Invoke the chain to generate prompt text
+                # results = self.chain.invoke({"instructions": self.instructions, "OCR_text": OCR_text, "empty": "", "model_kwargs": model_kwargs})
+
+                # Use retry_parser to parse the response with retry logic
+                # output = self.retry_parser.parse_with_prompt(results, prompt_value=OCR_text)
+                results = self.local_model.invoke(OCR_text)
+                output = self.retry_parser.parse_with_prompt(results, prompt_value=OCR_text)
+
+
+                # Should work:
+                # output = self.model.generate(**self.inputs, eos_token_id=self.eos_token_id, max_new_tokens=512)  # Adjust max_length as needed
+
+                # Decode the generated text
+                # generated_text = self.tokenizer.decode(output[0], skip_special_tokens=True)
+
+                # json_str, json_dict = self.extract_json(generated_text)
+                if self.print_output:
+                    # print("\nJSON String:")
+                    # print(json_str)
+                    print("\nJSON Dictionary:")
+                    print(output)
+
+
+                
+                if output is None:
+                    self.logger.error(f'Failed to extract JSON from:\n{results}')
+                    self._adjust_config()
+                    del results
+
+                else:
+                    nt_in = count_tokens(self.instructions+OCR_text, self.VENDOR, self.TOKENIZER_NAME)
+                    nt_out = count_tokens(results, self.VENDOR, self.TOKENIZER_NAME)
+
+                    output = validate_and_align_JSON_keys_with_template(output, json.loads(self.JSON_dict_structure_str))
+                    
+                    if output is None:
+                        self.logger.error(f'[Attempt {ind}] Failed to extract JSON from:\n{results}')
+                        self._adjust_config()
+                    else:
+                        self.monitor.stop_inference_timer() # Starts tool timer too
+                        
+                        if self.json_report:
+                            self.json_report.set_text(text_main=f'Working on WFO, Geolocation, Links')
+                        output_WFO, WFO_record, output_GEO, GEO_record = run_tools(output, self.tool_WFO, self.tool_GEO, self.tool_wikipedia, json_file_path_wiki)
+
+                        save_individual_prompt(sanitize_prompt(self.instructions+OCR_text), txt_file_path_ind_prompt)
+
+                        self.logger.info(f"Formatted JSON:\n{json.dumps(output,indent=4)}")
+
+                        usage_report = self.monitor.stop_monitoring_report_usage()    
+
+                        if self.adjust_temp != self.starting_temp:            
+                            self._reset_config()
+
+                        if self.json_report:
+                            self.json_report.set_text(text_main=f'LLM call successful')
+                        del results
+                        return output, nt_in, nt_out, WFO_record, GEO_record, usage_report
+
+            except Exception as e:
+                self.logger.error(f'{e}')
+
+
+        self.logger.info(f"Failed to extract valid JSON after [{ind}] attempts")
+        if self.json_report:
+            self.json_report.set_text(text_main=f'Failed to extract valid JSON after [{ind}] attempts')
+
+        self.monitor.stop_inference_timer() # Starts tool timer too
+        usage_report = self.monitor.stop_monitoring_report_usage()                
+        if self.json_report:
+            self.json_report.set_text(text_main=f'LLM call failed')
+
+        return None, nt_in, nt_out, None, None, usage_report
+
+
+
+        # # Create a prompt from the template so we can use it with Langchain
+        # self.prompt = PromptTemplate(template=template, input_variables=["query"])
+
+        # # Set up a parser
+        # self.parser = JsonOutputParser()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+model_name = "unsloth/mistral-7b-instruct-v0.2-bnb-4bit"
+sltp_version = 'HLT_MICH_Angiospermae_SLTPvA_v1-0_medium__OCR-C25-L25-E50-R05'
+lora_name = "phyloforfun/mistral-7b-instruct-v2-bnb-4bit__HLT_MICH_Angiospermae_SLTPvA_v1-0_medium__OCR-C25-L25-E50-R05"
+
+OCR_test = "HERBARIUM OF MARCUS W. LYON , JR . Tracaulon sagittatum Indiana : Porter Co. Mincral Springs edge wet subdural woods 1927 TX 11 Flowers pink UNIVERSIT HERBARIUM MICHIGAN MICH University of Michigan Herbarium 1439649 copyright reserved PERSICARIA FEB 26 1965 cm "
+
+
+
+
+
+# model.merge_and_unload()
+
+
+
+# Generate the output
+

vouchervision/OCR_Florence_2.py

@@ -0,0 +1,88 @@
+import random, os
+from PIL import Image
+import copy
+import matplotlib.pyplot as plt  
+import matplotlib.patches as patches  
+from PIL import Image, ImageDraw, ImageFont 
+import numpy as np
+import warnings
+from transformers import AutoProcessor, AutoModelForCausalLM, AutoTokenizer
+from vouchervision.utils_LLM import SystemLoadMonitor
+
+warnings.filterwarnings("ignore", category=UserWarning, message="TypedStorage is deprecated")
+
+class FlorenceOCR:
+    def __init__(self, logger, model_id='microsoft/Florence-2-large'):
+        self.MAX_TOKENS = 1024
+        self.logger = logger
+        self.model_id = model_id
+
+        self.monitor = SystemLoadMonitor(logger)
+
+        self.model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True).eval().cuda()
+        self.processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+
+        # self.model_id_clean = "mistralai/Mistral-7B-v0.3"
+        self.model_id_clean = "unsloth/mistral-7b-instruct-v0.3-bnb-4bit"
+        self.tokenizer_clean = AutoTokenizer.from_pretrained(self.model_id_clean)
+        self.model_clean = AutoModelForCausalLM.from_pretrained(self.model_id_clean)
+        
+
+    def ocr_florence(self, image, task_prompt='<OCR>', text_input=None):
+        self.monitor.start_monitoring_usage()
+
+        # Open image if a path is provided
+        if isinstance(image, str):
+            image = Image.open(image)
+
+        if text_input is None:
+            prompt = task_prompt
+        else:
+            prompt = task_prompt + text_input
+
+        inputs = self.processor(text=prompt, images=image, return_tensors="pt")
+
+        # Move input_ids and pixel_values to the same device as the model
+        inputs = {key: value.to(self.model.device) for key, value in inputs.items()}
+
+        generated_ids = self.model.generate(
+            input_ids=inputs["input_ids"],
+            pixel_values=inputs["pixel_values"],
+            max_new_tokens=self.MAX_TOKENS,
+            early_stopping=False,
+            do_sample=False,
+            num_beams=3,
+        )
+        generated_text = self.processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+        parsed_answer_dirty = self.processor.post_process_generation(
+            generated_text, 
+            task=task_prompt, 
+            image_size=(image.width, image.height)
+        )
+
+        inputs = self.tokenizer_clean(f"Insert spaces into this text to make all the words valid. This text contains scientific names of plants, locations, habitat, coordinate words: {parsed_answer_dirty[task_prompt]}", return_tensors="pt")
+        inputs = {key: value.to(self.model_clean.device) for key, value in inputs.items()}
+
+        outputs = self.model_clean.generate(**inputs, max_new_tokens=self.MAX_TOKENS)
+        parsed_answer = self.tokenizer_clean.decode(outputs[0], skip_special_tokens=True)
+        print(parsed_answer_dirty)
+        print(parsed_answer)
+
+        self.monitor.stop_inference_timer() # Starts tool timer too
+        usage_report = self.monitor.stop_monitoring_report_usage()   
+
+        return parsed_answer, parsed_answer_dirty[task_prompt], parsed_answer_dirty, usage_report
+
+
+def main():
+    img_path = '/home/brlab/Downloads/gem_2024_06_26__02-26-02/Cropped_Images/By_Class/label/1.jpg'
+    # img = 'D:/D_Desktop/BR_1839468565_Ochnaceae_Campylospermum_reticulatum_label.jpg'
+
+    image = Image.open(img_path)
+
+    ocr = FlorenceOCR(logger = None)
+    results_text, results, usage_report = ocr.ocr_florence(image, task_prompt='<OCR>', text_input=None)
+    print(results_text)
+
+if __name__ == '__main__':
+    main()

vouchervision/OCR_google_cloud_vision (DESKTOP-548UDCR's conflicted copy 2024-06-15).py

@@ -0,0 +1,850 @@
+import os, io, sys, inspect, statistics, json, cv2
+from statistics import mean 
+# from google.cloud import vision, storage
+from google.cloud import vision
+from google.cloud import vision_v1p3beta1 as vision_beta
+from PIL import Image, ImageDraw, ImageFont
+import colorsys
+from tqdm import tqdm
+from google.oauth2 import service_account
+
+### LLaVA should only be installed if the user will actually use it.
+### It requires the most recent pytorch/Python and can mess with older systems
+
+
+'''
+@misc{li2021trocr,
+      title={TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models}, 
+      author={Minghao Li and Tengchao Lv and Lei Cui and Yijuan Lu and Dinei Florencio and Cha Zhang and Zhoujun Li and Furu Wei},
+      year={2021},
+      eprint={2109.10282},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@inproceedings{baek2019character,
+  title={Character Region Awareness for Text Detection},
+  author={Baek, Youngmin and Lee, Bado and Han, Dongyoon and Yun, Sangdoo and Lee, Hwalsuk},
+  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={9365--9374},
+  year={2019}
+}
+'''
+
+class OCREngine:
+
+    BBOX_COLOR = "black"
+
+    def __init__(self, logger, json_report, dir_home, is_hf, path, cfg, trOCR_model_version, trOCR_model, trOCR_processor, device):
+        self.is_hf = is_hf
+        self.logger = logger
+
+        self.json_report = json_report
+
+        self.path = path
+        self.cfg = cfg
+        self.do_use_trOCR = self.cfg['leafmachine']['project']['do_use_trOCR']
+        self.OCR_option = self.cfg['leafmachine']['project']['OCR_option']
+        self.double_OCR = self.cfg['leafmachine']['project']['double_OCR']
+        self.dir_home = dir_home
+
+        # Initialize TrOCR components
+        self.trOCR_model_version = trOCR_model_version
+        self.trOCR_processor = trOCR_processor
+        self.trOCR_model = trOCR_model
+        self.device = device
+
+        self.hand_cleaned_text = None
+        self.hand_organized_text = None
+        self.hand_bounds = None
+        self.hand_bounds_word = None
+        self.hand_bounds_flat = None
+        self.hand_text_to_box_mapping = None
+        self.hand_height = None
+        self.hand_confidences = None
+        self.hand_characters = None
+
+        self.normal_cleaned_text = None
+        self.normal_organized_text = None
+        self.normal_bounds = None
+        self.normal_bounds_word = None
+        self.normal_text_to_box_mapping = None
+        self.normal_bounds_flat = None
+        self.normal_height = None
+        self.normal_confidences = None
+        self.normal_characters = None
+
+        self.trOCR_texts = None
+        self.trOCR_text_to_box_mapping = None
+        self.trOCR_bounds_flat = None
+        self.trOCR_height = None
+        self.trOCR_confidences = None
+        self.trOCR_characters = None
+        self.set_client()
+        self.init_craft()
+
+        self.multimodal_prompt = """I need you to transcribe all of the text in this image. 
+        Place the transcribed text into a JSON dictionary with this form {"Transcription_Printed_Text": "text","Transcription_Handwritten_Text": "text"}"""
+        self.init_llava()
+
+        
+    def set_client(self):
+        if self.is_hf:
+            self.client_beta = vision_beta.ImageAnnotatorClient(credentials=self.get_google_credentials())
+            self.client = vision.ImageAnnotatorClient(credentials=self.get_google_credentials())
+        else:
+            self.client_beta = vision_beta.ImageAnnotatorClient(credentials=self.get_google_credentials()) 
+            self.client = vision.ImageAnnotatorClient(credentials=self.get_google_credentials())
+
+
+    def get_google_credentials(self):
+        creds_json_str = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+        credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
+        return credentials
+    
+    def init_craft(self):
+        if 'CRAFT' in self.OCR_option:
+            from craft_text_detector import load_craftnet_model, load_refinenet_model
+
+            try:
+                self.refine_net = load_refinenet_model(cuda=True)
+                self.use_cuda = True
+            except:
+                self.refine_net = load_refinenet_model(cuda=False)
+                self.use_cuda = False
+
+            if self.use_cuda:
+                self.craft_net = load_craftnet_model(weight_path=os.path.join(self.dir_home,'vouchervision','craft','craft_mlt_25k.pth'), cuda=True)
+            else:
+                self.craft_net = load_craftnet_model(weight_path=os.path.join(self.dir_home,'vouchervision','craft','craft_mlt_25k.pth'), cuda=False)
+
+    def init_llava(self):
+        if 'LLaVA' in self.OCR_option:
+            from vouchervision.OCR_llava import OCRllava
+
+            self.model_path = "liuhaotian/" + self.cfg['leafmachine']['project']['OCR_option_llava']
+            self.model_quant = self.cfg['leafmachine']['project']['OCR_option_llava_bit']
+            
+            if self.json_report:
+                self.json_report.set_text(text_main=f'Loading LLaVA model: {self.model_path} Quantization: {self.model_quant}')
+
+            if self.model_quant == '4bit':
+                use_4bit = True
+            elif self.model_quant == 'full':
+                use_4bit = False
+            else:
+                self.logger.info(f"Provided model quantization invlid. Using 4bit.")
+                use_4bit = True
+
+            self.Llava = OCRllava(self.logger, model_path=self.model_path, load_in_4bit=use_4bit, load_in_8bit=False)
+
+    def init_gemini_vision(self):
+        pass
+
+    def init_gpt4_vision(self):
+        pass
+            
+
+    def detect_text_craft(self):
+        from craft_text_detector import read_image, get_prediction
+
+        # Perform prediction using CRAFT
+        image = read_image(self.path)
+
+        link_threshold = 0.85
+        text_threshold = 0.4
+        low_text = 0.4
+
+        if self.use_cuda:
+            self.prediction_result = get_prediction(
+                image=image,
+                craft_net=self.craft_net,
+                refine_net=self.refine_net,
+                text_threshold=text_threshold,
+                link_threshold=link_threshold,
+                low_text=low_text,
+                cuda=True,
+                long_size=1280
+            )
+        else:
+            self.prediction_result = get_prediction(
+                image=image,
+                craft_net=self.craft_net,
+                refine_net=self.refine_net,
+                text_threshold=text_threshold,
+                link_threshold=link_threshold,
+                low_text=low_text,
+                cuda=False,
+                long_size=1280
+            )
+
+        # Initialize metadata structures
+        bounds = []
+        bounds_word = []  # CRAFT gives bounds for text regions, not individual words
+        text_to_box_mapping = []
+        bounds_flat = []
+        height_flat = []
+        confidences = []  # CRAFT does not provide confidences per character, so this might be uniformly set or estimated
+        characters = []  # Simulating as CRAFT doesn't provide character-level details
+        organized_text = ""
+        
+        total_b = len(self.prediction_result["boxes"])
+        i=0
+        # Process each detected text region
+        for box in self.prediction_result["boxes"]:
+            i+=1
+            if self.json_report:
+                self.json_report.set_text(text_main=f'Locating text using CRAFT --- {i}/{total_b}')
+
+            vertices = [{"x": int(vertex[0]), "y": int(vertex[1])} for vertex in box]
+            
+            # Simulate a mapping for the whole detected region as a word
+            text_to_box_mapping.append({
+                "vertices": vertices,
+                "text": "detected_text"  # Placeholder, as CRAFT does not provide the text content directly
+            })
+
+            # Assuming each box is a word for the sake of this example
+            bounds_word.append({"vertices": vertices})
+
+            # For simplicity, we're not dividing text regions into characters as CRAFT doesn't provide this
+            # Instead, we create a single large 'character' per detected region
+            bounds.append({"vertices": vertices})
+            
+            # Simulate flat bounds and height for each detected region
+            x_positions = [vertex["x"] for vertex in vertices]
+            y_positions = [vertex["y"] for vertex in vertices]
+            min_x, max_x = min(x_positions), max(x_positions)
+            min_y, max_y = min(y_positions), max(y_positions)
+            avg_height = max_y - min_y
+            height_flat.append(avg_height)
+            
+            # Assuming uniform confidence for all detected regions
+            confidences.append(1.0)  # Placeholder confidence
+            
+            # Adding dummy character for each box
+            characters.append("X")  # Placeholder character
+            
+            # Organize text as a single string (assuming each box is a word)
+            # organized_text += "detected_text "  # Placeholder text    
+
+        # Update class attributes with processed data
+        self.normal_bounds = bounds
+        self.normal_bounds_word = bounds_word
+        self.normal_text_to_box_mapping = text_to_box_mapping
+        self.normal_bounds_flat = bounds_flat  # This would be similar to bounds if not processing characters individually
+        self.normal_height = height_flat
+        self.normal_confidences = confidences
+        self.normal_characters = characters
+        self.normal_organized_text = organized_text.strip() 
+    
+
+    def detect_text_with_trOCR_using_google_bboxes(self, do_use_trOCR, logger):
+        CONFIDENCES = 0.80
+        MAX_NEW_TOKENS = 50
+
+        self.OCR_JSON_to_file = {}
+        
+        ocr_parts = ''
+        if not do_use_trOCR:
+            if 'normal' in self.OCR_option:
+                self.OCR_JSON_to_file['OCR_printed'] = self.normal_organized_text
+                # logger.info(f"Google_OCR_Standard:\n{self.normal_organized_text}")
+                # ocr_parts = ocr_parts + f"Google_OCR_Standard:\n{self.normal_organized_text}"
+                ocr_parts = self.normal_organized_text
+            
+            if 'hand' in self.OCR_option:
+                self.OCR_JSON_to_file['OCR_handwritten'] = self.hand_organized_text
+                # logger.info(f"Google_OCR_Handwriting:\n{self.hand_organized_text}")
+                # ocr_parts = ocr_parts +  f"Google_OCR_Handwriting:\n{self.hand_organized_text}"
+                ocr_parts = self.hand_organized_text
+
+            # if self.OCR_option in ['both',]:
+            #     logger.info(f"Google_OCR_Standard:\n{self.normal_organized_text}\n\nGoogle_OCR_Handwriting:\n{self.hand_organized_text}")
+            #     return f"Google_OCR_Standard:\n{self.normal_organized_text}\n\nGoogle_OCR_Handwriting:\n{self.hand_organized_text}"
+            return ocr_parts
+        else:
+            logger.info(f'Supplementing with trOCR')
+
+            self.trOCR_texts = []
+            original_image = Image.open(self.path).convert("RGB")
+
+            if 'normal' in self.OCR_option or 'CRAFT' in self.OCR_option:
+                available_bounds = self.normal_bounds_word
+            elif 'hand' in self.OCR_option:
+                available_bounds = self.hand_bounds_word 
+            # elif self.OCR_option in ['both',]:
+            #     available_bounds = self.hand_bounds_word 
+            else:
+                raise
+
+            text_to_box_mapping = []
+            characters = []
+            height = []
+            confidences = []
+            total_b = len(available_bounds)
+            i=0
+            for bound in tqdm(available_bounds, desc="Processing words using Google Vision bboxes"):
+                i+=1
+                if self.json_report:
+                    self.json_report.set_text(text_main=f'Working on trOCR :construction: {i}/{total_b}')
+
+                vertices = bound["vertices"]
+
+                left = min([v["x"] for v in vertices])
+                top = min([v["y"] for v in vertices])
+                right = max([v["x"] for v in vertices])
+                bottom = max([v["y"] for v in vertices])
+
+                # Crop image based on Google's bounding box
+                cropped_image = original_image.crop((left, top, right, bottom))
+                pixel_values = self.trOCR_processor(cropped_image, return_tensors="pt").pixel_values
+
+                # Move pixel values to the appropriate device
+                pixel_values = pixel_values.to(self.device)
+
+                generated_ids = self.trOCR_model.generate(pixel_values, max_new_tokens=MAX_NEW_TOKENS)
+                extracted_text = self.trOCR_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+                self.trOCR_texts.append(extracted_text)
+
+                # For plotting 
+                word_length = max(vertex.get('x') for vertex in vertices) - min(vertex.get('x') for vertex in vertices)
+                num_symbols = len(extracted_text)
+
+                Yw = max(vertex.get('y') for vertex in vertices)
+                Yo = Yw - min(vertex.get('y') for vertex in vertices)
+                X = word_length / num_symbols if num_symbols > 0 else 0
+                H = int(X+(Yo*0.1))
+                height.append(H)
+
+                map_dict = {
+                    "vertices": vertices,
+                    "text": extracted_text  # Use the text extracted by trOCR
+                    }
+                text_to_box_mapping.append(map_dict)
+
+                characters.append(extracted_text)
+                confidences.append(CONFIDENCES)
+            
+            median_height = statistics.median(height) if height else 0
+            median_heights = [median_height * 1.5] * len(characters)
+
+            self.trOCR_texts = ' '.join(self.trOCR_texts)
+
+            self.trOCR_text_to_box_mapping = text_to_box_mapping
+            self.trOCR_bounds_flat = available_bounds
+            self.trOCR_height = median_heights
+            self.trOCR_confidences = confidences
+            self.trOCR_characters = characters
+
+            if 'normal' in self.OCR_option:
+                self.OCR_JSON_to_file['OCR_printed'] = self.normal_organized_text
+                self.OCR_JSON_to_file['OCR_trOCR'] = self.trOCR_texts
+                # logger.info(f"Google_OCR_Standard:\n{self.normal_organized_text}\n\ntrOCR:\n{self.trOCR_texts}")
+                # ocr_parts = ocr_parts +  f"\nGoogle_OCR_Standard:\n{self.normal_organized_text}\n\ntrOCR:\n{self.trOCR_texts}"
+                ocr_parts = self.trOCR_texts
+            if 'hand' in self.OCR_option:
+                self.OCR_JSON_to_file['OCR_handwritten'] = self.hand_organized_text
+                self.OCR_JSON_to_file['OCR_trOCR'] = self.trOCR_texts
+                # logger.info(f"Google_OCR_Handwriting:\n{self.hand_organized_text}\n\ntrOCR:\n{self.trOCR_texts}")
+                # ocr_parts = ocr_parts +  f"\nGoogle_OCR_Handwriting:\n{self.hand_organized_text}\n\ntrOCR:\n{self.trOCR_texts}"
+                ocr_parts = self.trOCR_texts
+            # if self.OCR_option in ['both',]:
+            #     self.OCR_JSON_to_file['OCR_printed'] = self.normal_organized_text
+            #     self.OCR_JSON_to_file['OCR_handwritten'] = self.hand_organized_text
+            #     self.OCR_JSON_to_file['OCR_trOCR'] = self.trOCR_texts
+            #     logger.info(f"Google_OCR_Standard:\n{self.normal_organized_text}\n\nGoogle_OCR_Handwriting:\n{self.hand_organized_text}\n\ntrOCR:\n{self.trOCR_texts}")
+            #     ocr_parts = ocr_parts +  f"\nGoogle_OCR_Standard:\n{self.normal_organized_text}\n\nGoogle_OCR_Handwriting:\n{self.hand_organized_text}\n\ntrOCR:\n{self.trOCR_texts}"
+            if 'CRAFT' in self.OCR_option:
+                # self.OCR_JSON_to_file['OCR_printed'] = self.normal_organized_text
+                self.OCR_JSON_to_file['OCR_CRAFT_trOCR'] = self.trOCR_texts
+                # logger.info(f"CRAFT_trOCR:\n{self.trOCR_texts}")
+                # ocr_parts = ocr_parts +  f"\nCRAFT_trOCR:\n{self.trOCR_texts}"
+                ocr_parts = self.trOCR_texts
+            return ocr_parts
+
+    @staticmethod
+    def confidence_to_color(confidence):
+        hue = (confidence - 0.5) * 120 / 0.5
+        r, g, b = colorsys.hls_to_rgb(hue/360, 0.5, 1)
+        return (int(r*255), int(g*255), int(b*255))
+
+
+    def render_text_on_black_image(self, option):
+        bounds_flat = getattr(self, f'{option}_bounds_flat', [])
+        heights = getattr(self, f'{option}_height', [])
+        confidences = getattr(self, f'{option}_confidences', [])
+        characters = getattr(self, f'{option}_characters', [])
+
+        original_image = Image.open(self.path)
+        width, height = original_image.size
+        black_image = Image.new("RGB", (width, height), "black")
+        draw = ImageDraw.Draw(black_image)
+
+        for bound, confidence, char_height, character in zip(bounds_flat, confidences, heights, characters):
+            font_size = int(char_height)
+            try:
+                font = ImageFont.truetype("arial.ttf", font_size)
+            except:
+                font = ImageFont.load_default().font_variant(size=font_size)
+            if option == 'trOCR':
+                color = (0, 170, 255)
+            else:
+                color = OCREngine.confidence_to_color(confidence)
+            position = (bound["vertices"][0]["x"], bound["vertices"][0]["y"] - char_height)
+            draw.text(position, character, fill=color, font=font)
+
+        return black_image
+
+
+    def merge_images(self, image1, image2):
+        width1, height1 = image1.size
+        width2, height2 = image2.size
+        merged_image = Image.new("RGB", (width1 + width2, max([height1, height2])))
+        merged_image.paste(image1, (0, 0))
+        merged_image.paste(image2, (width1, 0))
+        return merged_image
+
+
+    def draw_boxes(self, option):
+        bounds = getattr(self, f'{option}_bounds', [])
+        bounds_word = getattr(self, f'{option}_bounds_word', [])
+        confidences = getattr(self, f'{option}_confidences', [])
+
+        draw = ImageDraw.Draw(self.image)
+        width, height = self.image.size
+        if min([width, height]) > 4000:
+            line_width_thick = int((width + height) / 2 * 0.0025)  # Adjust line width for character level
+            line_width_thin = 1
+        else:
+            line_width_thick = int((width + height) / 2 * 0.005)  # Adjust line width for character level
+            line_width_thin = 1 #int((width + height) / 2 * 0.001)
+
+        for bound in bounds_word:
+            draw.polygon(
+                [
+                    bound["vertices"][0]["x"], bound["vertices"][0]["y"],
+                    bound["vertices"][1]["x"], bound["vertices"][1]["y"],
+                    bound["vertices"][2]["x"], bound["vertices"][2]["y"],
+                    bound["vertices"][3]["x"], bound["vertices"][3]["y"],
+                ],
+                outline=OCREngine.BBOX_COLOR,
+                width=line_width_thin
+            )
+
+        # Draw a line segment at the bottom of each handwritten character
+        for bound, confidence in zip(bounds, confidences):  
+            color = OCREngine.confidence_to_color(confidence)
+            # Use the bottom two vertices of the bounding box for the line
+            bottom_left = (bound["vertices"][3]["x"], bound["vertices"][3]["y"] + line_width_thick)
+            bottom_right = (bound["vertices"][2]["x"], bound["vertices"][2]["y"] + line_width_thick)
+            draw.line([bottom_left, bottom_right], fill=color, width=line_width_thick)
+
+        return self.image
+
+
+    def detect_text(self):
+        
+        with io.open(self.path, 'rb') as image_file:
+            content = image_file.read()
+        image = vision.Image(content=content)
+        response = self.client.document_text_detection(image=image)
+        texts = response.text_annotations
+
+        if response.error.message:
+            raise Exception(
+                '{}\nFor more info on error messages, check: '
+                'https://cloud.google.com/apis/design/errors'.format(
+                    response.error.message))
+
+        bounds = []
+        bounds_word = []
+        text_to_box_mapping = []
+        bounds_flat = []
+        height_flat = []
+        confidences = []
+        characters = []
+        organized_text = ""
+        paragraph_count = 0
+        
+        for text in texts[1:]:
+            vertices = [{"x": vertex.x, "y": vertex.y} for vertex in text.bounding_poly.vertices]
+            map_dict = {
+                "vertices": vertices,
+                "text": text.description
+            }
+            text_to_box_mapping.append(map_dict)
+
+        for page in response.full_text_annotation.pages:
+            for block in page.blocks:
+                # paragraph_count += 1
+                # organized_text += f'OCR_paragraph_{paragraph_count}:\n'  # Add paragraph label
+                for paragraph in block.paragraphs:
+
+                    avg_H_list = []
+                    for word in paragraph.words:
+                        Yw = max(vertex.y for vertex in word.bounding_box.vertices)
+                        # Calculate the width of the word and divide by the number of symbols
+                        word_length = max(vertex.x for vertex in word.bounding_box.vertices) - min(vertex.x for vertex in word.bounding_box.vertices)
+                        num_symbols = len(word.symbols)
+                        if num_symbols <= 3:
+                            H = int(Yw - min(vertex.y for vertex in word.bounding_box.vertices))
+                        else:
+                            Yo = Yw - min(vertex.y for vertex in word.bounding_box.vertices)
+                            X = word_length / num_symbols if num_symbols > 0 else 0
+                            H = int(X+(Yo*0.1))
+                        avg_H_list.append(H)
+                    avg_H = int(mean(avg_H_list))
+
+                    words_in_para = []
+                    for word in paragraph.words:
+                        # Get word-level bounding box
+                        bound_word_dict = {
+                            "vertices": [
+                                {"x": vertex.x, "y": vertex.y} for vertex in word.bounding_box.vertices
+                            ]
+                        }
+                        bounds_word.append(bound_word_dict)
+                        
+                        Y = max(vertex.y for vertex in word.bounding_box.vertices)
+                        word_x_start = min(vertex.x for vertex in word.bounding_box.vertices)
+                        word_x_end = max(vertex.x for vertex in word.bounding_box.vertices)
+                        num_symbols = len(word.symbols)
+                        symbol_width = (word_x_end - word_x_start) / num_symbols if num_symbols > 0 else 0
+
+                        current_x_position = word_x_start
+
+                        characters_ind = []
+                        for symbol in word.symbols:
+                            bound_dict = {
+                                "vertices": [
+                                    {"x": vertex.x, "y": vertex.y} for vertex in symbol.bounding_box.vertices
+                                ]
+                            }
+                            bounds.append(bound_dict)
+
+                            # Create flat bounds with adjusted x position
+                            bounds_flat_dict = {
+                                "vertices": [
+                                    {"x": current_x_position, "y": Y}, 
+                                    {"x": current_x_position + symbol_width, "y": Y}
+                                ]
+                            }
+                            bounds_flat.append(bounds_flat_dict)
+                            current_x_position += symbol_width
+
+                            height_flat.append(avg_H)
+                            confidences.append(round(symbol.confidence, 4))
+
+                            characters_ind.append(symbol.text)
+                            characters.append(symbol.text)
+
+                        words_in_para.append(''.join(characters_ind))
+                    paragraph_text = ' '.join(words_in_para)  # Join words in paragraph
+                    organized_text += paragraph_text + ' ' #+ '\n' 
+
+        # median_height = statistics.median(height_flat) if height_flat else 0
+        # median_heights = [median_height] * len(characters)
+
+        self.normal_cleaned_text = texts[0].description if texts else ''
+        self.normal_organized_text = organized_text
+        self.normal_bounds = bounds
+        self.normal_bounds_word = bounds_word
+        self.normal_text_to_box_mapping = text_to_box_mapping
+        self.normal_bounds_flat = bounds_flat
+        # self.normal_height = median_heights #height_flat
+        self.normal_height = height_flat
+        self.normal_confidences = confidences
+        self.normal_characters = characters
+        return self.normal_cleaned_text
+
+
+    def detect_handwritten_ocr(self):
+        
+        with open(self.path, "rb") as image_file:
+            content = image_file.read()
+
+        image = vision_beta.Image(content=content)
+        image_context = vision_beta.ImageContext(language_hints=["en-t-i0-handwrit"])
+        response = self.client_beta.document_text_detection(image=image, image_context=image_context)
+        texts = response.text_annotations
+        
+        if response.error.message:
+            raise Exception(
+                "{}\nFor more info on error messages, check: "
+                "https://cloud.google.com/apis/design/errors".format(response.error.message)
+            )
+
+        bounds = []
+        bounds_word = []
+        bounds_flat = []
+        height_flat = []
+        confidences = []
+        characters = []
+        organized_text = ""
+        paragraph_count = 0
+        text_to_box_mapping = []
+
+        for text in texts[1:]:
+            vertices = [{"x": vertex.x, "y": vertex.y} for vertex in text.bounding_poly.vertices]
+            map_dict = {
+                "vertices": vertices,
+                "text": text.description
+            }
+            text_to_box_mapping.append(map_dict)
+
+        for page in response.full_text_annotation.pages:
+            for block in page.blocks:
+                # paragraph_count += 1
+                # organized_text += f'\nOCR_paragraph_{paragraph_count}:\n'  # Add paragraph label
+                for paragraph in block.paragraphs:
+                    
+                    avg_H_list = []
+                    for word in paragraph.words:
+                        Yw = max(vertex.y for vertex in word.bounding_box.vertices)
+                        # Calculate the width of the word and divide by the number of symbols
+                        word_length = max(vertex.x for vertex in word.bounding_box.vertices) - min(vertex.x for vertex in word.bounding_box.vertices)
+                        num_symbols = len(word.symbols)
+                        if num_symbols <= 3:
+                            H = int(Yw - min(vertex.y for vertex in word.bounding_box.vertices))
+                        else:
+                            Yo = Yw - min(vertex.y for vertex in word.bounding_box.vertices)
+                            X = word_length / num_symbols if num_symbols > 0 else 0
+                            H = int(X+(Yo*0.1))
+                        avg_H_list.append(H)
+                    avg_H = int(mean(avg_H_list))
+
+                    words_in_para = []
+                    for word in paragraph.words:
+                        # Get word-level bounding box
+                        bound_word_dict = {
+                            "vertices": [
+                                {"x": vertex.x, "y": vertex.y} for vertex in word.bounding_box.vertices
+                            ]
+                        }
+                        bounds_word.append(bound_word_dict)
+
+                        Y = max(vertex.y for vertex in word.bounding_box.vertices)
+                        word_x_start = min(vertex.x for vertex in word.bounding_box.vertices)
+                        word_x_end = max(vertex.x for vertex in word.bounding_box.vertices)
+                        num_symbols = len(word.symbols)
+                        symbol_width = (word_x_end - word_x_start) / num_symbols if num_symbols > 0 else 0
+
+                        current_x_position = word_x_start
+
+                        characters_ind = []
+                        for symbol in word.symbols:
+                            bound_dict = {
+                                "vertices": [
+                                    {"x": vertex.x, "y": vertex.y} for vertex in symbol.bounding_box.vertices
+                                ]
+                            }
+                            bounds.append(bound_dict)
+
+                            # Create flat bounds with adjusted x position
+                            bounds_flat_dict = {
+                                "vertices": [
+                                    {"x": current_x_position, "y": Y}, 
+                                    {"x": current_x_position + symbol_width, "y": Y}
+                                ]
+                            }
+                            bounds_flat.append(bounds_flat_dict)
+                            current_x_position += symbol_width
+
+                            height_flat.append(avg_H)
+                            confidences.append(round(symbol.confidence, 4))
+
+                            characters_ind.append(symbol.text)
+                            characters.append(symbol.text)
+
+                        words_in_para.append(''.join(characters_ind))
+                    paragraph_text = ' '.join(words_in_para)  # Join words in paragraph
+                    organized_text += paragraph_text + ' ' #+ '\n' 
+
+        # median_height = statistics.median(height_flat) if height_flat else 0
+        # median_heights = [median_height] * len(characters)
+
+        self.hand_cleaned_text = response.text_annotations[0].description if response.text_annotations else ''
+        self.hand_organized_text = organized_text
+        self.hand_bounds = bounds
+        self.hand_bounds_word = bounds_word
+        self.hand_bounds_flat = bounds_flat
+        self.hand_text_to_box_mapping = text_to_box_mapping
+        # self.hand_height = median_heights #height_flat
+        self.hand_height = height_flat
+        self.hand_confidences = confidences
+        self.hand_characters = characters
+        return self.hand_cleaned_text
+
+
+    def process_image(self, do_create_OCR_helper_image, logger):
+        # Can stack options, so solitary if statements
+        self.OCR = 'OCR:\n'
+        if 'CRAFT' in self.OCR_option:
+            self.do_use_trOCR = True
+            self.detect_text_craft()
+            ### Optionally add trOCR to the self.OCR for additional context
+            if self.double_OCR:
+                part_OCR = "\CRAFT trOCR:\n" + self.detect_text_with_trOCR_using_google_bboxes(self.do_use_trOCR, logger)
+                self.OCR = self.OCR + part_OCR + part_OCR
+            else:
+                self.OCR = self.OCR + "\CRAFT trOCR:\n" + self.detect_text_with_trOCR_using_google_bboxes(self.do_use_trOCR, logger)
+            # logger.info(f"CRAFT trOCR:\n{self.OCR}")
+
+        if 'LLaVA' in self.OCR_option: # This option does not produce an OCR helper image
+            if self.json_report:
+                self.json_report.set_text(text_main=f'Working on LLaVA {self.Llava.model_path} transcription :construction:')
+
+            image, json_output, direct_output, str_output, usage_report = self.Llava.transcribe_image(self.path, self.multimodal_prompt)
+            self.logger.info(f"LLaVA Usage Report for Model {self.Llava.model_path}:\n{usage_report}")
+
+            try:
+                self.OCR_JSON_to_file['OCR_LLaVA'] = str_output
+            except:
+                self.OCR_JSON_to_file = {}
+                self.OCR_JSON_to_file['OCR_LLaVA'] = str_output
+
+            if self.double_OCR:
+                self.OCR = self.OCR + f"\nLLaVA OCR:\n{str_output}" + f"\nLLaVA OCR:\n{str_output}"
+            else:
+                self.OCR = self.OCR + f"\nLLaVA OCR:\n{str_output}"
+            # logger.info(f"LLaVA OCR:\n{self.OCR}")
+
+        if 'normal' in self.OCR_option or 'hand' in self.OCR_option:
+            if 'normal' in self.OCR_option:
+                if self.double_OCR:
+                    part_OCR = self.OCR + "\nGoogle Printed OCR:\n" + self.detect_text()
+                    self.OCR = self.OCR + part_OCR + part_OCR
+                else:
+                    self.OCR = self.OCR + "\nGoogle Printed OCR:\n" + self.detect_text()
+            if 'hand' in self.OCR_option:
+                if self.double_OCR:
+                    part_OCR = self.OCR + "\nGoogle Handwritten OCR:\n" + self.detect_handwritten_ocr()
+                    self.OCR = self.OCR + part_OCR + part_OCR
+                else:
+                    self.OCR = self.OCR + "\nGoogle Handwritten OCR:\n" + self.detect_handwritten_ocr()
+            # if self.OCR_option not in ['normal', 'hand', 'both']:
+            #     self.OCR_option = 'both'
+            #     self.detect_text()
+            #     self.detect_handwritten_ocr()
+
+            ### Optionally add trOCR to the self.OCR for additional context
+            if self.do_use_trOCR:
+                if self.double_OCR:
+                    part_OCR = "\ntrOCR:\n" + self.detect_text_with_trOCR_using_google_bboxes(self.do_use_trOCR, logger)
+                    self.OCR = self.OCR + part_OCR + part_OCR
+                else:
+                    self.OCR = self.OCR + "\ntrOCR:\n" + self.detect_text_with_trOCR_using_google_bboxes(self.do_use_trOCR, logger)
+            # logger.info(f"OCR:\n{self.OCR}")
+            else:
+                # populate self.OCR_JSON_to_file = {}
+                _ = self.detect_text_with_trOCR_using_google_bboxes(self.do_use_trOCR, logger)
+
+
+        if do_create_OCR_helper_image and ('LLaVA' not in self.OCR_option):
+            self.image = Image.open(self.path)
+
+            if 'normal' in self.OCR_option:
+                image_with_boxes_normal = self.draw_boxes('normal')
+                text_image_normal = self.render_text_on_black_image('normal')
+                self.merged_image_normal = self.merge_images(image_with_boxes_normal, text_image_normal)
+
+            if 'hand' in self.OCR_option:
+                image_with_boxes_hand = self.draw_boxes('hand')
+                text_image_hand = self.render_text_on_black_image('hand')
+                self.merged_image_hand = self.merge_images(image_with_boxes_hand, text_image_hand)
+
+            if self.do_use_trOCR:
+                text_image_trOCR = self.render_text_on_black_image('trOCR') 
+
+            if 'CRAFT' in self.OCR_option:
+                image_with_boxes_normal = self.draw_boxes('normal')
+                self.merged_image_normal = self.merge_images(image_with_boxes_normal, text_image_trOCR)
+
+            ### Merge final overlay image
+            ### [original, normal bboxes, normal text]
+            if 'CRAFT' in self.OCR_option or 'normal' in self.OCR_option:
+                self.overlay_image = self.merge_images(Image.open(self.path), self.merged_image_normal)
+            ### [original, hand bboxes, hand text]
+            elif 'hand' in self.OCR_option:
+                self.overlay_image = self.merge_images(Image.open(self.path), self.merged_image_hand)
+            ### [original, normal bboxes, normal text, hand bboxes, hand text]
+            else:
+                self.overlay_image = self.merge_images(Image.open(self.path), self.merge_images(self.merged_image_normal, self.merged_image_hand))
+            
+            if self.do_use_trOCR:
+                if 'CRAFT' in self.OCR_option:
+                    heat_map_text = Image.fromarray(cv2.cvtColor(self.prediction_result["heatmaps"]["text_score_heatmap"], cv2.COLOR_BGR2RGB))
+                    heat_map_link = Image.fromarray(cv2.cvtColor(self.prediction_result["heatmaps"]["link_score_heatmap"], cv2.COLOR_BGR2RGB))
+                    self.overlay_image = self.merge_images(self.overlay_image, heat_map_text)
+                    self.overlay_image = self.merge_images(self.overlay_image, heat_map_link)
+
+                else:
+                    self.overlay_image = self.merge_images(self.overlay_image, text_image_trOCR)
+
+        else:
+            self.merged_image_normal = None
+            self.merged_image_hand = None
+            self.overlay_image = Image.open(self.path)
+        
+        try:
+            from craft_text_detector import empty_cuda_cache
+            empty_cuda_cache()
+        except:
+            pass
+
+class SafetyCheck():
+    def __init__(self, is_hf) -> None:
+        self.is_hf = is_hf
+        self.set_client()
+
+    def set_client(self):
+        if self.is_hf:
+            self.client = vision.ImageAnnotatorClient(credentials=self.get_google_credentials())
+        else:
+            self.client = vision.ImageAnnotatorClient(credentials=self.get_google_credentials())
+
+    def get_google_credentials(self):
+        creds_json_str = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+        credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
+        return credentials
+    
+    def check_for_inappropriate_content(self, file_stream):
+        try:
+            LEVEL = 2
+            # content = file_stream.read()
+            file_stream.seek(0)  # Reset file stream position to the beginning
+            content = file_stream.read()
+            image = vision.Image(content=content)
+            response = self.client.safe_search_detection(image=image)
+            safe = response.safe_search_annotation
+
+            likelihood_name = (
+                "UNKNOWN",
+                "VERY_UNLIKELY",
+                "UNLIKELY",
+                "POSSIBLE",
+                "LIKELY",
+                "VERY_LIKELY",
+            )
+            print("Safe search:")
+
+            print(f"    adult*: {likelihood_name[safe.adult]}")
+            print(f"    medical*: {likelihood_name[safe.medical]}")
+            print(f"    spoofed: {likelihood_name[safe.spoof]}")
+            print(f"    violence*: {likelihood_name[safe.violence]}")
+            print(f"    racy: {likelihood_name[safe.racy]}")
+
+            # Check the levels of adult, violence, racy, etc. content.
+            if (safe.adult > LEVEL or
+                safe.medical > LEVEL or
+                # safe.spoof > LEVEL or
+                safe.violence > LEVEL #or
+                # safe.racy > LEVEL
+                ):
+                print("Found violation")
+                return True  # The image violates safe search guidelines.
+
+            print("Found NO violation")
+            return False  # The image is considered safe.
+        except:
+            return False  # The image is considered safe. TEMPOROARY FIX TODO

vouchervision/OCR_google_cloud_vision.py

@@ -7,7 +7,7 @@ from PIL import Image, ImageDraw, ImageFont
 import colorsys
 from tqdm import tqdm
 from google.oauth2 import service_account
-
+from OCR_Florence_2 import FlorenceOCR
 ### LLaVA should only be installed if the user will actually use it.
 ### It requires the most recent pytorch/Python and can mess with older systems
 
@@ -43,6 +43,7 @@ class OCREngine:
         self.path = path
         self.cfg = cfg
         self.do_use_trOCR = self.cfg['leafmachine']['project']['do_use_trOCR']
+        self.do_use_florence = self.cfg['leafmachine']['project']['do_use_florence']
         self.OCR_option = self.cfg['leafmachine']['project']['OCR_option']
         self.double_OCR = self.cfg['leafmachine']['project']['double_OCR']
         self.dir_home = dir_home
@@ -53,6 +54,8 @@ class OCREngine:
         self.trOCR_model = trOCR_model
         self.device = device
 
+        self.OCR_JSON_to_file = {}
+
         self.hand_cleaned_text = None
         self.hand_organized_text = None
         self.hand_bounds = None
@@ -80,6 +83,7 @@ class OCREngine:
         self.trOCR_confidences = None
         self.trOCR_characters = None
         self.set_client()
+        self.init_florence()
         self.init_craft()
 
         self.multimodal_prompt = """I need you to transcribe all of the text in this image. 
@@ -117,6 +121,10 @@ class OCREngine:
             else:
                 self.craft_net = load_craftnet_model(weight_path=os.path.join(self.dir_home,'vouchervision','craft','craft_mlt_25k.pth'), cuda=False)
 
+    def init_florence(self):
+        if 'Florence-2' in self.OCR_option:
+            self.Florence = FlorenceOCR(logger=self.logger, model_id=self.cfg['leafmachine']['project']['florence_model_path'])
+
     def init_llava(self):
         if 'LLaVA' in self.OCR_option:
             from vouchervision.OCR_llava import OCRllava
@@ -241,8 +249,6 @@ class OCREngine:
     def detect_text_with_trOCR_using_google_bboxes(self, do_use_trOCR, logger):
         CONFIDENCES = 0.80
         MAX_NEW_TOKENS = 50
-
-        self.OCR_JSON_to_file = {}
         
         ocr_parts = ''
         if not do_use_trOCR:
@@ -677,6 +683,9 @@ class OCREngine:
 
 
     def process_image(self, do_create_OCR_helper_image, logger):
+        if 'hand' not in self.OCR_option and 'normal' not in self.OCR_option:
+            do_create_OCR_helper_image = False
+            
         # Can stack options, so solitary if statements
         self.OCR = 'OCR:\n'
         if 'CRAFT' in self.OCR_option:
@@ -697,11 +706,7 @@ class OCREngine:
             image, json_output, direct_output, str_output, usage_report = self.Llava.transcribe_image(self.path, self.multimodal_prompt)
             self.logger.info(f"LLaVA Usage Report for Model {self.Llava.model_path}:\n{usage_report}")
 
-            try:
-                self.OCR_JSON_to_file['OCR_LLaVA'] = str_output
-            except:
-                self.OCR_JSON_to_file = {}
-                self.OCR_JSON_to_file['OCR_LLaVA'] = str_output
+            self.OCR_JSON_to_file['OCR_LLaVA'] = str_output
 
             if self.double_OCR:
                 self.OCR = self.OCR + f"\nLLaVA OCR:\n{str_output}" + f"\nLLaVA OCR:\n{str_output}"
@@ -709,6 +714,20 @@ class OCREngine:
                 self.OCR = self.OCR + f"\nLLaVA OCR:\n{str_output}"
             # logger.info(f"LLaVA OCR:\n{self.OCR}")
 
+        if 'Florence-2' in self.OCR_option: # This option does not produce an OCR helper image
+            if self.json_report:
+                self.json_report.set_text(text_main=f'Working on Florence-2 [{self.Florence.model_id}] transcription :construction:')
+
+            self.logger.info(f"Florence-2 Usage Report for Model [{self.Florence.model_id}]")
+            results_text, results_text_dirty, results, usage_report = self.Florence.ocr_florence(self.path, task_prompt='<OCR>', text_input=None)
+
+            self.OCR_JSON_to_file['OCR_Florence'] = results_text
+
+            if self.double_OCR:
+                self.OCR = self.OCR + f"\nFlorence-2 OCR:\n{results_text}" + f"\nFlorence-2 OCR:\n{results_text}"
+            else:
+                self.OCR = self.OCR + f"\nFlorence-2 OCR:\n{results_text}"
+
         if 'normal' in self.OCR_option or 'hand' in self.OCR_option:
             if 'normal' in self.OCR_option:
                 if self.double_OCR:
@@ -762,14 +781,16 @@ class OCREngine:
 
             ### Merge final overlay image
             ### [original, normal bboxes, normal text]
-            if 'CRAFT' in self.OCR_option or 'normal' in self.OCR_option:
-                self.overlay_image = self.merge_images(Image.open(self.path), self.merged_image_normal)
-            ### [original, hand bboxes, hand text]
-            elif 'hand' in self.OCR_option:
-                self.overlay_image = self.merge_images(Image.open(self.path), self.merged_image_hand)
-            ### [original, normal bboxes, normal text, hand bboxes, hand text]
-            else:
-                self.overlay_image = self.merge_images(Image.open(self.path), self.merge_images(self.merged_image_normal, self.merged_image_hand))
+            if 'hand' in self.OCR_option or 'normal' in self.OCR_option:
+                if 'CRAFT' in self.OCR_option or 'normal' in self.OCR_option:
+                    self.overlay_image = self.merge_images(Image.open(self.path), self.merged_image_normal)
+                ### [original, hand bboxes, hand text]
+                elif 'hand' in self.OCR_option:
+                    self.overlay_image = self.merge_images(Image.open(self.path), self.merged_image_hand)
+                ### [original, normal bboxes, normal text, hand bboxes, hand text]
+                else:
+                    self.overlay_image = self.merge_images(Image.open(self.path), self.merge_images(self.merged_image_normal, self.merged_image_hand))
+                
             
             if self.do_use_trOCR:
                 if 'CRAFT' in self.OCR_option:

vouchervision/VoucherVision_Config_Builder.py

@@ -36,21 +36,22 @@ def build_VV_config(loaded_cfg=None):
         save_cropped_annotations = ['label','barcode']
 
         do_use_trOCR = False
+        do_use_florence = False
         trOCR_model_path = "microsoft/trocr-large-handwritten"
+        florence_model_path = "microsoft/Florence-2-large"
         OCR_option = 'hand'
         OCR_option_llava = 'llava-v1.6-mistral-7b' # "llava-v1.6-mistral-7b", "llava-v1.6-34b", "llava-v1.6-vicuna-13b", "llava-v1.6-vicuna-7b",
         OCR_option_llava_bit = 'full' # full or 4bit
         double_OCR = False
 
-
         tool_GEO = True
         tool_WFO = True
         tool_wikipedia = True
 
         check_for_illegal_filenames = False
 
-        LLM_version_user = 'Azure GPT 4' #'Azure GPT 4 Turbo 1106-preview'
-        prompt_version = 'SLTPvA_long.yaml' # from ["Version 1", "Version 1 No Domain Knowledge", "Version 2"]
+        LLM_version_user = 'Gemini 1.5 Flash' # 'Azure GPT 4' #'Azure GPT 4 Turbo 1106-preview'
+        prompt_version = 'SLTPvM_long.yaml' # from ["Version 1", "Version 1 No Domain Knowledge", "Version 2"]
         use_LeafMachine2_collage_images = True # Use LeafMachine2 collage images
         do_create_OCR_helper_image = True
 
@@ -71,7 +72,7 @@ def build_VV_config(loaded_cfg=None):
         return assemble_config(dir_home, run_name, dir_images_local,dir_output,
                         prefix_removal,suffix_removal,catalog_numerical_only,LLM_version_user,batch_size,num_workers,
                         path_domain_knowledge,embeddings_database_name,use_LeafMachine2_collage_images,
-                        prompt_version, do_create_OCR_helper_image, do_use_trOCR, trOCR_model_path, OCR_option, OCR_option_llava, 
+                        prompt_version, do_create_OCR_helper_image, do_use_trOCR, do_use_florence, trOCR_model_path, florence_model_path, OCR_option, OCR_option_llava, 
                         OCR_option_llava_bit, double_OCR, save_cropped_annotations, 
                         tool_GEO, tool_WFO, tool_wikipedia,
                         check_for_illegal_filenames, skip_vertical, pdf_conversion_dpi, use_domain_knowledge=False)
@@ -88,7 +89,9 @@ def build_VV_config(loaded_cfg=None):
         catalog_numerical_only = loaded_cfg['leafmachine']['project']['catalog_numerical_only']
 
         do_use_trOCR = loaded_cfg['leafmachine']['project']['do_use_trOCR']
+        do_use_florence = loaded_cfg['leafmachine']['project']['do_use_florence']
         trOCR_model_path = loaded_cfg['leafmachine']['project']['trOCR_model_path']
+        florence_model_path = loaded_cfg['leafmachine']['project']['florence_model_path']
         OCR_option = loaded_cfg['leafmachine']['project']['OCR_option']
         OCR_option_llava = loaded_cfg['leafmachine']['project']['OCR_option_llava']
         OCR_option_llava_bit  = loaded_cfg['leafmachine']['project']['OCR_option_llava_bit']
@@ -118,7 +121,7 @@ def build_VV_config(loaded_cfg=None):
         return assemble_config(dir_home, run_name, dir_images_local,dir_output,
                         prefix_removal,suffix_removal,catalog_numerical_only,LLM_version_user,batch_size,num_workers,
                         path_domain_knowledge,embeddings_database_name,use_LeafMachine2_collage_images,
-                        prompt_version, do_create_OCR_helper_image, do_use_trOCR, trOCR_model_path, OCR_option, OCR_option_llava, 
+                        prompt_version, do_create_OCR_helper_image, do_use_trOCR, do_use_florence, trOCR_model_path, florence_model_path, OCR_option, OCR_option_llava, 
                         OCR_option_llava_bit, double_OCR, save_cropped_annotations,
                         tool_GEO, tool_WFO, tool_wikipedia,
                         check_for_illegal_filenames, skip_vertical, pdf_conversion_dpi, use_domain_knowledge=False)
@@ -127,7 +130,7 @@ def build_VV_config(loaded_cfg=None):
 def assemble_config(dir_home, run_name, dir_images_local,dir_output,
                     prefix_removal,suffix_removal,catalog_numerical_only,LLM_version_user,batch_size,num_workers,
                     path_domain_knowledge,embeddings_database_name,use_LeafMachine2_collage_images,
-                    prompt_version, do_create_OCR_helper_image_user, do_use_trOCR, trOCR_model_path, OCR_option, OCR_option_llava, 
+                    prompt_version, do_create_OCR_helper_image_user, do_use_trOCR, do_use_florence, trOCR_model_path, florence_model_path, OCR_option, OCR_option_llava, 
                     OCR_option_llava_bit, double_OCR, save_cropped_annotations, 
                     tool_GEO, tool_WFO, tool_wikipedia,
                     check_for_illegal_filenames, skip_vertical, pdf_conversion_dpi, use_domain_knowledge=False):
@@ -174,7 +177,9 @@ def assemble_config(dir_home, run_name, dir_images_local,dir_output,
         'delete_all_temps': False,
         'delete_temps_keep_VVE': False,
         'do_use_trOCR': do_use_trOCR,
+        'do_use_florence': do_use_florence,
         'trOCR_model_path': trOCR_model_path,
+        'florence_model_path': florence_model_path,
         'OCR_option': OCR_option,
         'OCR_option_llava': OCR_option_llava,
         'OCR_option_llava_bit': OCR_option_llava_bit,

vouchervision/fetch_data.py

@@ -7,7 +7,7 @@ import urllib.request
 from tqdm import tqdm
 import subprocess
 
-VERSION = 'v-2-1'
+VERSION = 'v-2-3'
 
 def fetch_data(logger, dir_home, cfg_file_path):
     logger.name = 'Fetch Data'

vouchervision/generate_partner_collage.py

@@ -0,0 +1,104 @@
+import requests
+from bs4 import BeautifulSoup
+from PIL import Image
+from io import BytesIO
+import time
+
+# Global variables
+H = 200
+ROWS = 6
+PADDING = 30
+
+# Step 1: Fetch the Images from the URL Folder
+def fetch_image_urls(url):
+    response = requests.get(url + '?t=' + str(time.time()))
+    soup = BeautifulSoup(response.content, 'html.parser')
+    images = {}
+    for node in soup.find_all('a'):
+        href = node.get('href')
+        if href.endswith(('.png', '.jpg', '.jpeg')):
+            try:
+                image_index = int(href.split('__')[0])
+                images[image_index] = url + '/' + href + '?t=' + str(time.time())
+            except ValueError:
+                print(f"Skipping invalid image: {href}")
+    return images
+
+# Step 2: Resize Images to Height H
+def fetch_image(url):
+    response = requests.get(url)
+    return Image.open(BytesIO(response.content))
+
+def resize_images(images, target_height):
+    resized_images = {}
+    for index, img in images.items():
+        ratio = target_height / img.height
+        new_width = int(img.width * ratio)
+        resized_img = img.resize((new_width, target_height), Image.BICUBIC)
+        resized_images[index] = resized_img
+    return resized_images
+
+# Step 3: Create a Collage with Efficient Placement Algorithm
+def create_collage(image_urls, collage_path, H, ROWS, PADDING):
+    images = {index: fetch_image(url) for index, url in image_urls.items()}
+    resized_images = resize_images(images, H)  # Resize to H pixels height
+
+    center_image = resized_images.pop(0)
+    other_images = list(resized_images.items())
+
+    # Calculate collage size based on the number of rows
+    collage_width = 3000  # 16:9 aspect ratio width
+    collage_height = (H + PADDING) * ROWS + 2 * PADDING  # Adjust height based on number of rows, add padding to top and bottom
+    collage = Image.new('RGB', (collage_width, collage_height), (255, 255, 255))
+
+    # Sort images by width and height
+    sorted_images = sorted(other_images, key=lambda x: x[1].width * x[1].height, reverse=True)
+
+    # Create alternate placement list and insert the center image in the middle
+    alternate_images = []
+    i, j = 0, len(sorted_images) - 1
+    halfway_point = (len(sorted_images) + 1) // 2
+    count = 0
+
+    while i <= j:
+        if count == halfway_point:
+            alternate_images.append((0, center_image))
+        if i == j:
+            alternate_images.append(sorted_images[i])
+        else:
+            alternate_images.append(sorted_images[i])
+            alternate_images.append(sorted_images[j])
+        i += 1
+        j -= 1
+        count += 2
+
+    # Calculate number of images per row
+    images_per_row = len(alternate_images) // ROWS
+    extra_images = len(alternate_images) % ROWS
+
+    # Place images in rows with only padding space between them
+    def place_images_in_rows(images, collage, max_width, padding, row_height, rows, images_per_row, extra_images):
+        y = padding
+        for current_row in range(rows):
+            row_images_count = images_per_row + (1 if extra_images > 0 else 0)
+            extra_images -= 1 if extra_images > 0 else 0
+            row_images = images[:row_images_count]
+            row_width = sum(img.width for idx, img in row_images) + padding * (row_images_count - 1)
+            x = (max_width - row_width) // 2
+            for idx, img in row_images:
+                collage.paste(img, (x, y))
+                x += img.width + padding
+            y += row_height + padding
+            images = images[row_images_count:]
+
+    place_images_in_rows(alternate_images, collage, collage_width, PADDING, H, ROWS, images_per_row, extra_images)
+
+    collage.save(collage_path)
+
+# Define the URL folder and other constants
+url_folder = 'https://leafmachine.org/partners/'
+collage_path = 'img/collage.jpg'
+
+# Fetch, Create, and Update
+image_urls = fetch_image_urls(url_folder)
+create_collage(image_urls, collage_path, H, ROWS, PADDING)

vouchervision/librarian_knowledge.json

@@ -0,0 +1,27 @@
+{
+    "catalogNumber": "barcode identifier, at least 6 digits, fewer than 30 digits.",
+    "order": "full scientific name of the Order in which the taxon is classified. Order must be capitalized.",
+    "family": "full scientific name of the Family in which the taxon is classified. Family must be capitalized.",
+    "scientificName": "scientific name of the taxon including Genus, specific epithet, and any lower classifications.",
+    "scientificNameAuthorship": "authorship information for the scientificName formatted according to the conventions of the applicable Darwin Core nomenclaturalCode.",
+    "genus": "taxonomic determination to Genus, Genus must be capitalized.",
+    "specificEpithet": "The name of the first or species epithet of the scientificName. Only include the species epithet.",
+    "identifiedBy": "list of names of people, doctors, professors, groups, or organizations who identified, determined the taxon name to the subject organism. This is not the specimen collector.",
+    "recordedBy": "list of names of people, doctors, professors, groups, or organizations.",
+    "recordNumber": "identifier given to the specimen at the time it was recorded.",
+    "verbatimEventDate": "The verbatim original representation of the date and time information for when the specimen was collected.",
+    "eventDate": "collection date formatted as year-month-day YYYY-MM-DD.",
+    "habitat": "habitat.",
+    "occurrenceRemarks": "all descriptive text in the OCR rearranged into sensible sentences or sentence fragments.",
+    "country": "country or major administrative unit.",
+    "stateProvince": "state, province, canton, department, region, etc.",
+    "county": "county, shire, department, parish etc.",
+    "municipality": "city, municipality, etc.",
+    "locality": "description of geographic information aiding in pinpointing the exact origin or location of the specimen.",
+    "degreeOfEstablishment": "cultivated plants are intentionally grown by humans. Use either - unknown or cultivated.",
+    "decimalLatitude": "latitude decimal coordinate.",
+    "decimalLongitude": "longitude decimal coordinate.",
+    "verbatimCoordinates": "verbatim location coordinates.",
+    "minimumElevationInMeters": "minimum elevation or altitude in meters.",
+    "maximumElevationInMeters": "maximum elevation or altitude in meters."
+}

vouchervision/save_dataset.py

@@ -0,0 +1,34 @@
+from datasets import load_dataset
+
+# Load the dataset
+dataset = load_dataset("phyloforfun/HLT_MICH_Angiospermae_SLTPvC_v1-0_medium_OCR-C25-L25-E50-R05")
+
+# Define the directory where you want to save the files
+save_dir = "D:/Dropbox/VoucherVision/datasets/SLTPvC_v1-0_medium_OCR-C25-L25-E50-R05"
+
+# Save each split as a JSONL file in the specified directory
+for split, split_dataset in dataset.items():
+    split_dataset.to_json(f"{save_dir}/SLTPvC_v1-0_medium_OCR-C25-L25-E50-R05-{split}.jsonl")
+
+
+'''import json # convert to google
+
+# Load the JSONL file
+input_file_path = '/mnt/data/SLTPvC_v1-0_medium_OCR-C25-L25-E50-R05-train.jsonl'
+output_file_path = '/mnt/data/SLTPvC_v1-0_medium_OCR-C25-L25-E50-R05-train-converted.jsonl'
+
+# Define the conversion function
+def convert_record(record):
+    return {
+        "input_text": record.get('instruction', '') + ' ' + record.get('input', ''),
+        "target_text": record.get('output', '')
+    }
+
+# Convert and save the new JSONL file
+with open(input_file_path, 'r', encoding='utf-8') as infile, open(output_file_path, 'w', encoding='utf-8') as outfile:
+    for line in infile:
+        record = json.loads(line)
+        converted_record = convert_record(record)
+        outfile.write(json.dumps(converted_record) + '\n')
+
+output_file_path'''

vouchervision/utils_VoucherVision.py

@@ -14,6 +14,7 @@ from vouchervision.LLM_GoogleGemini import GoogleGeminiHandler
 from vouchervision.LLM_MistralAI import MistralHandler
 from vouchervision.LLM_local_cpu_MistralAI import LocalCPUMistralHandler
 from vouchervision.LLM_local_MistralAI import LocalMistralHandler 
+from vouchervision.LLM_local_custom_fine_tune import LocalFineTuneHandler 
 from vouchervision.prompt_catalog import PromptCatalog
 from vouchervision.model_maps import ModelMaps
 from vouchervision.general_utils import get_cfg_from_full_path
@@ -449,6 +450,8 @@ class VoucherVision():
             k_openai = os.getenv('OPENAI_API_KEY')
             k_openai_azure = os.getenv('AZURE_API_VERSION')
 
+            k_huggingface = None
+
             k_google_project_id = os.getenv('GOOGLE_PROJECT_ID')
             k_google_location = os.getenv('GOOGLE_LOCATION')
             k_google_application_credentials = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
@@ -464,6 +467,8 @@ class VoucherVision():
             k_openai = self.cfg_private['openai']['OPENAI_API_KEY']
             k_openai_azure = self.cfg_private['openai_azure']['OPENAI_API_KEY_AZURE']
 
+            k_huggingface = self.cfg_private['huggingface']['hf_token']
+
             k_google_project_id = self.cfg_private['google']['GOOGLE_PROJECT_ID']
             k_google_location = self.cfg_private['google']['GOOGLE_LOCATION']
             k_google_application_credentials = self.cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS']
@@ -478,6 +483,8 @@ class VoucherVision():
         self.has_key_azure_openai = self.has_API_key(k_openai_azure)
         self.llm = None
         
+        self.has_key_huggingface = self.has_API_key(k_huggingface)
+
         self.has_key_google_project_id = self.has_API_key(k_google_project_id)
         self.has_key_google_location = self.has_API_key(k_google_location)
         self.has_key_google_application_credentials = self.has_API_key(k_google_application_credentials)
@@ -505,6 +512,11 @@ class VoucherVision():
                 openai.api_key = self.cfg_private['openai']['OPENAI_API_KEY']
                 os.environ["OPENAI_API_KEY"] = self.cfg_private['openai']['OPENAI_API_KEY']
 
+        if self.has_key_huggingface:
+            if self.is_hf:
+                pass
+            else:
+                os.environ["HUGGING_FACE_KEY"] = self.cfg_private['huggingface']['hf_token']
 
         ### OpenAI - Azure
         if self.has_key_azure_openai:
@@ -738,6 +750,10 @@ class VoucherVision():
                             response_candidate, nt_in, nt_out, WFO_record, GEO_record, usage_report = llm_model.call_llm_local_cpu_MistralAI(prompt, json_report, paths) 
                         else:
                             response_candidate, nt_in, nt_out, WFO_record, GEO_record, usage_report = llm_model.call_llm_local_MistralAI(prompt, json_report, paths) 
+                
+                elif "/" in ''.join(name_parts):
+                    response_candidate, nt_in, nt_out, WFO_record, GEO_record, usage_report = llm_model.call_llm_local_custom_fine_tune(self.OCR, json_report, paths)  ###
+
                 else:
                     response_candidate, nt_in, nt_out, WFO_record, GEO_record, usage_report = llm_model.call_llm_api_OpenAI(prompt, json_report, paths)
 
@@ -771,6 +787,8 @@ class VoucherVision():
                     return LocalCPUMistralHandler(cfg, logger, model_name, JSON_dict_structure, config_vals_for_permutation)
                 else:
                     return LocalMistralHandler(cfg, logger, model_name, JSON_dict_structure, config_vals_for_permutation)
+        elif "/" in ''.join(name_parts):
+            return LocalFineTuneHandler(cfg, logger, model_name, JSON_dict_structure, config_vals_for_permutation)
         else:
             if 'PALM2' in name_parts:
                 return GooglePalm2Handler(cfg, logger, model_name, JSON_dict_structure, config_vals_for_permutation)

vouchervision/utils_hf (DESKTOP-548UDCR's conflicted copy 2024-06-15).py

@@ -0,0 +1,266 @@
+import os, json, re, datetime, tempfile, yaml
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from google.oauth2 import service_account
+import base64
+from PIL import Image
+from PIL import Image
+from io import BytesIO
+from shutil import copyfileobj, copyfile
+
+# from vouchervision.general_utils import get_cfg_from_full_path
+
+
+def setup_streamlit_config(dir_home):
+    # Define the directory path and filename
+    dir_path = os.path.join(dir_home, ".streamlit")
+    file_path = os.path.join(dir_path, "config.toml")
+
+    # Check if directory exists, if not create it
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+    
+    # Create or modify the file with the provided content
+    config_content = f"""
+    [theme]
+    base = "dark"
+    primaryColor = "#00ff00"
+
+    [server]
+    enableStaticServing = false
+    runOnSave = true
+    port = 8524
+    maxUploadSize = 5000
+    """
+
+    with open(file_path, "w") as f:
+        f.write(config_content.strip())
+
+
+def save_uploaded_file_local(directory_in, directory_out, img_file_name, image=None):
+    if not os.path.exists(directory_out):
+        os.makedirs(directory_out)
+
+    # Assuming img_file_name includes the extension
+    img_file_base, img_file_ext = os.path.splitext(img_file_name)
+    
+    full_path_out = os.path.join(directory_out, img_file_name)
+    full_path_in = os.path.join(directory_in, img_file_name)
+
+    # Check if the file extension is .pdf (or add other conditions for different file types)
+    if img_file_ext.lower() == '.pdf':
+        # Copy the file from the input directory to the output directory
+        copyfile(full_path_in, full_path_out)
+        return full_path_out
+    else:
+        if image is None:
+            try:
+                with Image.open(full_path_in) as image:
+                    image.save(full_path_out, "JPEG")
+                # Return the full path of the saved image
+                return full_path_out
+            except:
+                pass
+        else:
+            try:
+                image.save(full_path_out, "JPEG")
+                return full_path_out
+            except:
+                pass
+            
+
+def save_uploaded_file(directory, img_file, image=None):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+    full_path = os.path.join(directory, img_file.name)
+
+    # Assuming the uploaded file is an image
+    if img_file.name.lower().endswith('.pdf'):
+        with open(full_path, 'wb') as out_file:
+            # If img_file is a file-like object (e.g., Django's UploadedFile),
+            # you can use copyfileobj or read chunks.
+            # If it's a path, you'd need to open and then save it.
+            if hasattr(img_file, 'read'):
+                # This is a file-like object
+                copyfileobj(img_file, out_file)
+            else:
+                # If img_file is a path string
+                with open(img_file, 'rb') as fd:
+                    copyfileobj(fd, out_file)    
+            return full_path
+    else:
+        if image is None:
+            try:
+                with Image.open(img_file) as image:
+                    full_path = os.path.join(directory, img_file.name)
+                    image.save(full_path, "JPEG")
+                # Return the full path of the saved image
+                return full_path
+            except:
+                try:
+                    with Image.open(os.path.join(directory,img_file)) as image:
+                        full_path = os.path.join(directory, img_file)
+                        image.save(full_path, "JPEG")
+                    # Return the full path of the saved image
+                    return full_path
+                except:
+                    with Image.open(img_file.name) as image:
+                        full_path = os.path.join(directory, img_file.name)
+                        image.save(full_path, "JPEG")
+                    # Return the full path of the saved image
+                    return full_path
+        else:
+            try:
+                full_path = os.path.join(directory, img_file.name)
+                image.save(full_path, "JPEG")
+                return full_path
+            except:
+                full_path = os.path.join(directory, img_file)
+                image.save(full_path, "JPEG")
+                return full_path
+# def save_uploaded_file(directory, uploaded_file, image=None):
+#     if not os.path.exists(directory):
+#         os.makedirs(directory)
+
+#     full_path = os.path.join(directory, uploaded_file.name)
+
+#     # Handle PDF files
+#     if uploaded_file.name.lower().endswith('.pdf'):
+#         with open(full_path, 'wb') as out_file:
+#             if hasattr(uploaded_file, 'read'):
+#                 copyfileobj(uploaded_file, out_file)
+#             else:
+#                 with open(uploaded_file, 'rb') as fd:
+#                     copyfileobj(fd, out_file)
+#         return full_path
+#     else:
+#         if image is None:
+#             try:
+#                 with Image.open(uploaded_file) as img:
+#                     img.save(full_path, "JPEG")
+#             except:
+#                 with Image.open(full_path) as img:
+#                     img.save(full_path, "JPEG")
+#         else:
+#             try:
+#                 image.save(full_path, "JPEG")
+#             except:
+#                 image.save(os.path.join(directory, uploaded_file.name), "JPEG")
+#         return full_path
+    
+def save_uploaded_local(directory, img_file, image=None):
+    name = img_file.split(os.path.sep)[-1]
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    
+    # Assuming the uploaded file is an image
+    if image is None:
+        with Image.open(img_file) as image:
+            full_path = os.path.join(directory, name)
+            image.save(full_path, "JPEG")
+        # Return the full path of the saved image
+        return os.path.join('uploads_small',name)
+    else:
+        full_path = os.path.join(directory, name)
+        image.save(full_path, "JPEG")
+        return os.path.join('.','uploads_small',name)
+    
+def image_to_base64(img):
+    buffered = BytesIO()
+    img.save(buffered, format="JPEG")
+    return base64.b64encode(buffered.getvalue()).decode()
+
+def check_prompt_yaml_filename(fname):
+    # Check if the filename only contains letters, numbers, underscores, and dashes
+    pattern = r'^[\w-]+$'
+    
+    # The \w matches any alphanumeric character and is equivalent to the character class [a-zA-Z0-9_].
+    # The hyphen - is literally matched.
+
+    if re.match(pattern, fname):
+        return True
+    else:
+        return False
+
+def report_violation(file_name, is_hf=True):
+    # Format the current date and time
+    current_time = datetime.datetime.now().strftime("%Y_%m_%d__%H_%M_%S")
+    violation_file_name = f"violation_{current_time}.yaml"  # Updated variable name to avoid confusion
+    
+    # Create a temporary YAML file in text mode
+    with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.yaml') as temp_file:
+        # Example content - customize as needed
+        content = {
+            'violation_time': current_time,
+            'notes': 'This is an autogenerated violation report.',
+            'name_of_file': file_name,
+        }
+        # Write the content to the temporary YAML file in text mode
+        yaml.dump(content, temp_file, default_flow_style=False)
+        temp_filepath = temp_file.name
+
+    # Now upload the temporary file
+    upload_to_drive(temp_filepath, violation_file_name, is_hf=is_hf)
+
+    # Optionally, delete the temporary file if you don't want it to remain on disk after uploading
+    os.remove(temp_filepath)
+
+# Function to upload files to Google Drive
+def upload_to_drive(filepath, filename, is_hf=True, cfg_private=None, do_upload = True):
+    if do_upload:
+        creds = get_google_credentials(is_hf=is_hf, cfg_private=cfg_private)
+        if creds:
+            service = build('drive', 'v3', credentials=creds)
+
+            # Get the folder ID from the environment variable
+            if is_hf:
+                folder_id = os.environ.get('GDRIVE_FOLDER_ID')  # Renamed for clarity
+            else:
+                folder_id = cfg_private['google']['GDRIVE_FOLDER_ID']  # Renamed for clarity
+
+
+            if folder_id:
+                file_metadata = {
+                    'name': filename,
+                    'parents': [folder_id]
+                }
+
+                # Determine the mimetype based on the file extension
+                if filename.endswith('.yaml') or filename.endswith('.yml') or filepath.endswith('.yaml') or filepath.endswith('.yml'):
+                    mimetype = 'application/x-yaml'
+                elif filepath.endswith('.zip'):
+                    mimetype = 'application/zip'
+                else:
+                    # Set a default mimetype if desired or handle the unsupported file type
+                    print("Unsupported file type")
+                    return None
+
+                # Upload the file
+                try:
+                    media = MediaFileUpload(filepath, mimetype=mimetype)
+                    file = service.files().create(
+                        body=file_metadata,
+                        media_body=media,
+                        fields='id'
+                    ).execute()
+                    print(f"Uploaded file with ID: {file.get('id')}")
+                except Exception as e:
+                    msg = f"If the following error is '404 cannot find file...' then you need to share the GDRIVE folder with your Google API service account's email address. Open your Google API JSON file, find the email account that ends with '@developer.gserviceaccount.com', go to your Google Drive, share the folder with this email account. {e}"
+                    print(msg)
+                    raise Exception(msg)
+        else:
+            print("GDRIVE_API environment variable not set.")
+
+def get_google_credentials(is_hf=True, cfg_private=None): # Also used for google drive
+    if is_hf:
+        creds_json_str = os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+        credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
+        return credentials
+    else:
+        with open(cfg_private['google']['GOOGLE_APPLICATION_CREDENTIALS'], 'r') as file:
+            data = json.load(file)
+            creds_json_str = json.dumps(data)
+            credentials = service_account.Credentials.from_service_account_info(json.loads(creds_json_str))
+            os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = creds_json_str
+            return credentials

vouchervision/utils_hf.py

@@ -73,7 +73,7 @@ def save_uploaded_file(directory, img_file, image=None):
     if not os.path.exists(directory):
         os.makedirs(directory)
 
-    full_path = os.path.join(directory, img_file.name)
+    full_path = os.path.join(directory, img_file.name) ########## TODO THIS MUST BE MOVED TO conditional specific location
 
     # Assuming the uploaded file is an image
     if img_file.name.lower().endswith('.pdf'):
@@ -98,11 +98,18 @@ def save_uploaded_file(directory, img_file, image=None):
                 # Return the full path of the saved image
                 return full_path
             except:
-                with Image.open(os.path.join(directory,img_file)) as image:
-                    full_path = os.path.join(directory, img_file)
-                    image.save(full_path, "JPEG")
-                # Return the full path of the saved image
-                return full_path
+                try:
+                    with Image.open(os.path.join(directory,img_file)) as image:
+                        full_path = os.path.join(directory, img_file)
+                        image.save(full_path, "JPEG")
+                    # Return the full path of the saved image
+                    return full_path
+                except:
+                    with Image.open(img_file.name) as image:
+                        full_path = os.path.join(directory, img_file.name)
+                        image.save(full_path, "JPEG")
+                    # Return the full path of the saved image
+                    return full_path
         else:
             try:
                 full_path = os.path.join(directory, img_file.name)
@@ -112,6 +119,35 @@ def save_uploaded_file(directory, img_file, image=None):
                 full_path = os.path.join(directory, img_file)
                 image.save(full_path, "JPEG")
                 return full_path
+# def save_uploaded_file(directory, uploaded_file, image=None):
+#     if not os.path.exists(directory):
+#         os.makedirs(directory)
+
+#     full_path = os.path.join(directory, uploaded_file.name)
+
+#     # Handle PDF files
+#     if uploaded_file.name.lower().endswith('.pdf'):
+#         with open(full_path, 'wb') as out_file:
+#             if hasattr(uploaded_file, 'read'):
+#                 copyfileobj(uploaded_file, out_file)
+#             else:
+#                 with open(uploaded_file, 'rb') as fd:
+#                     copyfileobj(fd, out_file)
+#         return full_path
+#     else:
+#         if image is None:
+#             try:
+#                 with Image.open(uploaded_file) as img:
+#                     img.save(full_path, "JPEG")
+#             except:
+#                 with Image.open(full_path) as img:
+#                     img.save(full_path, "JPEG")
+#         else:
+#             try:
+#                 image.save(full_path, "JPEG")
+#             except:
+#                 image.save(os.path.join(directory, uploaded_file.name), "JPEG")
+#         return full_path
     
 def save_uploaded_local(directory, img_file, image=None):
     name = img_file.split(os.path.sep)[-1]