update on code

README.md

@@ -1,5 +1,5 @@
 ---
-title: EyeSee_v2
+title: EyeSee_chi
 emoji: 👀
 colorFrom: green
 colorTo: yellow

app.py

@@ -8,7 +8,6 @@ import json
 import gradio as gr
 import numpy as np
 from PIL import Image
-import functools
 import emoji
 from langchain_community.chat_models import ChatOpenAI
 from langchain.schema import HumanMessage
@@ -17,13 +16,12 @@ from caption_anything.utils.utils import mask_painter, seg_model_map, prepare_se
 from caption_anything.utils.parser import parse_augment
 from caption_anything.captioner import build_captioner
 from caption_anything.segmenter import build_segmenter
-from backend.chatbox import ConversationBot, build_chatbot_tools, get_new_image_name
+from backend.chatbox import build_chatbot_tools, get_new_image_name
 from segment_anything import sam_model_registry
 import easyocr
 import re
 from langchain import __version__
 import pandas as pd
-import requests
 import os
 import json
 import numpy as np
@@ -32,6 +30,7 @@ from backend.prompts import generate_prompt
 from backend.recommendation import RecommendationConfig, ImageRecommender
 from backend.gpt_service import get_gpt_response, get_artistinfo, get_yearinfo
 from backend.texttospeech.tts import texttospeech
+from backend.prompts.prompt_templates import PromptTemplates
 
 recommendation_config = RecommendationConfig()
 recommender = ImageRecommender(recommendation_config)
@@ -108,30 +107,16 @@ def validate_api_key(api_key):
         return False
 
 
-def init_openai_api_key(api_key=""):
-    text_refiner = None
-    visual_chatgpt = None
+async def init_openai_api_key(api_key=""):
     if api_key and len(api_key) > 30:
         print(api_key)
         if validate_api_key(api_key):
-            try:
-                text_refiner = None
-                print("text refiner")
-                visual_chatgpt = ConversationBot(shared_chatbot_tools, api_key=api_key)
-            except Exception as e:
-                print(f"Error initializing TextRefiner or ConversationBot: {e}")
-                text_refiner = None
-                visual_chatgpt = None
+            return [gr.update(visible=True)]*4 + [gr.update(visible=False)]*2 + [gr.update(visible=True)]*2
         else:
             print("Invalid API key.")
     else:
         print("API key is too short.")
-    print(text_refiner)
-
-    if visual_chatgpt:
-        return [gr.update(visible=True)]+[gr.update(visible=False)]+[gr.update(visible=True)]* 3 + [gr.update(visible=False)]*2 + [text_refiner, visual_chatgpt, None]+[gr.update(visible=True)]*4+[gr.update(visible=False)]+[gr.update(visible=True)]*2
-    else:
-        return [gr.update(visible=False)]*5 + [gr.update(visible=True)]*2 + [text_refiner, visual_chatgpt, 'Your OpenAI API Key is not available']+[gr.update(visible=False)]*7
+    return [gr.update(visible=False)]*5 + [gr.update(visible=True)]*2 + ['Your OpenAI API Key is not available']+[gr.update(visible=False)]*7
         
         
 def get_click_prompt(chat_input, click_state, click_mode):
@@ -171,7 +156,7 @@ def update_click_state(click_state, caption, click_mode):
         raise NotImplementedError
 
 async def chat_input_callback(*args):
-    visual_chatgpt, chat_input, click_state, state, aux_state ,language , autoplay,gender,api_key,image_input,log_state,history,persona = args
+    chat_input, state, aux_state ,language , autoplay,gender,api_key,image_input,log_state,history,persona = args
     message = chat_input["text"]
     if persona == "Narrator":
         prompt="Please help me answer the question with this painting {question} in {language}, with a response length of about 70 words.."
@@ -181,32 +166,26 @@ async def chat_input_callback(*args):
         prompt="When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. Please help me answer the question with this painting {question} in {language}, with a response length of about 70 words."
     prompt=prompt.format(question=message, language=language)
     
-    if visual_chatgpt is not None:
-        result=get_gpt_response(api_key, image_input,prompt+message,history)
-        read_info = re.sub(r'[#[\]!*]','',result)
-        read_info = emoji.replace_emoji(read_info,replace="")   
-        state = state + [(message,result)]
-        log_state += [(message,"/////")]
-        log_state += [("/////",result)]
-        # log_state += [("%% chat messahe %%",None)]
-        
-        history.append({"role": "user", "content": message})
-        history.append({"role": "assistant", "content": result})
+    result=get_gpt_response(api_key, image_input,prompt+message,history)
+    read_info = re.sub(r'[#[\]!*]','',result)
+    read_info = emoji.replace_emoji(read_info,replace="")   
+    state = state + [(message,result)]
+    log_state += [(message,"/////")]
+    log_state += [("/////",result)]
+    
+    history.append({"role": "user", "content": message})
+    history.append({"role": "assistant", "content": result})
+    
+    if autoplay==False:
+        return state, state, aux_state, None,log_state,history
         
-        if autoplay==False:
-            return state, state, aux_state, None,log_state,history
-            
-        else: 
-            audio = await texttospeech(read_info,language,gender)
-            return state, state, aux_state, audio,log_state,history
-    else:
-        response = "Text refiner is not initilzed, please input openai api key."
-        state = state + [(chat_input, response)]
-        audio = await texttospeech(response,language,gender)
-        return state, state, None, audio,log_state,history
+    else: 
+        audio = await texttospeech(read_info,language,gender)
+        return state, state, aux_state, audio,log_state,history
+
 
         
-async def upload_callback(image_input,state, log_state, task_type, visual_chatgpt=None, openai_api_key=None,language="English",narritive=None,history=None,autoplay=True,session="Session 1"):
+async def upload_callback(image_input,state, log_state, task_type, openai_api_key=None,language="English",narritive=None,history=None,autoplay=True,session="Session 1"):
     print("narritive", narritive)
     if isinstance(image_input, dict):  # if upload from sketcher_input, input contains image and mask
         image_input = image_input['background']
@@ -216,10 +195,7 @@ async def upload_callback(image_input,state, log_state, task_type, visual_chatgp
     elif isinstance(image_input, bytes):
         image_input = Image.open(io.BytesIO(image_input))
 
-    
     click_state = [[], [], []]
-    
-    
     image_input = image_resize(image_input, res=1024)
 
     model = build_caption_anything_with_models(
@@ -235,42 +211,39 @@ async def upload_callback(image_input,state, log_state, task_type, visual_chatgp
     original_size = model.original_size
     input_size = model.input_size
 
-    if visual_chatgpt is not None:
-        print('upload_callback: add caption to chatGPT memory')
-        new_image_path = get_new_image_name('chat_image', func_name='upload')
-        image_input.save(new_image_path)
-        print("img_path",new_image_path)
-        visual_chatgpt.current_image = new_image_path
-        paragraph = get_gpt_response(openai_api_key, new_image_path,f"What's going on in this picture? in {language}")
-        # visual_chatgpt.agent.memory.buffer = visual_chatgpt.agent.memory.buffer + visual_chatgpt.global_prompt 
-        if task_type=="task 3":
-            name="Along the River During the Qingming Festival"
-            artist="Zhang Zeduan"
-            year="12th century (Song Dynasty)"
-            material="Chinese painting"
-            gender="male"
-        
-        elif task_type=="task 1":
-            name ="The Ambassadors"
-            artist ="Hans Holbein the Younger"
-            year = "1533 (Northern Renaissance)"
-            material="Realism"
-            gender = "male"
-                
-        elif task_type=="task 2":
-            name = "The Football Players"
-            artist= "Albert Gleizes"
-            year= "1912 (Cubism)"
-            material="Cubism"
-            gender= "male"
+    print('upload_callback: add caption to chatGPT memory')
+    new_image_path = get_new_image_name('chat_image', func_name='upload')
+    image_input.save(new_image_path)
+    print("img_path",new_image_path)
+    paragraph = get_gpt_response(openai_api_key, new_image_path,f"What's going on in this picture? in {language}")
+    if task_type=="task 3":
+        name="Along the River During the Qingming Festival"
+        artist="Zhang Zeduan"
+        year="12th century (Song Dynasty)"
+        material="Chinese painting"
+        gender="male"
+    
+    elif task_type=="task 1":
+        name ="The Ambassadors"
+        artist ="Hans Holbein the Younger"
+        year = "1533 (Northern Renaissance)"
+        material="Realism"
+        gender = "male"
             
-        else:
-            parsed_data = get_gpt_response(openai_api_key, new_image_path,"Please provide the name, artist, year of creation (including the art historical period), and painting style used for this painting. Return the information in dictionary format without any newline characters. Format as follows: { \"name\": \"Name of the painting\", \"artist\": \"Name of the artist\", \"year\": \"Year of creation (Art historical period)\", \"style\": \"Painting style used in the painting\"}")
-            print(parsed_data)
-            parsed_data = json.loads(parsed_data.replace("'", "\""))
-            name, artist, year, material= parsed_data["name"],parsed_data["artist"],parsed_data["year"], parsed_data["style"]
-            gender="male"
-            gender=gender.lower()
+    elif task_type=="task 2":
+        name = "The Football Players"
+        artist= "Albert Gleizes"
+        year= "1912 (Cubism)"
+        material="Cubism"
+        gender= "male"
+        
+    else:
+        parsed_data = get_gpt_response(openai_api_key, new_image_path,"Please provide the name, artist, year of creation (including the art historical period), and painting style used for this painting. Return the information in dictionary format without any newline characters. Format as follows: { \"name\": \"Name of the painting\", \"artist\": \"Name of the artist\", \"year\": \"Year of creation (Art historical period)\", \"style\": \"Painting style used in the painting\"}")
+        print(parsed_data)
+        parsed_data = json.loads(parsed_data.replace("'", "\""))
+        name, artist, year, material= parsed_data["name"],parsed_data["artist"],parsed_data["year"], parsed_data["style"]
+        gender="male"
+        gender=gender.lower()
                         
     if language=="English":
         if naritive_mapping[narritive]==0 :
@@ -317,7 +290,7 @@ async def upload_callback(image_input,state, log_state, task_type, visual_chatgp
 
 
 def inference_click(image_input, point_prompt, click_mode, enable_wiki, language, sentiment, factuality,
-                    length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt,
+                    length, image_embedding, state, click_state, original_size, input_size, text_refiner,
                     out_state, click_index_state, input_mask_state, input_points_state, input_labels_state, evt: gr.SelectData):
     click_index = evt.index
 
@@ -375,10 +348,9 @@ def inference_click(image_input, point_prompt, click_mode, enable_wiki, language
     input_labels_state = input_labels
     out_state = out  
 
-    if visual_chatgpt is not None:
-        new_crop_save_path = get_new_image_name('chat_image', func_name='crop')
-        Image.open(out["crop_save_path"]).save(new_crop_save_path)
-        print("new crop save",new_crop_save_path)
+    new_crop_save_path = get_new_image_name('chat_image', func_name='crop')
+    Image.open(out["crop_save_path"]).save(new_crop_save_path)
+    print("new crop save",new_crop_save_path)
 
     return state, state, click_state, image_input_nobackground, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state,new_crop_save_path,image_input_nobackground
     
@@ -403,7 +375,7 @@ async def submit_caption(naritive, state,length, sentiment, factuality, language
                    autoplay,paragraph,focus_type,openai_api_key,new_crop_save_path, gender,log_state,history):
 
     
-    focus_value=focus_map[focus_type]
+    focus_value = PromptTemplates.FOCUS_MAP[focus_type]
     click_index = click_index_state
     
     print("click_index",click_index)
@@ -418,14 +390,11 @@ async def submit_caption(naritive, state,length, sentiment, factuality, language
         
     log_state =  log_state + [("Selected image point: {}, Input label: {}".format(input_points_state, input_labels_state), None)] 
     
-    
     print("Prompt:", prompt)
     print("click",click_index)
-    
     log_state = log_state + [(naritive, None)]
     
 
-    # if not args.disable_gpt and text_refiner:
     if not args.disable_gpt:
         print("new crop save",new_crop_save_path)
         focus_info=get_gpt_response(openai_api_key,new_crop_save_path,prompt)
@@ -448,7 +417,6 @@ async def submit_caption(naritive, state,length, sentiment, factuality, language
         history.append({"role": "assistant", "content": focus_info})
       
         
-        
         print("new_cap",focus_info)
         read_info = re.sub(r'[#[\]!*]','',focus_info)
         read_info = emoji.replace_emoji(read_info,replace="")    
@@ -479,19 +447,6 @@ async def submit_caption(naritive, state,length, sentiment, factuality, language
 
 
 
-
-def clear_chat_memory(visual_chatgpt, keep_global=False):
-    if visual_chatgpt is not None:
-        visual_chatgpt.memory.clear()
-        visual_chatgpt.point_prompt = ""
-        if keep_global:
-            # visual_chatgpt.agent.memory.buffer = visual_chatgpt.global_prompt
-            visual_chatgpt.agent.memory.save_context({"input": visual_chatgpt.global_prompt}, {"output": ""})
-        else:
-            visual_chatgpt.current_image = None
-            visual_chatgpt.global_prompt = ""
-
-
 def export_chat_log(chat_state,log_list,narrative):
     try:
         chat_log=""
@@ -637,8 +592,6 @@ def get_recommendationscore(index,score,log_state):
     return log_state
     
   
-    
-
 def toggle_icons_and_update_prompt(point_prompt):
     new_prompt = "Negative" if point_prompt == "Positive" else "Positive"
     new_add_icon = "assets/icons/plus-square-blue.png" if new_prompt == "Positive" else "assets/icons/plus-square.png"
@@ -681,7 +634,6 @@ def create_ui():
         origin_image = gr.State(None)
         image_embedding = gr.State(None)
         text_refiner = gr.State(None)
-        visual_chatgpt = gr.State(None)
         original_size = gr.State(None)
         input_size = gr.State(None)
         paragraph = gr.State("")
@@ -920,11 +872,11 @@ def create_ui():
    
         openai_api_key.submit(init_openai_api_key, inputs=[openai_api_key],
                               outputs=[export, modules_need_gpt1, modules_need_gpt3, modules_not_need_gpt,
-                                       modules_not_need_gpt2, module_key_input ,module_notification_box, text_refiner, visual_chatgpt, notification_box,top_row,recommend,reco_reasons,instruct,modules_not_need_gpt3,sort_rec,reco_preview])
+                                       modules_not_need_gpt2, module_key_input ,module_notification_box, text_refiner, notification_box,top_row,recommend,reco_reasons,instruct,modules_not_need_gpt3,sort_rec,reco_preview])
         enable_chatGPT_button.click(init_openai_api_key, inputs=[openai_api_key],
                                     outputs=[export,modules_need_gpt1, modules_need_gpt3,
                                              modules_not_need_gpt,
-                                             modules_not_need_gpt2,module_key_input,module_notification_box, text_refiner, visual_chatgpt, notification_box,top_row,recommend,reco_reasons,instruct,modules_not_need_gpt3,sort_rec,reco_preview])
+                                             modules_not_need_gpt2,module_key_input,module_notification_box, text_refiner, notification_box,top_row,recommend,reco_reasons,instruct,modules_not_need_gpt3,sort_rec,reco_preview])
         
         artist_label.click(
             get_artistinfo,
@@ -953,7 +905,7 @@ def create_ui():
             queue=False,
             show_progress=False
         )
-        clear_button_click.click(functools.partial(clear_chat_memory, keep_global=True), inputs=[visual_chatgpt])
+
 
         clear_button_text.click(
             lambda: ([], [], [[], [], [], []],[]),
@@ -962,7 +914,7 @@ def create_ui():
             queue=False,
             show_progress=False
         )
-        clear_button_text.click(clear_chat_memory, inputs=[visual_chatgpt])
+
         
         image_input.clear(
             lambda: (None, [], [], [[], [], []], "", None, []),
@@ -972,25 +924,24 @@ def create_ui():
             show_progress=False
         )
 
-        image_input.clear(clear_chat_memory, inputs=[visual_chatgpt])
+
           
         
-        image_input.upload(upload_callback, [image_input, state, log_state,task_type, visual_chatgpt,openai_api_key,language,naritive,history_log,auto_play,session_type],
+        image_input.upload(upload_callback, [image_input, state, log_state,task_type,openai_api_key,language,naritive,history_log,auto_play,session_type],
                            [chatbot, state, origin_image, click_state, image_input,image_embedding, original_size, input_size,name_label,artist_label,year_label,material_label,\
                             paragraph,artist,gender,image_path,log_state,history_log,output_audio])
         
         
         
-        chat_input.submit(chat_input_callback, [visual_chatgpt, chat_input, click_state, state, aux_state,language,auto_play,gender,openai_api_key,image_path,log_state,history_log,naritive],
+        chat_input.submit(chat_input_callback, [chat_input,state, aux_state,language,auto_play,gender,openai_api_key,image_path,log_state,history_log,naritive],
                           [chatbot, state, aux_state,output_audio,log_state,history_log])
 
         chat_input.submit(lambda: {"text": ""}, None, chat_input)
 
-        example_image.change(upload_callback, [example_image, state, log_state, task_type,  visual_chatgpt, openai_api_key,language,naritive,history_log,auto_play,session_type],
+        example_image.change(upload_callback, [example_image, state, log_state, task_type, openai_api_key,language,naritive,history_log,auto_play,session_type],
                              [chatbot, state, origin_image, click_state, image_input, image_embedding, original_size, input_size,name_label,artist_label,year_label,material_label,\
                             paragraph,artist,gender,image_path, log_state,history_log,output_audio])
 
-        example_image.change(clear_chat_memory, inputs=[visual_chatgpt])
         example_image.change(
             lambda:([],[],[],None,[],gr.update(value="Preview")), 
             [],
@@ -1000,7 +951,7 @@ def create_ui():
             inference_click,
             inputs=[
                 origin_image, point_prompt, click_mode, enable_wiki, language, sentiment, factuality, length,
-                image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt,
+                image_embedding, state, click_state, original_size, input_size, text_refiner,
                 out_state, click_index_state, input_mask_state, input_points_state, input_labels_state
             ],
             outputs=[chatbot, state, click_state, image_input, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state,new_crop_save_path,image_input_nobackground],