Spaces:
Running
Running
Niki Zhang
commited on
Update app.py
Browse files
app.py
CHANGED
@@ -187,12 +187,13 @@ def inference_click(image_input, point_prompt, click_mode, enable_wiki, language
|
|
187 |
|
188 |
click_state[0] = input_points
|
189 |
click_state[1] = input_labels
|
|
|
190 |
|
191 |
return state, click_state
|
192 |
|
193 |
def submit_caption(image_input, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt):
|
194 |
-
|
195 |
-
prompt = get_click_prompt(
|
196 |
input_points = prompt['input_point']
|
197 |
input_labels = prompt['input_label']
|
198 |
|
@@ -216,15 +217,16 @@ def submit_caption(image_input, enable_wiki, language, sentiment, factuality, le
|
|
216 |
enable_wiki = True if enable_wiki in ['True', 'TRUE', 'true', True, 'Yes', 'YES', 'yes'] else False
|
217 |
out = model.inference(image_input, prompt, controls, disable_gpt=True, enable_wiki=enable_wiki, verbose=True, args={'clip_filter': False})[0]
|
218 |
|
219 |
-
state = state + [("Image point: {}, Input label: {}".format(prompt["input_point"], prompt["input_label"]), None)]
|
220 |
state = state + [(None, "raw_caption: {}".format(out['generated_captions']['raw_caption']))]
|
221 |
-
|
|
|
222 |
text = out['generated_captions']['raw_caption']
|
223 |
input_mask = np.array(out['mask'].convert('P'))
|
224 |
image_input = mask_painter(np.array(image_input), input_mask)
|
225 |
origin_image_input = image_input
|
226 |
image_input = create_bubble_frame(image_input, text, (input_points[-1][0], input_points[-1][1]), input_mask,
|
227 |
input_points=input_points, input_labels=input_labels)
|
|
|
228 |
if visual_chatgpt is not None:
|
229 |
new_image_path = get_new_image_name('chat_image', func_name='upload')
|
230 |
image_input.save(new_image_path)
|
@@ -246,9 +248,10 @@ def submit_caption(image_input, enable_wiki, language, sentiment, factuality, le
|
|
246 |
input_mask,
|
247 |
input_points=input_points, input_labels=input_labels)
|
248 |
return state, state, click_state, refined_image_input
|
249 |
-
|
250 |
return state, state, click_state, image_input
|
251 |
|
|
|
252 |
def get_sketch_prompt(mask: Image.Image):
|
253 |
"""
|
254 |
Get the prompt for the sketcher.
|
@@ -556,7 +559,7 @@ def create_ui():
|
|
556 |
submit_button_click.click(submit_caption,
|
557 |
[origin_image, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt],
|
558 |
[chatbot, state, click_state, image_input])
|
559 |
-
|
560 |
image_input.clear(
|
561 |
lambda: (None, [], [], [[], [], []], "", "", ""),
|
562 |
[],
|
|
|
187 |
|
188 |
click_state[0] = input_points
|
189 |
click_state[1] = input_labels
|
190 |
+
state = state + [("Image point: {}, Input label: {}".format(click_state[0], click_state[1]), None)]
|
191 |
|
192 |
return state, click_state
|
193 |
|
194 |
def submit_caption(image_input, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt):
|
195 |
+
coordinates = json.dumps(list(zip(click_state[0], click_state[1])))
|
196 |
+
prompt = get_click_prompt(coordinates, click_state, 'Single')
|
197 |
input_points = prompt['input_point']
|
198 |
input_labels = prompt['input_label']
|
199 |
|
|
|
217 |
enable_wiki = True if enable_wiki in ['True', 'TRUE', 'true', True, 'Yes', 'YES', 'yes'] else False
|
218 |
out = model.inference(image_input, prompt, controls, disable_gpt=True, enable_wiki=enable_wiki, verbose=True, args={'clip_filter': False})[0]
|
219 |
|
|
|
220 |
state = state + [(None, "raw_caption: {}".format(out['generated_captions']['raw_caption']))]
|
221 |
+
|
222 |
+
update_click_state(click_state, out['generated_captions']['raw_caption'], 'Single')
|
223 |
text = out['generated_captions']['raw_caption']
|
224 |
input_mask = np.array(out['mask'].convert('P'))
|
225 |
image_input = mask_painter(np.array(image_input), input_mask)
|
226 |
origin_image_input = image_input
|
227 |
image_input = create_bubble_frame(image_input, text, (input_points[-1][0], input_points[-1][1]), input_mask,
|
228 |
input_points=input_points, input_labels=input_labels)
|
229 |
+
|
230 |
if visual_chatgpt is not None:
|
231 |
new_image_path = get_new_image_name('chat_image', func_name='upload')
|
232 |
image_input.save(new_image_path)
|
|
|
248 |
input_mask,
|
249 |
input_points=input_points, input_labels=input_labels)
|
250 |
return state, state, click_state, refined_image_input
|
251 |
+
|
252 |
return state, state, click_state, image_input
|
253 |
|
254 |
+
|
255 |
def get_sketch_prompt(mask: Image.Image):
|
256 |
"""
|
257 |
Get the prompt for the sketcher.
|
|
|
559 |
submit_button_click.click(submit_caption,
|
560 |
[origin_image, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt],
|
561 |
[chatbot, state, click_state, image_input])
|
562 |
+
|
563 |
image_input.clear(
|
564 |
lambda: (None, [], [], [[], [], []], "", "", ""),
|
565 |
[],
|