Update app.py
Browse files
app.py
CHANGED
@@ -39,7 +39,7 @@ VISION_SYSTEM_PROMPT = """You are an AI assistant specialized in analyzing image
|
|
39 |
2. Identify any error messages, warnings, or highlighting that indicates bugs
|
40 |
3. Describe the programming language and context if visible.
|
41 |
Be thorough and accurate in your description, as this will be used to fix the code.
|
42 |
-
Note: In video,
|
43 |
"""
|
44 |
|
45 |
CODE_SYSTEM_PROMPT = """You are an expert code debugging assistant. Based on the description of code and errors provided, your task is to:
|
@@ -50,14 +50,49 @@ Be thorough in your explanation and ensure the corrected code is complete and fu
|
|
50 |
Note: Please provide the output in a well-structured Markdown format. Remove all unnecessary information and exclude any additional code formatting such as triple backticks or language identifiers. The response should be ready to be rendered as Markdown content.
|
51 |
"""
|
52 |
|
53 |
-
def
|
54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
vision_messages = [
|
56 |
{
|
57 |
"role": "user",
|
58 |
"content": [
|
59 |
{"type": "image", "image": image},
|
60 |
-
{"type": "text", "text": f"{VISION_SYSTEM_PROMPT}\n\nDescribe the code and any errors you see in this image."},
|
61 |
],
|
62 |
}
|
63 |
]
|
@@ -82,13 +117,13 @@ def process_image_for_code(image):
|
|
82 |
vision_output_trimmed = [
|
83 |
out_ids[len(in_ids):] for in_ids, out_ids in zip(vision_inputs.input_ids, vision_output_ids)
|
84 |
]
|
85 |
-
|
86 |
vision_output_trimmed,
|
87 |
skip_special_tokens=True,
|
88 |
clean_up_tokenization_spaces=False
|
89 |
)[0]
|
90 |
|
91 |
-
|
92 |
code_messages = [
|
93 |
{"role": "system", "content": CODE_SYSTEM_PROMPT},
|
94 |
{"role": "user", "content": f"Here's a description of code with errors:\n\n{vision_description}\n\nPlease analyze and fix the code."}
|
@@ -113,63 +148,32 @@ def process_image_for_code(image):
|
|
113 |
code_output_trimmed = [
|
114 |
out_ids[len(in_ids):] for in_ids, out_ids in zip(code_inputs.input_ids, code_output_ids)
|
115 |
]
|
116 |
-
|
117 |
code_output_trimmed,
|
118 |
skip_special_tokens=True
|
119 |
)[0]
|
120 |
-
|
121 |
-
return vision_description, fixed_code_response
|
122 |
-
|
123 |
-
def process_video_for_code(video_path, max_frames=16, frame_interval=30):
|
124 |
-
cap = cv2.VideoCapture(video_path)
|
125 |
-
frames = []
|
126 |
-
frame_count = 0
|
127 |
-
|
128 |
-
while len(frames) < max_frames:
|
129 |
-
ret, frame = cap.read()
|
130 |
-
if not ret:
|
131 |
-
break
|
132 |
-
|
133 |
-
if frame_count % frame_interval == 0:
|
134 |
-
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
135 |
-
frame = Image.fromarray(frame)
|
136 |
-
frames.append(frame)
|
137 |
-
|
138 |
-
frame_count += 1
|
139 |
-
|
140 |
-
cap.release()
|
141 |
-
|
142 |
-
# Process the first frame for now (you could extend this to handle multiple frames)
|
143 |
-
if frames:
|
144 |
-
return process_image_for_code(frames[0])
|
145 |
-
else:
|
146 |
-
return "No frames could be extracted from the video.", "No code could be analyzed."
|
147 |
|
148 |
@spaces.GPU
|
149 |
-
def process_content(
|
150 |
-
if
|
151 |
-
return "Please upload
|
152 |
-
|
153 |
-
if content.name.lower().endswith(('.png', '.jpg', '.jpeg')):
|
154 |
-
image = Image.open(content.name)
|
155 |
-
vision_output, code_output = process_image_for_code(image)
|
156 |
-
elif content.name.lower().endswith(('.mp4', '.avi', '.mov')):
|
157 |
-
vision_output, code_output = process_video_for_code(content.name)
|
158 |
-
else:
|
159 |
-
return "Unsupported file type. Please provide an image or video file.", ""
|
160 |
|
|
|
161 |
return vision_output, code_output
|
162 |
|
163 |
# Gradio interface
|
164 |
iface = gr.Interface(
|
165 |
fn=process_content,
|
166 |
-
inputs=
|
|
|
|
|
|
|
167 |
outputs=[
|
168 |
gr.Textbox(label="Vision Model Output (Code Description)"),
|
169 |
gr.Code(label="Fixed Code", language="python")
|
170 |
],
|
171 |
title="Vision Code Debugger",
|
172 |
-
description="Upload
|
173 |
)
|
174 |
|
175 |
if __name__ == "__main__":
|
|
|
39 |
2. Identify any error messages, warnings, or highlighting that indicates bugs
|
40 |
3. Describe the programming language and context if visible.
|
41 |
Be thorough and accurate in your description, as this will be used to fix the code.
|
42 |
+
Note: In video, irrelevant frames may occur (e.g., other windows tabs, eterniq website, etc.) in video. Please focus on code-specific frames as we have to extract that content only.
|
43 |
"""
|
44 |
|
45 |
CODE_SYSTEM_PROMPT = """You are an expert code debugging assistant. Based on the description of code and errors provided, your task is to:
|
|
|
50 |
Note: Please provide the output in a well-structured Markdown format. Remove all unnecessary information and exclude any additional code formatting such as triple backticks or language identifiers. The response should be ready to be rendered as Markdown content.
|
51 |
"""
|
52 |
|
53 |
+
def process_video_for_code(video_path, transcribed_text, max_frames=16, frame_interval=30):
|
54 |
+
cap = cv2.VideoCapture(video_path)
|
55 |
+
frames = []
|
56 |
+
frame_count = 0
|
57 |
+
|
58 |
+
while len(frames) < max_frames:
|
59 |
+
ret, frame = cap.read()
|
60 |
+
if not ret:
|
61 |
+
break
|
62 |
+
|
63 |
+
if frame_count % frame_interval == 0:
|
64 |
+
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
65 |
+
frame = Image.fromarray(frame)
|
66 |
+
frames.append(frame)
|
67 |
+
|
68 |
+
frame_count += 1
|
69 |
+
|
70 |
+
cap.release()
|
71 |
+
|
72 |
+
if not frames:
|
73 |
+
return "No frames could be extracted from the video.", "No code could be analyzed."
|
74 |
+
|
75 |
+
# Process all frames
|
76 |
+
vision_descriptions = []
|
77 |
+
for frame in frames:
|
78 |
+
vision_description = process_image_for_vision(frame, transcribed_text)
|
79 |
+
vision_descriptions.append(vision_description)
|
80 |
+
|
81 |
+
# Combine all vision descriptions
|
82 |
+
combined_vision_description = "\n\n".join(vision_descriptions)
|
83 |
+
|
84 |
+
# Use code model to fix the code based on combined description
|
85 |
+
fixed_code_response = process_for_code(combined_vision_description)
|
86 |
+
|
87 |
+
return combined_vision_description, fixed_code_response
|
88 |
+
|
89 |
+
def process_image_for_vision(image, transcribed_text):
|
90 |
vision_messages = [
|
91 |
{
|
92 |
"role": "user",
|
93 |
"content": [
|
94 |
{"type": "image", "image": image},
|
95 |
+
{"type": "text", "text": f"{VISION_SYSTEM_PROMPT}\n\nDescribe the code and any errors you see in this image. User's description: {transcribed_text}"},
|
96 |
],
|
97 |
}
|
98 |
]
|
|
|
117 |
vision_output_trimmed = [
|
118 |
out_ids[len(in_ids):] for in_ids, out_ids in zip(vision_inputs.input_ids, vision_output_ids)
|
119 |
]
|
120 |
+
return vision_processor.batch_decode(
|
121 |
vision_output_trimmed,
|
122 |
skip_special_tokens=True,
|
123 |
clean_up_tokenization_spaces=False
|
124 |
)[0]
|
125 |
|
126 |
+
def process_for_code(vision_description):
|
127 |
code_messages = [
|
128 |
{"role": "system", "content": CODE_SYSTEM_PROMPT},
|
129 |
{"role": "user", "content": f"Here's a description of code with errors:\n\n{vision_description}\n\nPlease analyze and fix the code."}
|
|
|
148 |
code_output_trimmed = [
|
149 |
out_ids[len(in_ids):] for in_ids, out_ids in zip(code_inputs.input_ids, code_output_ids)
|
150 |
]
|
151 |
+
return code_tokenizer.batch_decode(
|
152 |
code_output_trimmed,
|
153 |
skip_special_tokens=True
|
154 |
)[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
155 |
|
156 |
@spaces.GPU
|
157 |
+
def process_content(video, transcribed_text):
|
158 |
+
if video is None:
|
159 |
+
return "Please upload a video file of code with errors.", ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
160 |
|
161 |
+
vision_output, code_output = process_video_for_code(video.name, transcribed_text)
|
162 |
return vision_output, code_output
|
163 |
|
164 |
# Gradio interface
|
165 |
iface = gr.Interface(
|
166 |
fn=process_content,
|
167 |
+
inputs=[
|
168 |
+
gr.File(label="Upload Video of Code with Errors"),
|
169 |
+
gr.Textbox(label="Transcribed Audio")
|
170 |
+
],
|
171 |
outputs=[
|
172 |
gr.Textbox(label="Vision Model Output (Code Description)"),
|
173 |
gr.Code(label="Fixed Code", language="python")
|
174 |
],
|
175 |
title="Vision Code Debugger",
|
176 |
+
description="Upload a video of code with errors and provide transcribed audio, and the AI will analyze and fix the issues."
|
177 |
)
|
178 |
|
179 |
if __name__ == "__main__":
|