whyumesh commited on
Commit
a40894f
·
verified ·
1 Parent(s): c2ed638

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -47
app.py CHANGED
@@ -39,7 +39,7 @@ VISION_SYSTEM_PROMPT = """You are an AI assistant specialized in analyzing image
39
  2. Identify any error messages, warnings, or highlighting that indicates bugs
40
  3. Describe the programming language and context if visible.
41
  Be thorough and accurate in your description, as this will be used to fix the code.
42
- Note: In video, irrelevent frames may be occur (eg. other windows tabs, eterniq website, etc.) in video. please focus on code specific frames as we have to extract that content only.
43
  """
44
 
45
  CODE_SYSTEM_PROMPT = """You are an expert code debugging assistant. Based on the description of code and errors provided, your task is to:
@@ -50,14 +50,49 @@ Be thorough in your explanation and ensure the corrected code is complete and fu
50
  Note: Please provide the output in a well-structured Markdown format. Remove all unnecessary information and exclude any additional code formatting such as triple backticks or language identifiers. The response should be ready to be rendered as Markdown content.
51
  """
52
 
53
- def process_image_for_code(image):
54
- # First, process with vision model
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  vision_messages = [
56
  {
57
  "role": "user",
58
  "content": [
59
  {"type": "image", "image": image},
60
- {"type": "text", "text": f"{VISION_SYSTEM_PROMPT}\n\nDescribe the code and any errors you see in this image."},
61
  ],
62
  }
63
  ]
@@ -82,13 +117,13 @@ def process_image_for_code(image):
82
  vision_output_trimmed = [
83
  out_ids[len(in_ids):] for in_ids, out_ids in zip(vision_inputs.input_ids, vision_output_ids)
84
  ]
85
- vision_description = vision_processor.batch_decode(
86
  vision_output_trimmed,
87
  skip_special_tokens=True,
88
  clean_up_tokenization_spaces=False
89
  )[0]
90
 
91
- # Then, use code model to fix the code
92
  code_messages = [
93
  {"role": "system", "content": CODE_SYSTEM_PROMPT},
94
  {"role": "user", "content": f"Here's a description of code with errors:\n\n{vision_description}\n\nPlease analyze and fix the code."}
@@ -113,63 +148,32 @@ def process_image_for_code(image):
113
  code_output_trimmed = [
114
  out_ids[len(in_ids):] for in_ids, out_ids in zip(code_inputs.input_ids, code_output_ids)
115
  ]
116
- fixed_code_response = code_tokenizer.batch_decode(
117
  code_output_trimmed,
118
  skip_special_tokens=True
119
  )[0]
120
-
121
- return vision_description, fixed_code_response
122
-
123
- def process_video_for_code(video_path, max_frames=16, frame_interval=30):
124
- cap = cv2.VideoCapture(video_path)
125
- frames = []
126
- frame_count = 0
127
-
128
- while len(frames) < max_frames:
129
- ret, frame = cap.read()
130
- if not ret:
131
- break
132
-
133
- if frame_count % frame_interval == 0:
134
- frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
135
- frame = Image.fromarray(frame)
136
- frames.append(frame)
137
-
138
- frame_count += 1
139
-
140
- cap.release()
141
-
142
- # Process the first frame for now (you could extend this to handle multiple frames)
143
- if frames:
144
- return process_image_for_code(frames[0])
145
- else:
146
- return "No frames could be extracted from the video.", "No code could be analyzed."
147
 
148
  @spaces.GPU
149
- def process_content(content):
150
- if content is None:
151
- return "Please upload an image or video file of code with errors.", ""
152
-
153
- if content.name.lower().endswith(('.png', '.jpg', '.jpeg')):
154
- image = Image.open(content.name)
155
- vision_output, code_output = process_image_for_code(image)
156
- elif content.name.lower().endswith(('.mp4', '.avi', '.mov')):
157
- vision_output, code_output = process_video_for_code(content.name)
158
- else:
159
- return "Unsupported file type. Please provide an image or video file.", ""
160
 
 
161
  return vision_output, code_output
162
 
163
  # Gradio interface
164
  iface = gr.Interface(
165
  fn=process_content,
166
- inputs=gr.File(label="Upload Image or Video of Code with Errors"),
 
 
 
167
  outputs=[
168
  gr.Textbox(label="Vision Model Output (Code Description)"),
169
  gr.Code(label="Fixed Code", language="python")
170
  ],
171
  title="Vision Code Debugger",
172
- description="Upload an image or video of code with errors, and the AI will analyze and fix the issues."
173
  )
174
 
175
  if __name__ == "__main__":
 
39
  2. Identify any error messages, warnings, or highlighting that indicates bugs
40
  3. Describe the programming language and context if visible.
41
  Be thorough and accurate in your description, as this will be used to fix the code.
42
+ Note: In video, irrelevant frames may occur (e.g., other windows tabs, eterniq website, etc.) in video. Please focus on code-specific frames as we have to extract that content only.
43
  """
44
 
45
  CODE_SYSTEM_PROMPT = """You are an expert code debugging assistant. Based on the description of code and errors provided, your task is to:
 
50
  Note: Please provide the output in a well-structured Markdown format. Remove all unnecessary information and exclude any additional code formatting such as triple backticks or language identifiers. The response should be ready to be rendered as Markdown content.
51
  """
52
 
53
+ def process_video_for_code(video_path, transcribed_text, max_frames=16, frame_interval=30):
54
+ cap = cv2.VideoCapture(video_path)
55
+ frames = []
56
+ frame_count = 0
57
+
58
+ while len(frames) < max_frames:
59
+ ret, frame = cap.read()
60
+ if not ret:
61
+ break
62
+
63
+ if frame_count % frame_interval == 0:
64
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
65
+ frame = Image.fromarray(frame)
66
+ frames.append(frame)
67
+
68
+ frame_count += 1
69
+
70
+ cap.release()
71
+
72
+ if not frames:
73
+ return "No frames could be extracted from the video.", "No code could be analyzed."
74
+
75
+ # Process all frames
76
+ vision_descriptions = []
77
+ for frame in frames:
78
+ vision_description = process_image_for_vision(frame, transcribed_text)
79
+ vision_descriptions.append(vision_description)
80
+
81
+ # Combine all vision descriptions
82
+ combined_vision_description = "\n\n".join(vision_descriptions)
83
+
84
+ # Use code model to fix the code based on combined description
85
+ fixed_code_response = process_for_code(combined_vision_description)
86
+
87
+ return combined_vision_description, fixed_code_response
88
+
89
+ def process_image_for_vision(image, transcribed_text):
90
  vision_messages = [
91
  {
92
  "role": "user",
93
  "content": [
94
  {"type": "image", "image": image},
95
+ {"type": "text", "text": f"{VISION_SYSTEM_PROMPT}\n\nDescribe the code and any errors you see in this image. User's description: {transcribed_text}"},
96
  ],
97
  }
98
  ]
 
117
  vision_output_trimmed = [
118
  out_ids[len(in_ids):] for in_ids, out_ids in zip(vision_inputs.input_ids, vision_output_ids)
119
  ]
120
+ return vision_processor.batch_decode(
121
  vision_output_trimmed,
122
  skip_special_tokens=True,
123
  clean_up_tokenization_spaces=False
124
  )[0]
125
 
126
+ def process_for_code(vision_description):
127
  code_messages = [
128
  {"role": "system", "content": CODE_SYSTEM_PROMPT},
129
  {"role": "user", "content": f"Here's a description of code with errors:\n\n{vision_description}\n\nPlease analyze and fix the code."}
 
148
  code_output_trimmed = [
149
  out_ids[len(in_ids):] for in_ids, out_ids in zip(code_inputs.input_ids, code_output_ids)
150
  ]
151
+ return code_tokenizer.batch_decode(
152
  code_output_trimmed,
153
  skip_special_tokens=True
154
  )[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
 
156
  @spaces.GPU
157
+ def process_content(video, transcribed_text):
158
+ if video is None:
159
+ return "Please upload a video file of code with errors.", ""
 
 
 
 
 
 
 
 
160
 
161
+ vision_output, code_output = process_video_for_code(video.name, transcribed_text)
162
  return vision_output, code_output
163
 
164
  # Gradio interface
165
  iface = gr.Interface(
166
  fn=process_content,
167
+ inputs=[
168
+ gr.File(label="Upload Video of Code with Errors"),
169
+ gr.Textbox(label="Transcribed Audio")
170
+ ],
171
  outputs=[
172
  gr.Textbox(label="Vision Model Output (Code Description)"),
173
  gr.Code(label="Fixed Code", language="python")
174
  ],
175
  title="Vision Code Debugger",
176
+ description="Upload a video of code with errors and provide transcribed audio, and the AI will analyze and fix the issues."
177
  )
178
 
179
  if __name__ == "__main__":