Update app.py

app.py

@@ -1,34 +1,46 @@
 import gradio as gr
-from PIL import Image, ImageDraw,ImageFont
+from PIL import Image, ImageDraw, ImageFont
 import scipy.io.wavfile as wavfile
 
 
 # Use a pipeline as a high-level helper
 from transformers import pipeline
 
-model_path = ("../Model/models--facebook--detr-resnet-50/snapshots"
-              "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
+# model_path = ("../Models/models--facebook--detr-resnet-50/snapshots"
+#               "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
+#
+# tts_model_path = ("../Models/models--kakao-enterprise--vits-ljs/snapshots"
+#                   "/3bcb8321394f671bd948ebf0d086d694dda95464")
 
-tts_model_path = ("../Model/models--kakao-enterprise--vits-ljs/snapshots"
-                  "/3bcb8321394f671bd948ebf0d086d694dda95464")
 
+narrator = pipeline("text-to-speech",
+                    model="kakao-enterprise/vits-ljs")
 
-narrator = pipeline("text-to-speech", model="kakao-enterprise/vits-ljs")
+object_detector = pipeline("object-detection",
+                model="facebook/detr-resnet-50")
 
-object_detector = pipeline("object-detection", model="facebook/detr-resnet-50")
-
-
-# object_detector = pipeline("object-detection", model=model_path)
-# narrator = pipeline("text-to-speech", model=tts_model_path)
+# object_detector = pipeline("object-detection",
+#                 model=model_path)
+#
+# narrator = pipeline("text-to-speech",
+#                     model=tts_model_path)
 
+# [{'score': 0.9996405839920044, 'label': 'person', 'box': {'xmin': 435, 'ymin': 282, 'xmax': 636, 'ymax': 927}}, {'score': 0.9995879530906677, 'label': 'dog', 'box': {'xmin': 570, 'ymin': 694, 'xmax': 833, 'ymax': 946}}]
 
+# Define the function to generate audio from text
 def generate_audio(text):
+    # Generate the narrated text
     narrated_text = narrator(text)
-    wavfile.write("finetuned_output.wav",
-                  rate=narrated_text["sampling_rate"],
+
+    # Save the audio to a WAV file
+    wavfile.write("output.wav", rate=narrated_text["sampling_rate"],
                   data=narrated_text["audio"][0])
-    return "finetuned_output.wav";
 
+    # Return the path to the saved audio file
+    return "output.wav"
+
+# Could you please write me a python code that will take list of detection object as an input and it will give the response that will include all the objects (labels) provided in the input. For example if the input is like this: [{'score': 0.9996405839920044, 'label': 'person', 'box': {'xmin': 435, 'ymin': 282, 'xmax': 636, 'ymax': 927}}, {'score': 0.9995879530906677, 'label': 'dog', 'box': {'xmin': 570, 'ymin': 694, 'xmax': 833, 'ymax': 946}}]
+# The output should be, This pictuture contains 1 person and 1 dog. If there are multiple objects, do not add 'and' between every objects but 'and' should be at the end only
 
 
 def read_objects(detection_objects):
@@ -56,56 +68,79 @@ def read_objects(detection_objects):
             response += " and"
 
     response += "."
+
     return response
 
 
 
-def draw_bounding_boxes(image, detection_results):
+def draw_bounding_boxes(image, detections, font_path=None, font_size=20):
     """
-    Draws bounding boxes on the provided image based on the detection results.
-
-    Parameters:
-        image (PIL.Image): The input image to be annotated.
-        detection_results (list): A list of dictionaries, each containing the detected object details.
-
-    Returns:
-        PIL.Image: The image with bounding boxes drawn around the detected objects.
+    Draws bounding boxes on the given image based on the detections.
+
+    :param image: PIL.Image object
+    :param detections: List of detection results, where each result is a dictionary containing
+                       'score', 'label', and 'box' keys. 'box' itself is a dictionary with 'xmin',
+                       'ymin', 'xmax', 'ymax'.
+    :param font_path: Path to the TrueType font file to use for text.
+    :param font_size: Size of the font to use for text.
+    :return: PIL.Image object with bounding boxes drawn.
     """
-    # Convert the input image to ImageDraw object to draw on it
-    draw = ImageDraw.Draw(image)
-
-    # Iterate through each detection result
-    for result in detection_results:
-        # Extract the bounding box coordinates and label
-        box = result['box']
-        label = result['label']
-        score = result['score']
+    # Make a copy of the image to draw on
+    draw_image = image.copy()
+    draw = ImageDraw.Draw(draw_image)
+
+    # Load custom font or default font if path not provided
+    if font_path:
+        font = ImageFont.truetype(font_path, font_size)
+    else:
+        # When font_path is not provided, load default font but it's size is fixed
+        font = ImageFont.load_default()
+        # Increase font size workaround by using a TTF font file, if needed, can download and specify the path
+
+    for detection in detections:
+        box = detection['box']
+        xmin = box['xmin']
+        ymin = box['ymin']
+        xmax = box['xmax']
+        ymax = box['ymax']
+
+        # Draw the bounding box
+        draw.rectangle([(xmin, ymin), (xmax, ymax)], outline="red", width=3)
+
+        # Optionally, you can also draw the label and score
+        label = detection['label']
+        score = detection['score']
+        text = f"{label} {score:.2f}"
 
-        # Define coordinates for the bounding box
-        xmin, ymin, xmax, ymax = box['xmin'], box['ymin'], box['xmax'], box['ymax']
+        # Draw text with background rectangle for visibility
+        if font_path:  # Use the custom font with increased size
+            text_size = draw.textbbox((xmin, ymin), text, font=font)
+        else:
+            # Calculate text size using the default font
+            text_size = draw.textbbox((xmin, ymin), text)
 
-        # Draw the bounding box (with a red outline)
-        draw.rectangle([xmin, ymin, xmax, ymax], outline="red", width=3)
+        draw.rectangle([(text_size[0], text_size[1]), (text_size[2], text_size[3])], fill="red")
+        draw.text((xmin, ymin), text, fill="white", font=font)
 
-        # Optionally, add label with score near the bounding box
-        text = f"{label} ({score * 100:.1f}%)"
-        draw.text((xmin, ymin - 10), text, fill="red")
+    return draw_image
 
-    return image
 
-def detect_objects(image):
+def detect_object(image):
     raw_image = image
     output = object_detector(raw_image)
     processed_image = draw_bounding_boxes(raw_image, output)
-    naturalized_text = read_objects(output)
-    processed_audio = generate_audio(naturalized_text)
+    natural_text = read_objects(output)
+    processed_audio = generate_audio(natural_text)
     return processed_image, processed_audio
 
 
-
-demo = gr.Interface(fn = detect_objects,
+demo = gr.Interface(fn=detect_object,
                     inputs=[gr.Image(label="Select Image",type="pil")],
-                    outputs=[gr.Image(label="Summarized Text ",type="pil"), gr.Audio(label="Generated Audio")],
-                    title="@SherryAhuja Project : Object Detection with Audio",
-                    description="This AI application will be used to Detect objects in an image and generate audio.",)
-demo.launch()
+                    outputs=[gr.Image(label="Processed Image", type="pil"), gr.Audio(label="Generated Audio")],
+                    title="@GenAILearniverse Project 7: Object Detector with Audio",
+                    description="THIS APPLICATION WILL BE USED TO HIGHLIGHT OBJECTS AND GIVES AUDIO DESCRIPTION FOR THE PROVIDED INPUT IMAGE.")
+demo.launch()
+
+# print(output)
+
+