deployment/Spark_Deployment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mM4DvqTJGzwt",
    "outputId": "f9e04f60-2edd-4c55-962b-b18f5dd37001"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rQkjmqCqRlVs"
   },
   "outputs": [],
   "source": [
    "!sudo apt update && sudo apt install ffmpeg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6qpSdD_WSiC3",
    "outputId": "bd8400dc-244a-4529-902e-8838775f431c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'ml-nlgma-body-cam'...\n",
      "remote: Enumerating objects: 395, done.\u001b[K\n",
      "remote: Counting objects: 100% (292/292), done.\u001b[K\n",
      "remote: Compressing objects: 100% (239/239), done.\u001b[K\n",
      "remote: Total 395 (delta 77), reused 224 (delta 47), pack-reused 103\u001b[K\n",
      "Receiving objects: 100% (395/395), 9.15 MiB | 6.64 MiB/s, done.\n",
      "Resolving deltas: 100% (117/117), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/k-sashank/ml-nlgma-body-cam.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "S1z-orpESvt7",
    "outputId": "5a2fdae3-455a-4234-e4cb-da7578ca8a6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/ml-nlgma-body-cam/deployment\n"
     ]
    }
   ],
   "source": [
    "cd ml-nlgma-body-cam/deployment/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "byk_2SlPsWAZ",
    "outputId": "6286e21b-41ce-40cc-861b-022f6275f4a5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: opencv-python in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 1)) (4.8.0.76)\n",
      "Requirement already satisfied: ultralytics in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (8.0.219)\n",
      "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (2.1.0+cu118)\n",
      "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 4)) (4.35.2)\n",
      "Requirement already satisfied: Cython in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (3.0.5)\n",
      "Requirement already satisfied: wget in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (3.2)\n",
      "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (0.24.1)\n",
      "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (1.3.5)\n",
      "Requirement already satisfied: openai-whisper in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (20231117)\n",
      "Requirement already satisfied: numpy>=1.21.2 in /usr/local/lib/python3.10/dist-packages (from opencv-python->-r requirements.txt (line 1)) (1.23.5)\n",
      "Requirement already satisfied: matplotlib>=3.3.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (3.7.1)\n",
      "Requirement already satisfied: pillow>=7.1.2 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (9.4.0)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (6.0.1)\n",
      "Requirement already satisfied: requests>=2.23.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (2.31.0)\n",
      "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (1.11.3)\n",
      "Requirement already satisfied: torchvision>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (0.16.0+cu118)\n",
      "Requirement already satisfied: tqdm>=4.64.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (4.66.1)\n",
      "Requirement already satisfied: pandas>=1.1.4 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (1.5.3)\n",
      "Requirement already satisfied: seaborn>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (0.12.2)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (5.9.5)\n",
      "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (9.0.0)\n",
      "Requirement already satisfied: thop>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics->-r requirements.txt (line 2)) (0.1.1.post2209072238)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (4.8.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (2023.6.0)\n",
      "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 3)) (2.1.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 4)) (0.19.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 4)) (23.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 4)) (2023.6.3)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 4)) (0.15.0)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 4)) (0.4.0)\n",
      "Requirement already satisfied: anyio<4,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 8)) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai->-r requirements.txt (line 8)) (1.7.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 8)) (0.25.2)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 8)) (2.5.2)\n",
      "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from openai-whisper->-r requirements.txt (line 9)) (0.58.1)\n",
      "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from openai-whisper->-r requirements.txt (line 9)) (10.1.0)\n",
      "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from openai-whisper->-r requirements.txt (line 9)) (0.5.1)\n",
      "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.5.0->openai->-r requirements.txt (line 8)) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.5.0->openai->-r requirements.txt (line 8)) (1.3.0)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4,>=3.5.0->openai->-r requirements.txt (line 8)) (1.1.3)\n",
      "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai->-r requirements.txt (line 8)) (2023.7.22)\n",
      "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai->-r requirements.txt (line 8)) (1.0.2)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai->-r requirements.txt (line 8)) (0.14.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (4.44.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (1.4.5)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.4->ultralytics->-r requirements.txt (line 2)) (2023.3.post1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 8)) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.14.5 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 8)) (2.14.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics->-r requirements.txt (line 2)) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics->-r requirements.txt (line 2)) (2.0.7)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->-r requirements.txt (line 3)) (2.1.3)\n",
      "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->openai-whisper->-r requirements.txt (line 9)) (0.41.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->-r requirements.txt (line 3)) (1.3.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=3.3.0->ultralytics->-r requirements.txt (line 2)) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "3ZrCX53_ssHy"
   },
   "outputs": [],
   "source": [
    "import whisper\n",
    "import cv2\n",
    "import os\n",
    "import urllib.request\n",
    "from PIL import Image\n",
    "from ultralytics import YOLO\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from transformers import pipeline\n",
    "import moviepy.editor as mp\n",
    "import json\n",
    "import re\n",
    "import gradio as gr\n",
    "from openai import OpenAI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lmWMJ_FeJDyD"
   },
   "source": [
    "# Full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "YMDeEOwLtGYN"
   },
   "outputs": [],
   "source": [
    "def video_transcription(video_path):\n",
    "  try:\n",
    "    model = whisper.load_model('large-v3')\n",
    "    transcript = model.transcribe(video_path, verbose = True)\n",
    "\n",
    "    # JSON Dump (Find a way to not create a file and just dump into a variable or something)\n",
    "    #json_file_path = video_path.split('/')[-1][:-4]+ \".json\"\n",
    "    #with open(json_file_path, 'w') as json_file:\n",
    "    #json.dump(transcript, json_file, indent = 2)\n",
    "    #json_file_path = video_path.split('/')[-1][:-4]+ \".json\"\n",
    "    #with open(json_file_path, 'w') as json_file:\n",
    "    return json.dump(transcript)\n",
    "\n",
    "  except Exception as e:\n",
    "    return e\n",
    "\n",
    "def action_detection(transcript, openai_key):\n",
    "  try:\n",
    "    # JSON Dump (Find a way to not create a file and just dump into a variable or something)\n",
    "    #with open(json_path, 'r') as f:\n",
    "    #  transcript = json.load(f)\n",
    "    transcript_string = ''\n",
    "    for segments in transcript['segments']:\n",
    "      transcript_string+=str(segments['id'])+str(segments['text']+'\\n')\n",
    "\n",
    "    client = OpenAI(api_key = openai_key)\n",
    "\n",
    "    completion = client.chat.completions.create(\n",
    "      model=\"gpt-3.5-turbo-1106\",\n",
    "      messages=[\n",
    "        {\"role\": \"system\", \"content\": f\"Given this {transcript_string} You are an AI system specialized in detecting planning issues, critiquing plans, and analyzing conversations between people regarding how to disperse. Additionally, identify any instances suggesting 1st Amendment violations or officers expressing the belief that this protest was anti-police. Finally, flag any aggressive comments found in the audio transcript.\"},\n",
    "        {\"role\": \"user\", \"content\":\"Give responce like this following examples: Sentence: '18: What do you got?' Explanation: This sentence may indicate confusion or a need for clarification, as the speaker is asking for information. It could potentially be a planning issue if the speaker is seeking information to execute a specific task.\"}\n",
    "      ]\n",
    "    )\n",
    "\n",
    "    output = completion.choices[0].message.content\n",
    "\n",
    "    paragraphs = re.split(r'\\n\\n', output)\n",
    "\n",
    "    sentences = []\n",
    "    explanations = []\n",
    "\n",
    "    for paragraph in paragraphs:\n",
    "        sentence_match = re.search(r\"Sentence: '(.+)'\", paragraph)\n",
    "        explanation_match = re.search(r\"Explanation: (.+)\", paragraph)\n",
    "\n",
    "        if sentence_match and explanation_match:\n",
    "            sentences.append(sentence_match.group(1).split(': ')[-1])\n",
    "            explanations.append(explanation_match.group(1))\n",
    "\n",
    "    #for i in range(len(sentences)):\n",
    "    #    print(f\"Sentence: '{sentences[i]}'\")\n",
    "    #    print(f\"Explanation: {explanations[i]}\\n\")\n",
    "\n",
    "    for sentence_to_search in sentences:\n",
    "        pattern = re.compile(re.escape(sentence_to_search), re.IGNORECASE)\n",
    "\n",
    "        matching_entries = [entry for entry in transcript['segments'] if re.search(pattern, entry['text'])]\n",
    "\n",
    "        sent_with_time = []\n",
    "        if matching_entries:\n",
    "            for entry in matching_entries:\n",
    "                sent_with_time.append(sentence_to_search + ' Start Time: ', entry['start'] + ' End Time: ', entry['end'])\n",
    "\n",
    "    return sent_with_time\n",
    "\n",
    "  except Exception as e:\n",
    "    return e\n",
    "\n",
    "def process_video(video_path, weights):\n",
    "    try:\n",
    "        # This code cell detects batons in the video\n",
    "        current_frame = 0\n",
    "        model = YOLO(weights)\n",
    "        cap = cv2.VideoCapture(video_path)\n",
    "        fps = int(cap.get(cv2.CAP_PROP_FPS))\n",
    "        conseq_frames = 0\n",
    "        start_time = \"\"\n",
    "        end_time = \"\"\n",
    "        res = []\n",
    "\n",
    "        while True:\n",
    "            ret, frame = cap.read()\n",
    "            if not ret:\n",
    "                break\n",
    "\n",
    "            # Detecting baton on one frame per second\n",
    "            if current_frame % fps == 0:\n",
    "                currect_sec = current_frame/fps\n",
    "\n",
    "                # Model prediction on current frame\n",
    "                results = model(frame, verbose = False)\n",
    "                count = 0\n",
    "                classes = results[0].boxes.data\n",
    "\n",
    "                # Formatting the time for printing\n",
    "                hours, remainder = divmod(currect_sec, 3600)\n",
    "                minutes, seconds = divmod(remainder, 60)\n",
    "                hours = str(int(hours)).zfill(2)\n",
    "                minutes = str(int(minutes)).zfill(2)\n",
    "                seconds = str(int(seconds)).zfill(2)\n",
    "\n",
    "                for i in classes:\n",
    "\n",
    "                   # Checking if baton is detected (i.e. if the class corresponding to baton is 1 or not)\n",
    "                    if float(i[5]) == 1:\n",
    "                        count+=1\n",
    "\n",
    "                # Marking the start_time if this is the first consecutive frame a baton is detected in\n",
    "                if count >= 1:\n",
    "                    conseq_frames+=1\n",
    "                    if conseq_frames == 1:\n",
    "                        start_time = hours + \":\" + minutes + \":\" + seconds\n",
    "\n",
    "                # Marking the end time if after one or multiple consecutive frames of detection, a baton is not detected\n",
    "                else:\n",
    "                    if conseq_frames > 0:\n",
    "                        conseq_frames = 0\n",
    "                        end_time = hours + \":\" + minutes + \":\" + seconds\n",
    "\n",
    "                        # Printing time intervals in which baton was detected\n",
    "                        res.append(start_time + \" to \" + end_time)\n",
    "                        start_time = \"\"\n",
    "                        end_time = \"\"\n",
    "\n",
    "            current_frame += 1\n",
    "        cap.release()\n",
    "\n",
    "        return \"\\n\".join(res)\n",
    "\n",
    "    except Exception as e:\n",
    "\n",
    "        return e\n",
    "\n",
    "def all_funcs(openai_key,video_path, yolo_weights):\n",
    "  transcript = video_transcription(video_path)\n",
    "  sentences = action_detection(json.loads(transcript), openai_key)\n",
    "  batons = process_video(video_path, yolo_weights)\n",
    "\n",
    "  return sentences, batons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 645
    },
    "id": "ZbZKUNl3Mttf",
    "outputId": "bb7e6e43-1bce-4b35-da0e-49ef39093d16"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
      "\n",
      "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
      "Running on public URL: https://868bba52379b375588.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://868bba52379b375588.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "btn = gr.Interface(\n",
    "    fn=all_funcs,\n",
    "    inputs=[\"text\",gr.Files(label=\"Select Video File\"), gr.Files(label=\"Select YOLOv8 Weights File\")],\n",
    "    outputs=[gr.Textbox(label=\"Audio analysis time stamps\",lines=20), gr.Textbox(label=\"Baton detection timestamps\",lines=20)]\n",
    ")\n",
    "\n",
    "btn.launch()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NMF48OxVJHLp"
   },
   "source": [
    "# Baton Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8VSlkVeNJQo4"
   },
   "outputs": [],
   "source": [
    "def process_video(video_path, weights):\n",
    "    try:\n",
    "        # This code cell detects batons in the video\n",
    "        current_frame = 0\n",
    "        model = YOLO(weights)\n",
    "        cap = cv2.VideoCapture(video_path)\n",
    "        fps = int(cap.get(cv2.CAP_PROP_FPS))\n",
    "        conseq_frames = 0\n",
    "        start_time = \"\"\n",
    "        end_time = \"\"\n",
    "        res = []\n",
    "\n",
    "        while True:\n",
    "            ret, frame = cap.read()\n",
    "            if not ret:\n",
    "                break\n",
    "\n",
    "            # Detecting baton on one frame per second\n",
    "            if current_frame % fps == 0:\n",
    "                currect_sec = current_frame/fps\n",
    "\n",
    "                # Model prediction on current frame\n",
    "                results = model(frame, verbose = False)\n",
    "                count = 0\n",
    "                classes = results[0].boxes.data\n",
    "\n",
    "                # Formatting the time for printing\n",
    "                hours, remainder = divmod(currect_sec, 3600)\n",
    "                minutes, seconds = divmod(remainder, 60)\n",
    "                hours = str(int(hours)).zfill(2)\n",
    "                minutes = str(int(minutes)).zfill(2)\n",
    "                seconds = str(int(seconds)).zfill(2)\n",
    "\n",
    "                for i in classes:\n",
    "\n",
    "                   # Checking if baton is detected (i.e. if the class corresponding to baton is 1 or not)\n",
    "                    if float(i[5]) == 1:\n",
    "                        count+=1\n",
    "\n",
    "                # Marking the start_time if this is the first consecutive frame a baton is detected in\n",
    "                if count >= 1:\n",
    "                    conseq_frames+=1\n",
    "                    if conseq_frames == 1:\n",
    "                        start_time = hours + \":\" + minutes + \":\" + seconds\n",
    "\n",
    "                # Marking the end time if after one or multiple consecutive frames of detection, a baton is not detected\n",
    "                else:\n",
    "                    if conseq_frames > 0:\n",
    "                        conseq_frames = 0\n",
    "                        end_time = hours + \":\" + minutes + \":\" + seconds\n",
    "\n",
    "                        # Printing time intervals in which baton was detected\n",
    "                        res.append(start_time + \" to \" + end_time)\n",
    "                        start_time = \"\"\n",
    "                        end_time = \"\"\n",
    "\n",
    "            current_frame += 1\n",
    "        cap.release()\n",
    "\n",
    "        return \"\\n\".join(res)\n",
    "\n",
    "    except Exception as e:\n",
    "\n",
    "        return e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 626
    },
    "id": "uSBK_3VBJQmC",
    "outputId": "30c43f6f-72c7-4416-db52-741064696ed1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
      "\n",
      "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
      "Running on public URL: https://7b4faa7028f75417d8.gradio.live\n",
      "\n",
      "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"https://7b4faa7028f75417d8.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with gr.Blocks() as demo:\n",
    "\n",
    "    video_path = gr.Textbox(label = \"Enter Path to Video\")\n",
    "    #openai_keys = gr.Textbox(label = \"Enter your OpenAI Key\")\n",
    "    weights = gr.Textbox(label = \"Enter Path to YOLOv8 Weights\")\n",
    "    #sentences = gr.Textbox(label = \"Sentences Detected\")\n",
    "    batons = gr.Textbox(label = \"Batons Detected\")\n",
    "    btn = gr.Button(value = \"Process Video\")\n",
    "    btn.click(process_video, inputs = [video_path, weights], outputs = batons)\n",
    "\n",
    "demo.launch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Iqdjv7QowrZW"
   },
   "outputs": [],
   "source": [
    "/content/drive/MyDrive/Spark Project/Test_Video.mp4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "v9-4b-gfrbwa"
   },
   "outputs": [],
   "source": [
    "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"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nPh9wSJvwvAt"
   },
   "outputs": [],
   "source": [
    "/content/drive/MyDrive/Spark Project/Data (For YOLOv8 Training)/Option 3 - Roboflow (60 Images)/YOLOv8 Best Weights.pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CvuZJI3-LGOU"
   },
   "outputs": [],
   "source": [
    "process_video(\"/content/drive/MyDrive/Spark Project/Test_Video.mp4\", \"/content/drive/MyDrive/Spark Project/Data (For YOLOv8 Training)/Option 3 - Roboflow (60 Images)/YOLOv8 Best Weights.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "t7ZZYQp_tbN4"
   },
   "outputs": [],
   "source": [
    "a = video_transcription(\"/content/drive/MyDrive/Spark Project/Test_Video.mp4\")\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mtv7izc3HQHP",
    "outputId": "d7fed4ac-3d97-4580-bd21-f698f84f9615"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ultralytics\n",
      "  Downloading ultralytics-8.0.214-py3-none-any.whl (645 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m645.5/645.5 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: matplotlib>=3.3.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (3.7.1)\n",
      "Requirement already satisfied: numpy>=1.22.2 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (1.23.5)\n",
      "Requirement already satisfied: opencv-python>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (4.8.0.76)\n",
      "Requirement already satisfied: pillow>=7.1.2 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (9.4.0)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (6.0.1)\n",
      "Requirement already satisfied: requests>=2.23.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (2.31.0)\n",
      "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (1.11.3)\n",
      "Requirement already satisfied: torch>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (2.1.0+cu118)\n",
      "Requirement already satisfied: torchvision>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (0.16.0+cu118)\n",
      "Requirement already satisfied: tqdm>=4.64.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (4.66.1)\n",
      "Requirement already satisfied: pandas>=1.1.4 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (1.5.3)\n",
      "Requirement already satisfied: seaborn>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from ultralytics) (0.12.2)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from ultralytics) (5.9.5)\n",
      "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from ultralytics) (9.0.0)\n",
      "Collecting thop>=0.1.1 (from ultralytics)\n",
      "  Downloading thop-0.1.1.post2209072238-py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (4.44.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (23.2)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.3.0->ultralytics) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.1.4->ultralytics) (2023.3.post1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.23.0->ultralytics) (2023.7.22)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (3.13.1)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (4.5.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (3.2.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (2023.6.0)\n",
      "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->ultralytics) (2.1.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=3.3.0->ultralytics) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8.0->ultralytics) (2.1.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8.0->ultralytics) (1.3.0)\n",
      "Installing collected packages: thop, ultralytics\n",
      "Successfully installed thop-0.1.1.post2209072238 ultralytics-8.0.214\n"
     ]
    }
   ],
   "source": [
    "!pip install ultralytics"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}