initial upload

README.md

@@ -1,13 +1,11 @@
 ---
-title: Timesformer
-emoji: 🐨
-colorFrom: gray
-colorTo: purple
+title: Zero Shot Video Classification
+emoji: 👀
+colorFrom: blue
+colorTo: pink
 sdk: gradio
-sdk_version: 3.12.0
+sdk_version: 2.9.1
 app_file: app.py
 pinned: false
-license: openrail
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+license: apache-2.0
+---

app.py

@@ -0,0 +1,98 @@
+import os
+import torch
+import gradio as gr
+from video_transformers import VideoModel
+from utils import (
+    convert_frames_to_gif,
+    download_youtube_video,
+    sample_frames_from_video_file,
+)
+
+video_model = VideoModel.from_transformers("facebook/timesformer-base-finetuned-k400")
+
+
+examples = [
+    ["https://www.youtube.com/watch?v=huAJ9dC5lmI"],
+    ["https://www.youtube.com/watch?v=wvcWt6u5HTg"],
+    ["https://www.youtube.com/watch?v=-3kZSi5qjRM"],
+    ["https://www.youtube.com/watch?v=-6usjfP8hys"],
+    ["https://www.youtube.com/watch?v=B8OdMwVwyXc"],
+    ["https://www.youtube.com/watch?v=B9ea7YyCP6E"],
+    ["https://www.youtube.com/watch?v=BBkpaeJBKmk"],
+    ["https://www.youtube.com/watch?v=BBqU8Apee_g"],
+    ["https://www.youtube.com/watch?v=BDHub0gBGtc"],
+    ["https://www.youtube.com/watch?v=I7cwq6_4QtM"],
+    ["https://www.youtube.com/watch?v=Z0mJDXpNhYA"],
+    ["https://www.youtube.com/watch?v=QkQQjFGnZlg"],
+    ["https://www.youtube.com/watch?v=IQaoRUQif14"],
+]
+
+
+def predict(youtube_url):
+
+    video_path = download_youtube_video(youtube_url)
+
+    frames = sample_frames_from_video_file(video_path, num_frames=16)
+    gif_path = convert_frames_to_gif(frames)
+
+    result = video_model.predict(video_or_folder_path=video_path)
+
+    os.remove(video_path)
+
+    return result["predictions"], gif_path
+
+
+app = gr.Blocks()
+with app:
+    gr.Markdown("# **<p align='center'>Video Classification with Timesformer</p>**")
+    gr.Markdown(
+        """
+        <p style='text-align: center'>
+        Timesformer is a video model that uses a Transformer architecture to process video frames.
+        <br>It is released by Facebook AI Research in ICML 2021.
+        <br>This version is trained on Kinetics-400 dataset and can classify videos into 400 classes.
+        </p>
+        """
+    )
+    gr.Markdown(
+        """
+        <p style='text-align: center'>
+        Follow me for more! 
+        <br> <a href='https://twitter.com/fcakyon' target='_blank'>twitter</a> | <a href='https://github.com/fcakyon' target='_blank'>github</a> | <a href='https://www.linkedin.com/in/fcakyon/' target='_blank'>linkedin</a> | <a href='https://fcakyon.medium.com/' target='_blank'>medium</a>
+        </p>
+        """
+    )
+
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("Provide a Youtube video URL.")
+            youtube_url = gr.Textbox(label="Youtube URL:", show_label=True)
+            predict_btn = gr.Button(value="Predict")
+        with gr.Column():
+            video_gif = gr.Image(
+                label="Input Clip",
+                show_label=True,
+            )
+        with gr.Column():
+            predictions = gr.Label(
+                label="Predictions:", show_label=True, num_top_classes=5
+            )
+
+    gr.Markdown("**Examples:**")
+    gr.Examples(
+        examples,
+        youtube_url,
+        [predictions, video_gif],
+        fn=predict,
+        cache_examples=True,
+    )
+
+    predict_btn.click(predict, inputs=youtube_url, outputs=[predictions, video_gif])
+    gr.Markdown(
+        """
+        \n Demo created by: <a href=\"https://github.com/fcakyon\">fcakyon</a>
+        <br> Based on this <a href=\"https://huggingface.co/facebook/timesformer-base-finetuned-k400\">HuggingFace model</a>
+        """
+    )
+
+app.launch()

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+torch
+decord
+pytube
+imageio
+transformers @ git+https://github.com/huggingface/transformers.git@c54646b13d468b7a21fd6ee18f943ad69daab48e
+video_transformers == 0.0.8

utils.py

@@ -0,0 +1,37 @@
+from pytube import YouTube
+import numpy as np
+from decord import VideoReader, cpu
+import imageio
+
+
+def download_youtube_video(url: str):
+    yt = YouTube(url)
+
+    streams = yt.streams.filter(file_extension="mp4")
+    file_path = streams[0].download()
+    return file_path
+
+
+def sample_frame_indices(clip_len, frame_sample_rate):
+    converted_len = int(clip_len * frame_sample_rate)
+    start_idx = 0
+    end_idx = converted_len
+    indices = np.linspace(start_idx, end_idx, num=clip_len)
+    indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
+    return indices
+
+
+def sample_frames_from_video_file(file_path: str, num_frames: int = 16):
+    videoreader = VideoReader(file_path, num_threads=1, ctx=cpu(0))
+
+    # sample frames
+    videoreader.seek(0)
+    indices = sample_frame_indices(clip_len=num_frames, frame_sample_rate=4)
+    frames = videoreader.get_batch(indices).asnumpy()
+    return frames
+
+
+def convert_frames_to_gif(frames):
+    converted_frames = frames.astype(np.uint8)
+    imageio.mimsave("frames.gif", converted_frames, fps=8)
+    return "frames.gif"