Duplicate from damo-vilab/modelscope-text-to-video-synthesis

Co-authored-by: hysts <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,162 @@
+weights/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.pre-commit-config.yaml

@@ -0,0 +1,37 @@
+exclude: patch
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.2.0
+  hooks:
+  - id: check-executables-have-shebangs
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-toml
+  - id: check-yaml
+  - id: double-quote-string-fixer
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: ['--fix=lf']
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/myint/docformatter
+  rev: v1.4
+  hooks:
+  - id: docformatter
+    args: ['--in-place']
+- repo: https://github.com/pycqa/isort
+  rev: 5.12.0
+  hooks:
+    - id: isort
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.991
+  hooks:
+    - id: mypy
+      args: ['--ignore-missing-imports']
+      additional_dependencies: ['types-python-slugify']
+- repo: https://github.com/google/yapf
+  rev: v0.32.0
+  hooks:
+  - id: yapf
+    args: ['--parallel', '--in-place']

.style.yapf

@@ -0,0 +1,5 @@
+[style]
+based_on_style = pep8
+blank_line_before_nested_class_or_def = false
+spaces_before_comment = 2
+split_before_logical_operator = true

README.md

@@ -0,0 +1,13 @@
+---
+title: ModelScope Text To Video Synthesis
+emoji: 🚀
+colorFrom: pink
+colorTo: pink
+sdk: gradio
+sdk_version: 3.23.0
+app_file: app.py
+pinned: false
+duplicated_from: damo-vilab/modelscope-text-to-video-synthesis
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import os
+import random
+import tempfile
+
+import gradio as gr
+import imageio
+import numpy as np
+import torch
+from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
+
+DESCRIPTION = '# [ModelScope Text to Video Synthesis](https://modelscope.cn/models/damo/text-to-video-synthesis/summary)'
+DESCRIPTION += '\n<p>For Colab usage, you can view <a href="https://colab.research.google.com/drive/1uW1ZqswkQ9Z9bp5Nbo5z59cAn7I0hE6R?usp=sharing" style="text-decoration: underline;" target="_blank">this webpage</a>.（the latest update on 2023.03.21）</p>'
+DESCRIPTION += '\n<p>This model can only be used for non-commercial purposes. To learn more about the model, take a look at the <a href="https://huggingface.co/damo-vilab/modelscope-damo-text-to-video-synthesis" style="text-decoration: underline;" target="_blank">model card</a>.</p>'
+if (SPACE_ID := os.getenv('SPACE_ID')) is not None:
+    DESCRIPTION += f'\n<p>For faster inference without waiting in queue, you may duplicate the space and upgrade to GPU in settings. <a href="https://huggingface.co/spaces/{SPACE_ID}?duplicate=true"><img style="display: inline; margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space" /></a></p>'
+
+MAX_NUM_FRAMES = int(os.getenv('MAX_NUM_FRAMES', '200'))
+DEFAULT_NUM_FRAMES = min(MAX_NUM_FRAMES,
+                         int(os.getenv('DEFAULT_NUM_FRAMES', '16')))
+
+pipe = DiffusionPipeline.from_pretrained('damo-vilab/text-to-video-ms-1.7b',
+                                         torch_dtype=torch.float16,
+                                         variant='fp16')
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+pipe.enable_vae_slicing()
+
+
+def to_video(frames: list[np.ndarray], fps: int) -> str:
+    out_file = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False)
+    writer = imageio.get_writer(out_file.name, format='FFMPEG', fps=fps)
+    for frame in frames:
+        writer.append_data(frame)
+    writer.close()
+    return out_file.name
+
+
+def generate(prompt: str, seed: int, num_frames: int,
+             num_inference_steps: int) -> str:
+    if seed == -1:
+        seed = random.randint(0, 1000000)
+    generator = torch.Generator().manual_seed(seed)
+    frames = pipe(prompt,
+                  num_inference_steps=num_inference_steps,
+                  num_frames=num_frames,
+                  generator=generator).frames
+    return to_video(frames, 8)
+
+
+examples = [
+    ['An astronaut riding a horse.', 0, 16, 25],
+    ['A panda eating bamboo on a rock.', 0, 16, 25],
+    ['Spiderman is surfing.', 0, 16, 25],
+]
+
+with gr.Blocks(css='style.css') as demo:
+    gr.Markdown(DESCRIPTION)
+    with gr.Group():
+        with gr.Box():
+            with gr.Row(elem_id='prompt-container').style(equal_height=True):
+                prompt = gr.Text(
+                    label='Prompt',
+                    show_label=False,
+                    max_lines=1,
+                    placeholder='Enter your prompt',
+                    elem_id='prompt-text-input').style(container=False)
+                run_button = gr.Button('Generate video').style(
+                    full_width=False)
+        result = gr.Video(label='Result', show_label=False, elem_id='gallery')
+        with gr.Accordion('Advanced options', open=False):
+            seed = gr.Slider(
+                label='Seed',
+                minimum=-1,
+                maximum=1000000,
+                step=1,
+                value=-1,
+                info='If set to -1, a different seed will be used each time.')
+            num_frames = gr.Slider(
+                label='Number of frames',
+                minimum=16,
+                maximum=MAX_NUM_FRAMES,
+                step=1,
+                value=16,
+                info=
+                'Note that the content of the video also changes when you change the number of frames.'
+            )
+            num_inference_steps = gr.Slider(label='Number of inference steps',
+                                            minimum=10,
+                                            maximum=50,
+                                            step=1,
+                                            value=25)
+
+    inputs = [
+        prompt,
+        seed,
+        num_frames,
+        num_inference_steps,
+    ]
+    gr.Examples(examples=examples,
+                inputs=inputs,
+                outputs=result,
+                fn=generate,
+                cache_examples=os.getenv('SYSTEM') == 'spaces')
+
+    prompt.submit(fn=generate, inputs=inputs, outputs=result)
+    run_button.click(fn=generate, inputs=inputs, outputs=result)
+
+    with gr.Accordion(label='Biases and content acknowledgment', open=False):
+        gr.HTML("""<div class="acknowledgments">
+                    <h4>Biases and content acknowledgment</h4>
+                    <p>
+                        Despite how impressive being able to turn text into video is, beware to the fact that this model may output content that reinforces or exacerbates societal biases. The training data includes LAION5B, ImageNet, Webvid and other public datasets. The model was not trained to realistically represent people or events, so using it to generate such content is beyond the model's capabilities.
+                    </p>
+                    <p>
+                        It is not intended to generate content that is demeaning or harmful to people or their environment, culture, religion, etc. Similarly, it is not allowed to generate pornographic, violent and bloody content generation. <b>The model is meant for research purposes</b>.
+                    </p>
+                    <p>
+                        To learn more about the model, head to its <a href="https://huggingface.co/damo-vilab/modelscope-damo-text-to-video-synthesis" style="text-decoration: underline;" target="_blank">model card</a>.
+                    </p>
+                   </div>
+                """)
+
+demo.queue(api_open=False, max_size=15).launch()

requirements.txt

@@ -0,0 +1,8 @@
+accelerate==0.17.1
+git+https://github.com/huggingface/diffusers@9dc8444
+gradio==3.23.0
+huggingface-hub==0.13.3
+imageio[ffmpeg]==2.26.1
+torch==2.0.0
+torchvision==0.15.1
+transformers==4.27.2

style.css

@@ -0,0 +1,191 @@
+/*
+This CSS file is copied from here:
+https://huggingface.co/spaces/stabilityai/stable-diffusion/blob/2794a3c3ba66115c307075098e713f572b08bf80/app.py
+*/
+
+h1 {
+    text-align: center;
+}
+
+.gradio-container {
+    font-family: 'IBM Plex Sans', sans-serif;
+}
+
+.gr-button {
+    color: white;
+    border-color: black;
+    background: black;
+}
+
+input[type='range'] {
+    accent-color: black;
+}
+
+.dark input[type='range'] {
+    accent-color: #dfdfdf;
+}
+
+.container {
+    max-width: 730px;
+    margin: auto;
+    padding-top: 1.5rem;
+}
+
+#gallery {
+    min-height: 22rem;
+    margin-bottom: 15px;
+    margin-left: auto;
+    margin-right: auto;
+    border-bottom-right-radius: .5rem !important;
+    border-bottom-left-radius: .5rem !important;
+}
+
+#gallery>div>.h-full {
+    min-height: 20rem;
+}
+
+.details:hover {
+    text-decoration: underline;
+}
+
+.gr-button {
+    white-space: nowrap;
+}
+
+.gr-button:focus {
+    border-color: rgb(147 197 253 / var(--tw-border-opacity));
+    outline: none;
+    box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000);
+    --tw-border-opacity: 1;
+    --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);
+    --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color);
+    --tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity));
+    --tw-ring-opacity: .5;
+}
+
+#advanced-btn {
+    font-size: .7rem !important;
+    line-height: 19px;
+    margin-top: 12px;
+    margin-bottom: 12px;
+    padding: 2px 8px;
+    border-radius: 14px !important;
+}
+
+#advanced-options {
+    display: none;
+    margin-bottom: 20px;
+}
+
+.footer {
+    margin-bottom: 45px;
+    margin-top: 35px;
+    text-align: center;
+    border-bottom: 1px solid #e5e5e5;
+}
+
+.footer>p {
+    font-size: .8rem;
+    display: inline-block;
+    padding: 0 10px;
+    transform: translateY(10px);
+    background: white;
+}
+
+.dark .footer {
+    border-color: #303030;
+}
+
+.dark .footer>p {
+    background: #0b0f19;
+}
+
+.acknowledgments h4 {
+    margin: 1.25em 0 .25em 0;
+    font-weight: bold;
+    font-size: 115%;
+}
+
+.animate-spin {
+    animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+    from {
+        transform: rotate(0deg);
+    }
+
+    to {
+        transform: rotate(360deg);
+    }
+}
+
+#share-btn-container {
+    display: flex;
+    padding-left: 0.5rem !important;
+    padding-right: 0.5rem !important;
+    background-color: #000000;
+    justify-content: center;
+    align-items: center;
+    border-radius: 9999px !important;
+    width: 13rem;
+    margin-top: 10px;
+    margin-left: auto;
+}
+
+#share-btn {
+    all: initial;
+    color: #ffffff;
+    font-weight: 600;
+    cursor: pointer;
+    font-family: 'IBM Plex Sans', sans-serif;
+    margin-left: 0.5rem !important;
+    padding-top: 0.25rem !important;
+    padding-bottom: 0.25rem !important;
+    right: 0;
+}
+
+#share-btn * {
+    all: unset;
+}
+
+#share-btn-container div:nth-child(-n+2) {
+    width: auto !important;
+    min-height: 0px !important;
+}
+
+#share-btn-container .wrap {
+    display: none !important;
+}
+
+.gr-form {
+    flex: 1 1 50%;
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+}
+
+#prompt-container {
+    gap: 0;
+}
+
+#prompt-text-input,
+#negative-prompt-text-input {
+    padding: .45rem 0.625rem
+}
+
+#component-16 {
+    border-top-width: 1px !important;
+    margin-top: 1em
+}
+
+.image_duplication {
+    position: absolute;
+    width: 100px;
+    left: 50px
+}
+
+#component-0 {
+    max-width: 730px;
+    margin: auto;
+    padding-top: 1.5rem;
+}