Super-squash branch 'main' using huggingface_hub

Co-authored-by: sayakpaul <[email protected]>
Co-authored-by: idlebg <[email protected]>
Co-authored-by: patrickvonplaten <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: SD-XL To Diffusers (fp16)
+emoji: 🎨➡️🧨
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+sdk_version: 4.36.1
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,40 @@
+import gradio as gr
+
+from convert import convert
+
+DESCRIPTION = """
+
+[Here is the full version of the converter by Huggingface official.](https://huggingface.co/spaces/diffusers/sdxl-to-diffusers)
+
+The steps are the following:
+
+- Paste a read-access token from hf.co/settings/tokens. Read access is enough given that we will open a PR against the source repo.
+- Input a model id from the Hub
+- Input the filename from the root dir of the repo that you would like to convert, e.g. 'xl-pruned.safetensors'
+- Click "Submit"
+- That's it! You'll get feedback if it works or not, and if it worked, you'll get the URL of the opened PR 🔥
+
+⚠️ If you encounter weird error messages, please have a look into the Logs and feel free to open a PR to correct the error messages.
+
+⚠️⚠️ Note that if you want your model to work with Huggingface's Inference API, you will need to create a model card (README.md) after the conversion.
+
+If you have a good PC, you can use the following script in your local environment. [kiyohken2000 /
+convert-safetensors-to-diffusers](https://github.com/kiyohken2000/convert-safetensors-to-diffusers)
+
+"""
+
+demo = gr.Interface(
+    title="Convert any Stable Diffusion XL checkpoint to Diffusers and open a PR",
+    description=DESCRIPTION,
+    allow_flagging="never",
+    article="Check out the [Diffusers repo on GitHub](https://github.com/huggingface/diffusers)",
+    inputs=[
+        gr.Text(max_lines=1, label="your_hf_token"),
+        gr.Text(max_lines=1, label="model_id"),
+        gr.Text(max_lines=1, label="filename"),
+    ],
+    outputs=[gr.Markdown(label="output")],
+    fn=convert,
+).queue(max_size=1)
+
+demo.launch(show_api=True)

convert.py

@@ -0,0 +1,70 @@
+import gradio as gr
+import requests
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+from tempfile import TemporaryDirectory
+from typing import Optional
+
+import torch
+from io import BytesIO
+
+from huggingface_hub import CommitInfo, Discussion, HfApi, hf_hub_download
+from huggingface_hub.file_download import repo_folder_name
+from diffusers import StableDiffusionXLPipeline
+from transformers import CONFIG_MAPPING
+
+
+COMMIT_MESSAGE = " This PR adds fp16 weights in safetensors format to {}"
+
+
+def convert_single(model_id: str, filename: str, folder: str, progress: Any, token: str):
+    progress(0, desc="Downloading model")
+    local_file = os.path.join(model_id, filename)
+    ckpt_file = local_file if os.path.isfile(local_file) else hf_hub_download(repo_id=model_id, filename=filename, token=token)
+
+    pipeline = StableDiffusionXLPipeline.from_single_file(ckpt_file, torch_dtype=torch.float16)
+
+    from diffusers import EulerAncestralDiscreteScheduler
+    pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(pipeline.scheduler.config)
+
+    pipeline.save_pretrained(folder, safe_serialization=True)
+    
+    return folder
+
+
+def previous_pr(api: "HfApi", model_id: str, pr_title: str) -> Optional["Discussion"]:
+    try:
+        discussions = api.get_repo_discussions(repo_id=model_id)
+    except Exception:
+        return None
+    for discussion in discussions:
+        if discussion.status == "open" and discussion.is_pull_request and discussion.title == pr_title:
+            details = api.get_discussion_details(repo_id=model_id, discussion_num=discussion.num)
+            if details.target_branch == "refs/heads/main":
+                return discussion
+
+
+def convert(token: str, model_id: str, filename: str, progress=gr.Progress()):
+    api = HfApi()
+
+    pr_title = "Adding `diffusers` weights of this model"
+
+    with TemporaryDirectory() as d:
+        folder = os.path.join(d, repo_folder_name(repo_id=model_id, repo_type="models"))
+        os.makedirs(folder)
+        new_pr = None
+        try:
+            folder = convert_single(model_id, filename, folder, progress, token)
+            progress(0.7, desc="Uploading to Hub")
+            new_pr  = api.upload_folder(folder_path=folder, path_in_repo="./", repo_id=model_id, repo_type="model", token=token, commit_message=pr_title, commit_description=COMMIT_MESSAGE.format(model_id), create_pr=True)
+            pr_number = new_pr.split("%2F")[-1].split("/")[0]
+            link = f"Pr created at: {'https://huggingface.co/' + os.path.join(model_id, 'discussions', pr_number)}"
+            progress(1, desc="Done")
+        except Exception as e:
+            raise gr.exceptions.Error(str(e))
+        finally:
+            shutil.rmtree(folder)
+
+        return link

hf_utils.py

@@ -0,0 +1,50 @@
+from huggingface_hub import get_hf_file_metadata, hf_hub_url, hf_hub_download, scan_cache_dir, whoami, list_models
+
+
+def get_my_model_names(token):
+    
+    try:
+        author = whoami(token=token)
+        model_infos = list_models(author=author["name"], use_auth_token=token)
+        return [model.modelId for model in model_infos], None
+        
+    except Exception as e:
+        return [], e
+
+def download_file(repo_id: str, filename: str, token: str):
+    """Download a file from a repo on the Hugging Face Hub.
+
+    Returns:
+        file_path (:obj:`str`): The path to the downloaded file.
+        revision (:obj:`str`): The commit hash of the file.
+        """
+
+    md = get_hf_file_metadata(hf_hub_url(repo_id=repo_id, filename=filename), token=token)
+    revision = md.commit_hash
+
+    file_path = hf_hub_download(repo_id=repo_id, filename=filename, revision=revision, token=token)
+
+    return file_path, revision
+
+def delete_file(revision: str):
+    """Delete a file from local cache.
+
+    Args:
+        revision (:obj:`str`): The commit hash of the file.
+    Returns:
+        None
+        """
+    scan_cache_dir().delete_revisions(revision).execute()
+
+def get_pr_url(api, repo_id, title):
+    try:
+        discussions = api.get_repo_discussions(repo_id=repo_id)
+    except Exception:
+        return None
+    for discussion in discussions:
+        if (
+            discussion.status == "open"
+            and discussion.is_pull_request
+            and discussion.title == title
+        ):
+            return f"https://huggingface.co/{repo_id}/discussions/{discussion.num}"

requirements.txt

@@ -0,0 +1,7 @@
+huggingface_hub
+safetensors
+transformers
+accelerate
+git+https://github.com/huggingface/diffusers
+omegaconf
+pytorch_lightning

utils.py

@@ -0,0 +1,6 @@
+def is_google_colab():
+    try:
+        import google.colab
+        return True
+    except:
+        return False