First model version

args.json

@@ -0,0 +1,58 @@
+{
+  "pretrained_model_name_or_path": "booth-ai/realisticVisionV20_v20_inpainting",
+  "pretrained_vae_name_or_path": "stabilityai/sd-vae-ft-mse",
+  "revision": "main",
+  "tokenizer_name": null,
+  "instance_data_dir": null,
+  "class_data_dir": null,
+  "instance_prompt": null,
+  "class_prompt": null,
+  "save_sample_negative_prompt": null,
+  "n_save_sample": 4,
+  "save_guidance_scale": 7.5,
+  "save_infer_steps": 50,
+  "pad_tokens": false,
+  "with_prior_preservation": false,
+  "prior_loss_weight": 1.0,
+  "num_class_images": 100,
+  "output_dir": "./models/outgrowths_7",
+  "seed": 3434554,
+  "resolution": 512,
+  "center_crop": false,
+  "train_text_encoder": false,
+  "train_batch_size": 4,
+  "sample_batch_size": 2,
+  "num_train_epochs": 176,
+  "max_train_steps": 10000,
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "learning_rate": 2e-06,
+  "scale_lr": false,
+  "lr_scheduler": "constant",
+  "lr_warmup_steps": 0,
+  "use_8bit_adam": false,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "adam_weight_decay": 0.01,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "push_to_hub": false,
+  "hub_token": null,
+  "hub_model_id": null,
+  "logging_dir": "logs",
+  "log_interval": 10,
+  "save_interval": 500,
+  "save_min_steps": 500,
+  "mixed_precision": "no",
+  "not_cache_latents": true,
+  "hflip": true,
+  "local_rank": -1,
+  "concepts_list": [
+    {
+      "instance_prompt": "photo of zwx dog",
+      "class_prompt": "photo of a dog",
+      "instance_data_dir": "./512",
+      "class_data_dir": "../../../data/dog"
+    }
+  ]
+}

feature_extractor/preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "feature_extractor_type": "CLIPFeatureExtractor",
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPFeatureExtractor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}

model_index.json

@@ -0,0 +1,34 @@
+{
+  "_class_name": "StableDiffusionInpaintPipeline",
+  "_diffusers_version": "0.21.0.dev0",
+  "_name_or_path": "booth-ai/realisticVisionV20_v20_inpainting",
+  "feature_extractor": [
+    "transformers",
+    "CLIPFeatureExtractor"
+  ],
+  "requires_safety_checker": true,
+  "safety_checker": [
+    null,
+    null
+  ],
+  "scheduler": [
+    "diffusers",
+    "DDIMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "diffusers",
+    "UNet2DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

scheduler/scheduler_config.json

@@ -0,0 +1,19 @@
+{
+  "_class_name": "DDIMScheduler",
+  "_diffusers_version": "0.21.0.dev0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "dynamic_thresholding_ratio": 0.995,
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "rescale_betas_zero_snr": false,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": false,
+  "steps_offset": 1,
+  "thresholding": false,
+  "timestep_spacing": "leading",
+  "trained_betas": null
+}

text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "booth-ai/realisticVisionV20_v20_inpainting",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float16",
+  "transformers_version": "4.32.1",
+  "vocab_size": 49408
+}

text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3183e9cd8a1c4cfd99fc68b0c6d1832d009ad672c3577ab0e81939ec1be15b
+size 246144152

tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

train_inpainting_dreambooth.py

@@ -0,0 +1,1032 @@
+import argparse
+import hashlib
+import itertools
+import json
+import math
+import os
+import random
+import shutil
+from contextlib import nullcontext
+from pathlib import Path
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from accelerate import Accelerator
+from accelerate.logging import get_logger
+from accelerate.utils import set_seed
+from huggingface_hub import HfFolder, Repository, whoami
+from PIL import Image, ImageOps
+from torch.utils.data import Dataset
+# from torchvision import transforms
+# from torchvision.transforms import v2
+from opencv_transforms import transforms
+from tqdm.auto import tqdm
+from transformers import CLIPTextModel, CLIPTokenizer
+
+from diffusers import (AutoencoderKL, DDIMScheduler, DDPMScheduler,
+                       StableDiffusionInpaintPipeline, UNet2DConditionModel)
+from diffusers.optimization import get_scheduler
+from diffusers.utils import is_wandb_available
+
+if is_wandb_available():
+    import wandb
+
+torch.backends.cudnn.benchmark = True
+
+
+logger = get_logger(__name__)
+
+
+def parse_args(input_args=None):
+    parser = argparse.ArgumentParser(description="Simple example of a training script.")
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        type=str,
+        default=None,
+        required=True,
+        help="Path to pretrained model or model identifier from huggingface.co/models.",
+    )
+    parser.add_argument(
+        "--pretrained_vae_name_or_path",
+        type=str,
+        default=None,
+        help="Path to pretrained vae or vae identifier from huggingface.co/models.",
+    )
+    parser.add_argument(
+        "--revision",
+        type=str,
+        default="fp16",
+        required=False,
+        help="Revision of pretrained model identifier from huggingface.co/models.",
+    )
+    parser.add_argument(
+        "--tokenizer_name",
+        type=str,
+        default=None,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--instance_data_dir",
+        type=str,
+        default=None,
+        help="A folder containing the training data of instance images.",
+    )
+    parser.add_argument(
+        "--class_data_dir",
+        type=str,
+        default=None,
+        help="A folder containing the training data of class images.",
+    )
+    parser.add_argument(
+        "--instance_prompt",
+        type=str,
+        default=None,
+        help="The prompt with identifier specifying the instance",
+    )
+    parser.add_argument(
+        "--class_prompt",
+        type=str,
+        default=None,
+        help="The prompt to specify images in the same class as provided instance images.",
+    )
+    # parser.add_argument(
+    #     "--save_sample_prompt",
+    #     type=str,
+    #     default=None,
+    #     help="The prompt used to generate sample outputs to save.",
+    # )
+    parser.add_argument(
+        "--save_sample_negative_prompt",
+        type=str,
+        default=None,
+        help="The negative prompt used to generate sample outputs to save.",
+    )
+    parser.add_argument(
+        "--n_save_sample",
+        type=int,
+        default=4,
+        help="The number of samples to save.",
+    )
+    parser.add_argument(
+        "--save_guidance_scale",
+        type=float,
+        default=7.5,
+        help="CFG for save sample.",
+    )
+    parser.add_argument(
+        "--save_infer_steps",
+        type=int,
+        default=50,
+        help="The number of inference steps for save sample.",
+    )
+    parser.add_argument(
+        "--pad_tokens",
+        default=False,
+        action="store_true",
+        help="Flag to pad tokens to length 77.",
+    )
+    parser.add_argument(
+        "--with_prior_preservation",
+        default=False,
+        action="store_true",
+        help="Flag to add prior preservation loss.",
+    )
+    parser.add_argument("--prior_loss_weight", type=float, default=1.0, help="The weight of prior preservation loss.")
+    parser.add_argument(
+        "--num_class_images",
+        type=int,
+        default=100,
+        help=(
+            "Minimal class images for prior preservation loss. If not have enough images, additional images will be"
+            " sampled with class_prompt."
+        ),
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="text-inversion-model",
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
+    parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.")
+    parser.add_argument(
+        "--resolution",
+        type=int,
+        default=512,
+        help=(
+            "The resolution for input images, all the images in the train/validation dataset will be resized to this"
+            " resolution"
+        ),
+    )
+    parser.add_argument(
+        "--center_crop", action="store_true", help="Whether to center crop images before resizing to resolution"
+    )
+    parser.add_argument("--train_text_encoder", action="store_true", help="Whether to train the text encoder")
+    parser.add_argument(
+        "--train_batch_size", type=int, default=4, help="Batch size (per device) for the training dataloader."
+    )
+    parser.add_argument(
+        "--sample_batch_size", type=int, default=4, help="Batch size (per device) for sampling images."
+    )
+    parser.add_argument("--num_train_epochs", type=int, default=1)
+    parser.add_argument(
+        "--max_train_steps",
+        type=int,
+        default=None,
+        help="Total number of training steps to perform.  If provided, overrides num_train_epochs.",
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument(
+        "--gradient_checkpointing",
+        action="store_true",
+        help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.",
+    )
+    parser.add_argument(
+        "--learning_rate",
+        type=float,
+        default=5e-6,
+        help="Initial learning rate (after the potential warmup period) to use.",
+    )
+    parser.add_argument(
+        "--scale_lr",
+        action="store_true",
+        default=False,
+        help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.",
+    )
+    parser.add_argument(
+        "--lr_scheduler",
+        type=str,
+        default="constant",
+        help=(
+            'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",'
+            ' "constant", "constant_with_warmup"]'
+        ),
+    )
+    parser.add_argument(
+        "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler."
+    )
+    parser.add_argument(
+        "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes."
+    )
+    parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.")
+    parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.")
+    parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.")
+    parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
+    parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.")
+    parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.")
+    parser.add_argument(
+        "--hub_model_id",
+        type=str,
+        default=None,
+        help="The name of the repository to keep in sync with the local `output_dir`.",
+    )
+    parser.add_argument(
+        "--logging_dir",
+        type=str,
+        default="logs",
+        help=(
+            "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to"
+            " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***."
+        ),
+    )
+    parser.add_argument("--log_interval", type=int, default=10, help="Log every N steps.")
+    parser.add_argument("--save_interval", type=int, default=10_000, help="Save weights every N steps.")
+    parser.add_argument("--save_min_steps", type=int, default=0, help="Start saving weights after N steps.")
+    parser.add_argument(
+        "--mixed_precision",
+        type=str,
+        default="no",
+        choices=["no", "fp16", "bf16"],
+        help=(
+            "Whether to use mixed precision. Choose"
+            "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
+            "and an Nvidia Ampere GPU."
+        ),
+    )
+    parser.add_argument("--not_cache_latents", action="store_true", help="Do not precompute and cache latents from VAE.")
+    parser.add_argument("--hflip", action="store_true", help="Apply horizontal flip data augmentation.")
+    parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
+    parser.add_argument(
+        "--concepts_list",
+        type=str,
+        default=None,
+        help="Path to json containing multiple concepts, will overwrite parameters like instance_prompt, class_prompt, etc.",
+    )
+
+    if input_args is not None:
+        args = parser.parse_args(input_args)
+    else:
+        args = parser.parse_args()
+
+    env_local_rank = int(os.environ.get("LOCAL_RANK", -1))
+    if env_local_rank != -1 and env_local_rank != args.local_rank:
+        args.local_rank = env_local_rank
+
+    return args
+
+def get_cutout_holes(height, width, min_holes=8, max_holes=32, min_height=16, max_height=128, min_width=16, max_width=128):
+    holes = []
+    for _n in range(random.randint(min_holes, max_holes)):
+        hole_height = random.randint(min_height, max_height)
+        hole_width = random.randint(min_width, max_width)
+        y1 = random.randint(0, height - hole_height)
+        x1 = random.randint(0, width - hole_width)
+        y2 = y1 + hole_height
+        x2 = x1 + hole_width
+        holes.append((x1, y1, x2, y2))
+    return holes
+
+import torchvision
+
+def generate_random_mask(image):
+    # mask = torch.zeros_like(image[:1])
+    # print('foobar', mask.shape)
+    mask = image[-1].unsqueeze(0)
+    # torchvision.transforms.functional.to_pil_image(mask).save('foomask2.png')
+
+    # if random.uniform(0, 1) < 0.25:
+    #     mask = torch.zeros_like(image[:1])
+    #     mask.fill_(1.)
+    # print('foobar',image.shape)
+    # torchvision.transforms.functional.to_pil_image(image[:4]).save('fooimageinside2.png')  
+    # print('foobar',image[:3].shape)
+    masked_image = image[:-1] * (mask < 0.5)
+    # torchvision.transforms.functional.to_pil_image(masked_image).save('foomaskedimage2.png')  
+    return mask, masked_image
+
+image_transforms = transforms.Compose(
+    [
+        torchvision.transforms.RandomHorizontalFlip(0.5 * True),
+        # transforms.Resize(size, interpolation=transforms.InterpolationMode.BILINEAR),
+        # transforms.CenterCrop(size) if center_crop else transforms.RandomCrop(size),
+        transforms.RandomResizedCrop(512, scale=(0.75,1.0)),
+        torchvision.transforms.ToTensor(),
+        torchvision.transforms.Normalize([0.5], [0.5]),
+    ]
+)
+# instance_image = Image.open("/home/neil/Documents/diffusers/examples/dreambooth/512_/a corner sofa in midnight blue in a room with vibrant walls and eclectic artwork, creating a lively and energetic atmosphere. .png")
+# alpha = instance_image.split()[-1]
+# instance_image.putalpha(ImageOps.invert(alpha))  
+# instance_image.save('foofull.png')
+# # rgb_instance_image = instance_image.convert('RGB')
+# # rgb_instance_image.save('foofullrgb.png')
+# example = {}
+# example["instance_images"] = image_transforms(instance_image)
+# torchvision.transforms.functional.to_pil_image(example["instance_images"]).save('fooimage.png')
+# example["instance_masks"], example["instance_masked_images"] = generate_random_mask(example["instance_images"])
+# example["instance_images"] = example["instance_images"][:3]
+# torchvision.transforms.functional.to_pil_image(example["instance_images"]).save('fooimage1.png')
+# breakpoint()
+
+# example = {}
+# instance_image = Image.open("/home/neil/Documents/diffusers/examples/dreambooth/512_/a corner sofa in midnight blue in a room with vibrant walls and eclectic artwork, creating a lively and energetic atmosphere. .png")
+# instance_image.save('fooinstance2.png')
+# alpha = instance_image.split()[-1]
+# alpha.save('fooalpha2.png')
+# instance_image.putalpha(ImageOps.invert(alpha)) 
+# instance_image.convert("RGB").save('foofull2rgb.png')
+# instance_image.save('foofull2.png')
+
+# example["instance_images"] = image_transforms(instance_image)
+# torchvision.transforms.functional.to_pil_image(example["instance_images"]).save('fooimage2.png')
+# example["instance_masks"], example["instance_masked_images"] = generate_random_mask(example["instance_images"])
+# torchvision.transforms.functional.to_pil_image(example["instance_masks"]).save('fooinstance_masks.png')
+# torchvision.transforms.functional.to_pil_image(example["instance_masked_images"]).save('fooinstance_masked_images.png')
+# example["instance_images"] = example["instance_images"][:3]
+# final = torchvision.transforms.functional.to_pil_image(example["instance_images"])
+# final.save('fooimage3.png')
+# breakpoint()
+
+class DreamBoothDataset(Dataset):
+    """
+    A dataset to prepare the instance and class images with the prompts for fine-tuning the model.
+    It pre-processes the images and the tokenizes prompts.
+    """
+
+    def __init__(
+        self,
+        concepts_list,
+        tokenizer,
+        with_prior_preservation=True,
+        size=512,
+        center_crop=False,
+        num_class_images=None,
+        pad_tokens=False,
+        hflip=False
+    ):
+        self.size = size
+        self.center_crop = center_crop
+        self.tokenizer = tokenizer
+        self.with_prior_preservation = with_prior_preservation
+        self.pad_tokens = pad_tokens
+
+        self.instance_images_path = []
+        self.class_images_path = []
+
+        for concept in concepts_list:
+            inst_img_path = [(x, concept["instance_prompt"]) for x in Path(concept["instance_data_dir"]).iterdir() if x.is_file()]
+            self.instance_images_path.extend(inst_img_path)
+
+            if with_prior_preservation:
+                class_img_path = [(x, concept["class_prompt"]) for x in Path(concept["class_data_dir"]).iterdir() if x.is_file()]
+                self.class_images_path.extend(class_img_path[:num_class_images])
+
+        random.shuffle(self.instance_images_path)
+        self.num_instance_images = len(self.instance_images_path)
+        self.num_class_images = len(self.class_images_path)
+        self._length = max(self.num_class_images, self.num_instance_images)
+
+        self.image_transforms = transforms.Compose(
+            [
+                torchvision.transforms.RandomHorizontalFlip(0.5 * hflip),
+                transforms.RandomResizedCrop(size, scale=(0.75,1.0)),
+                torchvision.transforms.ToTensor(),
+                torchvision.transforms.Normalize([0.5], [0.5]),
+            ]
+        )
+
+    def __len__(self):
+        return self._length
+
+    def __getitem__(self, index):
+        example = {}
+        instance_path, instance_prompt = self.instance_images_path[index % self.num_instance_images]
+        instance_image = Image.open(instance_path)
+        alpha = instance_image.split()[-1]
+        instance_image.putalpha(ImageOps.invert(alpha)) 
+ 
+        example["instance_images"] = self.image_transforms(instance_image)
+        example["instance_masks"], example["instance_masked_images"] = generate_random_mask(example["instance_images"])
+        # torchvision.transforms.functional.to_pil_image(example["instance_masks"][0]).save('foo_instance_masks.png')
+        # torchvision.transforms.functional.to_pil_image(example["instance_masked_images"][0]).save('foo_instance_masked_images.png')
+        example["instance_images"] = example["instance_images"][:3]
+        # torchvision.transforms.functional.to_pil_image(example["instance_images"]).save('foo_instance_image.png')
+        example["instance_prompt_ids"] = self.tokenizer(
+            # instance_prompt,
+            os.path.basename(instance_prompt),
+            padding="max_length" if self.pad_tokens else "do_not_pad",
+            truncation=True,
+            max_length=self.tokenizer.model_max_length,
+        ).input_ids
+        if self.with_prior_preservation:
+            class_path, class_prompt = self.class_images_path[index % self.num_class_images]
+            class_image = Image.open(class_path)
+            alpha = instance_image.split()[-1]
+            instance_image.putalpha(ImageOps.invert(alpha)) 
+            example["instance_images"] = self.image_transforms(instance_image)
+            example["instance_masks"], example["instance_masked_images"] = generate_random_mask(example["instance_images"])
+            example["instance_images"] = self.image_transforms(instance_image.convert("RGB"))
+            example["class_prompt_ids"] = self.tokenizer(
+                class_prompt,
+                padding="max_length" if self.pad_tokens else "do_not_pad",
+                truncation=True,
+                max_length=self.tokenizer.model_max_length,
+            ).input_ids
+
+        return example
+
+
+class PromptDataset(Dataset):
+    "A simple dataset to prepare the prompts to generate class images on multiple GPUs."
+
+    def __init__(self, prompt, num_samples):
+        self.prompt = prompt
+        self.num_samples = num_samples
+
+    def __len__(self):
+        return self.num_samples
+
+    def __getitem__(self, index):
+        example = {}
+        example["prompt"] = self.prompt
+        example["index"] = index
+        return example
+
+
+class LatentsDataset(Dataset):
+    def __init__(self, latents_cache, text_encoder_cache):
+        self.latents_cache = latents_cache
+        self.text_encoder_cache = text_encoder_cache
+
+    def __len__(self):
+        return len(self.latents_cache)
+
+    def __getitem__(self, index):
+        return self.latents_cache[index], self.text_encoder_cache[index]
+
+
+class AverageMeter:
+    def __init__(self, name=None):
+        self.name = name
+        self.reset()
+
+    def reset(self):
+        self.sum = self.count = self.avg = 0
+
+    def update(self, val, n=1):
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None):
+    if token is None:
+        token = HfFolder.get_token()
+    if organization is None:
+        username = whoami(token)["name"]
+        return f"{username}/{model_id}"
+    else:
+        return f"{organization}/{model_id}"
+
+
+def main(args):
+    logging_dir = Path(args.output_dir, "0", args.logging_dir)
+
+    accelerator = Accelerator(
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        mixed_precision=args.mixed_precision,
+        log_with="tensorboard",
+        project_dir=logging_dir,
+    )
+
+    # Currently, it's not possible to do gradient accumulation when training two models with accelerate.accumulate
+    # This will be enabled soon in accelerate. For now, we don't allow gradient accumulation when training two models.
+    # TODO (patil-suraj): Remove this check when gradient accumulation with two models is enabled in accelerate.
+    if args.train_text_encoder and args.gradient_accumulation_steps > 1 and accelerator.num_processes > 1:
+        raise ValueError(
+            "Gradient accumulation is not supported when training the text encoder in distributed training. "
+            "Please set gradient_accumulation_steps to 1. This feature will be supported in the future."
+        )
+
+    if args.seed is not None:
+        set_seed(args.seed)
+
+    if args.concepts_list is None:
+        args.concepts_list = [
+            {
+                "instance_prompt": args.instance_prompt,
+                "class_prompt": args.class_prompt,
+                "instance_data_dir": args.instance_data_dir,
+                "class_data_dir": args.class_data_dir
+            }
+        ]
+    else:
+        with open(args.concepts_list, "r") as f:
+            args.concepts_list = json.load(f)
+
+    if args.with_prior_preservation:
+        pipeline = None
+        for concept in args.concepts_list:
+            class_images_dir = Path(concept["class_data_dir"])
+            class_images_dir.mkdir(parents=True, exist_ok=True)
+            cur_class_images = len(list(class_images_dir.iterdir()))
+
+            if cur_class_images < args.num_class_images:
+                torch_dtype = torch.float16 if accelerator.device.type == "cuda" else torch.float32
+                if pipeline is None:
+                    pipeline = StableDiffusionInpaintPipeline.from_pretrained(
+                        args.pretrained_model_name_or_path,
+                        vae=AutoencoderKL.from_pretrained(
+                            args.pretrained_vae_name_or_path or args.pretrained_model_name_or_path,
+                            revision=None if args.pretrained_vae_name_or_path else args.revision,
+                            torch_dtype=torch_dtype
+                        ),
+                        torch_dtype=torch_dtype,
+                        safety_checker=None,
+                        revision=args.revision
+                    )
+                    pipeline.set_progress_bar_config(disable=True)
+                    pipeline.to(accelerator.device)
+
+                num_new_images = args.num_class_images - cur_class_images
+                logger.info(f"Number of class images to sample: {num_new_images}.")
+
+                sample_dataset = PromptDataset(concept["class_prompt"], num_new_images)
+                sample_dataloader = torch.utils.data.DataLoader(sample_dataset, batch_size=args.sample_batch_size)
+
+                sample_dataloader = accelerator.prepare(sample_dataloader)
+
+                inp_img = Image.new("RGB", (512, 512), color=(0, 0, 0))
+                inp_mask = Image.new("L", (512, 512), color=255)
+
+                with torch.autocast("cuda"),torch.inference_mode():
+                    for example in tqdm(
+                        sample_dataloader, desc="Generating class images", disable=not accelerator.is_local_main_process
+                    ):
+                        images = pipeline(
+                            prompt=example["prompt"],
+                            image=inp_img,
+                            mask_image=inp_mask,
+                            num_inference_steps=args.save_infer_steps
+                        ).images
+
+                        for i, image in enumerate(images):
+                            hash_image = hashlib.sha1(image.tobytes()).hexdigest()
+                            image_filename = class_images_dir / f"{example['index'][i] + cur_class_images}-{hash_image}.jpg"
+                            image.save(image_filename)
+
+        del pipeline
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+    # Load the tokenizer
+    if args.tokenizer_name:
+        tokenizer = CLIPTokenizer.from_pretrained(
+            args.tokenizer_name,
+            revision=args.revision,
+        )
+    elif args.pretrained_model_name_or_path:
+        tokenizer = CLIPTokenizer.from_pretrained(
+            args.pretrained_model_name_or_path,
+            subfolder="tokenizer",
+            revision=args.revision,
+        )
+
+    # Load models and create wrapper for stable diffusion
+    text_encoder = CLIPTextModel.from_pretrained(
+        args.pretrained_model_name_or_path,
+        subfolder="text_encoder",
+        revision=args.revision,
+    )
+    vae = AutoencoderKL.from_pretrained(
+        args.pretrained_model_name_or_path,
+        subfolder="vae",
+        revision=args.revision,
+    )
+    unet = UNet2DConditionModel.from_pretrained(
+        args.pretrained_model_name_or_path,
+        subfolder="unet",
+        revision=args.revision,
+        torch_dtype=torch.float32
+    )
+
+    vae.requires_grad_(False)
+    if not args.train_text_encoder:
+        text_encoder.requires_grad_(False)
+
+    if args.gradient_checkpointing:
+        unet.enable_gradient_checkpointing()
+        if args.train_text_encoder:
+            text_encoder.gradient_checkpointing_enable()
+
+    if args.scale_lr:
+        args.learning_rate = (
+            args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes
+        )
+
+    # Use 8-bit Adam for lower memory usage or to fine-tune the model in 16GB GPUs
+    if args.use_8bit_adam:
+        try:
+            import bitsandbytes as bnb
+        except ImportError:
+            raise ImportError(
+                "To use 8-bit Adam, please install the bitsandbytes library: `pip install bitsandbytes`."
+            )
+
+        optimizer_class = bnb.optim.AdamW8bit
+    else:
+        optimizer_class = torch.optim.AdamW
+
+    params_to_optimize = (
+        itertools.chain(unet.parameters(), text_encoder.parameters()) if args.train_text_encoder else unet.parameters()
+    )
+    optimizer = optimizer_class(
+        params_to_optimize,
+        lr=args.learning_rate,
+        betas=(args.adam_beta1, args.adam_beta2),
+        weight_decay=args.adam_weight_decay,
+        eps=args.adam_epsilon,
+    )
+
+    noise_scheduler = DDPMScheduler.from_config(args.pretrained_model_name_or_path, subfolder="scheduler")
+
+    train_dataset = DreamBoothDataset(
+        concepts_list=args.concepts_list,
+        tokenizer=tokenizer,
+        with_prior_preservation=args.with_prior_preservation,
+        size=args.resolution,
+        center_crop=args.center_crop,
+        num_class_images=args.num_class_images,
+        pad_tokens=args.pad_tokens,
+        hflip=args.hflip
+    )
+
+    def collate_fn(examples):
+        input_ids = [example["instance_prompt_ids"] for example in examples]
+        pixel_values = [example["instance_images"] for example in examples]
+        mask_values = [example["instance_masks"] for example in examples]
+        masked_image_values = [example["instance_masked_images"] for example in examples]
+
+        # Concat class and instance examples for prior preservation.
+        # We do this to avoid doing two forward passes.
+        if args.with_prior_preservation:
+            input_ids += [example["class_prompt_ids"] for example in examples]
+            pixel_values += [example["class_images"] for example in examples]
+            mask_values += [example["class_masks"] for example in examples]
+            masked_image_values += [example["class_masked_images"] for example in examples]
+
+        pixel_values = torch.stack(pixel_values).to(memory_format=torch.contiguous_format).float()
+        mask_values = torch.stack(mask_values).to(memory_format=torch.contiguous_format).float()
+        masked_image_values = torch.stack(masked_image_values).to(memory_format=torch.contiguous_format).float()
+
+        input_ids = tokenizer.pad(
+            {"input_ids": input_ids},
+            padding=True,
+            return_tensors="pt",
+        ).input_ids
+
+        batch = {
+            "input_ids": input_ids,
+            "pixel_values": pixel_values,
+            "mask_values": mask_values,
+            "masked_image_values": masked_image_values
+        }
+        return batch
+
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset, batch_size=args.train_batch_size, shuffle=True, collate_fn=collate_fn, pin_memory=True, num_workers=8
+    )
+
+    weight_dtype = torch.float32
+    if args.mixed_precision == "fp16":
+        weight_dtype = torch.float16
+    elif args.mixed_precision == "bf16":
+        weight_dtype = torch.bfloat16
+
+    # Move text_encode and vae to gpu.
+    # For mixed precision training we cast the text_encoder and vae weights to half-precision
+    # as these models are only used for inference, keeping weights in full precision is not required.
+    vae.to(accelerator.device, dtype=weight_dtype)
+    if not args.train_text_encoder:
+        text_encoder.to(accelerator.device, dtype=weight_dtype)
+
+    if not args.not_cache_latents:
+        latents_cache = []
+        text_encoder_cache = []
+        for batch in tqdm(train_dataloader, desc="Caching latents"):
+            with torch.no_grad():
+                batch["pixel_values"] = batch["pixel_values"].to(accelerator.device, non_blocking=True, dtype=weight_dtype)
+                batch["input_ids"] = batch["input_ids"].to(accelerator.device, non_blocking=True)
+                latents_cache.append(vae.encode(batch["pixel_values"]).latent_dist)
+                if args.train_text_encoder:
+                    text_encoder_cache.append(batch["input_ids"])
+                else:
+                    text_encoder_cache.append(text_encoder(batch["input_ids"])[0])
+        train_dataset = LatentsDataset(latents_cache, text_encoder_cache)
+        train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=1, collate_fn=lambda x: x, shuffle=True)
+
+        del vae
+        if not args.train_text_encoder:
+            del text_encoder
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+    # Scheduler and math around the number of training steps.
+    overrode_max_train_steps = False
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    if args.max_train_steps is None:
+        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
+        overrode_max_train_steps = True
+
+    lr_scheduler = get_scheduler(
+        args.lr_scheduler,
+        optimizer=optimizer,
+        num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps,
+        num_training_steps=args.max_train_steps * args.gradient_accumulation_steps,
+    )
+
+    if args.train_text_encoder:
+        unet, text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+            unet, text_encoder, optimizer, train_dataloader, lr_scheduler
+        )
+    else:
+        unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
+            unet, optimizer, train_dataloader, lr_scheduler
+        )
+
+    # We need to recalculate our total training steps as the size of the training dataloader may have changed.
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    if overrode_max_train_steps:
+        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
+    # Afterwards we recalculate our number of training epochs
+    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+
+    # We need to initialize the trackers we use, and also store our configuration.
+    # The trackers initializes automatically on the main process.
+    if accelerator.is_main_process:
+        accelerator.init_trackers("dreambooth")
+
+    # Train!
+    total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+
+    logger.info("***** Running training *****")
+    logger.info(f"  Num examples = {len(train_dataset)}")
+    logger.info(f"  Num batches each epoch = {len(train_dataloader)}")
+    logger.info(f"  Num Epochs = {args.num_train_epochs}")
+    logger.info(f"  Instantaneous batch size per device = {args.train_batch_size}")
+    logger.info(f"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
+    logger.info(f"  Gradient Accumulation steps = {args.gradient_accumulation_steps}")
+    logger.info(f"  Total optimization steps = {args.max_train_steps}")
+
+    def save_weights(step):
+        # Create the pipeline using using the trained modules and save it.
+        if accelerator.is_main_process:
+            if args.train_text_encoder:
+                text_enc_model = accelerator.unwrap_model(text_encoder, keep_fp32_wrapper=True)
+            else:
+                text_enc_model = CLIPTextModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision)
+            scheduler = DDIMScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", clip_sample=False, set_alpha_to_one=False, steps_offset=1)
+            pipeline = StableDiffusionInpaintPipeline.from_pretrained(
+                args.pretrained_model_name_or_path,
+                unet=accelerator.unwrap_model(unet, keep_fp32_wrapper=True).to(torch.float16),
+                text_encoder=text_enc_model.to(torch.float16),
+                vae=AutoencoderKL.from_pretrained(
+                    args.pretrained_vae_name_or_path or args.pretrained_model_name_or_path,
+                    subfolder=None if args.pretrained_vae_name_or_path else "vae",
+                    revision=None if args.pretrained_vae_name_or_path else args.revision,
+                    # torch_dtype=torch.float16
+                ).half(),
+                safety_checker=None,
+                scheduler=scheduler,
+                torch_dtype=torch.float16,
+                revision=args.revision,
+            )
+            save_dir = os.path.join(args.output_dir, f"{step}")
+            pipeline.save_pretrained(save_dir)
+            with open(os.path.join(save_dir, "args.json"), "w") as f:
+                json.dump(args.__dict__, f, indent=2)
+
+            shutil.copy("train_inpainting_dreambooth.py", save_dir)
+
+            pipeline = pipeline.to(accelerator.device)
+            pipeline.set_progress_bar_config(disable=True)
+            for idx, concept in enumerate(args.concepts_list):
+                g_cuda = torch.Generator(device=accelerator.device).manual_seed(args.seed)
+                sample_dir = os.path.join(save_dir, "samples", str(idx))
+                os.makedirs(sample_dir, exist_ok=True)
+                # inp_img = Image.new("RGB", (512, 512), color=(0, 0, 0))  
+                inp_img = Image.open("/home/neil/Documents/diffusers/examples/dreambooth/512/a olive leather couch sofa Inside a Bedouin-inspired setting, soft sandy tones, hanging lanterns, and an endless view of sand dunes. .png")
+                inp_mask = ImageOps.invert(inp_img.split()[-1])
+                # inp_mask = Image.new("L", (512, 512), color=255)
+                with torch.inference_mode():
+                    for i in tqdm(range(args.n_save_sample), desc="Generating samples"):
+                        if i>1:
+                            images = pipeline(
+                                # prompt=concept["instance_prompt"],
+                                prompt="a olive leather couch sofa in a sleek and sophisticated mid century home interior, emphasizing clean lines and subtle design details.",
+                                image=inp_img.convert('RGB'),
+                                mask_image=inp_mask,
+                                negative_prompt=args.save_sample_negative_prompt,#"minimalist, sad, bad teeth, reflection, lens, lens flare, drawn, airbrush, blur, makeup, out of focus, hands, arms, veil, bad anatomy, mutation, deformed, cropped, signed, Mediocre, Screenshot, UI, watermark, cartoon",
+                                guidance_scale=args.save_guidance_scale,
+                                num_inference_steps=args.save_infer_steps,
+                                generator=g_cuda
+                            ).images
+                        else:
+                            images = pipeline(
+                                # prompt=concept["instance_prompt"],
+                                prompt="a olive leather couch sofa Inside a Bedouin-inspired setting, soft sandy tones, hanging lanterns, and an endless view of sand dunes. ",
+                                image=inp_img.convert('RGB'),
+                                mask_image=inp_mask,
+                                negative_prompt=args.save_sample_negative_prompt,
+                                guidance_scale=args.save_guidance_scale,
+                                num_inference_steps=args.save_infer_steps,
+                                generator=g_cuda
+                            ).images
+                        images[0].save(os.path.join(sample_dir, f"{i}.png"))
+            del pipeline
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+            print(f"[*] Weights saved at {save_dir}")
+            unet.to(torch.float32)
+            text_enc_model.to(torch.float32)
+
+    # Only show the progress bar once on each machine.
+    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)
+    progress_bar.set_description("Steps")
+    global_step = 0
+    loss_avg = AverageMeter()
+    text_enc_context = nullcontext() if args.train_text_encoder else torch.no_grad()
+    for epoch in range(args.num_train_epochs):
+        unet.train()
+        if args.train_text_encoder:
+            text_encoder.train()
+        random.shuffle(train_dataset.class_images_path)
+        for step, batch in enumerate(train_dataloader):
+            with accelerator.accumulate(unet):
+                # Convert images to latent space
+                with torch.no_grad():
+                    if not args.not_cache_latents:
+                        latent_dist = batch[0][0]
+                    else:
+                        latent_dist = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist
+                        masked_latent_dist = vae.encode(batch["masked_image_values"].to(dtype=weight_dtype)).latent_dist
+                        # torchvision.transforms.functional.to_pil_image(transforms.Normalize(mean=[-0.485/0.229, -0.456/0.224, -0.406/0.225],std=[1/0.229, 1/0.224, 1/0.255])(batch["pixel_values"][0])).save('batchim.png')
+                        # breakpoint()
+                    latents = latent_dist.sample() * 0.18215
+                    masked_image_latents = masked_latent_dist.sample() * 0.18215
+                    mask = F.interpolate(batch["mask_values"], scale_factor=1 / 8)
+
+                # Sample noise that we'll add to the latents
+                noise = torch.randn_like(latents)
+                bsz = latents.shape[0]
+                # Sample a random timestep for each image
+                timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device)
+                timesteps = timesteps.long()
+                
+                latents_copy = torch.clone(latents)
+                # Add noise to the latents according to the noise magnitude at each timestep
+                # (this is the forward diffusion process)
+                noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
+
+                # Get the text embedding for conditioning
+                with text_enc_context:
+                    if not args.not_cache_latents:
+                        if args.train_text_encoder:
+                            encoder_hidden_states = text_encoder(batch[0][1])[0]
+                        else:
+                            encoder_hidden_states = batch[0][1]
+                    else:
+                        encoder_hidden_states = text_encoder(batch["input_ids"])[0]
+
+                latent_model_input = torch.cat([noisy_latents, mask, masked_image_latents], dim=1)
+                # Predict the noise residual
+                noise_pred = unet(latent_model_input, timesteps, encoder_hidden_states).sample
+                
+                def latents_to_pil(latents):
+                    # bath of latents -> list of images
+                    latents = (1 / 0.18215) * latents
+                    with torch.no_grad():
+                        image = vae.decode(latents).sample
+                    image = (image / 2 + 0.5).clamp(0, 1)
+                    image = image.detach().cpu().permute(0, 2, 3, 1).numpy()
+                    images = (image * 255).round().astype("uint8")
+                    pil_images = [Image.fromarray(image) for image in images]
+                    return pil_images
+
+                def pil_to_latent(input_im):
+                    # Single image -> single latent in a batch (so size 1, 4, 64, 64)
+                    with torch.no_grad():
+                        latent = vae.encode(tfms.ToTensor()(input_im).unsqueeze(0).to(torch_device)*2-1) # Note scaling
+                    return 0.18215 * latent.latent_dist.sample()
+
+                if args.with_prior_preservation:
+                    # Chunk the noise and noise_pred into two parts and compute the loss on each part separately.
+                    noise_pred, noise_pred_prior = torch.chunk(noise_pred, 2, dim=0)
+                    noise, noise_prior = torch.chunk(noise, 2, dim=0)
+
+                    # Compute instance loss
+                    loss = F.mse_loss(noise_pred.float(), noise.float(), reduction="none").mean([1, 2, 3]).mean()
+
+                    # Compute prior loss
+                    prior_loss = F.mse_loss(noise_pred_prior.float(), noise_prior.float(), reduction="mean")
+
+                    # Add the prior loss to the instance loss.
+                    loss = loss + args.prior_loss_weight * prior_loss
+                else:
+                    loss = F.mse_loss(noise_pred.float(), noise.float(), reduction="mean")
+
+                    # im_pred = torch.cat([noise_scheduler.step(noise_pred[:1], timesteps[0], noisy_latents[:1]).prev_sample,
+                    # noise_scheduler.step(noise_pred[1:2], timesteps[1], noisy_latents[1:2]).prev_sample,
+                    # noise_scheduler.step(noise_pred[2:3], timesteps[2], noisy_latents[2:3]).prev_sample,
+                    # noise_scheduler.step(noise_pred[3:4], timesteps[3], noisy_latents[3:4]).prev_sample])
+
+                    # im_real = torch.cat([noise_scheduler.step(noise[:1], timesteps[0], noisy_latents[:1]).prev_sample,
+                    # noise_scheduler.step(noise[1:2], timesteps[1], noisy_latents[1:2]).prev_sample,
+                    # noise_scheduler.step(noise[2:3], timesteps[2], noisy_latents[2:3]).prev_sample,
+                    # noise_scheduler.step(noise[3:4], timesteps[3], noisy_latents[3:4]).prev_sample])
+
+                    # import numpy as np
+                    # import cv2 
+
+                    # # def tensor_to_canny(tensor):
+                    # #     canny_image = np.asarray(tensor.to('cpu'))
+                    # #     canny_image = cv2.Canny(np.uint8(canny_image), 100, 200)
+                    # #     # return Image.fromarray(canny_image)
+                    # #     return canny_image
+                    # # canny = batch["mask_values"][0][0]
+                    # # canny = tensor_to_canny(canny)
+                    # # Image.fromarray(canny).save('pcanny.png')
+                    # # canny = cv2.dilate(canny, np.ones((5,5),np.uint8), iterations=5)
+                    # # Image.fromarray(canny).save('pdilatecanny.png')
+                    # # breakpoint()
+                    # def get_band(tensor):
+                    #     canny_image = np.uint8(np.asarray(tensor.to('cpu')))
+                    #     dilated = cv2.dilate(canny_image, np.ones((5,5),np.uint8), iterations=10)
+                    #     # return torchvision.transforms.ToTensor()(dilated) * (tensor < 0.5)
+                    #     return dilated, canny_image
+                    # dilated, canny_image = get_band(batch["mask_values"][0][0])
+                    # # Image.fromarray(dilated).save('pdilated.png')
+                    # # Image.fromarray(canny_image).save('pcanny.png')
+                    # # Image.fromarray(dilated-canny_image).save('pxor.png')
+                    # band = dilated-canny_image
+                    # band = torch.from_numpy(band*0.3 + 1)
+                    # breakpoint()
+                    # im_pred = im_pred*band
+                    # im_real = im_real*band
+                    # breakpoint()
+                    # loss = F.mse_loss(im_real.float(), im_pred.float(), reduction="mean")
+
+                    # # latents_to_pil(latents_copy[:1])[0].save('ppp1.png')
+                    # # noise_scheduler.step(noise_pred, timesteps, latents, return_dict=False)[0]
+                    # # loss = F.mse_loss(vae.decode(noise_pred.float()).sample, vae.decode(noise.float()).sample, reduction="mean")
+
+# def generate_random_mask(image):
+#     # mask = torch.zeros_like(image[:1])
+#     # print('foobar', mask.shape)
+#     mask = image[-1].unsqueeze(0)
+#     # torchvision.transforms.functional.to_pil_image(mask).save('foomask2.png')
+
+#     # if random.uniform(0, 1) < 0.25:
+#     #     mask = torch.zeros_like(image[:1])
+#     #     mask.fill_(1.)
+#     # print('foobar',image.shape)
+#     # torchvision.transforms.functional.to_pil_image(image[:4]).save('fooimageinside2.png')  
+#     # print('foobar',image[:3].shape)
+#     masked_image = image[:-1] * (mask < 0.5)
+#     # torchvision.transforms.functional.to_pil_image(masked_image).save('foomaskedimage2.png')  
+#     return mask, masked_image
+
+                accelerator.backward(loss)
+                # if accelerator.sync_gradients:
+                #     params_to_clip = (
+                #         itertools.chain(unet.parameters(), text_encoder.parameters())
+                #         if args.train_text_encoder
+                #         else unet.parameters()
+                #     )
+                #     accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad(set_to_none=True)
+                loss_avg.update(loss.detach_(), bsz)
+
+            if not global_step % args.log_interval:
+                logs = {"loss": loss_avg.avg.item(), "lr": lr_scheduler.get_last_lr()[0]}
+                progress_bar.set_postfix(**logs)
+                accelerator.log(logs, step=global_step)
+
+            if global_step > 0 and not global_step % args.save_interval and global_step >= args.save_min_steps:
+                save_weights(global_step)
+
+            progress_bar.update(1)
+            global_step += 1
+
+            if global_step >= args.max_train_steps:
+                break
+
+        accelerator.wait_for_everyone()
+
+    save_weights(global_step)
+
+    accelerator.end_training()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

unet/config.json

@@ -0,0 +1,66 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.21.0.dev0",
+  "_name_or_path": "booth-ai/realisticVisionV20_v20_inpainting",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": 8,
+  "attention_type": "default",
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 768,
+  "cross_attention_norm": null,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 9,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 64,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D"
+  ],
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057028cbeb1ca5aacd36fc81ed7f2c5ae063330d4bfd37c1a42b63cc77d0e50d
+size 1719154104

vae/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.21.0.dev0",
+  "_name_or_path": "stabilityai/sd-vae-ft-mse",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 256,
+  "scaling_factor": 0.18215,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e4c08995484ee61270175e9e7a072b66a6e4eeb5f0c266667fe1f45b90daf9a
+size 167335342