init

.gitattributes

@@ -0,0 +1,39 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+.vscode/
+.DS_Store
+*.gif
+*.png
+*.jpg

README.md

@@ -0,0 +1,13 @@
+---
+title: MotionCtrl SVD
+emoji: 📉
+colorFrom: yellow
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.37.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,861 @@
+import argparse
+import os
+import tempfile
+
+import gradio as gr
+import numpy as np
+import torch
+from glob import glob
+from torchvision.transforms import CenterCrop, Compose, Resize
+
+from gradio_utils.camera_utils import CAMERA_MOTION_MODE, process_camera, create_relative
+
+from gradio_utils.utils import vis_camera
+from gradio_utils.motionctrl_cmcm_gradio import build_model, motionctrl_sample
+
+os.environ['KMP_DUPLICATE_LIB_OK']='True'
+SPACE_ID = os.environ.get('SPACE_ID', '')
+
+
+#### Description ####
+title = r"""<h1 align="center">MotionCtrl: A Unified and Flexible Motion Controller for Video Generation</h1>"""
+subtitle = r"""<h2 align="center">Deployed on SVD Generation</h2>"""
+
+description = r"""
+<b>Official Gradio demo</b> for <a href='https://github.com/TencentARC/MotionCtrl' target='_blank'><b>MotionCtrl: A Unified and Flexible Motion Controller for Video Generation</b></a>.<br>
+🔥 MotionCtrl is capable of independently and flexibly controling the camera motion and object motion of a generated video, with only a unified model.<br>
+🤗 Try to control the motion of the generated videos yourself!<br>
+❗❗❗ Please note **ONLY** Camera Motion Control in the current version of **MotionCtrl** deployed on **SVD** is avaliable.<br>
+"""
+# <div>
+# <img src="https://raw.githubusercontent.com/TencentARC/MotionCtrl/main/assets/svd/00_ibzz5-dxv2h.gif", width="300">
+# <img src="https://raw.githubusercontent.com/TencentARC/MotionCtrl/main/assets/svd/01_5guvn-0x6v2.gif", width="300">
+# <img src="https://raw.githubusercontent.com/TencentARC/MotionCtrl/main/assets/svd/12_sn7bz-0hcaf.gif", width="300">
+# <img src="https://raw.githubusercontent.com/TencentARC/MotionCtrl/main/assets/svd/13_3lyco-4ru8j.gif", width="300">
+# </div>
+article = r"""
+If MotionCtrl is helpful, please help to ⭐ the <a href='https://github.com/TencentARC/MotionCtrl' target='_blank'>Github Repo</a>. Thanks! 
+[![GitHub Stars](https://img.shields.io/github/stars/TencentARC%2FMotionCtrl
+)](https://github.com/TencentARC/MotionCtrl)
+
+---
+
+📝 **Citation**
+<br>
+If our work is useful for your research, please consider citing:
+```bibtex
+@inproceedings{wang2023motionctrl,
+  title={MotionCtrl: A Unified and Flexible Motion Controller for Video Generation},
+  author={Wang, Zhouxia and Yuan, Ziyang and Wang, Xintao and Chen, Tianshui and Xia, Menghan and Luo, Ping and Shan, Yin},
+  booktitle={arXiv preprint arXiv:2312.03641},
+  year={2023}
+}
+```
+
+📧 **Contact**
+<br>
+If you have any questions, please feel free to reach me out at <b>[email protected]</b>.
+
+"""
+css = """
+.gradio-container {width: 85% !important}
+.gr-monochrome-group {border-radius: 5px !important; border: revert-layer !important; border-width: 2px !important; color: black !important;}
+span.svelte-s1r2yt {font-size: 17px !important; font-weight: bold !important; color: #d30f2f !important;}
+button {border-radius: 8px !important;}
+.add_button {background-color: #4CAF50 !important;}
+.remove_button {background-color: #f44336 !important;}
+.clear_button {background-color: gray !important;}
+.mask_button_group {gap: 10px !important;}
+.video {height: 300px !important;}
+.image {height: 300px !important;}
+.video .wrap.svelte-lcpz3o {display: flex !important; align-items: center !important; justify-content: center !important;}
+.video .wrap.svelte-lcpz3o > :first-child {height: 100% !important;}
+.margin_center {width: 50% !important; margin: auto !important;}
+.jc_center {justify-content: center !important;}
+"""
+
+
+T_base = [
+            [1.,0.,0.],             ## W2C  x 的正方向： 相机朝左  left
+            [-1.,0.,0.],            ## W2C  x 的负方向： 相机朝右  right
+            [0., 1., 0.],           ## W2C  y 的正方向： 相机朝上  up     
+            [0.,-1.,0.],            ## W2C  y 的负方向： 相机朝下  down
+            [0.,0.,1.],             ## W2C  z 的正方向： 相机往前  zoom out
+            [0.,0.,-1.],            ## W2C  z 的负方向： 相机往前  zoom in
+        ]   
+radius = 1
+n = 16
+# step = 
+look_at = np.array([0, 0, 0.8]).reshape(3,1)
+# look_at = np.array([0, 0, 0.2]).reshape(3,1)
+
+T_list = []
+base_R = np.array([[1., 0., 0.],
+                [0., 1., 0.],
+                [0., 0., 1.]])
+res = [] 
+res_forsave = []
+T_range = 1.8
+
+
+
+for i in range(0, 16):
+    # theta = (1)*np.pi*i/n
+
+    R = base_R[:,:3]
+    T = np.array([0.,0.,1.]).reshape(3,1) * (i/n)*2
+    RT = np.concatenate([R,T], axis=1)
+    res.append(RT)
+    
+fig = vis_camera(res)
+    
+# MODE = ["camera motion control", "object motion control", "camera + object motion control"]
+MODE = ["control camera poses", "control object trajectory", "control both camera and object motion"]
+RESIZE_MODE = ['Center Crop To 576x1024', 'Keep original spatial ratio']
+DIY_MODE = ['Customized Mode 1: First A then B', 
+            'Customized Mode 2: Both A and B', 
+            'Customized Mode 3: RAW Camera Poses']
+
+## load default model
+num_frames = 14
+num_steps = 25
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device {device}")
+
+config = "configs/inference/config_motionctrl_cmcm.yaml"
+ckpt='checkpoints/motionctrl_svd.ckpt'
+if not os.path.exists(ckpt):
+    os.system(f'wget https://huggingface.co/TencentARC/MotionCtrl/resolve/main/motionctrl_svd.ckpt?download=true -P .')
+    os.system(f'mkdir checkpoints')
+    os.system(f'mv motionctrl_svd.ckpt?download=true {ckpt}')
+model = build_model(config, ckpt, device, num_frames, num_steps)
+width, height = 1024, 576 
+
+traj_list = [] 
+camera_dict = {
+                "motion":[],
+                "mode": "Customized Mode 1: First A then B",  # "First A then B", "Both A and B", "Custom"
+                "speed": 1.0,
+                "complex": None
+                }   
+
+def fn_vis_camera(camera_args):
+    global camera_dict, num_frames, width, height
+    RT = process_camera(camera_dict, camera_args, num_frames=num_frames, width=width, height=height) # [t, 3, 4]
+
+    rescale_T = 1.0
+    rescale_T = max(rescale_T, np.max(np.abs(RT[:,:,-1])) / 1.9)
+
+    fig = vis_camera(create_relative(RT), rescale_T=rescale_T)
+
+    vis_step3_prompt_generate = True
+    vis_generation_dec = True
+    vis_prompt = True
+    vis_num_samples = True
+    vis_seed = True
+    vis_start = True
+    vis_gen_video = True
+    vis_repeat_highlight = True
+
+    return fig, \
+            gr.update(visible=vis_step3_prompt_generate), \
+            gr.update(visible=vis_generation_dec), \
+            gr.update(visible=vis_prompt), \
+            gr.update(visible=vis_num_samples), \
+            gr.update(visible=vis_seed), \
+            gr.update(visible=vis_start), \
+            gr.update(visible=vis_gen_video, value=None), \
+            gr.update(visible=vis_repeat_highlight)
+
+def display_camera_info(camera_dict, camera_mode=None):
+    if camera_dict['complex'] is not None:
+        res = f"complex : {camera_dict['complex']}. "
+        res += f"speed : {camera_dict['speed']}. "
+    else:
+        res = ""
+        res += f"motion : {[_ for _ in camera_dict['motion']]}. "
+        res += f"speed : {camera_dict['speed']}. "
+        if camera_mode == CAMERA_MOTION_MODE[2]:
+            res += f"mode : {camera_dict['mode']}. "
+    return res
+
+def add_camera_motion(camera_motion, camera_mode):  
+    global camera_dict
+    if camera_dict['complex'] is not None:
+        camera_dict['complex'] = None
+    if camera_mode == CAMERA_MOTION_MODE[2] and len(camera_dict['motion']) <2:
+        camera_dict['motion'].append(camera_motion)
+    else:
+        camera_dict['motion']=[camera_motion]
+    
+    return display_camera_info(camera_dict, camera_mode)
+
+def add_complex_camera_motion(camera_motion):
+    global camera_dict
+    camera_dict['complex']=camera_motion
+    return display_camera_info(camera_dict)
+
+def input_raw_camera_pose(combine_type, camera_mode):
+    global camera_dict
+    camera_dict['mode'] = combine_type
+
+    vis_U = False
+    vis_D = False
+    vis_L = False
+    vis_R = False
+    vis_I = False
+    vis_O = False
+    vis_ACW = False
+    vis_CW = False
+    vis_speed = True
+    vis_combine3_des = True
+
+    return gr.update(value='1 0 0 0 0 1 0 0 0 0 1 0\n1 0 0 0 0 1 0 0 0 0 1 -0.225\n1 0 0 0 0 1 0 0 0 0 1 -0.45\n1 0 0 0 0 1 0 0 0 0 1 -0.675\n1 0 0 0 0 1 0 0 0 0 1 -0.9\n1 0 0 0 0 1 0 0 0 0 1 -1.125\n1 0 0 0 0 1 0 0 0 0 1 -1.35\n1 0 0 0 0 1 0 0 0 0 1 -1.575\n1 0 0 0 0 1 0 0 0 0 1 -1.8\n1 0 0 0 0 1 0 0 0 0 1 -2.025\n1 0 0 0 0 1 0 0 0 0 1 -2.25\n1 0 0 0 0 1 0 0 0 0 1 -2.475\n1 0 0 0 0 1 0 0 0 0 1 -2.7\n1 0 0 0 0 1 0 0 0 0 1 -2.925\n', max_lines=16, interactive=True), \
+            gr.update(visible=vis_U), \
+            gr.update(visible=vis_D), \
+            gr.update(visible=vis_L),\
+            gr.update(visible=vis_R), \
+            gr.update(visible=vis_I), \
+            gr.update(visible=vis_O), \
+            gr.update(visible=vis_ACW), \
+            gr.update(visible=vis_CW), \
+            gr.update(visible=vis_speed), \
+            gr.update(visible=vis_combine3_des)
+
+def change_camera_mode(combine_type, camera_mode):
+    global camera_dict
+    camera_dict['mode'] = combine_type
+
+    vis_U = True
+    vis_D = True
+    vis_L = True
+    vis_R = True
+    vis_I = True
+    vis_O = True
+    vis_ACW = True
+    vis_CW = True
+    vis_speed = True
+    vis_combine3_des = False
+
+    return display_camera_info(camera_dict, camera_mode), \
+            gr.update(visible=vis_U), \
+            gr.update(visible=vis_D), \
+            gr.update(visible=vis_L),\
+            gr.update(visible=vis_R), \
+            gr.update(visible=vis_I), \
+            gr.update(visible=vis_O), \
+            gr.update(visible=vis_ACW), \
+            gr.update(visible=vis_CW), \
+            gr.update(visible=vis_speed), \
+            gr.update(visible=vis_combine3_des)
+
+def change_camera_speed(camera_speed):
+    global camera_dict
+    camera_dict['speed'] = camera_speed
+    return display_camera_info(camera_dict)
+
+def reset_camera():
+    global camera_dict
+    camera_dict = {
+                    "motion":[],
+                    "mode": "Customized Mode 1: First A then B",
+                    "speed": 1.0,
+                    "complex": None
+                    }   
+    return display_camera_info(camera_dict)
+
+
+def visualized_camera_poses(step2_camera_motion):
+    reset_camera()
+
+    # generate video
+    vis_step3_prompt_generate = False
+    vis_generation_dec = False
+    vis_prompt = False
+    vis_num_samples = False
+    vis_seed = False
+    vis_start = False
+    vis_gen_video = False
+    vis_repeat_highlight = False
+
+    if step2_camera_motion == CAMERA_MOTION_MODE[0]:
+        vis_basic_camera_motion = True
+        vis_basic_camera_motion_des = True
+        vis_custom_camera_motion = False
+        vis_custom_run_status = False
+        vis_complex_camera_motion = False
+        vis_complex_camera_motion_des = False
+        vis_U = True
+        vis_D = True
+        vis_L = True
+        vis_R = True
+        vis_I = True
+        vis_O = True
+        vis_ACW = True
+        vis_CW = True
+        vis_combine1 = False
+        vis_combine2 = False
+        vis_combine3 = False
+        vis_combine3_des = False
+        vis_speed = True
+
+        vis_Pose_1, vis_Pose_2, vis_Pose_3, vis_Pose_4 = False, False, False, False
+        vis_Pose_5, vis_Pose_6, vis_Pose_7, vis_Pose_8 = False, False, False, False
+
+    elif step2_camera_motion == CAMERA_MOTION_MODE[1]:
+        vis_basic_camera_motion = False
+        vis_basic_camera_motion_des = False
+        vis_custom_camera_motion = False
+        vis_custom_run_status = False
+        vis_complex_camera_motion = True
+        vis_complex_camera_motion_des = True
+        vis_U = False
+        vis_D = False
+        vis_L = False
+        vis_R = False
+        vis_I = False
+        vis_O = False
+        vis_ACW = False
+        vis_CW = False
+        vis_combine1 = False
+        vis_combine2 = False
+        vis_combine3 = False
+        vis_combine3_des = False
+        vis_speed = True
+
+        vis_Pose_1, vis_Pose_2, vis_Pose_3, vis_Pose_4 = True, True, True, True
+        vis_Pose_5, vis_Pose_6, vis_Pose_7, vis_Pose_8 = True, True, True, True
+
+    else: # step2_camera_motion = CAMERA_MOTION_MODE[2]:
+        vis_basic_camera_motion = False
+        vis_basic_camera_motion_des = False
+        vis_custom_camera_motion = True
+        vis_custom_run_status = True
+        vis_complex_camera_motion = False
+        vis_complex_camera_motion_des = False
+        vis_U = False
+        vis_D = False
+        vis_L = False
+        vis_R = False
+        vis_I = False
+        vis_O = False
+        vis_ACW = False
+        vis_CW = False
+        vis_combine1 = True
+        vis_combine2 = True
+        vis_combine3 = True
+        vis_combine3_des = False
+        vis_speed = False
+
+        vis_Pose_1, vis_Pose_2, vis_Pose_3, vis_Pose_4 = False, False, False, False
+        vis_Pose_5, vis_Pose_6, vis_Pose_7, vis_Pose_8 = False, False, False, False
+
+    vis_camera_args = True
+    vis_camera_reset = True
+    vis_camera_vis = True
+    vis_vis_camera = True
+
+    return gr.update(visible=vis_basic_camera_motion), \
+            gr.update(visible=vis_basic_camera_motion_des), \
+            gr.update(visible=vis_custom_camera_motion), \
+            gr.update(visible=vis_custom_run_status), \
+            gr.update(visible=vis_complex_camera_motion), \
+            gr.update(visible=vis_complex_camera_motion_des), \
+            gr.update(visible=vis_U), gr.update(visible=vis_D), gr.update(visible=vis_L), gr.update(visible=vis_R), \
+            gr.update(visible=vis_I), gr.update(visible=vis_O), gr.update(visible=vis_ACW), gr.update(visible=vis_CW), \
+            gr.update(visible=vis_combine1), gr.update(visible=vis_combine2), gr.update(visible=vis_combine3), \
+            gr.update(visible=vis_combine3_des), \
+            gr.update(visible=vis_speed), \
+            gr.update(visible=vis_Pose_1), gr.update(visible=vis_Pose_2), gr.update(visible=vis_Pose_3), gr.update(visible=vis_Pose_4), \
+            gr.update(visible=vis_Pose_5), gr.update(visible=vis_Pose_6), gr.update(visible=vis_Pose_7), gr.update(visible=vis_Pose_8), \
+            gr.update(visible=vis_camera_args, value=None), \
+            gr.update(visible=vis_camera_reset), gr.update(visible=vis_camera_vis), \
+            gr.update(visible=vis_vis_camera, value=None), \
+            gr.update(visible=vis_step3_prompt_generate), \
+            gr.update(visible=vis_generation_dec), \
+            gr.update(visible=vis_prompt), \
+            gr.update(visible=vis_num_samples), \
+            gr.update(visible=vis_seed), \
+            gr.update(visible=vis_start), \
+            gr.update(visible=vis_gen_video), \
+            gr.update(visible=vis_repeat_highlight)
+
+
+def process_input_image(input_image, resize_mode):
+    global width, height
+    if resize_mode == RESIZE_MODE[0]:
+        height = 576
+        width = 1024
+        w, h = input_image.size
+        h_ratio = h / height
+        w_ratio = w / width
+
+        if h_ratio > w_ratio:
+            h = int(h / w_ratio)
+            if h < height:
+                h = height
+            input_image = Resize((h, width))(input_image)
+            
+        else:
+            w = int(w / h_ratio)
+            if w < width:
+                w = width
+            input_image = Resize((height, w))(input_image)
+
+        transformer = Compose([
+            # Resize(width),
+            CenterCrop((height, width)),
+        ])
+
+        input_image = transformer(input_image)
+    else:
+        w, h = input_image.size
+        if h > w:
+            height = 576
+            width = int(w * height / h)
+        else:
+            width = 1024
+            height = int(h * width / w)
+
+        input_image = Resize((height, width))(input_image)
+        # print(f'input_image size: {input_image.size}')
+
+    vis_step2_camera_motion = True
+    vis_step2_camera_motion_des = True
+    vis_camera_mode = True
+    vis_camera_info = True
+
+    ####
+    # camera motion control
+    vis_basic_camera_motion = False
+    vis_basic_camera_motion_des = False
+    vis_custom_camera_motion = False
+    vis_custom_run_status = False
+    vis_complex_camera_motion = False
+    vis_complex_camera_motion_des = False
+    vis_U = False
+    vis_D = False
+    vis_L = False
+    vis_R = False
+    vis_I = False
+    vis_O = False
+    vis_ACW = False
+    vis_CW = False
+    vis_combine1 = False
+    vis_combine2 = False
+    vis_combine3 = False
+    vis_combine3_des = False
+    vis_speed = False
+
+    vis_Pose_1, vis_Pose_2, vis_Pose_3, vis_Pose_4 = False, False, False, False
+    vis_Pose_5, vis_Pose_6, vis_Pose_7, vis_Pose_8 = False, False, False, False
+
+    vis_camera_args = False
+    vis_camera_reset = False
+    vis_camera_vis = False
+    vis_vis_camera = False
+
+    # generate video
+    vis_step3_prompt_generate = False
+    vis_generation_dec = False
+    vis_prompt = False
+    vis_num_samples = False
+    vis_seed = False
+    vis_start = False
+    vis_gen_video = False
+    vis_repeat_highlight = False
+    
+    return gr.update(visible=True, value=input_image, height=height, width=width), \
+            gr.update(visible=vis_step2_camera_motion), \
+            gr.update(visible=vis_step2_camera_motion_des), \
+            gr.update(visible=vis_camera_mode), \
+            gr.update(visible=vis_camera_info), \
+            gr.update(visible=vis_basic_camera_motion), \
+            gr.update(visible=vis_basic_camera_motion_des), \
+            gr.update(visible=vis_custom_camera_motion), \
+            gr.update(visible=vis_custom_run_status), \
+            gr.update(visible=vis_complex_camera_motion), \
+            gr.update(visible=vis_complex_camera_motion_des), \
+            gr.update(visible=vis_U), gr.update(visible=vis_D), gr.update(visible=vis_L), gr.update(visible=vis_R), \
+            gr.update(visible=vis_I), gr.update(visible=vis_O), gr.update(visible=vis_ACW), gr.update(visible=vis_CW), \
+            gr.update(visible=vis_combine1), gr.update(visible=vis_combine2), gr.update(visible=vis_combine3), \
+            gr.update(visible=vis_combine3_des), \
+            gr.update(visible=vis_speed), \
+            gr.update(visible=vis_Pose_1), gr.update(visible=vis_Pose_2), gr.update(visible=vis_Pose_3), gr.update(visible=vis_Pose_4), \
+            gr.update(visible=vis_Pose_5), gr.update(visible=vis_Pose_6), gr.update(visible=vis_Pose_7), gr.update(visible=vis_Pose_8), \
+            gr.update(visible=vis_camera_args, value=None), \
+            gr.update(visible=vis_camera_reset), gr.update(visible=vis_camera_vis), \
+            gr.update(visible=vis_vis_camera, value=None), \
+            gr.update(visible=vis_step3_prompt_generate), \
+            gr.update(visible=vis_generation_dec), \
+            gr.update(visible=vis_prompt), \
+            gr.update(visible=vis_num_samples), \
+            gr.update(visible=vis_seed), \
+            gr.update(visible=vis_start), \
+            gr.update(visible=vis_gen_video), \
+            gr.update(visible=vis_repeat_highlight)
+
+def model_run(input_image, fps_id, seed, n_samples, camera_args):
+    global model, device, camera_dict, num_frames, num_steps, width, height
+    RT = process_camera(camera_dict, camera_args, num_frames=num_frames, width=width, height=height).reshape(-1,12)
+
+    video_path = motionctrl_sample(
+        model=model,
+        image=input_image,
+        RT=RT,
+        num_frames=num_frames,
+        fps_id=fps_id,
+        decoding_t=1,
+        seed=seed,
+        sample_num=n_samples,
+        device=device
+    )
+
+    return video_path
+
+def main(args):
+    demo = gr.Blocks()
+    with demo:
+
+        gr.Markdown(title)
+        gr.Markdown(subtitle)
+        gr.Markdown(description)
+
+        with gr.Column():
+            
+            # step 0: Some useful tricks
+            gr.Markdown("## Step 0/3: Some Useful Tricks", show_label=False)
+            gr.HighlightedText(value=[("",""), (f"1. If the motion control is not obvious, try to increase the `Motion Speed`. \
+                                                \n 2. If the generated videos are distored severely, try to descrease the `Motion Speed` \
+                                                or increase `FPS`.", "Normal")],
+                                color_map={"Normal": "green", "Error": "red", "Clear clicks": "gray", "Add mask": "green", "Remove mask": "red"}, visible=True)
+
+            # step 2: input an image
+            step2_title = gr.Markdown("---\n## Step 1/3: Input an Image", show_label=False, visible=True)
+            step2_dec = gr.Markdown(f"\n 1. Upload an Image by `Drag` or Click `Upload Image`; \
+                                    \n 2. Click `{RESIZE_MODE[0]}` or `{RESIZE_MODE[1]}` to select the image resize mode. \
+                                    You will get a processed image and go into the next step. \
+                                    \n - `{RESIZE_MODE[0]}`: Our MotionCtrl is train on image with spatial size 576x1024. Choose `{RESIZE_MODE[0]}` can get better generated video. \
+                                    \n - `{RESIZE_MODE[1]}`: Choose `{RESIZE_MODE[1]}` if you want to generate video with the same spatial ratio as the input image.", 
+                                    show_label=False, visible=True)
+                                    
+            with gr.Row(equal_height=True):
+                with gr.Column(scale=2):
+                    input_image = gr.Image(type="pil", interactive=True, elem_id="input_image", elem_classes='image', visible=True)
+                    # process_input_image_button = gr.Button(value="Process Input Image", visible=False)
+                    with gr.Row():
+                        center_crop_botton = gr.Button(value=RESIZE_MODE[0], visible=True)
+                        keep_spatial_raition_botton = gr.Button(value=RESIZE_MODE[1], visible=True)
+                with gr.Column(scale=2):
+                    process_image = gr.Image(type="pil", interactive=False, elem_id="process_image", elem_classes='image', visible=False)
+            # step2_proceed_button = gr.Button(value="Proceed", visible=False)
+
+            
+            # step3 - camera motion control
+            step2_camera_motion = gr.Markdown("---\n## Step 2/3: Select the camera poses", show_label=False, visible=False)
+            step2_camera_motion_des = gr.Markdown(f"\n - {CAMERA_MOTION_MODE[0]}: Including 8 basic camera poses, such as pan up, pan down, zoom in, and zoom out. \
+                                                    \n - {CAMERA_MOTION_MODE[1]}: Complex camera poses extracted from the real videos. \
+                                                    \n - {CAMERA_MOTION_MODE[2]}: You can customize complex camera poses yourself by combining or fusing two of the eight basic camera poses or input RAW RT matrix. \
+                                                    \n - Click `Proceed` to go into next step", 
+                                                  show_label=False, visible=False)
+            camera_mode = gr.Radio(choices=CAMERA_MOTION_MODE, value=CAMERA_MOTION_MODE[0], label="Camera Motion Control Mode", interactive=True, visible=False)
+            camera_info = gr.Button(value="Proceed", visible=False)
+
+            with gr.Row():
+                with gr.Column():
+                    # step3.1 - camera motion control - basic
+                    basic_camera_motion = gr.Markdown("---\n### Basic Camera Poses", show_label=False, visible=False)
+                    basic_camera_motion_des = gr.Markdown(f"\n 1. Click one of the basic camera poses, such as `Pan Up`; \
+                                                            \n 2. Slide the `Motion speed` to get a speed value. The large the value, the fast the camera motion; \
+                                                            \n 3. Click `Visualize Camera and Proceed` to visualize the camera poses and go proceed; \
+                                                            \n 4. Click `Reset Camera` to reset the camera poses (If needed). ",
+                                                        show_label=False, visible=False)
+                    
+                    
+                    # step3.2 - camera motion control - provided complex
+                    complex_camera_motion = gr.Markdown("---\n### Provided Complex Camera Poses", show_label=False, visible=False)
+                    complex_camera_motion_des = gr.Markdown(f"\n 1. Click one of the complex camera poses, such as `Pose_1`; \
+                                                            \n 2. Click `Visualize Camera and Proceed` to visualize the camera poses and go proceed; \
+                                                            \n 3. Click `Reset Camera` to reset the camera poses (If needed). ",
+                                                        show_label=False, visible=False)
+
+                    # step3.3 - camera motion control - custom
+                    custom_camera_motion = gr.Markdown(f"---\n### {CAMERA_MOTION_MODE[2]}", show_label=False, visible=False)
+                    custom_run_status = gr.Markdown(f"\n 1. Click `{DIY_MODE[0]}`, `{DIY_MODE[1]}`, or `{DIY_MODE[2]}` \
+                                                    \n - `Customized Mode 1: First A then B`: For example, click `Pan Up` and `Pan Left`, the camera will first `Pan Up` and then `Pan Left`; \
+                                                    \n - `Customized Mode 2: Both A and B`: For example, click `Pan Up` and `Pan Left`, the camera will move towards the upper left corner; \
+                                                    \n - `{DIY_MODE[2]}`: Input the RAW RT matrix yourselves. \
+                                                    \n 2. Slide the `Motion speed` to get a speed value. The large the value, the fast the camera motion; \
+                                                    \n 3. Click `Visualize Camera and Proceed` to visualize the camera poses and go proceed; \
+                                                    \n 4. Click `Reset Camera` to reset the camera poses (If needed). ",
+                                                        show_label=False, visible=False)
+
+                    gr.HighlightedText(value=[("",""), ("1. Select two of the basic camera poses; 2. Select Customized Mode 1 OR Customized Mode 2. 3. Visualized Camera to show the customized camera poses", "Normal")],
+                                                        color_map={"Normal": "green", "Error": "red", "Clear clicks": "gray", "Add mask": "green", "Remove mask": "red"}, visible=False)
+                    
+                    with gr.Row():
+                        combine1 = gr.Button(value=DIY_MODE[0], visible=False)
+                        combine2 = gr.Button(value=DIY_MODE[1], visible=False)
+                        combine3 = gr.Button(value=DIY_MODE[2], visible=False)
+                    with gr.Row():
+                        combine3_des = gr.Markdown(f"---\n#### Input your camera pose in the following textbox. \
+                                                A total of 14 lines and each line contains 12 float number, indicated \
+                                                the RT matrix in the shape of 1x12. \
+                                                The example is RT matrix of ZOOM IN.", show_label=False, visible=False)
+
+                    with gr.Row():
+                        U = gr.Button(value="Pan Up", visible=False)
+                        D = gr.Button(value="Pan Down", visible=False)
+                        L = gr.Button(value="Pan Left", visible=False)
+                        R = gr.Button(value="Pan Right", visible=False)
+                    with gr.Row():
+                        I = gr.Button(value="Zoom In", visible=False)
+                        O = gr.Button(value="Zoom Out", visible=False)
+                        ACW = gr.Button(value="ACW", visible=False)
+                        CW = gr.Button(value="CW", visible=False)
+
+                    with gr.Row():    
+                        speed = gr.Slider(minimum=0, maximum=8, step=0.2, label="Motion Speed", value=1.0, visible=False)
+
+                    with gr.Row():
+                        Pose_1 = gr.Button(value="Pose_1", visible=False)
+                        Pose_2 = gr.Button(value="Pose_2", visible=False)
+                        Pose_3 = gr.Button(value="Pose_3", visible=False)
+                        Pose_4 = gr.Button(value="Pose_4", visible=False)
+                    with gr.Row():
+                        Pose_5 = gr.Button(value="Pose_5", visible=False)
+                        Pose_6 = gr.Button(value="Pose_6", visible=False)
+                        Pose_7 = gr.Button(value="Pose_7", visible=False)
+                        Pose_8 = gr.Button(value="Pose_8", visible=False)
+                
+                    with gr.Row():
+                        camera_args = gr.Textbox(value="Camera Type", label="Camera Type", visible=False)
+                    with gr.Row():
+                        camera_vis= gr.Button(value="Visualize Camera and Proceed", visible=False)
+                        camera_reset = gr.Button(value="Reset Camera", visible=False)
+                with gr.Column():
+                    vis_camera = gr.Plot(fig, label='Camera Poses', visible=False)
+
+            
+            # step4 - Generate videos
+            with gr.Row():
+                with gr.Column():
+                    step3_prompt_generate = gr.Markdown("---\n## Step 3/3: Generate videos", show_label=False, visible=False)
+                    generation_dec = gr.Markdown(f"\n 1. Set `FPS`.; \
+                                                    \n 2. Set `n_samples`; \
+                                                    \n 3. Set `seed`; \
+                                                    \n 4. Click `Start generation !` to generate videos; ", visible=False)
+                    # prompt = gr.Textbox(value="a dog sitting on grass", label="Prompt", interactive=True, visible=False)
+                    prompt = gr.Slider(minimum=5, maximum=30, step=1, label="FPS", value=10, visible=False)
+                    n_samples = gr.Number(value=2, precision=0, interactive=True, label="n_samples", visible=False)
+                    seed = gr.Number(value=1234, precision=0, interactive=True, label="Seed", visible=False)
+                    start = gr.Button(value="Start generation !", visible=False)
+                with gr.Column():
+                    gen_video = gr.Video(value=None, label="Generate Video", visible=False)
+                    repeat_highlight=gr.HighlightedText(value=[("",""), (f"1. If the motion control is not obvious, try to increase the `Motion Speed`. \
+                                                \n 2. If the generated videos are distored severely, try to descrease the `Motion Speed` \
+                                                or increase `FPS`.", "Normal")],
+                                color_map={"Normal": "green", "Error": "red", "Clear clicks": "gray", "Add mask": "green", "Remove mask": "red"}, visible=False)
+
+        center_crop_botton.click(
+            fn=process_input_image, 
+            inputs=[input_image, center_crop_botton], 
+            outputs=[
+                process_image,
+                step2_camera_motion, 
+                step2_camera_motion_des,
+                camera_mode, 
+                camera_info,
+                basic_camera_motion,
+                basic_camera_motion_des,
+                custom_camera_motion,
+                custom_run_status,
+                complex_camera_motion,
+                complex_camera_motion_des,
+                U, D, L, R, 
+                I, O, ACW, CW, 
+                combine1, combine2, combine3, combine3_des,
+                speed, 
+                Pose_1, Pose_2, Pose_3, Pose_4, 
+                Pose_5, Pose_6, Pose_7, Pose_8,
+                camera_args, 
+                camera_reset, camera_vis,
+                vis_camera,
+
+                step3_prompt_generate, 
+                generation_dec, 
+                prompt, 
+                n_samples, 
+                seed, start, gen_video, repeat_highlight])
+
+        keep_spatial_raition_botton.click(
+            fn=process_input_image, 
+            inputs=[input_image, keep_spatial_raition_botton], 
+            outputs=[
+                process_image,
+                step2_camera_motion, 
+                step2_camera_motion_des,
+                camera_mode, 
+                camera_info,
+                basic_camera_motion,
+                basic_camera_motion_des,
+                custom_camera_motion,
+                custom_run_status,
+                complex_camera_motion,
+                complex_camera_motion_des,
+                U, D, L, R, 
+                I, O, ACW, CW, 
+                combine1, combine2, combine3, combine3_des,
+                speed, 
+                Pose_1, Pose_2, Pose_3, Pose_4, 
+                Pose_5, Pose_6, Pose_7, Pose_8,
+                camera_args, 
+                camera_reset, camera_vis,
+                vis_camera,
+
+                step3_prompt_generate, 
+                generation_dec, 
+                prompt, 
+                n_samples, 
+                seed, start, gen_video, repeat_highlight])
+        
+
+        camera_info.click(
+            fn=visualized_camera_poses,
+            inputs=[camera_mode],
+            outputs=[basic_camera_motion,
+                     basic_camera_motion_des,
+                     custom_camera_motion,
+                     custom_run_status,
+                     complex_camera_motion,
+                     complex_camera_motion_des,
+                     U, D, L, R, 
+                     I, O, ACW, CW, 
+                     combine1, combine2, combine3, combine3_des,
+                     speed, 
+                     Pose_1, Pose_2, Pose_3, Pose_4, 
+                     Pose_5, Pose_6, Pose_7, Pose_8,
+                     camera_args, 
+                     camera_reset, camera_vis,
+                     vis_camera,
+                     step3_prompt_generate, generation_dec, prompt, n_samples, seed, start, gen_video, repeat_highlight],
+        )
+
+
+        U.click(fn=add_camera_motion, inputs=[U, camera_mode], outputs=camera_args)
+        D.click(fn=add_camera_motion, inputs=[D, camera_mode], outputs=camera_args)
+        L.click(fn=add_camera_motion, inputs=[L, camera_mode], outputs=camera_args)
+        R.click(fn=add_camera_motion, inputs=[R, camera_mode], outputs=camera_args)
+        I.click(fn=add_camera_motion, inputs=[I, camera_mode], outputs=camera_args)
+        O.click(fn=add_camera_motion, inputs=[O, camera_mode], outputs=camera_args)
+        ACW.click(fn=add_camera_motion, inputs=[ACW, camera_mode], outputs=camera_args)
+        CW.click(fn=add_camera_motion, inputs=[CW, camera_mode], outputs=camera_args)
+        speed.change(fn=change_camera_speed, inputs=speed, outputs=camera_args)
+        camera_reset.click(fn=reset_camera, inputs=None, outputs=[camera_args])
+
+        combine1.click(fn=change_camera_mode, 
+                       inputs=[combine1, camera_mode], 
+                       outputs=[camera_args,
+                                U, D, L, R, 
+                                I, O, ACW, CW, speed,
+                                combine3_des])
+        combine2.click(fn=change_camera_mode, 
+                       inputs=[combine2, camera_mode], 
+                       outputs=[camera_args,
+                                U, D, L, R, 
+                                I, O, ACW, CW, 
+                                speed,
+                                combine3_des])
+        combine3.click(fn=input_raw_camera_pose, 
+                       inputs=[combine3, camera_mode], 
+                       outputs=[camera_args,
+                                U, D, L, R, 
+                                I, O, ACW, CW, 
+                                speed, 
+                                combine3_des])
+
+        camera_vis.click(fn=fn_vis_camera, inputs=[camera_args], 
+                         outputs=[vis_camera, 
+                                  step3_prompt_generate, 
+                                  generation_dec,
+                                  prompt, 
+                                  n_samples, 
+                                  seed, 
+                                  start, 
+                                  gen_video,
+                                  repeat_highlight])
+
+        Pose_1.click(fn=add_complex_camera_motion, inputs=Pose_1, outputs=camera_args)
+        Pose_2.click(fn=add_complex_camera_motion, inputs=Pose_2, outputs=camera_args)
+        Pose_3.click(fn=add_complex_camera_motion, inputs=Pose_3, outputs=camera_args)
+        Pose_4.click(fn=add_complex_camera_motion, inputs=Pose_4, outputs=camera_args)
+        Pose_5.click(fn=add_complex_camera_motion, inputs=Pose_5, outputs=camera_args)
+        Pose_6.click(fn=add_complex_camera_motion, inputs=Pose_6, outputs=camera_args)
+        Pose_7.click(fn=add_complex_camera_motion, inputs=Pose_7, outputs=camera_args)
+        Pose_8.click(fn=add_complex_camera_motion, inputs=Pose_8, outputs=camera_args)
+
+
+        start.click(fn=model_run, 
+                    inputs=[process_image, prompt, seed, n_samples, camera_args], 
+                    outputs=gen_video)
+
+        # set example
+        gr.Markdown("## Examples")
+        examples = glob(os.path.join(os.path.dirname(__file__), "./assets/demo/images", "*.png"))
+        gr.Examples(
+            examples=examples,
+            inputs=[input_image],
+        )
+
+        gr.Markdown(article)
+
+    # demo.launch(server_name='0.0.0.0', share=False, server_port=args['server_port'])
+    # demo.queue(concurrency_count=1, max_size=10)
+    # demo.launch()
+    demo.queue(max_size=10).launch(**args)
+
+
+if __name__=="__main__":
+    parser = argparse.ArgumentParser()
+    # parser.add_argument("--port", type=int, default=12345)
+
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='0.0.0.0' if 'SPACE_ID' in os.environ else '127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+
+    args = parser.parse_args()
+
+    launch_kwargs = {}
+    launch_kwargs['server_name'] = args.listen
+
+    if args.username and args.password:
+        launch_kwargs['auth'] = (args.username, args.password)
+    if args.server_port:
+        launch_kwargs['server_port'] = args.server_port
+    if args.inbrowser:
+        launch_kwargs['inbrowser'] = args.inbrowser
+    if args.share:
+        launch_kwargs['share'] = args.share
+
+    main(launch_kwargs)

configs/inference/config_motionctrl_cmcm.yaml

@@ -0,0 +1,169 @@
+model:
+  base_learning_rate: 3.0e-5
+  target: sgm.motionctrl.camera_motion_control.CameraMotionControl
+  params:
+    ckpt_path: /group/30098/zhouxiawang/env/share/weights/svd/stable-video-diffusion-img2vid/svd.safetensors
+    scale_factor: 0.18215
+    input_key: video
+    no_cond_log: true
+    en_and_decode_n_samples_a_time: 1
+    use_ema: false
+    disable_first_stage_autocast: true
+
+    denoiser_config:
+      target: sgm.modules.diffusionmodules.denoiser.Denoiser
+      params:
+        scaling_config:
+          target: sgm.modules.diffusionmodules.denoiser_scaling.VScalingWithEDMcNoise
+
+    network_config:
+      target: sgm.modules.diffusionmodules.video_model.VideoUNet
+      params:
+        num_frames: 14
+        adm_in_channels: 768
+        num_classes: sequential
+        use_checkpoint: false
+        in_channels: 8
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [4, 2, 1]
+        num_res_blocks: 2
+        channel_mult: [1, 2, 4, 4]
+        num_head_channels: 64
+        use_linear_in_transformer: true
+        transformer_depth: 1
+        context_dim: 1024
+        spatial_transformer_attn_type: softmax-xformers
+        extra_ff_mix_layer: true
+        use_spatial_context: true
+        merge_strategy: learned_with_images
+        video_kernel_size: [3, 1, 1]
+
+    conditioner_config:
+      target: sgm.modules.GeneralConditioner
+      params:
+        emb_models:
+        - is_trainable: false
+          input_key: cond_frames_without_noise
+          ucg_rate: 0.1
+          target: sgm.modules.encoders.modules.FrozenOpenCLIPImagePredictionEmbedder
+          params:
+            n_cond_frames: 1
+            n_copies: 1
+            open_clip_embedding_config:
+              target: sgm.modules.encoders.modules.FrozenOpenCLIPImageEmbedder
+              params:
+                freeze: true
+                # version: "/apdcephfs_cq3/share_1290939/vg_zoo/dependencies/OpenCLIP-ViT-H-14-laion2B-s32B-b79K/blobs/9a78ef8e8c73fd0df621682e7a8e8eb36c6916cb3c16b291a082ecd52ab79cc4"
+
+        - input_key: fps_id
+          is_trainable: false
+          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
+          params:
+            outdim: 256
+
+        - input_key: motion_bucket_id
+          is_trainable: false
+          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
+          params:
+            outdim: 256
+
+        - input_key: cond_frames
+          is_trainable: false
+          ucg_rate: 0.1
+          target: sgm.modules.encoders.modules.VideoPredictionEmbedderWithEncoder
+          params:
+            disable_encoder_autocast: true
+            n_cond_frames: 1
+            n_copies: 1
+            is_ae: true
+            encoder_config:
+              target: sgm.models.autoencoder.AutoencoderKLModeOnly
+              params:
+                embed_dim: 4
+                monitor: val/rec_loss
+                ddconfig:
+                  attn_type: vanilla-xformers
+                  double_z: true
+                  z_channels: 4
+                  resolution: 256
+                  in_channels: 3
+                  out_ch: 3
+                  ch: 128
+                  ch_mult: [1, 2, 4, 4]
+                  num_res_blocks: 2
+                  attn_resolutions: []
+                  dropout: 0.0
+                lossconfig:
+                  target: torch.nn.Identity
+
+        - input_key: cond_aug
+          is_trainable: false
+          target: sgm.modules.encoders.modules.ConcatTimestepEmbedderND
+          params:
+            outdim: 256
+
+    first_stage_config:
+      target: sgm.models.autoencoder.AutoencodingEngine
+      params:
+        loss_config:
+          target: torch.nn.Identity
+        regularizer_config:
+          target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
+        encoder_config: 
+          target: sgm.modules.diffusionmodules.model.Encoder
+          params:
+            attn_type: vanilla
+            double_z: true
+            z_channels: 4
+            resolution: 256
+            in_channels: 3
+            out_ch: 3
+            ch: 128
+            ch_mult: [1, 2, 4, 4]
+            num_res_blocks: 2
+            attn_resolutions: []
+            dropout: 0.0
+        decoder_config:
+          target: sgm.modules.autoencoding.temporal_ae.VideoDecoder
+          params:
+            attn_type: vanilla
+            double_z: true
+            z_channels: 4
+            resolution: 256
+            in_channels: 3
+            out_ch: 3
+            ch: 128
+            ch_mult: [1, 2, 4, 4]
+            num_res_blocks: 2
+            attn_resolutions: []
+            dropout: 0.0
+            video_kernel_size: [3, 1, 1]
+
+    # loss_fn_config:
+    #   target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
+    #   params:
+    #     batch2model_keys: ['RT']  
+    #     loss_weighting_config:
+    #       target: sgm.modules.diffusionmodules.loss_weighting.VWeighting
+    #     sigma_sampler_config:
+    #       target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
+    #       params:
+    #         p_mean: 1.0
+    #         p_std: 1.6
+
+    sampler_config:
+      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
+      params:
+        num_steps: 25
+        discretization_config:
+          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
+          params:
+            sigma_max: 700.0
+
+        guider_config:
+          target: sgm.modules.diffusionmodules.guiders.LinearPredictionGuider
+          params:
+            num_frames: 14
+            max_scale: 2.5
+            min_scale: 1.0

configs/inference/motionctrl_run.sh

@@ -0,0 +1,36 @@
+
+
+ckpt='checkpoints/motionctrl_svd.ckpt'
+config='configs/inference/config_motionctrl_cmcm.yaml'
+
+height=576
+width=1024
+cond_aug=0.02
+
+fps=10
+
+image_input='examples/basic/eduardo-gorghetto-5auIBbcoRNw-unsplash.jpg'
+
+res_dir="outputs/motionctrl_svd"
+if [ ! -d $res_dir ]; then
+    mkdir -p $res_dir
+fi
+
+CUDA_VISIBLE_DEVICES=7 python main/inference/motionctrl_cmcm.py \
+--seed 12345 \
+--ckpt $ckpt \
+--config $config \
+--savedir $res_dir \
+--savefps 10 \
+--ddim_steps 25 \
+--frames 14 \
+--input $image_input \
+--fps $fps \
+--motion 127 \
+--cond_aug $cond_aug \
+--decoding_t 1 --resize \
+--height $height --width $width \
+--sample_num 2 \
+--transform \
+--pose_dir 'examples/camera_poses' \
+--speed 2.0 \

examples/camera_poses/test.txt

@@ -0,0 +1,42 @@
+gr.update(value=f'1.0 -4.493872218791495e-10 5.58983348497577e-09 1.9967236752904682e-09 \
+                     -4.493872218791495e-10 1.0 -6.144247333139674e-10 1.0815730533408896e-09 \
+                     5.58983348497577e-09 -6.144247333139674e-10 1.0 -7.984015226725205e-09 \n\
+                     0.9982863664627075 -0.0024742060340940952 0.05846544727683067 -0.024547122418880463 \
+                     0.002410230925306678 0.9999964237213135 0.0011647245846688747 -0.003784072818234563 \
+                     -0.05846811458468437 -0.0010218139505013824 0.9982887506484985 -0.09103696048259735 \n\
+                     0.9933298230171204 -0.006303737405687571 0.11513543128967285 -0.053876250982284546 \
+                     0.00586089538410306 0.9999741315841675 0.004184383898973465 -0.006566310301423073 \
+                    -0.115158811211586 -0.0034816779661923647 0.9933409690856934 -0.18525512516498566\n \
+                     0.9849286675453186 -0.013619760051369667 0.17242403328418732 -0.08322551101446152 \
+                     0.01256392989307642 0.9998950958251953 0.0072133541107177734 -0.004579910542815924 \
+                     -0.17250417172908783 -0.004938316997140646 0.9849964380264282 -0.28701746463775635 \n\
+                    0.9731453657150269 -0.022617166861891747 0.2290775030851364 -0.11655563861131668 \
+                     0.02060025744140148 0.9997251629829407 0.011192308738827705 -0.0017426757840439677\
+                     -0.2292676568031311 -0.006172688212245703 0.9733438491821289 -0.37736839056015015\n \
+                    0.9582399725914001 -0.03294993191957474 0.2840607464313507 -0.15743066370487213 \
+                     0.030182993039488792 0.9994447827339172 0.014113469049334526 -0.002769832033663988 \
+                     -0.28436803817749023 -0.004950287751853466 0.9587023854255676 -0.46959081292152405 \n \
+                    0.940129816532135 -0.03991429880261421 0.3384712040424347 -0.22889098525047302 \
+                     0.03725311905145645 0.9992027282714844 0.01435780432075262 -0.0028311305213719606 \
+                     -0.3387744128704071 -0.0008890923927538097 0.9408671855926514 -0.5631460547447205\n \
+                     0.9222924709320068 -0.044258520007133484 0.38395029306411743 -0.2986142039299011 \
+                     0.04110203683376312 0.9990199208259583 0.01642671786248684 0.0013055746676400304 \
+                     -0.38430097699165344 0.000630900904070586 0.9232076406478882 -0.6414245367050171\n \
+                     0.9061535000801086 -0.04851173609495163 0.4201577305793762 -0.3483412563800812 \
+                     0.04521748423576355 0.9988185167312622 0.017803886905312538 0.0010280977003276348 \
+                     -0.4205249547958374 0.0028654206544160843 0.907276451587677 -0.7144853472709656\n \
+                     0.8919307589530945 -0.05171844735741615 0.4492044746875763 -0.37905213236808777 \
+                     0.04818608984351158 0.9986518621444702 0.019300933927297592 0.00036871168413199484 \
+                     -0.44959715008735657 0.004430312197655439 0.8932204246520996,-0.7976372241973877\n \
+                     0.8792291879653931 -0.05425972864031792 0.47329893708229065 -0.39671003818511963 \
+                     0.05076585337519646 0.998507022857666 0.02016463316977024 0.001104982104152441 \
+                     -0.4736863970756531 0.00629808846861124 0.8806710243225098 -0.8874085545539856\n \
+                     0.8659296035766602 -0.0567130371928215 0.49694016575813293 -0.4097800552845001 \
+                     0.05366959795355797 0.9983500838279724 0.020415671169757843 0.0009228077251464128 \
+                     -0.497278094291687 0.008992047980427742 0.8675445914268494 -0.9762357473373413\n \
+                     0.8503361940383911 -0.055699657648801804 0.5232837200164795 -0.44268566370010376 \
+                     0.054582174867391586 0.9983546733856201 0.01757136546075344 0.005412018392235041 \
+                     -0.5234014391899109, 0.013620397076010704, 0.8519773483276367, -1.069865107536316 \n \
+                     0.836037814617157 -0.05214058235287666 0.5461887717247009 -0.4671085774898529 \
+                    0.05177384987473488 0.9985294938087463 0.01607322134077549 0.008980141952633858 \
+                      -0.5462236404418945 0.014840473420917988 0.8375079035758972 -1.1569048166275024\n', interactive=True), \

examples/camera_poses/test_camera_018f7907401f2fef.json

@@ -0,0 +1 @@
+[[1.0, -4.493872218791495e-10, 5.58983348497577e-09, 1.9967236752904682e-09, -4.493872218791495e-10, 1.0, -6.144247333139674e-10, 1.0815730533408896e-09, 5.58983348497577e-09, -6.144247333139674e-10, 1.0, -7.984015226725205e-09], [0.9982863664627075, -0.0024742060340940952, 0.05846544727683067, -0.024547122418880463, 0.002410230925306678, 0.9999964237213135, 0.0011647245846688747, -0.003784072818234563, -0.05846811458468437, -0.0010218139505013824, 0.9982887506484985, -0.09103696048259735], [0.9933298230171204, -0.006303737405687571, 0.11513543128967285, -0.053876250982284546, 0.00586089538410306, 0.9999741315841675, 0.004184383898973465, -0.006566310301423073, -0.115158811211586, -0.0034816779661923647, 0.9933409690856934, -0.18525512516498566], [0.9849286675453186, -0.013619760051369667, 0.17242403328418732, -0.08322551101446152, 0.01256392989307642, 0.9998950958251953, 0.0072133541107177734, -0.004579910542815924, -0.17250417172908783, -0.004938316997140646, 0.9849964380264282, -0.28701746463775635], [0.9731453657150269, -0.022617166861891747, 0.2290775030851364, -0.11655563861131668, 0.02060025744140148, 0.9997251629829407, 0.011192308738827705, -0.0017426757840439677, -0.2292676568031311, -0.006172688212245703, 0.9733438491821289, -0.37736839056015015], [0.9582399725914001, -0.03294993191957474, 0.2840607464313507, -0.15743066370487213, 0.030182993039488792, 0.9994447827339172, 0.014113469049334526, -0.002769832033663988, -0.28436803817749023, -0.004950287751853466, 0.9587023854255676, -0.46959081292152405], [0.940129816532135, -0.03991429880261421, 0.3384712040424347, -0.22889098525047302, 0.03725311905145645, 0.9992027282714844, 0.01435780432075262, -0.0028311305213719606, -0.3387744128704071, -0.0008890923927538097, 0.9408671855926514, -0.5631460547447205], [0.9222924709320068, -0.044258520007133484, 0.38395029306411743, -0.2986142039299011, 0.04110203683376312, 0.9990199208259583, 0.01642671786248684, 0.0013055746676400304, -0.38430097699165344, 0.000630900904070586, 0.9232076406478882, -0.6414245367050171], [0.9061535000801086, -0.04851173609495163, 0.4201577305793762, -0.3483412563800812, 0.04521748423576355, 0.9988185167312622, 0.017803886905312538, 0.0010280977003276348, -0.4205249547958374, 0.0028654206544160843, 0.907276451587677, -0.7144853472709656], [0.8919307589530945, -0.05171844735741615, 0.4492044746875763, -0.37905213236808777, 0.04818608984351158, 0.9986518621444702, 0.019300933927297592, 0.00036871168413199484, -0.44959715008735657, 0.004430312197655439, 0.8932204246520996, -0.7976372241973877], [0.8792291879653931, -0.05425972864031792, 0.47329893708229065, -0.39671003818511963, 0.05076585337519646, 0.998507022857666, 0.02016463316977024, 0.001104982104152441, -0.4736863970756531, 0.00629808846861124, 0.8806710243225098, -0.8874085545539856], [0.8659296035766602, -0.0567130371928215, 0.49694016575813293, -0.4097800552845001, 0.05366959795355797, 0.9983500838279724, 0.020415671169757843, 0.0009228077251464128, -0.497278094291687, 0.008992047980427742, 0.8675445914268494, -0.9762357473373413], [0.8503361940383911, -0.055699657648801804, 0.5232837200164795, -0.44268566370010376, 0.054582174867391586, 0.9983546733856201, 0.01757136546075344, 0.005412018392235041, -0.5234014391899109, 0.013620397076010704, 0.8519773483276367, -1.069865107536316], [0.836037814617157, -0.05214058235287666, 0.5461887717247009, -0.4671085774898529, 0.05177384987473488, 0.9985294938087463, 0.01607322134077549, 0.008980141952633858, -0.5462236404418945, 0.014840473420917988, 0.8375079035758972, -1.1569048166275024], [0.82603919506073, -0.04987695440649986, 0.5614013671875, -0.4677649438381195, 0.05124447122216225, 0.9985973834991455, 0.013318539597094059, 0.012170637026429176, -0.5612781643867493, 0.017767081037163734, 0.8274364471435547, -1.2651430368423462], [0.8179472088813782, -0.0496118925511837, 0.573150098323822, -0.45822662115097046, 0.052784956991672516, 0.9985441565513611, 0.011104168370366096, 0.018991567194461823, -0.5728666186332703, 0.0211710836738348, 0.8193751573562622, -1.3895009756088257]]

examples/camera_poses/test_camera_088b93f15ca8745d.json

@@ -0,0 +1 @@
+[[0.9999999403953552, 3.8618797049139175e-10, -1.3441345814158012e-08, 1.3928219289027766e-07, 3.8618797049139175e-10, 1.0, -4.134579345560496e-10, -6.074658998045379e-09, -1.3441345814158012e-08, -4.134579345560496e-10, 1.0, 7.038884319854333e-08], [0.9994913339614868, 0.003077245783060789, -0.031741149723529816, 0.08338673412799835, -0.0030815028585493565, 0.999995231628418, -8.520588744431734e-05, 0.006532138213515282, 0.0317407064139843, 0.00018297435599379241, 0.9994961619377136, -0.02256060019135475], [0.9979938268661499, 0.0051255361177027225, -0.06310292333364487, 0.18344485759735107, -0.005117486696690321, 0.9999868869781494, 0.00028916727751493454, 0.018134046345949173, 0.06310353428125381, 3.434090831433423e-05, 0.9980069994926453, -0.030579563230276108], [0.9954646825790405, 0.00820203311741352, -0.0947771891951561, 0.29663264751434326, -0.00811922550201416, 0.9999662041664124, 0.0012593322899192572, 0.02404301054775715, 0.09478426724672318, -0.0004841022891923785, 0.9954977035522461, -0.02678978443145752], [0.9913660883903503, 0.012001598253846169, -0.13057230412960052, 0.4076530337333679, -0.011968829669058323, 0.999927818775177, 0.0010357286082580686, 0.024977533146739006, 0.1305752843618393, 0.0005360084469430149, 0.9914382100105286, -0.010779343545436859], [0.985666811466217, 0.017323914915323257, -0.16781197488307953, 0.509911060333252, -0.017399737611413002, 0.9998481273651123, 0.0010186078725382686, 0.023117201402783394, 0.16780413687229156, 0.0019158748909831047, 0.9858185052871704, 0.018053216859698296], [0.9784473180770874, 0.022585421800613403, -0.20525763928890228, 0.5957884192466736, -0.022850200533866882, 0.9997382760047913, 0.0010805513011291623, 0.020451901480555534, 0.2052282989025116, 0.003632916137576103, 0.9787073731422424, 0.03460140898823738], [0.9711515307426453, 0.026846906170248985, -0.23694702982902527, 0.6832671165466309, -0.02745947800576687, 0.999622642993927, 0.0007151798927225173, 0.012211678549647331, 0.23687675595283508, 0.005811895243823528, 0.971522331237793, 0.03236595541238785], [0.9641746878623962, 0.030338184908032417, -0.26352745294570923, 0.7764986157417297, -0.031404945999383926, 0.9995067715644836, 0.0001645474840188399, 0.0011497576488181949, 0.26340243220329285, 0.008117412216961384, 0.964651882648468, 0.022656364366412163], [0.9573631882667542, 0.0335896760225296, -0.2869274914264679, 0.8815275430679321, -0.03532479330897331, 0.9993755221366882, -0.0008711823611520231, -0.003618708113208413, 0.2867189943790436, 0.010969695635139942, 0.9579519033432007, 0.005283573176711798], [0.9507063627243042, 0.036557890474796295, -0.3079299330711365, 0.9931321740150452, -0.03846294432878494, 0.9992600679397583, -0.00011733790597645566, 0.0018704120302572846, 0.30769774317741394, 0.01195544097572565, 0.9514090418815613, -0.035360634326934814], [0.9448517560958862, 0.039408694952726364, -0.3251185715198517, 1.1025006771087646, -0.041503626853227615, 0.9991382360458374, 0.0004919985658489168, 0.007425118237733841, 0.32485777139663696, 0.013028733432292938, 0.9456731081008911, -0.09869624674320221], [0.940796971321106, 0.04081147164106369, -0.33650481700897217, 1.1961394548416138, -0.0429220013320446, 0.9990777373313904, 0.0011677180882543325, 0.019955899566411972, 0.336242139339447, 0.013344875536859035, 0.9416810274124146, -0.16835527122020721], [0.9376427531242371, 0.04111124947667122, -0.3451607823371887, 1.2392503023147583, -0.043144747614860535, 0.9990671873092651, 0.0017920633545145392, 0.03982722759246826, 0.34491249918937683, 0.013211555778980255, 0.938541829586029, -0.24618202447891235], [0.9353355765342712, 0.04122937470674515, -0.3513509929180145, 1.285768747329712, -0.043183211237192154, 0.9990646243095398, 0.0022769556380808353, 0.06841164082288742, 0.3511161506175995, 0.01304274145513773, 0.936241090297699, -0.3213619291782379], [0.9342393279075623, 0.041213057935237885, -0.3542574644088745, 1.3363462686538696, -0.04236872121691704, 0.9990919232368469, 0.0044970144517719746, 0.08925694227218628, 0.35412102937698364, 0.010808154009282589, 0.9351370930671692, -0.40201041102409363]]

examples/camera_poses/test_camera_1424acd0007d40b5.json

@@ -0,0 +1 @@
+[[1.0, 9.44418099280142e-10, 3.889182664806867e-08, 6.214055492392845e-09, 9.44418099280142e-10, 1.0, -1.0644604121756718e-11, -7.621465680784922e-10, 3.889182664806867e-08, -1.0644604121756718e-11, 1.0, -2.7145965475483536e-08], [0.9873979091644287, -0.007892023772001266, 0.15806053578853607, 0.4749181270599365, 0.008024877868592739, 0.9999678134918213, -0.00020230526570230722, 0.1585356593132019, -0.15805381536483765, 0.0014681711327284575, 0.9874294400215149, -0.2091633826494217], [0.9708925485610962, -0.011486345902085304, 0.23923994600772858, 0.8120080828666687, 0.012198254466056824, 0.9999244809150696, -0.0014952132478356361, 0.2486257702112198, -0.23920467495918274, 0.004370000213384628, 0.9709593057632446, -0.5957822799682617], [0.9619541168212891, -0.013188007287681103, 0.2728927433490753, 1.1486873626708984, 0.014017474837601185, 0.9999011754989624, -0.001090032048523426, 0.3114692270755768, -0.2728513777256012, 0.0048738280311226845, 0.962043821811676, -1.0323039293289185], [0.9586812257766724, -0.013936692848801613, 0.284140944480896, 1.5948307514190674, 0.014867136254906654, 0.9998888373374939, -0.0011181083973497152, 0.36000898480415344, -0.2840937674045563, 0.0052962712943553925, 0.958781898021698, -1.4377187490463257], [0.9583359360694885, -0.011928150430321693, 0.28539448976516724, 2.002793788909912, 0.014221147634088993, 0.9998810887336731, -0.0059633455239236355, 0.35464340448379517, -0.28528934717178345, 0.009773525409400463, 0.9583916068077087, -1.8953297138214111], [0.9584393501281738, -0.010862396098673344, 0.28508952260017395, 2.3351645469665527, 0.012857729569077492, 0.9999041557312012, -0.005128204356878996, 0.38934090733528137, -0.28500640392303467, 0.008580676279962063, 0.9584871530532837, -2.4214961528778076], [0.9587277173995972, -0.009760312736034393, 0.28415825963020325, 2.6858017444610596, 0.012186127714812756, 0.9999027848243713, -0.0067702098749578, 0.4173329174518585, -0.28406453132629395, 0.009953574277460575, 0.9587535262107849, -3.030754327774048], [0.9589635729789734, -0.0070899901911616325, 0.2834406793117523, 2.8917219638824463, 0.010482418350875378, 0.9998904466629028, -0.01045384630560875, 0.4001043438911438, -0.28333547711372375, 0.012996001169085503, 0.9589328169822693, -3.6960957050323486], [0.9590328931808472, -0.005921780597418547, 0.2832328677177429, 3.034579038619995, 0.00947173498570919, 0.9998928308486938, -0.01116593275219202, 0.4193899631500244, -0.2831363379955292, 0.013391205109655857, 0.9589861631393433, -4.384733200073242], [0.9593284726142883, -0.004661113955080509, 0.28225383162498474, 3.2042288780212402, 0.0076906089670956135, 0.9999240636825562, -0.009626304730772972, 0.4752484858036041, -0.28218749165534973, 0.011405492201447487, 0.9592914581298828, -5.098723411560059], [0.9591755867004395, -0.0035665074829012156, 0.2827887237071991, 3.263953924179077, 0.0062992447055876255, 0.9999418258666992, -0.008754877373576164, 0.4543868899345398, -0.282740980386734, 0.010178821161389351, 0.95914226770401, -5.7807512283325195], [0.9591742753982544, -0.003413048107177019, 0.2827949523925781, 3.3116462230682373, 0.003146615345031023, 0.9999940991401672, 0.0013963348465040326, 0.4299861788749695, -0.28279799222946167, -0.0004494813329074532, 0.9591793417930603, -6.478931903839111], [0.9585762619972229, -0.002857929328456521, 0.28482162952423096, 3.4120190143585205, -0.008201238699257374, 0.9992581605911255, 0.0376281812787056, 0.21596357226371765, -0.2847178280353546, -0.03840537369251251, 0.957841694355011, -7.178638935089111], [0.9572952389717102, -0.002719884505495429, 0.28909942507743835, 3.4662365913391113, -0.030634215101599693, 0.9933721423149109, 0.11078491061925888, -0.29767531156539917, -0.2874845862388611, -0.11491020023822784, 0.9508671164512634, -7.794575214385986], [0.9545961618423462, -0.005338957067579031, 0.29785510897636414, 3.5083835124969482, -0.06037351116538048, 0.9756243824958801, 0.2109788954257965, -1.0968165397644043, -0.29172107577323914, -0.21938219666481018, 0.9310050010681152, -8.306528091430664]]

examples/camera_poses/test_camera_D.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.2,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.28750000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.37500000000000006,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.4625000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.55,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.6375000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.7250000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.8125000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.9000000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -0.9875000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -1.0750000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -1.1625000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -1.2500000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -1.3375000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -1.4250000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        -1.5125000000000004,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ]
+]

examples/camera_poses/test_camera_I.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.2
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.28750000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.37500000000000006
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.4625000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.55
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.6375000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.7250000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.8125000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.9000000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.9875000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.0750000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.1625000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.2500000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.3375000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.4250000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.5125000000000004
+    ]
+]

examples/camera_poses/test_camera_I_0.2x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.022500000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.045000000000000005
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.0675
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.09000000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.11250000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.135
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.15750000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.18000000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.2025
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.22500000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.24750000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.27
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.29250000000000004
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.31500000000000006
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.3375
+    ]
+]

examples/camera_poses/test_camera_I_0.4x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.045000000000000005
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.09000000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.135
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.18000000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.22500000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.27
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.31500000000000006
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.36000000000000004
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.405
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.45000000000000007
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.49500000000000005
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.54
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.5850000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.6300000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.675
+    ]
+]

examples/camera_poses/test_camera_I_1.0x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.1125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.225
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.3375
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.45
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.5625
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.675
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.7875
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.9
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.0125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.2375
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.35
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.4625000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.575
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.6875
+    ]
+]

examples/camera_poses/test_camera_I_2.0x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.225
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.45
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.675
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -0.9
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.35
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.575
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -1.8
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -2.025
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -2.25
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -2.475
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -2.7
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -2.9250000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -3.15
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        -3.375
+    ]
+]

examples/camera_poses/test_camera_L.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.2,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.28750000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.37500000000000006,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.4625000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.55,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.6375000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.7250000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.8125000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.9000000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.9875000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        1.0750000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        1.1625000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        1.2500000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        1.3375000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        1.4250000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        1.5125000000000004,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ]
+]

examples/camera_poses/test_camera_O.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.2
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.28750000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.37500000000000006
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.4625000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.55
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.6375000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.7250000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.8125000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.9000000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.9875000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0750000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.1625000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.2500000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.3375000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.4250000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.5125000000000004
+    ]
+]

examples/camera_poses/test_camera_O_0.2x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.022500000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.045000000000000005
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0675
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.09000000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.11250000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.135
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.15750000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.18000000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.2025
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.22500000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.24750000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.27
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.29250000000000004
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.31500000000000006
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.3375
+    ]
+]

examples/camera_poses/test_camera_O_0.4x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.045000000000000005
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.09000000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.135
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.18000000000000002
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.22500000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.27
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.31500000000000006
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.36000000000000004
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.405
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.45000000000000007
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.49500000000000005
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.54
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.5850000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.6300000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.675
+    ]
+]

examples/camera_poses/test_camera_O_1.0x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.1125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.225
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.3375
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.45
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.5625
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.675
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.7875
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.9
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.0125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.2375
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.35
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.4625000000000001
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.575
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.6875
+    ]
+]

examples/camera_poses/test_camera_O_2.0x.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.225
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.45
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.675
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.9
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.125
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.35
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.575
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.8
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        2.025
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        2.25
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        2.475
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        2.7
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        2.9250000000000003
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        3.15
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        3.375
+    ]
+]

examples/camera_poses/test_camera_R.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.2,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.28750000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.37500000000000006,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.4625000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.55,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.6375000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.7250000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.8125000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.9000000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -0.9875000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -1.0750000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -1.1625000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -1.2500000000000002,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -1.3375000000000001,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -1.4250000000000003,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        -1.5125000000000004,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ]
+]

examples/camera_poses/test_camera_Round-RI-120.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.8
+    ],
+    [
+        0.9914448613738104,
+        0.0,
+        0.13052619222005157,
+        0.13052619222005157,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.13052619222005157,
+        0.0,
+        0.9914448613738104,
+        1.7914448613738103
+    ],
+    [
+        0.9659258262890683,
+        0.0,
+        0.25881904510252074,
+        0.25881904510252074,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.25881904510252074,
+        0.0,
+        0.9659258262890683,
+        1.7659258262890685
+    ],
+    [
+        0.9238795325112867,
+        0.0,
+        0.3826834323650898,
+        0.3826834323650898,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.3826834323650898,
+        0.0,
+        0.9238795325112867,
+        1.7238795325112868
+    ],
+    [
+        0.8660254037844387,
+        0.0,
+        0.49999999999999994,
+        0.49999999999999994,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.49999999999999994,
+        0.0,
+        0.8660254037844387,
+        1.6660254037844386
+    ],
+    [
+        0.7933533402912353,
+        0.0,
+        0.6087614290087205,
+        0.6087614290087205,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.6087614290087205,
+        0.0,
+        0.7933533402912353,
+        1.5933533402912352
+    ],
+    [
+        0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        0.7071067811865476,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        1.5071067811865477
+    ],
+    [
+        0.6087614290087207,
+        0.0,
+        0.7933533402912352,
+        0.7933533402912352,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7933533402912352,
+        0.0,
+        0.6087614290087207,
+        1.4087614290087207
+    ],
+    [
+        0.5000000000000001,
+        0.0,
+        0.8660254037844386,
+        0.8660254037844386,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.8660254037844386,
+        0.0,
+        0.5000000000000001,
+        1.3000000000000003
+    ],
+    [
+        0.38268343236508984,
+        0.0,
+        0.9238795325112867,
+        0.9238795325112867,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9238795325112867,
+        0.0,
+        0.38268343236508984,
+        1.1826834323650899
+    ],
+    [
+        0.25881904510252096,
+        0.0,
+        0.9659258262890682,
+        0.9659258262890682,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9659258262890682,
+        0.0,
+        0.25881904510252096,
+        1.058819045102521
+    ],
+    [
+        0.1305261922200517,
+        0.0,
+        0.9914448613738104,
+        0.9914448613738104,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9914448613738104,
+        0.0,
+        0.1305261922200517,
+        0.9305261922200517
+    ],
+    [
+        6.123233995736766e-17,
+        0.0,
+        1.0,
+        1.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -1.0,
+        0.0,
+        6.123233995736766e-17,
+        0.8000000000000002
+    ],
+    [
+        -0.13052619222005138,
+        0.0,
+        0.9914448613738105,
+        0.9914448613738105,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9914448613738105,
+        0.0,
+        -0.13052619222005138,
+        0.6694738077799487
+    ],
+    [
+        -0.25881904510252063,
+        0.0,
+        0.9659258262890683,
+        0.9659258262890683,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9659258262890683,
+        0.0,
+        -0.25881904510252063,
+        0.5411809548974794
+    ],
+    [
+        -0.3826834323650895,
+        0.0,
+        0.9238795325112868,
+        0.9238795325112868,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9238795325112868,
+        0.0,
+        -0.3826834323650895,
+        0.41731656763491054
+    ]
+]

examples/camera_poses/test_camera_Round-RI.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.7000000000000002
+    ],
+    [
+        0.9807852804032304,
+        0.0,
+        0.19509032201612825,
+        0.17558128981451543,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.19509032201612825,
+        0.0,
+        0.9807852804032304,
+        1.6827067523629076
+    ],
+    [
+        0.9238795325112867,
+        0.0,
+        0.3826834323650898,
+        0.3444150891285808,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.3826834323650898,
+        0.0,
+        0.9238795325112867,
+        1.631491579260158
+    ],
+    [
+        0.8314696123025452,
+        0.0,
+        0.5555702330196022,
+        0.500013209717642,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.5555702330196022,
+        0.0,
+        0.8314696123025452,
+        1.5483226510722907
+    ],
+    [
+        0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        0.6363961030678928,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        1.436396103067893
+    ],
+    [
+        0.5555702330196023,
+        0.0,
+        0.8314696123025452,
+        0.7483226510722907,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.8314696123025452,
+        0.0,
+        0.5555702330196023,
+        1.3000132097176422
+    ],
+    [
+        0.38268343236508984,
+        0.0,
+        0.9238795325112867,
+        0.831491579260158,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9238795325112867,
+        0.0,
+        0.38268343236508984,
+        1.144415089128581
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.8827067523629074,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.9755812898145155
+    ],
+    [
+        6.123233995736766e-17,
+        0.0,
+        1.0,
+        0.9,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -1.0,
+        0.0,
+        6.123233995736766e-17,
+        0.8
+    ],
+    [
+        -0.1950903220161282,
+        0.0,
+        0.9807852804032304,
+        0.8827067523629074,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        -0.1950903220161282,
+        0.6244187101854847
+    ],
+    [
+        -0.3826834323650897,
+        0.0,
+        0.9238795325112867,
+        0.831491579260158,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9238795325112867,
+        0.0,
+        -0.3826834323650897,
+        0.4555849108714193
+    ],
+    [
+        -0.555570233019602,
+        0.0,
+        0.8314696123025453,
+        0.7483226510722908,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.8314696123025453,
+        0.0,
+        -0.555570233019602,
+        0.2999867902823583
+    ],
+    [
+        -0.7071067811865475,
+        0.0,
+        0.7071067811865476,
+        0.6363961030678928,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7071067811865476,
+        0.0,
+        -0.7071067811865475,
+        0.1636038969321073
+    ],
+    [
+        -0.8314696123025453,
+        0.0,
+        0.5555702330196022,
+        0.500013209717642,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.5555702330196022,
+        0.0,
+        -0.8314696123025453,
+        0.051677348927709255
+    ],
+    [
+        -0.9238795325112867,
+        0.0,
+        0.3826834323650899,
+        0.34441508912858093,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.3826834323650899,
+        0.0,
+        -0.9238795325112867,
+        -0.031491579260158
+    ],
+    [
+        -0.9807852804032304,
+        0.0,
+        0.1950903220161286,
+        0.17558128981451576,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.1950903220161286,
+        0.0,
+        -0.9807852804032304,
+        -0.08270675236290737
+    ]
+]

examples/camera_poses/test_camera_Round-RI_90.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        1.7000000000000002
+    ],
+    [
+        0.9951847266721969,
+        0.0,
+        0.0980171403295606,
+        0.08821542629660455,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.0980171403295606,
+        0.0,
+        0.9951847266721969,
+        1.6956662540049772
+    ],
+    [
+        0.9807852804032304,
+        0.0,
+        0.19509032201612825,
+        0.17558128981451543,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.19509032201612825,
+        0.0,
+        0.9807852804032304,
+        1.6827067523629076
+    ],
+    [
+        0.9569403357322088,
+        0.0,
+        0.29028467725446233,
+        0.2612562095290161,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.29028467725446233,
+        0.0,
+        0.9569403357322088,
+        1.661246302158988
+    ],
+    [
+        0.9238795325112867,
+        0.0,
+        0.3826834323650898,
+        0.3444150891285808,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.3826834323650898,
+        0.0,
+        0.9238795325112867,
+        1.631491579260158
+    ],
+    [
+        0.881921264348355,
+        0.0,
+        0.47139673682599764,
+        0.4242570631433979,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.47139673682599764,
+        0.0,
+        0.881921264348355,
+        1.5937291379135194
+    ],
+    [
+        0.8314696123025452,
+        0.0,
+        0.5555702330196022,
+        0.500013209717642,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.5555702330196022,
+        0.0,
+        0.8314696123025452,
+        1.5483226510722907
+    ],
+    [
+        0.773010453362737,
+        0.0,
+        0.6343932841636455,
+        0.5709539557472809,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.6343932841636455,
+        0.0,
+        0.773010453362737,
+        1.4957094080264635
+    ],
+    [
+        0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        0.6363961030678928,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        1.436396103067893
+    ],
+    [
+        0.6343932841636455,
+        0.0,
+        0.7730104533627369,
+        0.6957094080264632,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7730104533627369,
+        0.0,
+        0.6343932841636455,
+        1.370953955747281
+    ],
+    [
+        0.5555702330196023,
+        0.0,
+        0.8314696123025452,
+        0.7483226510722907,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.8314696123025452,
+        0.0,
+        0.5555702330196023,
+        1.3000132097176422
+    ],
+    [
+        0.4713967368259978,
+        0.0,
+        0.8819212643483549,
+        0.7937291379135195,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.8819212643483549,
+        0.0,
+        0.4713967368259978,
+        1.2242570631433982
+    ],
+    [
+        0.38268343236508984,
+        0.0,
+        0.9238795325112867,
+        0.831491579260158,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9238795325112867,
+        0.0,
+        0.38268343236508984,
+        1.144415089128581
+    ],
+    [
+        0.29028467725446233,
+        0.0,
+        0.9569403357322089,
+        0.861246302158988,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9569403357322089,
+        0.0,
+        0.29028467725446233,
+        1.0612562095290161
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.8827067523629074,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.9755812898145155
+    ],
+    [
+        0.09801714032956077,
+        0.0,
+        0.9951847266721968,
+        0.8956662540049771,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9951847266721968,
+        0.0,
+        0.09801714032956077,
+        0.8882154262966048
+    ]
+]

examples/camera_poses/test_camera_Round-ZoomIn.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.81
+    ],
+    [
+        0.9807852804032304,
+        0.0,
+        0.19509032201612825,
+        0.0019509032201612826,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.19509032201612825,
+        0.0,
+        0.9807852804032304,
+        0.8098078528040323
+    ],
+    [
+        0.9238795325112867,
+        0.0,
+        0.3826834323650898,
+        0.003826834323650898,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.3826834323650898,
+        0.0,
+        0.9238795325112867,
+        0.8092387953251129
+    ],
+    [
+        0.8314696123025452,
+        0.0,
+        0.5555702330196022,
+        0.005555702330196022,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.5555702330196022,
+        0.0,
+        0.8314696123025452,
+        0.8083146961230255
+    ],
+    [
+        0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        0.007071067811865476,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.7071067811865476,
+        0.0,
+        0.7071067811865476,
+        0.8070710678118656
+    ],
+    [
+        0.5555702330196023,
+        0.0,
+        0.8314696123025452,
+        0.008314696123025453,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.8314696123025452,
+        0.0,
+        0.5555702330196023,
+        0.805555702330196
+    ],
+    [
+        0.38268343236508984,
+        0.0,
+        0.9238795325112867,
+        0.009238795325112868,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9238795325112867,
+        0.0,
+        0.38268343236508984,
+        0.803826834323651
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.8019509032201614
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.8019509032201614
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.7019509032201614
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.6019509032201614
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.5019509032201613
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.4019509032201613
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.3019509032201613
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.2019509032201613
+    ],
+    [
+        0.19509032201612833,
+        0.0,
+        0.9807852804032304,
+        0.009807852804032305,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        -0.9807852804032304,
+        0.0,
+        0.19509032201612833,
+        0.10195090322016129
+    ]
+]

examples/camera_poses/test_camera_SPIN-ACW-60.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9978589232386035,
+        -0.06540312923014306,
+        0.0,
+        0.0,
+        0.06540312923014306,
+        0.9978589232386035,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9914448613738104,
+        -0.13052619222005157,
+        0.0,
+        0.0,
+        0.13052619222005157,
+        0.9914448613738104,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9807852804032304,
+        -0.19509032201612825,
+        0.0,
+        0.0,
+        0.19509032201612825,
+        0.9807852804032304,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9659258262890683,
+        -0.25881904510252074,
+        0.0,
+        0.0,
+        0.25881904510252074,
+        0.9659258262890683,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9469301294951057,
+        -0.32143946530316153,
+        0.0,
+        0.0,
+        0.32143946530316153,
+        0.9469301294951057,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9238795325112867,
+        -0.3826834323650898,
+        0.0,
+        0.0,
+        0.3826834323650898,
+        0.9238795325112867,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.8968727415326884,
+        -0.44228869021900125,
+        0.0,
+        0.0,
+        0.44228869021900125,
+        0.8968727415326884,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.8660254037844387,
+        -0.49999999999999994,
+        0.0,
+        0.0,
+        0.49999999999999994,
+        0.8660254037844387,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.8314696123025452,
+        -0.5555702330196022,
+        0.0,
+        0.0,
+        0.5555702330196022,
+        0.8314696123025452,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.7933533402912353,
+        -0.6087614290087205,
+        0.0,
+        0.0,
+        0.6087614290087205,
+        0.7933533402912353,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.7518398074789774,
+        -0.6593458151000688,
+        0.0,
+        0.0,
+        0.6593458151000688,
+        0.7518398074789774,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.7071067811865476,
+        -0.7071067811865476,
+        0.0,
+        0.0,
+        0.7071067811865476,
+        0.7071067811865476,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.659345815100069,
+        -0.7518398074789773,
+        0.0,
+        0.0,
+        0.7518398074789773,
+        0.659345815100069,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.6087614290087207,
+        -0.7933533402912352,
+        0.0,
+        0.0,
+        0.7933533402912352,
+        0.6087614290087207,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.5555702330196024,
+        -0.8314696123025451,
+        0.0,
+        0.0,
+        0.8314696123025451,
+        0.5555702330196024,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ]
+]

examples/camera_poses/test_camera_SPIN-CW-60.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9978589232386035,
+        0.06540312923014306,
+        0.0,
+        0.0,
+        -0.06540312923014306,
+        0.9978589232386035,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9914448613738104,
+        0.13052619222005157,
+        0.0,
+        0.0,
+        -0.13052619222005157,
+        0.9914448613738104,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9807852804032304,
+        0.19509032201612825,
+        0.0,
+        0.0,
+        -0.19509032201612825,
+        0.9807852804032304,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9659258262890683,
+        0.25881904510252074,
+        0.0,
+        0.0,
+        -0.25881904510252074,
+        0.9659258262890683,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9469301294951057,
+        0.32143946530316153,
+        0.0,
+        0.0,
+        -0.32143946530316153,
+        0.9469301294951057,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.9238795325112867,
+        0.3826834323650898,
+        0.0,
+        0.0,
+        -0.3826834323650898,
+        0.9238795325112867,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.8968727415326884,
+        0.44228869021900125,
+        0.0,
+        0.0,
+        -0.44228869021900125,
+        0.8968727415326884,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.8660254037844387,
+        0.49999999999999994,
+        0.0,
+        0.0,
+        -0.49999999999999994,
+        0.8660254037844387,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.8314696123025452,
+        0.5555702330196022,
+        0.0,
+        0.0,
+        -0.5555702330196022,
+        0.8314696123025452,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.7933533402912353,
+        0.6087614290087205,
+        0.0,
+        0.0,
+        -0.6087614290087205,
+        0.7933533402912353,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.7518398074789774,
+        0.6593458151000688,
+        0.0,
+        0.0,
+        -0.6593458151000688,
+        0.7518398074789774,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.7071067811865476,
+        0.7071067811865476,
+        0.0,
+        0.0,
+        -0.7071067811865476,
+        0.7071067811865476,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.659345815100069,
+        0.7518398074789773,
+        0.0,
+        0.0,
+        -0.7518398074789773,
+        0.659345815100069,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.6087614290087207,
+        0.7933533402912352,
+        0.0,
+        0.0,
+        -0.7933533402912352,
+        0.6087614290087207,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        0.5555702330196024,
+        0.8314696123025451,
+        0.0,
+        0.0,
+        -0.8314696123025451,
+        0.5555702330196024,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ]
+]

examples/camera_poses/test_camera_U.json

@@ -0,0 +1,226 @@
+[
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.2,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.28750000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.37500000000000006,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.4625000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.55,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.6375000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.7250000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.8125000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.9000000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        0.9875000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.0750000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.1625000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.2500000000000002,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.3375000000000001,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.4250000000000003,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ],
+    [
+        1.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        1.0,
+        0.0,
+        1.5125000000000004,
+        0.0,
+        0.0,
+        1.0,
+        0.0
+    ]
+]

examples/camera_poses/test_camera_b133a504fc90a2d1.json

@@ -0,0 +1 @@
+[[0.9999999403953552, -3.2563843288535566e-10, 8.624932434919685e-10, -5.431840754965833e-09, -3.2563843288535566e-10, 1.0, -7.078895802870022e-10, -2.678919752696629e-09, 8.624932434919685e-10, -7.078895802870022e-10, 1.0, 7.303774474110014e-09], [0.9999998807907104, 0.00026603759033605456, -0.0003414043167140335, -0.030107486993074417, -0.0002659278397914022, 0.9999999403953552, 0.00032293255208060145, 0.011978899128735065, 0.0003414914826862514, -0.0003228419227525592, 0.9999999403953552, -0.07893969118595123], [0.9999994039535522, -0.0006866907933726907, -0.0008186722989194095, -0.050785429775714874, 0.0006877407431602478, 0.9999989867210388, 0.0012827541213482618, 0.023323407396674156, 0.0008177922572940588, -0.0012833180371671915, 0.9999988675117493, -0.1486724615097046], [0.999999463558197, -0.0005364732351154089, -0.0008647883078083396, -0.0673225075006485, 0.0005379383219406009, 0.9999984502792358, 0.0016958877677097917, 0.03169107437133789, 0.0008638793369755149, -0.0016963522648438811, 0.9999982118606567, -0.21477271616458893], [0.9999995231628418, -0.0005853328038938344, 0.000731398060452193, -0.08539554476737976, 0.0005843221442773938, 0.9999988675117493, 0.0013807439245283604, 0.037947509437799454, -0.0007322035962715745, -0.0013803173787891865, 0.9999987483024597, -0.2858566641807556], [0.9999935030937195, -0.0007727851625531912, 0.00352613371796906, -0.10591986775398254, 0.0007684561423957348, 0.9999989867210388, 0.0012290359009057283, 0.0426139310002327, -0.003527079476043582, -0.0012263187672942877, 0.999993085861206, -0.3645589053630829], [0.9999805688858032, -0.0012497249990701675, 0.006103655323386192, -0.12257411330938339, 0.0012419418198987842, 0.9999984502792358, 0.001278668874874711, 0.05143251642584801, -0.006105243694037199, -0.0012710647424682975, 0.9999805688858032, -0.44212815165519714], [0.9999697804450989, -0.001364423311315477, 0.0076533216051757336, -0.13916294276714325, 0.0013530527940019965, 0.9999980330467224, 0.0014907552395015955, 0.06076823174953461, -0.0076553407125175, -0.0014803558588027954, 0.9999696016311646, -0.5131306648254395], [0.9999656081199646, -0.001309191924519837, 0.00818517804145813, -0.1577269434928894, 0.0012902054004371166, 0.9999964833259583, 0.0023244714830070734, 0.07221967726945877, -0.008188189007341862, -0.0023138325195759535, 0.9999638199806213, -0.5706046223640442], [0.9999715089797974, -0.000632039678748697, 0.0075194560922682285, -0.1842648684978485, 0.0006101227481849492, 0.9999955892562866, 0.002916603581979871, 0.0780157595872879, -0.0075212628580629826, -0.0029119334649294615, 0.9999675154685974, -0.6207911372184753], [0.999971330165863, 9.749359742272645e-05, 0.007570990361273289, -0.2200128734111786, -0.00012177121971035376, 0.9999948740005493, 0.0032062893733382225, 0.07083319127559662, -0.007570638321340084, -0.003207120578736067, 0.9999662041664124, -0.671477735042572], [0.9999842047691345, 0.0011085873702540994, 0.005505275446921587, -0.265299528837204, -0.0011242963373661041, 0.9999953508377075, 0.002851187251508236, 0.06430258601903915, -0.00550208892673254, -0.002857332583516836, 0.9999808073043823, -0.7294802069664001], [0.9999891519546509, 0.0030521126464009285, -0.003507567336782813, -0.3109421133995056, -0.0030410285107791424, 0.9999904632568359, 0.003161099273711443, 0.06503432989120483, 0.003517185803502798, -0.0031503995414823294, 0.999988853931427, -0.7917969226837158], [0.9995473623275757, 0.00647346256300807, -0.029379382729530334, -0.3455933630466461, -0.006376372650265694, 0.9999739527702332, 0.0033971993252635, 0.06981948018074036, 0.02940060943365097, -0.003208327107131481, 0.9995625615119934, -0.8474093675613403], [0.9966378808021545, 0.011493867263197899, -0.08112204819917679, -0.3555213510990143, -0.011167873628437519, 0.9999276399612427, 0.004471173509955406, 0.0734858587384224, 0.08116757124662399, -0.00355018163099885, 0.9966941475868225, -0.9062771201133728], [0.9889613389968872, 0.01762101612985134, -0.14712226390838623, -0.33937495946884155, -0.01693679392337799, 0.999839186668396, 0.0059022014029324055, 0.0776127353310585, 0.14720259606838226, -0.0033452697098255157, 0.9891006946563721, -0.9548948407173157]]

examples/camera_poses/test_camera_d9642c8efc01481d.json

@@ -0,0 +1 @@
+[[1.0, -1.4532828274127496e-09, 3.928266045782891e-10, -8.700192566379883e-09, -1.4532828274127496e-09, 1.0, -1.7456260048565042e-10, 9.261455491405002e-11, 3.928266045782891e-10, -1.7456260048565042e-10, 1.0, -1.5881864712241622e-08], [0.9850640296936035, -0.04082970321178436, 0.16727784276008606, 0.20439539849758148, 0.03502606600522995, 0.9986825585365295, 0.03750050067901611, -0.15185177326202393, -0.16858860850334167, -0.03108130767941475, 0.9851963520050049, 0.10121102631092072], [0.9168016910552979, -0.09482394903898239, 0.387921541929245, 0.44365406036376953, 0.06679922342300415, 0.9941277503967285, 0.08513444662094116, -0.26796984672546387, -0.39371633529663086, -0.0521385557949543, 0.9177521467208862, 0.06753730028867722], [0.7643709182739258, -0.15802493691444397, 0.6251122951507568, 0.6479341983795166, 0.09576083719730377, 0.9865723252296448, 0.13230620324611664, -0.33551496267318726, -0.6376261115074158, -0.04126973822712898, 0.7692397236824036, -0.07632352411746979], [0.5624101758003235, -0.2065712958574295, 0.8006392121315002, 0.8095709085464478, 0.10458854585886002, 0.9782856106758118, 0.17893710732460022, -0.3449772298336029, -0.8202170729637146, -0.016898371279239655, 0.5718027949333191, -0.267223596572876], [0.3860713243484497, -0.23765023052692413, 0.8913311958312988, 1.017128586769104, 0.10842984914779663, 0.9712379574775696, 0.21198998391628265, -0.34939509630203247, -0.9160742163658142, 0.014803661964833736, 0.4007355272769928, -0.5046621561050415], [0.25270596146583557, -0.2560441493988037, 0.9330493807792664, 1.2611535787582397, 0.09987718611955643, 0.9661006331443787, 0.23806333541870117, -0.36809343099594116, -0.9623742699623108, 0.03303031623363495, 0.26971235871315, -0.8164812922477722], [0.12896282970905304, -0.26742202043533325, 0.9549105167388916, 1.4084848165512085, 0.09037449955940247, 0.9621138572692871, 0.25723403692245483, -0.3597045838832855, -0.9875227212905884, 0.05312592536211014, 0.14824506640434265, -1.1886308193206787], [-0.037818778306245804, -0.2752484083175659, 0.9606289863586426, 1.3755683898925781, 0.08146493136882782, 0.957267701625824, 0.277492493391037, -0.353816956281662, -0.9959584474563599, 0.08875200152397156, -0.013779602944850922, -1.6891316175460815], [-0.1970304548740387, -0.2752428948879242, 0.9409677982330322, 1.1786020994186401, 0.07280438393354416, 0.9530242681503296, 0.2940141260623932, -0.36850038170814514, -0.9776904582977295, 0.12643630802631378, -0.16773590445518494, -2.261430025100708], [-0.3677733540534973, -0.26668044924736023, 0.8908559679985046, 0.8089978098869324, 0.06243090331554413, 0.9487544894218445, 0.30978602170944214, -0.37012383341789246, -0.9278174042701721, 0.16954797506332397, -0.33227747678756714, -2.8139760494232178], [-0.5150132775306702, -0.2527574300765991, 0.8190696835517883, 0.4462648332118988, 0.050335872918367386, 0.9449706673622131, 0.32325950264930725, -0.381427526473999, -0.8557030558586121, 0.20771150290966034, -0.473949670791626, -3.2886314392089844], [-0.6352092623710632, -0.23657281696796417, 0.7352160215377808, 0.0826139897108078, 0.036419764161109924, 0.9416990280151367, 0.33447936177253723, -0.3719184398651123, -0.771480917930603, 0.23924078047275543, -0.5895600318908691, -3.677316904067993], [-0.7218965888023376, -0.21918001770973206, 0.6563730239868164, -0.2607730031013489, 0.024064550176262856, 0.9399895071983337, 0.340353786945343, -0.359468549489975, -0.6915825009346008, 0.26149556040763855, -0.6733006834983826, -3.9900803565979004], [-0.7831082344055176, -0.2019210308790207, 0.5881916880607605, -0.609000563621521, 0.015634668990969658, 0.9391284584999084, 0.3432103097438812, -0.35386255383491516, -0.621688961982727, 0.2779669761657715, -0.732282280921936, -4.316582202911377], [-0.8348898887634277, -0.1848600059747696, 0.5184455513954163, -0.9689992070198059, 0.007868430577218533, 0.93780916929245, 0.3470619320869446, -0.3402447998523712, -0.5503608584403992, 0.2938378155231476, -0.7815127968788147, -4.561704635620117]]

examples/camera_poses/test_camera_d971457c81bca597.json

@@ -0,0 +1 @@
+[[1.0, -1.9455232563858615e-11, -8.611562019034125e-10, -1.0473515388298438e-09, -1.9455232563858615e-11, 1.0, 8.674055917978762e-10, 1.8375034827045056e-09, -8.611562019034125e-10, 8.674055917978762e-10, 1.0, 8.45981773522908e-09], [0.9973401427268982, -0.0032562706619501114, 0.07281485944986343, -0.07197967171669006, 0.00329946493729949, 0.9999944567680359, -0.00047293686657212675, -0.0565447174012661, -0.07281292229890823, 0.0007119301008060575, 0.9973453879356384, -0.3188611567020416], [0.9906007051467896, -0.0066957552917301655, 0.13662146031856537, -0.15415722131729126, 0.006810691673308611, 0.9999766945838928, -0.00037385127507150173, -0.10014614462852478, -0.1366157978773117, 0.0013008243404328823, 0.9906232953071594, -0.6077051758766174], [0.9827210903167725, -0.009423106908798218, 0.18485280871391296, -0.23053960502147675, 0.009786692447960377, 0.9999515414237976, -0.0010545575059950352, -0.11272216588258743, -0.18483391404151917, 0.002845433307811618, 0.9827656745910645, -0.8897562623023987], [0.9756309986114502, -0.011318936944007874, 0.2191258817911148, -0.2929331362247467, 0.01199379749596119, 0.9999265670776367, -0.0017497432418167591, -0.10348402708768845, -0.21908999979496002, 0.004335256293416023, 0.9756950736045837, -1.1368639469146729], [0.9685831069946289, -0.012191502377390862, 0.24839124083518982, -0.379209041595459, 0.013217172585427761, 0.9999096393585205, -0.0024619612377136946, -0.12078691273927689, -0.24833877384662628, 0.005667644087225199, 0.9686566591262817, -1.3666095733642578], [0.9615083932876587, -0.012519675306975842, 0.2744903266429901, -0.5242342352867126, 0.013806014321744442, 0.9999009370803833, -0.0027547888457775116, -0.13015854358673096, -0.2744286358356476, 0.006438371259719133, 0.9615859389305115, -1.5762972831726074], [0.9528889656066895, -0.014083069749176502, 0.3029923439025879, -0.6702165007591248, 0.015383570455014706, 0.9998798370361328, -0.0019058430334553123, -0.14860114455223083, -0.3029291331768036, 0.00647716224193573, 0.9529911279678345, -1.7173497676849365], [0.9457509517669678, -0.015159872360527515, 0.3245387375354767, -0.8001917600631714, 0.016729505732655525, 0.9998579621315002, -0.0020466779824346304, -0.15521110594272614, -0.32446160912513733, 0.007365020923316479, 0.9458702206611633, -1.8571592569351196], [0.9417706727981567, -0.015540024265646935, 0.33589670062065125, -0.9244012236595154, 0.017394419759511948, 0.9998455047607422, -0.0025124624371528625, -0.1416105479001999, -0.3358057737350464, 0.008208892308175564, 0.9418954849243164, -1.9969842433929443], [0.9381415247917175, -0.016401933506131172, 0.34586337208747864, -1.0128529071807861, 0.01851990446448326, 0.9998245239257812, -0.0028197101783007383, -0.11503250896930695, -0.3457564413547516, 0.009050644934177399, 0.9382806420326233, -2.080850124359131], [0.9357938766479492, -0.016873901709914207, 0.3521435558795929, -1.1008079051971436, 0.01865064539015293, 0.9998247027397156, -0.0016533475136384368, -0.048780668526887894, -0.3520539104938507, 0.008114897646009922, 0.9359445571899414, -2.088500499725342], [0.9334627985954285, -0.016492612659931183, 0.3582947850227356, -1.2417148351669312, 0.017931735143065453, 0.9998390078544617, -0.0006939812446944416, 0.01625901833176613, -0.35822567343711853, 0.007072653621435165, 0.9336082339286804, -2.0250792503356934], [0.9281281232833862, -0.016213543713092804, 0.3719078302383423, -1.4365860223770142, 0.01746317930519581, 0.9998475313186646, 8.084020009846427e-06, 0.060201361775398254, -0.37185123562812805, 0.006487190257757902, 0.9282696843147278, -1.8963005542755127], [0.9220678806304932, -0.017285069450736046, 0.3866421580314636, -1.604330062866211, 0.01880805380642414, 0.9998230934143066, -0.00015593231364618987, 0.0799601748585701, -0.38657107949256897, 0.007415766827762127, 0.9222298264503479, -1.7388361692428589], [0.9185494184494019, -0.018356993794441223, 0.3948797583580017, -1.7530856132507324, 0.019847355782985687, 0.9998030066490173, 0.0003104731731582433, 0.10000917315483093, -0.3948076665401459, 0.007552135270088911, 0.9187328219413757, -1.5337872505187988]]

gradio_utils/camera_utils.py

@@ -0,0 +1,157 @@
+import copy
+# import plotly.express as px
+# import plotly.graph_objects as go
+import json
+
+import numpy as np
+
+CAMERA_MOTION_MODE = ["Basic Camera Poses", "Provided Complex Camera Poses", "Custom Camera Poses"]
+
+CAMERA = {
+    # T
+    "base_T_norm": 1.5,
+    "base_angle": np.pi/3,
+
+    "Pan Up": {     "angle":[0., 0., 0.],   "T":[0., 1., 0.]},
+    "Pan Down": {   "angle":[0., 0., 0.],   "T":[0.,-1.,0.]},
+    "Pan Left": {   "angle":[0., 0., 0.],   "T":[3.,0.,0.]},
+    "Pan Right": {  "angle":[0., 0., 0.],   "T": [-3.,0.,0.]},
+    "Zoom In": {    "angle":[0., 0., 0.],   "T": [0.,0.,-4.]},
+    "Zoom Out": {   "angle":[0., 0., 0.],   "T": [0.,0.,4.]},
+    "ACW": {        "angle": [0., 0., 1.],  "T":[0., 0., 0.]},
+    "CW": {         "angle": [0., 0., -1.], "T":[0., 0., 0.]},
+}
+
+COMPLEX_CAMERA = {
+    "Pose_1": "examples/camera_poses/test_camera_1424acd0007d40b5.json",
+    "Pose_2": "examples/camera_poses/test_camera_d971457c81bca597.json",
+    "Pose_3": "examples/camera_poses/test_camera_Round-ZoomIn.json",
+    "Pose_4": "examples/camera_poses/test_camera_Round-RI_90.json",
+    "Pose_5": "examples/camera_poses/test_camera_Round-RI-120.json",
+    "Pose_6": "examples/camera_poses/test_camera_018f7907401f2fef.json",
+    "Pose_7": "examples/camera_poses/test_camera_088b93f15ca8745d.json",
+    "Pose_8": "examples/camera_poses/test_camera_b133a504fc90a2d1.json",
+}
+
+
+
+def compute_R_form_rad_angle(angles):
+    theta_x, theta_y, theta_z = angles
+    Rx = np.array([[1, 0, 0],
+                   [0, np.cos(theta_x), -np.sin(theta_x)],
+                   [0, np.sin(theta_x), np.cos(theta_x)]])
+    
+    Ry = np.array([[np.cos(theta_y), 0, np.sin(theta_y)],
+                   [0, 1, 0],
+                   [-np.sin(theta_y), 0, np.cos(theta_y)]])
+    
+    Rz = np.array([[np.cos(theta_z), -np.sin(theta_z), 0],
+                   [np.sin(theta_z), np.cos(theta_z), 0],
+                   [0, 0, 1]])
+    
+    # 计算相机外参的旋转矩阵
+    R = np.dot(Rz, np.dot(Ry, Rx))
+    return R
+
+def get_camera_motion(angle, T, speed, n=16):
+    RT = []
+    for i in range(n):
+        _angle = (i/n)*speed*(CAMERA["base_angle"])*angle
+        R = compute_R_form_rad_angle(_angle) 
+        # _T = (i/n)*speed*(T.reshape(3,1))
+        _T=(i/n)*speed*(CAMERA["base_T_norm"])*(T.reshape(3,1))
+        _RT = np.concatenate([R,_T], axis=1)
+        RT.append(_RT)
+    RT = np.stack(RT)
+    return RT
+    
+def create_relative(RT_list, K_1=4.7, dataset="syn"):
+    RT = copy.deepcopy(RT_list[0])
+    R_inv = RT[:,:3].T
+    T =  RT[:,-1]
+
+    temp = []
+    for _RT in RT_list:
+        _RT[:,:3] = np.dot(_RT[:,:3], R_inv)
+        _RT[:,-1] =  _RT[:,-1] - np.dot(_RT[:,:3], T)
+        temp.append(_RT)
+    RT_list = temp
+
+    return RT_list
+    
+def combine_camera_motion(RT_0, RT_1):
+    RT = copy.deepcopy(RT_0[-1])
+    R = RT[:,:3]
+    R_inv = RT[:,:3].T
+    T =  RT[:,-1]
+
+    temp = []
+    for _RT in RT_1:
+        _RT[:,:3] = np.dot(_RT[:,:3], R)
+        _RT[:,-1] =  _RT[:,-1] + np.dot(np.dot(_RT[:,:3], R_inv), T) 
+        temp.append(_RT)
+
+    RT_1 = np.stack(temp)
+
+    return np.concatenate([RT_0, RT_1], axis=0)
+
+def process_camera(camera_dict, camera_args, num_frames=16, width=256, height=256):
+    speed = camera_dict['speed']
+    motion_list = camera_dict['motion']
+    mode = camera_dict['mode']
+
+    if mode == 'Customized Mode 3: RAW Camera Poses':
+        # print(camera_args)
+        RT = camera_args.strip().split()
+        assert(len(RT) == num_frames*12), "The number of camera poses should be equal to the number of frames"
+        RT = [float(x) for x in RT]
+        RT = np.array(RT).reshape(-1, 3, 4)
+        RT[:, :, -1] = RT[:, :, -1] * np.array([1.5, 1, 1.3]) * speed
+        return RT
+
+    if camera_dict['complex'] is not None:
+        with open(COMPLEX_CAMERA[camera_dict['complex']]) as f:
+            RT = json.load(f) # [16, 12]
+        if num_frames < len(RT):
+            half = (len(RT) - num_frames) // 2
+            RT = RT[half:half+num_frames]
+        RT = np.array(RT).reshape(-1, 3, 4)
+        RT[:, :, -1] = RT[:, :, -1] * np.array([1.5, 1, 1.3]) * speed
+        return RT
+    
+    half_num_frames = num_frames//2
+
+    
+    
+    print(len(motion_list))
+    if len(motion_list) == 0:
+        angle = np.array([0,0,0])
+        T = np.array([0,0,0])
+        RT = get_camera_motion(angle, T, speed, num_frames)
+        
+    elif len(motion_list) == 1:
+        angle = np.array(CAMERA[motion_list[0]]["angle"])
+        T = np.array(CAMERA[motion_list[0]]["T"])
+        print(angle, T)
+        RT = get_camera_motion(angle, T, speed, num_frames)
+        
+    
+    elif len(motion_list) == 2:
+        if mode == "Customized Mode 1: First A then B":
+            angle = np.array(CAMERA[motion_list[0]]["angle"]) 
+            T = np.array(CAMERA[motion_list[0]]["T"]) 
+            RT_0 = get_camera_motion(angle, T, speed, half_num_frames)
+
+            angle = np.array(CAMERA[motion_list[1]]["angle"]) 
+            T = np.array(CAMERA[motion_list[1]]["T"]) 
+            RT_1 = get_camera_motion(angle, T, speed, num_frames-half_num_frames)
+
+            RT = combine_camera_motion(RT_0, RT_1)
+
+        elif mode == "Customized Mode 2: Both A and B":
+            angle = np.array(CAMERA[motion_list[0]]["angle"]) + np.array(CAMERA[motion_list[1]]["angle"])
+            T = np.array(CAMERA[motion_list[0]]["T"]) + np.array(CAMERA[motion_list[1]]["T"])
+            RT = get_camera_motion(angle, T, speed, num_frames)
+
+    return RT
+

gradio_utils/flow_utils.py

@@ -0,0 +1,69 @@
+import numpy as np
+
+
+def sigma_matrix2(sig_x, sig_y, theta):
+    """Calculate the rotated sigma matrix (two dimensional matrix).
+    Args:
+        sig_x (float):
+        sig_y (float):
+        theta (float): Radian measurement.
+    Returns:
+        ndarray: Rotated sigma matrix.
+    """
+    d_matrix = np.array([[sig_x**2, 0], [0, sig_y**2]])
+    u_matrix = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
+    return np.dot(u_matrix, np.dot(d_matrix, u_matrix.T))
+
+
+def mesh_grid(kernel_size):
+    """Generate the mesh grid, centering at zero.
+    Args:
+        kernel_size (int):
+    Returns:
+        xy (ndarray): with the shape (kernel_size, kernel_size, 2)
+        xx (ndarray): with the shape (kernel_size, kernel_size)
+        yy (ndarray): with the shape (kernel_size, kernel_size)
+    """
+    ax = np.arange(-kernel_size // 2 + 1., kernel_size // 2 + 1.)
+    xx, yy = np.meshgrid(ax, ax)
+    xy = np.hstack((xx.reshape((kernel_size * kernel_size, 1)), yy.reshape(kernel_size * kernel_size,
+                                                                           1))).reshape(kernel_size, kernel_size, 2)
+    return xy, xx, yy
+
+
+def pdf2(sigma_matrix, grid):
+    """Calculate PDF of the bivariate Gaussian distribution.
+    Args:
+        sigma_matrix (ndarray): with the shape (2, 2)
+        grid (ndarray): generated by :func:`mesh_grid`,
+            with the shape (K, K, 2), K is the kernel size.
+    Returns:
+        kernel (ndarrray): un-normalized kernel.
+    """
+    inverse_sigma = np.linalg.inv(sigma_matrix)
+    kernel = np.exp(-0.5 * np.sum(np.dot(grid, inverse_sigma) * grid, 2))
+    return kernel
+
+def bivariate_Gaussian(kernel_size, sig_x, sig_y, theta, grid=None, isotropic=True):
+    """Generate a bivariate isotropic or anisotropic Gaussian kernel.
+    In the isotropic mode, only `sig_x` is used. `sig_y` and `theta` is ignored.
+    Args:
+        kernel_size (int):
+        sig_x (float):
+        sig_y (float):
+        theta (float): Radian measurement.
+        grid (ndarray, optional): generated by :func:`mesh_grid`,
+            with the shape (K, K, 2), K is the kernel size. Default: None
+        isotropic (bool):
+    Returns:
+        kernel (ndarray): normalized kernel.
+    """
+    if grid is None:
+        grid, _, _ = mesh_grid(kernel_size)
+    if isotropic:
+        sigma_matrix = np.array([[sig_x**2, 0], [0, sig_x**2]])
+    else:
+        sigma_matrix = sigma_matrix2(sig_x, sig_y, theta)
+    kernel = pdf2(sigma_matrix, grid)
+    kernel = kernel / np.sum(kernel)
+    return kernel

gradio_utils/motionctrl_cmcm_gradio.py

@@ -0,0 +1,276 @@
+import argparse
+import datetime
+import json
+import math
+import os
+import sys
+import time
+from glob import glob
+from pathlib import Path
+from typing import Optional
+
+import cv2
+import numpy as np
+import torch
+import torchvision
+from einops import rearrange, repeat
+from fire import Fire
+from omegaconf import OmegaConf
+from PIL import Image
+from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor
+import tempfile
+
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+from sgm.util import default, instantiate_from_config
+
+
+
+def to_relative_RT2(org_pose, keyframe_idx=0, keyframe_zero=False):
+        org_pose = org_pose.reshape(-1, 3, 4) # [t, 3, 4]
+        R_dst = org_pose[:, :, :3]
+        T_dst = org_pose[:, :, 3:]
+
+        R_src = R_dst[keyframe_idx: keyframe_idx+1].repeat(org_pose.shape[0], axis=0) # [t, 3, 3]
+        T_src = T_dst[keyframe_idx: keyframe_idx+1].repeat(org_pose.shape[0], axis=0)
+
+        R_src_inv = R_src.transpose(0, 2, 1) # [t, 3, 3]
+        
+        R_rel = R_dst @ R_src_inv # [t, 3, 3]
+        T_rel = T_dst - R_rel@T_src
+
+        RT_rel = np.concatenate([R_rel, T_rel], axis=-1) # [t, 3, 4]
+        RT_rel = RT_rel.reshape(-1, 12) # [t, 12]
+
+        if keyframe_zero:
+            RT_rel[keyframe_idx] = np.zeros_like(RT_rel[keyframe_idx])
+
+        return RT_rel
+
+def build_model(config, ckpt, device, num_frames, num_steps):
+    num_frames = default(num_frames, 14)
+    num_steps = default(num_steps, 25)
+    model_config = default(config, "configs/inference/config_motionctrl_cmcm.yaml")
+
+    print(f"Loading model from {ckpt}")
+    model, filter = load_model(
+        model_config,
+        ckpt,
+        device,
+        num_frames,
+        num_steps,
+    )
+
+    model.eval()
+
+    return model
+
+def motionctrl_sample(
+    model,
+    image: Image = None,  # Can either be image file or folder with image files
+    RT: np.ndarray = None,
+    num_frames: Optional[int] = None,
+    fps_id: int = 6,
+    motion_bucket_id: int = 127,
+    cond_aug: float = 0.02,
+    seed: int = 23,
+    decoding_t: int = 1,  # Number of frames decoded at a time! This eats most VRAM. Reduce if necessary.
+    save_fps: int = 10,
+    sample_num: int = 1,
+    device: str = "cuda",
+):
+    """
+    Simple script to generate a single sample conditioned on an image `input_path` or multiple images, one for each
+    image file in folder `input_path`. If you run out of VRAM, try decreasing `decoding_t`.
+    """
+
+    torch.manual_seed(seed)
+
+    w, h = image.size
+
+    # RT: [t, 3, 4]
+    # RT = RT.reshape(-1, 3, 4) # [t, 3, 4]
+    # adaptive to different spatial ratio
+    # base_len = min(w, h) * 0.5
+    # K = np.array([[w/base_len, 0, w/base_len],
+    #               [0, h/base_len, h/base_len],
+    #               [0, 0, 1]])
+    # for i in range(RT.shape[0]):
+    #     RT[i,:,:] = np.dot(K, RT[i,:,:])
+
+    RT = to_relative_RT2(RT) # [t, 12]
+    RT = torch.tensor(RT).float().to(device) # [t, 12]
+    RT = RT.unsqueeze(0).repeat(2,1,1)
+
+    if h % 64 != 0 or w % 64 != 0:
+        width, height = map(lambda x: x - x % 64, (w, h))
+        image = image.resize((width, height))
+        print(
+            f"WARNING: Your image is of size {h}x{w} which is not divisible by 64. We are resizing to {height}x{width}!"
+        )
+
+    image = ToTensor()(image)
+    image = image * 2.0 - 1.0
+
+    image = image.unsqueeze(0).to(device)
+    H, W = image.shape[2:]
+    assert image.shape[1] == 3
+    F = 8
+    C = 4
+    shape = (num_frames, C, H // F, W // F)
+
+    if motion_bucket_id > 255:
+        print(
+            "WARNING: High motion bucket! This may lead to suboptimal performance."
+        )
+
+    if fps_id < 5:
+        print("WARNING: Small fps value! This may lead to suboptimal performance.")
+
+    if fps_id > 30:
+        print("WARNING: Large fps value! This may lead to suboptimal performance.")
+
+    value_dict = {}
+    value_dict["motion_bucket_id"] = motion_bucket_id
+    value_dict["fps_id"] = fps_id
+    value_dict["cond_aug"] = cond_aug
+    value_dict["cond_frames_without_noise"] = image
+    value_dict["cond_frames"] = image + cond_aug * torch.randn_like(image)
+
+    with torch.no_grad():
+        with torch.autocast(device):
+            batch, batch_uc = get_batch(
+                get_unique_embedder_keys_from_conditioner(model.conditioner),
+                value_dict,
+                [1, num_frames],
+                T=num_frames,
+                device=device,
+            )
+            c, uc = model.conditioner.get_unconditional_conditioning(
+                batch,
+                batch_uc=batch_uc,
+                force_uc_zero_embeddings=[
+                    "cond_frames",
+                    "cond_frames_without_noise",
+                ],
+            )
+
+            for k in ["crossattn", "concat"]:
+                uc[k] = repeat(uc[k], "b ... -> b t ...", t=num_frames)
+                uc[k] = rearrange(uc[k], "b t ... -> (b t) ...", t=num_frames)
+                c[k] = repeat(c[k], "b ... -> b t ...", t=num_frames)
+                c[k] = rearrange(c[k], "b t ... -> (b t) ...", t=num_frames)
+
+            
+
+            additional_model_inputs = {}
+            additional_model_inputs["image_only_indicator"] = torch.zeros(
+                2, num_frames
+            ).to(device)
+            #additional_model_inputs["image_only_indicator"][:,0] = 1
+            additional_model_inputs["num_video_frames"] = batch["num_video_frames"]
+
+            
+            additional_model_inputs["RT"] = RT.clone()
+
+            def denoiser(input, sigma, c):
+                return model.denoiser(
+                    model.model, input, sigma, c, **additional_model_inputs
+                )
+
+            results = []
+            for j in range(sample_num):
+                randn = torch.randn(shape, device=device)
+                samples_z = model.sampler(denoiser, randn, cond=c, uc=uc)
+                model.en_and_decode_n_samples_a_time = decoding_t
+                samples_x = model.decode_first_stage(samples_z)
+                samples = torch.clamp((samples_x + 1.0) / 2.0, min=0.0, max=1.0) # [1*t, c, h, w]
+                results.append(samples)
+
+            samples = torch.stack(results, dim=0) # [sample_num, t, c, h, w]
+            samples = samples.data.cpu()
+
+            video_path = tempfile.NamedTemporaryFile(suffix='.mp4').name
+            save_results(samples, video_path, fps=save_fps)
+    
+    return video_path
+
+def save_results(resutls, filename, fps=10):
+    video = resutls.permute(1, 0, 2, 3, 4) # [t, sample_num, c, h, w]
+    frame_grids = [torchvision.utils.make_grid(framesheet, nrow=int(video.shape[1])) for framesheet in video] #[3, 1*h, n*w]
+    grid = torch.stack(frame_grids, dim=0) # stack in temporal dim [t, 3, n*h, w]
+    # already in [0,1]
+    grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1)
+    torchvision.io.write_video(filename, grid, fps=fps, video_codec='h264', options={'crf': '10'})
+
+def get_unique_embedder_keys_from_conditioner(conditioner):
+    return list(set([x.input_key for x in conditioner.embedders]))
+
+
+def get_batch(keys, value_dict, N, T, device):
+    batch = {}
+    batch_uc = {}
+
+    for key in keys:
+        if key == "fps_id":
+            batch[key] = (
+                torch.tensor([value_dict["fps_id"]])
+                .to(device)
+                .repeat(int(math.prod(N)))
+            )
+        elif key == "motion_bucket_id":
+            batch[key] = (
+                torch.tensor([value_dict["motion_bucket_id"]])
+                .to(device)
+                .repeat(int(math.prod(N)))
+            )
+        elif key == "cond_aug":
+            batch[key] = repeat(
+                torch.tensor([value_dict["cond_aug"]]).to(device),
+                "1 -> b",
+                b=math.prod(N),
+            )
+        elif key == "cond_frames":
+            batch[key] = repeat(value_dict["cond_frames"], "1 ... -> b ...", b=N[0])
+        elif key == "cond_frames_without_noise":
+            batch[key] = repeat(
+                value_dict["cond_frames_without_noise"], "1 ... -> b ...", b=N[0]
+            )
+        else:
+            batch[key] = value_dict[key]
+
+    if T is not None:
+        batch["num_video_frames"] = T
+
+    for key in batch.keys():
+        if key not in batch_uc and isinstance(batch[key], torch.Tensor):
+            batch_uc[key] = torch.clone(batch[key])
+    return batch, batch_uc
+
+
+def load_model(
+    config: str,
+    ckpt: str,
+    device: str,
+    num_frames: int,
+    num_steps: int,
+):
+
+    config = OmegaConf.load(config)
+    config.model.params.ckpt_path = ckpt
+    if device == "cuda":
+        config.model.params.conditioner_config.params.emb_models[
+            0
+        ].params.open_clip_embedding_config.params.init_device = device
+
+    config.model.params.sampler_config.params.num_steps = num_steps
+    config.model.params.sampler_config.params.guider_config.params.num_frames = (
+        num_frames
+    )
+
+    model = instantiate_from_config(config.model)
+
+    model = model.to(device).eval()    
+
+    filter = None #DeepFloydDataFiltering(verbose=False, device=device)
+    return model, filter
+

gradio_utils/traj_utils.py

@@ -0,0 +1,104 @@
+import cv2
+import numpy as np
+
+from gradio_utils.flow_utils import bivariate_Gaussian
+
+OBJECT_MOTION_MODE = ["Provided Trajectory", "Custom Trajectory"]
+
+PROVIDED_TRAJS = {
+    "horizon_1": "examples/trajectories/horizon_2.txt",
+    "swaying_1": "examples/trajectories/shake_1.txt",
+    "swaying_2": "examples/trajectories/shake_2.txt",
+    "swaying_3": "examples/trajectories/shaking_10.txt",
+    "curve_1": "examples/trajectories/curve_1.txt",
+    "curve_2": "examples/trajectories/curve_2.txt",
+    "curve_3": "examples/trajectories/curve_3.txt",
+    "curve_4": "examples/trajectories/curve_4.txt",
+}
+
+
+def read_points(file, video_len=16, reverse=False):
+    with open(file, 'r') as f:
+        lines = f.readlines()
+    points = []
+    for line in lines:
+        x, y = line.strip().split(',')
+        points.append((int(x), int(y)))
+    if reverse:
+        points = points[::-1]
+
+    if len(points) > video_len:
+        skip = len(points) // video_len
+        points = points[::skip]
+    points = points[:video_len]
+    
+    return points
+
+def get_provided_traj(traj_name):
+    traj = read_points(PROVIDED_TRAJS[traj_name])
+    # xrange from 256 to 1024
+    traj = [[int(1024*x/256), int(1024*y/256)] for x,y in traj]
+    return traj
+
+blur_kernel = bivariate_Gaussian(99, 10, 10, 0, grid=None, isotropic=True)
+
+def process_points(points):
+    frames = 16
+    defualt_points = [[512,512]]*16
+
+    if len(points) < 2:
+        return defualt_points
+    elif len(points) >= frames:
+        skip = len(points)//frames
+        return points[::skip][:15] + points[-1:]
+    else:
+        insert_num = frames - len(points)
+        insert_num_dict = {}
+        interval = len(points) - 1
+        n = insert_num // interval
+        m = insert_num % interval
+        for i in range(interval):
+            insert_num_dict[i] = n
+        for i in range(m):
+            insert_num_dict[i] += 1
+
+        res = []
+        for i in range(interval):
+            insert_points = []
+            x0,y0 = points[i]
+            x1,y1 = points[i+1]
+
+            delta_x = x1 - x0
+            delta_y = y1 - y0
+            for j in range(insert_num_dict[i]):
+                x = x0 + (j+1)/(insert_num_dict[i]+1)*delta_x
+                y = y0 + (j+1)/(insert_num_dict[i]+1)*delta_y
+                insert_points.append([int(x), int(y)])
+
+            res += points[i:i+1] + insert_points
+        res += points[-1:]
+        return res
+
+def get_flow(points, video_len=16):
+    optical_flow = np.zeros((video_len, 256, 256, 2), dtype=np.float32)
+    for i in range(video_len-1):
+        p = points[i]
+        p1 = points[i+1]
+        optical_flow[i+1, p[1], p[0], 0] = p1[0] - p[0]
+        optical_flow[i+1, p[1], p[0], 1] = p1[1] - p[1]
+    for i in range(1, video_len):
+        optical_flow[i] = cv2.filter2D(optical_flow[i], -1, blur_kernel)
+
+
+    return optical_flow
+
+
+def process_traj(points, device='cpu'):
+    xy_range = 1024
+    points = process_points(points)
+    points = [[int(256*x/xy_range), int(256*y/xy_range)] for x,y in points]
+    
+    optical_flow = get_flow(points)
+    # optical_flow = torch.tensor(optical_flow).to(device)
+
+    return optical_flow

gradio_utils/utils.py

@@ -0,0 +1,175 @@
+import numpy as np
+import plotly.express as px
+import plotly.graph_objects as go
+
+def vis_camera(RT_list, rescale_T=1):
+    fig = go.Figure()
+    showticklabels = True
+    visible = True
+    scene_bounds = 2
+    base_radius = 2.5
+    zoom_scale = 1.5
+    fov_deg = 50.0
+    
+    edges = [(0, 1), (0, 2), (0, 3), (1, 2), (2, 3), (3, 1), (3, 4)] 
+    
+    colors = px.colors.qualitative.Plotly
+    
+    cone_list = []
+    n = len(RT_list)
+    for i, RT in enumerate(RT_list):
+        R = RT[:,:3]
+        T = RT[:,-1]/rescale_T
+        cone = calc_cam_cone_pts_3d(R, T, fov_deg)
+        cone_list.append((cone, (i*1/n, "green"), f"view_{i}"))
+
+    
+    for (cone, clr, legend) in cone_list:
+        for (i, edge) in enumerate(edges):
+            (x1, x2) = (cone[edge[0], 0], cone[edge[1], 0])
+            (y1, y2) = (cone[edge[0], 1], cone[edge[1], 1])
+            (z1, z2) = (cone[edge[0], 2], cone[edge[1], 2])
+            fig.add_trace(go.Scatter3d(
+                x=[x1, x2], y=[y1, y2], z=[z1, z2], mode='lines',
+                line=dict(color=clr, width=3),
+                name=legend, showlegend=(i == 0))) 
+    fig.update_layout(
+                    height=500,
+                    autosize=True,
+                    # hovermode=False,
+                    margin=go.layout.Margin(l=0, r=0, b=0, t=0),
+                    
+                    showlegend=True,
+                    legend=dict(
+                        yanchor='bottom',
+                        y=0.01,
+                        xanchor='right',
+                        x=0.99,
+                    ),
+                    scene=dict(
+                        aspectmode='manual',
+                        aspectratio=dict(x=1, y=1, z=1.0),
+                        camera=dict(
+                            center=dict(x=0.0, y=0.0, z=0.0),
+                            up=dict(x=0.0, y=-1.0, z=0.0),
+                            eye=dict(x=scene_bounds/2, y=-scene_bounds/2, z=-scene_bounds/2),
+                            ),
+
+                        xaxis=dict(
+                            range=[-scene_bounds, scene_bounds],
+                            showticklabels=showticklabels,
+                            visible=visible,
+                        ),
+                            
+                        
+                        yaxis=dict(
+                            range=[-scene_bounds, scene_bounds],
+                            showticklabels=showticklabels,
+                            visible=visible,
+                        ),
+                            
+                        
+                        zaxis=dict(
+                            range=[-scene_bounds, scene_bounds],
+                            showticklabels=showticklabels,
+                            visible=visible,
+                        )
+                    ))
+    return fig
+
+
+def calc_cam_cone_pts_3d(R_W2C, T_W2C, fov_deg, scale=0.1, set_canonical=False, first_frame_RT=None):
+    fov_rad = np.deg2rad(fov_deg)
+    R_W2C_inv = np.linalg.inv(R_W2C)
+
+    # Camera pose center:
+    T = np.zeros_like(T_W2C) - T_W2C
+    T = np.dot(R_W2C_inv, T)
+    cam_x = T[0]
+    cam_y = T[1]
+    cam_z = T[2]
+    if set_canonical:
+        T = np.zeros_like(T_W2C)
+        T = np.dot(first_frame_RT[:,:3], T) + first_frame_RT[:,-1]
+        T = T - T_W2C 
+        T = np.dot(R_W2C_inv, T)
+        cam_x = T[0]
+        cam_y = T[1]
+        cam_z = T[2]
+
+    # vertex
+    corn1 = np.array([np.tan(fov_rad / 2.0), 0.5*np.tan(fov_rad / 2.0), 1.0]) *scale 
+    corn2 = np.array([-np.tan(fov_rad / 2.0), 0.5*np.tan(fov_rad / 2.0), 1.0]) *scale
+    corn3 = np.array([0, -0.25*np.tan(fov_rad / 2.0), 1.0]) *scale
+    corn4 = np.array([0, -0.5*np.tan(fov_rad / 2.0), 1.0]) *scale
+
+    corn1 = corn1 - T_W2C
+    corn2 = corn2 - T_W2C
+    corn3 = corn3 - T_W2C
+    corn4 = corn4 - T_W2C
+    
+    corn1 = np.dot(R_W2C_inv, corn1)
+    corn2 = np.dot(R_W2C_inv, corn2)
+    corn3 = np.dot(R_W2C_inv, corn3) 
+    corn4 = np.dot(R_W2C_inv, corn4) 
+
+    # Now attach as offset to actual 3D camera position:
+    corn_x1 = corn1[0]
+    corn_y1 = corn1[1]
+    corn_z1 = corn1[2]
+    
+    corn_x2 = corn2[0]
+    corn_y2 = corn2[1]
+    corn_z2 = corn2[2]
+    
+    corn_x3 = corn3[0]
+    corn_y3 = corn3[1]
+    corn_z3 = corn3[2]
+    
+    corn_x4 = corn4[0]
+    corn_y4 = corn4[1]
+    corn_z4 = corn4[2]
+            
+
+    xs = [cam_x, corn_x1, corn_x2, corn_x3, corn_x4, ]
+    ys = [cam_y, corn_y1, corn_y2, corn_y3, corn_y4, ]
+    zs = [cam_z, corn_z1, corn_z2, corn_z3, corn_z4, ]
+
+    return np.array([xs, ys, zs]).T
+
+
+
+    # T_base = [
+    #             [1.,0.,0.],             ## W2C  x 的正方向： 相机朝左  left
+    #             [-1.,0.,0.],            ## W2C  x 的负方向： 相机朝右  right
+    #             [0., 1., 0.],           ## W2C  y 的正方向： 相机朝上  up     
+    #             [0.,-1.,0.],            ## W2C  y 的负方向： 相机朝下  down
+    #             [0.,0.,1.],             ## W2C  z 的正方向： 相机往前  zoom out
+    #             [0.,0.,-1.],            ## W2C  z 的负方向： 相机往前  zoom in
+    #         ]   
+    # radius = 1
+    # n = 16
+    # # step = 
+    # look_at = np.array([0, 0, 0.8]).reshape(3,1)
+    # # look_at = np.array([0, 0, 0.2]).reshape(3,1)
+
+    # T_list = []
+    # base_R = np.array([[1., 0., 0.],
+    #                 [0., 1., 0.],
+    #                 [0., 0., 1.]])
+    # res = [] 
+    # res_forsave = []
+    # T_range = 1.8
+
+
+
+    # for i in range(0, 16):
+    #     # theta = (1)*np.pi*i/n
+
+    #     R = base_R[:,:3]
+    #     T = np.array([0.,0.,1.]).reshape(3,1) * (i/n)*2
+    #     RT = np.concatenate([R,T], axis=1)
+    #     res.append(RT)
+        
+    # fig = vis_camera(res)
+    

main.py

@@ -0,0 +1,943 @@
+import argparse
+import datetime
+import glob
+import inspect
+import os
+import sys
+from inspect import Parameter
+from typing import Union
+
+import numpy as np
+import pytorch_lightning as pl
+import torch
+import torchvision
+import wandb
+from matplotlib import pyplot as plt
+from natsort import natsorted
+from omegaconf import OmegaConf
+from packaging import version
+from PIL import Image
+from pytorch_lightning import seed_everything
+from pytorch_lightning.callbacks import Callback
+from pytorch_lightning.loggers import WandbLogger
+from pytorch_lightning.trainer import Trainer
+from pytorch_lightning.utilities import rank_zero_only
+
+from sgm.util import exists, instantiate_from_config, isheatmap
+
+MULTINODE_HACKS = True
+
+
+def default_trainer_args():
+    argspec = dict(inspect.signature(Trainer.__init__).parameters)
+    argspec.pop("self")
+    default_args = {
+        param: argspec[param].default
+        for param in argspec
+        if argspec[param] != Parameter.empty
+    }
+    return default_args
+
+
+def get_parser(**parser_kwargs):
+    def str2bool(v):
+        if isinstance(v, bool):
+            return v
+        if v.lower() in ("yes", "true", "t", "y", "1"):
+            return True
+        elif v.lower() in ("no", "false", "f", "n", "0"):
+            return False
+        else:
+            raise argparse.ArgumentTypeError("Boolean value expected.")
+
+    parser = argparse.ArgumentParser(**parser_kwargs)
+    parser.add_argument(
+        "-n",
+        "--name",
+        type=str,
+        const=True,
+        default="",
+        nargs="?",
+        help="postfix for logdir",
+    )
+    parser.add_argument(
+        "--no_date",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,
+        help="if True, skip date generation for logdir and only use naming via opt.base or opt.name (+ opt.postfix, optionally)",
+    )
+    parser.add_argument(
+        "-r",
+        "--resume",
+        type=str,
+        const=True,
+        default="",
+        nargs="?",
+        help="resume from logdir or checkpoint in logdir",
+    )
+    parser.add_argument(
+        "-b",
+        "--base",
+        nargs="*",
+        metavar="base_config.yaml",
+        help="paths to base configs. Loaded from left-to-right. "
+        "Parameters can be overwritten or added with command-line options of the form `--key value`.",
+        default=list(),
+    )
+    parser.add_argument(
+        "-t",
+        "--train",
+        type=str2bool,
+        const=True,
+        default=True,
+        nargs="?",
+        help="train",
+    )
+    parser.add_argument(
+        "--no-test",
+        type=str2bool,
+        const=True,
+        default=False,
+        nargs="?",
+        help="disable test",
+    )
+    parser.add_argument(
+        "-p", "--project", help="name of new or path to existing project"
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,
+        help="enable post-mortem debugging",
+    )
+    parser.add_argument(
+        "-s",
+        "--seed",
+        type=int,
+        default=23,
+        help="seed for seed_everything",
+    )
+    parser.add_argument(
+        "-f",
+        "--postfix",
+        type=str,
+        default="",
+        help="post-postfix for default name",
+    )
+    parser.add_argument(
+        "--projectname",
+        type=str,
+        default="stablediffusion",
+    )
+    parser.add_argument(
+        "-l",
+        "--logdir",
+        type=str,
+        default="logs",
+        help="directory for logging dat shit",
+    )
+    parser.add_argument(
+        "--scale_lr",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,
+        help="scale base-lr by ngpu * batch_size * n_accumulate",
+    )
+    parser.add_argument(
+        "--legacy_naming",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,
+        help="name run based on config file name if true, else by whole path",
+    )
+    parser.add_argument(
+        "--enable_tf32",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,
+        help="enables the TensorFloat32 format both for matmuls and cuDNN for pytorch 1.12",
+    )
+    parser.add_argument(
+        "--startup",
+        type=str,
+        default=None,
+        help="Startuptime from distributed script",
+    )
+    parser.add_argument(
+        "--wandb",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,  # TODO: later default to True
+        help="log to wandb",
+    )
+    parser.add_argument(
+        "--no_base_name",
+        type=str2bool,
+        nargs="?",
+        const=True,
+        default=False,  # TODO: later default to True
+        help="log to wandb",
+    )
+    if version.parse(torch.__version__) >= version.parse("2.0.0"):
+        parser.add_argument(
+            "--resume_from_checkpoint",
+            type=str,
+            default=None,
+            help="single checkpoint file to resume from",
+        )
+    default_args = default_trainer_args()
+    for key in default_args:
+        parser.add_argument("--" + key, default=default_args[key])
+    return parser
+
+
+def get_checkpoint_name(logdir):
+    ckpt = os.path.join(logdir, "checkpoints", "last**.ckpt")
+    ckpt = natsorted(glob.glob(ckpt))
+    print('available "last" checkpoints:')
+    print(ckpt)
+    if len(ckpt) > 1:
+        print("got most recent checkpoint")
+        ckpt = sorted(ckpt, key=lambda x: os.path.getmtime(x))[-1]
+        print(f"Most recent ckpt is {ckpt}")
+        with open(os.path.join(logdir, "most_recent_ckpt.txt"), "w") as f:
+            f.write(ckpt + "\n")
+        try:
+            version = int(ckpt.split("/")[-1].split("-v")[-1].split(".")[0])
+        except Exception as e:
+            print("version confusion but not bad")
+            print(e)
+            version = 1
+        # version = last_version + 1
+    else:
+        # in this case, we only have one "last.ckpt"
+        ckpt = ckpt[0]
+        version = 1
+    melk_ckpt_name = f"last-v{version}.ckpt"
+    print(f"Current melk ckpt name: {melk_ckpt_name}")
+    return ckpt, melk_ckpt_name
+
+
+class SetupCallback(Callback):
+    def __init__(
+        self,
+        resume,
+        now,
+        logdir,
+        ckptdir,
+        cfgdir,
+        config,
+        lightning_config,
+        debug,
+        ckpt_name=None,
+    ):
+        super().__init__()
+        self.resume = resume
+        self.now = now
+        self.logdir = logdir
+        self.ckptdir = ckptdir
+        self.cfgdir = cfgdir
+        self.config = config
+        self.lightning_config = lightning_config
+        self.debug = debug
+        self.ckpt_name = ckpt_name
+
+    def on_exception(self, trainer: pl.Trainer, pl_module, exception):
+        if not self.debug and trainer.global_rank == 0:
+            print("Summoning checkpoint.")
+            if self.ckpt_name is None:
+                ckpt_path = os.path.join(self.ckptdir, "last.ckpt")
+            else:
+                ckpt_path = os.path.join(self.ckptdir, self.ckpt_name)
+            trainer.save_checkpoint(ckpt_path)
+
+    def on_fit_start(self, trainer, pl_module):
+        if trainer.global_rank == 0:
+            # Create logdirs and save configs
+            os.makedirs(self.logdir, exist_ok=True)
+            os.makedirs(self.ckptdir, exist_ok=True)
+            os.makedirs(self.cfgdir, exist_ok=True)
+
+            if "callbacks" in self.lightning_config:
+                if (
+                    "metrics_over_trainsteps_checkpoint"
+                    in self.lightning_config["callbacks"]
+                ):
+                    os.makedirs(
+                        os.path.join(self.ckptdir, "trainstep_checkpoints"),
+                        exist_ok=True,
+                    )
+            print("Project config")
+            print(OmegaConf.to_yaml(self.config))
+            if MULTINODE_HACKS:
+                import time
+
+                time.sleep(5)
+            OmegaConf.save(
+                self.config,
+                os.path.join(self.cfgdir, "{}-project.yaml".format(self.now)),
+            )
+
+            print("Lightning config")
+            print(OmegaConf.to_yaml(self.lightning_config))
+            OmegaConf.save(
+                OmegaConf.create({"lightning": self.lightning_config}),
+                os.path.join(self.cfgdir, "{}-lightning.yaml".format(self.now)),
+            )
+
+        else:
+            # ModelCheckpoint callback created log directory --- remove it
+            if not MULTINODE_HACKS and not self.resume and os.path.exists(self.logdir):
+                dst, name = os.path.split(self.logdir)
+                dst = os.path.join(dst, "child_runs", name)
+                os.makedirs(os.path.split(dst)[0], exist_ok=True)
+                try:
+                    os.rename(self.logdir, dst)
+                except FileNotFoundError:
+                    pass
+
+
+class ImageLogger(Callback):
+    def __init__(
+        self,
+        batch_frequency,
+        max_images,
+        clamp=True,
+        increase_log_steps=True,
+        rescale=True,
+        disabled=False,
+        log_on_batch_idx=False,
+        log_first_step=False,
+        log_images_kwargs=None,
+        log_before_first_step=False,
+        enable_autocast=True,
+    ):
+        super().__init__()
+        self.enable_autocast = enable_autocast
+        self.rescale = rescale
+        self.batch_freq = batch_frequency
+        self.max_images = max_images
+        self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+        if not increase_log_steps:
+            self.log_steps = [self.batch_freq]
+        self.clamp = clamp
+        self.disabled = disabled
+        self.log_on_batch_idx = log_on_batch_idx
+        self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+        self.log_first_step = log_first_step
+        self.log_before_first_step = log_before_first_step
+
+    @rank_zero_only
+    def log_local(
+        self,
+        save_dir,
+        split,
+        images,
+        global_step,
+        current_epoch,
+        batch_idx,
+        pl_module: Union[None, pl.LightningModule] = None,
+    ):
+        root = os.path.join(save_dir, "images", split)
+        for k in images:
+            if isheatmap(images[k]):
+                fig, ax = plt.subplots()
+                ax = ax.matshow(
+                    images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+                )
+                plt.colorbar(ax)
+                plt.axis("off")
+
+                filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+                    k, global_step, current_epoch, batch_idx
+                )
+                os.makedirs(root, exist_ok=True)
+                path = os.path.join(root, filename)
+                plt.savefig(path)
+                plt.close()
+                # TODO: support wandb
+            else:
+                grid = torchvision.utils.make_grid(images[k], nrow=4)
+                if self.rescale:
+                    grid = (grid + 1.0) / 2.0  # -1,1 -> 0,1; c,h,w
+                grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+                grid = grid.numpy()
+                grid = (grid * 255).astype(np.uint8)
+                filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+                    k, global_step, current_epoch, batch_idx
+                )
+                path = os.path.join(root, filename)
+                os.makedirs(os.path.split(path)[0], exist_ok=True)
+                img = Image.fromarray(grid)
+                img.save(path)
+                if exists(pl_module):
+                    assert isinstance(
+                        pl_module.logger, WandbLogger
+                    ), "logger_log_image only supports WandbLogger currently"
+                    pl_module.logger.log_image(
+                        key=f"{split}/{k}",
+                        images=[
+                            img,
+                        ],
+                        step=pl_module.global_step,
+                    )
+
+    @rank_zero_only
+    def log_img(self, pl_module, batch, batch_idx, split="train"):
+        check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+        if (
+            self.check_frequency(check_idx)
+            and hasattr(pl_module, "log_images")  # batch_idx % self.batch_freq == 0
+            and callable(pl_module.log_images)
+            and
+            # batch_idx > 5 and
+            self.max_images > 0
+        ):
+            logger = type(pl_module.logger)
+            is_train = pl_module.training
+            if is_train:
+                pl_module.eval()
+
+            gpu_autocast_kwargs = {
+                "enabled": self.enable_autocast,  # torch.is_autocast_enabled(),
+                "dtype": torch.get_autocast_gpu_dtype(),
+                "cache_enabled": torch.is_autocast_cache_enabled(),
+            }
+            with torch.no_grad(), torch.cuda.amp.autocast(**gpu_autocast_kwargs):
+                images = pl_module.log_images(
+                    batch, split=split, **self.log_images_kwargs
+                )
+
+            for k in images:
+                N = min(images[k].shape[0], self.max_images)
+                if not isheatmap(images[k]):
+                    images[k] = images[k][:N]
+                if isinstance(images[k], torch.Tensor):
+                    images[k] = images[k].detach().float().cpu()
+                    if self.clamp and not isheatmap(images[k]):
+                        images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+            self.log_local(
+                pl_module.logger.save_dir,
+                split,
+                images,
+                pl_module.global_step,
+                pl_module.current_epoch,
+                batch_idx,
+                pl_module=pl_module
+                if isinstance(pl_module.logger, WandbLogger)
+                else None,
+            )
+
+            if is_train:
+                pl_module.train()
+
+    def check_frequency(self, check_idx):
+        if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+            check_idx > 0 or self.log_first_step
+        ):
+            try:
+                self.log_steps.pop(0)
+            except IndexError as e:
+                print(e)
+                pass
+            return True
+        return False
+
+    @rank_zero_only
+    def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+        if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+            self.log_img(pl_module, batch, batch_idx, split="train")
+
+    @rank_zero_only
+    def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+        if self.log_before_first_step and pl_module.global_step == 0:
+            print(f"{self.__class__.__name__}: logging before training")
+            self.log_img(pl_module, batch, batch_idx, split="train")
+
+    @rank_zero_only
+    def on_validation_batch_end(
+        self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+    ):
+        if not self.disabled and pl_module.global_step > 0:
+            self.log_img(pl_module, batch, batch_idx, split="val")
+        if hasattr(pl_module, "calibrate_grad_norm"):
+            if (
+                pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+            ) and batch_idx > 0:
+                self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+@rank_zero_only
+def init_wandb(save_dir, opt, config, group_name, name_str):
+    print(f"setting WANDB_DIR to {save_dir}")
+    os.makedirs(save_dir, exist_ok=True)
+
+    os.environ["WANDB_DIR"] = save_dir
+    if opt.debug:
+        wandb.init(project=opt.projectname, mode="offline", group=group_name)
+    else:
+        wandb.init(
+            project=opt.projectname,
+            config=config,
+            settings=wandb.Settings(code_dir="./sgm"),
+            group=group_name,
+            name=name_str,
+        )
+
+
+if __name__ == "__main__":
+    # custom parser to specify config files, train, test and debug mode,
+    # postfix, resume.
+    # `--key value` arguments are interpreted as arguments to the trainer.
+    # `nested.key=value` arguments are interpreted as config parameters.
+    # configs are merged from left-to-right followed by command line parameters.
+
+    # model:
+    #   base_learning_rate: float
+    #   target: path to lightning module
+    #   params:
+    #       key: value
+    # data:
+    #   target: main.DataModuleFromConfig
+    #   params:
+    #      batch_size: int
+    #      wrap: bool
+    #      train:
+    #          target: path to train dataset
+    #          params:
+    #              key: value
+    #      validation:
+    #          target: path to validation dataset
+    #          params:
+    #              key: value
+    #      test:
+    #          target: path to test dataset
+    #          params:
+    #              key: value
+    # lightning: (optional, has sane defaults and can be specified on cmdline)
+    #   trainer:
+    #       additional arguments to trainer
+    #   logger:
+    #       logger to instantiate
+    #   modelcheckpoint:
+    #       modelcheckpoint to instantiate
+    #   callbacks:
+    #       callback1:
+    #           target: importpath
+    #           params:
+    #               key: value
+
+    now = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+
+    # add cwd for convenience and to make classes in this file available when
+    # running as `python main.py`
+    # (in particular `main.DataModuleFromConfig`)
+    sys.path.append(os.getcwd())
+
+    parser = get_parser()
+
+    opt, unknown = parser.parse_known_args()
+
+    if opt.name and opt.resume:
+        raise ValueError(
+            "-n/--name and -r/--resume cannot be specified both."
+            "If you want to resume training in a new log folder, "
+            "use -n/--name in combination with --resume_from_checkpoint"
+        )
+    melk_ckpt_name = None
+    name = None
+    if opt.resume:
+        if not os.path.exists(opt.resume):
+            raise ValueError("Cannot find {}".format(opt.resume))
+        if os.path.isfile(opt.resume):
+            paths = opt.resume.split("/")
+            # idx = len(paths)-paths[::-1].index("logs")+1
+            # logdir = "/".join(paths[:idx])
+            logdir = "/".join(paths[:-2])
+            ckpt = opt.resume
+            _, melk_ckpt_name = get_checkpoint_name(logdir)
+        else:
+            assert os.path.isdir(opt.resume), opt.resume
+            logdir = opt.resume.rstrip("/")
+            ckpt, melk_ckpt_name = get_checkpoint_name(logdir)
+
+        print("#" * 100)
+        print(f'Resuming from checkpoint "{ckpt}"')
+        print("#" * 100)
+
+        opt.resume_from_checkpoint = ckpt
+        base_configs = sorted(glob.glob(os.path.join(logdir, "configs/*.yaml")))
+        opt.base = base_configs + opt.base
+        _tmp = logdir.split("/")
+        nowname = _tmp[-1]
+    else:
+        if opt.name:
+            name = "_" + opt.name
+        elif opt.base:
+            if opt.no_base_name:
+                name = ""
+            else:
+                if opt.legacy_naming:
+                    cfg_fname = os.path.split(opt.base[0])[-1]
+                    cfg_name = os.path.splitext(cfg_fname)[0]
+                else:
+                    assert "configs" in os.path.split(opt.base[0])[0], os.path.split(
+                        opt.base[0]
+                    )[0]
+                    cfg_path = os.path.split(opt.base[0])[0].split(os.sep)[
+                        os.path.split(opt.base[0])[0].split(os.sep).index("configs")
+                        + 1 :
+                    ]  # cut away the first one (we assert all configs are in "configs")
+                    cfg_name = os.path.splitext(os.path.split(opt.base[0])[-1])[0]
+                    cfg_name = "-".join(cfg_path) + f"-{cfg_name}"
+                name = "_" + cfg_name
+        else:
+            name = ""
+        if not opt.no_date:
+            nowname = now + name + opt.postfix
+        else:
+            nowname = name + opt.postfix
+            if nowname.startswith("_"):
+                nowname = nowname[1:]
+        logdir = os.path.join(opt.logdir, nowname)
+        print(f"LOGDIR: {logdir}")
+
+    ckptdir = os.path.join(logdir, "checkpoints")
+    cfgdir = os.path.join(logdir, "configs")
+    seed_everything(opt.seed, workers=True)
+
+    # move before model init, in case a torch.compile(...) is called somewhere
+    if opt.enable_tf32:
+        # pt_version = version.parse(torch.__version__)
+        torch.backends.cuda.matmul.allow_tf32 = True
+        torch.backends.cudnn.allow_tf32 = True
+        print(f"Enabling TF32 for PyTorch {torch.__version__}")
+    else:
+        print(f"Using default TF32 settings for PyTorch {torch.__version__}:")
+        print(
+            f"torch.backends.cuda.matmul.allow_tf32={torch.backends.cuda.matmul.allow_tf32}"
+        )
+        print(f"torch.backends.cudnn.allow_tf32={torch.backends.cudnn.allow_tf32}")
+
+    try:
+        # init and save configs
+        configs = [OmegaConf.load(cfg) for cfg in opt.base]
+        cli = OmegaConf.from_dotlist(unknown)
+        config = OmegaConf.merge(*configs, cli)
+        lightning_config = config.pop("lightning", OmegaConf.create())
+        # merge trainer cli with config
+        trainer_config = lightning_config.get("trainer", OmegaConf.create())
+
+        # default to gpu
+        trainer_config["accelerator"] = "gpu"
+        #
+        standard_args = default_trainer_args()
+        for k in standard_args:
+            if getattr(opt, k) != standard_args[k]:
+                trainer_config[k] = getattr(opt, k)
+
+        ckpt_resume_path = opt.resume_from_checkpoint
+
+        if not "devices" in trainer_config and trainer_config["accelerator"] != "gpu":
+            del trainer_config["accelerator"]
+            cpu = True
+        else:
+            gpuinfo = trainer_config["devices"]
+            print(f"Running on GPUs {gpuinfo}")
+            cpu = False
+        trainer_opt = argparse.Namespace(**trainer_config)
+        lightning_config.trainer = trainer_config
+
+        # model
+        model = instantiate_from_config(config.model)
+
+        # trainer and callbacks
+        trainer_kwargs = dict()
+
+        # default logger configs
+        default_logger_cfgs = {
+            "wandb": {
+                "target": "pytorch_lightning.loggers.WandbLogger",
+                "params": {
+                    "name": nowname,
+                    # "save_dir": logdir,
+                    "offline": opt.debug,
+                    "id": nowname,
+                    "project": opt.projectname,
+                    "log_model": False,
+                    # "dir": logdir,
+                },
+            },
+            "csv": {
+                "target": "pytorch_lightning.loggers.CSVLogger",
+                "params": {
+                    "name": "testtube",  # hack for sbord fanatics
+                    "save_dir": logdir,
+                },
+            },
+        }
+        default_logger_cfg = default_logger_cfgs["wandb" if opt.wandb else "csv"]
+        if opt.wandb:
+            # TODO change once leaving "swiffer" config directory
+            try:
+                group_name = nowname.split(now)[-1].split("-")[1]
+            except:
+                group_name = nowname
+            default_logger_cfg["params"]["group"] = group_name
+            init_wandb(
+                os.path.join(os.getcwd(), logdir),
+                opt=opt,
+                group_name=group_name,
+                config=config,
+                name_str=nowname,
+            )
+        if "logger" in lightning_config:
+            logger_cfg = lightning_config.logger
+        else:
+            logger_cfg = OmegaConf.create()
+        logger_cfg = OmegaConf.merge(default_logger_cfg, logger_cfg)
+        trainer_kwargs["logger"] = instantiate_from_config(logger_cfg)
+
+        # modelcheckpoint - use TrainResult/EvalResult(checkpoint_on=metric) to
+        # specify which metric is used to determine best models
+        default_modelckpt_cfg = {
+            "target": "pytorch_lightning.callbacks.ModelCheckpoint",
+            "params": {
+                "dirpath": ckptdir,
+                "filename": "{epoch:06}",
+                "verbose": True,
+                "save_last": True,
+            },
+        }
+        if hasattr(model, "monitor"):
+            print(f"Monitoring {model.monitor} as checkpoint metric.")
+            default_modelckpt_cfg["params"]["monitor"] = model.monitor
+            default_modelckpt_cfg["params"]["save_top_k"] = 3
+
+        if "modelcheckpoint" in lightning_config:
+            modelckpt_cfg = lightning_config.modelcheckpoint
+        else:
+            modelckpt_cfg = OmegaConf.create()
+        modelckpt_cfg = OmegaConf.merge(default_modelckpt_cfg, modelckpt_cfg)
+        print(f"Merged modelckpt-cfg: \n{modelckpt_cfg}")
+
+        # https://pytorch-lightning.readthedocs.io/en/stable/extensions/strategy.html
+        # default to ddp if not further specified
+        default_strategy_config = {"target": "pytorch_lightning.strategies.DDPStrategy"}
+
+        if "strategy" in lightning_config:
+            strategy_cfg = lightning_config.strategy
+        else:
+            strategy_cfg = OmegaConf.create()
+            default_strategy_config["params"] = {
+                "find_unused_parameters": False,
+                # "static_graph": True,
+                # "ddp_comm_hook": default.fp16_compress_hook  # TODO: experiment with this, also for DDPSharded
+            }
+        strategy_cfg = OmegaConf.merge(default_strategy_config, strategy_cfg)
+        print(
+            f"strategy config: \n ++++++++++++++ \n {strategy_cfg} \n ++++++++++++++ "
+        )
+        trainer_kwargs["strategy"] = instantiate_from_config(strategy_cfg)
+
+        # add callback which sets up log directory
+        default_callbacks_cfg = {
+            "setup_callback": {
+                "target": "main.SetupCallback",
+                "params": {
+                    "resume": opt.resume,
+                    "now": now,
+                    "logdir": logdir,
+                    "ckptdir": ckptdir,
+                    "cfgdir": cfgdir,
+                    "config": config,
+                    "lightning_config": lightning_config,
+                    "debug": opt.debug,
+                    "ckpt_name": melk_ckpt_name,
+                },
+            },
+            "image_logger": {
+                "target": "main.ImageLogger",
+                "params": {"batch_frequency": 1000, "max_images": 4, "clamp": True},
+            },
+            "learning_rate_logger": {
+                "target": "pytorch_lightning.callbacks.LearningRateMonitor",
+                "params": {
+                    "logging_interval": "step",
+                    # "log_momentum": True
+                },
+            },
+        }
+        if version.parse(pl.__version__) >= version.parse("1.4.0"):
+            default_callbacks_cfg.update({"checkpoint_callback": modelckpt_cfg})
+
+        if "callbacks" in lightning_config:
+            callbacks_cfg = lightning_config.callbacks
+        else:
+            callbacks_cfg = OmegaConf.create()
+
+        if "metrics_over_trainsteps_checkpoint" in callbacks_cfg:
+            print(
+                "Caution: Saving checkpoints every n train steps without deleting. This might require some free space."
+            )
+            default_metrics_over_trainsteps_ckpt_dict = {
+                "metrics_over_trainsteps_checkpoint": {
+                    "target": "pytorch_lightning.callbacks.ModelCheckpoint",
+                    "params": {
+                        "dirpath": os.path.join(ckptdir, "trainstep_checkpoints"),
+                        "filename": "{epoch:06}-{step:09}",
+                        "verbose": True,
+                        "save_top_k": -1,
+                        "every_n_train_steps": 10000,
+                        "save_weights_only": True,
+                    },
+                }
+            }
+            default_callbacks_cfg.update(default_metrics_over_trainsteps_ckpt_dict)
+
+        callbacks_cfg = OmegaConf.merge(default_callbacks_cfg, callbacks_cfg)
+        if "ignore_keys_callback" in callbacks_cfg and ckpt_resume_path is not None:
+            callbacks_cfg.ignore_keys_callback.params["ckpt_path"] = ckpt_resume_path
+        elif "ignore_keys_callback" in callbacks_cfg:
+            del callbacks_cfg["ignore_keys_callback"]
+
+        trainer_kwargs["callbacks"] = [
+            instantiate_from_config(callbacks_cfg[k]) for k in callbacks_cfg
+        ]
+        if not "plugins" in trainer_kwargs:
+            trainer_kwargs["plugins"] = list()
+
+        # cmd line trainer args (which are in trainer_opt) have always priority over config-trainer-args (which are in trainer_kwargs)
+        trainer_opt = vars(trainer_opt)
+        trainer_kwargs = {
+            key: val for key, val in trainer_kwargs.items() if key not in trainer_opt
+        }
+        trainer = Trainer(**trainer_opt, **trainer_kwargs)
+
+        trainer.logdir = logdir  ###
+
+        # data
+        data = instantiate_from_config(config.data)
+        # NOTE according to https://pytorch-lightning.readthedocs.io/en/latest/datamodules.html
+        # calling these ourselves should not be necessary but it is.
+        # lightning still takes care of proper multiprocessing though
+        data.prepare_data()
+        # data.setup()
+        print("#### Data #####")
+        try:
+            for k in data.datasets:
+                print(
+                    f"{k}, {data.datasets[k].__class__.__name__}, {len(data.datasets[k])}"
+                )
+        except:
+            print("datasets not yet initialized.")
+
+        # configure learning rate
+        if "batch_size" in config.data.params:
+            bs, base_lr = config.data.params.batch_size, config.model.base_learning_rate
+        else:
+            bs, base_lr = (
+                config.data.params.train.loader.batch_size,
+                config.model.base_learning_rate,
+            )
+        if not cpu:
+            ngpu = len(lightning_config.trainer.devices.strip(",").split(","))
+        else:
+            ngpu = 1
+        if "accumulate_grad_batches" in lightning_config.trainer:
+            accumulate_grad_batches = lightning_config.trainer.accumulate_grad_batches
+        else:
+            accumulate_grad_batches = 1
+        print(f"accumulate_grad_batches = {accumulate_grad_batches}")
+        lightning_config.trainer.accumulate_grad_batches = accumulate_grad_batches
+        if opt.scale_lr:
+            model.learning_rate = accumulate_grad_batches * ngpu * bs * base_lr
+            print(
+                "Setting learning rate to {:.2e} = {} (accumulate_grad_batches) * {} (num_gpus) * {} (batchsize) * {:.2e} (base_lr)".format(
+                    model.learning_rate, accumulate_grad_batches, ngpu, bs, base_lr
+                )
+            )
+        else:
+            model.learning_rate = base_lr
+            print("++++ NOT USING LR SCALING ++++")
+            print(f"Setting learning rate to {model.learning_rate:.2e}")
+
+        # allow checkpointing via USR1
+        def melk(*args, **kwargs):
+            # run all checkpoint hooks
+            if trainer.global_rank == 0:
+                print("Summoning checkpoint.")
+                if melk_ckpt_name is None:
+                    ckpt_path = os.path.join(ckptdir, "last.ckpt")
+                else:
+                    ckpt_path = os.path.join(ckptdir, melk_ckpt_name)
+                trainer.save_checkpoint(ckpt_path)
+
+        def divein(*args, **kwargs):
+            if trainer.global_rank == 0:
+                import pudb
+
+                pudb.set_trace()
+
+        import signal
+
+        signal.signal(signal.SIGUSR1, melk)
+        signal.signal(signal.SIGUSR2, divein)
+
+        # run
+        if opt.train:
+            try:
+                trainer.fit(model, data, ckpt_path=ckpt_resume_path)
+            except Exception:
+                if not opt.debug:
+                    melk()
+                raise
+        if not opt.no_test and not trainer.interrupted:
+            trainer.test(model, data)
+    except RuntimeError as err:
+        if MULTINODE_HACKS:
+            import datetime
+            import os
+            import socket
+
+            import requests
+
+            device = os.environ.get("CUDA_VISIBLE_DEVICES", "?")
+            hostname = socket.gethostname()
+            ts = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+            resp = requests.get("http://169.254.169.254/latest/meta-data/instance-id")
+            print(
+                f"ERROR at {ts} on {hostname}/{resp.text} (CUDA_VISIBLE_DEVICES={device}): {type(err).__name__}: {err}",
+                flush=True,
+            )
+        raise err
+    except Exception:
+        if opt.debug and trainer.global_rank == 0:
+            try:
+                import pudb as debugger
+            except ImportError:
+                import pdb as debugger
+            debugger.post_mortem()
+        raise
+    finally:
+        # move newly created debug project to debug_runs
+        if opt.debug and not opt.resume and trainer.global_rank == 0:
+            dst, name = os.path.split(logdir)
+            dst = os.path.join(dst, "debug_runs", name)
+            os.makedirs(os.path.split(dst)[0], exist_ok=True)
+            os.rename(logdir, dst)
+
+        if opt.wandb:
+            wandb.finish()
+        # if trainer.global_rank == 0:
+        #    print(trainer.profiler.summary())

main/inference/motionctrl_cmcm.py

@@ -0,0 +1,416 @@
+import argparse
+import datetime
+import json
+import math
+import os
+import sys
+import time
+from glob import glob
+from pathlib import Path
+from typing import Optional
+
+import cv2
+import numpy as np
+import torch
+import torchvision
+from einops import rearrange, repeat
+from fire import Fire
+from omegaconf import OmegaConf
+from PIL import Image
+from torchvision.transforms import CenterCrop, Compose, Resize, ToTensor
+
+sys.path.insert(1, os.path.join(sys.path[0], '..', '..'))
+from sgm.util import default, instantiate_from_config
+
+camera_poses = [
+    'test_camera_L',
+    'test_camera_D',
+    'test_camera_I',
+    'test_camera_O',
+    'test_camera_R',
+    'test_camera_U',
+    'test_camera_Round-ZoomIn',
+    'test_camera_Round-RI_90',
+]
+
+def to_relative_RT2(org_pose, keyframe_idx=0, keyframe_zero=False):
+        org_pose = org_pose.reshape(-1, 3, 4) # [t, 3, 4]
+        R_dst = org_pose[:, :, :3]
+        T_dst = org_pose[:, :, 3:]
+
+        R_src = R_dst[keyframe_idx: keyframe_idx+1].repeat(org_pose.shape[0], axis=0) # [t, 3, 3]
+        T_src = T_dst[keyframe_idx: keyframe_idx+1].repeat(org_pose.shape[0], axis=0)
+
+        R_src_inv = R_src.transpose(0, 2, 1) # [t, 3, 3]
+        
+        R_rel = R_dst @ R_src_inv # [t, 3, 3]
+        T_rel = T_dst - R_rel@T_src
+
+        RT_rel = np.concatenate([R_rel, T_rel], axis=-1) # [t, 3, 4]
+        RT_rel = RT_rel.reshape(-1, 12) # [t, 12]
+
+        if keyframe_zero:
+            RT_rel[keyframe_idx] = np.zeros_like(RT_rel[keyframe_idx])
+
+        return RT_rel
+
+def get_RT(pose_dir='', video_frames=14, frame_stride=1, speed=1.0, **kwargs):
+    pose_file = [f'{pose_dir}/{pose}.json' for pose in camera_poses]
+    pose_sample_num = len(pose_file)
+
+    pose_sample_num = len(pose_file)
+
+    data_list = []
+    pose_name = []
+
+
+    for idx in range(pose_sample_num):
+        cur_pose_name = camera_poses[idx].replace('test_camera_', '')
+        pose_name.append(cur_pose_name)
+
+        with open(pose_file[idx], 'r') as f:
+            pose = json.load(f)
+        pose = np.array(pose) # [t, 12]
+        
+        while frame_stride * video_frames > pose.shape[0]:
+            frame_stride -= 1
+
+        pose = pose[::frame_stride]
+        if video_frames < 16:
+            half = (pose.shape[0] - video_frames) // 2
+            pose = pose[half:half+video_frames]
+        # pose = pose[:video_frames]
+        pose = pose.reshape(-1, 3, 4) # [t, 3, 4]
+        # rescale
+        pose[:, :, -1] = pose[:, :, -1] * np.array([3, 1, 4]) * speed
+        pose = to_relative_RT2(pose)
+        
+            
+        pose = torch.tensor(pose).float() # [t, 12]
+        data_list.append(pose)
+
+    # data_list = torch.stack(data_list, dim=0) # [pose_sample_num, t, 12]
+    return data_list, pose_name
+
+def sample(
+    input_path: str = "examples/camera_poses",  # Can either be image file or folder with image files
+    ckpt: str = "checkpoints/motionctrl_svd.ckpt",
+    config: str = None,
+    num_frames: Optional[int] = None,
+    num_steps: Optional[int] = None,
+    version: str = "svd",
+    fps_id: int = 6,
+    motion_bucket_id: int = 127,
+    cond_aug: float = 0.02,
+    seed: int = 23,
+    decoding_t: int = 1,  # Number of frames decoded at a time! This eats most VRAM. Reduce if necessary.
+    device: str = "cuda",
+    output_folder: Optional[str] = None,
+    save_fps: int = 10,
+    resize: Optional[bool] = False,
+    pose_dir: str = '',
+    sample_num: int = 1,
+    height: int = 576,
+    width: int = 1024,
+    transform: Optional[bool] = False,
+    save_images: Optional[bool] = False,
+    speed: float = 1.0,
+):
+    """
+    Simple script to generate a single sample conditioned on an image `input_path` or multiple images, one for each
+    image file in folder `input_path`. If you run out of VRAM, try decreasing `decoding_t`.
+    """
+
+    assert (version == "svd"), "Only SVD is supported for now."
+    num_frames = default(num_frames, 14)
+    num_steps = default(num_steps, 25)
+    output_folder = default(output_folder, "outputs/motionctrl_svd/")
+    model_config = default(config, "configs/inference/config_motionctrl_cmcm.yaml")
+
+    model, filter = load_model(
+        model_config,
+        ckpt,
+        device,
+        num_frames,
+        num_steps,
+    )
+    torch.manual_seed(seed)
+
+    path = Path(input_path)
+    all_img_paths = []
+    if path.is_file():
+        if any([input_path.endswith(x) for x in ["jpg", "jpeg", "png"]]):
+            all_img_paths = [input_path]
+        else:
+            raise ValueError("Path is not valid image file.")
+    elif path.is_dir():
+        all_img_paths = sorted(
+            [
+                f
+                for f in path.iterdir()
+                if f.is_file() and f.suffix.lower() in [".jpg", ".jpeg", ".png"]
+            ]
+        )
+        if len(all_img_paths) == 0:
+            raise ValueError("Folder does not contain any images.")
+    else:
+        raise ValueError
+    
+    if transform:
+        spatial_transform = Compose([
+            Resize(size=width),
+            CenterCrop(size=(height, width)),
+        ])
+    
+    # get camera poses
+    RTs, pose_name = get_RT(pose_dir=pose_dir, video_frames=num_frames, frame_stride=1, speed=speed)
+
+    print(f'loaded {len(all_img_paths)} images.')
+    os.makedirs(output_folder, exist_ok=True)
+    for no, input_img_path in enumerate(all_img_paths):
+        
+        filepath, fullflname = os.path.split(input_img_path)
+        filename, ext = os.path.splitext(fullflname)
+        print(f'-sample {no+1}: {filename} ...')
+
+        # RTs = RTs[0:1]
+        for RT_idx in range(len(RTs)):
+            cur_pose_name = pose_name[RT_idx]
+            print(f'--pose: {cur_pose_name} ...')
+            RT = RTs[RT_idx]
+            RT = RT.unsqueeze(0).repeat(2,1,1)
+            RT = RT.to(device)
+
+            with Image.open(input_img_path) as image:
+                if image.mode == "RGBA":
+                    image = image.convert("RGB")
+                if transform:
+                    image = spatial_transform(image)
+                if resize:
+                    image = image.resize((width, height))
+                w, h = image.size
+
+                if h % 64 != 0 or w % 64 != 0:
+                    width, height = map(lambda x: x - x % 64, (w, h))
+                    image = image.resize((width, height))
+                    print(
+                        f"WARNING: Your image is of size {h}x{w} which is not divisible by 64. We are resizing to {height}x{width}!"
+                    )
+
+                image = ToTensor()(image)
+                image = image * 2.0 - 1.0
+
+            image = image.unsqueeze(0).to(device)
+            H, W = image.shape[2:]
+            assert image.shape[1] == 3
+            F = 8
+            C = 4
+            shape = (num_frames, C, H // F, W // F)
+            if (H, W) != (576, 1024):
+                print(
+                    "WARNING: The conditioning frame you provided is not 576x1024. This leads to suboptimal performance as model was only trained on 576x1024. Consider increasing `cond_aug`."
+                )
+            if motion_bucket_id > 255:
+                print(
+                    "WARNING: High motion bucket! This may lead to suboptimal performance."
+                )
+
+            if fps_id < 5:
+                print("WARNING: Small fps value! This may lead to suboptimal performance.")
+
+            if fps_id > 30:
+                print("WARNING: Large fps value! This may lead to suboptimal performance.")
+
+            value_dict = {}
+            value_dict["motion_bucket_id"] = motion_bucket_id
+            value_dict["fps_id"] = fps_id
+            value_dict["cond_aug"] = cond_aug
+            value_dict["cond_frames_without_noise"] = image
+            value_dict["cond_frames"] = image + cond_aug * torch.randn_like(image)
+
+            with torch.no_grad():
+                with torch.autocast(device):
+                    batch, batch_uc = get_batch(
+                        get_unique_embedder_keys_from_conditioner(model.conditioner),
+                        value_dict,
+                        [1, num_frames],
+                        T=num_frames,
+                        device=device,
+                    )
+                    c, uc = model.conditioner.get_unconditional_conditioning(
+                        batch,
+                        batch_uc=batch_uc,
+                        force_uc_zero_embeddings=[
+                            "cond_frames",
+                            "cond_frames_without_noise",
+                        ],
+                    )
+
+                    for k in ["crossattn", "concat"]:
+                        uc[k] = repeat(uc[k], "b ... -> b t ...", t=num_frames)
+                        uc[k] = rearrange(uc[k], "b t ... -> (b t) ...", t=num_frames)
+                        c[k] = repeat(c[k], "b ... -> b t ...", t=num_frames)
+                        c[k] = rearrange(c[k], "b t ... -> (b t) ...", t=num_frames)
+
+                    
+
+                    additional_model_inputs = {}
+                    additional_model_inputs["image_only_indicator"] = torch.zeros(
+                        2, num_frames
+                    ).to(device)
+                    #additional_model_inputs["image_only_indicator"][:,0] = 1
+                    additional_model_inputs["num_video_frames"] = batch["num_video_frames"]
+
+                    
+                    additional_model_inputs["RT"] = RT
+
+                    def denoiser(input, sigma, c):
+                        return model.denoiser(
+                            model.model, input, sigma, c, **additional_model_inputs
+                        )
+
+                    results = []
+                    for j in range(sample_num):
+                        randn = torch.randn(shape, device=device)
+                        samples_z = model.sampler(denoiser, randn, cond=c, uc=uc)
+                        model.en_and_decode_n_samples_a_time = decoding_t
+                        samples_x = model.decode_first_stage(samples_z)
+                        samples = torch.clamp((samples_x + 1.0) / 2.0, min=0.0, max=1.0) # [1*t, c, h, w]
+                        results.append(samples)
+
+                    samples = torch.stack(results, dim=0) # [sample_num, t, c, h, w]
+                    samples = samples.data.cpu()
+
+                    video_path = os.path.join(output_folder, f"{filename}_{cur_pose_name}.mp4")
+                    save_results(samples, video_path, fps=save_fps)
+
+                    if save_images:
+                        for i in range(sample_num):
+                            cur_output_folder = os.path.join(output_folder, f"{filename}", f"{cur_pose_name}", f"{i}")
+                            os.makedirs(cur_output_folder, exist_ok=True)
+                            for j in range(num_frames):
+                                cur_img_path = os.path.join(cur_output_folder, f"{j:06d}.png")
+                                torchvision.utils.save_image(samples[i,j], cur_img_path)
+    
+    print(f'Done! results saved in {output_folder}.')
+
+def save_results(resutls, filename, fps=10):
+    video = resutls.permute(1, 0, 2, 3, 4) # [t, sample_num, c, h, w]
+    frame_grids = [torchvision.utils.make_grid(framesheet, nrow=int(video.shape[1])) for framesheet in video] #[3, 1*h, n*w]
+    grid = torch.stack(frame_grids, dim=0) # stack in temporal dim [t, 3, n*h, w]
+    # already in [0,1]
+    grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1)
+    torchvision.io.write_video(filename, grid, fps=fps, video_codec='h264', options={'crf': '10'})
+
+def get_unique_embedder_keys_from_conditioner(conditioner):
+    return list(set([x.input_key for x in conditioner.embedders]))
+
+
+def get_batch(keys, value_dict, N, T, device):
+    batch = {}
+    batch_uc = {}
+
+    for key in keys:
+        if key == "fps_id":
+            batch[key] = (
+                torch.tensor([value_dict["fps_id"]])
+                .to(device)
+                .repeat(int(math.prod(N)))
+            )
+        elif key == "motion_bucket_id":
+            batch[key] = (
+                torch.tensor([value_dict["motion_bucket_id"]])
+                .to(device)
+                .repeat(int(math.prod(N)))
+            )
+        elif key == "cond_aug":
+            batch[key] = repeat(
+                torch.tensor([value_dict["cond_aug"]]).to(device),
+                "1 -> b",
+                b=math.prod(N),
+            )
+        elif key == "cond_frames":
+            batch[key] = repeat(value_dict["cond_frames"], "1 ... -> b ...", b=N[0])
+        elif key == "cond_frames_without_noise":
+            batch[key] = repeat(
+                value_dict["cond_frames_without_noise"], "1 ... -> b ...", b=N[0]
+            )
+        else:
+            batch[key] = value_dict[key]
+
+    if T is not None:
+        batch["num_video_frames"] = T
+
+    for key in batch.keys():
+        if key not in batch_uc and isinstance(batch[key], torch.Tensor):
+            batch_uc[key] = torch.clone(batch[key])
+    return batch, batch_uc
+
+
+def load_model(
+    config: str,
+    ckpt: str,
+    device: str,
+    num_frames: int,
+    num_steps: int,
+):
+
+    config = OmegaConf.load(config)
+    config.model.params.ckpt_path = ckpt
+    if device == "cuda":
+        config.model.params.conditioner_config.params.emb_models[
+            0
+        ].params.open_clip_embedding_config.params.init_device = device
+
+    config.model.params.sampler_config.params.num_steps = num_steps
+    config.model.params.sampler_config.params.guider_config.params.num_frames = (
+        num_frames
+    )
+
+    model = instantiate_from_config(config.model)
+
+    model = model.to(device).eval()    
+
+    filter = None #DeepFloydDataFiltering(verbose=False, device=device)
+    return model, filter
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--seed", type=int, default=23, help="seed for seed_everything")
+    parser.add_argument("--ckpt", type=str, default=None, help="checkpoint path")
+    parser.add_argument("--config", type=str, help="config (yaml) path")
+    parser.add_argument("--input", type=str, default=None, help="image path or folder")
+    parser.add_argument("--savedir", type=str, default=None, help="results saving path")
+    parser.add_argument("--savefps", type=int, default=10, help="video fps to generate")
+    parser.add_argument("--n_samples", type=int, default=1, help="num of samples per prompt",)
+    parser.add_argument("--ddim_steps", type=int, default=50, help="steps of ddim if positive, otherwise use DDPM",)
+    parser.add_argument("--ddim_eta", type=float, default=1.0, help="eta for ddim sampling (0.0 yields deterministic sampling)",)
+    parser.add_argument("--frames", type=int, default=-1, help="frames num to inference")
+    parser.add_argument("--fps", type=int, default=6, help="control the fps")
+    parser.add_argument("--motion", type=int, default=127, help="control the motion magnitude")
+    parser.add_argument("--cond_aug", type=float, default=0.02, help="adding noise to input image")
+    parser.add_argument("--decoding_t", type=int, default=1, help="frames num to decoding per time")
+    parser.add_argument("--resize", action='store_true', default=False, help="resize all input to default resolution")
+    parser.add_argument("--sample_num", type=int, default=1, help="frames num to decoding per time")
+    parser.add_argument("--pose_dir", type=str, default='', help="checkpoint path")
+    parser.add_argument("--height", type=int, default=576, help="frames num to decoding per time")
+    parser.add_argument("--width", type=int, default=1024, help="frames num to decoding per time")
+    parser.add_argument("--transform", action='store_true', default=False, help="resize all input to specific resolution")
+    parser.add_argument("--save_images", action='store_true', default=False, help="save images")
+    parser.add_argument("--speed", type=float, default=1.0, help="speed of camera motion")
+    return parser
+
+
+if __name__ == "__main__":
+    now = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+    print("@MotionCrl+SVD Inference: %s"%now)
+    #Fire(sample)
+    parser = get_parser()
+    args = parser.parse_args()
+    sample(input_path=args.input, ckpt=args.ckpt, config=args.config, num_frames=args.frames, num_steps=args.ddim_steps, \
+        fps_id=args.fps, motion_bucket_id=args.motion, cond_aug=args.cond_aug, seed=args.seed, \
+        decoding_t=args.decoding_t, output_folder=args.savedir, save_fps=args.savefps, resize=args.resize,
+        pose_dir=args.pose_dir, sample_num=args.sample_num, height=args.height, width=args.width,
+        transform=args.transform, save_images=args.save_images, speed=args.speed)
+    

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers = 
+  inference: mark as inference test (deselect with '-m "not inference"')

requirements.txt

@@ -0,0 +1,32 @@
+Pytorch-Lightning==1.9.0
+decord
+kornia
+timm
+open_clip_torch
+av
+omegaconf
+transformers
+einops
+scikit-learn
+taming-transformers-rom1504
+pandas
+triton
+xformers==0.0.16
+torch==1.13.1
+torchvision
+fairscale
+psutil==5.9.5
+annotated-types==0.5.0
+plotly
+imageio==2.14.1
+imageio-ffmpeg==0.4.7
+opencv-python==4.8.0.74
+moviepy
+Pillow
+tqdm
+gradio==3.37.0
+webdataset
+Fire
+natsort
+wandb
+clip @ git+https://github.com/openai/CLIP.git

scripts/demo/detect.py

@@ -0,0 +1,156 @@
+import argparse
+
+import cv2
+import numpy as np
+
+try:
+    from imwatermark import WatermarkDecoder
+except ImportError as e:
+    try:
+        # Assume some of the other dependencies such as torch are not fulfilled
+        # import file without loading unnecessary libraries.
+        import importlib.util
+        import sys
+
+        spec = importlib.util.find_spec("imwatermark.maxDct")
+        assert spec is not None
+        maxDct = importlib.util.module_from_spec(spec)
+        sys.modules["maxDct"] = maxDct
+        spec.loader.exec_module(maxDct)
+
+        class WatermarkDecoder(object):
+            """A minimal version of
+            https://github.com/ShieldMnt/invisible-watermark/blob/main/imwatermark/watermark.py
+            to only reconstruct bits using dwtDct"""
+
+            def __init__(self, wm_type="bytes", length=0):
+                assert wm_type == "bits", "Only bits defined in minimal import"
+                self._wmType = wm_type
+                self._wmLen = length
+
+            def reconstruct(self, bits):
+                if len(bits) != self._wmLen:
+                    raise RuntimeError("bits are not matched with watermark length")
+
+                return bits
+
+            def decode(self, cv2Image, method="dwtDct", **configs):
+                (r, c, channels) = cv2Image.shape
+                if r * c < 256 * 256:
+                    raise RuntimeError("image too small, should be larger than 256x256")
+
+                bits = []
+                assert method == "dwtDct"
+                embed = maxDct.EmbedMaxDct(watermarks=[], wmLen=self._wmLen, **configs)
+                bits = embed.decode(cv2Image)
+                return self.reconstruct(bits)
+
+    except:
+        raise e
+
+
+# A fixed 48-bit message that was choosen at random
+# WATERMARK_MESSAGE = 0xB3EC907BB19E
+WATERMARK_MESSAGE = 0b101100111110110010010000011110111011000110011110
+# bin(x)[2:] gives bits of x as str, use int to convert them to 0/1
+WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]]
+MATCH_VALUES = [
+    [27, "No watermark detected"],
+    [33, "Partial watermark match. Cannot determine with certainty."],
+    [
+        35,
+        (
+            "Likely watermarked. In our test 0.02% of real images were "
+            'falsely detected as "Likely watermarked"'
+        ),
+    ],
+    [
+        49,
+        (
+            "Very likely watermarked. In our test no real images were "
+            'falsely detected as "Very likely watermarked"'
+        ),
+    ],
+]
+
+
+class GetWatermarkMatch:
+    def __init__(self, watermark):
+        self.watermark = watermark
+        self.num_bits = len(self.watermark)
+        self.decoder = WatermarkDecoder("bits", self.num_bits)
+
+    def __call__(self, x: np.ndarray) -> np.ndarray:
+        """
+        Detects the number of matching bits the predefined watermark with one
+        or multiple images. Images should be in cv2 format, e.g. h x w x c BGR.
+
+        Args:
+            x: ([B], h w, c) in range [0, 255]
+
+        Returns:
+           number of matched bits ([B],)
+        """
+        squeeze = len(x.shape) == 3
+        if squeeze:
+            x = x[None, ...]
+
+        bs = x.shape[0]
+        detected = np.empty((bs, self.num_bits), dtype=bool)
+        for k in range(bs):
+            detected[k] = self.decoder.decode(x[k], "dwtDct")
+        result = np.sum(detected == self.watermark, axis=-1)
+        if squeeze:
+            return result[0]
+        else:
+            return result
+
+
+get_watermark_match = GetWatermarkMatch(WATERMARK_BITS)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "filename",
+        nargs="+",
+        type=str,
+        help="Image files to check for watermarks",
+    )
+    opts = parser.parse_args()
+
+    print(
+        """
+        This script tries to detect watermarked images. Please be aware of
+        the following:
+        - As the watermark is supposed to be invisible, there is the risk that
+          watermarked images may not be detected.
+        - To maximize the chance of detection make sure that the image has the same
+          dimensions as when the watermark was applied (most likely 1024x1024
+          or 512x512).
+        - Specific image manipulation may drastically decrease the chance that
+          watermarks can be detected.
+        - There is also the chance that an image has the characteristics of the
+          watermark by chance.
+        - The watermark script is public, anybody may watermark any images, and
+          could therefore claim it to be generated.
+        - All numbers below are based on a test using 10,000 images without any
+          modifications after applying the watermark.
+        """
+    )
+
+    for fn in opts.filename:
+        image = cv2.imread(fn)
+        if image is None:
+            print(f"Couldn't read {fn}. Skipping")
+            continue
+
+        num_bits = get_watermark_match(image)
+        k = 0
+        while num_bits > MATCH_VALUES[k][0]:
+            k += 1
+        print(
+            f"{fn}: {MATCH_VALUES[k][1]}",
+            f"Bits that matched the watermark {num_bits} from {len(WATERMARK_BITS)}\n",
+            sep="\n\t",
+        )