Spaces:
Runtime error
Runtime error
| import copy | |
| import functools | |
| import os | |
| import time | |
| from types import SimpleNamespace | |
| import numpy as np | |
| import blobfile as bf | |
| import torch | |
| from torch.optim import AdamW | |
| from diffusion import logger | |
| from utils import dist_util | |
| from diffusion.fp16_util import MixedPrecisionTrainer | |
| from diffusion.resample import LossAwareSampler, UniformSampler | |
| from tqdm import tqdm | |
| from diffusion.resample import create_named_schedule_sampler | |
| # from data_loaders.humanml.networks.evaluator_wrapper import EvaluatorMDMWrapper | |
| # from eval import eval_humanml, eval_humanact12_uestc | |
| # from data_loaders.get_data import get_dataset_loader | |
| # For ImageNet experiments, this was a good default value. | |
| # We found that the lg_loss_scale quickly climbed to | |
| # 20-21 within the first ~1K steps of training. | |
| ## intial log loss scale ## | |
| INITIAL_LOG_LOSS_SCALE = 20.0 | |
| # do not # | |
| class TrainLoop: | |
| def __init__(self, args, train_platform, model, diffusion, data): | |
| self.args = args | |
| self.dataset = args.dataset | |
| self.train_platform = train_platform | |
| self.model = model | |
| self.diffusion = diffusion | |
| if self.args.nprocs > 1: | |
| self.cond_mode = model.module.cond_mode | |
| else: | |
| self.cond_mode = model.cond_mode | |
| self.data = data | |
| self.batch_size = args.batch_size | |
| self.microbatch = args.batch_size # deprecating this option | |
| self.lr = args.lr | |
| self.log_interval = args.log_interval | |
| self.save_interval = args.save_interval | |
| self.resume_checkpoint = args.resume_checkpoint | |
| self.use_fp16 = False # deprecating this option | |
| self.fp16_scale_growth = 1e-3 # deprecating this option | |
| self.weight_decay = args.weight_decay | |
| self.lr_anneal_steps = args.lr_anneal_steps | |
| self.step = 0 | |
| # self.resume_step = 0 | |
| self.resume_step = False | |
| self.global_batch = self.batch_size # * dist.get_world_size() | |
| self.num_steps = args.num_steps | |
| self.num_epochs = self.num_steps // len(self.data) + 1 | |
| self.sync_cuda = torch.cuda.is_available() | |
| if self.args.finetune_with_cond: # finetune_with_cond -> | |
| self._load_and_sync_parameters_cond() # load parameters here | |
| print(f"Setting trans linear layer to zero for conditioning...") | |
| self.model.set_trans_linear_layer_to_zero() # | |
| else: # finetune_with_cond | |
| self._load_and_sync_parameters() | |
| self.mp_trainer = MixedPrecisionTrainer( # mixed | |
| model=self.model, # | |
| use_fp16=self.use_fp16, | |
| fp16_scale_growth=self.fp16_scale_growth, | |
| args=args, | |
| ) | |
| self.save_dir = args.save_dir | |
| self.overwrite = args.overwrite | |
| self.opt = AdamW( | |
| self.mp_trainer.master_params, lr=self.lr, weight_decay=self.weight_decay | |
| ) | |
| if self.resume_step and not args.not_load_opt: | |
| self._load_optimizer_state() | |
| # Model was resumed, either due to a restart or a checkpoint | |
| # being specified at the command line. | |
| print(f"dist_utils: {dist_util.dev()}") | |
| self.device = torch.device("cpu") | |
| if torch.cuda.is_available() and dist_util.dev() != 'cpu': | |
| self.device = torch.device(dist_util.dev()) | |
| self.schedule_sampler_type = 'uniform' | |
| self.schedule_sampler = create_named_schedule_sampler(self.schedule_sampler_type, diffusion) | |
| self.eval_wrapper, self.eval_data, self.eval_gt_data = None, None, None | |
| # if args.dataset in ['kit', 'humanml', 'motion_ours'] and args.eval_during_training: | |
| # mm_num_samples = 0 # mm is super slow hence we won't run it during training | |
| # mm_num_repeats = 0 # mm is super slow hence we won't run it during training | |
| # gen_loader = get_dataset_loader(name=args.dataset, batch_size=args.eval_batch_size, num_frames=None, | |
| # split=args.eval_split, | |
| # hml_mode='eval') | |
| # self.eval_gt_data = get_dataset_loader(name=args.dataset, batch_size=args.eval_batch_size, num_frames=None, | |
| # split=args.eval_split, | |
| # hml_mode='gt') | |
| # self.eval_wrapper = EvaluatorMDMWrapper(args.dataset, dist_util.dev()) | |
| # self.eval_data = { | |
| # 'test': lambda: eval_humanml.get_mdm_loader( | |
| # model, diffusion, args.eval_batch_size, | |
| # gen_loader, mm_num_samples, mm_num_repeats, gen_loader.dataset.opt.max_motion_length, | |
| # args.eval_num_samples, scale=1., | |
| # ) | |
| # } | |
| self.use_ddp = False if self.args.nprocs == 1 else True | |
| self.ddp_model = self.model | |
| def safe_load_ckpt(self, model, state_dicts): | |
| ori_dict = state_dicts | |
| part_dict = dict() | |
| model_dict = model.state_dict() | |
| tot_params_n = 0 | |
| for k in ori_dict: | |
| if self.args.resume_diff: | |
| if k in model_dict: | |
| v = ori_dict[k] | |
| part_dict[k] = v | |
| tot_params_n += 1 | |
| else: | |
| if k in model_dict and "denoising" not in k: | |
| v = ori_dict[k] | |
| part_dict[k] = v | |
| tot_params_n += 1 | |
| model_dict.update(part_dict) | |
| model.load_state_dict(model_dict) | |
| print(f"Resume glb-backbone finished!! Total number of parameters: {tot_params_n}.") | |
| # | |
| def _load_and_sync_parameters_cond(self): | |
| resume_checkpoint = find_resume_checkpoint() or self.resume_checkpoint | |
| if resume_checkpoint: | |
| self.resume_step = parse_resume_step_from_filename(resume_checkpoint) | |
| logger.log(f"loading model from checkpoint: {resume_checkpoint}...") | |
| state_dicts = dist_util.load_state_dict( | |
| resume_checkpoint, map_location=dist_util.dev() | |
| ) | |
| if self.args.diff_basejtsrel: | |
| # if self.args.finetune_with_cond_rel: | |
| model_dict = self.model.state_dict() | |
| # elif self.args.finetune_with_cond_jtsobj: | |
| model_dict.update(state_dicts) | |
| self.model.load_state_dict(model_dict) | |
| if self.args.finetune_with_cond_jtsobj: # finetune_with_cond_jtsobj --> finetune_with_cond_jtsobj | |
| # cond_joints_offset_input_process <- joints_offset_input_process; cond_sequence_pos_encoder <- sequence_pos_encoder; cond_seqTransEncoder <- seqTransEncoder | |
| self.model.cond_joints_offset_input_process.load_state_dict(self.model.joints_offset_input_process.state_dict()) | |
| self.model.cond_sequence_pos_encoder.load_state_dict(self.model.sequence_pos_encoder.state_dict()) | |
| self.model.cond_seqTransEncoder.load_state_dict(self.model.seqTransEncoder.state_dict()) | |
| else: | |
| raise ValueError(f"Must have diff_basejtsrel setting, others not implemented yet!") | |
| # self.safe_load_ckpt(self.model, | |
| # dist_util.load_state_dict( | |
| # resume_checkpoint, map_location=dist_util.dev() | |
| # ) | |
| # ) | |
| def _load_and_sync_parameters(self): | |
| resume_checkpoint = find_resume_checkpoint() or self.resume_checkpoint | |
| if resume_checkpoint: | |
| self.resume_step = parse_resume_step_from_filename(resume_checkpoint) | |
| logger.log(f"loading model from checkpoint: {resume_checkpoint}...") | |
| # self.model.load_state_dict( | |
| # dist_util.load_state_dict( | |
| # resume_checkpoint, map_location=dist_util.dev() | |
| # ) | |
| # ) | |
| self.safe_load_ckpt(self.model, | |
| dist_util.load_state_dict( | |
| resume_checkpoint, map_location=dist_util.dev() | |
| ) | |
| ) | |
| def _load_optimizer_state(self): | |
| main_checkpoint = find_resume_checkpoint() or self.resume_checkpoint | |
| opt_checkpoint = bf.join( | |
| bf.dirname(main_checkpoint), f"opt{self.resume_step:09}.pt" | |
| ) | |
| if bf.exists(opt_checkpoint): | |
| logger.log(f"loading optimizer state from checkpoint: {opt_checkpoint}") | |
| state_dict = dist_util.load_state_dict( | |
| opt_checkpoint, map_location=dist_util.dev() | |
| ) | |
| self.opt.load_state_dict(state_dict) | |
| def run_loop(self): | |
| for epoch in range(self.num_epochs): | |
| print(f'Starting epoch {epoch}') | |
| for batch in tqdm(self.data): | |
| # for motion, cond in tqdm(self.data): ## motion; cond; data ## | |
| if not (not self.lr_anneal_steps or self.step + self.resume_step < self.lr_anneal_steps): | |
| break | |
| # print(f"motion: {motion.size()}, ") ## motion.to(self.device) | |
| # motion = motion.to(self.device) | |
| # cond['y'] = {key: val.to(self.device) if torch.is_tensor(val) else val for key, val in cond['y'].items()} | |
| # batch = { | |
| # key: val.to(self.device) if torch.is_tensor(val) else ([subval.to(self.device) for subval in val] if isinstance(val, list) else val) for key, val in batch.items() | |
| # } | |
| for k in batch: | |
| if isinstance(batch[k], torch.Tensor): | |
| batch[k] = batch[k].to(self.device) | |
| elif isinstance(batch[k], list): | |
| batch[k] = [subval.to(self.device) if isinstance(subval, torch.Tensor) else subval for subval in batch[k]] | |
| else: | |
| batch[k] = batch[k] | |
| ## run current motion and cond ## | |
| ## run step ## | |
| self.run_step(batch) ## run step for the motion and cond ## | |
| ## ===== log useful things ==== ## | |
| if self.step % self.log_interval == 0: # | |
| loss_dict = logger.get_current().name2val | |
| print('step[{}]: loss[{:0.5f}]'.format(self.step+self.resume_step, loss_dict["loss"])) | |
| for k in loss_dict: | |
| v = loss_dict[k] | |
| if k in ['rel_pred_loss', 'dist_pred_loss', 'dec_e_along_normals_loss', 'dec_e_vt_normals_loss', 'joints_pred_loss', 'jts_pred_loss', 'jts_latent_denoising_loss', 'basejtsrel_pred_loss', 'basejtsrel_latent_denoising_loss', 'basejtse_along_normals_pred_loss', 'basejtse_vt_normals_pred_loss', 'basejtse_latent_denoising_loss', "KL_loss", "avg_joints_pred_loss", "basejtrel_denoising_loss", "avgjts_denoising_loss"]: ## avg_joints_pred_loss # avg joints pred loss # | |
| print(f"\t{k}: {loss_dict[k].mean().item() if isinstance(loss_dict[k], torch.Tensor) else loss_dict[k]}") | |
| if k in ['step', 'samples'] or '_q' in k: # step samples # | |
| continue | |
| else: | |
| self.train_platform.report_scalar(name=k, value=v, iteration=self.step, group_name='Loss') | |
| # for k,v in logger.get_current().name2val.items(): | |
| # if k == 'loss': | |
| # print('step[{}]: loss[{:0.5f}]'.format(self.step+self.resume_step, v)) | |
| # if k in ['step', 'samples'] or '_q' in k: # step samples # | |
| # continue | |
| # else: | |
| # self.train_platform.report_scalar(name=k, value=v, iteration=self.step, group_name='Loss') | |
| ## ===== save checkpoints ===== ## | |
| if self.step % self.save_interval == 0: | |
| ## save; model.eval; | |
| self.save() | |
| if self.args.nprocs > 1: | |
| self.model.module.eval() | |
| else: | |
| self.model.eval() | |
| self.evaluate() | |
| if self.args.nprocs > 1: | |
| self.model.module.train() | |
| else: | |
| self.model.train() | |
| # Run for a finite amount of time in integration tests. | |
| if os.environ.get("DIFFUSION_TRAINING_TEST", "") and self.step > 0: | |
| return | |
| self.step += 1 | |
| if not (not self.lr_anneal_steps or self.step + self.resume_step < self.lr_anneal_steps): | |
| break | |
| # Save the last checkpoint if it wasn't already saved. | |
| if (self.step - 1) % self.save_interval != 0: | |
| self.save() | |
| self.evaluate() | |
| def evaluate(self): | |
| if not self.args.eval_during_training: | |
| return | |
| start_eval = time.time() | |
| if self.eval_wrapper is not None: | |
| print('Running evaluation loop: [Should take about 90 min]') | |
| log_file = os.path.join(self.save_dir, f'eval_humanml_{(self.step + self.resume_step):09d}.log') | |
| diversity_times = 300 | |
| mm_num_times = 0 # mm is super slow hence we won't run it during training | |
| eval_dict = eval_humanml.evaluation( | |
| self.eval_wrapper, self.eval_gt_data, self.eval_data, log_file, | |
| replication_times=self.args.eval_rep_times, diversity_times=diversity_times, mm_num_times=mm_num_times, run_mm=False) | |
| print(eval_dict) | |
| for k, v in eval_dict.items(): | |
| if k.startswith('R_precision'): | |
| for i in range(len(v)): | |
| self.train_platform.report_scalar(name=f'top{i + 1}_' + k, value=v[i], | |
| iteration=self.step + self.resume_step, | |
| group_name='Eval') | |
| else: | |
| self.train_platform.report_scalar(name=k, value=v, iteration=self.step + self.resume_step, | |
| group_name='Eval') | |
| elif self.dataset in ['humanact12', 'uestc']: | |
| eval_args = SimpleNamespace(num_seeds=self.args.eval_rep_times, num_samples=self.args.eval_num_samples, | |
| batch_size=self.args.eval_batch_size, device=self.device, guidance_param = 1, | |
| dataset=self.dataset, unconstrained=self.args.unconstrained, | |
| model_path=os.path.join(self.save_dir, self.ckpt_file_name())) | |
| eval_dict = eval_humanact12_uestc.evaluate(eval_args, model=self.model, diffusion=self.diffusion, data=self.data.dataset) | |
| print(f'Evaluation results on {self.dataset}: {sorted(eval_dict["feats"].items())}') | |
| for k, v in eval_dict["feats"].items(): | |
| if 'unconstrained' not in k: | |
| self.train_platform.report_scalar(name=k, value=np.array(v).astype(float).mean(), iteration=self.step, group_name='Eval') | |
| else: | |
| self.train_platform.report_scalar(name=k, value=np.array(v).astype(float).mean(), iteration=self.step, group_name='Eval Unconstrained') | |
| end_eval = time.time() | |
| print(f'Evaluation time: {round(end_eval-start_eval)/60}min') | |
| def run_step(self, batch): | |
| self.forward_backward(batch) ## forward | |
| self.mp_trainer.optimize(self.opt) | |
| self._anneal_lr() | |
| self.log_step() | |
| def forward_backward(self, batch): | |
| self.mp_trainer.zero_grad() | |
| for i in range(0, batch['base_pts'].shape[0], self.microbatch): | |
| # print(f"batch_device: {batch['base_pts'].device}") ## base pts device | |
| # Eliminates the microbatch feature | |
| assert i == 0 | |
| assert self.microbatch == self.batch_size | |
| micro = batch | |
| # micro_cond = cond | |
| ## micro-batch # base_pts; base_pts # | |
| last_batch = (i + self.microbatch) >= batch['base_pts'].shape[0] | |
| t, weights = self.schedule_sampler.sample(micro['base_pts'].shape[0], dist_util.dev()) | |
| # print(f"t: {t.size()}, weights: {weights.size()}, t_device: {t.device}, weights_device: {weights.device}") | |
| # compute_losses = functools.partial( | |
| # self.diffusion.training_losses, | |
| # self.ddp_model, | |
| # micro, # [bs, ch, image_size, image_size] | |
| # t, # [bs](int) sampled timesteps | |
| # model_kwargs={'y': batch}, # | |
| # dataset=self.data.dataset | |
| # ) | |
| # # if last_batch or not self.use_ddp: | |
| # # losses = compute_losses() ## compute lossses | |
| # # else: | |
| # # with self.ddp_model.no_sync(): | |
| # # losses = compute_losses() | |
| # if not self.use_ddp: | |
| # losses = compute_losses() ## compute lossses | |
| # else: | |
| # with self.ddp_model.no_sync(): | |
| # losses = compute_losses() | |
| ### training losses ### | |
| losses = self.diffusion.training_losses( | |
| self.ddp_model, | |
| micro, # [bs, ch, image_size, image_size] | |
| t, # [bs](int) sampled timesteps | |
| model_kwargs={'y': batch}, | |
| dataset=self.data.dataset | |
| ) | |
| # loss aware sampler # | |
| if isinstance(self.schedule_sampler, LossAwareSampler): | |
| self.schedule_sampler.update_with_local_losses( | |
| t, losses["loss"].detach() | |
| ) | |
| # print(losses["loss"].size(), f"weights: {weights.size()}") | |
| loss = (losses["loss"] * weights).mean() | |
| if self.args.nprocs > 1: | |
| torch.distributed.barrier() | |
| dist_util.reduce_mean(loss, self.args.nprocs) ## args nprocs ## | |
| log_loss_dict( | |
| self.diffusion, t, {k: v * weights for k, v in losses.items()} | |
| ) | |
| self.mp_trainer.backward(loss) | |
| def predict_single_step(self, batch, use_t=None): | |
| # self.mp_trainer.zero_grad() | |
| # use_t is not Noen | |
| tot_samples = [] | |
| tot_targets = [] | |
| tot_dec_disp_e_along_normals = [] | |
| tot_dec_disp_e_vt_normals = [] | |
| tot_pred_joints_quant = [] | |
| # | |
| for i in range(0, batch['base_pts'].shape[0], self.microbatch): | |
| # Eliminates the microbatch feature | |
| assert i == 0 | |
| assert self.microbatch == self.batch_size | |
| micro = batch | |
| # ## micro batch ## | |
| rhand_joints = micro['rhand_joints'] | |
| # micro_cond = cond # micro_cond and cond ## | |
| ## micro-batch ## | |
| last_batch = (i + self.microbatch) >= batch['base_pts'].shape[0] | |
| t, weights = self.schedule_sampler.sample(micro['base_pts'].shape[0], dist_util.dev()) | |
| if use_t is not None: | |
| t = torch.zeros_like(t) + use_t | |
| # batch: bsz x nnjoints x 3 x nnframes # | |
| ## === original sampling === ## | |
| # terms, model_output, target, t = self.diffusion.predict_sample_single_step(self.ddp_model, micro, t, model_kwargs=micro_cond, noise=None, dataset=self.data.dataset) ## restricted by those things ## | |
| ### use p_sample_loop from the diffusion model ### | |
| sample_fn = self.diffusion.p_sample_loop | |
| samples = sample_fn( | |
| self.ddp_model, | |
| rhand_joints.shape, | |
| clip_denoised=False, | |
| model_kwargs=micro, | |
| skip_timesteps=0, | |
| init_image=micro, | |
| progress=True, | |
| dump_steps=None, | |
| noise=None, ## noise ## # | |
| # const_noise=False, # whether to cond on noise ## | |
| const_noise=self.args.const_noise, ## const noise ! | |
| st_timestep=use_t, | |
| ) | |
| # sample either as joints or as relative positions for each base pts # | |
| tot_samples.append(samples['sampled_rhand_joints']) | |
| # tot_samples = tot_samples + samples # samples rhand_joints; targets rhand_joints | |
| ### add rhand joints | |
| tot_targets.append(micro['rhand_joints']) | |
| if 'e_disp_rel_to_base_along_normals' in samples: | |
| tot_dec_disp_e_along_normals.append(samples['e_disp_rel_to_base_along_normals']) | |
| tot_dec_disp_e_vt_normals.append(samples['e_disp_rel_to_baes_vt_normals']) | |
| if 'pred_joint_quants' in samples: | |
| tot_pred_joints_quant.append(samples['pred_joint_quants']) | |
| # tot_targets.append(samples['rhand_joints']) | |
| # all of them target at joints samples ## | |
| model_output = torch.cat(tot_samples, dim=0) | |
| # model_output = tot_samples | |
| target = torch.cat(tot_targets, dim=0) | |
| if len(tot_dec_disp_e_along_normals) > 0: | |
| tot_dec_disp_e_along_normals = torch.cat(tot_dec_disp_e_along_normals, dim=0) | |
| tot_dec_disp_e_vt_normals = torch.cat(tot_dec_disp_e_vt_normals, dim=0) ### tot_dec_disp_e_vt_normals # | |
| if len(tot_pred_joints_quant) > 0: | |
| tot_pred_joints_quant = torch.cat(tot_pred_joints_quant, dim=0) | |
| # print(f"Returning with model_output; {model_output.size()}, target: {target.size()}") | |
| print(f"Returning with target: {target.size()}") | |
| ### returning the samples and tarets ### | |
| if isinstance(tot_pred_joints_quant, torch.Tensor): | |
| return model_output, target, tot_pred_joints_quant | |
| elif isinstance(tot_dec_disp_e_along_normals, torch.Tensor): | |
| return model_output, target, tot_dec_disp_e_along_normals, tot_dec_disp_e_vt_normals | |
| else: | |
| return model_output, target | |
| # return model_output, target | |
| ### predict from data ### | |
| def predict_from_data(self): | |
| # for epoch in range(self.num_epochs): # | |
| # print(f'Starting epoch {epoch}') # the | |
| ## ==== a single pass for a single sequence ==== ## | |
| tot_model_outputs = [] | |
| tot_targets = [] | |
| tot_st_idxes = [] | |
| tot_ed_idxes = [] | |
| tot_pert_verts = [] | |
| tot_verts = [] | |
| tot_dec_disp_e_along_normals = [] | |
| tot_dec_disp_e_vt_normals = [] | |
| ## motion; cond; data ## | |
| tot_pred_joints_quant = [] | |
| for batch in tqdm(self.data): # batch data # | |
| for k in batch: | |
| if isinstance(batch[k], torch.Tensor): | |
| batch[k] = batch[k].to(self.device) | |
| elif isinstance(batch[k], list): | |
| # batch[k] = [subval.to(self.device) for subval in batch[k]] | |
| batch[k] = [subval.to(self.device) if isinstance(subval, torch.Tensor) else subval for subval in batch[k]] | |
| else: | |
| batch[k] = batch[k] | |
| # motion = motion.to(self.device) | |
| # cond['y'] = {key: val.to(self.device) if torch.is_tensor(val) else val for key, val in cond['y'].items()} | |
| # st_idxes = cond['y']['st_idx'] # st_idxes | |
| # ed_idxes = cond['y']['ed_idx'] # ed_idxes | |
| # pert_verts = cond['y']['pert_verts'] | |
| # verts = cond['y']['verts'] | |
| # if 'avg_joints' in cond['y']: | |
| # avg_joints = cond['y']['avg_joints'] | |
| # std_joints = cond['y']['std_joints'] | |
| # else: | |
| # avg_joints = None | |
| # std_joints = None | |
| st_idxes = batch['st_idx'] | |
| ed_idxes = batch['ed_idx'] | |
| pert_verts = batch['pert_verts'] | |
| verts = batch['verts'] | |
| # tot pert verts | |
| tot_pert_verts.append(pert_verts) | |
| tot_verts.append(verts) | |
| ## generative denoising -> we want to use it for the denoising task ## | |
| # std_joints: bsz x 1 | |
| # avg_joints: bsz x 1 x 3 --> mean of joints for each batch | |
| ## predict_single_step ## | |
| # model_output, target = self.predict_single_step(batch, use_t=1) ### trainingjloop ours | |
| use_t = self.args.use_t | |
| tot_pred_outputs = self.predict_single_step(batch, use_t=use_t) | |
| #### diff baes jts e ## | |
| if len(tot_pred_outputs) == 3: | |
| model_output, target, pred_joints_quant = tot_pred_outputs | |
| tot_pred_joints_quant.append(pred_joints_quant) | |
| elif self.args.diff_basejtse: | |
| model_output, target, dec_disp_e_along_normals, dec_disp_e_vt_normals = tot_pred_outputs | |
| else: | |
| model_output, target = tot_pred_outputs[:2] | |
| # model output; target # | |
| ## model_output: ([6, 21, 3, 60]), target: torch.Size([6, 21, 3, 60]) | |
| # if avg_joints is not None: | |
| # ### model_output, target ### | |
| # model_output = (model_output * std_joints.unsqueeze(-1).unsqueeze(-1)) + avg_joints.unsqueeze(-1) | |
| # target = (target * std_joints.unsqueeze(-1).unsqueeze(-1)) + avg_joints.unsqueeze(-1) | |
| # 10 -> the output sequence is still a little bit noisy # | |
| # 100 -> 60 | |
| # the difficulty of predicting base pts rel position information # | |
| # the difficulty of the prediction problem # base pts rel information p | |
| ## predicting base pts relative positions to the base_pts predictions ## ### base pts predictions ## wu le ## | |
| if self.args.diff_basejtse: | |
| tot_dec_disp_e_along_normals.append(dec_disp_e_along_normals) | |
| tot_dec_disp_e_vt_normals.append(dec_disp_e_vt_normals) | |
| tot_st_idxes.append(st_idxes) | |
| tot_ed_idxes.append(ed_idxes) | |
| tot_targets.append(target) | |
| tot_model_outputs.append(model_output) | |
| # tot_model_outputs.extend(model_output) | |
| # tot_model_outputs = tot_model_outputs + model_output | |
| tot_st_idxes = torch.cat(tot_st_idxes, dim=0) | |
| tot_ed_idxes = torch.cat(tot_ed_idxes, dim=0) | |
| tot_targets = torch.cat(tot_targets, dim=0) | |
| tot_model_outputs = torch.cat(tot_model_outputs, dim=0) | |
| if self.args.diff_basejtse: | |
| tot_dec_disp_e_along_normals = torch.cat(tot_dec_disp_e_along_normals, dim=0) | |
| tot_dec_disp_e_vt_normals = torch.cat(tot_dec_disp_e_vt_normals, dim=0) | |
| if len(tot_pred_joints_quant) > 0: | |
| tot_pred_joints_quant = torch.cat(tot_pred_joints_quant, dim=0) | |
| tot_pert_verts = torch.cat(tot_pert_verts, dim=0) | |
| tot_verts = torch.cat(tot_verts, dim=0) | |
| if isinstance(tot_pred_joints_quant, torch.Tensor): | |
| return tot_targets, tot_model_outputs, tot_st_idxes, tot_ed_idxes, tot_pert_verts, tot_verts, tot_pred_joints_quant | |
| elif self.args.diff_basejtse: | |
| return tot_targets, tot_model_outputs, tot_st_idxes, tot_ed_idxes, tot_pert_verts, tot_verts, tot_dec_disp_e_along_normals, tot_dec_disp_e_vt_normals | |
| else: | |
| return tot_targets, tot_model_outputs, tot_st_idxes, tot_ed_idxes, tot_pert_verts, tot_verts | |
| def _anneal_lr(self): | |
| if not self.lr_anneal_steps: | |
| return | |
| frac_done = (self.step + self.resume_step) / self.lr_anneal_steps | |
| lr = self.lr * (1 - frac_done) | |
| for param_group in self.opt.param_groups: | |
| param_group["lr"] = lr | |
| def log_step(self): | |
| logger.logkv("step", self.step + self.resume_step) | |
| logger.logkv("samples", (self.step + self.resume_step + 1) * self.global_batch) | |
| def ckpt_file_name(self): | |
| return f"model{(self.step+self.resume_step):09d}.pt" | |
| def save(self): | |
| def save_checkpoint(params): | |
| if self.args.finetune_with_cond: # | |
| state_dict = self.mp_trainer.model.state_dict() | |
| else: | |
| state_dict = self.mp_trainer.master_params_to_state_dict(params) | |
| # Do not save CLIP weights | |
| clip_weights = [e for e in state_dict.keys() if e.startswith('clip_model.')] | |
| for e in clip_weights: | |
| del state_dict[e] | |
| # logger.log(f"saving model...") | |
| filename = self.ckpt_file_name() | |
| model_sv_fn = os.path.join(self.save_dir, filename) | |
| logger.log(f"saving model to {model_sv_fn}...") | |
| with bf.BlobFile(bf.join(self.save_dir, filename), "wb") as f: | |
| torch.save(state_dict, f) | |
| save_checkpoint(self.mp_trainer.master_params) | |
| with bf.BlobFile( | |
| bf.join(self.save_dir, f"opt{(self.step+self.resume_step):09d}.pt"), | |
| "wb", | |
| ) as f: | |
| torch.save(self.opt.state_dict(), f) | |
| def parse_resume_step_from_filename(filename): | |
| """ | |
| Parse filenames of the form path/to/modelNNNNNN.pt, where NNNNNN is the | |
| checkpoint's number of steps. | |
| """ | |
| split = filename.split("model") | |
| if len(split) < 2: | |
| return 0 | |
| split1 = split[-1].split(".")[0] | |
| try: | |
| return int(split1) | |
| except ValueError: | |
| return 0 | |
| def get_blob_logdir(): | |
| # You can change this to be a separate path to save checkpoints to | |
| # a blobstore or some external drive. | |
| return logger.get_dir() | |
| def find_resume_checkpoint(): | |
| # On your infrastructure, you may want to override this to automatically | |
| # discover the latest checkpoint on your blob storage, etc. | |
| return None | |
| def log_loss_dict(diffusion, ts, losses): | |
| for key, values in losses.items(): | |
| logger.logkv_mean(key, values.mean().item()) | |
| # Log the quantiles (four quartiles, in particular). | |
| for sub_t, sub_loss in zip(ts.cpu().numpy(), values.detach().cpu().numpy()): | |
| quartile = int(4 * sub_t / diffusion.num_timesteps) | |
| logger.logkv_mean(f"{key}_q{quartile}", sub_loss) | |