Spaces:
Runtime error
Runtime error
| # import tensorflow_probability as tfp | |
| import numpy as np | |
| import torch.nn as nn | |
| # import layer_utils | |
| import torch | |
| # import data_utils_torch as data_utils | |
| import math ## | |
| import pickle | |
| import os | |
| import utils.model_util as model_util | |
| import time | |
| class DMTet: | |
| def __init__(self): # triangle_table -> the triangle table # | |
| self.triangle_table = torch.tensor([ | |
| [-1, -1, -1, -1, -1, -1], | |
| [ 1, 0, 2, -1, -1, -1], | |
| [ 4, 0, 3, -1, -1, -1], | |
| [ 1, 4, 2, 1, 3, 4], | |
| [ 3, 1, 5, -1, -1, -1], | |
| [ 2, 3, 0, 2, 5, 3], | |
| [ 1, 4, 0, 1, 5, 4], | |
| [ 4, 2, 5, -1, -1, -1], | |
| [ 4, 5, 2, -1, -1, -1], | |
| [ 4, 1, 0, 4, 5, 1], | |
| [ 3, 2, 0, 3, 5, 2], | |
| [ 1, 3, 5, -1, -1, -1], | |
| [ 4, 1, 2, 4, 3, 1], | |
| [ 3, 0, 4, -1, -1, -1], | |
| [ 2, 0, 1, -1, -1, -1], | |
| [-1, -1, -1, -1, -1, -1] | |
| ], dtype=torch.long, device='cuda') | |
| # triangles table; # base tet edges # | |
| self.num_triangles_table = torch.tensor([0,1,1,2,1,2,2,1,1,2,2,1,2,1,1,0], dtype=torch.long, device='cuda') | |
| self.base_tet_edges = torch.tensor([0,1,0,2,0,3,1,2,1,3,2,3], dtype=torch.long, device='cuda') | |
| ############################################################################### | |
| # Utility functions | |
| ############################################################################### | |
| # sorted edges # | |
| def sort_edges(self, edges_ex2): | |
| with torch.no_grad(): | |
| order = (edges_ex2[:,0] > edges_ex2[:,1]).long() | |
| order = order.unsqueeze(dim=1) | |
| a = torch.gather(input=edges_ex2, index=order, dim=1) | |
| b = torch.gather(input=edges_ex2, index=1-order, dim=1) | |
| return torch.stack([a, b],-1) | |
| def map_uv(self, faces, face_gidx, max_idx): | |
| N = int(np.ceil(np.sqrt((max_idx+1)//2))) | |
| tex_y, tex_x = torch.meshgrid( | |
| torch.linspace(0, 1 - (1 / N), N, dtype=torch.float32, device="cuda"), | |
| torch.linspace(0, 1 - (1 / N), N, dtype=torch.float32, device="cuda"), | |
| indexing='ij' | |
| ) | |
| pad = 0.9 / N | |
| uvs = torch.stack([ | |
| tex_x , tex_y, | |
| tex_x + pad, tex_y, | |
| tex_x + pad, tex_y + pad, | |
| tex_x , tex_y + pad | |
| ], dim=-1).view(-1, 2) | |
| def _idx(tet_idx, N): | |
| x = tet_idx % N | |
| y = torch.div(tet_idx, N, rounding_mode='trunc') | |
| return y * N + x | |
| tet_idx = _idx(torch.div(face_gidx, 2, rounding_mode='trunc'), N) | |
| tri_idx = face_gidx % 2 | |
| uv_idx = torch.stack(( | |
| tet_idx * 4, tet_idx * 4 + tri_idx + 1, tet_idx * 4 + tri_idx + 2 | |
| ), dim = -1). view(-1, 3) | |
| return uvs, uv_idx | |
| ############################################################################### | |
| # Marching tets implementation | |
| ############################################################################### | |
| def __call__(self, pos_nx3, sdf_n, tet_fx4): # po | |
| with torch.no_grad(): | |
| occ_n = sdf_n > 0 | |
| occ_fx4 = occ_n[tet_fx4.reshape(-1)].reshape(-1,4) | |
| occ_sum = torch.sum(occ_fx4, -1) | |
| valid_tets = (occ_sum>0) & (occ_sum<4) | |
| occ_sum = occ_sum[valid_tets] | |
| # find all vertices | |
| all_edges = tet_fx4[valid_tets][:,self.base_tet_edges].reshape(-1,2) | |
| all_edges = self.sort_edges(all_edges) | |
| unique_edges, idx_map = torch.unique(all_edges,dim=0, return_inverse=True) | |
| unique_edges = unique_edges.long() | |
| mask_edges = occ_n[unique_edges.reshape(-1)].reshape(-1,2).sum(-1) == 1 | |
| mapping = torch.ones((unique_edges.shape[0]), dtype=torch.long, device="cuda") * -1 | |
| mapping[mask_edges] = torch.arange(mask_edges.sum(), dtype=torch.long,device="cuda") | |
| idx_map = mapping[idx_map] # map edges to verts | |
| interp_v = unique_edges[mask_edges] | |
| edges_to_interp = pos_nx3[interp_v.reshape(-1)].reshape(-1,2,3) | |
| edges_to_interp_sdf = sdf_n[interp_v.reshape(-1)].reshape(-1,2,1) | |
| edges_to_interp_sdf[:,-1] *= -1 | |
| denominator = edges_to_interp_sdf.sum(1,keepdim = True) | |
| edges_to_interp_sdf = torch.flip(edges_to_interp_sdf, [1])/denominator | |
| verts = (edges_to_interp * edges_to_interp_sdf).sum(1) | |
| idx_map = idx_map.reshape(-1,6) | |
| v_id = torch.pow(2, torch.arange(4, dtype=torch.long, device="cuda")) | |
| tetindex = (occ_fx4[valid_tets] * v_id.unsqueeze(0)).sum(-1) | |
| num_triangles = self.num_triangles_table[tetindex] | |
| # Generate triangle indices | |
| faces = torch.cat(( | |
| torch.gather(input=idx_map[num_triangles == 1], dim=1, index=self.triangle_table[tetindex[num_triangles == 1]][:, :3]).reshape(-1,3), | |
| torch.gather(input=idx_map[num_triangles == 2], dim=1, index=self.triangle_table[tetindex[num_triangles == 2]][:, :6]).reshape(-1,3), | |
| ), dim=0) | |
| # Get global face index (static, does not depend on topology) | |
| num_tets = tet_fx4.shape[0] | |
| tet_gidx = torch.arange(num_tets, dtype=torch.long, device="cuda")[valid_tets] | |
| face_gidx = torch.cat(( | |
| tet_gidx[num_triangles == 1]*2, | |
| torch.stack((tet_gidx[num_triangles == 2]*2, tet_gidx[num_triangles == 2]*2 + 1), dim=-1).view(-1) | |
| ), dim=0) | |
| uvs, uv_idx = self.map_uv(faces, face_gidx, num_tets*2) | |
| face_to_valid_tet = torch.cat(( | |
| tet_gidx[num_triangles == 1], | |
| torch.stack((tet_gidx[num_triangles == 2], tet_gidx[num_triangles == 2]), dim=-1).view(-1) | |
| ), dim=0) | |
| valid_vert_idx = tet_fx4[tet_gidx[num_triangles > 0]].long().unique() | |
| return verts, faces, uvs, uv_idx, face_to_valid_tet.long(), valid_vert_idx | |
| def test_pickle(pkl_fn): | |
| pkl_data = pickle.load(open(pkl_fn, "rb"), encoding='latin1') | |
| # encoding='latin1' | |
| print(pkl_data.keys()) | |
| for k in pkl_data: | |
| print(f"key: {k}, value: {pkl_data[k].shape}") | |
| def load_data_fr_th_sv(th_sv_fn, grid_res=64): | |
| # /home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/dmt_dict_00001.pt | |
| # th_sv_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/dmt_dict_00001.pt" | |
| th_data = torch.load(th_sv_fn, map_location="cpu") # map location # | |
| # repo # th_data # | |
| sdf = th_data["sdf"] | |
| deform = th_data["deform"] | |
| deform_unmasked = th_data["deform_unmasked"] | |
| root = "/home/xueyi/sim/MeshDiffusion/nvdiffrec" # get the root path # | |
| # grid_res = 64 | |
| # grid_res of tets to be loaded # | |
| tets = np.load(os.path.join(root, 'data/tets/{}_tets_cropped.npz'.format(grid_res))) | |
| tet_verts = tets['vertices'] # tet _verts -> pose | |
| tet_indices = tets['indices'] # indices | |
| # sdf for each grids # | |
| print(f"tet_verts: {tet_verts.shape}, tet_indices: {tet_indices.shape}") | |
| dmt_net = DMTet() | |
| # grid_res = 64 | |
| # 1) deform but not deformed -> so tet_verts + deform as the deformed pos # | |
| # 2) deform but not | |
| tet_verts = torch.from_numpy(tet_verts).float() # nn_verts x 3 | |
| # sdf = torch.from_numpy(sdf).float() # nn_verts --> the size of nn_verts # | |
| tet_indices = torch.from_numpy(tet_indices).long() # nn_tets x 4 # | |
| # __call__(self, pos_nx3, sdf_n, tet_fx4) | |
| deform = deform.float() | |
| deformed_verts = tet_verts + deform | |
| # deformed_verts = deform | |
| print(deform_unmasked) | |
| deformed_verts[deform_unmasked.bool()] = tet_verts[deform_unmasked.bool()] | |
| # verts, faces, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(tet_verts.cuda(), sdf.cuda(), tet_indices.cuda()) | |
| verts, faces, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(deformed_verts.cuda(), sdf.cuda(), tet_indices.cuda()) | |
| print(f"verts: {verts.size()}, faces: {faces.size()}") | |
| sv_mesh_dict = { | |
| 'verts': verts.detach().cpu().numpy(), | |
| 'faces': faces.detach().cpu().numpy(), # deformed_verts # | |
| } | |
| # sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/mesh_extracted_00000.npy" | |
| sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/mesh_extracted_00000_res_128.npy" | |
| sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results_seq/tets/mesh_extracted_00002_res_128.npy" | |
| np.save(sv_mesh_fn, sv_mesh_dict) | |
| print(f"extracted mesh saved to {sv_mesh_fn}") | |
| # self.verts = torch.tensor(tets['vertices'], dtype=torch.float32, device='cuda') # * scale | |
| # self.indices = torch.tensor(tets['indices'], dtype=torch.long, device='cuda') # | |
| # | |
| # print() | |
| def load_data_fr_th_sv_fr_pred(th_sv_fn, grid_res=64): | |
| # /home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/dmt_dict_00001.pt | |
| # th_sv_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/dmt_dict_00001.pt" | |
| # th_data = torch.load(th_sv_fn, map_location="cpu") # map location # | |
| # repo # th_data # | |
| # cur_step_sv_dict = { | |
| # "obj_sdf_inputs": sdf_obj.detach().cpu().numpy(), | |
| # "hand_sdf_inputs": sdf_hand.detach().cpu().numpy(), | |
| # "obj_sdf_outputs": sdf.squeeze(-1).detach().cpu().numpy(), | |
| # "hand_sdf_outputs": sdf_obj2.squeeze(-1).detach().cpu().numpy(), | |
| # } | |
| cur_step_data = np.load(th_sv_fn, allow_pickle=True).item() | |
| obj_sdf_inputs = cur_step_data["obj_sdf_inputs"] | |
| obj_sdf_outputs = cur_step_data["obj_sdf_outputs"] | |
| hand_sdf_inputs = cur_step_data["hand_sdf_inputs"] | |
| hand_sdf_outputs = cur_step_data["hand_sdf_outputs"] | |
| # sdf = th_data["sdf"] | |
| # deform = th_data["deform"] | |
| # deform_unmasked = th_data["deform_unmasked"] | |
| root = "/home/xueyi/sim/MeshDiffusion/nvdiffrec" # get the root path # | |
| # grid_res = 64 | |
| # grid_res of tets to be loaded # | |
| tets = np.load(os.path.join(root, 'data/tets/{}_tets_cropped.npz'.format(grid_res))) | |
| tet_verts = tets['vertices'] # tet _verts -> pose | |
| tet_indices = tets['indices'] # indices | |
| # sdf for each grids # | |
| print(f"tet_verts: {tet_verts.shape}, tet_indices: {tet_indices.shape}") | |
| dmt_net = DMTet() | |
| # grid_res = 64 | |
| # 1) deform but not deformed -> so tet_verts + deform as the deformed pos # | |
| # 2) deform but not | |
| tet_verts = torch.from_numpy(tet_verts).float() # nn_verts x 3 | |
| # sdf = torch.from_numpy(sdf).float() # nn_verts --> the size of nn_verts # | |
| tet_indices = torch.from_numpy(tet_indices).long() # nn_tets x 4 # | |
| # __call__(self, pos_nx3, sdf_n, tet_fx4) | |
| obj_sdf_inputs = torch.from_numpy(obj_sdf_inputs).float().squeeze(0) | |
| obj_sdf_outputs = torch.from_numpy(obj_sdf_outputs).float().squeeze(0) | |
| hand_sdf_inputs = torch.from_numpy(hand_sdf_inputs).float().squeeze(0) | |
| hand_sdf_outputs = torch.from_numpy(hand_sdf_outputs).float().squeeze(0) | |
| # print() | |
| print(f"hand_sdf_inputs: {hand_sdf_inputs.size()}, hand_sdf_outputs: {hand_sdf_outputs.size()}") | |
| hand_verts_inputs, hand_faces_inputs, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(tet_verts.cuda(), hand_sdf_inputs.cuda(), tet_indices.cuda()) | |
| print(f"verts: {hand_verts_inputs.size()}, faces: {hand_faces_inputs.size()}") | |
| hand_verts_outputs, hand_faces_outputs, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(tet_verts.cuda(), hand_sdf_outputs.cuda(), tet_indices.cuda()) | |
| print(f"hand_verts_outputs: {hand_verts_outputs.size()}, hand_faces_outputs: {hand_faces_outputs.size()}") | |
| obj_verts_inputs, obj_faces_inputs, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(tet_verts.cuda(), obj_sdf_inputs.cuda(), tet_indices.cuda()) | |
| print(f"verts: {obj_verts_inputs.size()}, faces: {obj_faces_inputs.size()}") | |
| obj_verts_outputs, obj_faces_outputs, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(tet_verts.cuda(), obj_sdf_outputs.cuda(), tet_indices.cuda()) | |
| print(f"obj_verts_outputs: {obj_verts_outputs.size()}, obj_faces_outputs: {obj_faces_outputs.size()}") | |
| sv_mesh_dict = { | |
| 'obj_input_verts': obj_verts_inputs.detach().cpu().numpy(), | |
| 'obj_input_faces': obj_faces_inputs.detach().cpu().numpy(), # deformed_verts # | |
| 'obj_verts_outputs': obj_verts_outputs.detach().cpu().numpy(), | |
| 'obj_faces_outputs': obj_faces_outputs.detach().cpu().numpy(), # deformed_verts # | |
| 'hand_verts_inputs': hand_verts_inputs.detach().cpu().numpy(), | |
| 'hand_faces_inputs': hand_faces_inputs.detach().cpu().numpy(), # deformed_verts # | |
| 'hand_verts_outputs': hand_verts_outputs.detach().cpu().numpy(), | |
| 'hand_faces_outputs': hand_faces_outputs.detach().cpu().numpy(), # deformed_verts # | |
| } | |
| # sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/mesh_extracted_00000.npy" | |
| # sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/mesh_extracted_00000_res_128.npy" | |
| logging_dir = "/data2/sim/implicit_ae/logging/00041-stylegan2-rendering-gpus1-batch4-gamma80" | |
| # sv_mesh_fn = os.path.join(logging_dir, "pred_out_iter_2_batch_5_extracted.npy") | |
| # sv_mesh_fn = os.path.join(logging_dir, "pred_out_iter_3_batch_0_extracted.npy") | |
| sv_mesh_fn = os.path.join(logging_dir, "pred_out_iter_56_batch_0_extracted.npy") | |
| # iter_39_batch_0_nreg | |
| sv_mesh_fn = os.path.join(logging_dir, "pred_out_iter_63_batch_0_nreg_extracted.npy") # | |
| np.save(sv_mesh_fn, sv_mesh_dict) | |
| print(f"extracted mesh saved to {sv_mesh_fn}") | |
| # deform = deform.float() | |
| # deformed_verts = tet_verts + deform | |
| # # deformed_verts = deform | |
| # print(deform_unmasked) | |
| # deformed_verts[deform_unmasked.bool()] = tet_verts[deform_unmasked.bool()] | |
| # verts, faces, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(tet_verts.cuda(), sdf.cuda(), tet_indices.cuda()) | |
| # verts, faces, uvs, uv_idx, face_to_valid_tet, valid_vert_idx = dmt_net(deformed_verts.cuda(), sdf.cuda(), tet_indices.cuda()) | |
| # print(f"verts: {verts.size()}, faces: {faces.size()}") | |
| # sv_mesh_dict = { | |
| # 'verts': verts.detach().cpu().numpy(), | |
| # 'faces': faces.detach().cpu().numpy(), # deformed_verts # | |
| # } | |
| # # sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/mesh_extracted_00000.npy" | |
| # sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/mesh_extracted_00000_res_128.npy" | |
| # sv_mesh_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results_seq/tets/mesh_extracted_00002_res_128.npy" | |
| # np.save(sv_mesh_fn, sv_mesh_dict) | |
| # print(f"extracted mesh saved to {sv_mesh_fn}") | |
| # self.verts = torch.tensor(tets['vertices'], dtype=torch.float32, device='cuda') # * scale | |
| # self.indices = torch.tensor(tets['indices'], dtype=torch.long, device='cuda') # | |
| # | |
| # print() | |
| import trimesh | |
| from open3d import io as o3dio | |
| def load_ply_data(ply_fn, rt_normals=False): | |
| # obj_mesh = o3dio.read_triangle_mesh(ply_fn) | |
| # obj_verts = np.array(obj_mesh.vertices, dtype=np.float32) | |
| # obj_faces = np.array(obj_mesh.triangles) | |
| # # obj_vertex_normals = np.array(obj_mesh.vertex_normals) | |
| # # obj_face_normals = np.array(obj_mesh.face_normals) | |
| obj_mesh = trimesh.load(ply_fn, process=False) | |
| # obj_mesh.remove_degenerate_faces(height=1e-06) | |
| verts_obj = np.array(obj_mesh.vertices) | |
| faces_obj = np.array(obj_mesh.faces) | |
| obj_face_normals = np.array(obj_mesh.face_normals) | |
| obj_vertex_normals = np.array(obj_mesh.vertex_normals) | |
| print(f"vertex: {verts_obj.shape}, obj_faces: {faces_obj.shape}, obj_face_normals: {obj_face_normals.shape}, obj_vertex_normals: {obj_vertex_normals.shape}") | |
| if not rt_normals: | |
| return verts_obj, faces_obj | |
| else: | |
| return verts_obj, faces_obj, obj_vertex_normals | |
| def load_and_save_verts(rt_path): | |
| ws = 60 | |
| tot_obj_verts =[] | |
| for i_fr in range(ws): | |
| cur_fr_obj_nm = f"object_{i_fr}.obj" | |
| cur_fr_obj_path = os.path.join(rt_path, cur_fr_obj_nm) | |
| cur_obj_verts, cur_obj_faces = load_ply_data(cur_fr_obj_path) | |
| tot_obj_verts.append(cur_obj_verts) | |
| tot_obj_verts = np.stack(tot_obj_verts, axis=0) # ws x nn_obj_verts x 3 -> for obj verts here # | |
| bundle_obj_verts_sv_fn = os.path.join(rt_path, f"obj_verts_ws_{ws}.npy") | |
| np.save(bundle_obj_verts_sv_fn, tot_obj_verts) | |
| print(f"Object vertices saved to {bundle_obj_verts_sv_fn}") | |
| def get_penetration_depth_rnk_data(sv_dict_fn): | |
| sv_dict = np.load(sv_dict_fn, allow_pickle=True).item() | |
| pred_fn_to_APD = {} | |
| for cur_fn in sv_dict: | |
| pred_fn_to_APD[cur_fn] = sv_dict[cur_fn][0] | |
| sorted_pred_fn_with_APD = sorted(pred_fn_to_APD.items(), key=lambda i: i[1]) | |
| # print(sorted_pred_fn_with_APD) | |
| # predicted_infos_seq_300_seed_169_tag_rep_res_jts_hoi4d_scissors_t_300_.npy | |
| sorted_seed_APD_pair = [] | |
| selected_seeds = [] | |
| for cur_item in sorted_pred_fn_with_APD: | |
| cur_fn = cur_item[0] | |
| cur_seed =cur_fn.split("/")[-1].split("_")[5] | |
| cur_seed = int(cur_seed) | |
| sorted_seed_APD_pair.append((cur_seed, cur_item[1])) | |
| if cur_item[1] == 0.: | |
| selected_seeds.append(cur_seed) | |
| print(sorted_seed_APD_pair) | |
| print(f"selected_seeds:") | |
| print(selected_seeds) | |
| # case_flag | |
| def get_meta_info(sv_dict_fn): | |
| meta_info = np.load(sv_dict_fn, allow_pickle=True).item() | |
| # ZY20210800001/H1/C9/N12/S33/s01/T2 | |
| case_flag = meta_info["case_flag"] | |
| print(case_flag) | |
| return case_flag | |
| # average on all hand vertices # | |
| ### not a very accurate metric for penetration here ### | |
| def calculate_penetration_depth(subj_seq, obj_verts, obj_faces): # subj seq -> can be any vertices as well # | |
| # obj_verts: nn_verts x 3 -> numpy array | |
| # obj_faces: nn_faces x 3 -> numpy array | |
| # obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| # process=False, use_embree=True) | |
| obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| ) | |
| # subj_seq: nf x nn_subj_pts x 3 # | |
| tot_penetration_depth = [] | |
| for i_f in range(subj_seq.shape[0]): ## total sequence length ## | |
| # for i_f in range(10): | |
| cur_subj_seq = subj_seq[i_f] | |
| cur_subj_seq_in_obj = obj_mesh.contains(cur_subj_seq) # nn_subj_pts # | |
| dist_cur_subj_to_obj_verts = np.sum( # nn_subj_pts x nn_obj_pts # | |
| (np.reshape(cur_subj_seq, (cur_subj_seq.shape[0], 1, 3)) - np.reshape(obj_verts, (1, obj_verts.shape[0], 3))) ** 2, axis=-1 | |
| ) | |
| # dist_cur_subj_to_obj_verts | |
| nearest_obj_dist = np.min(dist_cur_subj_to_obj_verts, axis=-1) # nn_subj_pts | |
| nearest_obj_dist = np.sqrt(nearest_obj_dist) | |
| cur_pene_depth = np.zeros_like(nearest_obj_dist) | |
| cur_pene_depth[cur_subj_seq_in_obj] = nearest_obj_dist[cur_subj_seq_in_obj] | |
| tot_penetration_depth.append(cur_pene_depth) | |
| tot_penetration_depth = np.stack(tot_penetration_depth, axis=0) # nf x nn_subj_pts | |
| tot_penetration_depth = np.mean(tot_penetration_depth).item() | |
| return tot_penetration_depth | |
| ### not a very accurate metric for penetration here ### | |
| def calculate_penetration_depth_obj_seq(subj_seq, tot_obj_verts, tot_frames_obj_normals, obj_faces): # subj seq -> can be any vertices as well # | |
| # obj_verts: nn_verts x 3 -> numpy array | |
| # obj_faces: nn_faces x 3 -> numpy array | |
| # obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| # process=False, use_embree=True) | |
| # obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| # ) | |
| # subj_seq: nf x nn_subj_pts x 3 # | |
| tot_penetration_depth = [] | |
| for i_f in range(subj_seq.shape[0]): ## total sequence length ## | |
| obj_verts = tot_obj_verts[i_f] | |
| obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| ) | |
| cur_subj_seq = subj_seq[i_f] | |
| cur_subj_seq_in_obj = obj_mesh.contains(cur_subj_seq) # nn_subj_pts # | |
| dist_cur_subj_to_obj_verts = np.sum( # nn_subj_pts x nn_obj_pts # | |
| (np.reshape(cur_subj_seq, (cur_subj_seq.shape[0], 1, 3)) - np.reshape(obj_verts, (1, obj_verts.shape[0], 3))) ** 2, axis=-1 | |
| ) | |
| # dist_cur_subj_to_obj_verts | |
| nearest_obj_dist = np.min(dist_cur_subj_to_obj_verts, axis=-1) # nn_subj_pts | |
| nearest_obj_dist = np.sqrt(nearest_obj_dist) | |
| cur_pene_depth = np.zeros_like(nearest_obj_dist) | |
| cur_pene_depth[cur_subj_seq_in_obj] = nearest_obj_dist[cur_subj_seq_in_obj] | |
| tot_penetration_depth.append(cur_pene_depth) | |
| tot_penetration_depth = np.stack(tot_penetration_depth, axis=0) # nf x nn_subj_pts | |
| tot_penetration_depth = np.mean(tot_penetration_depth).item() | |
| return tot_penetration_depth | |
| ### not a very accurate metric for penetration here ### | |
| def calculate_penetration_depth_obj_seq_v2(subj_seq, tot_obj_verts, tot_frames_obj_normals, obj_faces): # subj seq -> can be any vertices as well # | |
| # obj_verts: nn_verts x 3 -> numpy array | |
| # obj_faces: nn_faces x 3 -> numpy array | |
| # obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| # process=False, use_embree=True) | |
| # obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| # ) | |
| # subj_seq: nf x nn_subj_pts x 3 # | |
| tot_penetration_depth = [] | |
| for i_f in range(subj_seq.shape[0]): ## total sequence length ## | |
| obj_verts = tot_obj_verts[i_f] | |
| obj_normals = tot_frames_obj_normals[i_f] | |
| # obj_mesh = trimesh.Trimesh(vertices=obj_verts, faces=obj_faces, | |
| # ) | |
| cur_subj_seq = subj_seq[i_f] | |
| # cur_subj_seq_in_obj = obj_mesh.contains(cur_subj_seq) # nn_subj_pts # | |
| dist_cur_subj_to_obj_verts = np.sum( # nn_subj_pts x nn_obj_pts # | |
| (np.reshape(cur_subj_seq, (cur_subj_seq.shape[0], 1, 3)) - np.reshape(obj_verts, (1, obj_verts.shape[0], 3))) ** 2, axis=-1 | |
| ) | |
| # dist_cur_subj_to_obj_verts | |
| nearest_obj_pts_idx = np.argmin(dist_cur_subj_to_obj_verts, axis=-1) # nn_subj_pts # | |
| nearest_obj_dist = np.min(dist_cur_subj_to_obj_verts, axis=-1) # nn_subj_pts | |
| nearest_obj_dist = np.sqrt(nearest_obj_dist) | |
| cur_pene_depth = np.zeros_like(nearest_obj_dist) | |
| nearest_obj_pts_idx_th = torch.from_numpy(nearest_obj_pts_idx).long().cuda() ### | |
| obj_verts_th = torch.from_numpy(obj_verts).float().cuda() ### nn_obj_verts x 3 | |
| obj_normals_th = torch.from_numpy(obj_normals).float().cuda() ### nn_obj_verts x 3 | |
| ## nn_hand_verts x 3 ## | |
| nearest_obj_pts = model_util.batched_index_select_ours(obj_verts_th, indices=nearest_obj_pts_idx_th, dim=0) ## nn_hand_verts x 3 | |
| nearest_obj_normals = model_util.batched_index_select_ours(obj_normals_th, indices=nearest_obj_pts_idx_th, dim=0) ## nn_hand_verts x 3 | |
| cur_subj_seq_th = torch.from_numpy(cur_subj_seq).float().cuda() | |
| rel_obj_verts_to_subj_pts = cur_subj_seq_th - nearest_obj_pts | |
| dot_rel_with_obj_normals = torch.sum(nearest_obj_normals * rel_obj_verts_to_subj_pts, dim=-1) | |
| cur_subj_seq_in_obj = dot_rel_with_obj_normals < 0. | |
| cur_subj_seq_in_obj = cur_subj_seq_in_obj.cpu().numpy().astype(np.bool8) | |
| cur_pene_depth[cur_subj_seq_in_obj] = nearest_obj_dist[cur_subj_seq_in_obj] | |
| tot_penetration_depth.append(cur_pene_depth) | |
| tot_penetration_depth = np.stack(tot_penetration_depth, axis=0) # nf x nn_subj_pts | |
| tot_penetration_depth = np.mean(tot_penetration_depth).item() | |
| return tot_penetration_depth | |
| def calculate_subj_smoothness(joint_seq): | |
| # joint_seq: nf x nnjoints x 3 | |
| disp_seq = joint_seq[1:] - joint_seq[:-1] # (nf - 1) x nnjoints x 3 # | |
| disp_seq = np.sum(disp_seq ** 2, axis=-1) | |
| disp_seq = np.mean(disp_seq) | |
| # disp_seq = np. | |
| disp_seq = disp_seq.item() | |
| return disp_seq | |
| def calculate_moving_consistency(base_pts_trans, joints_trans): | |
| # base_pts_trans: nf x nn_base_pts x 3 # | |
| # joints_trans: nf x nn_jts x 3 # | |
| base_pts_trans = torch.from_numpy(base_pts_trans).float() | |
| joints_trans = torch.from_numpy(joints_trans).float() | |
| # dist_joints_to_base_pts = np.sum | |
| dist_joints_to_base_pts = torch.sum( | |
| (joints_trans.unsqueeze(2) - base_pts_trans.unsqueeze(1)) ** 2, dim=-1 # nf x nn_jts x nn_base_pts # | |
| ) | |
| dist_joints_to_base_pts = torch.sqrt(dist_joints_to_base_pts) | |
| dist_joints_to_base_pts, joints_to_base_pts_minn_idxes = torch.min(dist_joints_to_base_pts, dim=-1) | |
| minn_dist_joints_to_base_pts_across_joints, minn_dist_joints_to_base_pts_idxes = torch.min(dist_joints_to_base_pts, dim=-1) # (nf - 1) | |
| minn_dist_joints_to_base_pts_idxes = minn_dist_joints_to_base_pts_idxes[:-1] | |
| disp_joints_to_base_pts_minn_idxes = joints_to_base_pts_minn_idxes[:-1] | |
| disp_base_pts = base_pts_trans[1:] - base_pts_trans[:-1] | |
| disp_joints = joints_trans[1:] - joints_trans[:-1] # (nf - 1) x nn_jts x 3 | |
| dist_joints_to_base_pts = dist_joints_to_base_pts[:-1] | |
| k_f = 100. | |
| k = torch.exp( | |
| -k_f * dist_joints_to_base_pts | |
| ) | |
| disp_joints_base_pts = model_util.batched_index_select_ours(disp_base_pts, indices=disp_joints_to_base_pts_minn_idxes, dim=1) # (nf - 1) x nn_jts x 3 | |
| nearest_joints_disp = model_util.batched_index_select_ours(disp_joints_base_pts, indices=minn_dist_joints_to_base_pts_idxes.unsqueeze(-1), dim=1) # (nf - 1) x 1 | |
| nearest_joints_disp = nearest_joints_disp.squeeze(1) # (nf - 1) x 3 # | |
| disp_joints = model_util.batched_index_select_ours(disp_joints, indices=minn_dist_joints_to_base_pts_idxes.unsqueeze(-1), dim=1).squeeze(1) # (nf - 1) x 3 | |
| nearest_k = model_util.batched_index_select_ours(k, indices=minn_dist_joints_to_base_pts_idxes.unsqueeze(-1), dim=1).squeeze(1) # (nf - 1) | |
| ##### use k for weighting disp ##### | |
| disp_joints_to_nearest_base_pts = disp_joints * nearest_k.unsqueeze(-1) # ### (nf - 1 ) x 3 | |
| diff_disp_joints_to_nearest_base_pts_disp = torch.sum( | |
| (disp_joints_to_nearest_base_pts - nearest_joints_disp) ** 2, dim=-1 | |
| ) | |
| diff_disp_joints_base_pts = diff_disp_joints_to_nearest_base_pts_disp.mean() | |
| ##### use k for weighting disp ##### | |
| return diff_disp_joints_base_pts.item() | |
| # i_test_seq | |
| def get_test_settings_to_statistics(i_test_seq, test_tag, start_idx=50, ws=60, use_toch=False): | |
| # optimized_sv_infos_sv_fn_nm = f"optimized_infos_sv_dict_seq_{i_test_seq}_seed_{seed}_tag_{test_tag}_dist_thres_{dist_thres}_with_proj_{with_proj}.npy" | |
| # optimized_infos_sv_dict_seq_0_seed_0_tag_jts_hoi4d_arti_t_400__dist_thres_0.005_with_proj_False.npy | |
| tot_dist_thres = [0.005, 0.01, 0.02, 0.05, 0.1] | |
| tot_dist_thres = [0.001, 0.005] | |
| tot_with_proj = [True, False] | |
| tot_seeds = range(0, 122, 11) | |
| if use_toch: | |
| tot_seeds = [0] | |
| # tot_with_proj = [False] | |
| tot_with_proj = [True, False] | |
| # tot_dist_thres = [0.005] | |
| tot_dist_thres = [0.001, 0.005] | |
| # test_tag = "" | |
| pred_infos_sv_folder = f"/data2/sim/eval_save/HOI_{cat_ty}/{cat_nm}" | |
| if cat_nm in ["Scissors"]: | |
| corr_infos_sv_folder = f"/data2/sim/HOI_Processed_Data_{cat_ty}/{cat_nm}/{cat_nm}" | |
| else: | |
| corr_infos_sv_folder = f"/data2/sim/HOI_Processed_Data_{cat_ty}/{cat_nm}" | |
| test_setting_to_pene_depth = {} | |
| corr_mesh_folder = os.path.join(corr_infos_sv_folder, f"case{i_test_seq}") | |
| corr_mesh_folder = os.path.join(corr_mesh_folder, "corr_mesh") | |
| tot_frames_obj_verts = [] | |
| tot_frames_obj_normals = [] | |
| # for i_idx in range(start_idx, start_idx + ws): | |
| # cur_obj_fn = os.path.join(corr_mesh_folder, f"object_{i_idx}.obj") | |
| # if not os.path.exists(cur_obj_fn): | |
| # return [] | |
| # cur_obj_verts, cur_obj_faces, cur_obj_verts_normals = load_ply_data(cur_obj_fn, rt_normals=True) #### load verts and faces jfrom the ply data ## | |
| # tot_frames_obj_verts.append(cur_obj_verts) | |
| # tot_frames_obj_normals.append(cur_obj_verts_normals) | |
| # tot_frames_obj_verts_np = np.stack(tot_frames_obj_verts, axis=0) | |
| for seed in tot_seeds: | |
| for dist_thres in tot_dist_thres: | |
| for with_proj in tot_with_proj: | |
| cur_optimized_sv_infos_fn = f"optimized_infos_sv_dict_seq_{i_test_seq}_seed_{seed}_tag_{test_tag}_dist_thres_{dist_thres}_with_proj_{with_proj}.npy" | |
| cur_optimized_sv_infos_fn = f"optimized_infos_sv_dict_seq_{i_test_seq}_seed_{seed}_tag_{test_tag}_dist_thres_{dist_thres}_with_proj_{with_proj}_wjts_0.01.npy" | |
| # f"optimized_infos_sv_dict_seq_{i_test_seq}_seed_{seed}_tag_{test_tag}_dist_thres_{dist_thres}_with_proj_{with_proj}.npy" | |
| cur_optimized_sv_infos_fn = os.path.join(pred_infos_sv_folder, cur_optimized_sv_infos_fn) | |
| print(f"cur_optimized_sv_infos_fn: {cur_optimized_sv_infos_fn}") | |
| if not os.path.exists(cur_optimized_sv_infos_fn): | |
| continue | |
| optimized_infos = np.load(cur_optimized_sv_infos_fn, allow_pickle=True).item() | |
| if len(tot_frames_obj_verts) == 0: | |
| for i_idx in range(start_idx, start_idx + ws): | |
| cur_obj_fn = os.path.join(corr_mesh_folder, f"object_{i_idx}.obj") | |
| if not os.path.exists(cur_obj_fn): | |
| return [] | |
| cur_obj_verts, cur_obj_faces, cur_obj_verts_normals = load_ply_data(cur_obj_fn, rt_normals=True) #### load verts and faces jfrom the ply data ## | |
| tot_frames_obj_verts.append(cur_obj_verts) | |
| tot_frames_obj_normals.append(cur_obj_verts_normals) | |
| tot_frames_obj_verts_np = np.stack(tot_frames_obj_verts, axis=0) | |
| tot_base_pts_trans = optimized_infos['tot_base_pts_trans'] # nn_frames x nn_base_pts x 3 | |
| tot_base_pts_trans_th = torch.from_numpy(tot_base_pts_trans).float() | |
| tot_frames_obj_verts_th = torch.from_numpy(tot_frames_obj_verts_np).float() | |
| diff_base_pts_trans_obj = torch.sum( | |
| (tot_base_pts_trans_th.unsqueeze(-2) - tot_frames_obj_verts_th.unsqueeze(1)) ** 2, dim=-1 # nn_frames x nn_base x nn_obj | |
| ) | |
| minn_diff_base_pts_trans_obj, minn_diff_base_pts_trans_obj_idxes = torch.min(diff_base_pts_trans_obj, dim=-1) # nn_frames x nn_base | |
| tot_frames_obj_verts_th = model_util.batched_index_select_ours(tot_frames_obj_verts_th.cuda(), minn_diff_base_pts_trans_obj_idxes.cuda(), dim=1) # nn_frames x nn_base x 3 | |
| tot_frames_obj_normals_np = np.stack(tot_frames_obj_normals, axis=0) | |
| tot_frames_obj_normals_th = torch.from_numpy(tot_frames_obj_normals_np).float() | |
| tot_frames_obj_normals_th = model_util.batched_index_select_ours(tot_frames_obj_normals_th.cuda(), minn_diff_base_pts_trans_obj_idxes.cuda(), dim=1) # nn_frames x nn_base x 3 | |
| # tot_frames_obj_normals = tot_frames_obj_normals_th.cpu().numpy().tolist() | |
| tot_frames_obj_normals = [tot_frames_obj_normals_th[ii].cpu().numpy() for ii in range(tot_frames_obj_normals_th.size(0))] | |
| tot_frames_obj_verts_np = tot_frames_obj_verts_th.cpu().numpy() | |
| # tot_frames_obj_verts = tot_frames_obj_verts_np.tolist() | |
| tot_frames_obj_verts = [tot_frames_obj_verts_np[ii] for ii in range(tot_frames_obj_verts_np.shape[0])] | |
| # if tot_frames_obj_verts | |
| optimized_verts = optimized_infos["hand_verts"] | |
| if use_toch: | |
| toch_eval_sv_fn = f"/data2/sim/eval_save/HOI_{cat_ty}_toch/{cat_nm}/{i_test_seq}.npy" | |
| toch_eval_sv_dict = np.load(toch_eval_sv_fn, allow_pickle=True).item() | |
| optimized_verts = toch_eval_sv_dict["hand_verts_tot"] | |
| ### calculate penetration depth obj seq ### | |
| # cur_penetration_depth = calculate_penetration_depth_obj_seq(optimized_verts, tot_frames_obj_verts, tot_frames_obj_normals, cur_obj_faces) | |
| # calculate_penetration_depth_obj_seq_v2 | |
| st_time = time.time() | |
| cur_penetration_depth = calculate_penetration_depth_obj_seq_v2(optimized_verts, tot_frames_obj_verts, tot_frames_obj_normals, cur_obj_faces) | |
| cur_subj_smoothness = calculate_subj_smoothness(optimized_verts) | |
| cur_moving_consistency = calculate_moving_consistency(tot_frames_obj_verts_np, optimized_verts) | |
| ed_time = time.time() | |
| print(f"Time used for calculating penetration depth (v2): {ed_time - st_time}") | |
| test_setting_to_pene_depth[(seed, dist_thres, with_proj) ] = (cur_penetration_depth, cur_subj_smoothness, cur_moving_consistency) | |
| print(f"i_test_seq: {i_test_seq}, seed: {seed}, dist_thres: {dist_thres}, with_proj: {with_proj}, penetration_depth: {cur_penetration_depth}, smoothness: {cur_subj_smoothness}, cur_moving_consistency: {cur_moving_consistency}") | |
| sorted_setting_to_pene_depth = sorted(test_setting_to_pene_depth.items(), key=lambda ii: ii[1][0], reverse=False) | |
| print(sorted_setting_to_pene_depth[:5]) | |
| return sorted_setting_to_pene_depth | |
| def get_setting_to_stats(st_idx, ed_idx, use_toch=False): | |
| # f"/data2/sim/eval_save/HOI_Arti/Scissors/setting_to_stats_seq_{i_test_seq}_toch.npy" | |
| tot_penetrations = [] | |
| tot_smoothness = [] | |
| tot_moving_consistency = [] | |
| for test_idx in range(st_idx, ed_idx): | |
| if test_idx == 12: | |
| continue | |
| if not use_toch: | |
| cur_sv_dict_fn = f"/data2/sim/eval_save/HOI_{cat_ty}/{cat_nm}/setting_to_stats_seq_{test_idx}.npy" | |
| else: | |
| cur_sv_dict_fn = f"/data2/sim/eval_save/HOI_{cat_ty}/{cat_nm}/setting_to_stats_seq_{test_idx}_toch.npy" | |
| sv_dict = np.load(cur_sv_dict_fn, allow_pickle=True) | |
| # print(f"Test idx: {test_idx}, statistics: {sv_dict}") | |
| # cur_sv_dict_fn = f"/data2/sim/eval_save/HOI_Arti/Scissors/setting_to_stats_seq_{test_idx}_toch.npy" | |
| # sv_dict = np.load(cur_sv_dict_fn, allow_pickle=True) | |
| # print(f"Test idx: {test_idx}, statistics: {sv_dict}") | |
| if len(sv_dict) > 0: | |
| print(sv_dict[0]) | |
| cur_stats = sv_dict[0][1] | |
| tot_penetrations.append(cur_stats[0]) | |
| tot_smoothness.append(cur_stats[1]) | |
| tot_moving_consistency.append(cur_stats[2]) | |
| # cur_sv_dict_fn = f"/data2/sim/eval_save/HOI_Arti/Scissors/setting_to_stats_seq_{test_idx}_toch.npy" | |
| # sv_dict = np.load(cur_sv_dict_fn, allow_pickle=True) | |
| # print(f"Test idx: {test_idx}, statistics: {sv_dict}") | |
| # the | |
| avg_penetration = sum(tot_penetrations) / float(len(tot_penetrations)) | |
| avg_smoothness = sum(tot_smoothness) / float(len(tot_smoothness)) | |
| avg_consis = sum(tot_moving_consistency) / float(len(tot_moving_consistency)) | |
| print(f"avg_penetration: {avg_penetration}, avg_smoothness: {avg_smoothness}, avg_consis: {avg_consis}") | |
| # [8, 11] | |
| # avg_penetration: 7.019575241429266e-05, avg_smoothness: 2.841970498934643e-06, avg_consis: 4.051069936394924e-06 | |
| # avg_penetration: 4.745464866573457e-05, avg_smoothness: 8.250485916505568e-05, avg_consis: 4.235470646563044e-05 | |
| # Toch | |
| # avg_penetration: 0.00011692142591831119, avg_smoothness: 6.375887634627968e-05, avg_consis: 3.023650837500706e-05 | |
| # Ours | |
| # avg_penetration: 5.615660193143412e-05, avg_smoothness: 3.3161883834509354e-06, avg_consis: 3.632244261098094e-06 | |
| from manopth.manolayer import ManoLayer | |
| def get_mano_model(): | |
| mano_path = "/data1/sim/mano_models/mano/models" | |
| mano_layer = ManoLayer( | |
| flat_hand_mean=True, | |
| side='right', | |
| mano_root=mano_path, # mano_root # | |
| ncomps=24, | |
| use_pca=True, | |
| root_rot_mode='axisang', | |
| joint_rot_mode='axisang' | |
| ) | |
| return mano_layer | |
| def get_arctic_mano_model(side='right'): | |
| mano_path = "/data1/sim/mano_models/mano/models" | |
| mano_layer = ManoLayer( | |
| flat_hand_mean=False, | |
| side=side, | |
| mano_root=mano_path, # mano_root # | |
| ncomps=45, | |
| use_pca=False, | |
| root_rot_mode='axisang', | |
| joint_rot_mode='axisang' | |
| ) | |
| return mano_layer | |
| # load clean obj clip data # | |
| import data_loaders.humanml.data.utils as utils | |
| # | |
| def load_grab_clip_data_clean_obj(clip_seq_idx, more_pert=False, other_noise=False, split='train'): | |
| mano_model = get_mano_model() | |
| grab_path = "/data1/sim/GRAB_extracted" # extracted # extracted | |
| # split = "test" | |
| window_size = 60 | |
| singe_seq_path = f"/data1/sim/GRAB_processed/{split}/{clip_seq_idx}.npy" | |
| clip_clean = np.load(singe_seq_path) | |
| subj_root_path = '/data1/sim/GRAB_processed_wsubj' | |
| subj_seq_path = f"{clip_seq_idx}_subj.npy" | |
| # load datas # grab path; grab sequences # | |
| grab_path = "/data1/sim/GRAB_extracted" | |
| obj_mesh_path = os.path.join(grab_path, 'tools/object_meshes/contact_meshes') | |
| id2objmesh = [] | |
| obj_meshes = sorted(os.listdir(obj_mesh_path)) | |
| for i, fn in enumerate(obj_meshes): | |
| id2objmesh.append(os.path.join(obj_mesh_path, fn)) | |
| subj_params_fn = os.path.join(subj_root_path, split, subj_seq_path) | |
| subj_params = np.load(subj_params_fn, allow_pickle=True).item() | |
| rhand_transl = subj_params["rhand_transl"][:window_size].astype(np.float32) | |
| rhand_betas = subj_params["rhand_betas"].astype(np.float32) | |
| rhand_pose = clip_clean['f2'][:window_size].astype(np.float32) ## rhand pose ## # | |
| rhand_global_orient = clip_clean['f1'][:window_size].astype(np.float32) | |
| # rhand_pose = clip_clean['f2'][:window_size].astype(np.float32) | |
| object_global_orient = clip_clean['f5'].astype(np.float32) ## clip_len x 3 --> orientation | |
| object_trcansl = clip_clean['f6'].astype(np.float32) ## cliplen x 3 --> translation | |
| object_idx = clip_clean['f7'][0].item() # clip len x 3 # clip len x 3 for translations # | |
| object_global_orient_mtx = utils.batched_get_orientation_matrices(object_global_orient) | |
| object_global_orient_mtx_th = torch.from_numpy(object_global_orient_mtx).float() | |
| object_trcansl_th = torch.from_numpy(object_trcansl).float() | |
| obj_nm = id2objmesh[object_idx] | |
| obj_mesh = trimesh.load(obj_nm, process=False) # obj mesh obj verts | |
| obj_verts = np.array(obj_mesh.vertices) | |
| obj_vertex_normals = np.array(obj_mesh.vertex_normals) | |
| obj_faces = np.array(obj_mesh.faces) | |
| obj_verts = torch.from_numpy(obj_verts).float() # | |
| obj_verts = torch.matmul(obj_verts.unsqueeze(0), object_global_orient_mtx_th) + object_trcansl_th.unsqueeze(1) ### nn_frames x nn_obj x 3 ### as the object transformed meshes ## | |
| obj_verts = obj_verts.detach().cpu().numpy() ### nn_frames x nn_obj_verts x 3 ### | |
| return obj_verts, obj_faces | |
| # | |
| def load_grab_clip_data_clean_subj(clip_seq_idx, split = "train", pert=False, more_pert=False, other_noise=False): | |
| mano_model = get_mano_model() | |
| grab_path = "/data1/sim/GRAB_extracted" # extracted | |
| # split = "test" | |
| window_size = 60 | |
| singe_seq_path = f"/data1/sim/GRAB_processed/{split}/{clip_seq_idx}.npy" | |
| clip_clean = np.load(singe_seq_path) | |
| subj_root_path = '/data1/sim/GRAB_processed_wsubj' | |
| subj_seq_path = f"{clip_seq_idx}_subj.npy" | |
| subj_params_fn = os.path.join(subj_root_path, split, subj_seq_path) | |
| subj_params = np.load(subj_params_fn, allow_pickle=True).item() | |
| rhand_transl = subj_params["rhand_transl"][:window_size].astype(np.float32) | |
| rhand_betas = subj_params["rhand_betas"].astype(np.float32) | |
| rhand_pose = clip_clean['f2'][:window_size].astype(np.float32) ## rhand pose ## # | |
| rhand_global_orient = clip_clean['f1'][:window_size].astype(np.float32) | |
| # rhand_pose = clip_clean['f2'][:window_size].astype(np.float32) | |
| rhand_global_orient_var = torch.from_numpy(rhand_global_orient).float() | |
| rhand_pose_var = torch.from_numpy(rhand_pose).float() | |
| rhand_beta_var = torch.from_numpy(rhand_betas).float() | |
| rhand_transl_var = torch.from_numpy(rhand_transl).float() | |
| aug_trans, aug_rot, aug_pose = 0.01, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.01, 0.1, 0.5 | |
| # aug_trans, aug_rot, aug_pose = 0.001, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.000, 0.05, 0.3 | |
| # | |
| if pert: | |
| if more_pert: | |
| aug_trans, aug_rot, aug_pose = 0.04, 0.2, 0.8 | |
| if other_noise: | |
| dist_beta = torch.distributions.beta.Beta(torch.tensor([8.]), torch.tensor([2.])) | |
| print(f"here!") | |
| aug_pose_var = dist_beta.sample(rhand_pose_var.size()).squeeze(-1) * aug_pose | |
| aug_global_orient_var = dist_beta.sample(rhand_global_orient_var.size()).squeeze(-1) * aug_rot | |
| print(f"aug_pose_var: {aug_pose_var.size()}, aug_global_orient_var: {aug_global_orient_var.size()}") | |
| aug_transl_var = dist_beta.sample(rhand_transl_var.size()).squeeze(-1) * aug_trans | |
| else: | |
| aug_global_orient_var = torch.randn_like(rhand_global_orient_var) * aug_rot ### sigma = aug_rot | |
| aug_pose_var = torch.randn_like(rhand_pose_var) * aug_pose ### sigma = aug_pose | |
| aug_transl_var = torch.randn_like(rhand_transl_var) * aug_trans ### sigma = aug_trans | |
| rnd_aug_global_orient_var = rhand_global_orient_var + aug_global_orient_var | |
| rnd_aug_pose_var = rhand_pose_var + aug_pose_var | |
| rnd_aug_transl_var = rhand_transl_var + aug_transl_var ### aug transl ### | |
| else: | |
| rnd_aug_global_orient_var = rhand_global_orient_var | |
| rnd_aug_pose_var = rhand_pose_var | |
| rnd_aug_transl_var = rhand_transl_var ### aug transl | |
| rhand_verts, rhand_joints = mano_model( | |
| torch.cat([rnd_aug_global_orient_var, rnd_aug_pose_var], dim=-1), | |
| rhand_beta_var.unsqueeze(0).repeat(window_size, 1).view(-1, 10), rnd_aug_transl_var | |
| ) | |
| ### rhand_joints: for joints ### | |
| rhand_verts = rhand_verts * 0.001 | |
| rhand_joints = rhand_joints * 0.001 | |
| return rhand_verts | |
| def load_grab_clip_data(clip_seq_idx, more_pert=False, other_noise=False): | |
| mano_model = get_mano_model() | |
| grab_path = "/data1/sim/GRAB_extracted" # extracted | |
| split = "test" | |
| window_size = 60 | |
| singe_seq_path = f"/data1/sim/GRAB_processed/{split}/{clip_seq_idx}.npy" | |
| clip_clean = np.load(singe_seq_path) | |
| subj_root_path = '/data1/sim/GRAB_processed_wsubj' | |
| subj_seq_path = f"{clip_seq_idx}_subj.npy" | |
| subj_params_fn = os.path.join(subj_root_path, split, subj_seq_path) | |
| subj_params = np.load(subj_params_fn, allow_pickle=True).item() | |
| rhand_transl = subj_params["rhand_transl"][:window_size].astype(np.float32) | |
| rhand_betas = subj_params["rhand_betas"].astype(np.float32) | |
| rhand_pose = clip_clean['f2'][:window_size].astype(np.float32) ## rhand pose ## # | |
| rhand_global_orient = clip_clean['f1'][:window_size].astype(np.float32) | |
| # rhand_pose = clip_clean['f2'][:window_size].astype(np.float32) | |
| rhand_global_orient_var = torch.from_numpy(rhand_global_orient).float() | |
| rhand_pose_var = torch.from_numpy(rhand_pose).float() | |
| rhand_beta_var = torch.from_numpy(rhand_betas).float() | |
| rhand_transl_var = torch.from_numpy(rhand_transl).float() | |
| aug_trans, aug_rot, aug_pose =0.01, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.01, 0.1, 0.5 | |
| # aug_trans, aug_rot, aug_pose = 0.001, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.000, 0.05, 0.3 | |
| if more_pert: | |
| aug_trans, aug_rot, aug_pose = 0.04, 0.2, 0.8 | |
| if other_noise: | |
| dist_beta = torch.distributions.beta.Beta(torch.tensor([8.]), torch.tensor([2.])) | |
| print(f"here!") | |
| aug_pose_var = dist_beta.sample(rhand_pose_var.size()).squeeze(-1) * aug_pose | |
| aug_global_orient_var = dist_beta.sample(rhand_global_orient_var.size()).squeeze(-1) * aug_rot | |
| print(f"aug_pose_var: {aug_pose_var.size()}, aug_global_orient_var: {aug_global_orient_var.size()}") | |
| aug_transl_var = dist_beta.sample(rhand_transl_var.size()).squeeze(-1) * aug_trans | |
| else: | |
| aug_global_orient_var = torch.randn_like(rhand_global_orient_var) * aug_rot ### sigma = aug_rot | |
| aug_pose_var = torch.randn_like(rhand_pose_var) * aug_pose ### sigma = aug_pose | |
| aug_transl_var = torch.randn_like(rhand_transl_var) * aug_trans ### sigma = aug_trans | |
| rnd_aug_global_orient_var = rhand_global_orient_var + aug_global_orient_var | |
| rnd_aug_pose_var = rhand_pose_var + aug_pose_var | |
| rnd_aug_transl_var = rhand_transl_var + aug_transl_var ### aug transl | |
| rhand_verts, rhand_joints = mano_model( | |
| torch.cat([rnd_aug_global_orient_var, rnd_aug_pose_var], dim=-1), | |
| rhand_beta_var.unsqueeze(0).repeat(window_size, 1).view(-1, 10), rnd_aug_transl_var | |
| ) | |
| ### rhand_joints: for joints ### | |
| rhand_verts = rhand_verts * 0.001 | |
| rhand_joints = rhand_joints * 0.001 | |
| return rhand_verts | |
| def get_idx_to_objname(): | |
| # load datas # grab path; grab sequences # | |
| grab_path = "/data1/sim/GRAB_extracted" | |
| ## grab contactmesh ## id2objmeshname | |
| obj_mesh_path = os.path.join(grab_path, 'tools/object_meshes/contact_meshes') | |
| id2objmeshname = [] | |
| obj_meshes = sorted(os.listdir(obj_mesh_path)) | |
| # objectmesh name # | |
| id2objmeshname = [obj_meshes[i].split(".")[0] for i in range(len(obj_meshes))] | |
| return id2objmeshname | |
| def get_test_idx_to_obj_name(): | |
| id2objmeshname = get_idx_to_objname() | |
| test_folder = "/data1/sim/GRAB_processed/test/" | |
| tot_test_seqs = os.listdir(test_folder) | |
| tot_test_seq_idxes = [ii for ii in range(246)] | |
| test_seq_idx_to_mesh_nm = {} | |
| for cur_test_seq_idx in tot_test_seq_idxes: | |
| cur_test_seq_path = os.path.join(test_folder, f"{cur_test_seq_idx}.npy") | |
| cur_test_seq = np.load(cur_test_seq_path) | |
| object_idx = cur_test_seq['f7'][0].item() | |
| cur_obj_mesh_nm = id2objmeshname[object_idx] | |
| test_seq_idx_to_mesh_nm[cur_test_seq_idx] = cur_obj_mesh_nm | |
| return test_seq_idx_to_mesh_nm | |
| def get_category_nns(): | |
| cat_root_folder = "/data2/sim/HOI_Processed_Data_Arti" | |
| tot_cat_nms = ["Bucket", "Laptop", "Pliers", "Scissors"] | |
| # Scissors/Scissors | |
| tot_cat_nms = ["Bucket", "Laptop", "Pliers", "Scissors/Scissors"] | |
| cat_nm_to_case_nns = {} | |
| for cur_cat_nm in tot_cat_nms: | |
| cur_cat_folder = os.path.join(cat_root_folder, cur_cat_nm) | |
| tot_cases = os.listdir(cur_cat_folder) | |
| tot_cases = [cur_case_nm for cur_case_nm in tot_cases if "case" in cur_case_nm] | |
| cat_nm_to_case_nns[cur_cat_nm] = len(tot_cases) | |
| print(cat_nm_to_case_nns) | |
| # {'Bucket': 42, 'Laptop': 155, 'Pliers': 187, 'Scissors/Scissors': 93} | |
| # {'Bottle': 214, 'Bowl': 217, 'Chair': 167, 'Knife': 58, 'Mug': 249, 'ToyCar': 257} | |
| cat_root_folder = "/data2/sim/HOI_Processed_Data_Rigid" | |
| tot_cat_nms = ["Bottle", "Bowl", "Chair", "Knife", "Mug", "ToyCar"] | |
| cat_nm_to_case_nns = {} | |
| for cur_cat_nm in tot_cat_nms: | |
| cur_cat_folder = os.path.join(cat_root_folder, cur_cat_nm) | |
| tot_cases = os.listdir(cur_cat_folder) | |
| tot_cases = [cur_case_nm for cur_case_nm in tot_cases if "case" in cur_case_nm] | |
| cat_nm_to_case_nns[cur_cat_nm] = len(tot_cases) | |
| print(cat_nm_to_case_nns) | |
| def get_cat_avg_values(): | |
| # cat_nm_to_arti_objs_nn = {'Bucket': 42, 'Laptop': 155, 'Pliers': 187, 'Scissors': 93} | |
| cat_nm_to_arti_objs_nn = {'Laptop': 155, 'Pliers': 187, 'Scissors': 93} | |
| # cat_nm_to_rigid_objs_nn = {'Bottle': 214, 'Bowl': 217, 'Chair': 167, 'Knife': 58, 'Mug': 249, 'ToyCar': 257, 'Kettle': 58} | |
| cat_nm_to_rigid_objs_nn = {'Bottle': 214, 'Bowl': 217, 'Chair': 167, 'Mug': 249, 'ToyCar': 257, 'Kettle': 58} | |
| cat_nm_to_objs_nn = {} | |
| cat_nm_to_objs_nn.update(cat_nm_to_arti_objs_nn) | |
| cat_nm_to_objs_nn.update(cat_nm_to_rigid_objs_nn) | |
| ### TOCH | |
| # cat_to_penetration_depth = { | |
| # "Knife": 29.56, "Bottle": 187.69, "Pliers": 667.96, "Scissors": 11.69, "Bowl": 28.66, "Kettle": 34.3, "Mug": 21.47, "ToyCar": 60.42 | |
| # } | |
| # cat_to_smoothness = { | |
| # "Knife": 9.885, "Bottle": 3.5871, "Pliers": 5.594, "Scissors": 6.376, "Bowl": 10.54, "Kettle": 6.81, "Mug": 10.56, "ToyCar": 2.404 | |
| # } | |
| # cat_to_ho_motion_consistency = { | |
| # "Knife": 20.896, "Bottle": 55.083, "Pliers": 14.3, "Scissors": 3.024, "Bowl": 18.37, "Kettle": 19.7, "Mug": 26.89, "ToyCar": 15.45 | |
| # } | |
| # tot_sum_value = 0. | |
| # tot_sum_nns = 0 | |
| # for cat_nm in cat_to_penetration_depth: | |
| # cur_cat_nn = cat_nm_to_objs_nn[cat_nm] | |
| # cur_cat_penetration_depth = cat_to_penetration_depth[cat_nm] | |
| # cur_cat_tot_pene_depth = cur_cat_penetration_depth * float(cur_cat_nn) | |
| # tot_sum_nns += cur_cat_nn | |
| # tot_sum_value += cur_cat_tot_pene_depth | |
| # avg_pene_depth = tot_sum_value / float(tot_sum_nns) | |
| # print(f"Avg_pene_depth: {avg_pene_depth}") | |
| # tot_sum_value = 0. | |
| # tot_sum_nns = 0 | |
| # for cat_nm in cat_to_smoothness: | |
| # cur_cat_nn = cat_nm_to_objs_nn[cat_nm] | |
| # cur_cat_smoothness = cat_to_smoothness[cat_nm] | |
| # cur_cat_tot_smoothness = cur_cat_smoothness * float(cur_cat_nn) | |
| # tot_sum_nns += cur_cat_nn | |
| # tot_sum_value += cur_cat_tot_smoothness | |
| # avg_smoothness = tot_sum_value / float(tot_sum_nns) | |
| # print(f"Avg_smoothness: {avg_smoothness}") | |
| # tot_sum_value = 0. | |
| # tot_sum_nns = 0 | |
| # for cat_nm in cat_to_ho_motion_consistency: | |
| # cur_cat_nn = cat_nm_to_objs_nn[cat_nm] | |
| # cur_cat_consistency = cat_to_ho_motion_consistency[cat_nm] | |
| # cur_cat_consistency = cur_cat_consistency * float(cur_cat_nn) | |
| # tot_sum_nns += cur_cat_nn | |
| # tot_sum_value += cur_cat_consistency | |
| # avg_consistency = tot_sum_value / float(tot_sum_nns) | |
| # print(f"Avg_consistency: {avg_consistency}") | |
| # # Avg_pene_depth: 147.75575393848462 | |
| # # Avg_smoothness: 6.683753488372093 | |
| # # Avg_consistency: 23.818613653413355 | |
| ### TOCH | |
| cat_to_penetration_depth = { | |
| "Knife": 1.5044, "Bottle": 135.51, "Pliers": 389.75, "Scissors": 5.616, "Bowl": 23.53, "Kettle": 38.64, "Mug": 7.446, "ToyCar": 19.19 | |
| } | |
| cat_to_smoothness = { | |
| "Knife":0.1232, "Bottle": 1.9689, "Pliers": 2.249, "Scissors": 0.3316, "Bowl": 1.186, "Kettle": 1.013, "Mug": 0.7445, "ToyCar": 1.066 | |
| } | |
| cat_to_ho_motion_consistency = { | |
| "Knife": 5.0841, "Bottle": 35.11, "Pliers": 10.86, "Scissors": 3.632, "Bowl": 4.983, "Kettle": 4.687, "Mug": 3.07, "ToyCar": 2.722 | |
| } | |
| tot_sum_value = 0. | |
| tot_sum_nns = 0 | |
| for cat_nm in cat_to_penetration_depth: | |
| cur_cat_nn = cat_nm_to_objs_nn[cat_nm] | |
| cur_cat_penetration_depth = cat_to_penetration_depth[cat_nm] | |
| cur_cat_tot_pene_depth = cur_cat_penetration_depth * float(cur_cat_nn) | |
| tot_sum_nns += cur_cat_nn | |
| tot_sum_value += cur_cat_tot_pene_depth | |
| avg_pene_depth = tot_sum_value / float(tot_sum_nns) | |
| print(f"Avg_pene_depth: {avg_pene_depth}") | |
| tot_sum_value = 0. | |
| tot_sum_nns = 0 | |
| for cat_nm in cat_to_smoothness: | |
| cur_cat_nn = cat_nm_to_objs_nn[cat_nm] | |
| cur_cat_smoothness = cat_to_smoothness[cat_nm] | |
| cur_cat_tot_smoothness = cur_cat_smoothness * float(cur_cat_nn) | |
| tot_sum_nns += cur_cat_nn | |
| tot_sum_value += cur_cat_tot_smoothness | |
| avg_smoothness = tot_sum_value / float(tot_sum_nns) | |
| print(f"Avg_smoothness: {avg_smoothness}") | |
| tot_sum_value = 0. | |
| tot_sum_nns = 0 | |
| for cat_nm in cat_to_ho_motion_consistency: | |
| cur_cat_nn = cat_nm_to_objs_nn[cat_nm] | |
| cur_cat_consistency = cat_to_ho_motion_consistency[cat_nm] | |
| cur_cat_consistency = cur_cat_consistency * float(cur_cat_nn) | |
| tot_sum_nns += cur_cat_nn | |
| tot_sum_value += cur_cat_consistency | |
| avg_consistency = tot_sum_value / float(tot_sum_nns) | |
| print(f"Avg_consistency: {avg_consistency}") | |
| # Avg_pene_depth: 87.49058304576144 | |
| # Avg_smoothness: 1.241823330832708 | |
| # Avg_consistency: 9.74805311327832 | |
| def get_obj_name_to_test_seqs(): | |
| id2objmeshname = get_idx_to_objname() | |
| test_folder = "/data1/sim/GRAB_processed/test/" | |
| test_folder = "/data1/sim/GRAB_processed/train/" | |
| tot_test_seqs = os.listdir(test_folder) | |
| # tot_test_seq_idxes = [ii for ii in range(246)] | |
| tot_test_seq_idxes = [ii for ii in range(1392)] | |
| test_seq_idx_to_mesh_nm = {} | |
| mesh_nm_to_test_seqs = {} | |
| for cur_test_seq_idx in tot_test_seq_idxes: | |
| cur_test_seq_path = os.path.join(test_folder, f"{cur_test_seq_idx}.npy") | |
| cur_test_seq = np.load(cur_test_seq_path) | |
| object_idx = cur_test_seq['f7'][0].item() | |
| cur_obj_mesh_nm = id2objmeshname[object_idx] | |
| if cur_obj_mesh_nm in mesh_nm_to_test_seqs: | |
| mesh_nm_to_test_seqs[cur_obj_mesh_nm].append(cur_test_seq_idx) | |
| else: | |
| mesh_nm_to_test_seqs[cur_obj_mesh_nm] = [cur_test_seq_idx] | |
| # test_seq_idx_to_mesh_nm[cur_test_seq_idx] = cur_obj_mesh_nm | |
| return mesh_nm_to_test_seqs | |
| # and the test seqs for | |
| # pert_rhand_verts, pert_rhand_verts_lft = load_arctic_clip_data(seq_path, more_pert=False, other_noise=False) | |
| def load_arctic_clip_data(seq_path, more_pert=False, other_noise=False): | |
| rgt_mano_model = get_arctic_mano_model(side='right') | |
| lft_manp_model = get_arctic_mano_model(side='left') ### get mano models ### | |
| grab_path = "/data1/sim/GRAB_extracted" # extracted | |
| split = "test" | |
| # window_size = 60 | |
| singe_seq_path = seq_path | |
| clip_clean = np.load(singe_seq_path, allow_pickle=True).item() | |
| c = clip_clean | |
| # subj_root_path = '/data1/sim/GRAB_processed_wsubj' | |
| # subj_seq_path = f"{clip_seq_idx}_subj.npy" | |
| # subj_params_fn = os.path.join(subj_root_path, split, subj_seq_path) | |
| # subj_params = np.load(subj_params_fn, allow_pickle=True).item() | |
| ### rot, pose, transl, betas for the rhand pose gt #### | |
| rhand_global_orient_gt, rhand_pose_gt = c["rot_r"], c["pose_r"] | |
| rhand_transl, rhand_betas = c["trans_r"], c["shape_r"][0] | |
| rhand_transl = rhand_transl.reshape(rhand_global_orient_gt.shape[0], -1).astype(np.float32) | |
| rhand_betas = rhand_betas.reshape(-1).astype(np.float32) | |
| rhand_global_orient_var = torch.from_numpy(rhand_global_orient_gt).float() | |
| rhand_pose_var = torch.from_numpy(rhand_pose_gt).float() | |
| rhand_beta_var = torch.from_numpy(rhand_betas).float() | |
| rhand_transl_var = torch.from_numpy(rhand_transl).float() # self.window_size x 3 | |
| # R.from_rotvec(obj_rot).as_matrix() | |
| aug_trans, aug_rot, aug_pose = 0.01, 0.05, 0.3 | |
| aug_trans, aug_rot, aug_pose = 0.001, 0.05, 0.3 | |
| aug_trans, aug_rot, aug_pose = 0.00, 0.00, 0.2 | |
| # aug_trans, aug_rot, aug_pose = 0.000, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.000, 0.00, 0.00 | |
| ## bsz x val_dim -> for each variable here # | |
| aug_global_orient_var = torch.randn_like(rhand_global_orient_var) * aug_rot ### sigma = aug_rot | |
| aug_pose_var = torch.randn_like(rhand_pose_var) * aug_pose ### sigma = aug_pose | |
| aug_transl_var = torch.randn_like(rhand_transl_var) * aug_trans ### sigma = aug_trans | |
| rnd_aug_global_orient_var = rhand_global_orient_var + aug_global_orient_var | |
| rnd_aug_pose_var = rhand_pose_var + aug_pose_var | |
| rnd_aug_transl_var = rhand_transl_var + aug_transl_var ### aug transl | |
| pert_rhand_verts, pert_rhand_joints = rgt_mano_model( | |
| torch.cat([rnd_aug_global_orient_var, rnd_aug_pose_var], dim=-1), | |
| rhand_beta_var.unsqueeze(0).repeat(rhand_global_orient_gt.shape[0], 1).view(-1, 10), rnd_aug_transl_var | |
| ) | |
| ### rhand_joints: for joints ### | |
| pert_rhand_verts = pert_rhand_verts * 0.001 | |
| pert_rhand_joints = pert_rhand_joints * 0.001 | |
| ### rot, pose, transl, betas for the rhand pose gt #### | |
| rhand_global_orient_gt_lft, rhand_pose_gt_lft = c["rot_l"], c["pose_l"] | |
| rhand_transl_lft, rhand_betas_lft = c["trans_l"], c["shape_l"][0] | |
| rhand_transl_lft = rhand_transl_lft.reshape(rhand_global_orient_gt_lft.shape[0], -1).astype(np.float32) | |
| rhand_betas_lft = rhand_betas_lft.reshape(-1).astype(np.float32) | |
| rhand_global_orient_var_lft = torch.from_numpy(rhand_global_orient_gt_lft).float() | |
| rhand_pose_var_lft = torch.from_numpy(rhand_pose_gt_lft).float() | |
| rhand_beta_var_lft = torch.from_numpy(rhand_betas_lft).float() | |
| rhand_transl_var_lft = torch.from_numpy(rhand_transl_lft).float() # self.window_size x 3 | |
| # R.from_rotvec(obj_rot).as_matrix() | |
| # aug_trans, aug_rot, aug_pose = 0.01, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.001, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.000, 0.05, 0.3 | |
| # aug_trans, aug_rot, aug_pose = 0.000, 0.00, 0.00 | |
| aug_global_orient_var = torch.randn_like(rhand_global_orient_var_lft) * aug_rot ### sigma = aug_rot | |
| aug_pose_var = torch.randn_like(rhand_pose_var_lft) * aug_pose ### sigma = aug_pose | |
| aug_transl_var = torch.randn_like(rhand_transl_var_lft) * aug_trans ### sigma = aug_trans | |
| rnd_aug_global_orient_var_lft = rhand_global_orient_var_lft + aug_global_orient_var | |
| rnd_aug_pose_var_lft = rhand_pose_var_lft + aug_pose_var | |
| rnd_aug_transl_var_lft = rhand_transl_var_lft + aug_transl_var ### aug transl | |
| pert_rhand_verts_lft, pert_rhand_joints_lft = lft_manp_model( | |
| torch.cat([rnd_aug_global_orient_var_lft, rnd_aug_pose_var_lft], dim=-1), | |
| rhand_beta_var_lft.unsqueeze(0).repeat(rhand_global_orient_gt.shape[0], 1).view(-1, 10), rnd_aug_transl_var_lft | |
| ) | |
| ### rhand_joints: for joints ### | |
| pert_rhand_verts_lft = pert_rhand_verts_lft * 0.001 | |
| pert_rhand_joints_lft = pert_rhand_joints_lft * 0.001 | |
| return pert_rhand_verts, pert_rhand_verts_lft | |
| # volume rendering and the sdf based rendering -> any differences # | |
| # give a reasonable range for those properties # | |
| # actions - actions # | |
| # from pytorch3d.transforms import matrix_to_axis_angle # mano maodel # # v template # infer from actions for object properties # properties | |
| def get_extracted_data(extracted_data_folder_fn, obj_ty): | |
| # /home/xueyi/sim/arctic/data/arctic_data/data/meta/object_vtemplates | |
| object_mesh_root_fn = "/home/xueyi/sim/arctic/data/arctic_data/data/meta/object_vtemplates" | |
| obj_mesh_fn = os.path.join(object_mesh_root_fn, obj_ty, "mesh.obj") | |
| ## | |
| print(f"Start loading form {obj_mesh_fn}") | |
| obj_verts, obj_faces = load_ply_data(obj_mesh_fn) | |
| # | |
| beta_l_fn = "pred.mano.beta.l.pt" | |
| beta_l_fn = os.path.join(extracted_data_folder_fn, beta_l_fn) | |
| beta_r_fn = "pred.mano.beta.r.pt" | |
| beta_r_fn = os.path.join(extracted_data_folder_fn, beta_r_fn) | |
| pose_l_fn = "pred.mano.pose.l.pt" | |
| pose_l_fn = os.path.join(extracted_data_folder_fn, pose_l_fn) | |
| pose_r_fn = "pred.mano.pose.r.pt" | |
| pose_r_fn = os.path.join(extracted_data_folder_fn, pose_r_fn) ## get pose_l and pose_r | |
| trans_l_fn = "pred.mano.cam_t.l.pt" | |
| trans_l_fn = os.path.join(extracted_data_folder_fn, trans_l_fn) | |
| trans_r_fn = "pred.mano.cam_t.r.pt" | |
| trans_r_fn = os.path.join(extracted_data_folder_fn, trans_r_fn) | |
| trans_l = torch.load(trans_l_fn, map_location='cpu') | |
| # print(trans_l.size()) | |
| print(f"trans_l: {trans_l.size()}") | |
| pose_l = torch.load(pose_l_fn, map_location='cpu') | |
| pose_l = matrix_to_axis_angle(pose_l) | |
| print(f"pose_l: {pose_l.size()}") | |
| beta_l = torch.load(beta_l_fn, map_location='cpu') | |
| print(f"beta_l: {beta_l.size()}") | |
| ### extracted target data ### | |
| def get_extracted_target_data(extracted_data_folder_fn, obj_ty): | |
| # /home/xueyi/sim/arctic/data/arctic_data/data/meta/object_vtemplates | |
| object_mesh_root_fn = "/home/xueyi/sim/arctic/data/arctic_data/data/meta/object_vtemplates" | |
| obj_mesh_fn = os.path.join(object_mesh_root_fn, obj_ty, "mesh.obj") | |
| ## | |
| print(f"Start loading form {obj_mesh_fn}") | |
| obj_verts, obj_faces = load_ply_data(obj_mesh_fn) | |
| #targets | |
| beta_l_fn = "targets.mano.beta.l.pt" | |
| beta_l_fn = os.path.join(extracted_data_folder_fn, beta_l_fn) | |
| beta_r_fn = "targets.mano.beta.r.pt" | |
| beta_r_fn = os.path.join(extracted_data_folder_fn, beta_r_fn) | |
| pose_l_fn = "targets.mano.pose.l.pt" | |
| pose_l_fn = os.path.join(extracted_data_folder_fn, pose_l_fn) | |
| pose_r_fn = "targets.mano.pose.r.pt" | |
| pose_r_fn = os.path.join(extracted_data_folder_fn, pose_r_fn) ## get pose_l and pose_r ## | |
| trans_l_fn = "targets.mano.cam_t.l.pt" | |
| trans_l_fn = os.path.join(extracted_data_folder_fn, trans_l_fn) | |
| trans_r_fn = "targets.mano.cam_t.r.pt" | |
| trans_r_fn = os.path.join(extracted_data_folder_fn, trans_r_fn) | |
| trans_l = torch.load(trans_l_fn, map_location='cpu') | |
| # print(trans_l.size()) | |
| print(f"trans_l: {trans_l.size()}") | |
| pose_l = torch.load(pose_l_fn, map_location='cpu') | |
| pose_l = matrix_to_axis_angle(pose_l) | |
| print(f"pose_l: {pose_l.size()}") | |
| beta_l = torch.load(beta_l_fn, map_location='cpu') | |
| # pose_r = matrix_to_axis_angle() | |
| print(f"beta_l: {beta_l.size()}") | |
| # 0353.pkl | |
| # | |
| def get_arctic_seq_paths(): | |
| processed_arctic_root = "/data/datasets/genn/sim/arctic_processed_data/processed_seqs" | |
| subj_folders = os.listdir(processed_arctic_root) | |
| tot_arctic_seq_paths = [] | |
| tot_arctic_seq_tags = [] | |
| for cur_subj_folder in subj_folders: | |
| full_cur_subj_folder = os.path.join(processed_arctic_root, cur_subj_folder) | |
| cur_subj_seq_nms = os.listdir(full_cur_subj_folder) | |
| cur_subj_seq_nms = [fn for fn in cur_subj_seq_nms if fn.endswith(".npy")] | |
| for cur_subj_seq_nm in cur_subj_seq_nms: | |
| full_seq_nm = os.path.join(full_cur_subj_folder, cur_subj_seq_nm) | |
| tot_arctic_seq_paths.append(full_seq_nm) | |
| cur_seq_tag = f"{cur_subj_folder}_{cur_subj_seq_nm.split('.')[0]}" | |
| tot_arctic_seq_tags.append(cur_seq_tag) | |
| return tot_arctic_seq_paths, tot_arctic_seq_tags | |
| def filter_arctic_seqs(): | |
| arctic_root_path = "/data/datasets/genn/sim/arctic_processed_data/processed_seqs" | |
| sv_path = "/data2/datasets/sim/arctic_processed_data/processed_split_seqs" | |
| valid_st_idxes = f"{sv_path}/valid_st_idxes.npy" | |
| tot_subjs = os.listdir(arctic_root_path) | |
| mano_path = "/data1/sim/mano_models/mano/models" | |
| cur_mano_layer = ManoLayer( | |
| flat_hand_mean=False, | |
| side='right', | |
| mano_root=mano_path, # mano_root # | |
| ncomps=45, | |
| use_pca=False, | |
| ) | |
| window_size = 60 | |
| step_size = 60 | |
| thres = 0.15 | |
| subj_to_seq_to_valid_st_idx = {} | |
| for cur_subj in tot_subjs: | |
| cur_subj_folder = os.path.join(arctic_root_path, cur_subj) | |
| cur_subj_sv_folder = os.path.join(sv_path, cur_subj) | |
| os.makedirs(cur_subj_sv_folder, exist_ok=True) | |
| cur_subj_seqs = os.listdir(cur_subj_folder) | |
| seq_to_valid_st_idx = {} | |
| for cur_seq in cur_subj_seqs: | |
| cur_seq_valid_st_idx = [] | |
| cur_subj_seq_path = os.path.join(cur_subj_folder, cur_seq) | |
| cur_subj_seq = np.load(cur_subj_seq_path, allow_pickle=True).item() | |
| obj_pc = cur_subj_seq["verts.object"] | |
| rhand_rot, rhand_pose = cur_subj_seq["rot_r"], cur_subj_seq["pose_r"] | |
| rhand_transl, rhand_betas =cur_subj_seq["trans_r"], cur_subj_seq["shape_r"][0] | |
| rhand_betas = rhand_betas.reshape(-1).astype(np.float32) | |
| obj_pc_var = torch.from_numpy(obj_pc).float() | |
| nn_frmaes = obj_pc.shape[0] | |
| rhand_rot_var = torch.from_numpy(rhand_rot).float() | |
| rhand_pose_var = torch.from_numpy(rhand_pose).float() | |
| rhand_beta_var = torch.from_numpy(rhand_betas).float() | |
| rhand_transl_var = torch.from_numpy(rhand_transl).float() # self.window_size x 3 | |
| # window_size = | |
| rhand_verts, rhand_joints = cur_mano_layer( | |
| torch.cat([rhand_rot_var, rhand_pose_var], dim=-1), | |
| rhand_beta_var.unsqueeze(0).repeat(nn_frmaes, 1).view(-1, 10), rhand_transl_var | |
| ) | |
| ### rhand_joints: for joints ### | |
| rhand_verts = rhand_verts * 0.001 | |
| rhand_joints = rhand_joints * 0.001 | |
| wrist_positions = rhand_joints[:, 0] # nframes x 3 | |
| dist_wrist_to_obj_pc = torch.sum( | |
| (wrist_positions.unsqueeze(1) - obj_pc_var) ** 2, dim=-1 ### nn_frames x nn_pc | |
| ) | |
| dist_wrist_to_obj_pc, _ = torch.min(dist_wrist_to_obj_pc, dim=-1) ## the wrist to pc ## | |
| for i_split in range(0, nn_frmaes - window_size, step_size): | |
| cur_window_wrist_to_pc = dist_wrist_to_obj_pc[i_split: i_split + window_size] ## windowi_size x 3 # | |
| maxx_dist = torch.max(cur_window_wrist_to_pc).item() | |
| if maxx_dist > thres: | |
| continue | |
| cur_seq_valid_st_idx.append(i_split) | |
| cur_sv_dict = { | |
| 'rhand_verts': rhand_verts[i_split: i_split + window_size].detach().cpu().numpy(), | |
| 'obj_pc': obj_pc[i_split: i_split + window_size], | |
| } | |
| cur_sv_dict_path = os.path.join(cur_subj_sv_folder, f"{cur_seq.split('.')[0]}_st_{i_split}_ed_{i_split + window_size}.npy") | |
| np.save(cur_sv_dict_path, cur_sv_dict) | |
| seq_to_valid_st_idx[cur_seq] = cur_seq_valid_st_idx | |
| subj_to_seq_to_valid_st_idx[cur_subj] = seq_to_valid_st_idx | |
| np.save(valid_st_idxes, subj_to_seq_to_valid_st_idx) | |
| print(f"valid st idxes saved to {valid_st_idxes}") | |
| def get_valid_seqs_nn(): | |
| valid_seqs_fn = "/data2/datasets/sim/arctic_processed_data/processed_split_seqs/valid_st_idxes.npy" | |
| valid_seqs = np.load(valid_seqs_fn, allow_pickle=True).item() | |
| train_nn = 0 | |
| for subj in valid_seqs: | |
| cur_subj_to_valid_idxes = valid_seqs[subj] | |
| tot_nn = 0 | |
| for seq_nm in cur_subj_to_valid_idxes: | |
| tot_nn += len(cur_subj_to_valid_idxes[seq_nm]) | |
| print(f"subj: {subj}, nn: {tot_nn}") | |
| if subj != "s01": | |
| train_nn += tot_nn | |
| print(f"train_nn: {train_nn}") | |
| if __name__=='__main__': | |
| # get_valid_seqs_nn() | |
| # exit(0) | |
| # filter_arctic_seqs() | |
| # exit(0) | |
| tot_arctic_seq_paths, tot_arctic_seq_tags = get_arctic_seq_paths() | |
| tot_arctic_seq_tags_dict = { | |
| idx: seq_tag for idx, seq_tag in enumerate(tot_arctic_seq_tags) | |
| } | |
| # raw_seq_sv_path = os.path.join("/data2/datasets/sim/arctic_save_res", f"arctic_seq_tags_dict.npy") | |
| # if not os.path.exists(raw_seq_sv_path): | |
| # np.save(raw_seq_sv_path, tot_arctic_seq_tags_dict) | |
| # print(f"seq tags dict saved to {raw_seq_sv_path}") | |
| # print(f"{tot_arctic_seq_tags_dict}") | |
| # exit(0) | |
| # extracted_data_folder_fn = "/home/xueyi/sim/arctic/logs/3558f1342/eval/s01_box_grab_01_1/targets" | |
| # obj_ty = "box" | |
| # get_extracted_target_data(extracted_data_folder_fn, obj_ty) | |
| # exit(0) | |
| # extracted_data_folder_fn = "/home/xueyi/sim/arctic/logs/3558f1342/eval/s01_box_grab_01_1/preds" | |
| # obj_ty = "box" | |
| # get_extracted_data(extracted_data_folder_fn, obj_ty) | |
| # exit(0) | |
| # get_cat_avg_values() | |
| # exit(0) | |
| # get_category_nns() | |
| # exit(0) | |
| # pkl_fn = "/data1/sim/oakink/OakInk-Shape/oakink_shape_v2/apple/C90001/0eec013c90/hand_param.pkl" | |
| # test_pickle(pkl_fn) | |
| # test_seq_idx_to_mesh_nm = get_test_idx_to_obj_name() | |
| # print(test_seq_idx_to_mesh_nm) | |
| # test_seq_idx_to_mesh_nm_sv_fn = "test_seq_idx_to_mesh_nm.npy" | |
| # np.save(test_seq_idx_to_mesh_nm_sv_fn, test_seq_idx_to_mesh_nm) | |
| # exit(0) | |
| # mesh_nm_to_test_seqs = get_obj_name_to_test_seqs() | |
| # print(mesh_nm_to_test_seqs) | |
| # train_mesh_nm_to_test_seqs = {} | |
| # test_mesh_nm_to_test_seqs = {} | |
| # for cur_mesh_nm in mesh_nm_to_test_seqs: # | |
| # cur_idxes = mesh_nm_to_test_seqs[cur_mesh_nm] | |
| # cur_idxes_nn = len(cur_idxes) | |
| # train_nns = int(float(cur_idxes_nn) * 0.8) | |
| # test_nns = cur_idxes_nn - train_nns | |
| # train_mesh_nm_to_test_seqs[cur_mesh_nm] = cur_idxes[:train_nns] | |
| # test_mesh_nm_to_test_seqs[cur_mesh_nm] = cur_idxes[train_nns:] | |
| # np.save("train_mesh_nm_to_test_seqs.npy", train_mesh_nm_to_test_seqs) | |
| # np.save("test_mesh_nm_to_test_seqs.npy", test_mesh_nm_to_test_seqs) | |
| # np.save("mesh_nm_to_test_seqs.npy", mesh_nm_to_test_seqs) | |
| # exit(0) | |
| split = 'train' ## | |
| clip_seq_idx = 3 | |
| clip_seq_idx = 100 | |
| clip_seq_idx = 200 | |
| clip_seq_idx = 110 | |
| other_noise = True | |
| other_noise = False # other noise # | |
| # split = 'test' | |
| # clip_seq_idx = 3 | |
| # clip_seq_idx = 80 | |
| # clip_seq_idx = 8 | |
| # split = 'test' | |
| # clip_seq_idx = 5 | |
| # sepqrate # | |
| # underlying template with some unknown parameters # | |
| # underlying template with some unknown parameters # | |
| # optimize the template to get the parameters # | |
| # optimize the parameters in the template -- the geometry # | |
| # optimize the parameters | |
| # /home/xueyi/sim/motion-diffusion-model/utils/common_tests.py | |
| ### get obj data ## | |
| # obj_verts, obj_faces = load_grab_clip_data_clean_obj(clip_seq_idx, more_pert=False, other_noise=False, split=split) | |
| # # load_grab_clip_data_clean_subj(clip_seq_idx, pert=False, more_pert=False, other_noise=False): | |
| # clean_rhand_verts = load_grab_clip_data_clean_subj(clip_seq_idx, pert=False, more_pert=False, other_noise=False, split=split) | |
| # pert_rhand_verts = load_grab_clip_data_clean_subj(clip_seq_idx, pert=True, more_pert=False, other_noise=False, split=split) | |
| ## optimize the geometry ## # optimize the geometry # | |
| ## ffmpeg -i input -vf "pad=w=iw:h='max(720,ih)':x=0:y=0:color=white" output ## | |
| #### perturb verts for the arctic dataset ### | |
| subj_idx = "s01" | |
| raw_seq_fn = "capsulemachine_use_01" | |
| seq_path = f"/data/datasets/genn/sim/arctic_processed_data/processed_seqs/{subj_idx}/{raw_seq_fn}.npy" | |
| # | |
| subj_idx = "s01" | |
| raw_seq_fn = "box_use_01" | |
| subj_idx = "s01" | |
| raw_seq_fn = "box_use_02" | |
| seq_path = f"/data/datasets/genn/sim/arctic_processed_data/processed_seqs/{subj_idx}/{raw_seq_fn}.npy" | |
| subj_idx = "s01" | |
| raw_seq_fn = "box_use_02" | |
| subj_idx = "s01" | |
| raw_seq_fn = "espressomachine_use_01" | |
| raw_seq_fn = "capsulemachine_grab_01" | |
| raw_seq_fn = "espressomachine_use_02" | |
| # needs train for the spatial diff as wel # | |
| for seq_idx in range(0, 12): | |
| cur_seq_raw_fn = tot_arctic_seq_tags_dict[seq_idx] | |
| subj_idx = cur_seq_raw_fn.split("_")[0] | |
| raw_seq_fn = "_".join(cur_seq_raw_fn.split("_")[1:]) | |
| # raw_seq_fn = "capsulemachine_use_01" # raw_seq_fn # # raw seq fn # # raw seq fn # | |
| # raw_seq_fn = "capsulemachine_use_02" | |
| seq_path = f"/data/datasets/genn/sim/arctic_processed_data/processed_seqs/{subj_idx}/{raw_seq_fn}.npy" | |
| ### seq_path ### | |
| pert_rhand_verts, pert_rhand_verts_lft = load_arctic_clip_data(seq_path, more_pert=False, other_noise=False) # | |
| cur_seq_pert_verts = { | |
| "pert_rhand_verts": pert_rhand_verts, | |
| "pert_rhand_verts_lft": pert_rhand_verts_lft, | |
| } | |
| sv_path = os.path.join("/data2/datasets/sim/arctic_save_res", f"pert_verts_{subj_idx}_{raw_seq_fn}.npy") | |
| np.save(sv_path, cur_seq_pert_verts) | |
| print(f"pert verts saved to {sv_path}") | |
| exit(0) | |
| ### get obj data ## | |
| obj_verts, obj_faces = load_grab_clip_data_clean_obj(clip_seq_idx, more_pert=False, other_noise=other_noise, split=split) | |
| # load_grab_clip_data_clean_subj(clip_seq_idx, pert=False, more_pert=False, other_noise=False): | |
| clean_rhand_verts = load_grab_clip_data_clean_subj(clip_seq_idx, pert=False, more_pert=False, other_noise=other_noise, split=split) | |
| pert_rhand_verts = load_grab_clip_data_clean_subj(clip_seq_idx, pert=True, more_pert=False, other_noise=other_noise, split=split) | |
| cur_clip_obj_hand_data = { | |
| 'pert_rhand_verts': pert_rhand_verts, | |
| 'clean_rhand_verts': clean_rhand_verts, | |
| 'obj_verts': obj_verts, | |
| 'obj_faces': obj_faces, | |
| } | |
| sv_dict_fn = f"tmp_sv_dict_grab_split_{split}_seq_{clip_seq_idx}.npy" | |
| np.save(sv_dict_fn, cur_clip_obj_hand_data) | |
| print(f"data with clip obj and subj saved to {sv_dict_fn}") | |
| exit(0) | |
| test_seq = 2 | |
| test_seq = 4 | |
| test_seq = 40 | |
| test_seq = 55 | |
| test_seq = 8 | |
| test_seq = 80 | |
| test_seq = 23 | |
| test_seq = 5 | |
| test_seq = 38 | |
| test_seq = 41 | |
| test_seq = 119 | |
| test_seq = 149 | |
| test_seq = 98 | |
| test_seq = 1 | |
| test_seq = 2 | |
| test_seq = 80 | |
| test_seq = 5 | |
| test_seq = 23 | |
| other_noise = True | |
| pert_rhand_verts = load_grab_clip_data(test_seq, other_noise=other_noise) | |
| pert_rhand_verts_more = load_grab_clip_data(test_seq, more_pert=True, other_noise=other_noise) | |
| pert_rhand_verts = pert_rhand_verts.detach().cpu().numpy() | |
| pert_rhand_verts_more = pert_rhand_verts_more.detach().cpu().numpy() | |
| sv_dict = { | |
| 'pert_rhand_verts': pert_rhand_verts, | |
| 'pert_rhand_verts_more': pert_rhand_verts_more | |
| } | |
| sv_dict_fn = f"tmp_sv_dict_pert_grab_seq_{test_seq}_other_noise_{other_noise}.npy" | |
| np.save(sv_dict_fn, sv_dict) | |
| print(f"pert rhand verts saved to {sv_dict_fn}") | |
| exit(0) | |
| # RIGHT_HAND_POSE_ROOT = /data1/sim/handpose/refinehandpose_right | |
| # SERVER_DATA_ROOT = /share/datasets/HOI4D_overall/ | |
| sv_dict_fn = "/data1/sim/HOI_Processed_Data_Arti/case7/meta_data.npy" | |
| sv_dict_fn = "/data1/sim/HOI_Processed_Data_Arti/case70/meta_data.npy" | |
| sv_dict_fn = "/data1/sim/HOI_Processed_Data_Arti/case175/meta_data.npy" | |
| sv_dict_fn = "/data1/sim/HOI_Processed_Data_Arti/case174/meta_data.npy" | |
| sv_dict_fn = "/data1/sim/HOI_Processed_Data_Arti/case173/meta_data.npy" | |
| # /data1/sim/HOI_Processed_Data_Arti/case194 | |
| sv_dict_fn = "/data1/sim/HOI_Processed_Data_Arti/case194/meta_data.npy" | |
| sv_dict_fn = "/data2/sim/HOI_Processed_Data_Arti/Scissors/Scissors/case47/meta_data.npy" | |
| # /data2/sim/HOI_Processed_Data_Arti # | |
| # get_meta_info(sv_dict_fn) | |
| # exit(0) | |
| tot_case_flag = [] | |
| ##### Bucket ##### | |
| # ['ZY20210800001/H1/C8/N11/S73/s01/T1', 'ZY20210800001/H1/C8/N12/S73/s01/T1', 'ZY20210800001/H1/C8/N13/S73/s02/T1', 'ZY20210800001/H1/C8/N13/S73/s02/T2', 'ZY20210800001/H1/C8/N14/S73/s02/T2', 'ZY20210800001/H1/C8/N15/S73/s03/T2', 'ZY20210800001/H1/C8/N19/S74/s02/T1', 'ZY20210800001/H1/C8/N21/S74/s03/T2', 'ZY20210800001/H1/C8/N23/S76/s01/T2', 'ZY20210800001/H1/C8/N24/S76/s02/T2', 'ZY20210800001/H1/C8/N25/S76/s02/T1', 'ZY20210800001/H1/C8/N25/S76/s02/T2', 'ZY20210800001/H1/C8/N26/S76/s03/T1', 'ZY20210800001/H1/C8/N28/S78/s01/T2', 'ZY20210800001/H1/C8/N29/S77/s05/T1', 'ZY20210800001/H1/C8/N31/S77/s04/T1', 'ZY20210800001/H1/C8/N31/S77/s04/T2', 'ZY20210800001/H1/C8/N32/S77/s03/T1', 'ZY20210800001/H1/C8/N32/S77/s03/T2', 'ZY20210800001/H1/C8/N33/S77/s03/T1', 'ZY20210800001/H1/C8/N33/S77/s03/T2', 'ZY20210800001/H1/C8/N34/S77/s04/T1', 'ZY20210800001/H1/C8/N34/S77/s04/T2', 'ZY20210800001/H1/C8/N40/S77/s01/T2', 'ZY20210800001/H1/C8/N41/S77/s02/T2', 'ZY20210800002/H2/C8/N11/S80/s01/T1', 'ZY20210800002/H2/C8/N11/S80/s01/T2', 'ZY20210800002/H2/C8/N12/S80/s01/T1', 'ZY20210800002/H2/C8/N12/S80/s01/T2', 'ZY20210800002/H2/C8/N13/S80/s02/T1', 'ZY20210800002/H2/C8/N13/S80/s02/T2', 'ZY20210800002/H2/C8/N14/S80/s02/T1', 'ZY20210800002/H2/C8/N15/S80/s03/T1', 'ZY20210800002/H2/C8/N15/S80/s03/T2', 'ZY20210800003/H3/C8/N38/S74/s02/T2', 'ZY20210800003/H3/C8/N39/S74/s02/T1', 'ZY20210800003/H3/C8/N42/S74/s04/T1', 'ZY20210800004/H4/C8/N12/S71/s02/T1', 'ZY20210800004/H4/C8/N12/S71/s02/T2', 'ZY20210800004/H4/C8/N13/S71/s02/T1', 'ZY20210800004/H4/C8/N14/S71/s03/T1', 'ZY20210800004/H4/C8/N14/S71/s03/T2'] | |
| # T2: 10 - 70 | |
| # st_idx = 8 | |
| # ed_idx = 12 | |
| # st_idx = 14 | |
| # ed_idx = 15 | |
| # cat_nm = "Scissors" | |
| # cat_nm = "Pliers" | |
| # cat_ty = "Arti" | |
| # cat_nm = "ToyCar" | |
| # cat_ty = "Rigid" | |
| # st_idx = 0 | |
| # ed_idx = 4 | |
| # use_toch = True | |
| # # use_toch = False | |
| # cat_nm = "Bottle" | |
| # cat_ty = "Rigid" | |
| # test_tag = "rep_res_jts_hoi4d_bottle_t_300_st_idx_0_" | |
| # st_idx = 1 | |
| # ed_idx = 4 | |
| # get_setting_to_stats(st_idx, ed_idx, use_toch=use_toch) | |
| # exit(0) | |
| # 154 | |
| # for case_idx in range(92): | |
| # for case_idx in range(42): | |
| # for case_idx in range(187): # jpliers | |
| # for case_idx in range(154): # | |
| # try: | |
| # print(f"Case idx: {case_idx}") | |
| # # sv_dict_fn = f"/data2/sim/HOI_Processed_Data_Arti/Scissors/Scissors/case{case_idx}/meta_data.npy" | |
| # # sv_dict_fn = f"/data2/sim/HOI_Processed_Data_Arti/Bucket/case{case_idx}/meta_data.npy" | |
| # # sv_dict_fn = f"/data2/sim/HOI_Processed_Data_Arti/Pliers/case{case_idx}/meta_data.npy" | |
| # sv_dict_fn = f"/data2/sim/HOI_Processed_Data_Arti/Laptop/case{case_idx}/meta_data.npy" | |
| # # sv_dict_fn = f"/data2/sim/HOI_Processed_Data_Rigid/Mug/case{case_idx}/meta_data.npy" | |
| # cur_case_flag = get_meta_info(sv_dict_fn) | |
| # tot_case_flag.append(cur_case_flag) | |
| # except: | |
| # continue | |
| # print(f"tot_case_flase") | |
| # print(tot_case_flag) | |
| # exit(0) | |
| i_test_seq = 0 | |
| test_tag = "jts_hoi4d_arti_t_400_" | |
| test_tag = "rep_res_jts_hoi4d_arti_scissors_t_400_" | |
| test_tag = "rep_res_jts_hoi4d_toycar_t_300_st_idx_0_" | |
| start_idx = 50 | |
| ws = 60 | |
| st_idx = 0 | |
| ed_idx = 44 | |
| st_idx = 8 | |
| ed_idx = 12 | |
| st_idx = 12 | |
| ed_idx = 15 | |
| cat_nm = "Scissors" | |
| cat_nm = "Pliers" | |
| cat_ty = "Arti" | |
| cat_nm = "ToyCar" | |
| cat_ty = "Rigid" | |
| st_idx = 0 | |
| ed_idx = 4 | |
| cat_nm = "Pliers" | |
| cat_ty = "Arti" | |
| test_tag = "rep_res_jts_hoi4d_pliers_t_300_st_idx_30_" | |
| st_idx = 1 | |
| ed_idx = 2 | |
| cat_nm = "Bottle" | |
| cat_ty = "Rigid" | |
| test_tag = "rep_res_jts_hoi4d_bottle_t_300_st_idx_0_" | |
| st_idx = 0 | |
| ed_idx = 5 | |
| cat_nm = "Scissors" | |
| cat_ty = "Arti" | |
| test_tag = "rep_res_jts_hoi4d_bottle_t_300_st_idx_0_" | |
| test_tag = "rep_res_jts_hoi4d_arti_scissors_t_300_st_idx_30_" | |
| test_tag = "rep_res_jts_hoi4d_arti_scissors_t_300_st_idx_0_" | |
| st_idx = 11 | |
| ed_idx = 12 | |
| cat_nm = "Knife" | |
| cat_ty = "Rigid" | |
| test_tag = "rep_res_jts_hoi4d_knife_t_300_st_idx_0_" | |
| st_idx = 0 | |
| ed_idx = 8 | |
| cat_nm = "Chair" | |
| cat_ty = "Rigid" | |
| test_tag = "rep_res_jts_hoi4d_chair_t_300_st_idx_0_" | |
| st_idx = 0 | |
| ed_idx = 8 | |
| # st_idx = 2 | |
| # ed_idx = 3 | |
| # seq_idx_to_setting_to_stats = {} ## get test settings to statistics | |
| for i_test_seq in range(st_idx, ed_idx): | |
| ### ours ### # get_test_settings_to_statistics | |
| cur_seq_setting_to_stats = get_test_settings_to_statistics(i_test_seq, test_tag, start_idx=start_idx, ws=ws) | |
| # # seq_idx_to_setting_to_stats[i_test_seq] = cur_seq_setting_to_stats | |
| cur_stats_sv_fn = f"/data2/sim/eval_save/HOI_{cat_ty}/{cat_nm}/setting_to_stats_seq_{i_test_seq}.npy" | |
| np.save(cur_stats_sv_fn, cur_seq_setting_to_stats) | |
| print(f"Setting to stats file saved to {cur_stats_sv_fn}") | |
| # # (0.0016338544664904475, 1.5728545577076147e-06, 3.436529596001492e-06) | |
| # (0.0017392054433003068, 4.550241555989487e-06, 7.405976703012129e-06)) | |
| # (0, 0.005, False), (0.0017126222373917699, 3.0001126560819102e-06, 6.4585701693431474e-06)) | |
| # knife | |
| # ours | |
| # i_test_seq: 2, seed: 0, dist_thres: 0.001, with_proj: True, penetration_depth: 1.5044701285660267e-05, smoothness: 1.232047111443535e-06, cur_moving_consistency: 5.0841890697483905e-06 | |
| # toch | |
| # i_test_seq: 2, seed: 0, dist_thres: 0.005, with_proj: True, penetration_depth: 0.002956472337245941, smoothness: 9.885283361654729e-05, cur_moving_consistency: 2.0896763089695014e-05 | |
| ### toch ### | |
| # use_toch = True | |
| # try: | |
| # cur_seq_setting_to_stats = get_test_settings_to_statistics(i_test_seq, test_tag, start_idx=start_idx, ws=ws, use_toch=use_toch) | |
| # # seq_idx_to_setting_to_stats[i_test_seq] = cur_seq_setting_to_stats | |
| # cur_stats_sv_fn = f"/data2/sim/eval_save/HOI_{cat_ty}/{cat_nm}/setting_to_stats_seq_{i_test_seq}_toch.npy" | |
| # np.save(cur_stats_sv_fn, cur_seq_setting_to_stats) | |
| # print(f"Setting to stats file saved to {cur_stats_sv_fn}") | |
| # except: | |
| # continue | |
| # /data2/sim/eval_save/HOI_Arti/Scissors | |
| ### stats to saved fn | |
| # seq_idx_to_setting_to_stats_sv_fn = f"/data2/sim/eval_save/HOI_Arti/Scissors/seq_idx_to_setting_to_stats_v2_basepts.npy" | |
| # np.save(seq_idx_to_setting_to_stats_sv_fn, seq_idx_to_setting_to_stats) | |
| # print(f"seq_idx_to_setting_to_stats saved to {seq_idx_to_setting_to_stats_sv_fn}") | |
| exit(0) | |
| # th_sv_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/dmt_dict_00000.pt" | |
| # dmt_dict_00000_res_128.pt | |
| # th_sv_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results/tets/dmt_dict_00000_res_128.pt" | |
| # th_sv_fn = "/home/xueyi/sim/MeshDiffusion/nvdiffrec/dmtet_results_seq/tets/dmt_dict_00002.pt" | |
| # load_data_fr_th_sv(th_sv_fn, grid_res=128) | |
| # rendering gpus, | |
| # incorporate dynamics into the process # th_sv_fn # # th_sv_fn # | |
| # dneoise accleration and | |
| # representations, voxels, | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00041-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_2_batch_5.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00045-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_3_batch_0.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00050-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_23_batch_0.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00050-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_56_batch_0.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00052-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_39_batch_0_nreg.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00054-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_22_batch_0_nreg.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00065-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_63_batch_0_nreg.npy" | |
| th_sv_fn = "/data2/sim/implicit_ae/logging/00065-stylegan2-rendering-gpus1-batch4-gamma80/out_iter_9420_batch_0_nreg.npy" | |
| load_data_fr_th_sv_fr_pred(th_sv_fn, grid_res=128) | |
| exit(0) | |
| # meta_data.npy | |
| # rt_path = "/data1/sim/mdm/tmp_data/case5" | |
| # load_and_save_verts(rt_path) | |
| rt_path = "/home/xueyi/sim/motion-diffusion-model/predicted_infos_fn_to_statistics.npy" | |
| get_penetration_depth_rnk_data(rt_path) | |