193 lines
7.5 KiB
Python
193 lines
7.5 KiB
Python
|
import numpy as np
|
|||
|
|
import cv2
|
|||
|
|
import scipy
|
|||
|
|
import torch
|
|||
|
|
|
|||
|
|
class MultilView_Merge:
|
|||
|
|
def __init__(self) -> None:
|
|||
|
|
pass
|
|||
|
|
def forward(self, data,ax=0):
|
|||
|
|
'''
|
|||
|
|
data - dict
|
|||
|
|
data[key] (nv,...)
|
|||
|
|
'''
|
|||
|
|
results={}
|
|||
|
|
for key in data.keys():
|
|||
|
|
results[key] = data[key].mean(axis=ax)
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Merge_hand(MultilView_Merge):
|
|||
|
|
def __init__(self, camtoworld) -> None:
|
|||
|
|
self.camtoworld = camtoworld
|
|||
|
|
# pass
|
|||
|
|
def __call__(self, posel , cameras, match3d_l):
|
|||
|
|
# ret = []
|
|||
|
|
# for nf in range(len(posel)):
|
|||
|
|
# breakpoint()
|
|||
|
|
hand_list=[]
|
|||
|
|
for pid in range(len(match3d_l)):
|
|||
|
|
dt = match3d_l[pid]
|
|||
|
|
if(isinstance(dt,int)):
|
|||
|
|
# TODO:处理-1的情况，也就是没有找到合适的匹配到的手
|
|||
|
|
hand_list.append(np.zeros((1,48)))
|
|||
|
|
break
|
|||
|
|
Merge_list=[]
|
|||
|
|
for cid in range(len(dt['views'])):
|
|||
|
|
nv = dt['views'][cid]
|
|||
|
|
poseid = dt['indices'][cid]
|
|||
|
|
pose = posel[nv][poseid].copy()
|
|||
|
|
|
|||
|
|
if self.camtoworld:
|
|||
|
|
Rh = pose[:,:3].copy()
|
|||
|
|
invR = np.linalg.inv(cameras['R'][nv])
|
|||
|
|
Rh_m_old = np.matrix(cv2.Rodrigues(Rh)[0])
|
|||
|
|
Rh_m_new = invR @ Rh_m_old
|
|||
|
|
Rh = cv2.Rodrigues(Rh_m_new)[0]
|
|||
|
|
Merge_list.append(np.hstack((Rh.reshape(3),pose[:,3:].reshape(-1))))
|
|||
|
|
else:
|
|||
|
|
Merge_list.append(pose)
|
|||
|
|
out = self.forward({'pose':np.stack(Merge_list)},0)
|
|||
|
|
|
|||
|
|
hand_list.append(out['pose'])
|
|||
|
|
pose_ = np.stack(hand_list)
|
|||
|
|
Rh = pose_[:,:3].copy()
|
|||
|
|
pose_[:,:3] = 0
|
|||
|
|
params={
|
|||
|
|
'Rh':Rh,
|
|||
|
|
'Th':np.zeros_like(Rh),
|
|||
|
|
'poses':pose_,
|
|||
|
|
'shapes':np.zeros((Rh.shape[0],10)),
|
|||
|
|
}
|
|||
|
|
# ret.append(params)
|
|||
|
|
return {'params': params}
|
|||
|
|
|
|||
|
|
class Merge_handlr(Merge_hand):
|
|||
|
|
def __call__(self, posel, poser, cameras, match3d_l, match3d_r):
|
|||
|
|
params_l = super().__call__(posel, cameras, match3d_l)
|
|||
|
|
params_r = super().__call__(poser, cameras, match3d_r)
|
|||
|
|
# breakpoint()
|
|||
|
|
return {'params_l':params_l['params'], 'params_r':params_r['params']}
|
|||
|
|
# return {'params_l':params_l['params'], 'params_r':params_r['params'], 'params':params_l['params']}
|
|||
|
|
|
|||
|
|
class Merge_bodyandhand:
|
|||
|
|
def __init__(self, tmp) -> None:
|
|||
|
|
pass
|
|||
|
|
def get_R(self, poses, cfg, st):
|
|||
|
|
res = st.copy()
|
|||
|
|
for i in cfg:
|
|||
|
|
res = res @ cv2.Rodrigues(poses[i,:])[0]
|
|||
|
|
return res
|
|||
|
|
def process_poses_mano(self, poses, hand_Rh, flag):
|
|||
|
|
if sum(flag) == 0:
|
|||
|
|
return poses
|
|||
|
|
|
|||
|
|
poses = poses.reshape((-1,3))
|
|||
|
|
cfg={'rt': [0,3,6,9],
|
|||
|
|
'r': [14,17,19],
|
|||
|
|
'l': [13,16,18]
|
|||
|
|
}
|
|||
|
|
RA = self.get_R(poses, cfg['rt'],np.eye(3))
|
|||
|
|
|
|||
|
|
if flag[0] :
|
|||
|
|
RL = self.get_R(poses, cfg['l'],RA)
|
|||
|
|
tmppose = np.matrix(RL).I @ cv2.Rodrigues(np.array(hand_Rh[0]))[0]
|
|||
|
|
tmppose = cv2.Rodrigues(tmppose)[0]
|
|||
|
|
poses[20,:] = tmppose.reshape(3)
|
|||
|
|
|
|||
|
|
e20 = scipy.spatial.transform.Rotation.from_rotvec(torch.from_numpy(poses[20,:]).reshape(-1,3))
|
|||
|
|
e20 = e20.as_euler('ZYX', degrees=True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
dt = scipy.spatial.transform.Rotation.from_euler('ZYX', np.array([0,0,e20[0,2]/2]), degrees=True)
|
|||
|
|
rot_dt = dt.as_matrix()
|
|||
|
|
rot18 = cv2.Rodrigues(poses[18,:])[0]
|
|||
|
|
rot18 = rot18@rot_dt
|
|||
|
|
vec18 = cv2.Rodrigues(rot18)[0].reshape((1,3))
|
|||
|
|
rot20 = cv2.Rodrigues(poses[20,:])[0]
|
|||
|
|
rot20 = np.linalg.inv(rot_dt) @ rot20
|
|||
|
|
vec20 = cv2.Rodrigues(rot20)[0].reshape((1,3))
|
|||
|
|
poses[20,:] = vec20
|
|||
|
|
poses[18,:] = vec18
|
|||
|
|
|
|||
|
|
# e18 = scipy.spatial.transform.Rotation.from_rotvec(torch.from_numpy(poses[18,:]).reshape(-1,3))
|
|||
|
|
# e18 = e18.as_euler('ZYX', degrees=True)
|
|||
|
|
# e20[0,2] = e20[0,2]/2
|
|||
|
|
# e18[0,2] += e20[0,2]
|
|||
|
|
# e20 = scipy.spatial.transform.Rotation.from_euler('ZYX', e20, degrees=True)
|
|||
|
|
# e20 = e20.as_rotvec()
|
|||
|
|
# e18 = scipy.spatial.transform.Rotation.from_euler('ZYX', e18, degrees=True)
|
|||
|
|
# e18 = e18.as_rotvec()
|
|||
|
|
# poses[20,:] = e20
|
|||
|
|
# poses[18,:] = e18
|
|||
|
|
if flag[1] : #and sum(np.array(hand_Rh[1])!=0)>0:
|
|||
|
|
RR = self.get_R(poses, cfg['r'],RA)
|
|||
|
|
tmppose = np.matrix(RR).I @ cv2.Rodrigues(np.array(hand_Rh[1]))[0]
|
|||
|
|
tmppose = cv2.Rodrigues(tmppose)[0]
|
|||
|
|
poses[21,:] = tmppose.reshape(3)
|
|||
|
|
|
|||
|
|
e21 = scipy.spatial.transform.Rotation.from_rotvec(torch.from_numpy(poses[21,:]).reshape(-1,3))
|
|||
|
|
e21 = e21.as_euler('ZYX', degrees=True)
|
|||
|
|
|
|||
|
|
dt = scipy.spatial.transform.Rotation.from_euler('ZYX', np.array([0,0,e21[0,2]/2]), degrees=True)
|
|||
|
|
rot_dt = dt.as_matrix()
|
|||
|
|
rot19 = cv2.Rodrigues(poses[19,:])[0]
|
|||
|
|
rot19 = rot19@rot_dt
|
|||
|
|
vec19 = cv2.Rodrigues(rot19)[0].reshape((1,3))
|
|||
|
|
rot21 = cv2.Rodrigues(poses[21,:])[0]
|
|||
|
|
rot21 = np.linalg.inv(rot_dt) @ rot21
|
|||
|
|
vec21 = cv2.Rodrigues(rot21)[0].reshape((1,3))
|
|||
|
|
poses[21,:] = vec21
|
|||
|
|
poses[19,:] = vec19
|
|||
|
|
|
|||
|
|
# e19 = scipy.spatial.transform.Rotation.from_rotvec(torch.from_numpy(poses[19,:]).reshape(-1,3))
|
|||
|
|
# e19 = e19.as_euler('ZYX', degrees=True)
|
|||
|
|
# e21[0,2] = e21[0,2]/2
|
|||
|
|
# e19[0,2] += e21[0,2]
|
|||
|
|
# e21 = scipy.spatial.transform.Rotation.from_euler('ZYX', e21, degrees=True)
|
|||
|
|
# e21 = e21.as_rotvec()
|
|||
|
|
# e19 = scipy.spatial.transform.Rotation.from_euler('ZYX', e19, degrees=True)
|
|||
|
|
# e19 = e19.as_rotvec()
|
|||
|
|
# poses[21,:] = e21
|
|||
|
|
# poses[19,:] = e19
|
|||
|
|
|
|||
|
|
return poses.reshape((1,-1))
|
|||
|
|
|
|||
|
|
def merge_pose(self, bodypose,handlpose,handrpose):
|
|||
|
|
flag=[True,True]
|
|||
|
|
if abs(handlpose).sum()==0:
|
|||
|
|
flag[0]=False
|
|||
|
|
if abs(handrpose).sum()==0:
|
|||
|
|
flag[1]=False
|
|||
|
|
|
|||
|
|
out_L = []
|
|||
|
|
pose = np.hstack((bodypose,handlpose[:,3:],handrpose[:,3:])) # (1,156)
|
|||
|
|
out_pose = self.process_poses_mano(pose, [handlpose[0,:3],handrpose[0,:3]], flag) # 如果没找到手，那么应该设置为全0 这里设置为false
|
|||
|
|
out_L.append(out_pose)
|
|||
|
|
return out_pose
|
|||
|
|
def __call__(self, params_l, params_r, params):
|
|||
|
|
# breakpoint()
|
|||
|
|
bz = params['Rh'].shape[0]
|
|||
|
|
ret = {
|
|||
|
|
'Rh': np.zeros((bz,3),dtype=np.float32),
|
|||
|
|
'Th': params['Th'],
|
|||
|
|
'poses': np.zeros((bz,156),dtype=np.float32),
|
|||
|
|
'shapes':np.zeros((bz,16),dtype=np.float32)
|
|||
|
|
}
|
|||
|
|
ret['shapes'][:,:10] = params['shapes']
|
|||
|
|
# breakpoint()
|
|||
|
|
#TODO for nframe nperson
|
|||
|
|
for i in range(bz):
|
|||
|
|
inpose = np.zeros((1,66))
|
|||
|
|
inpose[:,3:] = params['poses'][i][:63].copy()
|
|||
|
|
inpose[:,:3] = params['Rh'][i].copy() # pose0:3有值 Rh 可能要合并
|
|||
|
|
|
|||
|
|
handlpose = params_l['poses'][i].reshape((1,-1)).copy()
|
|||
|
|
handrpose = params_r['poses'][i].reshape((1,-1)).copy()
|
|||
|
|
handlpose[:,:3] = params_l['Rh'][i]
|
|||
|
|
handrpose[:,:3] = params_r['Rh'][i]
|
|||
|
|
|
|||
|
|
out = self.merge_pose(inpose.reshape((1,-1)), handlpose, handrpose)
|
|||
|
|
ret['Rh'][i] = out[:,:3]
|
|||
|
|
ret['poses'][i,3:] = out[:,3:]
|
|||
|
|
return {'params_smplh': ret}
|