290 lines
11 KiB
Python
290 lines
11 KiB
Python
'''
|
|
@ Date: 2020-12-01 22:14:11
|
|
@ Author: Qing Shuai
|
|
@ LastEditors: Qing Shuai
|
|
@ LastEditTime: 2021-05-30 21:33:40
|
|
@ FilePath: /EasyMocap/scripts/postprocess/eval_shelf.py
|
|
'''
|
|
import os
|
|
import sys
|
|
from os.path import join
|
|
import re
|
|
import json
|
|
import time
|
|
import scipy.io as scio
|
|
import numpy as np
|
|
from tqdm import tqdm
|
|
|
|
def save_json(output, json_path):
|
|
os.system('mkdir -p {}'.format(os.path.dirname(json_path)))
|
|
with open(json_path, 'w') as f:
|
|
json.dump(output, f, indent=4)
|
|
|
|
def is_right(model_start_point, model_end_point, gt_strat_point, gt_end_point, alpha=0.5):
|
|
bone_lenth = np.linalg.norm ( gt_end_point - gt_strat_point )
|
|
start_difference = np.linalg.norm ( gt_strat_point - model_start_point )
|
|
end_difference = np.linalg.norm ( gt_end_point - model_end_point )
|
|
return ((start_difference + end_difference) / 2) <= alpha * bone_lenth
|
|
|
|
def openpose2shelf3D(pose3d, score):
|
|
"""
|
|
transform coco order(our method output) 3d pose to shelf dataset order with interpolation
|
|
:param pose3d: np.array with shape nJx3
|
|
:return: 3D pose in shelf order with shape 14x3
|
|
"""
|
|
shelf_pose = np.zeros ( (14, 3) )
|
|
shelf_score = np.zeros ( (14, 1) )
|
|
|
|
# coco2shelf = np.array ( [16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9] )
|
|
openpose2shelf = np.array([11, 10, 9, 12, 13, 14, 4, 3, 2, 5, 6, 7])
|
|
shelf_pose[0: 12] += pose3d[openpose2shelf]
|
|
shelf_score[0: 12] += score[openpose2shelf]
|
|
if True:
|
|
shelf_pose[12] = pose3d[1] # Use middle of shoulder to init
|
|
shelf_pose[13] = pose3d[0] # use nose to init
|
|
shelf_pose[13] = shelf_pose[12] + (shelf_pose[13] - shelf_pose[12]) * np.array ( [0.75, 0.75, 1.5] )
|
|
shelf_pose[12] = shelf_pose[12] + (pose3d[0] - shelf_pose[12]) * np.array ( [1. / 2., 1. / 2., 1. / 2.] )
|
|
shelf_score[12] = score[0]*score[1]
|
|
shelf_score[13] = score[0]*score[1]
|
|
else:
|
|
shelf_pose[12] = pose3d[1]
|
|
shelf_pose[13] = pose3d[0]
|
|
return shelf_pose, shelf_score
|
|
|
|
def convert_openpose_shelf(keypoints3d):
|
|
shelf15 = np.zeros((15, 4))
|
|
openpose2shelf = np.array([11, 10, 9, 12, 13, 14, 4, 3, 2, 5, 6, 7, 1, 0, 8])
|
|
shelf15 = keypoints3d[openpose2shelf].copy()
|
|
# interp head
|
|
faceDir = np.cross(shelf15[12, :3] - shelf15[14, :3], shelf15[8, :3] - shelf15[9, :3])
|
|
faceDir = faceDir/np.linalg.norm(faceDir)
|
|
zDir = np.array([0., 0., 1.])
|
|
shoulderCenter = (keypoints3d[2, :3] + keypoints3d[5, :3])/2.
|
|
# headCenter = (keypoints3d[15, :3] + keypoints3d[16, :3])/2.
|
|
headCenter = (keypoints3d[17, :3] + keypoints3d[18, :3])/2.
|
|
|
|
shelf15[12, :3] = shoulderCenter + (headCenter - shoulderCenter) * 0.5
|
|
shelf15[13, :3] = shelf15[12, :3] + faceDir * 0.125 + zDir * 0.145
|
|
return shelf15
|
|
|
|
def convert_openpose_shelf1(keypoints3d):
|
|
shelf15 = np.zeros((15, 4))
|
|
openpose2shelf = np.array([11, 10, 9, 12, 13, 14, 4, 3, 2, 5, 6, 7, 1, 0, 8])
|
|
shelf15 = keypoints3d[openpose2shelf].copy()
|
|
# interp head
|
|
faceDir = np.cross(keypoints3d[1, :3] - keypoints3d[8, :3], keypoints3d[2, :3] - shelf15[5, :3])
|
|
faceDir = faceDir/np.linalg.norm(faceDir)
|
|
|
|
upDir = keypoints3d[1, :3] - keypoints3d[8, :3]
|
|
upDir = upDir/np.linalg.norm(upDir)
|
|
|
|
shoulderCenter = keypoints3d[1, :3]
|
|
ear = (keypoints3d[17, :3] + keypoints3d[18, :3])/2 - keypoints3d[1, :3]
|
|
eye = (keypoints3d[15, :3] + keypoints3d[16, :3])/2 - keypoints3d[1, :3]
|
|
nose = keypoints3d[0, :3] - keypoints3d[1, :3]
|
|
head = (ear + eye + nose)/3.
|
|
noseLen = np.linalg.norm(head)
|
|
noseDir = head / noseLen
|
|
headDir = (noseDir * 2 + upDir)
|
|
headDir = headDir / np.linalg.norm(headDir)
|
|
|
|
neck = shoulderCenter + noseLen*headDir * 0.5
|
|
|
|
shelf15[12, :3] = neck
|
|
shelf15[13, :3] = neck + headDir * noseLen * 0.8
|
|
return shelf15
|
|
|
|
def convert_shelf_shelfgt(keypoints):
|
|
gt_hip = (keypoints[2] + keypoints[3]) / 2
|
|
gt = np.vstack((keypoints, gt_hip))
|
|
return gt
|
|
|
|
def vectorize_distance(a, b):
|
|
"""
|
|
Calculate euclid distance on each row of a and b
|
|
:param a: Nx... np.array
|
|
:param b: Mx... np.array
|
|
:return: MxN np.array representing correspond distance
|
|
"""
|
|
N = a.shape[0]
|
|
a = a.reshape ( N, -1 )
|
|
M = b.shape[0]
|
|
b = b.reshape ( M, -1 )
|
|
a2 = np.tile ( np.sum ( a ** 2, axis=1 ).reshape ( -1, 1 ), (1, M) )
|
|
b2 = np.tile ( np.sum ( b ** 2, axis=1 ), (N, 1) )
|
|
dist = a2 + b2 - 2 * (a @ b.T)
|
|
return np.sqrt ( dist )
|
|
|
|
def distance(a, b, score):
|
|
# a: (N, J, 3)
|
|
# b: (M, J, 3)
|
|
# score: (M, J, 1)
|
|
# return: (M, N)
|
|
a = a[None, :, :, :]
|
|
b = b[:, None, :, :]
|
|
score = score[:, None, :, 0]
|
|
diff = np.sum((a - b)**2, axis=3)*score
|
|
dist = diff.sum(axis=2)/score.sum(axis=2)
|
|
return np.sqrt(dist)
|
|
|
|
def _readResult(filename, isA4d):
|
|
import json
|
|
with open(filename, "r") as file:
|
|
datas = json.load(file)
|
|
res_ = []
|
|
for data in datas:
|
|
trackId = data['id']
|
|
keypoints3d = np.array(data['keypoints3d'])
|
|
if (keypoints3d[:, 3]>0).sum() > 1:
|
|
res_.append({'id':trackId, 'keypoints3d': keypoints3d})
|
|
if isA4d:
|
|
# association4d 的关节顺序和正常的定义不一样
|
|
for r in res_:
|
|
r['keypoints3d'] = r['keypoints3d'][[4, 1, 5, 9, 13, 6, 10, 14, 0, 2, 7, 11, 3, 8, 12], :]
|
|
return res_
|
|
|
|
def readResult(filePath, range_=None, isA4d=None):
|
|
res = {}
|
|
if range_ is None:
|
|
from glob import glob
|
|
filelists = glob(join(filePath, '*.txt'))
|
|
range_ = [i for i in range(len(filelists))]
|
|
if isA4d is None:
|
|
isA4d = args.a4d
|
|
for imgId in tqdm(range_):
|
|
res[imgId] = _readResult(join(filePath, '{:06d}.json'.format(imgId)), isA4d)
|
|
return res
|
|
|
|
class ShelfGT:
|
|
def __init__(self, actor3D) -> None:
|
|
self.actor3D = actor3D
|
|
self.actor3D = self.actor3D[:3]
|
|
|
|
def __getitem__(self, index):
|
|
results = []
|
|
for pid in range(len(self.actor3D)):
|
|
gt_pose = self.actor3D[pid][index-2][0]
|
|
if gt_pose.shape == (1, 0) or gt_pose.shape == (0, 0):
|
|
continue
|
|
keypoints3d = convert_shelf_shelfgt(gt_pose)
|
|
results.append({'id': pid, 'keypoints3d': keypoints3d})
|
|
return results
|
|
|
|
def write_to_csv(filename, results, id_wise=True):
|
|
keys = [key for key in results[0].keys() if isinstance(results[0][key], float)]
|
|
if id_wise:
|
|
ids = list(set([res['id'] for res in results]))
|
|
header = [''] + ['{:s}'.format(key.replace(' ', '')) for key in keys]
|
|
contents = []
|
|
if id_wise:
|
|
for pid in ids:
|
|
content = ['{}'.format(pid)]
|
|
for key in keys:
|
|
vals = [res[key] for res in results if res['id'] == pid]
|
|
content.append('{:.3f}'.format(sum(vals)/len(vals)))
|
|
contents.append(content)
|
|
# 计算平均值
|
|
content = ['Mean']
|
|
for i, key in enumerate(keys):
|
|
content.append('{:.3f}'.format(sum([float(con[i+1]) for con in contents])/len(ids)))
|
|
contents.append(content)
|
|
else:
|
|
content = ['Mean']
|
|
for key in keys:
|
|
content.append('{:.3f}'.format(sum([res[key] for res in results])/len(results)))
|
|
contents.append(content)
|
|
import tabulate
|
|
print(tabulate.tabulate(contents, header, tablefmt='fancy_grid'))
|
|
print(tabulate.tabulate(contents, header, tablefmt='fancy_grid'), file=open(filename.replace('.csv', '.txt'), 'w'))
|
|
|
|
with open(filename, 'w') as f:
|
|
# 写入头
|
|
header = list(results[0].keys())
|
|
f.write(','.join(header) + '\n')
|
|
for res in results:
|
|
f.write(','.join(['{}'.format(res[key]) for key in header]) + '\n')
|
|
|
|
def evaluate(actor3D, range_, out):
|
|
shelfgt = ShelfGT(actor3D)
|
|
check_result = np.zeros ( (len ( actor3D[0] ), len ( actor3D ), 10), dtype=np.int32 )
|
|
result = readResult(out, range_)
|
|
bones = [[0, 1], [1, 2], [3, 4], [4, 5], [6, 7], [7, 8], [9, 10], [10, 11], [12, 13], [12, 14]]
|
|
start = [ 9, 8, 10, 7, 3, 2, 4, 1, 12, 12,]
|
|
end = [10, 7, 11, 6, 4, 1, 5, 0, 13, 14]
|
|
names = ["Left Upper Arm", "Right Upper Arm", "Left Lower Arm", "Right Lower Arm", "Left Upper Leg", "Right Upper Leg", "Left Lower Leg", "Right Lower Leg", "Head", "Torso" ]
|
|
|
|
results = []
|
|
for img_id in tqdm(range_):
|
|
# 转化成model_poses
|
|
ests = []
|
|
for res in result[img_id]:
|
|
ests.append({'id': res['id'], 'keypoints3d': convert_openpose_shelf1(res['keypoints3d'])})
|
|
gts = shelfgt[img_id]
|
|
if len(gts) < 1:
|
|
continue
|
|
# 匹配最近的
|
|
kpts_gt = np.stack([v['keypoints3d'] for v in gts])
|
|
kpts_dt = np.stack([v['keypoints3d'] for v in ests])
|
|
distances = np.linalg.norm(kpts_gt[:, None, :, :3] - kpts_dt[None, :, :, :3], axis=-1)
|
|
conf = (kpts_gt[:, None, :, -1] > 0) * (kpts_dt[None, :, :, -1] > 0)
|
|
dist = (distances * conf).sum(axis=-1)/conf.sum(axis=-1)
|
|
# 贪婪的匹配
|
|
ests_new = []
|
|
for igt, gt in enumerate(gts):
|
|
bestid = np.argmin(dist[igt])
|
|
ests_new.append(ests[bestid])
|
|
ests = ests_new
|
|
# 计算误差
|
|
for i, data in enumerate(gts):
|
|
kpts_gt = data['keypoints3d']
|
|
kpts_est = ests[i]['keypoints3d']
|
|
# 计算各种误差，存成字典
|
|
da = np.linalg.norm(kpts_gt[start, :3] - kpts_est[start, :3], axis=1)
|
|
db = np.linalg.norm(kpts_gt[end, :3] - kpts_est[end, :3], axis=1)
|
|
l = np.linalg.norm(kpts_gt[start, :3] - kpts_gt[end, :3], axis=1)
|
|
isright = 1.0*((da + db) < l)
|
|
if args.joint:
|
|
res = {name: isright[i] for i, name in enumerate(names)}
|
|
else:
|
|
res = {}
|
|
res['Mean'] = isright.mean()
|
|
res['nf'] = img_id
|
|
res['id'] = data['id']
|
|
results.append(res)
|
|
write_to_csv(join(out, '..', 'report.csv'), results)
|
|
return 0
|
|
|
|
if __name__ == '__main__':
|
|
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser ()
|
|
parser.add_argument('--out', type=str, default='output/')
|
|
parser.add_argument('--gt_path', type=str, default='config/evaluation/actorsGT_shelf.mat')
|
|
parser.add_argument('--setting', type=str, default='shelf')
|
|
parser.add_argument('--a4d', action='store_true')
|
|
parser.add_argument('--joint', action='store_true')
|
|
|
|
args = parser.parse_args ()
|
|
if args.setting == 'shelf':
|
|
test_range = range ( 302, 602)
|
|
# test_range = range (2000, 3200)
|
|
elif args.setting == 'campus':
|
|
test_range = [i for i in range ( 350, 471 )] + [i for i in range ( 650, 751 )]
|
|
else:
|
|
raise NotImplementedError
|
|
|
|
actorsGT = scio.loadmat (args.gt_path)
|
|
test_actor3D = actorsGT['actor3D'][0]
|
|
if False:
|
|
valid = np.zeros((3200, 4))
|
|
for nf in range(3200):
|
|
for pid in range(4):
|
|
if test_actor3D[pid][nf].item().shape[0] == 14:
|
|
valid[nf, pid] = 1
|
|
import matplotlib.pyplot as plt
|
|
plt.plot(valid.sum(axis=1))
|
|
plt.show()
|
|
import ipdb; ipdb.set_trace()
|
|
evaluate(test_actor3D, test_range, args.out)
|