''' @ Date: 2020-12-01 22:14:11 @ Author: Qing Shuai @ LastEditors: Qing Shuai @ LastEditTime: 2021-05-30 21:33:40 @ FilePath: /EasyMocap/scripts/postprocess/eval_shelf.py ''' import os import sys from os.path import join import re import json import time import scipy.io as scio import numpy as np from tqdm import tqdm def save_json(output, json_path): os.system('mkdir -p {}'.format(os.path.dirname(json_path))) with open(json_path, 'w') as f: json.dump(output, f, indent=4) def is_right(model_start_point, model_end_point, gt_strat_point, gt_end_point, alpha=0.5): bone_lenth = np.linalg.norm ( gt_end_point - gt_strat_point ) start_difference = np.linalg.norm ( gt_strat_point - model_start_point ) end_difference = np.linalg.norm ( gt_end_point - model_end_point ) return ((start_difference + end_difference) / 2) <= alpha * bone_lenth def openpose2shelf3D(pose3d, score): """ transform coco order(our method output) 3d pose to shelf dataset order with interpolation :param pose3d: np.array with shape nJx3 :return: 3D pose in shelf order with shape 14x3 """ shelf_pose = np.zeros ( (14, 3) ) shelf_score = np.zeros ( (14, 1) ) # coco2shelf = np.array ( [16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9] ) openpose2shelf = np.array([11, 10, 9, 12, 13, 14, 4, 3, 2, 5, 6, 7]) shelf_pose[0: 12] += pose3d[openpose2shelf] shelf_score[0: 12] += score[openpose2shelf] if True: shelf_pose[12] = pose3d[1] # Use middle of shoulder to init shelf_pose[13] = pose3d[0] # use nose to init shelf_pose[13] = shelf_pose[12] + (shelf_pose[13] - shelf_pose[12]) * np.array ( [0.75, 0.75, 1.5] ) shelf_pose[12] = shelf_pose[12] + (pose3d[0] - shelf_pose[12]) * np.array ( [1. / 2., 1. / 2., 1. / 2.] ) shelf_score[12] = score[0]*score[1] shelf_score[13] = score[0]*score[1] else: shelf_pose[12] = pose3d[1] shelf_pose[13] = pose3d[0] return shelf_pose, shelf_score def convert_openpose_shelf(keypoints3d): shelf15 = np.zeros((15, 4)) openpose2shelf = np.array([11, 10, 9, 12, 13, 14, 4, 3, 2, 5, 6, 7, 1, 0, 8]) shelf15 = keypoints3d[openpose2shelf].copy() # interp head faceDir = np.cross(shelf15[12, :3] - shelf15[14, :3], shelf15[8, :3] - shelf15[9, :3]) faceDir = faceDir/np.linalg.norm(faceDir) zDir = np.array([0., 0., 1.]) shoulderCenter = (keypoints3d[2, :3] + keypoints3d[5, :3])/2. # headCenter = (keypoints3d[15, :3] + keypoints3d[16, :3])/2. headCenter = (keypoints3d[17, :3] + keypoints3d[18, :3])/2. shelf15[12, :3] = shoulderCenter + (headCenter - shoulderCenter) * 0.5 shelf15[13, :3] = shelf15[12, :3] + faceDir * 0.125 + zDir * 0.145 return shelf15 def convert_openpose_shelf1(keypoints3d): shelf15 = np.zeros((15, 4)) openpose2shelf = np.array([11, 10, 9, 12, 13, 14, 4, 3, 2, 5, 6, 7, 1, 0, 8]) shelf15 = keypoints3d[openpose2shelf].copy() # interp head faceDir = np.cross(keypoints3d[1, :3] - keypoints3d[8, :3], keypoints3d[2, :3] - shelf15[5, :3]) faceDir = faceDir/np.linalg.norm(faceDir) upDir = keypoints3d[1, :3] - keypoints3d[8, :3] upDir = upDir/np.linalg.norm(upDir) shoulderCenter = keypoints3d[1, :3] ear = (keypoints3d[17, :3] + keypoints3d[18, :3])/2 - keypoints3d[1, :3] eye = (keypoints3d[15, :3] + keypoints3d[16, :3])/2 - keypoints3d[1, :3] nose = keypoints3d[0, :3] - keypoints3d[1, :3] head = (ear + eye + nose)/3. noseLen = np.linalg.norm(head) noseDir = head / noseLen headDir = (noseDir * 2 + upDir) headDir = headDir / np.linalg.norm(headDir) neck = shoulderCenter + noseLen*headDir * 0.5 shelf15[12, :3] = neck shelf15[13, :3] = neck + headDir * noseLen * 0.8 return shelf15 def convert_shelf_shelfgt(keypoints): gt_hip = (keypoints[2] + keypoints[3]) / 2 gt = np.vstack((keypoints, gt_hip)) return gt def vectorize_distance(a, b): """ Calculate euclid distance on each row of a and b :param a: Nx... np.array :param b: Mx... np.array :return: MxN np.array representing correspond distance """ N = a.shape[0] a = a.reshape ( N, -1 ) M = b.shape[0] b = b.reshape ( M, -1 ) a2 = np.tile ( np.sum ( a ** 2, axis=1 ).reshape ( -1, 1 ), (1, M) ) b2 = np.tile ( np.sum ( b ** 2, axis=1 ), (N, 1) ) dist = a2 + b2 - 2 * (a @ b.T) return np.sqrt ( dist ) def distance(a, b, score): # a: (N, J, 3) # b: (M, J, 3) # score: (M, J, 1) # return: (M, N) a = a[None, :, :, :] b = b[:, None, :, :] score = score[:, None, :, 0] diff = np.sum((a - b)**2, axis=3)*score dist = diff.sum(axis=2)/score.sum(axis=2) return np.sqrt(dist) def _readResult(filename, isA4d): import json with open(filename, "r") as file: datas = json.load(file) res_ = [] for data in datas: trackId = data['id'] keypoints3d = np.array(data['keypoints3d']) if (keypoints3d[:, 3]>0).sum() > 1: res_.append({'id':trackId, 'keypoints3d': keypoints3d}) if isA4d: # association4d 的关节顺序和正常的定义不一样 for r in res_: r['keypoints3d'] = r['keypoints3d'][[4, 1, 5, 9, 13, 6, 10, 14, 0, 2, 7, 11, 3, 8, 12], :] return res_ def readResult(filePath, range_=None, isA4d=None): res = {} if range_ is None: from glob import glob filelists = glob(join(filePath, '*.txt')) range_ = [i for i in range(len(filelists))] if isA4d is None: isA4d = args.a4d for imgId in tqdm(range_): res[imgId] = _readResult(join(filePath, '{:06d}.json'.format(imgId)), isA4d) return res class ShelfGT: def __init__(self, actor3D) -> None: self.actor3D = actor3D self.actor3D = self.actor3D[:3] def __getitem__(self, index): results = [] for pid in range(len(self.actor3D)): gt_pose = self.actor3D[pid][index-2][0] if gt_pose.shape == (1, 0) or gt_pose.shape == (0, 0): continue keypoints3d = convert_shelf_shelfgt(gt_pose) results.append({'id': pid, 'keypoints3d': keypoints3d}) return results def write_to_csv(filename, results, id_wise=True): keys = [key for key in results[0].keys() if isinstance(results[0][key], float)] if id_wise: ids = list(set([res['id'] for res in results])) header = [''] + ['{:s}'.format(key.replace(' ', '')) for key in keys] contents = [] if id_wise: for pid in ids: content = ['{}'.format(pid)] for key in keys: vals = [res[key] for res in results if res['id'] == pid] content.append('{:.3f}'.format(sum(vals)/len(vals))) contents.append(content) # 计算平均值 content = ['Mean'] for i, key in enumerate(keys): content.append('{:.3f}'.format(sum([float(con[i+1]) for con in contents])/len(ids))) contents.append(content) else: content = ['Mean'] for key in keys: content.append('{:.3f}'.format(sum([res[key] for res in results])/len(results))) contents.append(content) import tabulate print(tabulate.tabulate(contents, header, tablefmt='fancy_grid')) print(tabulate.tabulate(contents, header, tablefmt='fancy_grid'), file=open(filename.replace('.csv', '.txt'), 'w')) with open(filename, 'w') as f: # 写入头 header = list(results[0].keys()) f.write(','.join(header) + '\n') for res in results: f.write(','.join(['{}'.format(res[key]) for key in header]) + '\n') def evaluate(actor3D, range_, out): shelfgt = ShelfGT(actor3D) check_result = np.zeros ( (len ( actor3D[0] ), len ( actor3D ), 10), dtype=np.int32 ) result = readResult(out, range_) bones = [[0, 1], [1, 2], [3, 4], [4, 5], [6, 7], [7, 8], [9, 10], [10, 11], [12, 13], [12, 14]] start = [ 9, 8, 10, 7, 3, 2, 4, 1, 12, 12,] end = [10, 7, 11, 6, 4, 1, 5, 0, 13, 14] names = ["Left Upper Arm", "Right Upper Arm", "Left Lower Arm", "Right Lower Arm", "Left Upper Leg", "Right Upper Leg", "Left Lower Leg", "Right Lower Leg", "Head", "Torso" ] results = [] for img_id in tqdm(range_): # 转化成model_poses ests = [] for res in result[img_id]: ests.append({'id': res['id'], 'keypoints3d': convert_openpose_shelf1(res['keypoints3d'])}) gts = shelfgt[img_id] if len(gts) < 1: continue # 匹配最近的 kpts_gt = np.stack([v['keypoints3d'] for v in gts]) kpts_dt = np.stack([v['keypoints3d'] for v in ests]) distances = np.linalg.norm(kpts_gt[:, None, :, :3] - kpts_dt[None, :, :, :3], axis=-1) conf = (kpts_gt[:, None, :, -1] > 0) * (kpts_dt[None, :, :, -1] > 0) dist = (distances * conf).sum(axis=-1)/conf.sum(axis=-1) # 贪婪的匹配 ests_new = [] for igt, gt in enumerate(gts): bestid = np.argmin(dist[igt]) ests_new.append(ests[bestid]) ests = ests_new # 计算误差 for i, data in enumerate(gts): kpts_gt = data['keypoints3d'] kpts_est = ests[i]['keypoints3d'] # 计算各种误差,存成字典 da = np.linalg.norm(kpts_gt[start, :3] - kpts_est[start, :3], axis=1) db = np.linalg.norm(kpts_gt[end, :3] - kpts_est[end, :3], axis=1) l = np.linalg.norm(kpts_gt[start, :3] - kpts_gt[end, :3], axis=1) isright = 1.0*((da + db) < l) if args.joint: res = {name: isright[i] for i, name in enumerate(names)} else: res = {} res['Mean'] = isright.mean() res['nf'] = img_id res['id'] = data['id'] results.append(res) write_to_csv(join(out, '..', 'report.csv'), results) return 0 if __name__ == '__main__': import argparse parser = argparse.ArgumentParser () parser.add_argument('--out', type=str, default='output/') parser.add_argument('--gt_path', type=str, default='config/evaluation/actorsGT_shelf.mat') parser.add_argument('--setting', type=str, default='shelf') parser.add_argument('--a4d', action='store_true') parser.add_argument('--joint', action='store_true') args = parser.parse_args () if args.setting == 'shelf': test_range = range ( 302, 602) # test_range = range (2000, 3200) elif args.setting == 'campus': test_range = [i for i in range ( 350, 471 )] + [i for i in range ( 650, 751 )] else: raise NotImplementedError actorsGT = scio.loadmat (args.gt_path) test_actor3D = actorsGT['actor3D'][0] if False: valid = np.zeros((3200, 4)) for nf in range(3200): for pid in range(4): if test_actor3D[pid][nf].item().shape[0] == 14: valid[nf, pid] = 1 import matplotlib.pyplot as plt plt.plot(valid.sum(axis=1)) plt.show() import ipdb; ipdb.set_trace() evaluate(test_actor3D, test_range, args.out)