EasyMocap/code/demo_mv1pmf_smpl.py

'''
  @ Date: 2021-01-12 17:08:25
  @ Author: Qing Shuai
  @ LastEditors: Qing Shuai
  @ LastEditTime: 2021-01-25 19:32:44
  @ FilePath: /EasyMocapRelease/code/demo_mv1pmf_smpl.py
'''
# show skeleton and reprojection
import pyrender # first import the pyrender
from pyfitting.optimize_simple import optimizeShape, optimizePose
from dataset.mv1pmf import MV1PMF
from dataset.config import CONFIG
from mytools.utils import Timer
from smplmodel import select_nf, init_params, Config, load_model, check_keypoints
from os.path import join
from tqdm import tqdm
import numpy as np

def load_weight_shape():
    weight = {'s3d': 1., 'reg_shapes': 5e-3}
    return weight

def load_weight_pose(model):
    if model == 'smpl':
        weight = {
            'k3d': 1., 'reg_poses_zero': 1e-2, 'smooth_body': 1e-2
        }
    elif model == 'smplh':
        weight = {
            'k3d': 1., 'reg_poses_zero': 1e-3,
            'smooth_body': 1e-2, 'smooth_hand': 1e-2
        }
    elif model == 'smplx':
        weight = {
            'k3d': 1., 'reg_poses_zero': 1e-3,
            'reg_expression': 1e-2,
            'smooth_body': 1e-2, 'smooth_hand': 1e-2
        }
    else:
        raise NotImplementedError
    return weight

def print_mean_skel(mode):
    with Timer('Loading {}, {}'.format(args.model, args.gender)):
        body_model = load_model(args.gender, model_type=args.model)
    params_init = init_params(nFrames=1, model_type=args.model)
    skel = body_model(return_verts=False, return_tensor=False, **params_init)[0]
    # skel: nJoints, 3
    config = CONFIG[mode]
    skeleton = {}
    for i, j_ in config['kintree']:
        if j_ == 25:
            j = 7
        elif j_ == 46:
            j = 4
        else:
            j = j_
        key = tuple(sorted([i, j]))
        limb_length = np.linalg.norm(skel[i] - skel[j])
        skeleton[key] = {'mean': limb_length, 'std': limb_length*0.2}
    print('{')
    for key, val in skeleton.items():
        res = '    ({:2d}, {:2d}): {{\'mean\': {:.3f}, \'std\': {:.3f}}}, '.format(*key, val['mean'], val['std'])
        if 'joint_names' in config.keys():
            res += '# {:9s}->{:9s}'.format(config['joint_names'][key[0]], config['joint_names'][key[1]])
        print(res)
    print('}')

def mv1pmf_smpl(path, sub, out, mode, args):
    config = CONFIG[mode]
    no_img = True
    dataset = MV1PMF(path, cams=sub, config=CONFIG[mode], mode=args.body,
        undis=args.undis, no_img=no_img, out=out)
    if args.skel is None:
        from demo_mv1pmf_skel import mv1pmf_skel
        mv1pmf_skel(path, sub, out, mode, args)
        args.skel = join(out, 'keypoints3d')
    dataset.skel_path = args.skel
    kp3ds = []
    start, end = args.start, min(args.end, len(dataset))
    dataset.no_img = True
    annots_all = []
    for nf in tqdm(range(start, end), desc='loading'):
        images, annots = dataset[nf]
        infos = dataset.read_skel(nf)
        kp3ds.append(infos[0]['keypoints3d'])
        annots_all.append(annots)
    kp3ds = np.stack(kp3ds)
    kp3ds = check_keypoints(kp3ds, 1)
    # optimize the human shape
    with Timer('Loading {}, {}'.format(args.model, args.gender)):
        body_model = load_model(args.gender, model_type=args.model)
    params_init = init_params(nFrames=1, model_type=args.model)
    weight = load_weight_shape()
    if args.model in ['smpl', 'smplh', 'smplx']:
        # when use SMPL model, optimize the shape only with first 14 limbs
        params_shape = optimizeShape(body_model, params_init, kp3ds, weight_loss=weight, kintree=CONFIG['body15']['kintree'])
    else:
        params_shape = optimizeShape(body_model, params_init, kp3ds, weight_loss=weight, kintree=config['kintree'])
    # optimize 3D pose
    cfg = Config()
    cfg.VERBOSE = args.verbose
    cfg.MODEL = args.model
    params = init_params(nFrames=kp3ds.shape[0], model_type=args.model)
    params['shapes'] = params_shape['shapes'].copy()
    weight = load_weight_pose(args.model)
    with Timer('Optimize global RT'):
        cfg.OPT_R = True
        cfg.OPT_T = True
        params = optimizePose(body_model, params, kp3ds, weight_loss=weight, kintree=config['kintree'], cfg=cfg)
    with Timer('Optimize Pose/{} frames'.format(end-start)):
        cfg.OPT_POSE = True
        params = optimizePose(body_model, params, kp3ds, weight_loss=weight, kintree=config['kintree'], cfg=cfg)
        if args.model in ['smplh', 'smplx']:
            cfg.OPT_HAND = True
            params = optimizePose(body_model, params, kp3ds, weight_loss=weight, kintree=config['kintree'], cfg=cfg)
        if args.model == 'smplx':
            cfg.OPT_EXPR = True
            params = optimizePose(body_model, params, kp3ds, weight_loss=weight, kintree=config['kintree'], cfg=cfg)
    # TODO:optimize 2D pose
    # write out the results
    dataset.no_img = not args.vis_smpl
    for nf in tqdm(range(start, end), desc='render'):
        images, annots = dataset[nf]
        dataset.write_smpl(select_nf(params, nf-start), nf)
        if args.vis_smpl:
            vertices = body_model(return_verts=True, return_tensor=False, **select_nf(params, nf-start))
            dataset.vis_smpl(vertices=vertices, faces=body_model.faces, images=images, nf=nf, sub_vis=args.sub_vis, add_back=True)

if __name__ == "__main__":
    from mytools.cmd_loader import load_parser
    parser = load_parser()
    parser.add_argument('--skel', type=str, default=None, 
        help='path to keypoints3d')
    parser.add_argument('--vis_smpl', action='store_true')
    args = parser.parse_args()
    # print_mean_skel(args.body)
    mv1pmf_smpl(args.path, args.sub, args.out, args.body, args)