EasyMocap/config/1v1p/hand_detect_finetune.yml

smooth: &smooth_keypoints
  weight: 50.
  module: myeasymocap.operations.loss.Smooth
  key_from_output: [keypoints, Th]
  key_from_infos: [] # TODO: 根据2D的置信度来计算smooth权重
  args:
    keys: [Th, keypoints]
    smooth_type: [Linear, Linear] # 这个depth似乎需要相机参数进行转换
    norm: [l2, l2]
    order: [2, 2]
    weights: [1000., 1000.]
    window_weight: [0.5, 0.3, 0.1, 0.1]

module: myeasymocap.stages.basestage.MultiStage
args:
  output: output/sv1p_hand
  at_step:
    detect_by_mediapipe:
      module: myeasymocap.backbone.mediapipe.hand.MediaPipe
      key_from_data: [images, imgnames]
      args:
        ckpt: models/mediapipe/hand_landmarker.task
    hand2d: 
      module: myeasymocap.backbone.hand2d.hand2d.MyHand2D
      key_from_data: [images, imgnames]
      key_from_previous: [bbox]
      args:
        # ckpt: /nas/public/EasyMocapModels/hrnetv2_w18_coco_wholebody_hand_256x256-1c028db7_20210908.pth
        ckpt: /nas/public/EasyMocapModels/hand/resnet_kp2d_clean.pt
        mode: resnet
    vis2d:
      module: myeasymocap.io.vis.Vis2D
      skip: False
      key_from_data: [images]
      key_from_previous: [keypoints, bbox]
      args:
        name: vis_keypoints2d
        scale: 0.5
    infer_mano: # 
      module: myeasymocap.backbone.hmr.hmr.MyHMR
      key_from_data: [images, imgnames]
      key_from_previous: [bbox]
      key_keep: [meta, cameras, imgnames] # 将这些参数都保留到最后的输出中
      args:
        ckpt: models/manol_pca45_noflat.ckpt
    # TODO: add visualize for Init MANO
  at_final:
    load_hand_model: # 载入身体模型
      module: myeasymocap.io.model.MANOLoader
      args:
        cfg_path: config/model/mano.yml
        model_path: models/manov1.2/MANO_LEFT.pkl #models/handmesh/data/MANO_RIGHT.pkl # load mano model
        regressor_path: models/manov1.2/J_regressor_mano_LEFT.txt #models/handmesh/data/J_regressor_mano_RIGHT.txt
        num_pca_comps: 45
        use_pca: True
        use_flat_mean: False
      # 这个模块返回两个内容：body_model, model; 其中的body_model是用来进行可视化的
    mean_param: # 初始化姿态，这里将poses和shapes都进行平均
      module: myeasymocap.operations.init.MeanShapes
      key_from_data: [params]
      args:
        keys: ['shapes']
    init_T:  # 初始化每一帧的位置
      module: myeasymocap.operations.optimizer.Optimizer
      key_from_data: [keypoints, cameras, params]
      key_from_previous: [model]
      args:
        optimizer_args: {optim_type: lbfgs}
        optimize_keys: [Th]
        loss:
          repro:
            weight: 100.
            module: myeasymocap.operations.loss.Keypoints2D
            key_from_output: [keypoints]
            key_from_infos: [keypoints, cameras]
            args:
              norm: l2
          smooth: *smooth_keypoints
    init_R:  # 初始化每一帧的旋转
      module: myeasymocap.operations.optimizer.Optimizer
      key_from_data: [keypoints, cameras]
      key_from_previous: [model, params]
      args:
        optimizer_args: {optim_type: lbfgs}
        optimize_keys: [Rh]
        loss:
          repro:
            weight: 100.
            module: myeasymocap.operations.loss.Keypoints2D
            key_from_output: [keypoints]
            key_from_infos: [keypoints, cameras]
            args:
              norm: l2
          smooth: *smooth_keypoints
    refine_poses: # 优化poses
      repeat: 2
      module: myeasymocap.operations.optimizer.Optimizer
      key_from_data: [keypoints, cameras]
      key_from_previous: [model, params]
      args:
        optimizer_args: {optim_type: lbfgs}
        optimize_keys: [poses, shapes, Rh, Th]
        loss:
          repro:
            weight: 100.
            module: myeasymocap.operations.loss.Keypoints2D
            key_from_output: [keypoints]
            key_from_infos: [keypoints, cameras]
            args:
              norm: l1
          reg:
            weight: 0.001
            module: myeasymocap.operations.loss.RegLoss
            key_from_output: [poses]
            key_from_infos: []
            args:
              key: poses
              norm: l2
          smooth: *smooth_keypoints
    write:
      module: myeasymocap.io.write.WriteSMPL
      key_from_data: [meta]
      key_from_previous: [params, model]
      args:
        name: smpl
    render:
      module: myeasymocap.io.vis3d.Render_multiview
      key_from_data: [cameras, imgnames]
      key_from_previous: [hand_model, params]
      args:
        model_name: hand_model
        backend: pyrender
        view_list: [0]
        scale: 0.5
    make_video:
      module: myeasymocap.io.video.MakeVideo
      args:
        fps: 50
        keep_image: False