EasyMocap/easymocap/annotator/vanish_callback.py

import numpy as np
from easymocap.dataset.mirror import flipPoint2D

CONF_VANISHING_ANNOT = 2.
def clear_vanish_points(self, param):
    "remove all vanishing points"
    annots = param['annots']
    annots['vanish_line'] = [[], [], []]
    annots['vanish_point'] = [[], [], []]

def clear_body_points(self, param):
    "remove vanish lines of body"
    annots = param['annots']
    for i in range(3):
        vanish_lines = []
        for data in annots['vanish_line'][i]:
            if data[0][-1] > 1 and data[1][-1] > 1:
                vanish_lines.append(data)
        annots['vanish_line'][i] = vanish_lines
        if len(vanish_lines) > 1:
            annots['vanish_point'][i] = update_vanish_points(vanish_lines)


def calc_vanishpoint(keypoints2d, thres=0.3):
    '''
        keypoints2d: (2, N, 3)
    '''
    valid_idx = []
    for nj in range(keypoints2d.shape[1]):
        if keypoints2d[0, nj, 2] > thres and keypoints2d[1, nj, 2] > thres:
            valid_idx.append(nj)
    assert len(valid_idx) > 0, 'ATTN: cannot calculate the mirror pose'

    keypoints2d = keypoints2d[:, valid_idx]
    # weight: (N, 1)
    weight = keypoints2d[:, :, 2:].mean(axis=0)
    conf = weight.mean()
    A = np.hstack([
        keypoints2d[1, :, 1:2] - keypoints2d[0, :, 1:2],
        -(keypoints2d[1, :, 0:1] - keypoints2d[0, :, 0:1])
    ])
    b = -keypoints2d[0, :, 0:1]*(keypoints2d[1, :, 1:2] - keypoints2d[0, :, 1:2]) \
        + keypoints2d[0, :, 1:2] * (keypoints2d[1, :, 0:1] - keypoints2d[0, :, 0:1])
    b = -b
    A = A * weight
    b = b * weight
    avgInsec = np.linalg.inv(A.T @ A) @ (A.T @ b)
    result = np.zeros(3)
    result[0] = avgInsec[0, 0]
    result[1] = avgInsec[1, 0]
    result[2] = conf
    return result
    
def update_vanish_points(lines):
    vline0 = np.array(lines).transpose(1, 0, 2)
    # vline0 = np.dstack((vline0, np.ones((vline0.shape[0], vline0.shape[1], 1))))
    dim1points = vline0.copy()
    points = calc_vanishpoint(dim1points)
    return points.tolist()
    
def get_record_vanish_lines(index):
    def record_vanish_lines(self, param, **kwargs):
        "record vanish lines, X: mirror edge, Y: into mirror, Z: Up"
        annots = param['annots']
        if 'vanish_line' not in annots.keys():
            annots['vanish_line'] = [[], [], []]
        if 'vanish_point' not in annots.keys():
            annots['vanish_point'] = [[], [], []]
        start, end = param['start'], param['end']
        if start is not None and end is not None:
            annots['vanish_line'][index].append([[start[0], start[1], CONF_VANISHING_ANNOT], [end[0], end[1], CONF_VANISHING_ANNOT]])
            # 更新vanish point
            param['start'] = None
            param['end'] = None
        if len(annots['vanish_line'][index]) > 1:
            for val in annots['vanish_line'][index]:
                if len(val[0]) == 2:
                    val[0].append(CONF_VANISHING_ANNOT)
                    val[1].append(CONF_VANISHING_ANNOT)
            annots['vanish_point'][index] = update_vanish_points(annots['vanish_line'][index])
    func = record_vanish_lines
    text = ['parallel to mirror edges', 'vertical to mirror', 'vertical to ground']
    func.__doc__ = 'vanish line of ' + text[index]
    return record_vanish_lines

def vanish_point_from_body(self, param, **kwargs):
    "calculating the vanish point from human keypoints"
    annots = param['annots']
    bodies = annots['annots']
    if len(bodies) < 2:
        return 0
    assert len(bodies) == 2, 'Please make sure that there are only two bboxes!'
    kpts0 = np.array(bodies[0]['keypoints'])
    kpts1 = flipPoint2D(np.array(bodies[1]['keypoints']))
    vanish_line = annots['vanish_line'][1] # the y-dim
    MIN_CONF = 0.5
    for i in range(15):
        conf = min(kpts0[i, -1], kpts1[i, -1])
        if kpts0[i, -1] > MIN_CONF and kpts1[i, -1] > MIN_CONF:
            vanish_line.append([[kpts0[i, 0], kpts0[i, 1], conf], [kpts1[i, 0], kpts1[i, 1], conf]])
    if len(vanish_line) > 1:
        annots['vanish_point'][1] = update_vanish_points(vanish_line)

def copy_edges(self, param, **kwargs):
    "copy the static edges from previous frame"
    if self.frame == 0:
        return 0
    previous = self.previous()
    annots = param['annots']

    # copy the vanish points
    vanish_lines_pre = previous['vanish_line']
    vanish_lines = param['annots']['vanish_line']
    for i in range(3):
        vanish_lines[i] = []
        for data in vanish_lines_pre[i]:
            if data[0][-1] > 1 and data[1][-1] > 1:
                vanish_lines[i].append(data)
        if len(vanish_lines[i]) > 1:
            annots['vanish_point'][i] = update_vanish_points(vanish_lines[i])

def get_calc_intrinsic(mode='xy'):
    def calc_intrinsic(self, param, **kwargs):
        "calculating intrinsic matrix according to vanish points"
        annots = param['annots']
        if mode == 'xy':
            point0 = annots['vanish_point'][0]
            point1 = annots['vanish_point'][1]
        elif mode == 'yz':
            point0 = annots['vanish_point'][1]
            point1 = annots['vanish_point'][2]
        else:
            import ipdb; ipdb.set_trace()
        if len(point0) < 1 or len(point1) < 1:
            return 0
        vanish_point = np.stack([np.array(point0), np.array(point1)])
        K = np.eye(3)
        H = annots['height']
        W = annots['width']
        K = np.eye(3)
        K[0, 2] = W/2
        K[1, 2] = H/2
        print(vanish_point)
        vanish_point[:, 0] -= W/2
        vanish_point[:, 1] -= H/2
        print(vanish_point)
        focal = np.sqrt(-(vanish_point[0][0]*vanish_point[1][0] + vanish_point[0][1]*vanish_point[1][1]))
        
        K[0, 0] = focal
        K[1, 1] = focal
        annots['K'] = K.tolist()
        print('>>> estimated K: ')
        print(K)
    calc_intrinsic.__doc__ = 'calculate K with {}'.format(mode)
    return calc_intrinsic