From 6c8d277c9e432a5c22c2ce0cffcabe4cad023a74 Mon Sep 17 00:00:00 2001 From: shuaiqing Date: Sat, 12 Jun 2021 15:31:04 +0800 Subject: [PATCH] [vis] add filter for realtime visualization --- config/{vis => vis3d}/o3d_scene.yml | 10 +++- config/vis3d/o3d_scene_412.yml | 30 +++++++++++ doc/realtime_visualization.md | 7 ++- easymocap/assignment/criterion.py | 83 +++++++++++++++++++++++++++++ easymocap/config/baseconfig.py | 1 - easymocap/config/vis_socket.py | 14 +++-- easymocap/mytools/cmd_loader.py | 5 +- easymocap/socket/base.py | 6 ++- easymocap/socket/o3d.py | 26 ++++++++- 9 files changed, 168 insertions(+), 14 deletions(-) rename config/{vis => vis3d}/o3d_scene.yml (85%) create mode 100644 config/vis3d/o3d_scene_412.yml create mode 100644 easymocap/assignment/criterion.py diff --git a/config/vis/o3d_scene.yml b/config/vis3d/o3d_scene.yml similarity index 85% rename from config/vis/o3d_scene.yml rename to config/vis3d/o3d_scene.yml index 5973639..1fd6b8c 100644 --- a/config/vis/o3d_scene.yml +++ b/config/vis3d/o3d_scene.yml @@ -1,4 +1,4 @@ -host: 'auto' +host: '127.0.0.1' port: 9999 width: 1920 @@ -36,4 +36,10 @@ scene: yrange: 3 white: [1., 1., 1.] black: [0.,0.,0.] - two_sides: True \ No newline at end of file + two_sides: True + +range: + minr: [-100, -100, -100] + maxr: [ 100, 100, 100] + rate_inlier: 0.8 + min_conf: 0.1 \ No newline at end of file diff --git a/config/vis3d/o3d_scene_412.yml b/config/vis3d/o3d_scene_412.yml new file mode 100644 index 0000000..531531d --- /dev/null +++ b/config/vis3d/o3d_scene_412.yml @@ -0,0 +1,30 @@ +parent: "config/vis3d/o3d_scene.yml" + +scene: + "easymocap.visualize.o3dwrapper.create_bbox": + min_bound: [-3, -3, 0] + max_bound: [3, 3, 2] + flip: False + "easymocap.visualize.o3dwrapper.create_ground": + center: [0, 0, 0] + xdir: [1, 0, 0] + ydir: [0, 1, 0] + step: 1 + xrange: 3 + yrange: 3 + white: [1., 1., 1.] + black: [0.,0.,0.] + two_sides: True + +camera: + cy: 0.6 + cz: 4. + theta: -30 + +range: + minr: [-2, -1, 0] + maxr: [ 2, 2, 2.5] + rate_inlier: 0.8 + min_conf: 0.1 + +new_frames: 10 \ No newline at end of file diff --git a/doc/realtime_visualization.md b/doc/realtime_visualization.md index 7e144a2..900f05c 100644 --- a/doc/realtime_visualization.md +++ b/doc/realtime_visualization.md @@ -2,7 +2,7 @@ * @Date: 2021-06-04 15:56:55 * @Author: Qing Shuai * @LastEditors: Qing Shuai - * @LastEditTime: 2021-06-04 17:11:48 + * @LastEditTime: 2021-06-12 15:29:23 * @FilePath: /EasyMocapRelease/doc/realtime_visualization.md --> # EasyMoCap -> Real-time Visualization @@ -21,7 +21,10 @@ python3 -m pip install open3d==0.9.0 Before any visualization, you should run a server: ```bash -python3 apps/vis/vis_server.py --cfg config/vis/o3d_scene.yml host port +# quick start: +python3 apps/vis/vis_server.py --cfg config/vis3d/o3d_scene.yml +# If you want to specify the host and port: +python3 apps/vis/vis_server.py --cfg config/vis3d/o3d_scene.yml host port ``` This step will open the visualization window: diff --git a/easymocap/assignment/criterion.py b/easymocap/assignment/criterion.py new file mode 100644 index 0000000..c70957e --- /dev/null +++ b/easymocap/assignment/criterion.py @@ -0,0 +1,83 @@ +''' + @ Date: 2021-05-28 16:36:45 + @ Author: Qing Shuai + @ LastEditors: Qing Shuai + @ LastEditTime: 2021-05-30 12:21:15 + @ FilePath: /EasyMocap/easymocap/assignment/criterion.py +''' +import numpy as np + +class BaseCrit: + def __init__(self, min_conf, min_joints=3) -> None: + self.min_conf = min_conf + self.min_joints = min_joints + self.name = self.__class__.__name__ + + def __call__(self, keypoints3d, **kwargs): + # keypoints3d: (N, 4) + conf = keypoints3d[..., -1] + conf[conf self.min_conf + return len(idx) > self.min_joints + +class CritWithTorso(BaseCrit): + def __init__(self, torso_idx, min_conf, **kwargs) -> None: + super().__init__(min_conf) + self.idx = torso_idx + self.min_conf = min_conf + + def __call__(self, keypoints3d, **kwargs) -> bool: + self.log = '{}'.format(keypoints3d[self.idx, -1]) + return (keypoints3d[self.idx, -1] > self.min_conf).all() + +class CritLenTorso(BaseCrit): + def __init__(self, src, dst, min_torso_length, max_torso_length, min_conf) -> None: + super().__init__(min_conf) + self.src = src + self.dst = dst + self.min_torso_length = min_torso_length + self.max_torso_length = max_torso_length + + def __call__(self, keypoints3d, **kwargs): + """length of torso""" + # eps = 0.1 + # MIN_TORSO_LENGTH = 0.3 + # MAX_TORSO_LENGTH = 0.8 + if (keypoints3d[[self.src, self.dst], -1] < self.min_conf).all(): + # low confidence, skip + return True + length = np.linalg.norm(keypoints3d[self.dst] - keypoints3d[self.src]) + self.log = '{}: {:.3f}'.format(self.name, length) + if length < self.min_torso_length or length > self.max_torso_length: + return False + return True + +class CritRange(BaseCrit): + def __init__(self, minr, maxr, rate_inlier, min_conf) -> None: + super().__init__(min_conf) + self.min = minr + self.max = maxr + self.rate = rate_inlier + + def __call__(self, keypoints3d, **kwargs): + idx = keypoints3d[..., -1] > self.min_conf + k3d = keypoints3d[idx, :3] + crit = (k3d[:, 0] > self.min[0]) & (k3d[:, 0] < self.max[0]) &\ + (k3d[:, 1] > self.min[1]) & (k3d[:, 1] < self.max[1]) &\ + (k3d[:, 2] > self.min[2]) & (k3d[:, 2] < self.max[2]) + self.log = '{}: {}'.format(self.name, k3d) + return crit.sum()/crit.shape[0] > self.rate + +class CritMinMax(BaseCrit): + def __init__(self, max_human_length, min_conf) -> None: + super().__init__(min_conf) + self.max_human_length = max_human_length + + def __call__(self, keypoints3d, **kwargs): + idx = keypoints3d[..., -1] > self.min_conf + k3d = keypoints3d[idx, :3] + mink = np.min(k3d, axis=0) + maxk = np.max(k3d, axis=0) + length = max(np.abs(maxk - mink)) + self.log = '{}: {:.3f}'.format(self.name, length) + return length < self.max_human_length \ No newline at end of file diff --git a/easymocap/config/baseconfig.py b/easymocap/config/baseconfig.py index 0cbb601..5f6e2f9 100644 --- a/easymocap/config/baseconfig.py +++ b/easymocap/config/baseconfig.py @@ -47,7 +47,6 @@ class Config: import importlib def load_object(module_name, module_args): module_path = '.'.join(module_name.split('.')[:-1]) - # scene_module = importlib.import_module(cfg.scene_module) module = importlib.import_module(module_path) name = module_name.split('.')[-1] obj = getattr(module, name)(**module_args) diff --git a/easymocap/config/vis_socket.py b/easymocap/config/vis_socket.py index 8cfed19..fd521ce 100644 --- a/easymocap/config/vis_socket.py +++ b/easymocap/config/vis_socket.py @@ -2,8 +2,8 @@ @ Date: 2021-05-30 11:17:18 @ Author: Qing Shuai @ LastEditors: Qing Shuai - @ LastEditTime: 2021-06-04 15:44:56 - @ FilePath: /EasyMocap/easymocap/config/vis_socket.py + @ LastEditTime: 2021-06-12 14:56:00 + @ FilePath: /EasyMocapRelease/easymocap/config/vis_socket.py ''' from .baseconfig import CN from .baseconfig import Config as BaseConfig @@ -19,7 +19,6 @@ class Config(BaseConfig): cfg.width = 1920 cfg.height = 1080 - cfg.body = 'body25' cfg.max_human = 5 cfg.track = True cfg.block = True # block visualization or not, True for visualize each frame, False in realtime applications @@ -30,6 +29,9 @@ class Config(BaseConfig): cfg.scene_module = "easymocap.visualize.o3dwrapper" cfg.scene = CN() cfg.extra = CN() + cfg.range = CN() + cfg.new_frames = 0 + # skel cfg.skel = CN() cfg.skel.joint_radius = 0.02 @@ -42,6 +44,12 @@ class Config(BaseConfig): cfg.camera.cz = 6. cfg.camera.set_camera = False cfg.camera.camera_pose = [] + # range + cfg.range = CN() + cfg.range.minr = [-100, -100, -10] + cfg.range.maxr = [ 100, 100, 10] + cfg.range.rate_inlier = 0.8 + cfg.range.min_conf = 0.1 return cfg @staticmethod diff --git a/easymocap/mytools/cmd_loader.py b/easymocap/mytools/cmd_loader.py index b0eb5ea..47d06fb 100644 --- a/easymocap/mytools/cmd_loader.py +++ b/easymocap/mytools/cmd_loader.py @@ -2,8 +2,8 @@ @ Date: 2021-01-15 12:09:27 @ Author: Qing Shuai @ LastEditors: Qing Shuai - @ LastEditTime: 2021-05-27 20:36:42 - @ FilePath: /EasyMocapRelease/easymocap/mytools/cmd_loader.py + @ LastEditTime: 2021-06-09 19:52:41 + @ FilePath: /EasyMocap/easymocap/mytools/cmd_loader.py ''' import os import argparse @@ -12,6 +12,7 @@ def load_parser(): parser = argparse.ArgumentParser('EasyMocap commond line tools') parser.add_argument('path', type=str) parser.add_argument('--out', type=str, default=None) + parser.add_argument('--cfg', type=str, default=None) parser.add_argument('--camera', type=str, default=None) parser.add_argument('--annot', type=str, default='annots', help="sub directory name to store the generated annotation files, default to be annots") parser.add_argument('--sub', type=str, nargs='+', default=[], diff --git a/easymocap/socket/base.py b/easymocap/socket/base.py index dd0dd77..f732d10 100644 --- a/easymocap/socket/base.py +++ b/easymocap/socket/base.py @@ -2,7 +2,7 @@ @ Date: 2021-05-25 11:14:48 @ Author: Qing Shuai @ LastEditors: Qing Shuai - @ LastEditTime: 2021-06-02 13:00:35 + @ LastEditTime: 2021-06-05 19:32:56 @ FilePath: /EasyMocap/easymocap/socket/base.py ''' import socket @@ -24,9 +24,9 @@ class BaseSocket: serversocket.bind((host, port)) serversocket.listen(1) self.serversocket = serversocket + self.queue = Queue() self.t = Thread(target=self.run) self.t.start() - self.queue = Queue() self.debug = debug self.disconnect = False @@ -55,10 +55,12 @@ class BaseSocket: while True: clientsocket, addr = self.serversocket.accept() print("[Info] Connect: %s" % str(addr)) + self.disconnect = False while True: flag, l = self.recvLine(clientsocket) if not flag: print("[Info] Disonnect: %s" % str(addr)) + self.disconnect = True break data = self.recvAll(clientsocket, l) if self.debug:log('[Info] Recv data') diff --git a/easymocap/socket/o3d.py b/easymocap/socket/o3d.py index 52cfb15..919d4ef 100644 --- a/easymocap/socket/o3d.py +++ b/easymocap/socket/o3d.py @@ -2,7 +2,7 @@ @ Date: 2021-05-25 11:15:53 @ Author: Qing Shuai @ LastEditors: Qing Shuai - @ LastEditTime: 2021-06-04 17:06:17 + @ LastEditTime: 2021-06-12 14:54:43 @ FilePath: /EasyMocapRelease/easymocap/socket/o3d.py ''' import open3d as o3d @@ -15,6 +15,7 @@ import json import numpy as np from os.path import join import os +from ..assignment.criterion import CritRange class VisOpen3DSocket(BaseSocket): def __init__(self, host, port, cfg) -> None: @@ -53,6 +54,9 @@ class VisOpen3DSocket(BaseSocket): self.add_human(zero_params) self.count = 0 + self.previous = {} + self.critrange = CritRange(**cfg.range) + self.new_frames = cfg.new_frames def add_human(self, zero_params): vertices = self.body_model(zero_params)[0] @@ -69,15 +73,31 @@ class VisOpen3DSocket(BaseSocket): init_param.extrinsic = np.array(camera_pose) ctr.convert_from_pinhole_camera_parameters(init_param) + def filter_human(self, datas): + datas_new = [] + for data in datas: + kpts3d = np.array(data['keypoints3d']) + data['keypoints3d'] = kpts3d + pid = data['id'] + if pid not in self.previous.keys(): + if not self.critrange(kpts3d): + continue + self.previous[pid] = 0 + self.previous[pid] += 1 + if self.previous[pid] > self.new_frames: + datas_new.append(data) + return datas_new + def main(self, datas): if self.debug:log('[Info] Load data {}'.format(self.count)) datas = json.loads(datas) + datas = self.filter_human(datas) with Timer('forward'): for i, data in enumerate(datas): if i >= len(self.meshes): print('[Error] the number of human exceeds!') self.add_human(np.array(data['keypoints3d'])) - vertices = self.body_model(np.array(data['keypoints3d'])) + vertices = self.body_model(data['keypoints3d']) self.vertices[i] = Vector3dVector(vertices[0]) for i in range(len(datas), len(self.meshes)): self.vertices[i] = self.zero_vertices @@ -90,6 +110,8 @@ class VisOpen3DSocket(BaseSocket): self.meshes[i].paint_uniform_color(col) def update(self): + if self.disconnect and not self.block: + self.previous.clear() if not self.queue.empty(): if self.debug:log('Update' + str(self.queue.qsize())) datas = self.queue.get()