[vis] add filter for realtime visualization

config/vis3d/o3d_scene.yml

@@ -1,4 +1,4 @@
-host: 'auto'
+host: '127.0.0.1'
 port: 9999
 
 width: 1920
@@ -36,4 +36,10 @@ scene:
     yrange: 3
     white: [1., 1., 1.]
     black: [0.,0.,0.]
-    two_sides: True
+    two_sides: True
+
+range:
+  minr: [-100, -100, -100]
+  maxr: [ 100,  100,  100]
+  rate_inlier: 0.8
+  min_conf: 0.1

config/vis3d/o3d_scene_412.yml

@@ -0,0 +1,30 @@
+parent: "config/vis3d/o3d_scene.yml"
+
+scene:
+  "easymocap.visualize.o3dwrapper.create_bbox":
+    min_bound: [-3, -3, 0]
+    max_bound: [3, 3, 2]
+    flip: False
+  "easymocap.visualize.o3dwrapper.create_ground":
+    center: [0, 0, 0]
+    xdir: [1, 0, 0]
+    ydir: [0, 1, 0]
+    step: 1
+    xrange: 3
+    yrange: 3
+    white: [1., 1., 1.]
+    black: [0.,0.,0.]
+    two_sides: True
+
+camera:
+  cy: 0.6
+  cz: 4.
+  theta: -30
+
+range:
+  minr: [-2, -1,   0]
+  maxr: [ 2,  2, 2.5]
+  rate_inlier: 0.8
+  min_conf: 0.1
+
+new_frames: 10

doc/realtime_visualization.md

@@ -2,7 +2,7 @@
  * @Date: 2021-06-04 15:56:55
  * @Author: Qing Shuai
  * @LastEditors: Qing Shuai
- * @LastEditTime: 2021-06-04 17:11:48
+ * @LastEditTime: 2021-06-12 15:29:23
  * @FilePath: /EasyMocapRelease/doc/realtime_visualization.md
 -->
 # EasyMoCap -> Real-time Visualization
@@ -21,7 +21,10 @@ python3 -m pip install open3d==0.9.0
 Before any visualization, you should run a server:
 
 ```bash
-python3 apps/vis/vis_server.py --cfg config/vis/o3d_scene.yml host <your_ip_address> port <set_a_port>
+# quick start:
+python3 apps/vis/vis_server.py --cfg config/vis3d/o3d_scene.yml
+# If you want to specify the host and port:
+python3 apps/vis/vis_server.py --cfg config/vis3d/o3d_scene.yml host <your_ip_address> port <set_a_port>
 ```
 
 This step will open the visualization window:

easymocap/assignment/criterion.py

@@ -0,0 +1,83 @@
+'''
+  @ Date: 2021-05-28 16:36:45
+  @ Author: Qing Shuai
+  @ LastEditors: Qing Shuai
+  @ LastEditTime: 2021-05-30 12:21:15
+  @ FilePath: /EasyMocap/easymocap/assignment/criterion.py
+'''
+import numpy as np
+
+class BaseCrit:
+    def __init__(self, min_conf, min_joints=3) -> None:
+        self.min_conf = min_conf
+        self.min_joints = min_joints
+        self.name = self.__class__.__name__
+
+    def __call__(self, keypoints3d, **kwargs):
+        # keypoints3d: (N, 4)
+        conf = keypoints3d[..., -1]
+        conf[conf<self.min_conf] = 0
+        idx = keypoints3d[..., -1] > self.min_conf
+        return len(idx) > self.min_joints
+
+class CritWithTorso(BaseCrit):
+    def __init__(self, torso_idx, min_conf, **kwargs) -> None:
+        super().__init__(min_conf)
+        self.idx = torso_idx
+        self.min_conf = min_conf
+    
+    def __call__(self, keypoints3d, **kwargs) -> bool:
+        self.log = '{}'.format(keypoints3d[self.idx, -1])
+        return (keypoints3d[self.idx, -1] > self.min_conf).all()
+
+class CritLenTorso(BaseCrit):
+    def __init__(self, src, dst, min_torso_length, max_torso_length, min_conf) -> None:
+        super().__init__(min_conf)
+        self.src = src
+        self.dst = dst
+        self.min_torso_length = min_torso_length
+        self.max_torso_length = max_torso_length
+    
+    def __call__(self, keypoints3d, **kwargs):
+        """length of torso"""
+        # eps = 0.1
+        # MIN_TORSO_LENGTH = 0.3
+        # MAX_TORSO_LENGTH = 0.8
+        if (keypoints3d[[self.src, self.dst], -1] < self.min_conf).all():
+            # low confidence, skip
+            return True
+        length = np.linalg.norm(keypoints3d[self.dst] - keypoints3d[self.src])
+        self.log = '{}: {:.3f}'.format(self.name, length)
+        if length < self.min_torso_length or length > self.max_torso_length:
+            return False
+        return True
+
+class CritRange(BaseCrit):
+    def __init__(self, minr, maxr, rate_inlier, min_conf) -> None:
+        super().__init__(min_conf)
+        self.min = minr
+        self.max = maxr
+        self.rate = rate_inlier
+    
+    def __call__(self, keypoints3d, **kwargs):
+        idx = keypoints3d[..., -1] > self.min_conf
+        k3d = keypoints3d[idx, :3]
+        crit = (k3d[:, 0] > self.min[0]) & (k3d[:, 0] < self.max[0]) &\
+        (k3d[:, 1] > self.min[1]) & (k3d[:, 1] < self.max[1]) &\
+        (k3d[:, 2] > self.min[2]) & (k3d[:, 2] < self.max[2])
+        self.log = '{}: {}'.format(self.name, k3d)
+        return crit.sum()/crit.shape[0] > self.rate
+
+class CritMinMax(BaseCrit):  
+    def __init__(self, max_human_length, min_conf) -> None:
+        super().__init__(min_conf)
+        self.max_human_length = max_human_length
+
+    def __call__(self, keypoints3d, **kwargs):
+        idx = keypoints3d[..., -1] > self.min_conf
+        k3d = keypoints3d[idx, :3]
+        mink = np.min(k3d, axis=0)
+        maxk = np.max(k3d, axis=0)
+        length = max(np.abs(maxk - mink))
+        self.log = '{}: {:.3f}'.format(self.name, length)
+        return length < self.max_human_length

easymocap/config/baseconfig.py

@@ -47,7 +47,6 @@ class Config:
 import importlib
 def load_object(module_name, module_args):
     module_path = '.'.join(module_name.split('.')[:-1])
-    # scene_module = importlib.import_module(cfg.scene_module)
     module = importlib.import_module(module_path)
     name = module_name.split('.')[-1]
     obj = getattr(module, name)(**module_args)

easymocap/config/vis_socket.py

@@ -2,8 +2,8 @@
   @ Date: 2021-05-30 11:17:18
   @ Author: Qing Shuai
   @ LastEditors: Qing Shuai
-  @ LastEditTime: 2021-06-04 15:44:56
-  @ FilePath: /EasyMocap/easymocap/config/vis_socket.py
+  @ LastEditTime: 2021-06-12 14:56:00
+  @ FilePath: /EasyMocapRelease/easymocap/config/vis_socket.py
 '''
 from .baseconfig import CN
 from .baseconfig import Config as BaseConfig
@@ -19,7 +19,6 @@ class Config(BaseConfig):
         cfg.width = 1920
         cfg.height = 1080
         
-        cfg.body = 'body25'
         cfg.max_human = 5
         cfg.track = True
         cfg.block = True # block visualization or not, True for visualize each frame, False in realtime applications
@@ -30,6 +29,9 @@ class Config(BaseConfig):
         cfg.scene_module = "easymocap.visualize.o3dwrapper"
         cfg.scene = CN()
         cfg.extra = CN()
+        cfg.range = CN()
+        cfg.new_frames = 0
+
         # skel
         cfg.skel = CN()
         cfg.skel.joint_radius = 0.02
@@ -42,6 +44,12 @@ class Config(BaseConfig):
         cfg.camera.cz = 6.
         cfg.camera.set_camera = False
         cfg.camera.camera_pose = []
+        # range
+        cfg.range = CN()
+        cfg.range.minr = [-100, -100, -10]
+        cfg.range.maxr = [ 100,  100,  10]
+        cfg.range.rate_inlier = 0.8
+        cfg.range.min_conf = 0.1
         return cfg
     
     @staticmethod

easymocap/mytools/cmd_loader.py

@@ -2,8 +2,8 @@
   @ Date: 2021-01-15 12:09:27
   @ Author: Qing Shuai
   @ LastEditors: Qing Shuai
-  @ LastEditTime: 2021-05-27 20:36:42
-  @ FilePath: /EasyMocapRelease/easymocap/mytools/cmd_loader.py
+  @ LastEditTime: 2021-06-09 19:52:41
+  @ FilePath: /EasyMocap/easymocap/mytools/cmd_loader.py
 '''
 import os
 import argparse
@@ -12,6 +12,7 @@ def load_parser():
     parser = argparse.ArgumentParser('EasyMocap commond line tools')
     parser.add_argument('path', type=str)
     parser.add_argument('--out', type=str, default=None)
+    parser.add_argument('--cfg', type=str, default=None)
     parser.add_argument('--camera', type=str, default=None)
     parser.add_argument('--annot', type=str, default='annots', help="sub directory name to store the generated annotation files, default to be annots")
     parser.add_argument('--sub', type=str, nargs='+', default=[],

easymocap/socket/base.py

@@ -2,7 +2,7 @@
   @ Date: 2021-05-25 11:14:48
   @ Author: Qing Shuai
   @ LastEditors: Qing Shuai
-  @ LastEditTime: 2021-06-02 13:00:35
+  @ LastEditTime: 2021-06-05 19:32:56
   @ FilePath: /EasyMocap/easymocap/socket/base.py
 '''
 import socket
@@ -24,9 +24,9 @@ class BaseSocket:
         serversocket.bind((host, port))
         serversocket.listen(1)
         self.serversocket = serversocket
+        self.queue = Queue()
         self.t = Thread(target=self.run)
         self.t.start()
-        self.queue = Queue()
         self.debug = debug
         self.disconnect = False
     
@@ -55,10 +55,12 @@ class BaseSocket:
         while True:
             clientsocket, addr = self.serversocket.accept()
             print("[Info] Connect: %s" % str(addr))
+            self.disconnect = False
             while True:
                 flag, l = self.recvLine(clientsocket)
                 if not flag:
                     print("[Info] Disonnect: %s" % str(addr))
+                    self.disconnect = True
                     break
                 data = self.recvAll(clientsocket, l)
                 if self.debug:log('[Info] Recv data')

easymocap/socket/o3d.py

@@ -2,7 +2,7 @@
   @ Date: 2021-05-25 11:15:53
   @ Author: Qing Shuai
   @ LastEditors: Qing Shuai
-  @ LastEditTime: 2021-06-04 17:06:17
+  @ LastEditTime: 2021-06-12 14:54:43
   @ FilePath: /EasyMocapRelease/easymocap/socket/o3d.py
 '''
 import open3d as o3d
@@ -15,6 +15,7 @@ import json
 import numpy as np
 from os.path import join
 import os
+from ..assignment.criterion import CritRange
 
 class VisOpen3DSocket(BaseSocket):
     def __init__(self, host, port, cfg) -> None:
@@ -53,6 +54,9 @@ class VisOpen3DSocket(BaseSocket):
             self.add_human(zero_params)
 
         self.count = 0
+        self.previous = {}
+        self.critrange = CritRange(**cfg.range)
+        self.new_frames  = cfg.new_frames
     
     def add_human(self, zero_params):
         vertices = self.body_model(zero_params)[0]
@@ -69,15 +73,31 @@ class VisOpen3DSocket(BaseSocket):
         init_param.extrinsic = np.array(camera_pose)
         ctr.convert_from_pinhole_camera_parameters(init_param) 
 
+    def filter_human(self, datas):
+        datas_new = []
+        for data in datas:
+            kpts3d = np.array(data['keypoints3d'])
+            data['keypoints3d'] = kpts3d
+            pid = data['id']
+            if pid not in self.previous.keys():
+                if not self.critrange(kpts3d):
+                    continue
+                self.previous[pid] = 0
+            self.previous[pid] += 1
+            if self.previous[pid] > self.new_frames:
+                datas_new.append(data)
+        return datas_new
+
     def main(self, datas):
         if self.debug:log('[Info] Load data {}'.format(self.count))
         datas = json.loads(datas)
+        datas = self.filter_human(datas)
         with Timer('forward'):
             for i, data in enumerate(datas):
                 if i >= len(self.meshes):
                     print('[Error] the number of human exceeds!')
                     self.add_human(np.array(data['keypoints3d']))
-                vertices = self.body_model(np.array(data['keypoints3d']))
+                vertices = self.body_model(data['keypoints3d'])
                 self.vertices[i] = Vector3dVector(vertices[0])
             for i in range(len(datas), len(self.meshes)):
                 self.vertices[i] = self.zero_vertices
@@ -90,6 +110,8 @@ class VisOpen3DSocket(BaseSocket):
                     self.meshes[i].paint_uniform_color(col)
 
     def update(self):
+        if self.disconnect and not self.block:
+            self.previous.clear()
         if not self.queue.empty():
             if self.debug:log('Update' + str(self.queue.qsize()))
             datas = self.queue.get()