265 lines
9.5 KiB
Python
265 lines
9.5 KiB
Python
|
import cv2
|
||
|
|
import numpy as np
|
||
|
|
from tqdm import tqdm
|
||
|
|
import os
|
||
|
|
|
||
|
|
class FileStorage(object):
|
||
|
|
def __init__(self, filename, isWrite=False):
|
||
|
|
version = cv2.__version__
|
||
|
|
self.major_version = int(version.split('.')[0])
|
||
|
|
self.second_version = int(version.split('.')[1])
|
||
|
|
|
||
|
|
if isWrite:
|
||
|
|
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||
|
|
self.fs = cv2.FileStorage(filename, cv2.FILE_STORAGE_WRITE)
|
||
|
|
else:
|
||
|
|
self.fs = cv2.FileStorage(filename, cv2.FILE_STORAGE_READ)
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
cv2.FileStorage.release(self.fs)
|
||
|
|
|
||
|
|
def write(self, key, value, dt='mat'):
|
||
|
|
if dt == 'mat':
|
||
|
|
cv2.FileStorage.write(self.fs, key, value)
|
||
|
|
elif dt == 'list':
|
||
|
|
if self.major_version == 4: # 4.4
|
||
|
|
self.fs.startWriteStruct(key, cv2.FileNode_SEQ)
|
||
|
|
for elem in value:
|
||
|
|
self.fs.write('', elem)
|
||
|
|
self.fs.endWriteStruct()
|
||
|
|
else: # 3.4
|
||
|
|
self.fs.write(key, '[')
|
||
|
|
for elem in value:
|
||
|
|
self.fs.write('none', elem)
|
||
|
|
self.fs.write('none', ']')
|
||
|
|
|
||
|
|
def read(self, key, dt='mat'):
|
||
|
|
if dt == 'mat':
|
||
|
|
output = self.fs.getNode(key).mat()
|
||
|
|
elif dt == 'list':
|
||
|
|
results = []
|
||
|
|
n = self.fs.getNode(key)
|
||
|
|
for i in range(n.size()):
|
||
|
|
val = n.at(i).string()
|
||
|
|
if val == '':
|
||
|
|
val = str(int(n.at(i).real()))
|
||
|
|
if val != 'none':
|
||
|
|
results.append(val)
|
||
|
|
output = results
|
||
|
|
else:
|
||
|
|
raise NotImplementedError
|
||
|
|
return output
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self.__del__(self)
|
||
|
|
|
||
|
|
def safe_mkdir(path):
|
||
|
|
if not os.path.exists(path):
|
||
|
|
os.makedirs(path)
|
||
|
|
|
||
|
|
def read_intri(intri_name):
|
||
|
|
assert os.path.exists(intri_name), intri_name
|
||
|
|
intri = FileStorage(intri_name)
|
||
|
|
camnames = intri.read('names', dt='list')
|
||
|
|
cameras = {}
|
||
|
|
for key in camnames:
|
||
|
|
cam = {}
|
||
|
|
cam['K'] = intri.read('K_{}'.format(key))
|
||
|
|
cam['invK'] = np.linalg.inv(cam['K'])
|
||
|
|
cam['dist'] = intri.read('dist_{}'.format(key))
|
||
|
|
cameras[key] = cam
|
||
|
|
return cameras
|
||
|
|
|
||
|
|
def write_intri(intri_name, cameras):
|
||
|
|
intri = FileStorage(intri_name, True)
|
||
|
|
results = {}
|
||
|
|
camnames = list(cameras.keys())
|
||
|
|
intri.write('names', camnames, 'list')
|
||
|
|
for key_, val in cameras.items():
|
||
|
|
key = key_.split('.')[0]
|
||
|
|
K, dist = val['K'], val['dist']
|
||
|
|
assert K.shape == (3, 3), K.shape
|
||
|
|
assert dist.shape == (1, 5) or dist.shape == (5, 1), dist.shape
|
||
|
|
intri.write('K_{}'.format(key), K)
|
||
|
|
intri.write('dist_{}'.format(key), dist.reshape(1, 5))
|
||
|
|
|
||
|
|
def write_extri(extri_name, cameras):
|
||
|
|
extri = FileStorage(extri_name, True)
|
||
|
|
results = {}
|
||
|
|
camnames = list(cameras.keys())
|
||
|
|
extri.write('names', camnames, 'list')
|
||
|
|
for key_, val in cameras.items():
|
||
|
|
key = key_.split('.')[0]
|
||
|
|
extri.write('R_{}'.format(key), val['Rvec'])
|
||
|
|
extri.write('Rot_{}'.format(key), val['R'])
|
||
|
|
extri.write('T_{}'.format(key), val['T'])
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def read_camera(intri_name, extri_name, cam_names=[]):
|
||
|
|
assert os.path.exists(intri_name), intri_name
|
||
|
|
assert os.path.exists(extri_name), extri_name
|
||
|
|
|
||
|
|
intri = FileStorage(intri_name)
|
||
|
|
extri = FileStorage(extri_name)
|
||
|
|
cams, P = {}, {}
|
||
|
|
cam_names = intri.read('names', dt='list')
|
||
|
|
for cam in cam_names:
|
||
|
|
# 内参只读子码流的
|
||
|
|
cams[cam] = {}
|
||
|
|
cams[cam]['K'] = intri.read('K_{}'.format( cam))
|
||
|
|
cams[cam]['invK'] = np.linalg.inv(cams[cam]['K'])
|
||
|
|
Rvec = extri.read('R_{}'.format(cam))
|
||
|
|
Tvec = extri.read('T_{}'.format(cam))
|
||
|
|
R = cv2.Rodrigues(Rvec)[0]
|
||
|
|
RT = np.hstack((R, Tvec))
|
||
|
|
|
||
|
|
cams[cam]['RT'] = RT
|
||
|
|
cams[cam]['R'] = R
|
||
|
|
cams[cam]['T'] = Tvec
|
||
|
|
P[cam] = cams[cam]['K'] @ cams[cam]['RT']
|
||
|
|
cams[cam]['P'] = P[cam]
|
||
|
|
|
||
|
|
cams[cam]['dist'] = intri.read('dist_{}'.format(cam))
|
||
|
|
cams['basenames'] = cam_names
|
||
|
|
return cams
|
||
|
|
|
||
|
|
def write_camera(camera, path):
|
||
|
|
from os.path import join
|
||
|
|
intri_name = join(path, 'intri.yml')
|
||
|
|
extri_name = join(path, 'extri.yml')
|
||
|
|
intri = FileStorage(intri_name, True)
|
||
|
|
extri = FileStorage(extri_name, True)
|
||
|
|
results = {}
|
||
|
|
camnames = [key_.split('.')[0] for key_ in camera.keys()]
|
||
|
|
intri.write('names', camnames, 'list')
|
||
|
|
extri.write('names', camnames, 'list')
|
||
|
|
for key_, val in camera.items():
|
||
|
|
if key_ == 'basenames':
|
||
|
|
continue
|
||
|
|
key = key_.split('.')[0]
|
||
|
|
intri.write('K_{}'.format(key), val['K'])
|
||
|
|
intri.write('dist_{}'.format(key), val['dist'])
|
||
|
|
if 'Rvec' not in val.keys():
|
||
|
|
val['Rvec'] = cv2.Rodrigues(val['R'])[0]
|
||
|
|
extri.write('R_{}'.format(key), val['Rvec'])
|
||
|
|
extri.write('Rot_{}'.format(key), val['R'])
|
||
|
|
extri.write('T_{}'.format(key), val['T'])
|
||
|
|
|
||
|
|
class Undistort:
|
||
|
|
@staticmethod
|
||
|
|
def image(frame, K, dist):
|
||
|
|
return cv2.undistort(frame, K, dist, None)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def points(keypoints, K, dist):
|
||
|
|
# keypoints: (N, 3)
|
||
|
|
assert len(keypoints.shape) == 2, keypoints.shape
|
||
|
|
kpts = keypoints[:, None, :2]
|
||
|
|
kpts = np.ascontiguousarray(kpts)
|
||
|
|
kpts = cv2.undistortPoints(kpts, K, dist, P=K)
|
||
|
|
keypoints[:, :2] = kpts[:, 0]
|
||
|
|
return keypoints
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def bbox(bbox, K, dist):
|
||
|
|
keypoints = np.array([[bbox[0], bbox[1], 1], [bbox[2], bbox[3], 1]])
|
||
|
|
kpts = Undistort.points(keypoints, K, dist)
|
||
|
|
bbox = np.array([kpts[0, 0], kpts[0, 1], kpts[1, 0], kpts[1, 1], bbox[4]])
|
||
|
|
return bbox
|
||
|
|
|
||
|
|
def undistort(camera, frame=None, keypoints=None, output=None, bbox=None):
|
||
|
|
# bbox: 1, 7
|
||
|
|
mtx = camera['K']
|
||
|
|
dist = camera['dist']
|
||
|
|
if frame is not None:
|
||
|
|
frame = cv2.undistort(frame, mtx, dist, None)
|
||
|
|
if output is not None:
|
||
|
|
output = cv2.undistort(output, mtx, dist, None)
|
||
|
|
if keypoints is not None:
|
||
|
|
for nP in range(keypoints.shape[0]):
|
||
|
|
kpts = keypoints[nP][:, None, :2]
|
||
|
|
kpts = np.ascontiguousarray(kpts)
|
||
|
|
kpts = cv2.undistortPoints(kpts, mtx, dist, P=mtx)
|
||
|
|
keypoints[nP, :, :2] = kpts[:, 0]
|
||
|
|
if bbox is not None:
|
||
|
|
kpts = np.zeros((2, 1, 2))
|
||
|
|
kpts[0, 0, 0] = bbox[0]
|
||
|
|
kpts[0, 0, 1] = bbox[1]
|
||
|
|
kpts[1, 0, 0] = bbox[2]
|
||
|
|
kpts[1, 0, 1] = bbox[3]
|
||
|
|
kpts = cv2.undistortPoints(kpts, mtx, dist, P=mtx)
|
||
|
|
bbox[0] = kpts[0, 0, 0]
|
||
|
|
bbox[1] = kpts[0, 0, 1]
|
||
|
|
bbox[2] = kpts[1, 0, 0]
|
||
|
|
bbox[3] = kpts[1, 0, 1]
|
||
|
|
return bbox
|
||
|
|
return frame, keypoints, output
|
||
|
|
|
||
|
|
def get_bbox(points_set, H, W, thres=0.1, scale=1.2):
|
||
|
|
bboxes = np.zeros((points_set.shape[0], 6))
|
||
|
|
for iv in range(points_set.shape[0]):
|
||
|
|
pose = points_set[iv, :, :]
|
||
|
|
use_idx = pose[:,2] > thres
|
||
|
|
if np.sum(use_idx) < 1:
|
||
|
|
continue
|
||
|
|
ll, rr = np.min(pose[use_idx, 0]), np.max(pose[use_idx, 0])
|
||
|
|
bb, tt = np.min(pose[use_idx, 1]), np.max(pose[use_idx, 1])
|
||
|
|
center = (int((ll + rr) / 2), int((bb + tt) / 2))
|
||
|
|
length = [int(scale*(rr-ll)/2), int(scale*(tt-bb)/2)]
|
||
|
|
l = max(0, center[0] - length[0])
|
||
|
|
r = min(W, center[0] + length[0]) # img.shape[1]
|
||
|
|
b = max(0, center[1] - length[1])
|
||
|
|
t = min(H, center[1] + length[1]) # img.shape[0]
|
||
|
|
conf = pose[:, 2].mean()
|
||
|
|
cls_conf = pose[use_idx, 2].mean()
|
||
|
|
bboxes[iv, 0] = l
|
||
|
|
bboxes[iv, 1] = r
|
||
|
|
bboxes[iv, 2] = b
|
||
|
|
bboxes[iv, 3] = t
|
||
|
|
bboxes[iv, 4] = conf
|
||
|
|
bboxes[iv, 5] = cls_conf
|
||
|
|
return bboxes
|
||
|
|
|
||
|
|
def filterKeypoints(keypoints, thres = 0.1, min_width=40, \
|
||
|
|
min_height=40, min_area= 50000, min_count=6):
|
||
|
|
add_list = []
|
||
|
|
# TODO:并行化
|
||
|
|
for ik in range(keypoints.shape[0]):
|
||
|
|
pose = keypoints[ik]
|
||
|
|
vis_count = np.sum(pose[:15, 2] > thres) #TODO:
|
||
|
|
if vis_count < min_count:
|
||
|
|
continue
|
||
|
|
ll, rr = np.min(pose[pose[:,2]>thres,0]), np.max(pose[pose[:,2]>thres,0])
|
||
|
|
bb, tt = np.min(pose[pose[:,2]>thres,1]), np.max(pose[pose[:,2]>thres,1])
|
||
|
|
center = (int((ll+rr)/2), int((bb+tt)/2))
|
||
|
|
length = [int(1.2*(rr-ll)/2), int(1.2*(tt-bb)/2)]
|
||
|
|
l = center[0] - length[0]
|
||
|
|
r = center[0] + length[0]
|
||
|
|
b = center[1] - length[1]
|
||
|
|
t = center[1] + length[1]
|
||
|
|
if (r - l) < min_width:
|
||
|
|
continue
|
||
|
|
if (t - b) < min_height:
|
||
|
|
continue
|
||
|
|
if (r - l)*(t - b) < min_area:
|
||
|
|
continue
|
||
|
|
add_list.append(ik)
|
||
|
|
keypoints = keypoints[add_list, :, :]
|
||
|
|
return keypoints, add_list
|
||
|
|
|
||
|
|
|
||
|
|
def get_fundamental_matrix(cameras, basenames):
|
||
|
|
skew_op = lambda x: np.array([[0, -x[2], x[1]], [x[2], 0, -x[0]], [-x[1], x[0], 0]])
|
||
|
|
fundamental_op = lambda K_0, R_0, T_0, K_1, R_1, T_1: np.linalg.inv(K_0).T @ (
|
||
|
|
R_0 @ R_1.T) @ K_1.T @ skew_op(K_1 @ R_1 @ R_0.T @ (T_0 - R_0 @ R_1.T @ T_1))
|
||
|
|
fundamental_RT_op = lambda K_0, RT_0, K_1, RT_1: fundamental_op (K_0, RT_0[:, :3], RT_0[:, 3], K_1,
|
||
|
|
RT_1[:, :3], RT_1[:, 3] )
|
||
|
|
F = np.zeros((len(basenames), len(basenames), 3, 3)) # N x N x 3 x 3 matrix
|
||
|
|
F = {(icam, jcam): np.zeros((3, 3)) for jcam in basenames for icam in basenames}
|
||
|
|
for icam in basenames:
|
||
|
|
for jcam in basenames:
|
||
|
|
F[(icam, jcam)] += fundamental_RT_op(cameras[icam]['K'], cameras[icam]['RT'], cameras[jcam]['K'], cameras[jcam]['RT'])
|
||
|
|
if F[(icam, jcam)].sum() == 0:
|
||
|
|
F[(icam, jcam)] += 1e-12 # to avoid nan
|
||
|
|
return F
|