diff --git a/Pose2Sim/Demo/S00_Demo_Session/Config.toml b/Pose2Sim/Demo/S00_Demo_Session/Config.toml index 91640d6..562effb 100644 --- a/Pose2Sim/Demo/S00_Demo_Session/Config.toml +++ b/Pose2Sim/Demo/S00_Demo_Session/Config.toml @@ -127,6 +127,7 @@ interpolation = 'cubic' #linear, slinear, quadratic, cubic, or none # 'none' if you don't want to interpolate missing points interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/Config.toml b/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/Config.toml index ca48237..3738864 100644 --- a/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/Config.toml +++ b/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T00_StaticTrial/Config.toml b/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T00_StaticTrial/Config.toml index ca48237..3738864 100644 --- a/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T00_StaticTrial/Config.toml +++ b/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T00_StaticTrial/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T01_BalancingTrial/Config.toml b/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T01_BalancingTrial/Config.toml index 0f00a42..d8eb7b0 100644 --- a/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T01_BalancingTrial/Config.toml +++ b/Pose2Sim/Demo/S00_Demo_Session/S00_P00_Participant/S00_P00_T01_BalancingTrial/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/Config.toml index 8f0c216..c56ceeb 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/Config.toml @@ -127,6 +127,7 @@ interpolation = 'cubic' #linear, slinear, quadratic, cubic, or none # 'none' if you don't want to interpolate missing points interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/Config.toml index 5f4c2d3..bbceb51 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T00_StaticTrial/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T00_StaticTrial/Config.toml index 5f4c2d3..bbceb51 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T00_StaticTrial/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T00_StaticTrial/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T01_MotionTrial1/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T01_MotionTrial1/Config.toml index bc65a50..075146e 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T01_MotionTrial1/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T01_MotionTrial1/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T02_MotionTrial2/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T02_MotionTrial2/Config.toml index fb2f597..bace4d6 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T02_MotionTrial2/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P00_Participant0/S01_P00_T02_MotionTrial2/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/Config.toml index 0eff317..5e1bdaa 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T00_StaticTrial/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T00_StaticTrial/Config.toml index 0eff317..5e1bdaa 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T00_StaticTrial/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T00_StaticTrial/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T01_MotionTrial1/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T01_MotionTrial1/Config.toml index 0493feb..422e2fa 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T01_MotionTrial1/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T01_MotionTrial1/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T02_MotionTrial2/Config.toml b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T02_MotionTrial2/Config.toml index 0493feb..422e2fa 100644 --- a/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T02_MotionTrial2/Config.toml +++ b/Pose2Sim/Demo/S01_Empty_Session/S01_P01_Participant1/S01_P01_T02_MotionTrial2/Config.toml @@ -128,6 +128,7 @@ ## 'none' if you don't want to interpolate missing points # interp_if_gap_smaller_than = 10 # do not interpolate bigger gaps # show_interp_indices = true # true or false (lowercase). For each keypoint, return the frames that need to be interpolated +# handle_LR_swap = true # Set to false if you use many cameras (eg more than 4), or if the pose estimation never confuses right and left limbs (eg no camera facing sagittal plane) # make_c3d = false # save triangulated data in c3d format in addition to trc diff --git a/Pose2Sim/triangulation.py b/Pose2Sim/triangulation.py index 09e8290..59d51f4 100644 --- a/Pose2Sim/triangulation.py +++ b/Pose2Sim/triangulation.py @@ -11,7 +11,7 @@ by OpenSim. The triangulation is weighted by the likelihood of each detected 2D keypoint - (if they meet the likelihood threshold). It the reprojection error is above a + (if they meet the likelihood threshold). If the reprojection error is above a threshold, right and left sides are swapped; if it is still above, a camera is removed for this point and this frame, until the threshold is met. If more cameras are removed than a predefined minimum, triangulation is skipped for @@ -280,6 +280,7 @@ def triangulation_from_best_cameras(config, coords_2D_kpt, coords_2D_kpt_swapped # Read config error_threshold_triangulation = config.get('triangulation').get('reproj_error_threshold_triangulation') min_cameras_for_triangulation = config.get('triangulation').get('min_cameras_for_triangulation') + handle_LR_swap = config.get('triangulation').get('handle_LR_swap') # Initialize x_files, y_files, likelihood_files = coords_2D_kpt @@ -296,7 +297,7 @@ def triangulation_from_best_cameras(config, coords_2D_kpt, coords_2D_kpt_swapped y_files_filt = np.vstack([y_files.copy()]*len(id_cams_off)) x_files_swapped_filt = np.vstack([x_files_swapped.copy()]*len(id_cams_off)) y_files_swapped_filt = np.vstack([y_files_swapped.copy()]*len(id_cams_off)) - likelihood_files_filt = np.vstack([likelihood_files.copy()]*len(id_cams_off)) + likelihood_files_filt = np.vstack([likelihood_files_swapped.copy()]*len(id_cams_off)) if nb_cams_off > 0: for i in range(len(id_cams_off)): @@ -338,57 +339,57 @@ def triangulation_from_best_cameras(config, coords_2D_kpt, coords_2D_kpt_swapped Q = Q_filt[best_cams][:-1] - # # Swap left and right sides if reprojection error still too high - # if error_min > error_threshold_triangulation: - # n_cams_swapped = 1 - # id_cams_swapped = np.array(list(it.combinations(range(n_cams-nb_cams_off), n_cams_swapped))) - # error_off_swap_min = error_min - # while error_off_swap_min > error_threshold_triangulation and n_cams_swapped < (n_cams - nb_cams_off) / 2: # more than half of the cameras switched: may triangulate twice the same side - # # Create subsets - # x_files_filt_off_swap = np.array([[x] * len(id_cams_swapped) for x in x_files_filt]) - # y_files_filt_off_swap = np.array([[y] * len(id_cams_swapped) for y in y_files_filt]) - # for id_off in range(len(id_cams_off)): # for each configuration with nb_cams_off removed - # for id_swapped, config_swapped in enumerate(id_cams_swapped): # for each of these configurations, test all subconfigurations with with n_cams_swapped swapped - # x_files_filt_off_swap[id_off, id_swapped, config_swapped] = x_files_swapped_filt[id_off, config_swapped] - # y_files_filt_off_swap[id_off, id_swapped, config_swapped] = y_files_swapped_filt[id_off, config_swapped] + # Swap left and right sides if reprojection error still too high + if handle_LR_swap and error_min > error_threshold_triangulation: + n_cams_swapped = 1 + error_off_swap_min = error_min + while error_off_swap_min > error_threshold_triangulation and n_cams_swapped < (n_cams - nb_cams_off) / 2: # more than half of the cameras switched: may triangulate twice the same side + # Create subsets + id_cams_swapped = np.array(list(it.combinations(range(n_cams-nb_cams_off), n_cams_swapped))) + x_files_filt_off_swap = np.array([[x] * len(id_cams_swapped) for x in x_files_filt]) + y_files_filt_off_swap = np.array([[y] * len(id_cams_swapped) for y in y_files_filt]) + for id_off in range(len(id_cams_off)): # for each configuration with nb_cams_off removed + for id_swapped, config_swapped in enumerate(id_cams_swapped): # for each of these configurations, test all subconfigurations with with n_cams_swapped swapped + x_files_filt_off_swap[id_off, id_swapped, config_swapped] = x_files_swapped_filt[id_off, config_swapped] + y_files_filt_off_swap[id_off, id_swapped, config_swapped] = y_files_swapped_filt[id_off, config_swapped] - # # Triangulate 2D points - # Q_filt_off_swap = np.array([[weighted_triangulation(projection_matrices_filt[id_off], x_files_filt_off_swap[id_off, id_swapped], y_files_filt_off_swap[id_off, id_swapped], likelihood_files_filt[id_off]) - # for id_swapped in range(len(id_cams_swapped))] - # for id_off in range(len(id_cams_off))] ) + # Triangulate 2D points + Q_filt_off_swap = np.array([[weighted_triangulation(projection_matrices_filt[id_off], x_files_filt_off_swap[id_off, id_swapped], y_files_filt_off_swap[id_off, id_swapped], likelihood_files_filt[id_off]) + for id_swapped in range(len(id_cams_swapped))] + for id_off in range(len(id_cams_off))] ) - # # Reprojection - # coords_2D_kpt_calc_off_swap = np.array([[reprojection(projection_matrices_filt[id_off], Q_filt_off_swap[id_off, id_swapped]) - # for id_swapped in range(len(id_cams_swapped))] - # for id_off in range(len(id_cams_off))]) - # x_calc_off_swap = coords_2D_kpt_calc_off_swap[:,:,0] - # y_calc_off_swap = coords_2D_kpt_calc_off_swap[:,:,1] + # Reprojection + coords_2D_kpt_calc_off_swap = np.array([[reprojection(projection_matrices_filt[id_off], Q_filt_off_swap[id_off, id_swapped]) + for id_swapped in range(len(id_cams_swapped))] + for id_off in range(len(id_cams_off))]) + x_calc_off_swap = coords_2D_kpt_calc_off_swap[:,:,0] + y_calc_off_swap = coords_2D_kpt_calc_off_swap[:,:,1] - # # Reprojection error - # error_off_swap = [] - # for id_off in range(len(id_cams_off)): - # q_file = [(x_files_filt[id_off,i], y_files_filt[id_off,i]) for i in range(len(x_files_filt[id_off]))] - # error_percam = [] - # for id_swapped, config_swapped in enumerate(id_cams_swapped): - # q_calc_off_swap = [(x_calc_off_swap[id_off,id_swapped,i], y_calc_off_swap[id_off,id_swapped,i]) for i in range(len(x_calc_off_swap[id_off]))] - # error_percam.append( np.mean( [euclidean_distance(q_file[i], q_calc_off_swap[i]) for i in range(len(q_file))] ) ) - # error_off_swap.append(error_percam) - # error_off_swap = np.array(error_off_swap) + # Reprojection error + error_off_swap = [] + for id_off in range(len(id_cams_off)): + error_percam = [] + for id_swapped, config_swapped in enumerate(id_cams_swapped): + q_file_off_swap = [(x_files_filt_off_swap[id_off,id_swapped,i], y_files_filt_off_swap[id_off,id_swapped,i]) for i in range(n_cams - nb_cams_off)] + q_calc_off_swap = [(x_calc_off_swap[id_off,id_swapped,i], y_calc_off_swap[id_off,id_swapped,i]) for i in range(n_cams - nb_cams_off)] + error_percam.append( np.mean( [euclidean_distance(q_file_off_swap[i], q_calc_off_swap[i]) for i in range(len(q_file_off_swap))] ) ) + error_off_swap.append(error_percam) + error_off_swap = np.array(error_off_swap) - # # Choosing best triangulation (with min reprojection error) - # error_off_swap_min = np.min(error_off_swap) - # best_off_swap_config = np.unravel_index(error_off_swap.argmin(), error_off_swap.shape) + # Choosing best triangulation (with min reprojection error) + error_off_swap_min = np.min(error_off_swap) + best_off_swap_config = np.unravel_index(error_off_swap.argmin(), error_off_swap.shape) - # id_off_cams = id_cams_off[best_off_swap_config[0]] - # id_swapped_cams = id_cams_swapped[best_off_swap_config[1]] - # Q_best = Q_filt_off_swap[best_off_swap_config][:-1] + id_off_cams = best_off_swap_config[0] + id_swapped_cams = id_cams_swapped[best_off_swap_config[1]] + Q_best = Q_filt_off_swap[best_off_swap_config][:-1] - # n_cams_swapped += 1 + n_cams_swapped += 1 - # if error_off_swap_min < error_min: - # error_min = error_off_swap_min - # best_cams = id_off_cams - # Q = Q_best + if error_off_swap_min < error_min: + error_min = error_off_swap_min + best_cams = id_off_cams + Q = Q_best nb_cams_off += 1 diff --git a/README.md b/README.md index e500431..9e5903e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ > **_News_: Version 0.5 released:** \ **Deep change in the folder structure to allow for automatic batch processing!**\ +Incidentally, right/left limb swapping is now handled, which is useful if few cameras are used.\ To upgrade, type `pip install pose2sim --upgrade`.\ *N.B.:* As always, I am more than happy to welcome contributors (see [How to contribute](#how-to-contribute)). @@ -365,6 +366,8 @@ Output:\ ### Triangulating keypoints > _**Triangulate your 2D coordinates in a robust way.**_ \ +> The triangulation is weighted by the likelihood of each detected 2D keypoint, provided that they meet a likelihood threshold.\ + If the reprojection error is above a threshold, right and left sides are swapped; if it is still above, cameras are removed until the threshold is met. If more cameras are removed than threshold, triangulation is skipped for this point and this frame. In the end, missing values are interpolated.\ > _**N.B.:**_ You can visualize your resulting 3D coordinates with my (experimental) [Maya-Mocap tool](https://github.com/davidpagnon/Maya-Mocap). Open an Anaconda prompt or a terminal in a `Session`, `Participant`, or `Trial` folder.\ @@ -385,6 +388,7 @@ Output:\ ### Filtering 3D coordinates > _**Filter your 3D coordinates.**_\ +> Numerous filter types are provided, and can be tuned accordingly.\ > _**N.B.:**_ You can visualize your resulting filtered 3D coordinates with my (experimental) [Maya-Mocap tool](https://github.com/davidpagnon/Maya-Mocap). Open an Anaconda prompt or a terminal in a `Session`, `Participant`, or `Trial` folder.\ diff --git a/setup.cfg b/setup.cfg index d66efb7..3720b1f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pose2sim -version = 0.5.2 +version = 0.5.3 author = David Pagnon author_email = contact@david-pagnon.com description = Perform a markerless kinematic analysis from multiple calibrated views as a unified workflow from an OpenPose input to an OpenSim result.