pose2sim/Pose2Sim/Utilities/synchronize_cams_draft.py

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import signal
from scipy import interpolate
import json
import os
import fnmatch
import pickle as pk


'''
    #########################################
    ## Synchronize cameras                 ##
    #########################################

    Steps undergone in this script
    0. Converting json files to pandas dataframe
    1. Computing speeds (either vertical, or 2D speeds)
    2. Plotting paired correlations of speeds from one camera viewpoint to another (work on one single keypoint, or on all keypoints, or on a weighted selection of keypoints)
    3. 
    Dans l'idéal, on fait ça automatiqueement pour toutes les vues, en coisissant les paires 2 à 2 avec le plus haut coefficient de corrélation, 
    et on demande confirmation avant de supprimer les frames en question (en réalité, renommées .json.del - option reset_sync dans le Config.toml)
'''


#############
# CONSTANTS #
#############

# pose_dir is populated with subfolders for each camera, each of them populated with json files 
pose_dir = r'GOp2AniPoitiersHalteroHaltero2pose-2d' 
fps = 120 # frame rate of the cameras (Hz)
reset_sync = True # Start synchronization over each time it is run

cut_off_frequency = 10 # cut-off frequency for a 4th order low-pass Butterworth filter

# Vertical speeds (on X, Y, or Z axis, or 2D speeds)
speed_kind = 'y' # 'x', 'y', 'z', or '2D'
vmax = 20 # px/s

cam1_nb = 4
cam2_nb = 3
id_kpt = [9,10] # Pour plus tard aller chercher numéro depuis keypoint name dans skeleton.py. 'RWrist' BLAZEPOSE 16, BODY_25B 10, BODY_25 4 ; 'LWrist' BLAZEPOSE 15, BODY_25B 9, BODY_25 7
weights_kpt = [1,1] # Pris en compte uniquement si on a plusieurs keypoints
frames = [2850,3490]


############
# FUNCTIONS#
############

def convert_json2csv(json_dir):
    json_files_names = fnmatch.filter(os.listdir(os.path.join(json_dir)), '.json')
    json_files_path = [os.path.join(json_dir, j_f) for j_f in json_files_names]
    json_coords = []
    for i, j_p in enumerate(json_files_path):
        # if i in range(frames)
            with open(j_p) as j_f:
                try:
                    json_data = json.load(j_f)['people'][0]['pose_keypoints_2d']
                except:
                    print(f'No person found in {os.path.basename(json_dir)}, frame {i}')
                    json_data = [0]*75
            json_coords.append(json_data)
    df_json_coords = pd.DataFrame(json_coords)
    return df_json_coords

def drop_col(df,col_nb):
    idx_col = list(range(col_nb-1, df.shape[1], col_nb)) 
    df_dropped = df.drop(idx_col, axis=1)
    df_dropped.columns = range(df_dropped.columns.size)
    return df_dropped

def speed_vert(df, axis='y'):
    axis_dict = {'x':0, 'y':1, 'z':2}
    df_diff = df.diff()
    df_diff = df_diff.fillna(df_diff.iloc[1]*2)
    df_vert_speed = pd.DataFrame([df_diff.loc[:, 2*k + axis_dict[axis]] for k in range(int(df_diff.shape[1]*2))]).T
    df_vert_speed.columns = np.arange(len(df_vert_speed.columns))
    return df_vert_speed

def speed_2D(df):
    df_diff = df.diff()
    df_diff = df_diff.fillna(df_diff.iloc[1]*2)
    df_2Dspeed = pd.DataFrame([np.sqrt(df_diff.loc[:,2*k]*2 + df_diff.loc[:,2*k+1]*2) for k in range(int(df_diff.shape[1]*2))]).T
    return df_2Dspeed

def interpolate_nans(col, kind):
    '''
    Interpolate missing points (of value nan)

    INPUTS
    - col pandas column of coordinates
    - kind 'linear', 'slinear', 'quadratic', 'cubic'. Default 'cubic'

    OUTPUT
    - col_interp interpolated pandas column
    '''

    idx = col.index
    idx_good = np.where(np.isfinite(col))[0] #index of non zeros
    if len(idx_good) == 10: return col
    # idx_notgood = np.delete(np.arange(len(col)), idx_good)

    if not kind: # 'linear', 'slinear', 'quadratic', 'cubic'
        f_interp = interpolate.interp1d(idx_good, col[idx_good], kind='cubic', bounds_error=False)
    else:
        f_interp = interpolate.interp1d(idx_good, col[idx_good], kind=kind[0], bounds_error=False)
    col_interp = np.where(np.isfinite(col), col, f_interp(idx)) #replace nans with interpolated values
    col_interp = np.where(np.isfinite(col_interp), col_interp, np.nanmean(col_interp)) #replace remaining nans

    return col_interp #, idx_notgood

def plot_time_lagged_cross_corr(camx, camy, ax):
    pearson_r = [camx.corr(camy.shift(lag)) for lag in range(-2*fps, 2*fps)] # lag -2 sec à +2 sec
    offset = int(np.floor(len(pearson_r)/2)-np.argmax(pearson_r))
    max_corr = np.max(pearson_r)
    ax.plot(list(range(-2*fps, 2*fps)), pearson_r)
    ax.axvline(np.ceil(len(pearson_r)/2)-2*fps,color='k',linestyle='--')
    ax.axvline(np.argmax(pearson_r)-2*fps,color='r',linestyle='--',label='Peak synchrony')
    plt.annotate(f'Max correlation={np.round(max_corr,2)}', xy=(0.05, 0.9), xycoords='axes fraction')
    ax.set(title=f'Offset = {offset} frames', xlabel='Offset (frames)',ylabel='Pearson r')
    plt.legend()
    return offset, max_corr


######################################
# 0. CONVERTING JSON FILES TO PANDAS #
######################################

# Also filter, and then save

pose_listdirs_names = next(os.walk(pose_dir))[1]
json_dirs_names = [k for k in pose_listdirs_names if 'json' in k]
json_dirs = [os.path.join(pose_dir, j_d) for j_d in json_dirs_names]

df_coords = []
for i, json_dir in enumerate(json_dirs):
    df_coords.append(convert_json2csv(json_dir))
    df_coords[i] = drop_col(df_coords[i],3) # drop likelihood

b, a = signal.butter(42, cut_off_frequency(fps*2), 'low', analog = False) 
for i in range(len(json_dirs)):
    df_coords[i] = pd.DataFrame(signal.filtfilt(b, a, df_coords[i], axis=0)) # filter 

## Pour sauvegarder et réouvrir au besoin
with open(os.path.join(pose_dir, 'coords'), 'wb') as fp:
    pk.dump(df_coords, fp)
# with open(os.path.join(pose_dir, 'coords'), 'rb') as fp
#     df_coords = pk.load(fp)


#############################
# 1. COMPUTING SPEEDS       #
#############################


# Vitesse verticale    
df_speed = []
for i in range(len(json_dirs)):
    if speed_kind == 'y':
        df_speed.append(speed_vert(df_coords[i]))
    elif speed_kind == '2D':
        df_speed.append(speed_2D(df_coords[i]))
    df_speed[i] = df_speed[i].where(df_speed[i]*vmax, other=np.nan)
    df_speed[i] = df_speed[i].apply(interpolate_nans, axis=0, args = ['cubic'])


#############################################
# 2. PLOTTING PAIRED CORRELATIONS OF SPEEDS #
#############################################

# Faire ça sur toutes les paires de cams
# Choisir paire avec corrélation la plus haute


# sur un point particulier (typiquement le poignet sur un mouvement vertical)
# ou sur tous les points
# ou sur une sélection de points pondérés

id_kpt_dict = {}

if len(id_kpt)==1 and id_kpt != ['all']:
    camx = df_speed[cam1_nb-1].loc[range(np.array(frames)),id_kpt[0]]
    camy = df_speed[cam2_nb-1].loc[range(np.array(frames)),id_kpt[0]]
elif id_kpt == ['all']:
    camx = df_speed[cam1_nb-1].loc[range(np.array(frames)),].sum(axis=1)
    camy = df_speed[cam2_nb-1].loc[range(np.array(frames)),].sum(axis=1)
elif len(id_kpt)==1 and len(id_kpt)==len(weights_kpt): # ex id_kpt1=9 set to 10, id_kpt2=10 to 15
    # ajouter frames
    dict_id_weights = {i:w for i, w in zip(id_kpt, weights_kpt)}
    camx = df_speed[cam1_nb-1] @ pd.Series(dict_id_weights).reindex(df_speed[cam1_nb-1].columns, fill_value=0)
    camy = df_speed[cam2_nb-1] @ pd.Series(dict_id_weights).reindex(df_speed[cam2_nb-1].columns, fill_value=0)
    camx = camx.loc[range(np.array(frames))]
    camy = camy.loc[range(np.array(frames))]
else:
    raise ValueError('wrong values for id_kpt or weights_kpt')



# camx = df_speed[1][16]
# camy = df_speed[2][16]
# camx = df_speed[1][10]
# camy = df_speed[2][10]
# camx = df_speed[1].sum(axis=1)
# camy = df_speed[2].sum(axis=1)
# camx.plot()
# camy.plot()
# plt.show()

for i in range(25):
    df_coords[1].iloc[:,i*2+1].plot(label='1')
    df_coords[2].iloc[:,i*2+1].plot(label='2')
    plt.title(i)
    plt.legend()
    plt.show()

for i in range(25):
    df_speed[1].iloc[:,i].plot(label='1')
    df_speed[2].iloc[:,i].plot(label='2')
    plt.title(i)
    plt.legend()
    plt.show()

for i in range(4):
    abs(df_speed[i]).sum(axis=1).plot(label=i)
plt.legend()
plt.show()

df_speed[0].plot()  # --> remove janky points
plt.show()



f, ax = plt.subplots(2,1)
# speed
camx.plot(ax=ax[0], label = f'cam {cam1_nb}')
camy.plot(ax=ax[0], label = f'cam {cam2_nb}')
ax[0].set(xlabel='Frame',ylabel='Speed (pxframe)')
ax[0].legend()

# time lagged cross-correlation
offset, max_corr = plot_time_lagged_cross_corr(camx, camy, ax[1])

f.tight_layout()
plt.show()


##################################################################
# 3. ON CHANGE LES EXTENSIONS DES FICHIERS POUR SIMULER UN OFFSET#
##################################################################

# et on relance tout le code


if offset == 0:
    json_dir_to_offset = json_dirs[cam2_nb-1]
else:
    json_dir_to_offset = json_dirs[cam1_nb-1]
    offset = -offset

json_files = fnmatch.filter(os.listdir(json_dir_to_offset), '.json')[offset]

[os.rename( os.path.join(json_dir_to_offset,json_file), os.path.join(json_dir_to_offset,json_file+'.old') ) for json_file in json_files]

# Reset remove all '.old'
json_files = fnmatch.filter(os.listdir(json_dir_to_offset), '.old')
[os.rename( os.path.join(json_dir_to_offset,json_file), os.path.join(json_dir_to_offset,json_file[-4]) ) for json_file in json_files]