sports2d 0.8.24__py3-none-any.whl → 0.8.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Content/huggingface_demo.png +0 -0
- Sports2D/Demo/Config_demo.toml +17 -11
- Sports2D/Sports2D.py +14 -8
- Sports2D/Utilities/common.py +3 -0
- Sports2D/Utilities/tests.py +3 -2
- Sports2D/process.py +436 -227
- {sports2d-0.8.24.dist-info → sports2d-0.8.26.dist-info}/METADATA +60 -38
- {sports2d-0.8.24.dist-info → sports2d-0.8.26.dist-info}/RECORD +12 -11
- {sports2d-0.8.24.dist-info → sports2d-0.8.26.dist-info}/WHEEL +0 -0
- {sports2d-0.8.24.dist-info → sports2d-0.8.26.dist-info}/entry_points.txt +0 -0
- {sports2d-0.8.24.dist-info → sports2d-0.8.26.dist-info}/licenses/LICENSE +0 -0
- {sports2d-0.8.24.dist-info → sports2d-0.8.26.dist-info}/top_level.txt +0 -0
Sports2D/process.py
CHANGED
|
@@ -90,6 +90,8 @@ from Pose2Sim.filtering import *
|
|
|
90
90
|
# Silence numpy "RuntimeWarning: Mean of empty slice"
|
|
91
91
|
import warnings
|
|
92
92
|
warnings.filterwarnings("ignore", category=RuntimeWarning, message="Mean of empty slice")
|
|
93
|
+
warnings.filterwarnings("ignore", category=RuntimeWarning, message="All-NaN slice encountered")
|
|
94
|
+
warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid value encountered in scalar divide")
|
|
93
95
|
|
|
94
96
|
# Not safe, but to be used until OpenMMLab/RTMlib's SSL certificates are updated
|
|
95
97
|
import ssl
|
|
@@ -240,7 +242,7 @@ def setup_model_class_mode(pose_model, mode, config_dict={}):
|
|
|
240
242
|
try: # from Config.toml
|
|
241
243
|
from anytree.importer import DictImporter
|
|
242
244
|
model_name = pose_model.upper()
|
|
243
|
-
pose_model = DictImporter().import_(config_dict.get('pose').get(pose_model))
|
|
245
|
+
pose_model = DictImporter().import_(config_dict.get('pose').get(pose_model)[0])
|
|
244
246
|
if pose_model.id == 'None':
|
|
245
247
|
pose_model.id = None
|
|
246
248
|
logging.info(f"Using model {model_name} for pose estimation.")
|
|
@@ -796,7 +798,6 @@ def make_mot_with_angles(angles, time, mot_path):
|
|
|
796
798
|
def pose_plots(trc_data_unfiltered, trc_data, person_id, show=True):
|
|
797
799
|
'''
|
|
798
800
|
Displays trc filtered and unfiltered data for comparison
|
|
799
|
-
⚠ Often crashes on the third window...
|
|
800
801
|
|
|
801
802
|
INPUTS:
|
|
802
803
|
- trc_data_unfiltered: pd.DataFrame. The unfiltered trc data
|
|
@@ -807,23 +808,26 @@ def pose_plots(trc_data_unfiltered, trc_data, person_id, show=True):
|
|
|
807
808
|
OUTPUT:
|
|
808
809
|
- matplotlib window with tabbed figures for each keypoint
|
|
809
810
|
'''
|
|
810
|
-
|
|
811
|
+
|
|
811
812
|
os_name = platform.system()
|
|
812
|
-
if os_name == 'Windows':
|
|
813
|
-
mpl.use('qt5agg') # windows
|
|
814
813
|
mpl.rc('figure', max_open_warning=0)
|
|
814
|
+
if show:
|
|
815
|
+
if os_name == 'Windows':
|
|
816
|
+
mpl.use('qt5agg') # windows
|
|
817
|
+
pw = plotWindow()
|
|
818
|
+
pw.MainWindow.setWindowTitle('Person'+ str(person_id) + ' coordinates')
|
|
819
|
+
else:
|
|
820
|
+
mpl.use('Agg') # Otherwise fails on Hugging-face
|
|
821
|
+
figures_list = []
|
|
815
822
|
|
|
816
823
|
keypoints_names = trc_data.columns[1::3]
|
|
817
|
-
|
|
818
|
-
pw = plotWindow()
|
|
819
|
-
pw.MainWindow.setWindowTitle('Person'+ str(person_id) + ' coordinates') # Main title
|
|
820
|
-
|
|
821
824
|
for id, keypoint in enumerate(keypoints_names):
|
|
822
825
|
f = plt.figure()
|
|
823
|
-
if
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
826
|
+
if show:
|
|
827
|
+
if os_name == 'Windows':
|
|
828
|
+
f.canvas.manager.window.setWindowTitle(keypoint + ' Plot')
|
|
829
|
+
elif os_name == 'Darwin':
|
|
830
|
+
f.canvas.manager.set_window_title(keypoint + ' Plot')
|
|
827
831
|
|
|
828
832
|
axX = plt.subplot(211)
|
|
829
833
|
plt.plot(trc_data_unfiltered.iloc[:,0], trc_data_unfiltered.iloc[:,id*3+1], label='unfiltered')
|
|
@@ -838,18 +842,21 @@ def pose_plots(trc_data_unfiltered, trc_data, person_id, show=True):
|
|
|
838
842
|
axY.set_xlabel('Time (seconds)')
|
|
839
843
|
axY.set_ylabel(keypoint+' Y')
|
|
840
844
|
|
|
841
|
-
|
|
845
|
+
if show:
|
|
846
|
+
pw.addPlot(keypoint, f)
|
|
847
|
+
else:
|
|
848
|
+
figures_list.append((keypoint, f))
|
|
842
849
|
|
|
843
850
|
if show:
|
|
844
851
|
pw.show()
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
852
|
+
return pw
|
|
853
|
+
else:
|
|
854
|
+
return figures_list
|
|
855
|
+
|
|
848
856
|
|
|
849
857
|
def angle_plots(angle_data_unfiltered, angle_data, person_id, show=True):
|
|
850
858
|
'''
|
|
851
859
|
Displays angle filtered and unfiltered data for comparison
|
|
852
|
-
⚠ Often crashes on the third window...
|
|
853
860
|
|
|
854
861
|
INPUTS:
|
|
855
862
|
- angle_data_unfiltered: pd.DataFrame. The unfiltered angle data
|
|
@@ -860,21 +867,24 @@ def angle_plots(angle_data_unfiltered, angle_data, person_id, show=True):
|
|
|
860
867
|
'''
|
|
861
868
|
|
|
862
869
|
os_name = platform.system()
|
|
863
|
-
if os_name == 'Windows':
|
|
864
|
-
mpl.use('qt5agg') # windows
|
|
865
870
|
mpl.rc('figure', max_open_warning=0)
|
|
871
|
+
if show:
|
|
872
|
+
if os_name == 'Windows':
|
|
873
|
+
mpl.use('qt5agg') # windows
|
|
874
|
+
pw = plotWindow()
|
|
875
|
+
pw.MainWindow.setWindowTitle('Person'+ str(person_id) + ' angles')
|
|
876
|
+
else:
|
|
877
|
+
mpl.use('Agg') # Otherwise fails on Hugging-face
|
|
878
|
+
figures_list = []
|
|
866
879
|
|
|
867
880
|
angles_names = angle_data.columns[1:]
|
|
868
|
-
|
|
869
|
-
pw = plotWindow()
|
|
870
|
-
pw.MainWindow.setWindowTitle('Person'+ str(person_id) + ' angles') # Main title
|
|
871
|
-
|
|
872
881
|
for id, angle in enumerate(angles_names):
|
|
873
882
|
f = plt.figure()
|
|
874
|
-
if
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
883
|
+
if show:
|
|
884
|
+
if os_name == 'Windows':
|
|
885
|
+
f.canvas.manager.window.setWindowTitle(angle + ' Plot') # windows
|
|
886
|
+
elif os_name == 'Darwin': # macOS
|
|
887
|
+
f.canvas.manager.set_window_title(angle + ' Plot') # mac
|
|
878
888
|
|
|
879
889
|
ax = plt.subplot(111)
|
|
880
890
|
plt.plot(angle_data_unfiltered.iloc[:,0], angle_data_unfiltered.iloc[:,id+1], label='unfiltered')
|
|
@@ -884,12 +894,16 @@ def angle_plots(angle_data_unfiltered, angle_data, person_id, show=True):
|
|
|
884
894
|
ax.set_ylabel(angle+' (°)')
|
|
885
895
|
plt.legend()
|
|
886
896
|
|
|
887
|
-
|
|
888
|
-
|
|
897
|
+
if show:
|
|
898
|
+
pw.addPlot(angle, f)
|
|
899
|
+
else:
|
|
900
|
+
figures_list.append((angle, f))
|
|
901
|
+
|
|
889
902
|
if show:
|
|
890
903
|
pw.show()
|
|
891
|
-
|
|
892
|
-
|
|
904
|
+
return pw
|
|
905
|
+
else:
|
|
906
|
+
return figures_list
|
|
893
907
|
|
|
894
908
|
|
|
895
909
|
def get_personIDs_with_highest_scores(all_frames_scores, nb_persons_to_detect):
|
|
@@ -1275,7 +1289,7 @@ def select_persons_on_vid(video_file_path, frame_range, all_pose_coords):
|
|
|
1275
1289
|
return selected_persons
|
|
1276
1290
|
|
|
1277
1291
|
|
|
1278
|
-
def compute_floor_line(trc_data, score_data, keypoint_names = ['LBigToe', 'RBigToe'], toe_speed_below = 7, score_threshold=0.
|
|
1292
|
+
def compute_floor_line(trc_data, score_data, keypoint_names = ['LBigToe', 'RBigToe'], toe_speed_below = 7, score_threshold=0.3):
|
|
1279
1293
|
'''
|
|
1280
1294
|
Compute the floor line equation, angle, and direction
|
|
1281
1295
|
from the feet keypoints when they have zero speed.
|
|
@@ -1304,7 +1318,7 @@ def compute_floor_line(trc_data, score_data, keypoint_names = ['LBigToe', 'RBigT
|
|
|
1304
1318
|
trc_data_kpt_trim = trc_data_kpt.iloc[start:end].reset_index(drop=True)
|
|
1305
1319
|
score_data_kpt_trim = score_data_kpt.iloc[start:end].reset_index(drop=True)
|
|
1306
1320
|
|
|
1307
|
-
# Compute
|
|
1321
|
+
# Compute euclidean speed
|
|
1308
1322
|
speeds = np.linalg.norm(trc_data_kpt_trim.diff(), axis=1)
|
|
1309
1323
|
|
|
1310
1324
|
# Remove speeds with low confidence
|
|
@@ -1325,7 +1339,7 @@ def compute_floor_line(trc_data, score_data, keypoint_names = ['LBigToe', 'RBigT
|
|
|
1325
1339
|
|
|
1326
1340
|
# Fit a line to the zero-speed coordinates
|
|
1327
1341
|
floor_line = np.polyfit(low_speeds_X, low_speeds_Y, 1) # (slope, intercept)
|
|
1328
|
-
angle = -np.arctan(floor_line[0]) # angle of the floor line in
|
|
1342
|
+
angle = -np.arctan(floor_line[0]) # angle of the floor line in radians
|
|
1329
1343
|
xy_origin = [0, floor_line[1]] # origin of the floor line
|
|
1330
1344
|
|
|
1331
1345
|
# Gait direction
|
|
@@ -1334,14 +1348,160 @@ def compute_floor_line(trc_data, score_data, keypoint_names = ['LBigToe', 'RBigT
|
|
|
1334
1348
|
return angle, xy_origin, gait_direction
|
|
1335
1349
|
|
|
1336
1350
|
|
|
1337
|
-
def
|
|
1351
|
+
def get_distance_from_camera(perspective_value=10, perspective_unit='distance_m', calib_file =None, height_px=1, height_m=1, cam_width=1, cam_height=1):
|
|
1352
|
+
'''
|
|
1353
|
+
Compute the distance between the camera and the person based on the chosen perspective unit.
|
|
1354
|
+
|
|
1355
|
+
INPUTS:
|
|
1356
|
+
- perspective_value: Value associated with the chosen perspective unit.
|
|
1357
|
+
- perspective_unit: Unit used to compute the distance. Can be 'distance_m', 'f_px', 'fov_rad', 'fov_deg', or 'from_calib'.
|
|
1358
|
+
- calib_file: Path to the toml calibration file.
|
|
1359
|
+
- height_px: Height of the person in pixels.
|
|
1360
|
+
- height_m: Height of the first person in meters.
|
|
1361
|
+
- cam_width: Width of the camera frame in pixels.
|
|
1362
|
+
- cam_height: Height of the camera frame in pixels.
|
|
1363
|
+
|
|
1364
|
+
OUTPUTS:
|
|
1365
|
+
- distance_m: Distance between the camera and the person in meters.
|
|
1366
|
+
'''
|
|
1367
|
+
|
|
1368
|
+
if perspective_unit == 'from_calib':
|
|
1369
|
+
if not calib_file:
|
|
1370
|
+
perspective_unit = 'distance_m'
|
|
1371
|
+
distance_m = 10.0
|
|
1372
|
+
logging.warning(f'No calibration file provided. Using a default distance of {distance_m} m between the camera and the person to convert px to meters.')
|
|
1373
|
+
else:
|
|
1374
|
+
calib_params_dict = retrieve_calib_params(calib_file)
|
|
1375
|
+
f_px = calib_params_dict['K'][0][0][0]
|
|
1376
|
+
distance_m = f_px / height_px * height_m
|
|
1377
|
+
elif perspective_unit == 'distance_m':
|
|
1378
|
+
distance_m = perspective_value
|
|
1379
|
+
elif perspective_unit == 'f_px':
|
|
1380
|
+
f_px = perspective_value
|
|
1381
|
+
distance_m = f_px / height_px * height_m
|
|
1382
|
+
elif perspective_unit == 'fov_rad':
|
|
1383
|
+
fov_rad = perspective_value
|
|
1384
|
+
f_px = max(cam_width, cam_height)/2 / np.tan(fov_rad / 2)
|
|
1385
|
+
distance_m = f_px / height_px * height_m
|
|
1386
|
+
elif perspective_unit == 'fov_deg':
|
|
1387
|
+
fov_rad = np.radians(perspective_value)
|
|
1388
|
+
f_px = max(cam_width, cam_height)/2 / np.tan(fov_rad / 2)
|
|
1389
|
+
distance_m = f_px / height_px * height_m
|
|
1390
|
+
|
|
1391
|
+
return distance_m
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def get_floor_params(floor_angle='auto', xy_origin=['auto'],
|
|
1395
|
+
calib_file=None, height_px=1, height_m=1,
|
|
1396
|
+
fps=30, trc_data=pd.DataFrame(), score_data=pd.DataFrame(), toe_speed_below=1, score_threshold=0.5,
|
|
1397
|
+
cam_width=1, cam_height=1):
|
|
1398
|
+
'''
|
|
1399
|
+
Compute the floor angle and the xy_origin based on calibration file, kinematics, or user input.
|
|
1400
|
+
|
|
1401
|
+
INPUTS:
|
|
1402
|
+
- floor_angle: Method to compute the floor angle. Can be 'auto', 'from_calib', 'from_kinematics', or a numeric value in degrees.
|
|
1403
|
+
- xy_origin: Method to compute the xy_origin. Can be ['auto'], ['from_calib'], ['from_kinematics'], or a list of two numeric values in pixels [cx, cy].
|
|
1404
|
+
- calib_file: Path to a toml calibration file.
|
|
1405
|
+
- height_px: Height of the person in pixels.
|
|
1406
|
+
- height_m: Height of the first person in meters.
|
|
1407
|
+
- fps: Framerate of the video in frames per second. Used if estimating floor line from kinematics.
|
|
1408
|
+
- trc_data: DataFrame containing the pose data in pixels for one person. Used if estimating floor line from kinematics.
|
|
1409
|
+
- score_data: DataFrame containing the keypoint scores for one person. Used if estimating floor line from kinematics.
|
|
1410
|
+
- toe_speed_below: Speed below which the foot is considered to be stationary, in m/s. Used if estimating floor line from kinematics.
|
|
1411
|
+
- score_threshold: Minimum average keypoint score to consider a frame for floor line estimation.
|
|
1412
|
+
- cam_width: Width of the camera frame in pixels. Used if failed to estimate floor line from kinematics.
|
|
1413
|
+
- cam_height: Height of the camera frame in pixels. Used if failed to estimate floor line from kinematics.
|
|
1414
|
+
|
|
1415
|
+
OUTPUTS:
|
|
1416
|
+
- floor_angle_estim: Estimated floor angle in radians.
|
|
1417
|
+
- xy_origin_estim: Estimated xy_origin as a list of two numeric values in pixels [cx, cy].
|
|
1418
|
+
- gait_direction: Estimated gait direction. 'left' if < 0, 'right' otherwise.
|
|
1419
|
+
'''
|
|
1420
|
+
|
|
1421
|
+
# Estimate floor angle from the calibration file
|
|
1422
|
+
if floor_angle == 'from_calib' or xy_origin == ['from_calib']:
|
|
1423
|
+
if not calib_file:
|
|
1424
|
+
if floor_angle == 'from_calib':
|
|
1425
|
+
floor_angle = 'auto'
|
|
1426
|
+
if xy_origin == ['from_calib']:
|
|
1427
|
+
xy_origin = ['auto']
|
|
1428
|
+
logging.warning(f'No calibration file provided. Estimating floor angle and xy_origin from the pose of the first selected person.')
|
|
1429
|
+
else:
|
|
1430
|
+
calib_params_dict = retrieve_calib_params(calib_file)
|
|
1431
|
+
|
|
1432
|
+
R90z = np.array([[0.0, -1.0, 0.0],
|
|
1433
|
+
[1.0, 0.0, 0.0],
|
|
1434
|
+
[0.0, 0.0, 1.0]])
|
|
1435
|
+
R270x = np.array([[1.0, 0.0, 0.0],
|
|
1436
|
+
[0.0, 0.0, 1.0],
|
|
1437
|
+
[0.0, -1.0, 0.0]])
|
|
1438
|
+
|
|
1439
|
+
R_cam = cv2.Rodrigues(calib_params_dict['R'][0])[0]
|
|
1440
|
+
T_cam = np.array(calib_params_dict['T'][0])
|
|
1441
|
+
R_world, T_world = world_to_camera_persp(R_cam, T_cam)
|
|
1442
|
+
Rfloory = R90z.T @ R_world @ R270x.T
|
|
1443
|
+
T_world = R90z.T @ T_world
|
|
1444
|
+
floor_angle_calib = np.arctan2(Rfloory[0,2], Rfloory[0,0])
|
|
1445
|
+
|
|
1446
|
+
cu = calib_params_dict['K'][0][0][2]
|
|
1447
|
+
cv = calib_params_dict['K'][0][1][2]
|
|
1448
|
+
cx = 0.0
|
|
1449
|
+
cy = cv + T_world[2]* height_px / height_m
|
|
1450
|
+
xy_origin_calib = [cx, cy]
|
|
1451
|
+
|
|
1452
|
+
# Estimate xy_origin from the line formed by the toes when they are on the ground (where speed = 0)
|
|
1453
|
+
px_per_m = height_px/height_m
|
|
1454
|
+
toe_speed_below_px_frame = toe_speed_below * px_per_m / fps # speed below which the foot is considered to be stationary
|
|
1455
|
+
try:
|
|
1456
|
+
if all(key in trc_data for key in ['LBigToe', 'RBigToe']):
|
|
1457
|
+
floor_angle_kin, xy_origin_kin, gait_direction = compute_floor_line(trc_data, score_data, keypoint_names=['LBigToe', 'RBigToe'], toe_speed_below=toe_speed_below_px_frame, score_threshold=score_threshold)
|
|
1458
|
+
else:
|
|
1459
|
+
floor_angle_kin, xy_origin_estim, gait_direction = compute_floor_line(trc_data, score_data, keypoint_names=['LAnkle', 'RAnkle'], toe_speed_below=toe_speed_below_px_frame, score_threshold=score_threshold)
|
|
1460
|
+
xy_origin_kin[1] = xy_origin_kin[1] + 0.13*px_per_m # approx. height of the ankle above the floor
|
|
1461
|
+
logging.warning(f'The RBigToe and LBigToe are missing from your pose estimation model. Using ankles - 13 cm to compute the floor line.')
|
|
1462
|
+
except:
|
|
1463
|
+
floor_angle_kin = 0
|
|
1464
|
+
xy_origin_kin = cam_width/2, cam_height/2
|
|
1465
|
+
gait_direction = 1
|
|
1466
|
+
logging.warning(f'Could not estimate the floor angle, xy_origin, and visible from person {0}. Make sure that the full body is visible. Using floor angle = 0°, xy_origin = [{cam_width/2}, {cam_height/2}] px, and visible_side = right.')
|
|
1467
|
+
|
|
1468
|
+
# Determine final floor angle estimation
|
|
1469
|
+
if floor_angle == 'from_calib':
|
|
1470
|
+
floor_angle_estim = floor_angle_calib
|
|
1471
|
+
elif floor_angle in ['auto', 'from_kinematics']:
|
|
1472
|
+
floor_angle_estim = floor_angle_kin
|
|
1473
|
+
else:
|
|
1474
|
+
try:
|
|
1475
|
+
floor_angle_estim = np.radians(float(floor_angle))
|
|
1476
|
+
except:
|
|
1477
|
+
raise ValueError(f'Invalid floor_angle: {floor_angle}. Must be "auto", "from_calib", "from_kinematics", or a numeric value in degrees.')
|
|
1478
|
+
|
|
1479
|
+
# Determine final xy_origin estimation
|
|
1480
|
+
if xy_origin == ['from_calib']:
|
|
1481
|
+
xy_origin_estim = xy_origin_calib
|
|
1482
|
+
elif xy_origin in [['auto'], ['from_kinematics']]:
|
|
1483
|
+
xy_origin_estim = xy_origin_kin
|
|
1484
|
+
else:
|
|
1485
|
+
try:
|
|
1486
|
+
xy_origin_estim = [float(v) for v in xy_origin]
|
|
1487
|
+
except:
|
|
1488
|
+
raise ValueError(f'Invalid xy_origin: {xy_origin}. Must be "auto", "from_calib", "from_kinematics", or a list of two numeric values in pixels.')
|
|
1489
|
+
|
|
1490
|
+
return floor_angle_estim, xy_origin_estim, gait_direction
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def convert_px_to_meters(Q_coords_kpt, first_person_height, height_px, distance_m, cam_width, cam_height, cx, cy, floor_angle, visible_side='none'):
|
|
1338
1494
|
'''
|
|
1339
1495
|
Convert pixel coordinates to meters.
|
|
1496
|
+
Corrects for floor angle, floor level, and depth perspective errors.
|
|
1340
1497
|
|
|
1341
1498
|
INPUTS:
|
|
1342
1499
|
- Q_coords_kpt: pd.DataFrame. The xyz coordinates of a keypoint in pixels, with z filled with zeros
|
|
1343
1500
|
- first_person_height: float. The height of the person in meters
|
|
1344
1501
|
- height_px: float. The height of the person in pixels
|
|
1502
|
+
- cam_width: float. The width of the camera frame in pixels
|
|
1503
|
+
- cam_height: float. The height of the camera frame in pixels
|
|
1504
|
+
- distance_m: float. The distance between the camera and the person in meters
|
|
1345
1505
|
- cx, cy: float. The origin of the image in pixels
|
|
1346
1506
|
- floor_angle: float. The angle of the floor in radians
|
|
1347
1507
|
- visible_side: str. The side of the person that is visible ('right', 'left', 'front', 'back', 'none')
|
|
@@ -1350,19 +1510,36 @@ def convert_px_to_meters(Q_coords_kpt, first_person_height, height_px, cx, cy, f
|
|
|
1350
1510
|
- Q_coords_kpt_m: pd.DataFrame. The XYZ coordinates of a keypoint in meters
|
|
1351
1511
|
'''
|
|
1352
1512
|
|
|
1513
|
+
# u,v coordinates
|
|
1353
1514
|
u = Q_coords_kpt.iloc[:,0]
|
|
1354
1515
|
v = Q_coords_kpt.iloc[:,1]
|
|
1516
|
+
cu = cam_width / 2
|
|
1517
|
+
cv = cam_height / 2
|
|
1355
1518
|
|
|
1356
|
-
|
|
1357
|
-
Y = - first_person_height / height_px * np.cos(floor_angle) * (v-cy - np.tan(floor_angle)*(u-cx))
|
|
1358
|
-
|
|
1519
|
+
# Normative Z coordinates
|
|
1359
1520
|
marker_name = Q_coords_kpt.columns[0]
|
|
1360
1521
|
if 'marker_Z_positions' in globals() and visible_side!='none' and marker_name in marker_Z_positions[visible_side].keys():
|
|
1361
|
-
Z =
|
|
1522
|
+
Z = u.copy()
|
|
1362
1523
|
Z[:] = marker_Z_positions[visible_side][marker_name]
|
|
1363
1524
|
else:
|
|
1364
|
-
Z = np.zeros_like(
|
|
1365
|
-
|
|
1525
|
+
Z = np.zeros_like(u)
|
|
1526
|
+
|
|
1527
|
+
## Compute X and Y coordinates in meters
|
|
1528
|
+
# X = first_person_height / height_px * (u-cu)
|
|
1529
|
+
# Y = - first_person_height / height_px * (v-cv)
|
|
1530
|
+
## With floor angle and level correction:
|
|
1531
|
+
# X = first_person_height / height_px * ((u-cx)*np.cos(floor_angle) + (v-cy)*np.sin(floor_angle))
|
|
1532
|
+
# Y = - first_person_height / height_px * ((v-cy)*np.cos(floor_angle) + (u-cx)*np.sin(floor_angle))
|
|
1533
|
+
## With floor angle and level correction, and depth perspective correction:
|
|
1534
|
+
scaling_factor = first_person_height / height_px
|
|
1535
|
+
X = scaling_factor * ( \
|
|
1536
|
+
((u-cx) - Z/distance_m * (u-cu)) * np.cos(floor_angle) + \
|
|
1537
|
+
((v-cy) - Z/distance_m * (v-cv)) * np.sin(floor_angle) )
|
|
1538
|
+
Y = - scaling_factor * ( \
|
|
1539
|
+
((v-cy) - Z/distance_m * (v-cv)) * np.cos(floor_angle) - \
|
|
1540
|
+
((u-cx) - Z/distance_m * (u-cu)) * np.sin(floor_angle) )
|
|
1541
|
+
|
|
1542
|
+
# Assemble results
|
|
1366
1543
|
Q_coords_kpt_m = pd.DataFrame(np.array([X, Y, Z]).T, columns=Q_coords_kpt.columns)
|
|
1367
1544
|
|
|
1368
1545
|
return Q_coords_kpt_m
|
|
@@ -1414,7 +1591,7 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1414
1591
|
|
|
1415
1592
|
# Base parameters
|
|
1416
1593
|
video_dir = Path(config_dict.get('base').get('video_dir'))
|
|
1417
|
-
|
|
1594
|
+
|
|
1418
1595
|
nb_persons_to_detect = config_dict.get('base').get('nb_persons_to_detect')
|
|
1419
1596
|
if nb_persons_to_detect != 'all':
|
|
1420
1597
|
try:
|
|
@@ -1455,8 +1632,9 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1455
1632
|
pose_model = config_dict.get('pose').get('pose_model')
|
|
1456
1633
|
mode = config_dict.get('pose').get('mode')
|
|
1457
1634
|
det_frequency = config_dict.get('pose').get('det_frequency')
|
|
1635
|
+
backend = config_dict.get('pose').get('backend')
|
|
1636
|
+
device = config_dict.get('pose').get('device')
|
|
1458
1637
|
tracking_mode = config_dict.get('pose').get('tracking_mode')
|
|
1459
|
-
max_distance = config_dict.get('pose').get('max_distance', None)
|
|
1460
1638
|
if tracking_mode == 'deepsort':
|
|
1461
1639
|
from deep_sort_realtime.deepsort_tracker import DeepSort
|
|
1462
1640
|
deepsort_params = config_dict.get('pose').get('deepsort_params')
|
|
@@ -1468,39 +1646,39 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1468
1646
|
deepsort_params = json.loads(deepsort_params)
|
|
1469
1647
|
deepsort_tracker = DeepSort(**deepsort_params)
|
|
1470
1648
|
deepsort_tracker.tracker.tracks.clear()
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1649
|
+
|
|
1650
|
+
keypoint_likelihood_threshold = config_dict.get('pose').get('keypoint_likelihood_threshold')
|
|
1651
|
+
average_likelihood_threshold = config_dict.get('pose').get('average_likelihood_threshold')
|
|
1652
|
+
keypoint_number_threshold = config_dict.get('pose').get('keypoint_number_threshold')
|
|
1653
|
+
max_distance = config_dict.get('pose').get('max_distance', None)
|
|
1654
|
+
|
|
1474
1655
|
# Pixel to meters conversion
|
|
1475
1656
|
to_meters = config_dict.get('px_to_meters_conversion').get('to_meters')
|
|
1476
1657
|
make_c3d = config_dict.get('px_to_meters_conversion').get('make_c3d')
|
|
1477
1658
|
save_calib = config_dict.get('px_to_meters_conversion').get('save_calib')
|
|
1659
|
+
# Correct perspective effects
|
|
1660
|
+
perspective_value = config_dict.get('px_to_meters_conversion', {}).get('perspective_value', 10.0)
|
|
1661
|
+
perspective_unit = config_dict.get('px_to_meters_conversion', {}).get('perspective_unit', 'distance_m')
|
|
1662
|
+
# Calibration from person height
|
|
1663
|
+
floor_angle = config_dict.get('px_to_meters_conversion').get('floor_angle', 'auto') # 'auto' or float
|
|
1664
|
+
xy_origin = config_dict.get('px_to_meters_conversion').get('xy_origin', ['auto']) # ['auto'] or [x, y]
|
|
1478
1665
|
# Calibration from file
|
|
1479
1666
|
calib_file = config_dict.get('px_to_meters_conversion').get('calib_file')
|
|
1480
1667
|
if calib_file == '':
|
|
1481
1668
|
calib_file = None
|
|
1482
1669
|
else:
|
|
1483
|
-
calib_file =
|
|
1670
|
+
calib_file = Path(calib_file).resolve()
|
|
1484
1671
|
if not calib_file.is_file():
|
|
1485
1672
|
raise FileNotFoundError(f'Error: Could not find calibration file {calib_file}. Check that the file exists.')
|
|
1486
|
-
# Calibration from person height
|
|
1487
|
-
floor_angle = config_dict.get('px_to_meters_conversion').get('floor_angle') # 'auto' or float
|
|
1488
|
-
floor_angle = np.radians(float(floor_angle)) if floor_angle != 'auto' else floor_angle
|
|
1489
|
-
xy_origin = config_dict.get('px_to_meters_conversion').get('xy_origin') # ['auto'] or [x, y]
|
|
1490
|
-
xy_origin = [float(o) for o in xy_origin] if xy_origin != ['auto'] else 'auto'
|
|
1491
|
-
|
|
1492
|
-
keypoint_likelihood_threshold = config_dict.get('pose').get('keypoint_likelihood_threshold')
|
|
1493
|
-
average_likelihood_threshold = config_dict.get('pose').get('average_likelihood_threshold')
|
|
1494
|
-
keypoint_number_threshold = config_dict.get('pose').get('keypoint_number_threshold')
|
|
1495
1673
|
|
|
1496
1674
|
# Angles advanced settings
|
|
1675
|
+
display_angle_values_on = config_dict.get('angles').get('display_angle_values_on')
|
|
1676
|
+
fontSize = config_dict.get('angles').get('fontSize')
|
|
1677
|
+
thickness = 1 if fontSize < 0.8 else 2
|
|
1497
1678
|
joint_angle_names = config_dict.get('angles').get('joint_angles')
|
|
1498
1679
|
segment_angle_names = config_dict.get('angles').get('segment_angles')
|
|
1499
1680
|
angle_names = joint_angle_names + segment_angle_names
|
|
1500
1681
|
angle_names = [angle_name.lower() for angle_name in angle_names]
|
|
1501
|
-
display_angle_values_on = config_dict.get('angles').get('display_angle_values_on')
|
|
1502
|
-
fontSize = config_dict.get('angles').get('fontSize')
|
|
1503
|
-
thickness = 1 if fontSize < 0.8 else 2
|
|
1504
1682
|
flip_left_right = config_dict.get('angles').get('flip_left_right')
|
|
1505
1683
|
correct_segment_angles_with_floor_angle = config_dict.get('angles').get('correct_segment_angles_with_floor_angle')
|
|
1506
1684
|
|
|
@@ -1660,6 +1838,7 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1660
1838
|
keypoints_names = [node.name for _, _, node in RenderTree(pose_model) if node.id!=None]
|
|
1661
1839
|
t0 = 0
|
|
1662
1840
|
tf = (cap.get(cv2.CAP_PROP_FRAME_COUNT)-1) / fps if cap.get(cv2.CAP_PROP_FRAME_COUNT)>0 else float('inf')
|
|
1841
|
+
kpt_id_max = max(keypoints_ids)+1
|
|
1663
1842
|
|
|
1664
1843
|
# Set up pose tracker
|
|
1665
1844
|
try:
|
|
@@ -1672,7 +1851,7 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1672
1851
|
except:
|
|
1673
1852
|
logging.error('Error: Pose estimation failed. Check in Config.toml that pose_model and mode are valid.')
|
|
1674
1853
|
raise ValueError('Error: Pose estimation failed. Check in Config.toml that pose_model and mode are valid.')
|
|
1675
|
-
logging.info(f'Persons
|
|
1854
|
+
logging.info(f'Persons detection is run every {det_frequency} frames (pose estimation is run at every frame). Tracking is done with {tracking_mode}.')
|
|
1676
1855
|
|
|
1677
1856
|
if tracking_mode == 'deepsort':
|
|
1678
1857
|
logging.info(f'Deepsort parameters: {deepsort_params}.')
|
|
@@ -1748,60 +1927,64 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1748
1927
|
if video_file == "webcam":
|
|
1749
1928
|
out_vid.write(frame)
|
|
1750
1929
|
|
|
1751
|
-
#
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
if '
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1930
|
+
try: # Frames with no detection cause errors on MacOS CoreMLExecutionProvider
|
|
1931
|
+
# Detect poses
|
|
1932
|
+
keypoints, scores = pose_tracker(frame)
|
|
1933
|
+
|
|
1934
|
+
# Non maximum suppression (at pose level, not detection, and only using likely keypoints)
|
|
1935
|
+
frame_shape = frame.shape
|
|
1936
|
+
mask_scores = np.mean(scores, axis=1) > 0.2
|
|
1937
|
+
|
|
1938
|
+
likely_keypoints = np.where(mask_scores[:, np.newaxis, np.newaxis], keypoints, np.nan)
|
|
1939
|
+
likely_scores = np.where(mask_scores[:, np.newaxis], scores, np.nan)
|
|
1940
|
+
likely_bboxes = bbox_xyxy_compute(frame_shape, likely_keypoints, padding=0)
|
|
1941
|
+
score_likely_bboxes = np.nanmean(likely_scores, axis=1)
|
|
1942
|
+
|
|
1943
|
+
valid_indices = np.where(~np.isnan(score_likely_bboxes))[0]
|
|
1944
|
+
if len(valid_indices) > 0:
|
|
1945
|
+
valid_bboxes = likely_bboxes[valid_indices]
|
|
1946
|
+
valid_scores = score_likely_bboxes[valid_indices]
|
|
1947
|
+
keep_valid = nms(valid_bboxes, valid_scores, nms_thr=0.45)
|
|
1948
|
+
keep = valid_indices[keep_valid]
|
|
1949
|
+
else:
|
|
1950
|
+
keep = []
|
|
1951
|
+
keypoints, scores = likely_keypoints[keep], likely_scores[keep]
|
|
1952
|
+
|
|
1953
|
+
# # Debugging: display detected keypoints on the frame
|
|
1954
|
+
# colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255), (0,255,255), (128,0,0), (0,128,0), (0,0,128), (128,128,0), (128,0,128), (0,128,128)]
|
|
1955
|
+
# bboxes = likely_bboxes[keep]
|
|
1956
|
+
# for person_idx in range(len(keypoints)):
|
|
1957
|
+
# for kpt_idx, kpt in enumerate(keypoints[person_idx]):
|
|
1958
|
+
# if not np.isnan(kpt).any():
|
|
1959
|
+
# cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3, colors[person_idx%len(colors)], -1)
|
|
1960
|
+
# if not np.isnan(bboxes[person_idx]).any():
|
|
1961
|
+
# cv2.rectangle(frame, (int(bboxes[person_idx][0]), int(bboxes[person_idx][1])), (int(bboxes[person_idx][2]), int(bboxes[person_idx][3])), colors[person_idx%len(colors)], 1)
|
|
1962
|
+
# cv2.imshow(f'{video_file} Sports2D', frame)
|
|
1963
|
+
|
|
1964
|
+
# Track poses across frames
|
|
1965
|
+
if tracking_mode == 'deepsort':
|
|
1966
|
+
keypoints, scores = sort_people_deepsort(keypoints, scores, deepsort_tracker, frame, frame_count)
|
|
1967
|
+
if tracking_mode == 'sports2d':
|
|
1968
|
+
if 'prev_keypoints' not in locals(): prev_keypoints = keypoints
|
|
1969
|
+
prev_keypoints, keypoints, scores = sort_people_sports2d(prev_keypoints, keypoints, scores=scores, max_dist=max_distance)
|
|
1970
|
+
else:
|
|
1971
|
+
pass
|
|
1972
|
+
|
|
1973
|
+
# # Debugging: display detected keypoints on the frame
|
|
1974
|
+
# colors = [(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255), (0,255,255), (128,0,0), (0,128,0), (0,0,128), (128,128,0), (128,0,128), (0,128,128)]
|
|
1975
|
+
# for person_idx in range(len(keypoints)):
|
|
1976
|
+
# for kpt_idx, kpt in enumerate(keypoints[person_idx]):
|
|
1977
|
+
# if not np.isnan(kpt).any():
|
|
1978
|
+
# cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3, colors[person_idx%len(colors)], -1)
|
|
1979
|
+
# # if not np.isnan(bboxes[person_idx]).any():
|
|
1980
|
+
# # cv2.rectangle(frame, (int(bboxes[person_idx][0]), int(bboxes[person_idx][1])), (int(bboxes[person_idx][2]), int(bboxes[person_idx][3])), colors[person_idx%len(colors)], 1)
|
|
1981
|
+
# cv2.imshow(f'{video_file} Sports2D', frame)
|
|
1982
|
+
# # if (cv2.waitKey(1) & 0xFF) == ord('q') or (cv2.waitKey(1) & 0xFF) == 27:
|
|
1983
|
+
# # break
|
|
1984
|
+
# # input()
|
|
1985
|
+
except:
|
|
1986
|
+
keypoints = np.full((1,kpt_id_max,2), fill_value=np.nan)
|
|
1987
|
+
scores = np.full((1,kpt_id_max), fill_value=np.nan)
|
|
1805
1988
|
|
|
1806
1989
|
# Process coordinates and compute angles
|
|
1807
1990
|
valid_X, valid_Y, valid_scores = [], [], []
|
|
@@ -1893,6 +2076,10 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
1893
2076
|
if (cv2.waitKey(1) & 0xFF) == ord('q') or (cv2.waitKey(1) & 0xFF) == 27:
|
|
1894
2077
|
break
|
|
1895
2078
|
|
|
2079
|
+
# # Debugging
|
|
2080
|
+
# img_output_path = img_output_dir / f'{video_file_stem}_frame{frame_nb:06d}.png'
|
|
2081
|
+
# cv2.imwrite(str(img_output_path), img)
|
|
2082
|
+
|
|
1896
2083
|
all_frames_X.append(np.array(valid_X))
|
|
1897
2084
|
all_frames_X_flipped.append(np.array(valid_X_flipped))
|
|
1898
2085
|
all_frames_Y.append(np.array(valid_Y))
|
|
@@ -2095,12 +2282,20 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2095
2282
|
if not to_meters and (show_plots or save_plots):
|
|
2096
2283
|
pw = pose_plots(trc_data_unfiltered_i, trc_data_i, i, show=show_plots)
|
|
2097
2284
|
if save_plots:
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2285
|
+
if show_plots:
|
|
2286
|
+
for n, f in enumerate(pw.figure_handles):
|
|
2287
|
+
dpi = pw.canvases[n].figure.dpi
|
|
2288
|
+
f.set_size_inches(1280/dpi, 720/dpi)
|
|
2289
|
+
title = pw.tabs.tabText(n)
|
|
2290
|
+
plot_path = plots_output_dir / (pose_output_path.stem + f'_person{i:02d}_px_{title.replace(" ","_").replace("/","_")}.png')
|
|
2291
|
+
f.savefig(plot_path, dpi=dpi, bbox_inches='tight')
|
|
2292
|
+
else: # Tabbed plots not used
|
|
2293
|
+
for title, f in pw:
|
|
2294
|
+
dpi = f.dpi
|
|
2295
|
+
f.set_size_inches(1280/dpi, 720/dpi)
|
|
2296
|
+
plot_path = plots_output_dir / (pose_output_path.stem + f'_person{i:02d}_px_{title.replace(" ","_").replace("/","_")}.png')
|
|
2297
|
+
f.savefig(plot_path, dpi=dpi, bbox_inches='tight')
|
|
2298
|
+
plt.close(f)
|
|
2104
2299
|
logging.info(f'Pose plots (px) saved in {plots_output_dir}.')
|
|
2105
2300
|
|
|
2106
2301
|
all_frames_X_processed[:,idx_person,:], all_frames_Y_processed[:,idx_person,:] = all_frames_X_person_filt, all_frames_Y_person_filt
|
|
@@ -2112,102 +2307,99 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2112
2307
|
trc_data_m = []
|
|
2113
2308
|
if to_meters and save_pose:
|
|
2114
2309
|
logging.info('\nConverting pose to meters:')
|
|
2115
|
-
|
|
2310
|
+
|
|
2116
2311
|
# Compute height in px of the first person
|
|
2117
2312
|
height_px = compute_height(trc_data[0].iloc[:,1:], new_keypoints_names,
|
|
2118
2313
|
fastest_frames_to_remove_percent=fastest_frames_to_remove_percent, close_to_zero_speed=close_to_zero_speed_px, large_hip_knee_angles=large_hip_knee_angles, trimmed_extrema_percent=trimmed_extrema_percent)
|
|
2119
2314
|
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2315
|
+
# Compute distance from camera to compensate for perspective effects
|
|
2316
|
+
distance_m = get_distance_from_camera(perspective_value=perspective_value, perspective_unit=perspective_unit,
|
|
2317
|
+
calib_file=calib_file, height_px=height_px, height_m=first_person_height,
|
|
2318
|
+
cam_width=cam_width, cam_height=cam_height)
|
|
2319
|
+
|
|
2320
|
+
# Compute floor angle and xy_origin to compensate for camera horizon and person position
|
|
2321
|
+
floor_angle_estim, xy_origin_estim, gait_direction = get_floor_params(floor_angle=floor_angle, xy_origin=xy_origin,
|
|
2322
|
+
calib_file=calib_file, height_px=height_px, height_m=first_person_height,
|
|
2323
|
+
fps=fps, trc_data=trc_data[0], score_data=score_data[0], toe_speed_below=1, score_threshold=average_likelihood_threshold,
|
|
2324
|
+
cam_width=cam_width, cam_height=cam_height)
|
|
2325
|
+
cx, cy = xy_origin_estim
|
|
2326
|
+
direction_person0 = 'right' if gait_direction > 0.3 \
|
|
2327
|
+
else 'left' if gait_direction < -0.3 \
|
|
2328
|
+
else 'front'
|
|
2329
|
+
|
|
2330
|
+
logging.info(f'Converting from pixels to meters using a person height of {first_person_height:.2f} in meters (manual input), and of {height_px:.2f} in pixels (calculated).')
|
|
2331
|
+
|
|
2332
|
+
perspective_messages = {
|
|
2333
|
+
"distance_m": f"(obtained from a manual input).",
|
|
2334
|
+
"f_px": f"(calculated from a focal length of {perspective_value:.2f} m).",
|
|
2335
|
+
"fov_deg": f"(calculated from a field of view of {perspective_value:.2f} deg).",
|
|
2336
|
+
"fov_rad": f"(calculated from a field of view of {perspective_value:.2f} rad).",
|
|
2337
|
+
"from_calib": f"(calculated from a calibration file: {calib_file}).",
|
|
2338
|
+
}
|
|
2339
|
+
message = perspective_messages.get(perspective_unit, "")
|
|
2340
|
+
logging.info(f'Perspective effects corrected using a camera-to-person distance of {distance_m:.2f} m {message}')
|
|
2341
|
+
|
|
2342
|
+
floor_angle_messages = {
|
|
2343
|
+
"manual": "manual input.",
|
|
2344
|
+
"auto": "gait kinematics.",
|
|
2345
|
+
"from_kinematics": "gait kinematics.",
|
|
2346
|
+
"from_calib": "a calibration file.",
|
|
2347
|
+
}
|
|
2348
|
+
if isinstance(floor_angle, (int, float)):
|
|
2349
|
+
key = "manual"
|
|
2155
2350
|
else:
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
logging.warning(f'The RBigToe and LBigToe are missing from your pose estimation model. Using ankles - 13 cm to compute the floor line.')
|
|
2168
|
-
except:
|
|
2169
|
-
floor_angle_estim = 0
|
|
2170
|
-
xy_origin_estim = cam_width/2, cam_height/2
|
|
2171
|
-
logging.warning(f'Could not estimate the floor angle and xy_origin from person {0}. Make sure that the full body is visible. Using floor angle = 0° and xy_origin = [{cam_width/2}, {cam_height/2}] px.')
|
|
2172
|
-
if not floor_angle == 'auto':
|
|
2173
|
-
floor_angle_estim = floor_angle
|
|
2174
|
-
if xy_origin == 'auto':
|
|
2175
|
-
cx, cy = xy_origin_estim
|
|
2351
|
+
key = floor_angle
|
|
2352
|
+
message = floor_angle_messages.get(key, "")
|
|
2353
|
+
logging.info(f'Camera horizon: {np.degrees(floor_angle_estim):.2f}°, corrected using {message}')
|
|
2354
|
+
|
|
2355
|
+
def get_correction_message(xy_origin):
|
|
2356
|
+
if all(isinstance(o, (int, float)) for o in xy_origin) and len(xy_origin) == 2:
|
|
2357
|
+
return "manual input."
|
|
2358
|
+
elif xy_origin == ["auto"] or xy_origin == ["from_kinematics"]:
|
|
2359
|
+
return "gait kinematics."
|
|
2360
|
+
elif xy_origin == ["from_calib"]:
|
|
2361
|
+
return "a calibration file."
|
|
2176
2362
|
else:
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2363
|
+
return "."
|
|
2364
|
+
message = get_correction_message(xy_origin)
|
|
2365
|
+
logging.info(f'Floor level: {cy:.2f} px (from the top of the image), gait starting at {cx:.2f} px in the {direction_person0} direction for the first person. Corrected using {message}\n')
|
|
2366
|
+
|
|
2367
|
+
# Prepare calibration data
|
|
2368
|
+
R90z = np.array([[0.0, -1.0, 0.0],
|
|
2369
|
+
[1.0, 0.0, 0.0],
|
|
2370
|
+
[0.0, 0.0, 1.0]])
|
|
2371
|
+
R270x = np.array([[1.0, 0.0, 0.0],
|
|
2372
|
+
[0.0, 0.0, 1.0],
|
|
2373
|
+
[0.0, -1.0, 0.0]])
|
|
2374
|
+
|
|
2375
|
+
calib_file_path = output_dir / f'{video_file_stem}_Sports2D_calib.toml'
|
|
2376
|
+
|
|
2377
|
+
# name, size, distortions
|
|
2378
|
+
N = [video_file_stem]
|
|
2379
|
+
S = [[cam_width, cam_height]]
|
|
2380
|
+
D = [[0.0, 0.0, 0.0, 0.0]]
|
|
2381
|
+
|
|
2382
|
+
# Intrinsics
|
|
2383
|
+
f = height_px / first_person_height * distance_m
|
|
2384
|
+
cu = cam_width/2
|
|
2385
|
+
cv = cam_height/2
|
|
2386
|
+
K = np.array([[[f, 0.0, cu], [0.0, f, cv], [0.0, 0.0, 1.0]]])
|
|
2387
|
+
|
|
2388
|
+
# Extrinsics
|
|
2389
|
+
Rfloory = np.array([[np.cos(floor_angle_estim), 0.0, np.sin(floor_angle_estim)],
|
|
2390
|
+
[0.0, 1.0, 0.0],
|
|
2391
|
+
[-np.sin(floor_angle_estim), 0.0, np.cos(floor_angle_estim)]])
|
|
2392
|
+
R_world = R90z @ Rfloory @ R270x
|
|
2393
|
+
T_world = R90z @ np.array([-(cx-cu)/f*distance_m, -distance_m, (cy-cv)/f*distance_m])
|
|
2394
|
+
|
|
2395
|
+
R_cam, T_cam = world_to_camera_persp(R_world, T_world)
|
|
2396
|
+
Tvec_cam = T_cam.reshape(1,3).tolist()
|
|
2397
|
+
Rvec_cam = cv2.Rodrigues(R_cam)[0].reshape(1,3).tolist()
|
|
2207
2398
|
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2399
|
+
# Save calibration file
|
|
2400
|
+
if save_calib and not calib_file:
|
|
2401
|
+
toml_write(calib_file_path, N, S, D, K, Rvec_cam, Tvec_cam)
|
|
2402
|
+
logging.info(f'Calibration saved to {calib_file_path}.')
|
|
2211
2403
|
|
|
2212
2404
|
|
|
2213
2405
|
# Coordinates in m
|
|
@@ -2220,9 +2412,9 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2220
2412
|
if visible_side_i == 'auto':
|
|
2221
2413
|
try:
|
|
2222
2414
|
if all(key in trc_data[i] for key in ['LBigToe', 'RBigToe']):
|
|
2223
|
-
_, _, gait_direction = compute_floor_line(trc_data[i], score_data[
|
|
2415
|
+
_, _, gait_direction = compute_floor_line(trc_data[i], score_data[i], keypoint_names=['LBigToe', 'RBigToe'], score_threshold=keypoint_likelihood_threshold) # toe_speed_below=1 bu default
|
|
2224
2416
|
else:
|
|
2225
|
-
_, _, gait_direction = compute_floor_line(trc_data[i], score_data[
|
|
2417
|
+
_, _, gait_direction = compute_floor_line(trc_data[i], score_data[i], keypoint_names=['LAnkle', 'RAnkle'], score_threshold=keypoint_likelihood_threshold)
|
|
2226
2418
|
logging.warning(f'The RBigToe and LBigToe are missing from your model. Gait direction will be determined from the ankle points.')
|
|
2227
2419
|
visible_side_i = 'right' if gait_direction > 0.3 \
|
|
2228
2420
|
else 'left' if gait_direction < -0.3 \
|
|
@@ -2237,8 +2429,9 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2237
2429
|
else:
|
|
2238
2430
|
logging.info(f'- Person {i}: Seen from the {visible_side_i}.')
|
|
2239
2431
|
|
|
2432
|
+
|
|
2240
2433
|
# Convert to meters
|
|
2241
|
-
px_to_m_i = [convert_px_to_meters(trc_data[i][kpt_name], first_person_height, height_px, cx, cy, -floor_angle_estim, visible_side=visible_side_i) for kpt_name in new_keypoints_names]
|
|
2434
|
+
px_to_m_i = [convert_px_to_meters(trc_data[i][kpt_name], first_person_height, height_px, distance_m, cam_width, cam_height, cx, cy, -floor_angle_estim, visible_side=visible_side_i) for kpt_name in new_keypoints_names]
|
|
2242
2435
|
trc_data_m_i = pd.concat([all_frames_time.rename('time')]+px_to_m_i, axis=1)
|
|
2243
2436
|
for c_id, c in enumerate(3*np.arange(len(trc_data_m_i.columns[3::3]))+1): # only X coordinates
|
|
2244
2437
|
first_run_start, last_run_end = first_run_starts_everyone[i][c_id], last_run_ends_everyone[i][c_id]
|
|
@@ -2247,18 +2440,26 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2247
2440
|
trc_data_m_i.iloc[first_run_start:last_run_end,c+2] = trc_data_m_i.iloc[first_run_start:last_run_end,c+2].ffill().bfill()
|
|
2248
2441
|
first_trim, last_trim = trc_data_m_i.isnull().any(axis=1).idxmin(), trc_data_m_i[::-1].isnull().any(axis=1).idxmin()
|
|
2249
2442
|
trc_data_m_i = trc_data_m_i.iloc[first_trim:last_trim+1,:]
|
|
2250
|
-
px_to_m_unfiltered_i = [convert_px_to_meters(trc_data_unfiltered[i][kpt_name], first_person_height, height_px, cx, cy, -floor_angle_estim) for kpt_name in new_keypoints_names]
|
|
2443
|
+
px_to_m_unfiltered_i = [convert_px_to_meters(trc_data_unfiltered[i][kpt_name], first_person_height, height_px, distance_m, cam_width, cam_height, cx, cy, -floor_angle_estim, visible_side=visible_side_i) for kpt_name in new_keypoints_names]
|
|
2251
2444
|
trc_data_unfiltered_m_i = pd.concat([all_frames_time.rename('time')]+px_to_m_unfiltered_i, axis=1)
|
|
2252
2445
|
|
|
2253
2446
|
if to_meters and (show_plots or save_plots):
|
|
2254
2447
|
pw = pose_plots(trc_data_unfiltered_m_i, trc_data_m_i, i, show=show_plots)
|
|
2255
2448
|
if save_plots:
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2449
|
+
if show_plots:
|
|
2450
|
+
for n, f in enumerate(pw.figure_handles):
|
|
2451
|
+
dpi = pw.canvases[n].figure.dpi
|
|
2452
|
+
f.set_size_inches(1280/dpi, 720/dpi)
|
|
2453
|
+
title = pw.tabs.tabText(n)
|
|
2454
|
+
plot_path = plots_output_dir / (pose_output_path.stem + f'_person{i:02d}_m_{title.replace(" ","_").replace("/","_")}.png')
|
|
2455
|
+
f.savefig(plot_path, dpi=dpi, bbox_inches='tight')
|
|
2456
|
+
else: # Tabbed plots not used
|
|
2457
|
+
for title, f in pw:
|
|
2458
|
+
dpi = f.dpi
|
|
2459
|
+
f.set_size_inches(1280/dpi, 720/dpi)
|
|
2460
|
+
plot_path = plots_output_dir / (pose_output_path.stem + f'_person{i:02d}_m_{title.replace(" ","_").replace("/","_")}.png')
|
|
2461
|
+
f.savefig(plot_path, dpi=dpi, bbox_inches='tight')
|
|
2462
|
+
plt.close(f)
|
|
2262
2463
|
logging.info(f'Pose plots (m) saved in {plots_output_dir}.')
|
|
2263
2464
|
|
|
2264
2465
|
# Write to trc file
|
|
@@ -2389,12 +2590,20 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2389
2590
|
if show_plots or save_plots:
|
|
2390
2591
|
pw = angle_plots(all_frames_angles_person, angle_data, i, show=show_plots) # i = current person
|
|
2391
2592
|
if save_plots:
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2593
|
+
if show_plots:
|
|
2594
|
+
for n, f in enumerate(pw.figure_handles):
|
|
2595
|
+
dpi = pw.canvases[n].figure.dpi
|
|
2596
|
+
f.set_size_inches(1280/dpi, 720/dpi)
|
|
2597
|
+
title = pw.tabs.tabText(n)
|
|
2598
|
+
plot_path = plots_output_dir / (pose_output_path.stem + f'_person{i:02d}_ang_{title.replace(" ","_").replace("/","_")}.png')
|
|
2599
|
+
f.savefig(plot_path, dpi=dpi, bbox_inches='tight')
|
|
2600
|
+
else: # Tabbed plots not used
|
|
2601
|
+
for title, f in pw:
|
|
2602
|
+
dpi = f.dpi
|
|
2603
|
+
f.set_size_inches(1280/dpi, 720/dpi)
|
|
2604
|
+
plot_path = plots_output_dir / (pose_output_path.stem + f'_person{i:02d}_ang_{title.replace(" ","_").replace("/","_")}.png')
|
|
2605
|
+
f.savefig(plot_path, dpi=dpi, bbox_inches='tight')
|
|
2606
|
+
plt.close(f)
|
|
2398
2607
|
logging.info(f'Pose plots (m) saved in {plots_output_dir}.')
|
|
2399
2608
|
|
|
2400
2609
|
|
|
@@ -2531,7 +2740,7 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
|
|
|
2531
2740
|
if (mot_file.parent/(mot_file.stem+'_ik.mot')).exists():
|
|
2532
2741
|
os.remove(mot_file.parent/(mot_file.stem+'_ik.mot'))
|
|
2533
2742
|
os.rename(mot_file, mot_file.parent/(mot_file.stem+'_ik.mot'))
|
|
2534
|
-
logging.info(f'.osim model and .mot motion file results saved to {kinematics_dir.resolve()}.\n')
|
|
2743
|
+
logging.info(f'.osim model and .mot motion file results saved to {kinematics_dir.resolve().parent}.\n')
|
|
2535
2744
|
|
|
2536
2745
|
# Move all files in pose-3d and kinematics to the output_dir
|
|
2537
2746
|
osim.Logger.removeFileSink()
|