sl-shared-assets 1.0.0rc12__py3-none-any.whl → 1.0.0rc14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sl-shared-assets might be problematic. Click here for more details.
- sl_shared_assets/__init__.py +8 -0
- sl_shared_assets/__init__.pyi +51 -0
- sl_shared_assets/ascension_tools.py +0 -4
- sl_shared_assets/ascension_tools.pyi +68 -0
- sl_shared_assets/cli.py +12 -13
- sl_shared_assets/cli.pyi +28 -0
- sl_shared_assets/data_classes.py +525 -435
- sl_shared_assets/data_classes.pyi +799 -0
- sl_shared_assets/packaging_tools.pyi +52 -0
- sl_shared_assets/server.py +22 -15
- sl_shared_assets/server.pyi +117 -0
- sl_shared_assets/suite2p.pyi +188 -0
- sl_shared_assets/transfer_tools.pyi +53 -0
- {sl_shared_assets-1.0.0rc12.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/METADATA +1 -1
- sl_shared_assets-1.0.0rc14.dist-info/RECORD +22 -0
- sl_shared_assets-1.0.0rc12.dist-info/RECORD +0 -14
- {sl_shared_assets-1.0.0rc12.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/WHEEL +0 -0
- {sl_shared_assets-1.0.0rc12.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/entry_points.txt +0 -0
- {sl_shared_assets-1.0.0rc12.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/licenses/LICENSE +0 -0
sl_shared_assets/data_classes.py
CHANGED
|
@@ -7,7 +7,6 @@ import re
|
|
|
7
7
|
import copy
|
|
8
8
|
import shutil as sh
|
|
9
9
|
from pathlib import Path
|
|
10
|
-
import warnings
|
|
11
10
|
from dataclasses import field, dataclass
|
|
12
11
|
|
|
13
12
|
import appdirs
|
|
@@ -15,10 +14,6 @@ from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
|
|
|
15
14
|
from ataraxis_data_structures import YamlConfig
|
|
16
15
|
from ataraxis_time.time_helpers import get_timestamp
|
|
17
16
|
|
|
18
|
-
# Ensures console is enabled when this file is imported
|
|
19
|
-
if not console.enabled:
|
|
20
|
-
console.enable()
|
|
21
|
-
|
|
22
17
|
|
|
23
18
|
def replace_root_path(path: Path) -> None:
|
|
24
19
|
"""Replaces the path to the local root directory used to store all Sun lab projects with the provided path.
|
|
@@ -104,8 +99,8 @@ class ProjectConfiguration(YamlConfig):
|
|
|
104
99
|
this field is configured automatically each time the class is instantiated through any method, so overwriting it
|
|
105
100
|
manually will not be respected."""
|
|
106
101
|
local_server_directory: str | Path = Path("/home/cybermouse/server/storage/sun_data")
|
|
107
|
-
"""The absolute path to the directory where all projects
|
|
108
|
-
locally accessible (mounted) using a network sharing protocol, such as SMB."""
|
|
102
|
+
"""The absolute path to the directory where the raw data portion of all projects is stored on the BioHPC server.
|
|
103
|
+
This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
|
|
109
104
|
local_nas_directory: str | Path = Path("/home/cybermouse/nas/rawdata")
|
|
110
105
|
"""The absolute path to the directory where all projects are stored on the Synology NAS. This directory should be
|
|
111
106
|
locally accessible (mounted) using a network sharing protocol, such as SMB."""
|
|
@@ -113,6 +108,9 @@ class ProjectConfiguration(YamlConfig):
|
|
|
113
108
|
"""The absolute path to the root mesoscope (ScanImagePC) directory where all mesoscope-acquired data is aggregated
|
|
114
109
|
during acquisition runtime. This directory should be locally accessible (mounted) using a network sharing
|
|
115
110
|
protocol, such as SMB."""
|
|
111
|
+
local_server_working_directory: str | Path = Path("/home/cybermouse/server/workdir/sun_data")
|
|
112
|
+
"""The absolute path to the directory where the processed data portion of all projects is stored on the BioHPC
|
|
113
|
+
server. This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
|
|
116
114
|
remote_storage_directory: str | Path = Path("/storage/sun_data")
|
|
117
115
|
"""The absolute path, relative to the BioHPC server root, to the directory where all projects are stored on the
|
|
118
116
|
slow (SSD) volume of the server. This path is used when running remote (server-side) jobs and, therefore, has to
|
|
@@ -257,6 +255,7 @@ class ProjectConfiguration(YamlConfig):
|
|
|
257
255
|
instance.local_mesoscope_directory = Path(instance.local_mesoscope_directory)
|
|
258
256
|
instance.local_nas_directory = Path(instance.local_nas_directory)
|
|
259
257
|
instance.local_server_directory = Path(instance.local_server_directory)
|
|
258
|
+
instance.local_server_working_directory = Path(instance.local_server_working_directory)
|
|
260
259
|
instance.remote_storage_directory = Path(instance.remote_storage_directory)
|
|
261
260
|
instance.remote_working_directory = Path(instance.remote_working_directory)
|
|
262
261
|
instance.google_credentials_path = Path(instance.google_credentials_path)
|
|
@@ -301,6 +300,7 @@ class ProjectConfiguration(YamlConfig):
|
|
|
301
300
|
original.local_mesoscope_directory = str(original.local_mesoscope_directory)
|
|
302
301
|
original.local_nas_directory = str(original.local_nas_directory)
|
|
303
302
|
original.local_server_directory = str(original.local_server_directory)
|
|
303
|
+
original.local_server_working_directory = str(original.local_server_working_directory)
|
|
304
304
|
original.remote_storage_directory = str(original.remote_storage_directory)
|
|
305
305
|
original.remote_working_directory = str(original.remote_working_directory)
|
|
306
306
|
original.google_credentials_path = str(original.google_credentials_path)
|
|
@@ -358,472 +358,510 @@ class ProjectConfiguration(YamlConfig):
|
|
|
358
358
|
|
|
359
359
|
@dataclass()
|
|
360
360
|
class RawData:
|
|
361
|
-
"""Stores the paths to the directories and files that make up the 'raw_data' session directory.
|
|
361
|
+
"""Stores the paths to the directories and files that make up the 'raw_data' session-specific directory.
|
|
362
362
|
|
|
363
363
|
The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
|
|
364
364
|
preprocessing does not alter the data, any data in that folder is considered 'raw'. The raw_data folder is initially
|
|
365
365
|
created on the VRPC and, after preprocessing, is copied to the BioHPC server and the Synology NAS for long-term
|
|
366
366
|
storage and further processing.
|
|
367
|
-
|
|
368
|
-
Notes:
|
|
369
|
-
The overall structure of the raw_data directory remains fixed for the entire lifetime of the data. It is reused
|
|
370
|
-
across all destinations.
|
|
371
367
|
"""
|
|
372
368
|
|
|
373
|
-
raw_data_path:
|
|
369
|
+
raw_data_path: Path = Path()
|
|
374
370
|
"""Stores the path to the root raw_data directory of the session. This directory stores all raw data during
|
|
375
371
|
acquisition and preprocessing. Note, preprocessing does not alter raw data, so at any point in time all data inside
|
|
376
372
|
the folder is considered 'raw'."""
|
|
377
|
-
camera_data_path:
|
|
373
|
+
camera_data_path: Path = Path()
|
|
378
374
|
"""Stores the path to the directory that contains all camera data acquired during the session. Primarily, this
|
|
379
375
|
includes .mp4 video files from each recorded camera."""
|
|
380
|
-
mesoscope_data_path:
|
|
376
|
+
mesoscope_data_path: Path = Path()
|
|
381
377
|
"""Stores the path to the directory that contains all Mesoscope data acquired during the session. Primarily, this
|
|
382
378
|
includes the mesoscope-acquired .tiff files (brain activity data) and the motion estimation data."""
|
|
383
|
-
behavior_data_path:
|
|
379
|
+
behavior_data_path: Path = Path()
|
|
384
380
|
"""Stores the path to the directory that contains all behavior data acquired during the session. Primarily, this
|
|
385
381
|
includes the .npz log files used by data-acquisition libraries to store all acquired data. The data stored in this
|
|
386
382
|
way includes the camera and mesoscope frame timestamps and the states of Mesoscope-VR components, such as lick
|
|
387
383
|
sensors, rotary encoders, and other modules."""
|
|
388
|
-
zaber_positions_path:
|
|
384
|
+
zaber_positions_path: Path = Path()
|
|
389
385
|
"""Stores the path to the zaber_positions.yaml file. This file contains the snapshot of all Zaber motor positions
|
|
390
386
|
at the end of the session. Zaber motors are used to position the LickPort and the HeadBar manipulators, which is
|
|
391
387
|
essential for supporting proper brain imaging and animal's running behavior during the session."""
|
|
392
|
-
session_descriptor_path:
|
|
388
|
+
session_descriptor_path: Path = Path()
|
|
393
389
|
"""Stores the path to the session_descriptor.yaml file. This file is partially filled by the system during runtime
|
|
394
390
|
and partially by the experimenter after the runtime. It contains session-specific information, such as the specific
|
|
395
391
|
training parameters, the positions of the Mesoscope objective and the notes made by the experimenter during
|
|
396
392
|
runtime."""
|
|
397
|
-
hardware_configuration_path:
|
|
393
|
+
hardware_configuration_path: Path = Path()
|
|
398
394
|
"""Stores the path to the hardware_configuration.yaml file. This file contains the partial snapshot of the
|
|
399
395
|
calibration parameters used by the Mesoscope-VR system components during runtime. Primarily, this is used during
|
|
400
396
|
data processing to read the .npz data log files generated during runtime."""
|
|
401
|
-
surgery_metadata_path:
|
|
397
|
+
surgery_metadata_path: Path = Path()
|
|
402
398
|
"""Stores the path to the surgery_metadata.yaml file. This file contains the most actual information about the
|
|
403
399
|
surgical intervention(s) performed on the animal prior to the session."""
|
|
404
|
-
project_configuration_path:
|
|
400
|
+
project_configuration_path: Path = Path()
|
|
405
401
|
"""Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
|
|
406
402
|
parameters for the session's project."""
|
|
407
|
-
session_data_path:
|
|
403
|
+
session_data_path: Path = Path()
|
|
408
404
|
"""Stores the path to the session_data.yaml file. This path is used b y the SessionData instance to save itself to
|
|
409
405
|
disk as a .yaml file. The file contains all paths used during data acquisition and processing on both the VRPC and
|
|
410
406
|
the BioHPC server."""
|
|
411
|
-
experiment_configuration_path:
|
|
407
|
+
experiment_configuration_path: Path = Path()
|
|
412
408
|
"""Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
|
|
413
409
|
experiment runtime configuration used by the session. This file is only created for experiment session. It does not
|
|
414
410
|
exist for behavior training sessions."""
|
|
415
|
-
mesoscope_positions_path:
|
|
411
|
+
mesoscope_positions_path: Path = Path()
|
|
416
412
|
"""Stores the path to the mesoscope_positions.yaml file. This file contains the snapshot of the positions used
|
|
417
413
|
by the Mesoscope at the end of the session. This includes both the physical position of the mesoscope objective and
|
|
418
414
|
the 'virtual' tip, tilt, and fastZ positions set via ScanImage software. This file is only created for experiment
|
|
419
415
|
sessions that use the mesoscope, it is omitted for behavior training sessions."""
|
|
420
|
-
window_screenshot_path:
|
|
416
|
+
window_screenshot_path: Path = Path()
|
|
421
417
|
"""Stores the path to the .png screenshot of the ScanImagePC screen. The screenshot should contain the image of the
|
|
422
418
|
cranial window and the red-dot alignment windows. This is used to generate a visual snapshot of the cranial window
|
|
423
419
|
alignment and appearance for each experiment session. This file is only created for experiment sessions that use
|
|
424
420
|
the mesoscope, it is omitted for behavior training sessions."""
|
|
425
|
-
telomere_path:
|
|
421
|
+
telomere_path: Path = Path()
|
|
426
422
|
"""Stores the path to the telomere.bin file. This file is created by the data processing pipelines running on the
|
|
427
|
-
BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission.
|
|
428
|
-
|
|
429
|
-
for the existence of the file when it purges redundant data."""
|
|
430
|
-
checksum_path: str | Path = "null"
|
|
423
|
+
BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission."""
|
|
424
|
+
checksum_path: Path = Path()
|
|
431
425
|
"""Stores the path to the ax_checksum.txt file. This file is generated as part of packaging the data for
|
|
432
426
|
transmission and stores the xxHash-128 checksum of the data. It is used to verify that the transmission did not
|
|
433
|
-
damage or otherwise alter the data
|
|
427
|
+
damage or otherwise alter the data."""
|
|
434
428
|
|
|
435
|
-
def
|
|
436
|
-
"""
|
|
437
|
-
|
|
438
|
-
|
|
429
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
430
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
431
|
+
|
|
432
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
433
|
+
machine that instantiates the class.
|
|
439
434
|
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
if self.camera_data_path == "null":
|
|
445
|
-
self.camera_data_path = self.raw_data_path.joinpath("camera_data")
|
|
446
|
-
if self.mesoscope_data_path == "null":
|
|
447
|
-
self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
|
|
448
|
-
if self.behavior_data_path == "null":
|
|
449
|
-
self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
|
|
450
|
-
if self.zaber_positions_path == "null":
|
|
451
|
-
self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
|
|
452
|
-
if self.session_descriptor_path == "null":
|
|
453
|
-
self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
|
|
454
|
-
if self.hardware_configuration_path == "null":
|
|
455
|
-
self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
|
|
456
|
-
if self.surgery_metadata_path == "null":
|
|
457
|
-
self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
|
|
458
|
-
if self.project_configuration_path == "null":
|
|
459
|
-
self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
|
|
460
|
-
if self.session_data_path == "null":
|
|
461
|
-
self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
|
|
462
|
-
if self.experiment_configuration_path == "null":
|
|
463
|
-
self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
|
|
464
|
-
if self.mesoscope_positions_path == "null":
|
|
465
|
-
self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
|
|
466
|
-
if self.window_screenshot_path == "null":
|
|
467
|
-
self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
|
|
468
|
-
if self.telomere_path == "null":
|
|
469
|
-
self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
|
|
470
|
-
if self.checksum_path == "null":
|
|
471
|
-
self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
|
|
472
|
-
|
|
473
|
-
def make_string(self) -> None:
|
|
474
|
-
"""Converts all Path objects stored inside the class to strings.
|
|
475
|
-
|
|
476
|
-
This transformation is required to support dumping class data into a .YAML file so that the data can be stored
|
|
477
|
-
on disk.
|
|
435
|
+
Args:
|
|
436
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
437
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
438
|
+
the managed session.
|
|
478
439
|
"""
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
self.
|
|
482
|
-
self.
|
|
483
|
-
self.
|
|
484
|
-
self.
|
|
485
|
-
self.
|
|
486
|
-
self.
|
|
487
|
-
self.
|
|
488
|
-
self.
|
|
489
|
-
self.
|
|
490
|
-
self.
|
|
491
|
-
self.
|
|
492
|
-
self.
|
|
493
|
-
self.
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
440
|
+
|
|
441
|
+
# Generates the managed paths
|
|
442
|
+
self.raw_data_path = root_directory_path
|
|
443
|
+
self.camera_data_path = self.raw_data_path.joinpath("camera_data")
|
|
444
|
+
self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
|
|
445
|
+
self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
|
|
446
|
+
self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
|
|
447
|
+
self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
|
|
448
|
+
self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
|
|
449
|
+
self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
|
|
450
|
+
self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
|
|
451
|
+
self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
|
|
452
|
+
self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
|
|
453
|
+
self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
|
|
454
|
+
self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
|
|
455
|
+
self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
|
|
456
|
+
self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
|
|
457
|
+
|
|
458
|
+
def make_directories(self) -> None:
|
|
459
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
460
|
+
ensure_directory_exists(self.raw_data_path)
|
|
461
|
+
ensure_directory_exists(self.camera_data_path)
|
|
462
|
+
ensure_directory_exists(self.mesoscope_data_path)
|
|
463
|
+
ensure_directory_exists(self.behavior_data_path)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
@dataclass()
|
|
467
|
+
class DeepLabCutData:
|
|
468
|
+
"""Stores the paths to the directories and files that make up the 'deeplabcut' project-specific directory.
|
|
469
|
+
|
|
470
|
+
DeepLabCut (DLC) is used to track animal body parts and poses in video data acquired during experiment and training
|
|
471
|
+
sessions. Since DLC is designed to work with projects, rather than single animals or sessions, each Sun lab
|
|
472
|
+
project data hierarchy contains a dedicated 'deeplabcut' directory under the root project directory. The contents of
|
|
473
|
+
that directory are largely managed by the DLC itself. Therefore, each session of a given project refers to and
|
|
474
|
+
uses the same 'deeplabcut' directory.
|
|
475
|
+
"""
|
|
476
|
+
|
|
477
|
+
deeplabcut_path: Path = Path()
|
|
478
|
+
"""Stores the path to the project-specific DeepLabCut directory. This folder stores all DeepLabCut data specific to
|
|
479
|
+
a single project, which is reused during the processing of all sessions of the project."""
|
|
480
|
+
|
|
481
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
482
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
483
|
+
|
|
484
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
485
|
+
machine that instantiates the class.
|
|
486
|
+
|
|
487
|
+
Args:
|
|
488
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
489
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
490
|
+
the managed session.
|
|
499
491
|
"""
|
|
500
|
-
ensure_directory_exists(Path(self.raw_data_path))
|
|
501
|
-
ensure_directory_exists(Path(self.camera_data_path))
|
|
502
|
-
ensure_directory_exists(Path(self.mesoscope_data_path))
|
|
503
|
-
ensure_directory_exists(Path(self.behavior_data_path))
|
|
504
492
|
|
|
505
|
-
|
|
506
|
-
|
|
493
|
+
# Generates the managed paths
|
|
494
|
+
self.deeplabcut_path = root_directory_path
|
|
495
|
+
|
|
496
|
+
def make_directories(self) -> None:
|
|
497
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
498
|
+
ensure_directory_exists(self.deeplabcut_path)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
@dataclass()
|
|
502
|
+
class ConfigurationData:
|
|
503
|
+
"""Stores the paths to the directories and files that make up the 'configuration' project-specific directory.
|
|
507
504
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
505
|
+
The configuration directory contains various configuration files and settings used by data acquisition,
|
|
506
|
+
preprocessing, and processing pipelines in the lab. Generally, all configuration settings are defined once for each
|
|
507
|
+
project and are reused for every session within the project. Therefore, this directory is created under each main
|
|
508
|
+
project directory.
|
|
512
509
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
510
|
+
Notes:
|
|
511
|
+
Some attribute names inside this section match the names in the RawData section. This is intentional, as some
|
|
512
|
+
configuration files are copied into the raw_data session directories to allow reinstating the session data
|
|
513
|
+
hierarchy across machines.
|
|
514
|
+
"""
|
|
515
|
+
|
|
516
|
+
configuration_path: Path = Path()
|
|
517
|
+
"""Stores the path to the project-specific configuration directory. This directory is used by all animals
|
|
518
|
+
and sessions of the project to store all pan-project configuration files. The configuration data is reused by all
|
|
519
|
+
sessions in the project."""
|
|
520
|
+
experiment_configuration_path: Path = Path()
|
|
521
|
+
"""Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
|
|
522
|
+
experiment runtime configuration used by the session. This file is only created for experiment session. It does not
|
|
523
|
+
exist for behavior training sessions."""
|
|
524
|
+
project_configuration_path: Path = Path()
|
|
525
|
+
"""Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
|
|
526
|
+
parameters for the session's project."""
|
|
527
|
+
suite2p_configuration_path: Path = Path()
|
|
528
|
+
"""Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
|
|
529
|
+
the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
|
|
530
|
+
registration pipeline, which is applied to each session that generates brain activity data."""
|
|
531
|
+
multiday_configuration_path: Path = Path()
|
|
532
|
+
"""Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
|
|
533
|
+
on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
|
|
534
|
+
sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
|
|
535
|
+
|
|
536
|
+
def resolve_paths(self, root_directory_path: Path, experiment_name: str | None = None) -> None:
|
|
537
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
538
|
+
|
|
539
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
540
|
+
machine that instantiates the class.
|
|
516
541
|
|
|
517
542
|
Args:
|
|
518
|
-
|
|
519
|
-
|
|
543
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
544
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
545
|
+
the managed session.
|
|
546
|
+
experiment_name: Optionally specifies the name of the experiment executed as part of the managed session's
|
|
547
|
+
runtime. This is used to correctly configure the path to the specific ExperimentConfiguration data file.
|
|
548
|
+
If the managed session is not an Experiment session, this parameter should be set to None.
|
|
520
549
|
"""
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
self.
|
|
529
|
-
self.
|
|
530
|
-
self.
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
self.project_configuration_path = new_root.joinpath(Path(self.project_configuration_path).relative_to(old_root))
|
|
536
|
-
self.session_data_path = new_root.joinpath(Path(self.session_data_path).relative_to(old_root))
|
|
537
|
-
self.experiment_configuration_path = new_root.joinpath(
|
|
538
|
-
Path(self.experiment_configuration_path).relative_to(old_root)
|
|
539
|
-
)
|
|
540
|
-
self.mesoscope_positions_path = new_root.joinpath(Path(self.mesoscope_positions_path).relative_to(old_root))
|
|
541
|
-
self.window_screenshot_path = new_root.joinpath(Path(self.window_screenshot_path).relative_to(old_root))
|
|
542
|
-
self.telomere_path = new_root.joinpath(Path(self.telomere_path).relative_to(old_root))
|
|
543
|
-
self.checksum_path = new_root.joinpath(Path(self.checksum_path).relative_to(old_root))
|
|
550
|
+
|
|
551
|
+
# Generates the managed paths
|
|
552
|
+
self.configuration_path = root_directory_path
|
|
553
|
+
if experiment_name is None:
|
|
554
|
+
self.experiment_configuration_path = self.configuration_path.joinpath("null")
|
|
555
|
+
else:
|
|
556
|
+
self.experiment_configuration_path = self.configuration_path.joinpath(f"{experiment_name}.yaml")
|
|
557
|
+
self.project_configuration_path = self.configuration_path.joinpath("project_configuration.yaml")
|
|
558
|
+
self.suite2p_configuration_path = self.configuration_path.joinpath("suite2p_configuration.yaml")
|
|
559
|
+
self.multiday_configuration_path = self.configuration_path.joinpath("multiday_configuration.yaml")
|
|
560
|
+
|
|
561
|
+
def make_directories(self) -> None:
|
|
562
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
563
|
+
ensure_directory_exists(self.configuration_path)
|
|
544
564
|
|
|
545
565
|
|
|
546
566
|
@dataclass()
|
|
547
567
|
class ProcessedData:
|
|
548
|
-
"""Stores the paths to the directories and files that make up the 'processed_data' session directory.
|
|
568
|
+
"""Stores the paths to the directories and files that make up the 'processed_data' session-specific directory.
|
|
549
569
|
|
|
550
|
-
The processed_data directory stores the data generated by various processing pipelines from the raw data
|
|
551
|
-
data represents an intermediate step between raw data and the dataset used in
|
|
552
|
-
designed to be analyzed.
|
|
570
|
+
The processed_data directory stores the data generated by various processing pipelines from the raw data (contents
|
|
571
|
+
of the raw_data directory). Processed data represents an intermediate step between raw data and the dataset used in
|
|
572
|
+
the data analysis, but is not itself designed to be analyzed.
|
|
553
573
|
|
|
554
574
|
Notes:
|
|
555
575
|
The paths from this section are typically used only on the BioHPC server. This is because most data processing
|
|
556
576
|
in the lab is performed using the processing server's resources. On the server, processed data is stored on
|
|
557
|
-
the fast volume, in contrast to raw data, which is stored on the slow volume.
|
|
558
|
-
the class resolves the paths in this section both locally and globally (on the server).
|
|
577
|
+
the fast (NVME) drive volume, in contrast to raw data, which is stored on the slow (SSD) drive volume.
|
|
559
578
|
|
|
579
|
+
When this class is instantiated on a machine other than BioHPC server, for example, to test processing
|
|
580
|
+
pipelines, it uses the same drive as the raw_data folder to create the processed_data folder. This relies on the
|
|
581
|
+
assumption that non-server machines in the lab only use fast NVME drives, so there is no need to separate
|
|
582
|
+
storage and processing volumes.
|
|
560
583
|
"""
|
|
561
584
|
|
|
562
|
-
processed_data_path:
|
|
585
|
+
processed_data_path: Path = Path()
|
|
563
586
|
"""Stores the path to the root processed_data directory of the session. This directory stores the processed data
|
|
564
587
|
as it is generated by various data processing pipelines."""
|
|
565
|
-
|
|
566
|
-
"""Stores the path to the
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
"""Stores the
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
processing_tracker_path: str | Path = "null"
|
|
585
|
-
"""Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
|
|
586
|
-
the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
|
|
587
|
-
session and the status (success / failure) of each processing step.
|
|
588
|
+
camera_data_path: Path = Path()
|
|
589
|
+
"""Stores the path to the directory that contains video tracking data generated by our DeepLabCut-based video
|
|
590
|
+
processing pipelines."""
|
|
591
|
+
mesoscope_data_path: Path = Path()
|
|
592
|
+
"""Stores path to the directory that contains processed brain activity (cell) data generated by our suite2p-based
|
|
593
|
+
photometry processing pipelines (single day and multi day)."""
|
|
594
|
+
behavior_data_path: Path = Path()
|
|
595
|
+
"""Stores the path to the directory that contains the non-video behavior and system runtime data extracted from
|
|
596
|
+
.npz log files by our in-house log parsing pipeline."""
|
|
597
|
+
job_logs_path: Path = Path()
|
|
598
|
+
"""Stores the path to the directory that stores the standard output and standard error data collected during
|
|
599
|
+
server-side data processing pipeline runtimes. Since we use SLURM job manager to execute multiple compute jobs on
|
|
600
|
+
the BioHPC server, all information sent to the terminal during runtime is redirected to text files stored in this
|
|
601
|
+
directory."""
|
|
602
|
+
processing_tracker_path: Path = Path()
|
|
603
|
+
"""Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory.
|
|
604
|
+
This file is used to track which processing pipelines need to be applied to the target session and the status
|
|
605
|
+
(success / failure) each of these pipelines. Primarily, this is used to optimize data processing to avoid redundant
|
|
606
|
+
operations.
|
|
588
607
|
"""
|
|
589
|
-
multiday_configuration_path: str | Path = "null"
|
|
590
|
-
"""Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
|
|
591
|
-
on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
|
|
592
|
-
sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
|
|
593
608
|
|
|
594
|
-
def
|
|
595
|
-
"""
|
|
596
|
-
are converted to Path objects.
|
|
597
|
-
"""
|
|
609
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
610
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
598
611
|
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
if self.camera_data_path == "null":
|
|
607
|
-
self.camera_data_path = self.processed_data_path.joinpath("camera_data")
|
|
608
|
-
if self.mesoscope_data_path == "null":
|
|
609
|
-
self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
|
|
610
|
-
if self.behavior_data_path == "null":
|
|
611
|
-
self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
|
|
612
|
-
if self.suite2p_configuration_path == "null":
|
|
613
|
-
self.suite2p_configuration_path = self.processed_data_path.joinpath("suite2p_configuration.yaml")
|
|
614
|
-
if self.processing_tracker_path == "null":
|
|
615
|
-
self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
|
|
616
|
-
if self.multiday_configuration_path == "null":
|
|
617
|
-
self.multiday_configuration_path = self.processed_data_path.joinpath("multiday_configuration.yaml")
|
|
618
|
-
|
|
619
|
-
def make_string(self) -> None:
|
|
620
|
-
"""Converts all Path objects stored inside the class to strings.
|
|
621
|
-
|
|
622
|
-
This transformation is required to support dumping class data into a .YAML file so that the data can be stored
|
|
623
|
-
on disk.
|
|
624
|
-
"""
|
|
625
|
-
self.processed_data_path = str(self.processed_data_path)
|
|
626
|
-
self.deeplabcut_path = str(self.deeplabcut_path)
|
|
627
|
-
self.configuration_path = str(self.configuration_path)
|
|
628
|
-
self.camera_data_path = str(self.camera_data_path)
|
|
629
|
-
self.mesoscope_data_path = str(self.mesoscope_data_path)
|
|
630
|
-
self.behavior_data_path = str(self.behavior_data_path)
|
|
631
|
-
self.suite2p_configuration_path = str(self.suite2p_configuration_path)
|
|
632
|
-
self.processing_tracker_path = str(self.processing_tracker_path)
|
|
633
|
-
self.multiday_configuration_path = str(self.multiday_configuration_path)
|
|
634
|
-
|
|
635
|
-
def make_dirs(self) -> None:
|
|
636
|
-
"""Ensures that all major subdirectories of the processed_data directory exist.
|
|
637
|
-
|
|
638
|
-
This method is used by the BioHPC server to generate the processed_data directory as part of the sl-forgery
|
|
639
|
-
library runtime.
|
|
612
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
613
|
+
machine that instantiates the class.
|
|
614
|
+
|
|
615
|
+
Args:
|
|
616
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
617
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
618
|
+
the managed session.
|
|
640
619
|
"""
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
620
|
+
# Generates the managed paths
|
|
621
|
+
self.processed_data_path = root_directory_path
|
|
622
|
+
self.camera_data_path = self.processed_data_path.joinpath("camera_data")
|
|
623
|
+
self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
|
|
624
|
+
self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
|
|
625
|
+
self.job_logs_path = self.processed_data_path.joinpath("job_logs")
|
|
626
|
+
self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
|
|
627
|
+
|
|
628
|
+
def make_directories(self) -> None:
|
|
629
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
630
|
+
|
|
631
|
+
ensure_directory_exists(self.processed_data_path)
|
|
632
|
+
ensure_directory_exists(self.camera_data_path)
|
|
633
|
+
ensure_directory_exists(self.mesoscope_data_path)
|
|
634
|
+
ensure_directory_exists(self.behavior_data_path)
|
|
635
|
+
ensure_directory_exists(self.job_logs_path)
|
|
647
636
|
|
|
648
637
|
|
|
649
638
|
@dataclass()
|
|
650
|
-
class
|
|
651
|
-
"""Stores the paths to the directories and files that make up the 'persistent_data'
|
|
652
|
-
the ScanImagePC.
|
|
639
|
+
class VRPCPersistentData:
|
|
640
|
+
"""Stores the paths to the directories and files that make up the 'persistent_data' directory on the VRPC.
|
|
653
641
|
|
|
654
|
-
Persistent data directories are used
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
previous session.
|
|
642
|
+
Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
|
|
643
|
+
are purposefully designed for specific PCs that participate in data acquisition. This section manages the
|
|
644
|
+
animal-specific persistent_data directory stored on the VRPC.
|
|
658
645
|
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
only used internally by the sl-experiment or sl-forgery libraries and is not intended for end-users.
|
|
646
|
+
VRPC persistent data directory is used to preserve configuration data, such as the positions of Zaber motors and
|
|
647
|
+
Meososcope objective, so that they can be reused across sessions of the same animals. The data in this directory
|
|
648
|
+
is read at the beginning of each session and replaced at the end of each session.
|
|
663
649
|
"""
|
|
664
650
|
|
|
665
|
-
|
|
651
|
+
persistent_data_path: Path = Path()
|
|
666
652
|
"""Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
|
|
667
|
-
belongs, relative to the
|
|
668
|
-
|
|
669
|
-
vrpc_persistent_path: str | Path
|
|
670
|
-
"""Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
|
|
671
|
-
belongs, relative to the VRPC root. This directory is primarily used to back up the Zaber and Mesoscope positions,
|
|
672
|
-
so that they can be restored between sessions of the same animals."""
|
|
673
|
-
zaber_positions_path: str | Path = "null"
|
|
653
|
+
belongs, relative to the VRPC root. This directory is exclusively used on the VRPC."""
|
|
654
|
+
zaber_positions_path: Path = Path()
|
|
674
655
|
"""Stores the path to the Zaber motor positions snapshot generated at the end of the previous session runtime. This
|
|
675
656
|
is used to automatically restore all Zaber motors to the same position across all sessions."""
|
|
676
|
-
mesoscope_positions_path:
|
|
657
|
+
mesoscope_positions_path: Path = Path()
|
|
677
658
|
"""Stores the path to the Mesoscope positions snapshot generated at the end of the previous session runtime. This
|
|
678
659
|
is used to help the user to (manually) restore the Mesoscope to the same position across all sessions."""
|
|
679
|
-
motion_estimator_path: str | Path = "null"
|
|
680
|
-
"""Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
|
|
681
|
-
file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
|
|
682
660
|
|
|
683
|
-
def
|
|
684
|
-
"""
|
|
685
|
-
|
|
661
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
662
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
663
|
+
|
|
664
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
665
|
+
machine that instantiates the class.
|
|
666
|
+
|
|
667
|
+
Args:
|
|
668
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
669
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
670
|
+
the managed session.
|
|
686
671
|
"""
|
|
687
672
|
|
|
688
|
-
#
|
|
689
|
-
self.
|
|
690
|
-
self.
|
|
673
|
+
# Generates the managed paths
|
|
674
|
+
self.persistent_data_path = root_directory_path
|
|
675
|
+
self.zaber_positions_path = self.persistent_data_path.joinpath("zaber_positions.yaml")
|
|
676
|
+
self.mesoscope_positions_path = self.persistent_data_path.joinpath("mesoscope_positions.yaml")
|
|
677
|
+
|
|
678
|
+
def make_directories(self) -> None:
|
|
679
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
691
680
|
|
|
692
|
-
|
|
693
|
-
if self.zaber_positions_path == "null":
|
|
694
|
-
self.zaber_positions_path = self.vrpc_persistent_path.joinpath("zaber_positions.yaml")
|
|
695
|
-
if self.mesoscope_positions_path == "null":
|
|
696
|
-
self.mesoscope_positions_path = self.vrpc_persistent_path.joinpath("mesoscope_positions.yaml")
|
|
697
|
-
if self.motion_estimator_path == "null":
|
|
698
|
-
self.motion_estimator_path = self.scanimage_persistent_path.joinpath("MotionEstimator.me")
|
|
681
|
+
ensure_directory_exists(self.persistent_data_path)
|
|
699
682
|
|
|
700
|
-
def make_string(self) -> None:
|
|
701
|
-
"""Converts all Path objects stored inside the class to strings.
|
|
702
683
|
|
|
703
|
-
|
|
704
|
-
|
|
684
|
+
@dataclass()
|
|
685
|
+
class ScanImagePCPersistentData:
|
|
686
|
+
"""Stores the paths to the directories and files that make up the 'persistent_data' directory on the ScanImagePC.
|
|
687
|
+
|
|
688
|
+
Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
|
|
689
|
+
are purposefully designed for specific PCs that participate in data acquisition. This section manages the
|
|
690
|
+
animal-specific persistent_data directory stored on the ScanImagePC (Mesoscope PC).
|
|
691
|
+
|
|
692
|
+
ScanImagePC persistent data directory is used to preserve the motion estimation snapshot, generated during the first
|
|
693
|
+
experiment session. This is necessary to align the brain recording field of view across sessions. In turn, this
|
|
694
|
+
is used to carry out 'online' motion and z-drift correction, improving the accuracy of across-day (multi-day)
|
|
695
|
+
cell tracking.
|
|
696
|
+
"""
|
|
697
|
+
|
|
698
|
+
persistent_data_path: Path = Path()
|
|
699
|
+
"""Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
|
|
700
|
+
belongs, relative to the ScanImagePC root. This directory is exclusively used on the ScanImagePC (Mesoscope PC)."""
|
|
701
|
+
motion_estimator_path: Path = Path()
|
|
702
|
+
"""Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
|
|
703
|
+
file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
|
|
704
|
+
|
|
705
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
706
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
707
|
+
|
|
708
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
709
|
+
machine that instantiates the class.
|
|
710
|
+
|
|
711
|
+
Args:
|
|
712
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
713
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
714
|
+
the managed session.
|
|
705
715
|
"""
|
|
706
|
-
self.scanimage_persistent_path = str(self.scanimage_persistent_path)
|
|
707
|
-
self.vrpc_persistent_path = str(self.vrpc_persistent_path)
|
|
708
|
-
self.zaber_positions_path = str(self.zaber_positions_path)
|
|
709
|
-
self.mesoscope_positions_path = str(self.mesoscope_positions_path)
|
|
710
|
-
self.motion_estimator_path = str(self.motion_estimator_path)
|
|
711
716
|
|
|
712
|
-
|
|
713
|
-
|
|
717
|
+
# Generates the managed paths
|
|
718
|
+
self.persistent_data_path = root_directory_path
|
|
719
|
+
self.motion_estimator_path = self.persistent_data_path.joinpath("MotionEstimator.me")
|
|
714
720
|
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
721
|
+
def make_directories(self) -> None:
|
|
722
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
723
|
+
|
|
724
|
+
ensure_directory_exists(self.persistent_data_path)
|
|
718
725
|
|
|
719
726
|
|
|
720
727
|
@dataclass()
|
|
721
728
|
class MesoscopeData:
|
|
722
|
-
"""Stores the paths to the directories
|
|
723
|
-
|
|
729
|
+
"""Stores the paths to the directories and files that make up the 'meso_data' directory on the ScanImagePC.
|
|
730
|
+
|
|
731
|
+
The meso_data directory is the root directory where all mesoscope-generated data is stored on the ScanImagePC. The
|
|
732
|
+
path to this directory should be given relative to the VRPC root and be mounted to the VRPC filesystem via the
|
|
733
|
+
SMB or equivalent protocol.
|
|
724
734
|
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
735
|
+
During runtime, the ScanImagePC should organize all collected data under this root directory. During preprocessing,
|
|
736
|
+
the VRPC uses SMB to access the data in this directory and merge it into the 'raw_data' session directory. The paths
|
|
737
|
+
in this section, therefore, are specific to the VRPC and are not used on other PCs.
|
|
728
738
|
"""
|
|
729
739
|
|
|
730
|
-
|
|
740
|
+
meso_data_path: Path = Path()
|
|
731
741
|
"""Stores the path to the root ScanImagePC data directory, mounted to the VRPC filesystem via the SMB or equivalent
|
|
732
|
-
protocol.
|
|
733
|
-
|
|
734
|
-
|
|
742
|
+
protocol. All mesoscope-generated data is stored under this root directory before it is merged into the VRPC-managed
|
|
743
|
+
raw_data directory of each session."""
|
|
744
|
+
mesoscope_data_path: Path = Path()
|
|
745
|
+
"""Stores the path to the 'default' mesoscope_data directory. All experiment sessions across all animals and
|
|
746
|
+
projects use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
|
|
747
|
+
software. This simplifies ScanImagePC configuration process during runtime, as all data is always saved in the same
|
|
748
|
+
directory. During preprocessing, the data is moved from the default directory first into a session-specific
|
|
749
|
+
ScanImagePC directory and then into the VRPC raw_data session directory."""
|
|
750
|
+
session_specific_path: Path = Path()
|
|
735
751
|
"""Stores the path to the session-specific data directory. This directory is generated at the end of each experiment
|
|
736
|
-
runtime to prepare mesoscope data for
|
|
737
|
-
runtime."""
|
|
738
|
-
|
|
739
|
-
"""Stores the path to the 'general' mesoscope_data directory. All experiment sessions (across all animals and
|
|
740
|
-
projects) use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
|
|
741
|
-
software. This simplifies ScanImagePC configuration process during runtime. The data is moved into a
|
|
742
|
-
session-specific directory during preprocessing."""
|
|
743
|
-
ubiquitin_path: str | Path = "null"
|
|
752
|
+
runtime to prepare mesoscope data for being moved to the VRPC-managed raw_data directory and to reset the 'default'
|
|
753
|
+
mesoscope_data directory for the next session's runtime."""
|
|
754
|
+
ubiquitin_path: Path = Path()
|
|
744
755
|
"""Stores the path to the 'ubiquitin.bin' file. This file is automatically generated inside the session-specific
|
|
745
|
-
data
|
|
746
|
-
removal, the VRPC searches for directories marked with ubiquitin.bin
|
|
756
|
+
data directory after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
|
|
757
|
+
removal step of preprocessing, the VRPC searches for directories marked with ubiquitin.bin and deletes them from the
|
|
747
758
|
ScanImagePC filesystem."""
|
|
748
759
|
|
|
749
|
-
def
|
|
750
|
-
"""
|
|
751
|
-
are converted to Path objects.
|
|
752
|
-
"""
|
|
753
|
-
# Converts the paths that have to be provided at class instantiation to the Path objects.
|
|
754
|
-
self.root_data_path = Path(self.root_data_path)
|
|
755
|
-
self.session_specific_path = Path(self.session_specific_path)
|
|
756
|
-
|
|
757
|
-
# Resolves the 'null' placeholders using the 'anchor' paths above
|
|
758
|
-
if self.mesoscope_data_path == "null":
|
|
759
|
-
self.mesoscope_data_path = self.root_data_path.joinpath("mesoscope_data")
|
|
760
|
-
if self.ubiquitin_path == "null":
|
|
761
|
-
self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
|
|
760
|
+
def resolve_paths(self, root_mesoscope_path: Path, session_name: str) -> None:
|
|
761
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
762
762
|
|
|
763
|
-
|
|
764
|
-
|
|
763
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
764
|
+
machine that instantiates the class.
|
|
765
765
|
|
|
766
|
-
|
|
767
|
-
|
|
766
|
+
Args:
|
|
767
|
+
root_mesoscope_path: The path to the top-level directory of the ScanImagePC data hierarchy mounted to the
|
|
768
|
+
VRPC via the SMB or equivalent protocol.
|
|
769
|
+
session_name: The name of the session for which this subclass is initialized.
|
|
768
770
|
"""
|
|
769
|
-
self.root_data_path = str(self.root_data_path)
|
|
770
|
-
self.session_specific_path = str(self.session_specific_path)
|
|
771
|
-
self.mesoscope_data_path = str(self.mesoscope_data_path)
|
|
772
|
-
self.ubiquitin_path = str(self.ubiquitin_path)
|
|
773
771
|
|
|
774
|
-
|
|
775
|
-
|
|
772
|
+
# Generates the managed paths
|
|
773
|
+
self.meso_data_path = root_mesoscope_path
|
|
774
|
+
self.session_specific_path = self.meso_data_path.joinpath(session_name)
|
|
775
|
+
self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
|
|
776
|
+
self.mesoscope_data_path = self.meso_data_path.joinpath("mesoscope_data")
|
|
776
777
|
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
ensure_directory_exists(Path(self.mesoscope_data_path))
|
|
778
|
+
def make_directories(self) -> None:
|
|
779
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
780
|
+
|
|
781
|
+
ensure_directory_exists(self.meso_data_path)
|
|
782
782
|
|
|
783
783
|
|
|
784
784
|
@dataclass()
|
|
785
|
-
class
|
|
786
|
-
"""Stores the paths to the VRPC filesystem-mounted Synology NAS and BioHPC server
|
|
785
|
+
class VRPCDestinations:
|
|
786
|
+
"""Stores the paths to the VRPC filesystem-mounted directories of the Synology NAS and BioHPC server.
|
|
787
|
+
|
|
788
|
+
The paths from this section are primarily used to transfer preprocessed data to the long-term storage destinations.
|
|
789
|
+
Additionally, they allow VRPC to interface with the configuration directory of the BioHPC server to start data
|
|
790
|
+
processing jobs and to read the data from the processed_data directory to remove redundant data from the VRPC
|
|
791
|
+
filesystem.
|
|
787
792
|
|
|
788
|
-
|
|
789
|
-
VRPC to the long-term storage destinations.
|
|
793
|
+
Overall, this section is intended solely for the VRPC and should not be used on other PCs.
|
|
790
794
|
"""
|
|
791
795
|
|
|
792
|
-
nas_raw_data_path:
|
|
796
|
+
nas_raw_data_path: Path = Path()
|
|
793
797
|
"""Stores the path to the session's raw_data directory on the Synology NAS, which is mounted to the VRPC via the
|
|
794
798
|
SMB or equivalent protocol."""
|
|
795
|
-
server_raw_data_path:
|
|
799
|
+
server_raw_data_path: Path = Path()
|
|
796
800
|
"""Stores the path to the session's raw_data directory on the BioHPC server, which is mounted to the VRPC via the
|
|
797
801
|
SMB or equivalent protocol."""
|
|
798
|
-
|
|
799
|
-
"""Stores the path to the session's
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
802
|
+
server_processed_data_path: Path = Path()
|
|
803
|
+
"""Stores the path to the session's processed_data directory on the BioHPC server, which is mounted to the VRPC via
|
|
804
|
+
the SMB or equivalent protocol."""
|
|
805
|
+
server_configuration_path: Path = Path()
|
|
806
|
+
"""Stores the path to the project-specific 'configuration' directory on the BioHPC server, which is mounted to the
|
|
807
|
+
VRPC via the SMB or equivalent protocol."""
|
|
808
|
+
telomere_path: Path = Path()
|
|
809
|
+
"""Stores the path to the session's telomere.bin marker. This marker is generated as part of data processing on the
|
|
810
|
+
BioHPC server to notify the VRPC that the server received preprocessed data intact. The presence of this marker is
|
|
811
|
+
used by the VRPC to determine which locally stored raw_data is safe to delete from the filesystem."""
|
|
812
|
+
suite2p_configuration_path: Path = Path()
|
|
813
|
+
"""Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
|
|
814
|
+
the BioHPC server. This configuration file specifies the parameters for the 'single day' sl-suite2p registration
|
|
815
|
+
pipeline, which is applied to each session that generates brain activity data."""
|
|
816
|
+
processing_tracker_path: Path = Path()
|
|
817
|
+
"""Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
|
|
818
|
+
the BioHPC server. This file tracks which processing pipelines need to be applied the target session and the status
|
|
819
|
+
(success / failure) of each applied pipeline.
|
|
820
|
+
"""
|
|
821
|
+
multiday_configuration_path: Path = Path()
|
|
822
|
+
"""Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
|
|
823
|
+
on the BioHPC server. This configuration file specifies the parameters for the 'multiday' sl-suite2p registration
|
|
824
|
+
pipeline used to track brain cells across multiple sessions."""
|
|
809
825
|
|
|
810
|
-
|
|
811
|
-
|
|
826
|
+
def resolve_paths(
|
|
827
|
+
self,
|
|
828
|
+
nas_raw_data_path: Path,
|
|
829
|
+
server_raw_data_path: Path,
|
|
830
|
+
server_processed_data_path: Path,
|
|
831
|
+
server_configuration_path: Path,
|
|
832
|
+
) -> None:
|
|
833
|
+
"""Resolves all paths managed by the class instance based on the input root directory paths.
|
|
812
834
|
|
|
813
|
-
|
|
814
|
-
|
|
835
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
836
|
+
machine that instantiates the class.
|
|
815
837
|
|
|
816
|
-
|
|
817
|
-
|
|
838
|
+
Args:
|
|
839
|
+
nas_raw_data_path: The path to the session's raw_data directory on the Synology NAS, relative to the VRPC
|
|
840
|
+
filesystem root.
|
|
841
|
+
server_raw_data_path: The path to the session's raw_data directory on the BioHPC server, relative to the
|
|
842
|
+
VRPC filesystem root.
|
|
843
|
+
server_processed_data_path: The path to the session's processed_data directory on the BioHPC server,
|
|
844
|
+
relative to the VRPC filesystem root.
|
|
845
|
+
server_configuration_path: The path to the project-specific 'configuration' directory on the BioHPC server,
|
|
846
|
+
relative to the VRPC filesystem root.
|
|
818
847
|
"""
|
|
819
|
-
self.nas_raw_data_path = str(self.nas_raw_data_path)
|
|
820
|
-
self.server_raw_data_path = str(self.server_raw_data_path)
|
|
821
|
-
self.server_telomere_path = str(self.server_telomere_path)
|
|
822
848
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
849
|
+
# Generates the managed paths
|
|
850
|
+
self.nas_raw_data_path = nas_raw_data_path
|
|
851
|
+
self.server_raw_data_path = server_raw_data_path
|
|
852
|
+
self.server_processed_data_path = server_processed_data_path
|
|
853
|
+
self.server_configuration_path = server_configuration_path
|
|
854
|
+
self.telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
|
|
855
|
+
self.suite2p_configuration_path = self.server_configuration_path.joinpath("suite2p_configuration.yaml")
|
|
856
|
+
self.processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
|
|
857
|
+
self.multiday_configuration_path = self.server_configuration_path.joinpath("multiday_configuration.yaml")
|
|
858
|
+
|
|
859
|
+
def make_directories(self) -> None:
|
|
860
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
861
|
+
ensure_directory_exists(self.nas_raw_data_path)
|
|
862
|
+
ensure_directory_exists(self.server_raw_data_path)
|
|
863
|
+
ensure_directory_exists(self.server_configuration_path)
|
|
864
|
+
ensure_directory_exists(self.server_processed_data_path)
|
|
827
865
|
|
|
828
866
|
|
|
829
867
|
@dataclass
|
|
@@ -869,32 +907,30 @@ class SessionData(YamlConfig):
|
|
|
869
907
|
field is not None (null), it communicates the specific experiment configuration used by the session. During runtime,
|
|
870
908
|
the name stored here is used to load the specific experiment configuration data stored in a .yaml file with the
|
|
871
909
|
same name. If the session is not an experiment session, this field is ignored."""
|
|
872
|
-
raw_data: RawData
|
|
873
|
-
"""
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
"""
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
destinations:
|
|
894
|
-
"""
|
|
895
|
-
|
|
896
|
-
should be mapped (mounted) to the VRPC's filesystem via the SMB or equivalent protocol. This section is not used
|
|
897
|
-
during data processing."""
|
|
910
|
+
raw_data: RawData = field(default_factory=lambda: RawData())
|
|
911
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/session/raw_data directory of any
|
|
912
|
+
PC used to work with Sun lab data."""
|
|
913
|
+
processed_data: ProcessedData = field(default_factory=lambda: ProcessedData())
|
|
914
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/session/processed_data directory of
|
|
915
|
+
any PC used to work with Sun lab data."""
|
|
916
|
+
deeplabcut_data: DeepLabCutData = field(default_factory=lambda: DeepLabCutData())
|
|
917
|
+
"""Stores the paths to all subfolders and files found under the /project/deeplabcut directory of any PC used to
|
|
918
|
+
work with Sun lab data."""
|
|
919
|
+
configuration_data: ConfigurationData = field(default_factory=lambda: ConfigurationData())
|
|
920
|
+
"""Stores the paths to all subfolders and files found under the /project/configuration directory of any PC used to
|
|
921
|
+
work with Sun lab data."""
|
|
922
|
+
vrpc_persistent_data: VRPCPersistentData = field(default_factory=lambda: VRPCPersistentData())
|
|
923
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
|
|
924
|
+
the VRPC used in the Sun lab to acquire behavior data."""
|
|
925
|
+
scanimagepc_persistent_data: ScanImagePCPersistentData = field(default_factory=lambda: ScanImagePCPersistentData())
|
|
926
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
|
|
927
|
+
the ScanImagePC used in the Sun lab to acquire brain activity data."""
|
|
928
|
+
mesoscope_data: MesoscopeData = field(default_factory=lambda: MesoscopeData())
|
|
929
|
+
"""Stores the paths to all subfolders and files found under the /meso_data (root mesoscope data) directory of
|
|
930
|
+
the ScanImagePC used in the Sun lab to acquire brain activity data."""
|
|
931
|
+
destinations: VRPCDestinations = field(default_factory=lambda: VRPCDestinations())
|
|
932
|
+
"""Stores the paths to all subfolders and files under various VRPC-filesystem-mounted directories of other machines
|
|
933
|
+
used in the Sun lab for long-term data storage."""
|
|
898
934
|
|
|
899
935
|
@classmethod
|
|
900
936
|
def create(
|
|
@@ -943,17 +979,18 @@ class SessionData(YamlConfig):
|
|
|
943
979
|
if session_name is None:
|
|
944
980
|
session_name = str(get_timestamp(time_separator="-"))
|
|
945
981
|
|
|
946
|
-
# Extracts the root directory paths stored inside the project configuration file.
|
|
947
|
-
#
|
|
982
|
+
# Extracts the root directory paths stored inside the project configuration file. These roots are then used to
|
|
983
|
+
# initialize this class instance.
|
|
948
984
|
vrpc_root = Path(project_configuration.local_root_directory)
|
|
949
985
|
mesoscope_root = Path(project_configuration.local_mesoscope_directory)
|
|
950
986
|
biohpc_root = Path(project_configuration.local_server_directory)
|
|
987
|
+
biohpc_workdir = Path(project_configuration.local_server_working_directory)
|
|
951
988
|
nas_root = Path(project_configuration.local_nas_directory)
|
|
952
989
|
|
|
953
990
|
# Extracts the name of the project stored inside the project configuration file.
|
|
954
991
|
project_name = project_configuration.project_name
|
|
955
992
|
|
|
956
|
-
# Constructs the session directory path
|
|
993
|
+
# Constructs the session directory path
|
|
957
994
|
session_path = vrpc_root.joinpath(project_name, animal_id, session_name)
|
|
958
995
|
|
|
959
996
|
# Handles potential session name conflicts
|
|
@@ -971,38 +1008,51 @@ class SessionData(YamlConfig):
|
|
|
971
1008
|
f"already exists. The newly created session directory uses a '_{counter}' postfix to distinguish "
|
|
972
1009
|
f"itself from the already existing session directory."
|
|
973
1010
|
)
|
|
974
|
-
|
|
1011
|
+
console.echo(message=message, level=LogLevel.ERROR)
|
|
975
1012
|
|
|
976
1013
|
# Generates subclasses stored inside the main class instance based on the data resolved above. Note; most fields
|
|
977
|
-
# of these classes are
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
)
|
|
981
|
-
raw_data.
|
|
982
|
-
|
|
983
|
-
processed_data = ProcessedData(
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
1014
|
+
# of these classes are resolved automatically, based on one or more 'root' paths provided to the 'resolve_paths'
|
|
1015
|
+
# method.
|
|
1016
|
+
raw_data = RawData()
|
|
1017
|
+
raw_data.resolve_paths(root_directory_path=session_path.joinpath("raw_data"))
|
|
1018
|
+
raw_data.make_directories() # Generates the local directory tree
|
|
1019
|
+
|
|
1020
|
+
processed_data = ProcessedData()
|
|
1021
|
+
processed_data.resolve_paths(root_directory_path=session_path.joinpath("processed_data"))
|
|
1022
|
+
processed_data.make_directories()
|
|
1023
|
+
|
|
1024
|
+
dlc_data = DeepLabCutData()
|
|
1025
|
+
dlc_data.resolve_paths(root_directory_path=vrpc_root.joinpath(project_name, "deeplabcut"))
|
|
1026
|
+
dlc_data.make_directories()
|
|
1027
|
+
|
|
1028
|
+
configuration_data = ConfigurationData()
|
|
1029
|
+
configuration_data.resolve_paths(
|
|
1030
|
+
root_directory_path=vrpc_root.joinpath(project_name, "configuration"), experiment_name=experiment_name
|
|
987
1031
|
)
|
|
1032
|
+
configuration_data.make_directories()
|
|
988
1033
|
|
|
1034
|
+
vrpc_persistent_data = VRPCPersistentData()
|
|
989
1035
|
vrpc_persistent_path = vrpc_root.joinpath(project_name, animal_id, "persistent_data")
|
|
1036
|
+
vrpc_persistent_data.resolve_paths(root_directory_path=vrpc_persistent_path)
|
|
1037
|
+
vrpc_persistent_data.make_directories()
|
|
1038
|
+
|
|
1039
|
+
scanimagepc_persistent_data = ScanImagePCPersistentData()
|
|
990
1040
|
scanimagepc_persistent_path = mesoscope_root.joinpath(project_name, animal_id, "persistent_data")
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
)
|
|
994
|
-
persistent_data.make_dirs() # Generates all persistent directory trees
|
|
1041
|
+
scanimagepc_persistent_data.resolve_paths(root_directory_path=scanimagepc_persistent_path)
|
|
1042
|
+
scanimagepc_persistent_data.make_directories()
|
|
995
1043
|
|
|
996
|
-
mesoscope_data = MesoscopeData(
|
|
997
|
-
|
|
998
|
-
)
|
|
999
|
-
mesoscope_data.make_dirs() # Generates all Mesoscope directory trees
|
|
1044
|
+
mesoscope_data = MesoscopeData()
|
|
1045
|
+
mesoscope_data.resolve_paths(root_mesoscope_path=mesoscope_root, session_name=session_name)
|
|
1046
|
+
mesoscope_data.make_directories()
|
|
1000
1047
|
|
|
1001
|
-
destinations =
|
|
1048
|
+
destinations = VRPCDestinations()
|
|
1049
|
+
destinations.resolve_paths(
|
|
1002
1050
|
nas_raw_data_path=nas_root.joinpath(project_name, animal_id, session_name, "raw_data"),
|
|
1003
1051
|
server_raw_data_path=biohpc_root.joinpath(project_name, animal_id, session_name, "raw_data"),
|
|
1052
|
+
server_configuration_path=biohpc_root.joinpath(project_name, "configuration"),
|
|
1053
|
+
server_processed_data_path=biohpc_workdir.joinpath(project_name, "processed_data"),
|
|
1004
1054
|
)
|
|
1005
|
-
destinations.
|
|
1055
|
+
destinations.make_directories()
|
|
1006
1056
|
|
|
1007
1057
|
# Packages the sections generated above into a SessionData instance
|
|
1008
1058
|
instance = SessionData(
|
|
@@ -1011,8 +1061,11 @@ class SessionData(YamlConfig):
|
|
|
1011
1061
|
session_name=session_name,
|
|
1012
1062
|
session_type=session_type,
|
|
1013
1063
|
raw_data=raw_data,
|
|
1064
|
+
deeplabcut_data=dlc_data,
|
|
1065
|
+
configuration_data=configuration_data,
|
|
1014
1066
|
processed_data=processed_data,
|
|
1015
|
-
|
|
1067
|
+
vrpc_persistent_data=vrpc_persistent_data,
|
|
1068
|
+
scanimagepc_persistent_data=scanimagepc_persistent_data,
|
|
1016
1069
|
mesoscope_data=mesoscope_data,
|
|
1017
1070
|
destinations=destinations,
|
|
1018
1071
|
experiment_name=experiment_name,
|
|
@@ -1026,19 +1079,16 @@ class SessionData(YamlConfig):
|
|
|
1026
1079
|
# classes is not exhaustive. More classes are saved as part of the session runtime management class start() and
|
|
1027
1080
|
# __init__() method runtimes:
|
|
1028
1081
|
|
|
1029
|
-
# Resolves the path to the project configuration folder
|
|
1030
|
-
vrpc_configuration_path = vrpc_root.joinpath(project_name, "configuration")
|
|
1031
|
-
|
|
1032
1082
|
# Discovers and saves the necessary configuration class instances to the raw_data folder of the managed session:
|
|
1033
1083
|
# Project Configuration
|
|
1034
1084
|
sh.copy2(
|
|
1035
|
-
src=
|
|
1085
|
+
src=instance.configuration_data.project_configuration_path,
|
|
1036
1086
|
dst=instance.raw_data.project_configuration_path,
|
|
1037
1087
|
)
|
|
1038
1088
|
# Experiment Configuration, if the session type is Experiment.
|
|
1039
1089
|
if experiment_name is not None:
|
|
1040
1090
|
sh.copy2(
|
|
1041
|
-
src=
|
|
1091
|
+
src=instance.configuration_data.experiment_configuration_path,
|
|
1042
1092
|
dst=instance.raw_data.experiment_configuration_path,
|
|
1043
1093
|
)
|
|
1044
1094
|
|
|
@@ -1062,9 +1112,10 @@ class SessionData(YamlConfig):
|
|
|
1062
1112
|
|
|
1063
1113
|
Args:
|
|
1064
1114
|
session_path: The path to the root directory of an existing session, e.g.: vrpc_root/project/animal/session.
|
|
1065
|
-
on_server: Determines whether the method is used to initialize an existing session on the
|
|
1066
|
-
|
|
1067
|
-
subfolders. BioHPC server runtimes use different volumes (drives) to store these
|
|
1115
|
+
on_server: Determines whether the method is used to initialize an existing session on the BioHPC server or
|
|
1116
|
+
a non-server machine. Note, non-server runtimes use the same 'root' directory to store raw_data and
|
|
1117
|
+
processed_data subfolders. BioHPC server runtimes use different volumes (drives) to store these
|
|
1118
|
+
subfolders.
|
|
1068
1119
|
|
|
1069
1120
|
Returns:
|
|
1070
1121
|
An initialized SessionData instance for the session whose data is stored at the provided path.
|
|
@@ -1084,48 +1135,100 @@ class SessionData(YamlConfig):
|
|
|
1084
1135
|
)
|
|
1085
1136
|
console.error(message=message, error=FileNotFoundError)
|
|
1086
1137
|
|
|
1087
|
-
# Loads class data from .yaml
|
|
1138
|
+
# Loads class data from .yaml file
|
|
1088
1139
|
instance: SessionData = cls.from_yaml(file_path=session_data_path) # type: ignore
|
|
1089
1140
|
|
|
1090
1141
|
# The method assumes that the 'donor' .yaml file is always stored inside the raw_data directory of the session
|
|
1091
1142
|
# to be processed. Since the directory itself might have moved (between or even within the same PC) relative to
|
|
1092
1143
|
# where it was when the SessionData snapshot was generated, reconfigures the paths to all raw_data files using
|
|
1093
1144
|
# the root from above.
|
|
1094
|
-
|
|
1145
|
+
local_root = session_path.parents[2]
|
|
1146
|
+
|
|
1147
|
+
# RAW DATA
|
|
1148
|
+
new_root = local_root.joinpath(instance.project_name, instance.animal_id, instance.session_name, "raw_data")
|
|
1149
|
+
instance.raw_data.resolve_paths(root_directory_path=new_root)
|
|
1150
|
+
|
|
1151
|
+
# Uses the adjusted raw_data section to load the ProjectConfiguration instance. This is used below to resolve
|
|
1152
|
+
# all other SessionData sections, as it stores various required root directories.
|
|
1153
|
+
project_configuration: ProjectConfiguration = ProjectConfiguration.load(
|
|
1154
|
+
project_name=instance.project_name,
|
|
1155
|
+
configuration_path=Path(instance.raw_data.project_configuration_path),
|
|
1156
|
+
)
|
|
1157
|
+
|
|
1158
|
+
# Resolves the new roots for all sections that use the same root as the raw_data directory:
|
|
1159
|
+
|
|
1160
|
+
# CONFIGURATION
|
|
1161
|
+
new_root = local_root.joinpath(instance.project_name, "configuration")
|
|
1162
|
+
instance.configuration_data.resolve_paths(
|
|
1163
|
+
root_directory_path=new_root, experiment_name=instance.experiment_name
|
|
1164
|
+
)
|
|
1165
|
+
|
|
1166
|
+
# DEEPLABCUT
|
|
1167
|
+
new_root = local_root.joinpath(instance.project_name, "deeplabcut")
|
|
1168
|
+
instance.deeplabcut_data.resolve_paths(root_directory_path=new_root)
|
|
1169
|
+
|
|
1170
|
+
# Resolves the roots for all VRPC-specific sections that use the data from the ProjectConfiguration instance:
|
|
1171
|
+
|
|
1172
|
+
# VRPC PERSISTENT DATA
|
|
1173
|
+
new_root = Path(project_configuration.local_root_directory).joinpath(
|
|
1174
|
+
instance.project_name, instance.animal_id, "persistent_data"
|
|
1175
|
+
)
|
|
1176
|
+
instance.vrpc_persistent_data.resolve_paths(root_directory_path=new_root)
|
|
1177
|
+
|
|
1178
|
+
# SCANIMAGEPC PERSISTENT DATA
|
|
1179
|
+
new_root = Path(project_configuration.local_mesoscope_directory).joinpath(
|
|
1180
|
+
instance.project_name, instance.animal_id, "persistent_data"
|
|
1181
|
+
)
|
|
1182
|
+
instance.scanimagepc_persistent_data.resolve_paths(root_directory_path=new_root)
|
|
1183
|
+
|
|
1184
|
+
# MESOSCOPE DATA
|
|
1185
|
+
instance.mesoscope_data.resolve_paths(
|
|
1186
|
+
root_mesoscope_path=Path(project_configuration.local_mesoscope_directory),
|
|
1187
|
+
session_name=instance.session_name,
|
|
1188
|
+
)
|
|
1189
|
+
|
|
1190
|
+
# DESTINATIONS
|
|
1191
|
+
instance.destinations.resolve_paths(
|
|
1192
|
+
nas_raw_data_path=Path(project_configuration.local_nas_directory).joinpath(
|
|
1193
|
+
instance.project_name, instance.animal_id, instance.session_name, "raw_data"
|
|
1194
|
+
),
|
|
1195
|
+
server_raw_data_path=Path(project_configuration.local_server_directory).joinpath(
|
|
1196
|
+
instance.project_name, instance.animal_id, instance.session_name, "raw_data"
|
|
1197
|
+
),
|
|
1198
|
+
server_configuration_path=Path(project_configuration.local_server_directory).joinpath(
|
|
1199
|
+
instance.project_name, "configuration"
|
|
1200
|
+
),
|
|
1201
|
+
server_processed_data_path=Path(project_configuration.local_server_working_directory).joinpath(
|
|
1202
|
+
instance.project_name, instance.animal_id, instance.session_name, "processed_data"
|
|
1203
|
+
),
|
|
1204
|
+
)
|
|
1095
1205
|
|
|
1096
1206
|
# Resolves the paths to the processed_data directories. The resolution strategy depends on whether the method is
|
|
1097
1207
|
# called on the VRPC (locally) or the BioHPC server (remotely).
|
|
1098
|
-
new_root: Path
|
|
1099
1208
|
if not on_server:
|
|
1100
1209
|
# Local runtimes use the same root session directory for both raw_data and processed_data. This stems from
|
|
1101
1210
|
# the assumption that most local machines in the lab only use NVME (fast) volumes and, therefore, do not
|
|
1102
1211
|
# need to separate 'storage' and 'working' data directories.
|
|
1103
|
-
|
|
1104
|
-
# Regenerates the processed_data paths using the information loaded from the session_data.yaml file and
|
|
1105
|
-
# static path definition rules used in the lab.
|
|
1106
|
-
new_root = session_path.parents[2] # Reuses the local root for non-server runtimes
|
|
1212
|
+
new_root = local_root # Reuses the local root for non-server runtimes
|
|
1107
1213
|
|
|
1108
1214
|
else:
|
|
1109
1215
|
# The BioHPC server stores raw_data on slow volume and processed_data on fast (NVME) volume. Therefore, to
|
|
1110
1216
|
# configure processed_data paths, the method first needs to load the fast volume root path from the
|
|
1111
1217
|
# project_configuration.yaml file stored in the raw_data folder.
|
|
1112
|
-
project_configuration: ProjectConfiguration = ProjectConfiguration.load(
|
|
1113
|
-
project_name=instance.project_name,
|
|
1114
|
-
configuration_path=Path(instance.raw_data.project_configuration_path),
|
|
1115
|
-
)
|
|
1116
1218
|
new_root = Path(project_configuration.remote_working_directory)
|
|
1117
1219
|
|
|
1118
1220
|
# Regenerates the processed_data path depending on the root resolution above
|
|
1119
|
-
instance.processed_data
|
|
1120
|
-
|
|
1221
|
+
instance.processed_data.resolve_paths(
|
|
1222
|
+
root_directory_path=new_root.joinpath(
|
|
1121
1223
|
instance.project_name, instance.animal_id, instance.session_name, "processed_data"
|
|
1122
|
-
)
|
|
1123
|
-
configuration_path=new_root.joinpath(instance.project_name, instance.animal_id, "configuration"),
|
|
1124
|
-
deeplabcut_path=new_root.joinpath(instance.project_name, "deeplabcut"),
|
|
1224
|
+
)
|
|
1125
1225
|
)
|
|
1126
1226
|
|
|
1127
|
-
# Generates
|
|
1128
|
-
instance.
|
|
1227
|
+
# Generates data directory hierarchies that may be missing on the local machine
|
|
1228
|
+
instance.raw_data.make_directories()
|
|
1229
|
+
instance.configuration_data.make_directories()
|
|
1230
|
+
instance.deeplabcut_data.make_directories()
|
|
1231
|
+
instance.processed_data.make_directories()
|
|
1129
1232
|
|
|
1130
1233
|
# Returns the initialized SessionData instance to caller
|
|
1131
1234
|
return instance
|
|
@@ -1138,21 +1241,8 @@ class SessionData(YamlConfig):
|
|
|
1138
1241
|
create() method runtime.
|
|
1139
1242
|
"""
|
|
1140
1243
|
|
|
1141
|
-
# Copies instance data to prevent it from being modified by reference when executing the steps below
|
|
1142
|
-
original = copy.deepcopy(self)
|
|
1143
|
-
|
|
1144
|
-
# Extracts the target file path before it is converted to a string.
|
|
1145
|
-
file_path: Path = copy.copy(self.raw_data.session_data_path) # type: ignore
|
|
1146
|
-
|
|
1147
|
-
# Converts all Paths objects to strings before dumping the data to YAML.
|
|
1148
|
-
original.raw_data.make_string()
|
|
1149
|
-
original.processed_data.make_string()
|
|
1150
|
-
original.persistent_data.make_string()
|
|
1151
|
-
original.mesoscope_data.make_string()
|
|
1152
|
-
original.destinations.make_string()
|
|
1153
|
-
|
|
1154
1244
|
# Saves instance data as a .YAML file
|
|
1155
|
-
|
|
1245
|
+
self.to_yaml(file_path=self.raw_data.session_data_path)
|
|
1156
1246
|
|
|
1157
1247
|
|
|
1158
1248
|
@dataclass()
|
|
@@ -1582,7 +1672,7 @@ class SurgeryData(YamlConfig):
|
|
|
1582
1672
|
|
|
1583
1673
|
@dataclass()
|
|
1584
1674
|
class ProcessingTracker(YamlConfig):
|
|
1585
|
-
"""
|
|
1675
|
+
"""Tracks the data processing status for a single session.
|
|
1586
1676
|
|
|
1587
1677
|
This class is used during BioHPC-server data processing runtimes to track which processing steps are enabled and
|
|
1588
1678
|
have been successfully applied to a given session. This is used to optimize data processing and avoid unnecessary
|