sl-shared-assets 1.0.0rc13__py3-none-any.whl → 1.0.0rc14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sl-shared-assets might be problematic. Click here for more details.
- sl_shared_assets/__init__.py +7 -1
- sl_shared_assets/__init__.pyi +51 -0
- sl_shared_assets/ascension_tools.py +0 -4
- sl_shared_assets/ascension_tools.pyi +68 -0
- sl_shared_assets/cli.py +12 -13
- sl_shared_assets/cli.pyi +28 -0
- sl_shared_assets/data_classes.py +508 -464
- sl_shared_assets/data_classes.pyi +799 -0
- sl_shared_assets/packaging_tools.pyi +52 -0
- sl_shared_assets/server.py +22 -15
- sl_shared_assets/server.pyi +117 -0
- sl_shared_assets/suite2p.pyi +188 -0
- sl_shared_assets/transfer_tools.pyi +53 -0
- {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/METADATA +1 -1
- sl_shared_assets-1.0.0rc14.dist-info/RECORD +22 -0
- sl_shared_assets-1.0.0rc13.dist-info/RECORD +0 -14
- {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/WHEEL +0 -0
- {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/entry_points.txt +0 -0
- {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc14.dist-info}/licenses/LICENSE +0 -0
sl_shared_assets/data_classes.py
CHANGED
|
@@ -7,7 +7,6 @@ import re
|
|
|
7
7
|
import copy
|
|
8
8
|
import shutil as sh
|
|
9
9
|
from pathlib import Path
|
|
10
|
-
import warnings
|
|
11
10
|
from dataclasses import field, dataclass
|
|
12
11
|
|
|
13
12
|
import appdirs
|
|
@@ -15,10 +14,6 @@ from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
|
|
|
15
14
|
from ataraxis_data_structures import YamlConfig
|
|
16
15
|
from ataraxis_time.time_helpers import get_timestamp
|
|
17
16
|
|
|
18
|
-
# Ensures console is enabled when this file is imported
|
|
19
|
-
if not console.enabled:
|
|
20
|
-
console.enable()
|
|
21
|
-
|
|
22
17
|
|
|
23
18
|
def replace_root_path(path: Path) -> None:
|
|
24
19
|
"""Replaces the path to the local root directory used to store all Sun lab projects with the provided path.
|
|
@@ -363,510 +358,510 @@ class ProjectConfiguration(YamlConfig):
|
|
|
363
358
|
|
|
364
359
|
@dataclass()
|
|
365
360
|
class RawData:
|
|
366
|
-
"""Stores the paths to the directories and files that make up the 'raw_data' session directory.
|
|
361
|
+
"""Stores the paths to the directories and files that make up the 'raw_data' session-specific directory.
|
|
367
362
|
|
|
368
363
|
The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
|
|
369
364
|
preprocessing does not alter the data, any data in that folder is considered 'raw'. The raw_data folder is initially
|
|
370
365
|
created on the VRPC and, after preprocessing, is copied to the BioHPC server and the Synology NAS for long-term
|
|
371
366
|
storage and further processing.
|
|
372
|
-
|
|
373
|
-
Notes:
|
|
374
|
-
The overall structure of the raw_data directory remains fixed for the entire lifetime of the data. It is reused
|
|
375
|
-
across all destinations.
|
|
376
367
|
"""
|
|
377
368
|
|
|
378
|
-
raw_data_path:
|
|
369
|
+
raw_data_path: Path = Path()
|
|
379
370
|
"""Stores the path to the root raw_data directory of the session. This directory stores all raw data during
|
|
380
371
|
acquisition and preprocessing. Note, preprocessing does not alter raw data, so at any point in time all data inside
|
|
381
372
|
the folder is considered 'raw'."""
|
|
382
|
-
camera_data_path:
|
|
373
|
+
camera_data_path: Path = Path()
|
|
383
374
|
"""Stores the path to the directory that contains all camera data acquired during the session. Primarily, this
|
|
384
375
|
includes .mp4 video files from each recorded camera."""
|
|
385
|
-
mesoscope_data_path:
|
|
376
|
+
mesoscope_data_path: Path = Path()
|
|
386
377
|
"""Stores the path to the directory that contains all Mesoscope data acquired during the session. Primarily, this
|
|
387
378
|
includes the mesoscope-acquired .tiff files (brain activity data) and the motion estimation data."""
|
|
388
|
-
behavior_data_path:
|
|
379
|
+
behavior_data_path: Path = Path()
|
|
389
380
|
"""Stores the path to the directory that contains all behavior data acquired during the session. Primarily, this
|
|
390
381
|
includes the .npz log files used by data-acquisition libraries to store all acquired data. The data stored in this
|
|
391
382
|
way includes the camera and mesoscope frame timestamps and the states of Mesoscope-VR components, such as lick
|
|
392
383
|
sensors, rotary encoders, and other modules."""
|
|
393
|
-
zaber_positions_path:
|
|
384
|
+
zaber_positions_path: Path = Path()
|
|
394
385
|
"""Stores the path to the zaber_positions.yaml file. This file contains the snapshot of all Zaber motor positions
|
|
395
386
|
at the end of the session. Zaber motors are used to position the LickPort and the HeadBar manipulators, which is
|
|
396
387
|
essential for supporting proper brain imaging and animal's running behavior during the session."""
|
|
397
|
-
session_descriptor_path:
|
|
388
|
+
session_descriptor_path: Path = Path()
|
|
398
389
|
"""Stores the path to the session_descriptor.yaml file. This file is partially filled by the system during runtime
|
|
399
390
|
and partially by the experimenter after the runtime. It contains session-specific information, such as the specific
|
|
400
391
|
training parameters, the positions of the Mesoscope objective and the notes made by the experimenter during
|
|
401
392
|
runtime."""
|
|
402
|
-
hardware_configuration_path:
|
|
393
|
+
hardware_configuration_path: Path = Path()
|
|
403
394
|
"""Stores the path to the hardware_configuration.yaml file. This file contains the partial snapshot of the
|
|
404
395
|
calibration parameters used by the Mesoscope-VR system components during runtime. Primarily, this is used during
|
|
405
396
|
data processing to read the .npz data log files generated during runtime."""
|
|
406
|
-
surgery_metadata_path:
|
|
397
|
+
surgery_metadata_path: Path = Path()
|
|
407
398
|
"""Stores the path to the surgery_metadata.yaml file. This file contains the most actual information about the
|
|
408
399
|
surgical intervention(s) performed on the animal prior to the session."""
|
|
409
|
-
project_configuration_path:
|
|
400
|
+
project_configuration_path: Path = Path()
|
|
410
401
|
"""Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
|
|
411
402
|
parameters for the session's project."""
|
|
412
|
-
session_data_path:
|
|
403
|
+
session_data_path: Path = Path()
|
|
413
404
|
"""Stores the path to the session_data.yaml file. This path is used b y the SessionData instance to save itself to
|
|
414
405
|
disk as a .yaml file. The file contains all paths used during data acquisition and processing on both the VRPC and
|
|
415
406
|
the BioHPC server."""
|
|
416
|
-
experiment_configuration_path:
|
|
407
|
+
experiment_configuration_path: Path = Path()
|
|
417
408
|
"""Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
|
|
418
409
|
experiment runtime configuration used by the session. This file is only created for experiment session. It does not
|
|
419
410
|
exist for behavior training sessions."""
|
|
420
|
-
mesoscope_positions_path:
|
|
411
|
+
mesoscope_positions_path: Path = Path()
|
|
421
412
|
"""Stores the path to the mesoscope_positions.yaml file. This file contains the snapshot of the positions used
|
|
422
413
|
by the Mesoscope at the end of the session. This includes both the physical position of the mesoscope objective and
|
|
423
414
|
the 'virtual' tip, tilt, and fastZ positions set via ScanImage software. This file is only created for experiment
|
|
424
415
|
sessions that use the mesoscope, it is omitted for behavior training sessions."""
|
|
425
|
-
window_screenshot_path:
|
|
416
|
+
window_screenshot_path: Path = Path()
|
|
426
417
|
"""Stores the path to the .png screenshot of the ScanImagePC screen. The screenshot should contain the image of the
|
|
427
418
|
cranial window and the red-dot alignment windows. This is used to generate a visual snapshot of the cranial window
|
|
428
419
|
alignment and appearance for each experiment session. This file is only created for experiment sessions that use
|
|
429
420
|
the mesoscope, it is omitted for behavior training sessions."""
|
|
430
|
-
telomere_path:
|
|
421
|
+
telomere_path: Path = Path()
|
|
431
422
|
"""Stores the path to the telomere.bin file. This file is created by the data processing pipelines running on the
|
|
432
|
-
BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission.
|
|
433
|
-
|
|
434
|
-
for the existence of the file when it purges redundant data."""
|
|
435
|
-
checksum_path: str | Path = "null"
|
|
423
|
+
BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission."""
|
|
424
|
+
checksum_path: Path = Path()
|
|
436
425
|
"""Stores the path to the ax_checksum.txt file. This file is generated as part of packaging the data for
|
|
437
426
|
transmission and stores the xxHash-128 checksum of the data. It is used to verify that the transmission did not
|
|
438
|
-
damage or otherwise alter the data
|
|
427
|
+
damage or otherwise alter the data."""
|
|
439
428
|
|
|
440
|
-
def
|
|
441
|
-
"""
|
|
442
|
-
are converted to Path objects.
|
|
443
|
-
"""
|
|
429
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
430
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
444
431
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
|
|
453
|
-
if self.behavior_data_path == "null":
|
|
454
|
-
self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
|
|
455
|
-
if self.zaber_positions_path == "null":
|
|
456
|
-
self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
|
|
457
|
-
if self.session_descriptor_path == "null":
|
|
458
|
-
self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
|
|
459
|
-
if self.hardware_configuration_path == "null":
|
|
460
|
-
self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
|
|
461
|
-
if self.surgery_metadata_path == "null":
|
|
462
|
-
self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
|
|
463
|
-
if self.project_configuration_path == "null":
|
|
464
|
-
self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
|
|
465
|
-
if self.session_data_path == "null":
|
|
466
|
-
self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
|
|
467
|
-
if self.experiment_configuration_path == "null":
|
|
468
|
-
self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
|
|
469
|
-
if self.mesoscope_positions_path == "null":
|
|
470
|
-
self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
|
|
471
|
-
if self.window_screenshot_path == "null":
|
|
472
|
-
self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
|
|
473
|
-
if self.telomere_path == "null":
|
|
474
|
-
self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
|
|
475
|
-
if self.checksum_path == "null":
|
|
476
|
-
self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
|
|
477
|
-
|
|
478
|
-
def make_string(self) -> None:
|
|
479
|
-
"""Converts all Path objects stored inside the class to strings.
|
|
480
|
-
|
|
481
|
-
This transformation is required to support dumping class data into a .YAML file so that the data can be stored
|
|
482
|
-
on disk.
|
|
432
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
433
|
+
machine that instantiates the class.
|
|
434
|
+
|
|
435
|
+
Args:
|
|
436
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
437
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
438
|
+
the managed session.
|
|
483
439
|
"""
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
self.
|
|
487
|
-
self.
|
|
488
|
-
self.
|
|
489
|
-
self.
|
|
490
|
-
self.
|
|
491
|
-
self.
|
|
492
|
-
self.
|
|
493
|
-
self.
|
|
494
|
-
self.
|
|
495
|
-
self.
|
|
496
|
-
self.
|
|
497
|
-
self.
|
|
498
|
-
self.
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
440
|
+
|
|
441
|
+
# Generates the managed paths
|
|
442
|
+
self.raw_data_path = root_directory_path
|
|
443
|
+
self.camera_data_path = self.raw_data_path.joinpath("camera_data")
|
|
444
|
+
self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
|
|
445
|
+
self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
|
|
446
|
+
self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
|
|
447
|
+
self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
|
|
448
|
+
self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
|
|
449
|
+
self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
|
|
450
|
+
self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
|
|
451
|
+
self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
|
|
452
|
+
self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
|
|
453
|
+
self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
|
|
454
|
+
self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
|
|
455
|
+
self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
|
|
456
|
+
self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
|
|
457
|
+
|
|
458
|
+
def make_directories(self) -> None:
|
|
459
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
460
|
+
ensure_directory_exists(self.raw_data_path)
|
|
461
|
+
ensure_directory_exists(self.camera_data_path)
|
|
462
|
+
ensure_directory_exists(self.mesoscope_data_path)
|
|
463
|
+
ensure_directory_exists(self.behavior_data_path)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
@dataclass()
|
|
467
|
+
class DeepLabCutData:
|
|
468
|
+
"""Stores the paths to the directories and files that make up the 'deeplabcut' project-specific directory.
|
|
469
|
+
|
|
470
|
+
DeepLabCut (DLC) is used to track animal body parts and poses in video data acquired during experiment and training
|
|
471
|
+
sessions. Since DLC is designed to work with projects, rather than single animals or sessions, each Sun lab
|
|
472
|
+
project data hierarchy contains a dedicated 'deeplabcut' directory under the root project directory. The contents of
|
|
473
|
+
that directory are largely managed by the DLC itself. Therefore, each session of a given project refers to and
|
|
474
|
+
uses the same 'deeplabcut' directory.
|
|
475
|
+
"""
|
|
476
|
+
|
|
477
|
+
deeplabcut_path: Path = Path()
|
|
478
|
+
"""Stores the path to the project-specific DeepLabCut directory. This folder stores all DeepLabCut data specific to
|
|
479
|
+
a single project, which is reused during the processing of all sessions of the project."""
|
|
480
|
+
|
|
481
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
482
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
483
|
+
|
|
484
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
485
|
+
machine that instantiates the class.
|
|
486
|
+
|
|
487
|
+
Args:
|
|
488
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
489
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
490
|
+
the managed session.
|
|
504
491
|
"""
|
|
505
|
-
ensure_directory_exists(Path(self.raw_data_path))
|
|
506
|
-
ensure_directory_exists(Path(self.camera_data_path))
|
|
507
|
-
ensure_directory_exists(Path(self.mesoscope_data_path))
|
|
508
|
-
ensure_directory_exists(Path(self.behavior_data_path))
|
|
509
492
|
|
|
510
|
-
|
|
511
|
-
|
|
493
|
+
# Generates the managed paths
|
|
494
|
+
self.deeplabcut_path = root_directory_path
|
|
495
|
+
|
|
496
|
+
def make_directories(self) -> None:
|
|
497
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
498
|
+
ensure_directory_exists(self.deeplabcut_path)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
@dataclass()
|
|
502
|
+
class ConfigurationData:
|
|
503
|
+
"""Stores the paths to the directories and files that make up the 'configuration' project-specific directory.
|
|
504
|
+
|
|
505
|
+
The configuration directory contains various configuration files and settings used by data acquisition,
|
|
506
|
+
preprocessing, and processing pipelines in the lab. Generally, all configuration settings are defined once for each
|
|
507
|
+
project and are reused for every session within the project. Therefore, this directory is created under each main
|
|
508
|
+
project directory.
|
|
509
|
+
|
|
510
|
+
Notes:
|
|
511
|
+
Some attribute names inside this section match the names in the RawData section. This is intentional, as some
|
|
512
|
+
configuration files are copied into the raw_data session directories to allow reinstating the session data
|
|
513
|
+
hierarchy across machines.
|
|
514
|
+
"""
|
|
515
|
+
|
|
516
|
+
configuration_path: Path = Path()
|
|
517
|
+
"""Stores the path to the project-specific configuration directory. This directory is used by all animals
|
|
518
|
+
and sessions of the project to store all pan-project configuration files. The configuration data is reused by all
|
|
519
|
+
sessions in the project."""
|
|
520
|
+
experiment_configuration_path: Path = Path()
|
|
521
|
+
"""Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
|
|
522
|
+
experiment runtime configuration used by the session. This file is only created for experiment session. It does not
|
|
523
|
+
exist for behavior training sessions."""
|
|
524
|
+
project_configuration_path: Path = Path()
|
|
525
|
+
"""Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
|
|
526
|
+
parameters for the session's project."""
|
|
527
|
+
suite2p_configuration_path: Path = Path()
|
|
528
|
+
"""Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
|
|
529
|
+
the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
|
|
530
|
+
registration pipeline, which is applied to each session that generates brain activity data."""
|
|
531
|
+
multiday_configuration_path: Path = Path()
|
|
532
|
+
"""Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
|
|
533
|
+
on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
|
|
534
|
+
sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
|
|
512
535
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
(another root). This method is only implemented for subclasses intended to be used both locally and on the
|
|
516
|
-
BioHPC server.
|
|
536
|
+
def resolve_paths(self, root_directory_path: Path, experiment_name: str | None = None) -> None:
|
|
537
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
517
538
|
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
from the RawData section, if necessary.
|
|
539
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
540
|
+
machine that instantiates the class.
|
|
521
541
|
|
|
522
542
|
Args:
|
|
523
|
-
|
|
524
|
-
|
|
543
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
544
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
545
|
+
the managed session.
|
|
546
|
+
experiment_name: Optionally specifies the name of the experiment executed as part of the managed session's
|
|
547
|
+
runtime. This is used to correctly configure the path to the specific ExperimentConfiguration data file.
|
|
548
|
+
If the managed session is not an Experiment session, this parameter should be set to None.
|
|
525
549
|
"""
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
self.
|
|
534
|
-
self.
|
|
535
|
-
self.
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
self.project_configuration_path = new_root.joinpath(Path(self.project_configuration_path).relative_to(old_root))
|
|
541
|
-
self.session_data_path = new_root.joinpath(Path(self.session_data_path).relative_to(old_root))
|
|
542
|
-
self.experiment_configuration_path = new_root.joinpath(
|
|
543
|
-
Path(self.experiment_configuration_path).relative_to(old_root)
|
|
544
|
-
)
|
|
545
|
-
self.mesoscope_positions_path = new_root.joinpath(Path(self.mesoscope_positions_path).relative_to(old_root))
|
|
546
|
-
self.window_screenshot_path = new_root.joinpath(Path(self.window_screenshot_path).relative_to(old_root))
|
|
547
|
-
self.telomere_path = new_root.joinpath(Path(self.telomere_path).relative_to(old_root))
|
|
548
|
-
self.checksum_path = new_root.joinpath(Path(self.checksum_path).relative_to(old_root))
|
|
550
|
+
|
|
551
|
+
# Generates the managed paths
|
|
552
|
+
self.configuration_path = root_directory_path
|
|
553
|
+
if experiment_name is None:
|
|
554
|
+
self.experiment_configuration_path = self.configuration_path.joinpath("null")
|
|
555
|
+
else:
|
|
556
|
+
self.experiment_configuration_path = self.configuration_path.joinpath(f"{experiment_name}.yaml")
|
|
557
|
+
self.project_configuration_path = self.configuration_path.joinpath("project_configuration.yaml")
|
|
558
|
+
self.suite2p_configuration_path = self.configuration_path.joinpath("suite2p_configuration.yaml")
|
|
559
|
+
self.multiday_configuration_path = self.configuration_path.joinpath("multiday_configuration.yaml")
|
|
560
|
+
|
|
561
|
+
def make_directories(self) -> None:
|
|
562
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
563
|
+
ensure_directory_exists(self.configuration_path)
|
|
549
564
|
|
|
550
565
|
|
|
551
566
|
@dataclass()
|
|
552
567
|
class ProcessedData:
|
|
553
|
-
"""Stores the paths to the directories and files that make up the 'processed_data' session directory.
|
|
568
|
+
"""Stores the paths to the directories and files that make up the 'processed_data' session-specific directory.
|
|
554
569
|
|
|
555
|
-
The processed_data directory stores the data generated by various processing pipelines from the raw data
|
|
556
|
-
data represents an intermediate step between raw data and the dataset used in
|
|
557
|
-
designed to be analyzed.
|
|
570
|
+
The processed_data directory stores the data generated by various processing pipelines from the raw data (contents
|
|
571
|
+
of the raw_data directory). Processed data represents an intermediate step between raw data and the dataset used in
|
|
572
|
+
the data analysis, but is not itself designed to be analyzed.
|
|
558
573
|
|
|
559
574
|
Notes:
|
|
560
575
|
The paths from this section are typically used only on the BioHPC server. This is because most data processing
|
|
561
576
|
in the lab is performed using the processing server's resources. On the server, processed data is stored on
|
|
562
|
-
the fast volume, in contrast to raw data, which is stored on the slow volume.
|
|
563
|
-
the class resolves the paths in this section both locally and globally (on the server).
|
|
577
|
+
the fast (NVME) drive volume, in contrast to raw data, which is stored on the slow (SSD) drive volume.
|
|
564
578
|
|
|
579
|
+
When this class is instantiated on a machine other than BioHPC server, for example, to test processing
|
|
580
|
+
pipelines, it uses the same drive as the raw_data folder to create the processed_data folder. This relies on the
|
|
581
|
+
assumption that non-server machines in the lab only use fast NVME drives, so there is no need to separate
|
|
582
|
+
storage and processing volumes.
|
|
565
583
|
"""
|
|
566
584
|
|
|
567
|
-
processed_data_path:
|
|
585
|
+
processed_data_path: Path = Path()
|
|
568
586
|
"""Stores the path to the root processed_data directory of the session. This directory stores the processed data
|
|
569
587
|
as it is generated by various data processing pipelines."""
|
|
570
|
-
|
|
571
|
-
"""Stores the path to the
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
"""Stores the
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
processing_tracker_path: str | Path = "null"
|
|
590
|
-
"""Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
|
|
591
|
-
the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
|
|
592
|
-
session and the status (success / failure) of each processing step.
|
|
588
|
+
camera_data_path: Path = Path()
|
|
589
|
+
"""Stores the path to the directory that contains video tracking data generated by our DeepLabCut-based video
|
|
590
|
+
processing pipelines."""
|
|
591
|
+
mesoscope_data_path: Path = Path()
|
|
592
|
+
"""Stores path to the directory that contains processed brain activity (cell) data generated by our suite2p-based
|
|
593
|
+
photometry processing pipelines (single day and multi day)."""
|
|
594
|
+
behavior_data_path: Path = Path()
|
|
595
|
+
"""Stores the path to the directory that contains the non-video behavior and system runtime data extracted from
|
|
596
|
+
.npz log files by our in-house log parsing pipeline."""
|
|
597
|
+
job_logs_path: Path = Path()
|
|
598
|
+
"""Stores the path to the directory that stores the standard output and standard error data collected during
|
|
599
|
+
server-side data processing pipeline runtimes. Since we use SLURM job manager to execute multiple compute jobs on
|
|
600
|
+
the BioHPC server, all information sent to the terminal during runtime is redirected to text files stored in this
|
|
601
|
+
directory."""
|
|
602
|
+
processing_tracker_path: Path = Path()
|
|
603
|
+
"""Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory.
|
|
604
|
+
This file is used to track which processing pipelines need to be applied to the target session and the status
|
|
605
|
+
(success / failure) each of these pipelines. Primarily, this is used to optimize data processing to avoid redundant
|
|
606
|
+
operations.
|
|
593
607
|
"""
|
|
594
|
-
multiday_configuration_path: str | Path = "null"
|
|
595
|
-
"""Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
|
|
596
|
-
on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
|
|
597
|
-
sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
|
|
598
608
|
|
|
599
|
-
def
|
|
600
|
-
"""
|
|
601
|
-
are converted to Path objects.
|
|
602
|
-
"""
|
|
609
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
610
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
603
611
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
if self.camera_data_path == "null":
|
|
612
|
-
self.camera_data_path = self.processed_data_path.joinpath("camera_data")
|
|
613
|
-
if self.mesoscope_data_path == "null":
|
|
614
|
-
self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
|
|
615
|
-
if self.behavior_data_path == "null":
|
|
616
|
-
self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
|
|
617
|
-
if self.suite2p_configuration_path == "null":
|
|
618
|
-
self.suite2p_configuration_path = self.processed_data_path.joinpath("suite2p_configuration.yaml")
|
|
619
|
-
if self.processing_tracker_path == "null":
|
|
620
|
-
self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
|
|
621
|
-
if self.multiday_configuration_path == "null":
|
|
622
|
-
self.multiday_configuration_path = self.processed_data_path.joinpath("multiday_configuration.yaml")
|
|
623
|
-
|
|
624
|
-
def make_string(self) -> None:
|
|
625
|
-
"""Converts all Path objects stored inside the class to strings.
|
|
626
|
-
|
|
627
|
-
This transformation is required to support dumping class data into a .YAML file so that the data can be stored
|
|
628
|
-
on disk.
|
|
629
|
-
"""
|
|
630
|
-
self.processed_data_path = str(self.processed_data_path)
|
|
631
|
-
self.deeplabcut_path = str(self.deeplabcut_path)
|
|
632
|
-
self.configuration_path = str(self.configuration_path)
|
|
633
|
-
self.camera_data_path = str(self.camera_data_path)
|
|
634
|
-
self.mesoscope_data_path = str(self.mesoscope_data_path)
|
|
635
|
-
self.behavior_data_path = str(self.behavior_data_path)
|
|
636
|
-
self.suite2p_configuration_path = str(self.suite2p_configuration_path)
|
|
637
|
-
self.processing_tracker_path = str(self.processing_tracker_path)
|
|
638
|
-
self.multiday_configuration_path = str(self.multiday_configuration_path)
|
|
639
|
-
|
|
640
|
-
def make_dirs(self) -> None:
|
|
641
|
-
"""Ensures that all major subdirectories of the processed_data directory exist.
|
|
642
|
-
|
|
643
|
-
This method is used by the BioHPC server to generate the processed_data directory as part of the sl-forgery
|
|
644
|
-
library runtime.
|
|
612
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
613
|
+
machine that instantiates the class.
|
|
614
|
+
|
|
615
|
+
Args:
|
|
616
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
617
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
618
|
+
the managed session.
|
|
645
619
|
"""
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
620
|
+
# Generates the managed paths
|
|
621
|
+
self.processed_data_path = root_directory_path
|
|
622
|
+
self.camera_data_path = self.processed_data_path.joinpath("camera_data")
|
|
623
|
+
self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
|
|
624
|
+
self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
|
|
625
|
+
self.job_logs_path = self.processed_data_path.joinpath("job_logs")
|
|
626
|
+
self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
|
|
627
|
+
|
|
628
|
+
def make_directories(self) -> None:
|
|
629
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
630
|
+
|
|
631
|
+
ensure_directory_exists(self.processed_data_path)
|
|
632
|
+
ensure_directory_exists(self.camera_data_path)
|
|
633
|
+
ensure_directory_exists(self.mesoscope_data_path)
|
|
634
|
+
ensure_directory_exists(self.behavior_data_path)
|
|
635
|
+
ensure_directory_exists(self.job_logs_path)
|
|
652
636
|
|
|
653
637
|
|
|
654
638
|
@dataclass()
|
|
655
|
-
class
|
|
656
|
-
"""Stores the paths to the directories and files that make up the 'persistent_data'
|
|
657
|
-
the ScanImagePC.
|
|
639
|
+
class VRPCPersistentData:
|
|
640
|
+
"""Stores the paths to the directories and files that make up the 'persistent_data' directory on the VRPC.
|
|
658
641
|
|
|
659
|
-
Persistent data directories are used
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
previous session.
|
|
642
|
+
Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
|
|
643
|
+
are purposefully designed for specific PCs that participate in data acquisition. This section manages the
|
|
644
|
+
animal-specific persistent_data directory stored on the VRPC.
|
|
663
645
|
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
only used internally by the sl-experiment or sl-forgery libraries and is not intended for end-users.
|
|
646
|
+
VRPC persistent data directory is used to preserve configuration data, such as the positions of Zaber motors and
|
|
647
|
+
Meososcope objective, so that they can be reused across sessions of the same animals. The data in this directory
|
|
648
|
+
is read at the beginning of each session and replaced at the end of each session.
|
|
668
649
|
"""
|
|
669
650
|
|
|
670
|
-
|
|
671
|
-
"""Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
|
|
672
|
-
belongs, relative to the ScanImagePC root. This directory is primarily used to store reference motion registration
|
|
673
|
-
data acquired during the first experiment session for each animal."""
|
|
674
|
-
vrpc_persistent_path: str | Path
|
|
651
|
+
persistent_data_path: Path = Path()
|
|
675
652
|
"""Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
|
|
676
|
-
belongs, relative to the VRPC root. This directory is
|
|
677
|
-
|
|
678
|
-
zaber_positions_path: str | Path = "null"
|
|
653
|
+
belongs, relative to the VRPC root. This directory is exclusively used on the VRPC."""
|
|
654
|
+
zaber_positions_path: Path = Path()
|
|
679
655
|
"""Stores the path to the Zaber motor positions snapshot generated at the end of the previous session runtime. This
|
|
680
656
|
is used to automatically restore all Zaber motors to the same position across all sessions."""
|
|
681
|
-
mesoscope_positions_path:
|
|
657
|
+
mesoscope_positions_path: Path = Path()
|
|
682
658
|
"""Stores the path to the Mesoscope positions snapshot generated at the end of the previous session runtime. This
|
|
683
659
|
is used to help the user to (manually) restore the Mesoscope to the same position across all sessions."""
|
|
684
|
-
motion_estimator_path: str | Path = "null"
|
|
685
|
-
"""Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
|
|
686
|
-
file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
|
|
687
660
|
|
|
688
|
-
def
|
|
689
|
-
"""
|
|
690
|
-
|
|
661
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
662
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
663
|
+
|
|
664
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
665
|
+
machine that instantiates the class.
|
|
666
|
+
|
|
667
|
+
Args:
|
|
668
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
669
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
670
|
+
the managed session.
|
|
691
671
|
"""
|
|
692
672
|
|
|
693
|
-
#
|
|
694
|
-
self.
|
|
695
|
-
self.
|
|
673
|
+
# Generates the managed paths
|
|
674
|
+
self.persistent_data_path = root_directory_path
|
|
675
|
+
self.zaber_positions_path = self.persistent_data_path.joinpath("zaber_positions.yaml")
|
|
676
|
+
self.mesoscope_positions_path = self.persistent_data_path.joinpath("mesoscope_positions.yaml")
|
|
677
|
+
|
|
678
|
+
def make_directories(self) -> None:
|
|
679
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
680
|
+
|
|
681
|
+
ensure_directory_exists(self.persistent_data_path)
|
|
682
|
+
|
|
696
683
|
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
684
|
+
@dataclass()
|
|
685
|
+
class ScanImagePCPersistentData:
|
|
686
|
+
"""Stores the paths to the directories and files that make up the 'persistent_data' directory on the ScanImagePC.
|
|
687
|
+
|
|
688
|
+
Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
|
|
689
|
+
are purposefully designed for specific PCs that participate in data acquisition. This section manages the
|
|
690
|
+
animal-specific persistent_data directory stored on the ScanImagePC (Mesoscope PC).
|
|
691
|
+
|
|
692
|
+
ScanImagePC persistent data directory is used to preserve the motion estimation snapshot, generated during the first
|
|
693
|
+
experiment session. This is necessary to align the brain recording field of view across sessions. In turn, this
|
|
694
|
+
is used to carry out 'online' motion and z-drift correction, improving the accuracy of across-day (multi-day)
|
|
695
|
+
cell tracking.
|
|
696
|
+
"""
|
|
697
|
+
|
|
698
|
+
persistent_data_path: Path = Path()
|
|
699
|
+
"""Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
|
|
700
|
+
belongs, relative to the ScanImagePC root. This directory is exclusively used on the ScanImagePC (Mesoscope PC)."""
|
|
701
|
+
motion_estimator_path: Path = Path()
|
|
702
|
+
"""Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
|
|
703
|
+
file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
|
|
704
704
|
|
|
705
|
-
def
|
|
706
|
-
"""
|
|
705
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
706
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
707
707
|
|
|
708
|
-
This
|
|
709
|
-
|
|
708
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
709
|
+
machine that instantiates the class.
|
|
710
|
+
|
|
711
|
+
Args:
|
|
712
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
713
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
714
|
+
the managed session.
|
|
710
715
|
"""
|
|
711
|
-
self.scanimage_persistent_path = str(self.scanimage_persistent_path)
|
|
712
|
-
self.vrpc_persistent_path = str(self.vrpc_persistent_path)
|
|
713
|
-
self.zaber_positions_path = str(self.zaber_positions_path)
|
|
714
|
-
self.mesoscope_positions_path = str(self.mesoscope_positions_path)
|
|
715
|
-
self.motion_estimator_path = str(self.motion_estimator_path)
|
|
716
716
|
|
|
717
|
-
|
|
718
|
-
|
|
717
|
+
# Generates the managed paths
|
|
718
|
+
self.persistent_data_path = root_directory_path
|
|
719
|
+
self.motion_estimator_path = self.persistent_data_path.joinpath("MotionEstimator.me")
|
|
719
720
|
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
721
|
+
def make_directories(self) -> None:
|
|
722
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
723
|
+
|
|
724
|
+
ensure_directory_exists(self.persistent_data_path)
|
|
723
725
|
|
|
724
726
|
|
|
725
727
|
@dataclass()
|
|
726
728
|
class MesoscopeData:
|
|
727
|
-
"""Stores the paths to the directories
|
|
728
|
-
|
|
729
|
+
"""Stores the paths to the directories and files that make up the 'meso_data' directory on the ScanImagePC.
|
|
730
|
+
|
|
731
|
+
The meso_data directory is the root directory where all mesoscope-generated data is stored on the ScanImagePC. The
|
|
732
|
+
path to this directory should be given relative to the VRPC root and be mounted to the VRPC filesystem via the
|
|
733
|
+
SMB or equivalent protocol.
|
|
729
734
|
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
735
|
+
During runtime, the ScanImagePC should organize all collected data under this root directory. During preprocessing,
|
|
736
|
+
the VRPC uses SMB to access the data in this directory and merge it into the 'raw_data' session directory. The paths
|
|
737
|
+
in this section, therefore, are specific to the VRPC and are not used on other PCs.
|
|
733
738
|
"""
|
|
734
739
|
|
|
735
|
-
|
|
740
|
+
meso_data_path: Path = Path()
|
|
736
741
|
"""Stores the path to the root ScanImagePC data directory, mounted to the VRPC filesystem via the SMB or equivalent
|
|
737
|
-
protocol.
|
|
738
|
-
|
|
739
|
-
|
|
742
|
+
protocol. All mesoscope-generated data is stored under this root directory before it is merged into the VRPC-managed
|
|
743
|
+
raw_data directory of each session."""
|
|
744
|
+
mesoscope_data_path: Path = Path()
|
|
745
|
+
"""Stores the path to the 'default' mesoscope_data directory. All experiment sessions across all animals and
|
|
746
|
+
projects use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
|
|
747
|
+
software. This simplifies ScanImagePC configuration process during runtime, as all data is always saved in the same
|
|
748
|
+
directory. During preprocessing, the data is moved from the default directory first into a session-specific
|
|
749
|
+
ScanImagePC directory and then into the VRPC raw_data session directory."""
|
|
750
|
+
session_specific_path: Path = Path()
|
|
740
751
|
"""Stores the path to the session-specific data directory. This directory is generated at the end of each experiment
|
|
741
|
-
runtime to prepare mesoscope data for
|
|
742
|
-
runtime."""
|
|
743
|
-
|
|
744
|
-
"""Stores the path to the 'general' mesoscope_data directory. All experiment sessions (across all animals and
|
|
745
|
-
projects) use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
|
|
746
|
-
software. This simplifies ScanImagePC configuration process during runtime. The data is moved into a
|
|
747
|
-
session-specific directory during preprocessing."""
|
|
748
|
-
ubiquitin_path: str | Path = "null"
|
|
752
|
+
runtime to prepare mesoscope data for being moved to the VRPC-managed raw_data directory and to reset the 'default'
|
|
753
|
+
mesoscope_data directory for the next session's runtime."""
|
|
754
|
+
ubiquitin_path: Path = Path()
|
|
749
755
|
"""Stores the path to the 'ubiquitin.bin' file. This file is automatically generated inside the session-specific
|
|
750
|
-
data
|
|
751
|
-
removal, the VRPC searches for directories marked with ubiquitin.bin
|
|
756
|
+
data directory after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
|
|
757
|
+
removal step of preprocessing, the VRPC searches for directories marked with ubiquitin.bin and deletes them from the
|
|
752
758
|
ScanImagePC filesystem."""
|
|
753
759
|
|
|
754
|
-
def
|
|
755
|
-
"""
|
|
756
|
-
are converted to Path objects.
|
|
757
|
-
"""
|
|
758
|
-
# Converts the paths that have to be provided at class instantiation to the Path objects.
|
|
759
|
-
self.root_data_path = Path(self.root_data_path)
|
|
760
|
-
self.session_specific_path = Path(self.session_specific_path)
|
|
761
|
-
|
|
762
|
-
# Resolves the 'null' placeholders using the 'anchor' paths above
|
|
763
|
-
if self.mesoscope_data_path == "null":
|
|
764
|
-
self.mesoscope_data_path = self.root_data_path.joinpath("mesoscope_data")
|
|
765
|
-
if self.ubiquitin_path == "null":
|
|
766
|
-
self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
|
|
760
|
+
def resolve_paths(self, root_mesoscope_path: Path, session_name: str) -> None:
|
|
761
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
767
762
|
|
|
768
|
-
|
|
769
|
-
|
|
763
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
764
|
+
machine that instantiates the class.
|
|
770
765
|
|
|
771
|
-
|
|
772
|
-
|
|
766
|
+
Args:
|
|
767
|
+
root_mesoscope_path: The path to the top-level directory of the ScanImagePC data hierarchy mounted to the
|
|
768
|
+
VRPC via the SMB or equivalent protocol.
|
|
769
|
+
session_name: The name of the session for which this subclass is initialized.
|
|
773
770
|
"""
|
|
774
|
-
self.root_data_path = str(self.root_data_path)
|
|
775
|
-
self.session_specific_path = str(self.session_specific_path)
|
|
776
|
-
self.mesoscope_data_path = str(self.mesoscope_data_path)
|
|
777
|
-
self.ubiquitin_path = str(self.ubiquitin_path)
|
|
778
771
|
|
|
779
|
-
|
|
780
|
-
|
|
772
|
+
# Generates the managed paths
|
|
773
|
+
self.meso_data_path = root_mesoscope_path
|
|
774
|
+
self.session_specific_path = self.meso_data_path.joinpath(session_name)
|
|
775
|
+
self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
|
|
776
|
+
self.mesoscope_data_path = self.meso_data_path.joinpath("mesoscope_data")
|
|
781
777
|
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
ensure_directory_exists(Path(self.mesoscope_data_path))
|
|
778
|
+
def make_directories(self) -> None:
|
|
779
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
780
|
+
|
|
781
|
+
ensure_directory_exists(self.meso_data_path)
|
|
787
782
|
|
|
788
783
|
|
|
789
784
|
@dataclass()
|
|
790
|
-
class
|
|
791
|
-
"""Stores the paths to the VRPC filesystem-mounted Synology NAS and BioHPC server
|
|
785
|
+
class VRPCDestinations:
|
|
786
|
+
"""Stores the paths to the VRPC filesystem-mounted directories of the Synology NAS and BioHPC server.
|
|
787
|
+
|
|
788
|
+
The paths from this section are primarily used to transfer preprocessed data to the long-term storage destinations.
|
|
789
|
+
Additionally, they allow VRPC to interface with the configuration directory of the BioHPC server to start data
|
|
790
|
+
processing jobs and to read the data from the processed_data directory to remove redundant data from the VRPC
|
|
791
|
+
filesystem.
|
|
792
792
|
|
|
793
|
-
|
|
794
|
-
VRPC to the long-term storage destinations.
|
|
793
|
+
Overall, this section is intended solely for the VRPC and should not be used on other PCs.
|
|
795
794
|
"""
|
|
796
795
|
|
|
797
|
-
nas_raw_data_path:
|
|
796
|
+
nas_raw_data_path: Path = Path()
|
|
798
797
|
"""Stores the path to the session's raw_data directory on the Synology NAS, which is mounted to the VRPC via the
|
|
799
798
|
SMB or equivalent protocol."""
|
|
800
|
-
server_raw_data_path:
|
|
799
|
+
server_raw_data_path: Path = Path()
|
|
801
800
|
"""Stores the path to the session's raw_data directory on the BioHPC server, which is mounted to the VRPC via the
|
|
802
801
|
SMB or equivalent protocol."""
|
|
803
|
-
server_processed_data_path:
|
|
802
|
+
server_processed_data_path: Path = Path()
|
|
804
803
|
"""Stores the path to the session's processed_data directory on the BioHPC server, which is mounted to the VRPC via
|
|
805
804
|
the SMB or equivalent protocol."""
|
|
806
|
-
server_configuration_path:
|
|
805
|
+
server_configuration_path: Path = Path()
|
|
807
806
|
"""Stores the path to the project-specific 'configuration' directory on the BioHPC server, which is mounted to the
|
|
808
807
|
VRPC via the SMB or equivalent protocol."""
|
|
809
|
-
|
|
810
|
-
"""Stores the path to the session's telomere.bin marker. This marker is generated as part of
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
808
|
+
telomere_path: Path = Path()
|
|
809
|
+
"""Stores the path to the session's telomere.bin marker. This marker is generated as part of data processing on the
|
|
810
|
+
BioHPC server to notify the VRPC that the server received preprocessed data intact. The presence of this marker is
|
|
811
|
+
used by the VRPC to determine which locally stored raw_data is safe to delete from the filesystem."""
|
|
812
|
+
suite2p_configuration_path: Path = Path()
|
|
814
813
|
"""Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
|
|
815
|
-
the
|
|
816
|
-
|
|
817
|
-
|
|
814
|
+
the BioHPC server. This configuration file specifies the parameters for the 'single day' sl-suite2p registration
|
|
815
|
+
pipeline, which is applied to each session that generates brain activity data."""
|
|
816
|
+
processing_tracker_path: Path = Path()
|
|
818
817
|
"""Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
|
|
819
|
-
the
|
|
820
|
-
|
|
818
|
+
the BioHPC server. This file tracks which processing pipelines need to be applied the target session and the status
|
|
819
|
+
(success / failure) of each applied pipeline.
|
|
821
820
|
"""
|
|
822
|
-
|
|
821
|
+
multiday_configuration_path: Path = Path()
|
|
823
822
|
"""Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
|
|
824
|
-
on the
|
|
825
|
-
|
|
823
|
+
on the BioHPC server. This configuration file specifies the parameters for the 'multiday' sl-suite2p registration
|
|
824
|
+
pipeline used to track brain cells across multiple sessions."""
|
|
826
825
|
|
|
827
|
-
def
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
if self.server_telomere_path == "null":
|
|
837
|
-
self.server_telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
|
|
838
|
-
if self.server_suite2p_configuration_path == "null":
|
|
839
|
-
self.server_suite2p_configuration_path = self.server_configuration_path.joinpath(
|
|
840
|
-
"suite2p_configuration.yaml"
|
|
841
|
-
)
|
|
842
|
-
if self.server_processing_tracker_path == "null":
|
|
843
|
-
self.server_processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
|
|
844
|
-
if self.server_multiday_configuration_path == "null":
|
|
845
|
-
self.server_multiday_configuration_path = self.server_configuration_path.joinpath(
|
|
846
|
-
"multiday_configuration.yaml"
|
|
847
|
-
)
|
|
826
|
+
def resolve_paths(
|
|
827
|
+
self,
|
|
828
|
+
nas_raw_data_path: Path,
|
|
829
|
+
server_raw_data_path: Path,
|
|
830
|
+
server_processed_data_path: Path,
|
|
831
|
+
server_configuration_path: Path,
|
|
832
|
+
) -> None:
|
|
833
|
+
"""Resolves all paths managed by the class instance based on the input root directory paths.
|
|
848
834
|
|
|
849
|
-
|
|
850
|
-
|
|
835
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
836
|
+
machine that instantiates the class.
|
|
851
837
|
|
|
852
|
-
|
|
853
|
-
|
|
838
|
+
Args:
|
|
839
|
+
nas_raw_data_path: The path to the session's raw_data directory on the Synology NAS, relative to the VRPC
|
|
840
|
+
filesystem root.
|
|
841
|
+
server_raw_data_path: The path to the session's raw_data directory on the BioHPC server, relative to the
|
|
842
|
+
VRPC filesystem root.
|
|
843
|
+
server_processed_data_path: The path to the session's processed_data directory on the BioHPC server,
|
|
844
|
+
relative to the VRPC filesystem root.
|
|
845
|
+
server_configuration_path: The path to the project-specific 'configuration' directory on the BioHPC server,
|
|
846
|
+
relative to the VRPC filesystem root.
|
|
854
847
|
"""
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
self.
|
|
858
|
-
self.
|
|
859
|
-
self.server_processed_data_path =
|
|
860
|
-
self.
|
|
861
|
-
self.
|
|
862
|
-
self.
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
ensure_directory_exists(
|
|
869
|
-
ensure_directory_exists(
|
|
848
|
+
|
|
849
|
+
# Generates the managed paths
|
|
850
|
+
self.nas_raw_data_path = nas_raw_data_path
|
|
851
|
+
self.server_raw_data_path = server_raw_data_path
|
|
852
|
+
self.server_processed_data_path = server_processed_data_path
|
|
853
|
+
self.server_configuration_path = server_configuration_path
|
|
854
|
+
self.telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
|
|
855
|
+
self.suite2p_configuration_path = self.server_configuration_path.joinpath("suite2p_configuration.yaml")
|
|
856
|
+
self.processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
|
|
857
|
+
self.multiday_configuration_path = self.server_configuration_path.joinpath("multiday_configuration.yaml")
|
|
858
|
+
|
|
859
|
+
def make_directories(self) -> None:
|
|
860
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
861
|
+
ensure_directory_exists(self.nas_raw_data_path)
|
|
862
|
+
ensure_directory_exists(self.server_raw_data_path)
|
|
863
|
+
ensure_directory_exists(self.server_configuration_path)
|
|
864
|
+
ensure_directory_exists(self.server_processed_data_path)
|
|
870
865
|
|
|
871
866
|
|
|
872
867
|
@dataclass
|
|
@@ -912,32 +907,30 @@ class SessionData(YamlConfig):
|
|
|
912
907
|
field is not None (null), it communicates the specific experiment configuration used by the session. During runtime,
|
|
913
908
|
the name stored here is used to load the specific experiment configuration data stored in a .yaml file with the
|
|
914
909
|
same name. If the session is not an experiment session, this field is ignored."""
|
|
915
|
-
raw_data: RawData
|
|
916
|
-
"""
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
"""
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
destinations:
|
|
937
|
-
"""
|
|
938
|
-
|
|
939
|
-
should be mapped (mounted) to the VRPC's filesystem via the SMB or equivalent protocol. This section is not used
|
|
940
|
-
during data processing."""
|
|
910
|
+
raw_data: RawData = field(default_factory=lambda: RawData())
|
|
911
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/session/raw_data directory of any
|
|
912
|
+
PC used to work with Sun lab data."""
|
|
913
|
+
processed_data: ProcessedData = field(default_factory=lambda: ProcessedData())
|
|
914
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/session/processed_data directory of
|
|
915
|
+
any PC used to work with Sun lab data."""
|
|
916
|
+
deeplabcut_data: DeepLabCutData = field(default_factory=lambda: DeepLabCutData())
|
|
917
|
+
"""Stores the paths to all subfolders and files found under the /project/deeplabcut directory of any PC used to
|
|
918
|
+
work with Sun lab data."""
|
|
919
|
+
configuration_data: ConfigurationData = field(default_factory=lambda: ConfigurationData())
|
|
920
|
+
"""Stores the paths to all subfolders and files found under the /project/configuration directory of any PC used to
|
|
921
|
+
work with Sun lab data."""
|
|
922
|
+
vrpc_persistent_data: VRPCPersistentData = field(default_factory=lambda: VRPCPersistentData())
|
|
923
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
|
|
924
|
+
the VRPC used in the Sun lab to acquire behavior data."""
|
|
925
|
+
scanimagepc_persistent_data: ScanImagePCPersistentData = field(default_factory=lambda: ScanImagePCPersistentData())
|
|
926
|
+
"""Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
|
|
927
|
+
the ScanImagePC used in the Sun lab to acquire brain activity data."""
|
|
928
|
+
mesoscope_data: MesoscopeData = field(default_factory=lambda: MesoscopeData())
|
|
929
|
+
"""Stores the paths to all subfolders and files found under the /meso_data (root mesoscope data) directory of
|
|
930
|
+
the ScanImagePC used in the Sun lab to acquire brain activity data."""
|
|
931
|
+
destinations: VRPCDestinations = field(default_factory=lambda: VRPCDestinations())
|
|
932
|
+
"""Stores the paths to all subfolders and files under various VRPC-filesystem-mounted directories of other machines
|
|
933
|
+
used in the Sun lab for long-term data storage."""
|
|
941
934
|
|
|
942
935
|
@classmethod
|
|
943
936
|
def create(
|
|
@@ -986,8 +979,8 @@ class SessionData(YamlConfig):
|
|
|
986
979
|
if session_name is None:
|
|
987
980
|
session_name = str(get_timestamp(time_separator="-"))
|
|
988
981
|
|
|
989
|
-
# Extracts the root directory paths stored inside the project configuration file.
|
|
990
|
-
#
|
|
982
|
+
# Extracts the root directory paths stored inside the project configuration file. These roots are then used to
|
|
983
|
+
# initialize this class instance.
|
|
991
984
|
vrpc_root = Path(project_configuration.local_root_directory)
|
|
992
985
|
mesoscope_root = Path(project_configuration.local_mesoscope_directory)
|
|
993
986
|
biohpc_root = Path(project_configuration.local_server_directory)
|
|
@@ -997,7 +990,7 @@ class SessionData(YamlConfig):
|
|
|
997
990
|
# Extracts the name of the project stored inside the project configuration file.
|
|
998
991
|
project_name = project_configuration.project_name
|
|
999
992
|
|
|
1000
|
-
# Constructs the session directory path
|
|
993
|
+
# Constructs the session directory path
|
|
1001
994
|
session_path = vrpc_root.joinpath(project_name, animal_id, session_name)
|
|
1002
995
|
|
|
1003
996
|
# Handles potential session name conflicts
|
|
@@ -1015,40 +1008,51 @@ class SessionData(YamlConfig):
|
|
|
1015
1008
|
f"already exists. The newly created session directory uses a '_{counter}' postfix to distinguish "
|
|
1016
1009
|
f"itself from the already existing session directory."
|
|
1017
1010
|
)
|
|
1018
|
-
|
|
1011
|
+
console.echo(message=message, level=LogLevel.ERROR)
|
|
1019
1012
|
|
|
1020
1013
|
# Generates subclasses stored inside the main class instance based on the data resolved above. Note; most fields
|
|
1021
|
-
# of these classes are
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
)
|
|
1025
|
-
raw_data.
|
|
1026
|
-
|
|
1027
|
-
processed_data = ProcessedData(
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1014
|
+
# of these classes are resolved automatically, based on one or more 'root' paths provided to the 'resolve_paths'
|
|
1015
|
+
# method.
|
|
1016
|
+
raw_data = RawData()
|
|
1017
|
+
raw_data.resolve_paths(root_directory_path=session_path.joinpath("raw_data"))
|
|
1018
|
+
raw_data.make_directories() # Generates the local directory tree
|
|
1019
|
+
|
|
1020
|
+
processed_data = ProcessedData()
|
|
1021
|
+
processed_data.resolve_paths(root_directory_path=session_path.joinpath("processed_data"))
|
|
1022
|
+
processed_data.make_directories()
|
|
1023
|
+
|
|
1024
|
+
dlc_data = DeepLabCutData()
|
|
1025
|
+
dlc_data.resolve_paths(root_directory_path=vrpc_root.joinpath(project_name, "deeplabcut"))
|
|
1026
|
+
dlc_data.make_directories()
|
|
1027
|
+
|
|
1028
|
+
configuration_data = ConfigurationData()
|
|
1029
|
+
configuration_data.resolve_paths(
|
|
1030
|
+
root_directory_path=vrpc_root.joinpath(project_name, "configuration"), experiment_name=experiment_name
|
|
1031
1031
|
)
|
|
1032
|
+
configuration_data.make_directories()
|
|
1032
1033
|
|
|
1034
|
+
vrpc_persistent_data = VRPCPersistentData()
|
|
1033
1035
|
vrpc_persistent_path = vrpc_root.joinpath(project_name, animal_id, "persistent_data")
|
|
1036
|
+
vrpc_persistent_data.resolve_paths(root_directory_path=vrpc_persistent_path)
|
|
1037
|
+
vrpc_persistent_data.make_directories()
|
|
1038
|
+
|
|
1039
|
+
scanimagepc_persistent_data = ScanImagePCPersistentData()
|
|
1034
1040
|
scanimagepc_persistent_path = mesoscope_root.joinpath(project_name, animal_id, "persistent_data")
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
)
|
|
1038
|
-
persistent_data.make_dirs() # Generates all persistent directory trees
|
|
1041
|
+
scanimagepc_persistent_data.resolve_paths(root_directory_path=scanimagepc_persistent_path)
|
|
1042
|
+
scanimagepc_persistent_data.make_directories()
|
|
1039
1043
|
|
|
1040
|
-
mesoscope_data = MesoscopeData(
|
|
1041
|
-
|
|
1042
|
-
)
|
|
1043
|
-
mesoscope_data.make_dirs() # Generates all Mesoscope directory trees
|
|
1044
|
+
mesoscope_data = MesoscopeData()
|
|
1045
|
+
mesoscope_data.resolve_paths(root_mesoscope_path=mesoscope_root, session_name=session_name)
|
|
1046
|
+
mesoscope_data.make_directories()
|
|
1044
1047
|
|
|
1045
|
-
destinations =
|
|
1048
|
+
destinations = VRPCDestinations()
|
|
1049
|
+
destinations.resolve_paths(
|
|
1046
1050
|
nas_raw_data_path=nas_root.joinpath(project_name, animal_id, session_name, "raw_data"),
|
|
1047
1051
|
server_raw_data_path=biohpc_root.joinpath(project_name, animal_id, session_name, "raw_data"),
|
|
1048
|
-
server_configuration_path=
|
|
1052
|
+
server_configuration_path=biohpc_root.joinpath(project_name, "configuration"),
|
|
1049
1053
|
server_processed_data_path=biohpc_workdir.joinpath(project_name, "processed_data"),
|
|
1050
1054
|
)
|
|
1051
|
-
destinations.
|
|
1055
|
+
destinations.make_directories()
|
|
1052
1056
|
|
|
1053
1057
|
# Packages the sections generated above into a SessionData instance
|
|
1054
1058
|
instance = SessionData(
|
|
@@ -1057,8 +1061,11 @@ class SessionData(YamlConfig):
|
|
|
1057
1061
|
session_name=session_name,
|
|
1058
1062
|
session_type=session_type,
|
|
1059
1063
|
raw_data=raw_data,
|
|
1064
|
+
deeplabcut_data=dlc_data,
|
|
1065
|
+
configuration_data=configuration_data,
|
|
1060
1066
|
processed_data=processed_data,
|
|
1061
|
-
|
|
1067
|
+
vrpc_persistent_data=vrpc_persistent_data,
|
|
1068
|
+
scanimagepc_persistent_data=scanimagepc_persistent_data,
|
|
1062
1069
|
mesoscope_data=mesoscope_data,
|
|
1063
1070
|
destinations=destinations,
|
|
1064
1071
|
experiment_name=experiment_name,
|
|
@@ -1072,19 +1079,16 @@ class SessionData(YamlConfig):
|
|
|
1072
1079
|
# classes is not exhaustive. More classes are saved as part of the session runtime management class start() and
|
|
1073
1080
|
# __init__() method runtimes:
|
|
1074
1081
|
|
|
1075
|
-
# Resolves the path to the project configuration folder
|
|
1076
|
-
vrpc_configuration_path = vrpc_root.joinpath(project_name, "configuration")
|
|
1077
|
-
|
|
1078
1082
|
# Discovers and saves the necessary configuration class instances to the raw_data folder of the managed session:
|
|
1079
1083
|
# Project Configuration
|
|
1080
1084
|
sh.copy2(
|
|
1081
|
-
src=
|
|
1085
|
+
src=instance.configuration_data.project_configuration_path,
|
|
1082
1086
|
dst=instance.raw_data.project_configuration_path,
|
|
1083
1087
|
)
|
|
1084
1088
|
# Experiment Configuration, if the session type is Experiment.
|
|
1085
1089
|
if experiment_name is not None:
|
|
1086
1090
|
sh.copy2(
|
|
1087
|
-
src=
|
|
1091
|
+
src=instance.configuration_data.experiment_configuration_path,
|
|
1088
1092
|
dst=instance.raw_data.experiment_configuration_path,
|
|
1089
1093
|
)
|
|
1090
1094
|
|
|
@@ -1108,9 +1112,10 @@ class SessionData(YamlConfig):
|
|
|
1108
1112
|
|
|
1109
1113
|
Args:
|
|
1110
1114
|
session_path: The path to the root directory of an existing session, e.g.: vrpc_root/project/animal/session.
|
|
1111
|
-
on_server: Determines whether the method is used to initialize an existing session on the
|
|
1112
|
-
|
|
1113
|
-
subfolders. BioHPC server runtimes use different volumes (drives) to store these
|
|
1115
|
+
on_server: Determines whether the method is used to initialize an existing session on the BioHPC server or
|
|
1116
|
+
a non-server machine. Note, non-server runtimes use the same 'root' directory to store raw_data and
|
|
1117
|
+
processed_data subfolders. BioHPC server runtimes use different volumes (drives) to store these
|
|
1118
|
+
subfolders.
|
|
1114
1119
|
|
|
1115
1120
|
Returns:
|
|
1116
1121
|
An initialized SessionData instance for the session whose data is stored at the provided path.
|
|
@@ -1130,48 +1135,100 @@ class SessionData(YamlConfig):
|
|
|
1130
1135
|
)
|
|
1131
1136
|
console.error(message=message, error=FileNotFoundError)
|
|
1132
1137
|
|
|
1133
|
-
# Loads class data from .yaml
|
|
1138
|
+
# Loads class data from .yaml file
|
|
1134
1139
|
instance: SessionData = cls.from_yaml(file_path=session_data_path) # type: ignore
|
|
1135
1140
|
|
|
1136
1141
|
# The method assumes that the 'donor' .yaml file is always stored inside the raw_data directory of the session
|
|
1137
1142
|
# to be processed. Since the directory itself might have moved (between or even within the same PC) relative to
|
|
1138
1143
|
# where it was when the SessionData snapshot was generated, reconfigures the paths to all raw_data files using
|
|
1139
1144
|
# the root from above.
|
|
1140
|
-
|
|
1145
|
+
local_root = session_path.parents[2]
|
|
1146
|
+
|
|
1147
|
+
# RAW DATA
|
|
1148
|
+
new_root = local_root.joinpath(instance.project_name, instance.animal_id, instance.session_name, "raw_data")
|
|
1149
|
+
instance.raw_data.resolve_paths(root_directory_path=new_root)
|
|
1150
|
+
|
|
1151
|
+
# Uses the adjusted raw_data section to load the ProjectConfiguration instance. This is used below to resolve
|
|
1152
|
+
# all other SessionData sections, as it stores various required root directories.
|
|
1153
|
+
project_configuration: ProjectConfiguration = ProjectConfiguration.load(
|
|
1154
|
+
project_name=instance.project_name,
|
|
1155
|
+
configuration_path=Path(instance.raw_data.project_configuration_path),
|
|
1156
|
+
)
|
|
1157
|
+
|
|
1158
|
+
# Resolves the new roots for all sections that use the same root as the raw_data directory:
|
|
1159
|
+
|
|
1160
|
+
# CONFIGURATION
|
|
1161
|
+
new_root = local_root.joinpath(instance.project_name, "configuration")
|
|
1162
|
+
instance.configuration_data.resolve_paths(
|
|
1163
|
+
root_directory_path=new_root, experiment_name=instance.experiment_name
|
|
1164
|
+
)
|
|
1165
|
+
|
|
1166
|
+
# DEEPLABCUT
|
|
1167
|
+
new_root = local_root.joinpath(instance.project_name, "deeplabcut")
|
|
1168
|
+
instance.deeplabcut_data.resolve_paths(root_directory_path=new_root)
|
|
1169
|
+
|
|
1170
|
+
# Resolves the roots for all VRPC-specific sections that use the data from the ProjectConfiguration instance:
|
|
1171
|
+
|
|
1172
|
+
# VRPC PERSISTENT DATA
|
|
1173
|
+
new_root = Path(project_configuration.local_root_directory).joinpath(
|
|
1174
|
+
instance.project_name, instance.animal_id, "persistent_data"
|
|
1175
|
+
)
|
|
1176
|
+
instance.vrpc_persistent_data.resolve_paths(root_directory_path=new_root)
|
|
1177
|
+
|
|
1178
|
+
# SCANIMAGEPC PERSISTENT DATA
|
|
1179
|
+
new_root = Path(project_configuration.local_mesoscope_directory).joinpath(
|
|
1180
|
+
instance.project_name, instance.animal_id, "persistent_data"
|
|
1181
|
+
)
|
|
1182
|
+
instance.scanimagepc_persistent_data.resolve_paths(root_directory_path=new_root)
|
|
1183
|
+
|
|
1184
|
+
# MESOSCOPE DATA
|
|
1185
|
+
instance.mesoscope_data.resolve_paths(
|
|
1186
|
+
root_mesoscope_path=Path(project_configuration.local_mesoscope_directory),
|
|
1187
|
+
session_name=instance.session_name,
|
|
1188
|
+
)
|
|
1189
|
+
|
|
1190
|
+
# DESTINATIONS
|
|
1191
|
+
instance.destinations.resolve_paths(
|
|
1192
|
+
nas_raw_data_path=Path(project_configuration.local_nas_directory).joinpath(
|
|
1193
|
+
instance.project_name, instance.animal_id, instance.session_name, "raw_data"
|
|
1194
|
+
),
|
|
1195
|
+
server_raw_data_path=Path(project_configuration.local_server_directory).joinpath(
|
|
1196
|
+
instance.project_name, instance.animal_id, instance.session_name, "raw_data"
|
|
1197
|
+
),
|
|
1198
|
+
server_configuration_path=Path(project_configuration.local_server_directory).joinpath(
|
|
1199
|
+
instance.project_name, "configuration"
|
|
1200
|
+
),
|
|
1201
|
+
server_processed_data_path=Path(project_configuration.local_server_working_directory).joinpath(
|
|
1202
|
+
instance.project_name, instance.animal_id, instance.session_name, "processed_data"
|
|
1203
|
+
),
|
|
1204
|
+
)
|
|
1141
1205
|
|
|
1142
1206
|
# Resolves the paths to the processed_data directories. The resolution strategy depends on whether the method is
|
|
1143
1207
|
# called on the VRPC (locally) or the BioHPC server (remotely).
|
|
1144
|
-
new_root: Path
|
|
1145
1208
|
if not on_server:
|
|
1146
1209
|
# Local runtimes use the same root session directory for both raw_data and processed_data. This stems from
|
|
1147
1210
|
# the assumption that most local machines in the lab only use NVME (fast) volumes and, therefore, do not
|
|
1148
1211
|
# need to separate 'storage' and 'working' data directories.
|
|
1149
|
-
|
|
1150
|
-
# Regenerates the processed_data paths using the information loaded from the session_data.yaml file and
|
|
1151
|
-
# static path definition rules used in the lab.
|
|
1152
|
-
new_root = session_path.parents[2] # Reuses the local root for non-server runtimes
|
|
1212
|
+
new_root = local_root # Reuses the local root for non-server runtimes
|
|
1153
1213
|
|
|
1154
1214
|
else:
|
|
1155
1215
|
# The BioHPC server stores raw_data on slow volume and processed_data on fast (NVME) volume. Therefore, to
|
|
1156
1216
|
# configure processed_data paths, the method first needs to load the fast volume root path from the
|
|
1157
1217
|
# project_configuration.yaml file stored in the raw_data folder.
|
|
1158
|
-
project_configuration: ProjectConfiguration = ProjectConfiguration.load(
|
|
1159
|
-
project_name=instance.project_name,
|
|
1160
|
-
configuration_path=Path(instance.raw_data.project_configuration_path),
|
|
1161
|
-
)
|
|
1162
1218
|
new_root = Path(project_configuration.remote_working_directory)
|
|
1163
1219
|
|
|
1164
1220
|
# Regenerates the processed_data path depending on the root resolution above
|
|
1165
|
-
instance.processed_data
|
|
1166
|
-
|
|
1221
|
+
instance.processed_data.resolve_paths(
|
|
1222
|
+
root_directory_path=new_root.joinpath(
|
|
1167
1223
|
instance.project_name, instance.animal_id, instance.session_name, "processed_data"
|
|
1168
|
-
)
|
|
1169
|
-
configuration_path=new_root.joinpath(instance.project_name, instance.animal_id, "configuration"),
|
|
1170
|
-
deeplabcut_path=new_root.joinpath(instance.project_name, "deeplabcut"),
|
|
1224
|
+
)
|
|
1171
1225
|
)
|
|
1172
1226
|
|
|
1173
|
-
# Generates
|
|
1174
|
-
instance.
|
|
1227
|
+
# Generates data directory hierarchies that may be missing on the local machine
|
|
1228
|
+
instance.raw_data.make_directories()
|
|
1229
|
+
instance.configuration_data.make_directories()
|
|
1230
|
+
instance.deeplabcut_data.make_directories()
|
|
1231
|
+
instance.processed_data.make_directories()
|
|
1175
1232
|
|
|
1176
1233
|
# Returns the initialized SessionData instance to caller
|
|
1177
1234
|
return instance
|
|
@@ -1184,21 +1241,8 @@ class SessionData(YamlConfig):
|
|
|
1184
1241
|
create() method runtime.
|
|
1185
1242
|
"""
|
|
1186
1243
|
|
|
1187
|
-
# Copies instance data to prevent it from being modified by reference when executing the steps below
|
|
1188
|
-
original = copy.deepcopy(self)
|
|
1189
|
-
|
|
1190
|
-
# Extracts the target file path before it is converted to a string.
|
|
1191
|
-
file_path: Path = copy.copy(self.raw_data.session_data_path) # type: ignore
|
|
1192
|
-
|
|
1193
|
-
# Converts all Paths objects to strings before dumping the data to YAML.
|
|
1194
|
-
original.raw_data.make_string()
|
|
1195
|
-
original.processed_data.make_string()
|
|
1196
|
-
original.persistent_data.make_string()
|
|
1197
|
-
original.mesoscope_data.make_string()
|
|
1198
|
-
original.destinations.make_string()
|
|
1199
|
-
|
|
1200
1244
|
# Saves instance data as a .YAML file
|
|
1201
|
-
|
|
1245
|
+
self.to_yaml(file_path=self.raw_data.session_data_path)
|
|
1202
1246
|
|
|
1203
1247
|
|
|
1204
1248
|
@dataclass()
|