sl-shared-assets 1.0.0rc13__py3-none-any.whl → 1.0.0rc14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

@@ -7,7 +7,6 @@ import re
7
7
  import copy
8
8
  import shutil as sh
9
9
  from pathlib import Path
10
- import warnings
11
10
  from dataclasses import field, dataclass
12
11
 
13
12
  import appdirs
@@ -15,10 +14,6 @@ from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
15
14
  from ataraxis_data_structures import YamlConfig
16
15
  from ataraxis_time.time_helpers import get_timestamp
17
16
 
18
- # Ensures console is enabled when this file is imported
19
- if not console.enabled:
20
- console.enable()
21
-
22
17
 
23
18
  def replace_root_path(path: Path) -> None:
24
19
  """Replaces the path to the local root directory used to store all Sun lab projects with the provided path.
@@ -363,510 +358,510 @@ class ProjectConfiguration(YamlConfig):
363
358
 
364
359
  @dataclass()
365
360
  class RawData:
366
- """Stores the paths to the directories and files that make up the 'raw_data' session directory.
361
+ """Stores the paths to the directories and files that make up the 'raw_data' session-specific directory.
367
362
 
368
363
  The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
369
364
  preprocessing does not alter the data, any data in that folder is considered 'raw'. The raw_data folder is initially
370
365
  created on the VRPC and, after preprocessing, is copied to the BioHPC server and the Synology NAS for long-term
371
366
  storage and further processing.
372
-
373
- Notes:
374
- The overall structure of the raw_data directory remains fixed for the entire lifetime of the data. It is reused
375
- across all destinations.
376
367
  """
377
368
 
378
- raw_data_path: str | Path
369
+ raw_data_path: Path = Path()
379
370
  """Stores the path to the root raw_data directory of the session. This directory stores all raw data during
380
371
  acquisition and preprocessing. Note, preprocessing does not alter raw data, so at any point in time all data inside
381
372
  the folder is considered 'raw'."""
382
- camera_data_path: str | Path = "null"
373
+ camera_data_path: Path = Path()
383
374
  """Stores the path to the directory that contains all camera data acquired during the session. Primarily, this
384
375
  includes .mp4 video files from each recorded camera."""
385
- mesoscope_data_path: str | Path = "null"
376
+ mesoscope_data_path: Path = Path()
386
377
  """Stores the path to the directory that contains all Mesoscope data acquired during the session. Primarily, this
387
378
  includes the mesoscope-acquired .tiff files (brain activity data) and the motion estimation data."""
388
- behavior_data_path: str | Path = "null"
379
+ behavior_data_path: Path = Path()
389
380
  """Stores the path to the directory that contains all behavior data acquired during the session. Primarily, this
390
381
  includes the .npz log files used by data-acquisition libraries to store all acquired data. The data stored in this
391
382
  way includes the camera and mesoscope frame timestamps and the states of Mesoscope-VR components, such as lick
392
383
  sensors, rotary encoders, and other modules."""
393
- zaber_positions_path: str | Path = "null"
384
+ zaber_positions_path: Path = Path()
394
385
  """Stores the path to the zaber_positions.yaml file. This file contains the snapshot of all Zaber motor positions
395
386
  at the end of the session. Zaber motors are used to position the LickPort and the HeadBar manipulators, which is
396
387
  essential for supporting proper brain imaging and animal's running behavior during the session."""
397
- session_descriptor_path: str | Path = "null"
388
+ session_descriptor_path: Path = Path()
398
389
  """Stores the path to the session_descriptor.yaml file. This file is partially filled by the system during runtime
399
390
  and partially by the experimenter after the runtime. It contains session-specific information, such as the specific
400
391
  training parameters, the positions of the Mesoscope objective and the notes made by the experimenter during
401
392
  runtime."""
402
- hardware_configuration_path: str | Path = "null"
393
+ hardware_configuration_path: Path = Path()
403
394
  """Stores the path to the hardware_configuration.yaml file. This file contains the partial snapshot of the
404
395
  calibration parameters used by the Mesoscope-VR system components during runtime. Primarily, this is used during
405
396
  data processing to read the .npz data log files generated during runtime."""
406
- surgery_metadata_path: str | Path = "null"
397
+ surgery_metadata_path: Path = Path()
407
398
  """Stores the path to the surgery_metadata.yaml file. This file contains the most actual information about the
408
399
  surgical intervention(s) performed on the animal prior to the session."""
409
- project_configuration_path: str | Path = "null"
400
+ project_configuration_path: Path = Path()
410
401
  """Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
411
402
  parameters for the session's project."""
412
- session_data_path: str | Path = "null"
403
+ session_data_path: Path = Path()
413
404
  """Stores the path to the session_data.yaml file. This path is used b y the SessionData instance to save itself to
414
405
  disk as a .yaml file. The file contains all paths used during data acquisition and processing on both the VRPC and
415
406
  the BioHPC server."""
416
- experiment_configuration_path: str | Path = "null"
407
+ experiment_configuration_path: Path = Path()
417
408
  """Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
418
409
  experiment runtime configuration used by the session. This file is only created for experiment session. It does not
419
410
  exist for behavior training sessions."""
420
- mesoscope_positions_path: str | Path = "null"
411
+ mesoscope_positions_path: Path = Path()
421
412
  """Stores the path to the mesoscope_positions.yaml file. This file contains the snapshot of the positions used
422
413
  by the Mesoscope at the end of the session. This includes both the physical position of the mesoscope objective and
423
414
  the 'virtual' tip, tilt, and fastZ positions set via ScanImage software. This file is only created for experiment
424
415
  sessions that use the mesoscope, it is omitted for behavior training sessions."""
425
- window_screenshot_path: str | Path = "null"
416
+ window_screenshot_path: Path = Path()
426
417
  """Stores the path to the .png screenshot of the ScanImagePC screen. The screenshot should contain the image of the
427
418
  cranial window and the red-dot alignment windows. This is used to generate a visual snapshot of the cranial window
428
419
  alignment and appearance for each experiment session. This file is only created for experiment sessions that use
429
420
  the mesoscope, it is omitted for behavior training sessions."""
430
- telomere_path: str | Path = "null"
421
+ telomere_path: Path = Path()
431
422
  """Stores the path to the telomere.bin file. This file is created by the data processing pipelines running on the
432
- BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission. This
433
- path is used by the BioHPC server pipeline. The VRPC uses the path stored in the 'destinations' section to check
434
- for the existence of the file when it purges redundant data."""
435
- checksum_path: str | Path = "null"
423
+ BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission."""
424
+ checksum_path: Path = Path()
436
425
  """Stores the path to the ax_checksum.txt file. This file is generated as part of packaging the data for
437
426
  transmission and stores the xxHash-128 checksum of the data. It is used to verify that the transmission did not
438
- damage or otherwise alter the data.."""
427
+ damage or otherwise alter the data."""
439
428
 
440
- def __post_init__(self) -> None:
441
- """This method is automatically called after class instantiation and ensures that all path fields of the class
442
- are converted to Path objects.
443
- """
429
+ def resolve_paths(self, root_directory_path: Path) -> None:
430
+ """Resolves all paths managed by the class instance based on the input root directory path.
444
431
 
445
- # Translates the root raw_data path to the Path object before using it to determine other Path objects.
446
- self.raw_data_path = Path(self.raw_data_path)
447
-
448
- # When the class is instantiated for the first time, resolves all 'null' path placeholders
449
- if self.camera_data_path == "null":
450
- self.camera_data_path = self.raw_data_path.joinpath("camera_data")
451
- if self.mesoscope_data_path == "null":
452
- self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
453
- if self.behavior_data_path == "null":
454
- self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
455
- if self.zaber_positions_path == "null":
456
- self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
457
- if self.session_descriptor_path == "null":
458
- self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
459
- if self.hardware_configuration_path == "null":
460
- self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
461
- if self.surgery_metadata_path == "null":
462
- self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
463
- if self.project_configuration_path == "null":
464
- self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
465
- if self.session_data_path == "null":
466
- self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
467
- if self.experiment_configuration_path == "null":
468
- self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
469
- if self.mesoscope_positions_path == "null":
470
- self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
471
- if self.window_screenshot_path == "null":
472
- self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
473
- if self.telomere_path == "null":
474
- self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
475
- if self.checksum_path == "null":
476
- self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
477
-
478
- def make_string(self) -> None:
479
- """Converts all Path objects stored inside the class to strings.
480
-
481
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
482
- on disk.
432
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
433
+ machine that instantiates the class.
434
+
435
+ Args:
436
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
437
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
438
+ the managed session.
483
439
  """
484
- self.raw_data_path = str(self.raw_data_path)
485
- self.camera_data_path = str(self.camera_data_path)
486
- self.mesoscope_data_path = str(self.mesoscope_data_path)
487
- self.behavior_data_path = str(self.behavior_data_path)
488
- self.zaber_positions_path = str(self.zaber_positions_path)
489
- self.session_descriptor_path = str(self.session_descriptor_path)
490
- self.hardware_configuration_path = str(self.hardware_configuration_path)
491
- self.surgery_metadata_path = str(self.surgery_metadata_path)
492
- self.project_configuration_path = str(self.project_configuration_path)
493
- self.session_data_path = str(self.session_data_path)
494
- self.experiment_configuration_path = str(self.experiment_configuration_path)
495
- self.mesoscope_positions_path = str(self.mesoscope_positions_path)
496
- self.window_screenshot_path = str(self.window_screenshot_path)
497
- self.telomere_path = str(self.telomere_path)
498
- self.checksum_path = str(self.checksum_path)
499
-
500
- def make_dirs(self) -> None:
501
- """Ensures that all major subdirectories and the root raw_data directory exist.
502
-
503
- This method is used by the VRPC to generate the raw_data directory when it creates a new session.
440
+
441
+ # Generates the managed paths
442
+ self.raw_data_path = root_directory_path
443
+ self.camera_data_path = self.raw_data_path.joinpath("camera_data")
444
+ self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
445
+ self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
446
+ self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
447
+ self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
448
+ self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
449
+ self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
450
+ self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
451
+ self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
452
+ self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
453
+ self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
454
+ self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
455
+ self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
456
+ self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
457
+
458
+ def make_directories(self) -> None:
459
+ """Ensures that all major subdirectories and the root directory exist."""
460
+ ensure_directory_exists(self.raw_data_path)
461
+ ensure_directory_exists(self.camera_data_path)
462
+ ensure_directory_exists(self.mesoscope_data_path)
463
+ ensure_directory_exists(self.behavior_data_path)
464
+
465
+
466
+ @dataclass()
467
+ class DeepLabCutData:
468
+ """Stores the paths to the directories and files that make up the 'deeplabcut' project-specific directory.
469
+
470
+ DeepLabCut (DLC) is used to track animal body parts and poses in video data acquired during experiment and training
471
+ sessions. Since DLC is designed to work with projects, rather than single animals or sessions, each Sun lab
472
+ project data hierarchy contains a dedicated 'deeplabcut' directory under the root project directory. The contents of
473
+ that directory are largely managed by the DLC itself. Therefore, each session of a given project refers to and
474
+ uses the same 'deeplabcut' directory.
475
+ """
476
+
477
+ deeplabcut_path: Path = Path()
478
+ """Stores the path to the project-specific DeepLabCut directory. This folder stores all DeepLabCut data specific to
479
+ a single project, which is reused during the processing of all sessions of the project."""
480
+
481
+ def resolve_paths(self, root_directory_path: Path) -> None:
482
+ """Resolves all paths managed by the class instance based on the input root directory path.
483
+
484
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
485
+ machine that instantiates the class.
486
+
487
+ Args:
488
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
489
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
490
+ the managed session.
504
491
  """
505
- ensure_directory_exists(Path(self.raw_data_path))
506
- ensure_directory_exists(Path(self.camera_data_path))
507
- ensure_directory_exists(Path(self.mesoscope_data_path))
508
- ensure_directory_exists(Path(self.behavior_data_path))
509
492
 
510
- def switch_root(self, new_root: Path) -> None:
511
- """Changes the root of the managed raw_data directory to the provided root path.
493
+ # Generates the managed paths
494
+ self.deeplabcut_path = root_directory_path
495
+
496
+ def make_directories(self) -> None:
497
+ """Ensures that all major subdirectories and the root directory exist."""
498
+ ensure_directory_exists(self.deeplabcut_path)
499
+
500
+
501
+ @dataclass()
502
+ class ConfigurationData:
503
+ """Stores the paths to the directories and files that make up the 'configuration' project-specific directory.
504
+
505
+ The configuration directory contains various configuration files and settings used by data acquisition,
506
+ preprocessing, and processing pipelines in the lab. Generally, all configuration settings are defined once for each
507
+ project and are reused for every session within the project. Therefore, this directory is created under each main
508
+ project directory.
509
+
510
+ Notes:
511
+ Some attribute names inside this section match the names in the RawData section. This is intentional, as some
512
+ configuration files are copied into the raw_data session directories to allow reinstating the session data
513
+ hierarchy across machines.
514
+ """
515
+
516
+ configuration_path: Path = Path()
517
+ """Stores the path to the project-specific configuration directory. This directory is used by all animals
518
+ and sessions of the project to store all pan-project configuration files. The configuration data is reused by all
519
+ sessions in the project."""
520
+ experiment_configuration_path: Path = Path()
521
+ """Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
522
+ experiment runtime configuration used by the session. This file is only created for experiment session. It does not
523
+ exist for behavior training sessions."""
524
+ project_configuration_path: Path = Path()
525
+ """Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
526
+ parameters for the session's project."""
527
+ suite2p_configuration_path: Path = Path()
528
+ """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
529
+ the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
530
+ registration pipeline, which is applied to each session that generates brain activity data."""
531
+ multiday_configuration_path: Path = Path()
532
+ """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
533
+ on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
534
+ sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
512
535
 
513
- This service method is used by the SessionData class to convert all paths in this class to be relative to the
514
- new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
515
- (another root). This method is only implemented for subclasses intended to be used both locally and on the
516
- BioHPC server.
536
+ def resolve_paths(self, root_directory_path: Path, experiment_name: str | None = None) -> None:
537
+ """Resolves all paths managed by the class instance based on the input root directory path.
517
538
 
518
- Note:
519
- This is the only class that contains this method. Other classes are re-initialized using the data loaded
520
- from the RawData section, if necessary.
539
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
540
+ machine that instantiates the class.
521
541
 
522
542
  Args:
523
- new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
524
- directory that stores all Sun lab projects on the target machine.
543
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
544
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
545
+ the managed session.
546
+ experiment_name: Optionally specifies the name of the experiment executed as part of the managed session's
547
+ runtime. This is used to correctly configure the path to the specific ExperimentConfiguration data file.
548
+ If the managed session is not an Experiment session, this parameter should be set to None.
525
549
  """
526
- # Gets current root from the raw_data_path.
527
- old_root = Path(self.raw_data_path).parents[3]
528
-
529
- # Updates all paths by replacing old_root with new_root
530
- self.raw_data_path = new_root.joinpath(Path(self.raw_data_path).relative_to(old_root))
531
- self.camera_data_path = new_root.joinpath(Path(self.camera_data_path).relative_to(old_root))
532
- self.mesoscope_data_path = new_root.joinpath(Path(self.mesoscope_data_path).relative_to(old_root))
533
- self.behavior_data_path = new_root.joinpath(Path(self.behavior_data_path).relative_to(old_root))
534
- self.zaber_positions_path = new_root.joinpath(Path(self.zaber_positions_path).relative_to(old_root))
535
- self.session_descriptor_path = new_root.joinpath(Path(self.session_descriptor_path).relative_to(old_root))
536
- self.hardware_configuration_path = new_root.joinpath(
537
- Path(self.hardware_configuration_path).relative_to(old_root)
538
- )
539
- self.surgery_metadata_path = new_root.joinpath(Path(self.surgery_metadata_path).relative_to(old_root))
540
- self.project_configuration_path = new_root.joinpath(Path(self.project_configuration_path).relative_to(old_root))
541
- self.session_data_path = new_root.joinpath(Path(self.session_data_path).relative_to(old_root))
542
- self.experiment_configuration_path = new_root.joinpath(
543
- Path(self.experiment_configuration_path).relative_to(old_root)
544
- )
545
- self.mesoscope_positions_path = new_root.joinpath(Path(self.mesoscope_positions_path).relative_to(old_root))
546
- self.window_screenshot_path = new_root.joinpath(Path(self.window_screenshot_path).relative_to(old_root))
547
- self.telomere_path = new_root.joinpath(Path(self.telomere_path).relative_to(old_root))
548
- self.checksum_path = new_root.joinpath(Path(self.checksum_path).relative_to(old_root))
550
+
551
+ # Generates the managed paths
552
+ self.configuration_path = root_directory_path
553
+ if experiment_name is None:
554
+ self.experiment_configuration_path = self.configuration_path.joinpath("null")
555
+ else:
556
+ self.experiment_configuration_path = self.configuration_path.joinpath(f"{experiment_name}.yaml")
557
+ self.project_configuration_path = self.configuration_path.joinpath("project_configuration.yaml")
558
+ self.suite2p_configuration_path = self.configuration_path.joinpath("suite2p_configuration.yaml")
559
+ self.multiday_configuration_path = self.configuration_path.joinpath("multiday_configuration.yaml")
560
+
561
+ def make_directories(self) -> None:
562
+ """Ensures that all major subdirectories and the root directory exist."""
563
+ ensure_directory_exists(self.configuration_path)
549
564
 
550
565
 
551
566
  @dataclass()
552
567
  class ProcessedData:
553
- """Stores the paths to the directories and files that make up the 'processed_data' session directory.
568
+ """Stores the paths to the directories and files that make up the 'processed_data' session-specific directory.
554
569
 
555
- The processed_data directory stores the data generated by various processing pipelines from the raw data. Processed
556
- data represents an intermediate step between raw data and the dataset used in the data analysis, but is not itself
557
- designed to be analyzed.
570
+ The processed_data directory stores the data generated by various processing pipelines from the raw data (contents
571
+ of the raw_data directory). Processed data represents an intermediate step between raw data and the dataset used in
572
+ the data analysis, but is not itself designed to be analyzed.
558
573
 
559
574
  Notes:
560
575
  The paths from this section are typically used only on the BioHPC server. This is because most data processing
561
576
  in the lab is performed using the processing server's resources. On the server, processed data is stored on
562
- the fast volume, in contrast to raw data, which is stored on the slow volume. However, to support local testing,
563
- the class resolves the paths in this section both locally and globally (on the server).
577
+ the fast (NVME) drive volume, in contrast to raw data, which is stored on the slow (SSD) drive volume.
564
578
 
579
+ When this class is instantiated on a machine other than BioHPC server, for example, to test processing
580
+ pipelines, it uses the same drive as the raw_data folder to create the processed_data folder. This relies on the
581
+ assumption that non-server machines in the lab only use fast NVME drives, so there is no need to separate
582
+ storage and processing volumes.
565
583
  """
566
584
 
567
- processed_data_path: str | Path
585
+ processed_data_path: Path = Path()
568
586
  """Stores the path to the root processed_data directory of the session. This directory stores the processed data
569
587
  as it is generated by various data processing pipelines."""
570
- deeplabcut_path: str | Path
571
- """Stores the path to the root DeepLabCut project directory. Since DeepLabCut adopts a project-based directory
572
- management hierarchy, it is easier to have a single DLC folder shared by all animals and sessions of a given
573
- project. This root folder is typically stored under the main project directory on the fast BioHPC server volume."""
574
- configuration_path: str | Path
575
- """Stores the path to the project's processed data configuration directory. This directory is used by all animals
576
- and sessions of the project to store data processing configuration files. Since processing configuration is
577
- typically reused for all data in the project, this allows updating a single file to control the processing for all
578
- data. This directory is typically stored on the fast BioHPC server volume."""
579
- camera_data_path: str | Path = "null"
580
- """Stores the output of the DeepLabCut pose estimation pipeline."""
581
- mesoscope_data_path: str | Path = "null"
582
- """Stores the output of the suite2p cell registration pipeline."""
583
- behavior_data_path: str | Path = "null"
584
- """Stores the output of the Sun lab behavior data extraction pipeline."""
585
- suite2p_configuration_path: str | Path = "null"
586
- """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
587
- the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
588
- registration pipeline, which is applied to each session that generates brain activity data."""
589
- processing_tracker_path: str | Path = "null"
590
- """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
591
- the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
592
- session and the status (success / failure) of each processing step.
588
+ camera_data_path: Path = Path()
589
+ """Stores the path to the directory that contains video tracking data generated by our DeepLabCut-based video
590
+ processing pipelines."""
591
+ mesoscope_data_path: Path = Path()
592
+ """Stores path to the directory that contains processed brain activity (cell) data generated by our suite2p-based
593
+ photometry processing pipelines (single day and multi day)."""
594
+ behavior_data_path: Path = Path()
595
+ """Stores the path to the directory that contains the non-video behavior and system runtime data extracted from
596
+ .npz log files by our in-house log parsing pipeline."""
597
+ job_logs_path: Path = Path()
598
+ """Stores the path to the directory that stores the standard output and standard error data collected during
599
+ server-side data processing pipeline runtimes. Since we use SLURM job manager to execute multiple compute jobs on
600
+ the BioHPC server, all information sent to the terminal during runtime is redirected to text files stored in this
601
+ directory."""
602
+ processing_tracker_path: Path = Path()
603
+ """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory.
604
+ This file is used to track which processing pipelines need to be applied to the target session and the status
605
+ (success / failure) each of these pipelines. Primarily, this is used to optimize data processing to avoid redundant
606
+ operations.
593
607
  """
594
- multiday_configuration_path: str | Path = "null"
595
- """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
596
- on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
597
- sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
598
608
 
599
- def __post_init__(self) -> None:
600
- """This method is automatically called after class instantiation and ensures that all path fields of the class
601
- are converted to Path objects.
602
- """
609
+ def resolve_paths(self, root_directory_path: Path) -> None:
610
+ """Resolves all paths managed by the class instance based on the input root directory path.
603
611
 
604
- # Ensures all objects that have to be provided externally are transformed to Path objects. Unlike 'null' fields,
605
- # these paths cannot be resolved automatically
606
- self.processed_data_path = Path(self.processed_data_path)
607
- self.deeplabcut_path = Path(self.deeplabcut_path)
608
- self.configuration_path = Path(self.configuration_path)
609
-
610
- # When the class is instantiated for the first time, resolves all 'null' path placeholders
611
- if self.camera_data_path == "null":
612
- self.camera_data_path = self.processed_data_path.joinpath("camera_data")
613
- if self.mesoscope_data_path == "null":
614
- self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
615
- if self.behavior_data_path == "null":
616
- self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
617
- if self.suite2p_configuration_path == "null":
618
- self.suite2p_configuration_path = self.processed_data_path.joinpath("suite2p_configuration.yaml")
619
- if self.processing_tracker_path == "null":
620
- self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
621
- if self.multiday_configuration_path == "null":
622
- self.multiday_configuration_path = self.processed_data_path.joinpath("multiday_configuration.yaml")
623
-
624
- def make_string(self) -> None:
625
- """Converts all Path objects stored inside the class to strings.
626
-
627
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
628
- on disk.
629
- """
630
- self.processed_data_path = str(self.processed_data_path)
631
- self.deeplabcut_path = str(self.deeplabcut_path)
632
- self.configuration_path = str(self.configuration_path)
633
- self.camera_data_path = str(self.camera_data_path)
634
- self.mesoscope_data_path = str(self.mesoscope_data_path)
635
- self.behavior_data_path = str(self.behavior_data_path)
636
- self.suite2p_configuration_path = str(self.suite2p_configuration_path)
637
- self.processing_tracker_path = str(self.processing_tracker_path)
638
- self.multiday_configuration_path = str(self.multiday_configuration_path)
639
-
640
- def make_dirs(self) -> None:
641
- """Ensures that all major subdirectories of the processed_data directory exist.
642
-
643
- This method is used by the BioHPC server to generate the processed_data directory as part of the sl-forgery
644
- library runtime.
612
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
613
+ machine that instantiates the class.
614
+
615
+ Args:
616
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
617
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
618
+ the managed session.
645
619
  """
646
- ensure_directory_exists(Path(self.processed_data_path))
647
- ensure_directory_exists(Path(self.deeplabcut_path))
648
- ensure_directory_exists(Path(self.configuration_path))
649
- ensure_directory_exists(Path(self.camera_data_path))
650
- ensure_directory_exists(Path(self.mesoscope_data_path))
651
- ensure_directory_exists(Path(self.behavior_data_path))
620
+ # Generates the managed paths
621
+ self.processed_data_path = root_directory_path
622
+ self.camera_data_path = self.processed_data_path.joinpath("camera_data")
623
+ self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
624
+ self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
625
+ self.job_logs_path = self.processed_data_path.joinpath("job_logs")
626
+ self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
627
+
628
+ def make_directories(self) -> None:
629
+ """Ensures that all major subdirectories and the root directory exist."""
630
+
631
+ ensure_directory_exists(self.processed_data_path)
632
+ ensure_directory_exists(self.camera_data_path)
633
+ ensure_directory_exists(self.mesoscope_data_path)
634
+ ensure_directory_exists(self.behavior_data_path)
635
+ ensure_directory_exists(self.job_logs_path)
652
636
 
653
637
 
654
638
  @dataclass()
655
- class PersistentData:
656
- """Stores the paths to the directories and files that make up the 'persistent_data' directories of the VRPC and
657
- the ScanImagePC.
639
+ class VRPCPersistentData:
640
+ """Stores the paths to the directories and files that make up the 'persistent_data' directory on the VRPC.
658
641
 
659
- Persistent data directories are used to keep certain files on the VRPC and the ScanImagePC. Typically, this data
660
- is reused during the following sessions. For example, a copy of Zaber motor positions is persisted on the VRPC for
661
- each animal after every session to support automatically restoring Zaber motors to the positions used during the
662
- previous session.
642
+ Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
643
+ are purposefully designed for specific PCs that participate in data acquisition. This section manages the
644
+ animal-specific persistent_data directory stored on the VRPC.
663
645
 
664
- Notes:
665
- Persistent data includes the project and experiment configuration data. Some persistent data is overwritten
666
- after each session, other data is generated once and kept through the animal's lifetime. Primarily, this data is
667
- only used internally by the sl-experiment or sl-forgery libraries and is not intended for end-users.
646
+ VRPC persistent data directory is used to preserve configuration data, such as the positions of Zaber motors and
647
+ Meososcope objective, so that they can be reused across sessions of the same animals. The data in this directory
648
+ is read at the beginning of each session and replaced at the end of each session.
668
649
  """
669
650
 
670
- scanimage_persistent_path: str | Path
671
- """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
672
- belongs, relative to the ScanImagePC root. This directory is primarily used to store reference motion registration
673
- data acquired during the first experiment session for each animal."""
674
- vrpc_persistent_path: str | Path
651
+ persistent_data_path: Path = Path()
675
652
  """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
676
- belongs, relative to the VRPC root. This directory is primarily used to back up the Zaber and Mesoscope positions,
677
- so that they can be restored between sessions of the same animals."""
678
- zaber_positions_path: str | Path = "null"
653
+ belongs, relative to the VRPC root. This directory is exclusively used on the VRPC."""
654
+ zaber_positions_path: Path = Path()
679
655
  """Stores the path to the Zaber motor positions snapshot generated at the end of the previous session runtime. This
680
656
  is used to automatically restore all Zaber motors to the same position across all sessions."""
681
- mesoscope_positions_path: str | Path = "null"
657
+ mesoscope_positions_path: Path = Path()
682
658
  """Stores the path to the Mesoscope positions snapshot generated at the end of the previous session runtime. This
683
659
  is used to help the user to (manually) restore the Mesoscope to the same position across all sessions."""
684
- motion_estimator_path: str | Path = "null"
685
- """Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
686
- file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
687
660
 
688
- def __post_init__(self) -> None:
689
- """This method is automatically called after class instantiation and ensures that all path fields of the class
690
- are converted to Path objects.
661
+ def resolve_paths(self, root_directory_path: Path) -> None:
662
+ """Resolves all paths managed by the class instance based on the input root directory path.
663
+
664
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
665
+ machine that instantiates the class.
666
+
667
+ Args:
668
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
669
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
670
+ the managed session.
691
671
  """
692
672
 
693
- # Converts 'anchor' paths that have to be provided at class instantiation
694
- self.scanimage_persistent_path = Path(self.scanimage_persistent_path)
695
- self.vrpc_persistent_path = Path(self.vrpc_persistent_path)
673
+ # Generates the managed paths
674
+ self.persistent_data_path = root_directory_path
675
+ self.zaber_positions_path = self.persistent_data_path.joinpath("zaber_positions.yaml")
676
+ self.mesoscope_positions_path = self.persistent_data_path.joinpath("mesoscope_positions.yaml")
677
+
678
+ def make_directories(self) -> None:
679
+ """Ensures that all major subdirectories and the root directory exist."""
680
+
681
+ ensure_directory_exists(self.persistent_data_path)
682
+
696
683
 
697
- # Replaces 'null' defaults with automatically resolved paths
698
- if self.zaber_positions_path == "null":
699
- self.zaber_positions_path = self.vrpc_persistent_path.joinpath("zaber_positions.yaml")
700
- if self.mesoscope_positions_path == "null":
701
- self.mesoscope_positions_path = self.vrpc_persistent_path.joinpath("mesoscope_positions.yaml")
702
- if self.motion_estimator_path == "null":
703
- self.motion_estimator_path = self.scanimage_persistent_path.joinpath("MotionEstimator.me")
684
+ @dataclass()
685
+ class ScanImagePCPersistentData:
686
+ """Stores the paths to the directories and files that make up the 'persistent_data' directory on the ScanImagePC.
687
+
688
+ Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
689
+ are purposefully designed for specific PCs that participate in data acquisition. This section manages the
690
+ animal-specific persistent_data directory stored on the ScanImagePC (Mesoscope PC).
691
+
692
+ ScanImagePC persistent data directory is used to preserve the motion estimation snapshot, generated during the first
693
+ experiment session. This is necessary to align the brain recording field of view across sessions. In turn, this
694
+ is used to carry out 'online' motion and z-drift correction, improving the accuracy of across-day (multi-day)
695
+ cell tracking.
696
+ """
697
+
698
+ persistent_data_path: Path = Path()
699
+ """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
700
+ belongs, relative to the ScanImagePC root. This directory is exclusively used on the ScanImagePC (Mesoscope PC)."""
701
+ motion_estimator_path: Path = Path()
702
+ """Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
703
+ file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
704
704
 
705
- def make_string(self) -> None:
706
- """Converts all Path objects stored inside the class to strings.
705
+ def resolve_paths(self, root_directory_path: Path) -> None:
706
+ """Resolves all paths managed by the class instance based on the input root directory path.
707
707
 
708
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
709
- on disk.
708
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
709
+ machine that instantiates the class.
710
+
711
+ Args:
712
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
713
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
714
+ the managed session.
710
715
  """
711
- self.scanimage_persistent_path = str(self.scanimage_persistent_path)
712
- self.vrpc_persistent_path = str(self.vrpc_persistent_path)
713
- self.zaber_positions_path = str(self.zaber_positions_path)
714
- self.mesoscope_positions_path = str(self.mesoscope_positions_path)
715
- self.motion_estimator_path = str(self.motion_estimator_path)
716
716
 
717
- def make_dirs(self) -> None:
718
- """Ensures that the VRPC and the ScanImagePC persistent_data directories exist."""
717
+ # Generates the managed paths
718
+ self.persistent_data_path = root_directory_path
719
+ self.motion_estimator_path = self.persistent_data_path.joinpath("MotionEstimator.me")
719
720
 
720
- # We need to call ensure_directory_exists one for each unique directory tree
721
- ensure_directory_exists(Path(self.vrpc_persistent_path)) # vrpc_root/project/animal/persistent_data
722
- ensure_directory_exists(Path(self.scanimage_persistent_path)) # scanimagepc_root/project/animal/persistent_data
721
+ def make_directories(self) -> None:
722
+ """Ensures that all major subdirectories and the root directory exist."""
723
+
724
+ ensure_directory_exists(self.persistent_data_path)
723
725
 
724
726
 
725
727
  @dataclass()
726
728
  class MesoscopeData:
727
- """Stores the paths to the directories used by the ScanImagePC to save mesoscope-generated data during session
728
- runtime.
729
+ """Stores the paths to the directories and files that make up the 'meso_data' directory on the ScanImagePC.
730
+
731
+ The meso_data directory is the root directory where all mesoscope-generated data is stored on the ScanImagePC. The
732
+ path to this directory should be given relative to the VRPC root and be mounted to the VRPC filesystem via the
733
+ SMB or equivalent protocol.
729
734
 
730
- The ScanImagePC is largely isolated from the VRPC during runtime. For the VRPC to pull the data acquired by the
731
- ScanImagePC, it has to use the predefined directory structure to save the data. This class stores the predefined
732
- path to various directories where ScanImagePC is expected to save the data and store it after acquisition.sers.
735
+ During runtime, the ScanImagePC should organize all collected data under this root directory. During preprocessing,
736
+ the VRPC uses SMB to access the data in this directory and merge it into the 'raw_data' session directory. The paths
737
+ in this section, therefore, are specific to the VRPC and are not used on other PCs.
733
738
  """
734
739
 
735
- root_data_path: str | Path
740
+ meso_data_path: Path = Path()
736
741
  """Stores the path to the root ScanImagePC data directory, mounted to the VRPC filesystem via the SMB or equivalent
737
- protocol. This path is used during experiment session runtimes to discover the cranial window screenshots
738
- taken by the user before starting the experiment."""
739
- session_specific_path: str | Path
742
+ protocol. All mesoscope-generated data is stored under this root directory before it is merged into the VRPC-managed
743
+ raw_data directory of each session."""
744
+ mesoscope_data_path: Path = Path()
745
+ """Stores the path to the 'default' mesoscope_data directory. All experiment sessions across all animals and
746
+ projects use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
747
+ software. This simplifies ScanImagePC configuration process during runtime, as all data is always saved in the same
748
+ directory. During preprocessing, the data is moved from the default directory first into a session-specific
749
+ ScanImagePC directory and then into the VRPC raw_data session directory."""
750
+ session_specific_path: Path = Path()
740
751
  """Stores the path to the session-specific data directory. This directory is generated at the end of each experiment
741
- runtime to prepare mesoscope data for further processing and to reset the 'shared' folder for the next session's
742
- runtime."""
743
- mesoscope_data_path: str | Path = "null"
744
- """Stores the path to the 'general' mesoscope_data directory. All experiment sessions (across all animals and
745
- projects) use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
746
- software. This simplifies ScanImagePC configuration process during runtime. The data is moved into a
747
- session-specific directory during preprocessing."""
748
- ubiquitin_path: str | Path = "null"
752
+ runtime to prepare mesoscope data for being moved to the VRPC-managed raw_data directory and to reset the 'default'
753
+ mesoscope_data directory for the next session's runtime."""
754
+ ubiquitin_path: Path = Path()
749
755
  """Stores the path to the 'ubiquitin.bin' file. This file is automatically generated inside the session-specific
750
- data folder after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
751
- removal, the VRPC searches for directories marked with ubiquitin.bin, so that they can be safely removed from the
756
+ data directory after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
757
+ removal step of preprocessing, the VRPC searches for directories marked with ubiquitin.bin and deletes them from the
752
758
  ScanImagePC filesystem."""
753
759
 
754
- def __post_init__(self) -> None:
755
- """This method is automatically called after class instantiation and ensures that all path fields of the class
756
- are converted to Path objects.
757
- """
758
- # Converts the paths that have to be provided at class instantiation to the Path objects.
759
- self.root_data_path = Path(self.root_data_path)
760
- self.session_specific_path = Path(self.session_specific_path)
761
-
762
- # Resolves the 'null' placeholders using the 'anchor' paths above
763
- if self.mesoscope_data_path == "null":
764
- self.mesoscope_data_path = self.root_data_path.joinpath("mesoscope_data")
765
- if self.ubiquitin_path == "null":
766
- self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
760
+ def resolve_paths(self, root_mesoscope_path: Path, session_name: str) -> None:
761
+ """Resolves all paths managed by the class instance based on the input root directory path.
767
762
 
768
- def make_string(self) -> None:
769
- """Converts all Path objects stored inside the class to strings.
763
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
764
+ machine that instantiates the class.
770
765
 
771
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
772
- on disk.
766
+ Args:
767
+ root_mesoscope_path: The path to the top-level directory of the ScanImagePC data hierarchy mounted to the
768
+ VRPC via the SMB or equivalent protocol.
769
+ session_name: The name of the session for which this subclass is initialized.
773
770
  """
774
- self.root_data_path = str(self.root_data_path)
775
- self.session_specific_path = str(self.session_specific_path)
776
- self.mesoscope_data_path = str(self.mesoscope_data_path)
777
- self.ubiquitin_path = str(self.ubiquitin_path)
778
771
 
779
- def make_dirs(self) -> None:
780
- """Ensures that the ScanImagePC data acquisition directories exist."""
772
+ # Generates the managed paths
773
+ self.meso_data_path = root_mesoscope_path
774
+ self.session_specific_path = self.meso_data_path.joinpath(session_name)
775
+ self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
776
+ self.mesoscope_data_path = self.meso_data_path.joinpath("mesoscope_data")
781
777
 
782
- # Does not create the session-specific directory. This is on purpose, as the session-specific directory
783
- # is generated during runtime by renaming the 'general' mesoscope_data directory. The 'general' directory is
784
- # then recreated from scratch. This ensures that the general directory is empty (ready for the next session)
785
- # with minimal I/O overhead.
786
- ensure_directory_exists(Path(self.mesoscope_data_path))
778
+ def make_directories(self) -> None:
779
+ """Ensures that all major subdirectories and the root directory exist."""
780
+
781
+ ensure_directory_exists(self.meso_data_path)
787
782
 
788
783
 
789
784
  @dataclass()
790
- class Destinations:
791
- """Stores the paths to the VRPC filesystem-mounted Synology NAS and BioHPC server directories.
785
+ class VRPCDestinations:
786
+ """Stores the paths to the VRPC filesystem-mounted directories of the Synology NAS and BioHPC server.
787
+
788
+ The paths from this section are primarily used to transfer preprocessed data to the long-term storage destinations.
789
+ Additionally, they allow VRPC to interface with the configuration directory of the BioHPC server to start data
790
+ processing jobs and to read the data from the processed_data directory to remove redundant data from the VRPC
791
+ filesystem.
792
792
 
793
- These directories are used during data preprocessing to transfer the preprocessed raw_data directory from the
794
- VRPC to the long-term storage destinations.
793
+ Overall, this section is intended solely for the VRPC and should not be used on other PCs.
795
794
  """
796
795
 
797
- nas_raw_data_path: str | Path
796
+ nas_raw_data_path: Path = Path()
798
797
  """Stores the path to the session's raw_data directory on the Synology NAS, which is mounted to the VRPC via the
799
798
  SMB or equivalent protocol."""
800
- server_raw_data_path: str | Path
799
+ server_raw_data_path: Path = Path()
801
800
  """Stores the path to the session's raw_data directory on the BioHPC server, which is mounted to the VRPC via the
802
801
  SMB or equivalent protocol."""
803
- server_processed_data_path: str | Path
802
+ server_processed_data_path: Path = Path()
804
803
  """Stores the path to the session's processed_data directory on the BioHPC server, which is mounted to the VRPC via
805
804
  the SMB or equivalent protocol."""
806
- server_configuration_path: str | Path
805
+ server_configuration_path: Path = Path()
807
806
  """Stores the path to the project-specific 'configuration' directory on the BioHPC server, which is mounted to the
808
807
  VRPC via the SMB or equivalent protocol."""
809
- server_telomere_path: str | Path = "null"
810
- """Stores the path to the session's telomere.bin marker. This marker is generated as part of BioHPC-side data
811
- processing pipeline to notify he VRPC that the server received the data intact. it is used to determine which
812
- data can be safely removed from the VRPC filesystem."""
813
- server_suite2p_configuration_path: str | Path = "null"
808
+ telomere_path: Path = Path()
809
+ """Stores the path to the session's telomere.bin marker. This marker is generated as part of data processing on the
810
+ BioHPC server to notify the VRPC that the server received preprocessed data intact. The presence of this marker is
811
+ used by the VRPC to determine which locally stored raw_data is safe to delete from the filesystem."""
812
+ suite2p_configuration_path: Path = Path()
814
813
  """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
815
- the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
816
- registration pipeline, which is applied to each session that generates brain activity data."""
817
- server_processing_tracker_path: str | Path = "null"
814
+ the BioHPC server. This configuration file specifies the parameters for the 'single day' sl-suite2p registration
815
+ pipeline, which is applied to each session that generates brain activity data."""
816
+ processing_tracker_path: Path = Path()
818
817
  """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
819
- the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
820
- session and the status (success / failure) of each processing step.
818
+ the BioHPC server. This file tracks which processing pipelines need to be applied the target session and the status
819
+ (success / failure) of each applied pipeline.
821
820
  """
822
- server_multiday_configuration_path: str | Path = "null"
821
+ multiday_configuration_path: Path = Path()
823
822
  """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
824
- on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
825
- sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
823
+ on the BioHPC server. This configuration file specifies the parameters for the 'multiday' sl-suite2p registration
824
+ pipeline used to track brain cells across multiple sessions."""
826
825
 
827
- def __post_init__(self) -> None:
828
- """This method is automatically called after class instantiation and ensures that all path fields of the class
829
- are converted to Path objects.
830
- """
831
- self.nas_raw_data_path = Path(self.nas_raw_data_path)
832
- self.server_raw_data_path = Path(self.server_raw_data_path)
833
- self.server_configuration_path = Path(self.server_configuration_path)
834
- self.server_processed_data_path = Path(self.server_processed_data_path)
835
-
836
- if self.server_telomere_path == "null":
837
- self.server_telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
838
- if self.server_suite2p_configuration_path == "null":
839
- self.server_suite2p_configuration_path = self.server_configuration_path.joinpath(
840
- "suite2p_configuration.yaml"
841
- )
842
- if self.server_processing_tracker_path == "null":
843
- self.server_processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
844
- if self.server_multiday_configuration_path == "null":
845
- self.server_multiday_configuration_path = self.server_configuration_path.joinpath(
846
- "multiday_configuration.yaml"
847
- )
826
+ def resolve_paths(
827
+ self,
828
+ nas_raw_data_path: Path,
829
+ server_raw_data_path: Path,
830
+ server_processed_data_path: Path,
831
+ server_configuration_path: Path,
832
+ ) -> None:
833
+ """Resolves all paths managed by the class instance based on the input root directory paths.
848
834
 
849
- def make_string(self) -> None:
850
- """Converts all Path objects stored inside the class to strings.
835
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
836
+ machine that instantiates the class.
851
837
 
852
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
853
- on disk.
838
+ Args:
839
+ nas_raw_data_path: The path to the session's raw_data directory on the Synology NAS, relative to the VRPC
840
+ filesystem root.
841
+ server_raw_data_path: The path to the session's raw_data directory on the BioHPC server, relative to the
842
+ VRPC filesystem root.
843
+ server_processed_data_path: The path to the session's processed_data directory on the BioHPC server,
844
+ relative to the VRPC filesystem root.
845
+ server_configuration_path: The path to the project-specific 'configuration' directory on the BioHPC server,
846
+ relative to the VRPC filesystem root.
854
847
  """
855
- self.nas_raw_data_path = str(self.nas_raw_data_path)
856
- self.server_raw_data_path = str(self.server_raw_data_path)
857
- self.server_telomere_path = str(self.server_telomere_path)
858
- self.server_configuration_path = str(self.server_configuration_path)
859
- self.server_processed_data_path = str(self.server_processed_data_path)
860
- self.server_suite2p_configuration_path = str(self.server_suite2p_configuration_path)
861
- self.server_processing_tracker_path = str(self.server_processing_tracker_path)
862
- self.server_multiday_configuration_path = str(self.server_multiday_configuration_path)
863
-
864
- def make_dirs(self) -> None:
865
- """Ensures that all destination directories exist."""
866
- ensure_directory_exists(Path(self.nas_raw_data_path))
867
- ensure_directory_exists(Path(self.server_raw_data_path))
868
- ensure_directory_exists(Path(self.server_configuration_path))
869
- ensure_directory_exists(Path(self.server_processed_data_path))
848
+
849
+ # Generates the managed paths
850
+ self.nas_raw_data_path = nas_raw_data_path
851
+ self.server_raw_data_path = server_raw_data_path
852
+ self.server_processed_data_path = server_processed_data_path
853
+ self.server_configuration_path = server_configuration_path
854
+ self.telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
855
+ self.suite2p_configuration_path = self.server_configuration_path.joinpath("suite2p_configuration.yaml")
856
+ self.processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
857
+ self.multiday_configuration_path = self.server_configuration_path.joinpath("multiday_configuration.yaml")
858
+
859
+ def make_directories(self) -> None:
860
+ """Ensures that all major subdirectories and the root directory exist."""
861
+ ensure_directory_exists(self.nas_raw_data_path)
862
+ ensure_directory_exists(self.server_raw_data_path)
863
+ ensure_directory_exists(self.server_configuration_path)
864
+ ensure_directory_exists(self.server_processed_data_path)
870
865
 
871
866
 
872
867
  @dataclass
@@ -912,32 +907,30 @@ class SessionData(YamlConfig):
912
907
  field is not None (null), it communicates the specific experiment configuration used by the session. During runtime,
913
908
  the name stored here is used to load the specific experiment configuration data stored in a .yaml file with the
914
909
  same name. If the session is not an experiment session, this field is ignored."""
915
- raw_data: RawData
916
- """This section stores the paths to various directories and files that make up the raw_data subfolder. This
917
- subfolder stores all data acquired during training or experiment runtimes before and after preprocessing. Note, the
918
- preprocessing does not change the raw data in any way other than lossless compression and minor format
919
- reorganization. Therefore, the data is considered 'raw' both before and after preprocessing."""
920
- processed_data: ProcessedData
921
- """This section stores the paths to various directories used to store processed session data. Processed data is
922
- generated from raw data by running various processing pipelines, such as suite2p, DeepLabCut and Sun lab's behavior
923
- parsing pipelines. Typically, all data is processed on the BioHPC server and may be stored on a filesystem volume
924
- different from the one that stores the raw data."""
925
- persistent_data: PersistentData
926
- """This section stores the paths to various files and directories that are held back on the VRPC and ScanImagePC
927
- after the session data is transferred to long-term storage destinations as part of preprocessing. Typically, this
928
- data is reused during the acquisition of future runtime session data. This section is not used during data
929
- processing."""
930
- mesoscope_data: MesoscopeData
931
- """This section stores the paths to various directories used by the ScanImagePC when acquiring mesoscope-related
932
- data. During runtime, the VRPC (behavior data and experiment control) and the ScanImagePC (brain activity data and
933
- Mesoscope control) operate mostly independently of each-other. During preprocessing, the VRPC pulls the data from
934
- the ScanImagePC, using the paths in this section to find the data to be transferred. This section is not used
935
- during data processing."""
936
- destinations: Destinations
937
- """This section stores the paths to the destination directories on the BioHPC server and Synology NAS, to which the
938
- data is copied as part of preprocessing for long-term storage and further processing. Both of these directories
939
- should be mapped (mounted) to the VRPC's filesystem via the SMB or equivalent protocol. This section is not used
940
- during data processing."""
910
+ raw_data: RawData = field(default_factory=lambda: RawData())
911
+ """Stores the paths to all subfolders and files found under the /project/animal/session/raw_data directory of any
912
+ PC used to work with Sun lab data."""
913
+ processed_data: ProcessedData = field(default_factory=lambda: ProcessedData())
914
+ """Stores the paths to all subfolders and files found under the /project/animal/session/processed_data directory of
915
+ any PC used to work with Sun lab data."""
916
+ deeplabcut_data: DeepLabCutData = field(default_factory=lambda: DeepLabCutData())
917
+ """Stores the paths to all subfolders and files found under the /project/deeplabcut directory of any PC used to
918
+ work with Sun lab data."""
919
+ configuration_data: ConfigurationData = field(default_factory=lambda: ConfigurationData())
920
+ """Stores the paths to all subfolders and files found under the /project/configuration directory of any PC used to
921
+ work with Sun lab data."""
922
+ vrpc_persistent_data: VRPCPersistentData = field(default_factory=lambda: VRPCPersistentData())
923
+ """Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
924
+ the VRPC used in the Sun lab to acquire behavior data."""
925
+ scanimagepc_persistent_data: ScanImagePCPersistentData = field(default_factory=lambda: ScanImagePCPersistentData())
926
+ """Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
927
+ the ScanImagePC used in the Sun lab to acquire brain activity data."""
928
+ mesoscope_data: MesoscopeData = field(default_factory=lambda: MesoscopeData())
929
+ """Stores the paths to all subfolders and files found under the /meso_data (root mesoscope data) directory of
930
+ the ScanImagePC used in the Sun lab to acquire brain activity data."""
931
+ destinations: VRPCDestinations = field(default_factory=lambda: VRPCDestinations())
932
+ """Stores the paths to all subfolders and files under various VRPC-filesystem-mounted directories of other machines
933
+ used in the Sun lab for long-term data storage."""
941
934
 
942
935
  @classmethod
943
936
  def create(
@@ -986,8 +979,8 @@ class SessionData(YamlConfig):
986
979
  if session_name is None:
987
980
  session_name = str(get_timestamp(time_separator="-"))
988
981
 
989
- # Extracts the root directory paths stored inside the project configuration file. All roots are expected to be
990
- # mounted on the local (VRPC) via SMB or equivalent protocol and be relative to the VRPC root.
982
+ # Extracts the root directory paths stored inside the project configuration file. These roots are then used to
983
+ # initialize this class instance.
991
984
  vrpc_root = Path(project_configuration.local_root_directory)
992
985
  mesoscope_root = Path(project_configuration.local_mesoscope_directory)
993
986
  biohpc_root = Path(project_configuration.local_server_directory)
@@ -997,7 +990,7 @@ class SessionData(YamlConfig):
997
990
  # Extracts the name of the project stored inside the project configuration file.
998
991
  project_name = project_configuration.project_name
999
992
 
1000
- # Constructs the session directory path and generates the directory
993
+ # Constructs the session directory path
1001
994
  session_path = vrpc_root.joinpath(project_name, animal_id, session_name)
1002
995
 
1003
996
  # Handles potential session name conflicts
@@ -1015,40 +1008,51 @@ class SessionData(YamlConfig):
1015
1008
  f"already exists. The newly created session directory uses a '_{counter}' postfix to distinguish "
1016
1009
  f"itself from the already existing session directory."
1017
1010
  )
1018
- warnings.warn(message=message)
1011
+ console.echo(message=message, level=LogLevel.ERROR)
1019
1012
 
1020
1013
  # Generates subclasses stored inside the main class instance based on the data resolved above. Note; most fields
1021
- # of these classes are initialized automatically, using the 'root' path arguments.
1022
- raw_data = RawData(
1023
- raw_data_path=session_path.joinpath("raw_data"),
1024
- )
1025
- raw_data.make_dirs() # Generates the local directory tree
1026
-
1027
- processed_data = ProcessedData(
1028
- processed_data_path=session_path.joinpath("processed_data"),
1029
- deeplabcut_path=vrpc_root.joinpath(project_name, "deeplabcut"),
1030
- configuration_path=vrpc_root.joinpath(project_name, "configuration"),
1014
+ # of these classes are resolved automatically, based on one or more 'root' paths provided to the 'resolve_paths'
1015
+ # method.
1016
+ raw_data = RawData()
1017
+ raw_data.resolve_paths(root_directory_path=session_path.joinpath("raw_data"))
1018
+ raw_data.make_directories() # Generates the local directory tree
1019
+
1020
+ processed_data = ProcessedData()
1021
+ processed_data.resolve_paths(root_directory_path=session_path.joinpath("processed_data"))
1022
+ processed_data.make_directories()
1023
+
1024
+ dlc_data = DeepLabCutData()
1025
+ dlc_data.resolve_paths(root_directory_path=vrpc_root.joinpath(project_name, "deeplabcut"))
1026
+ dlc_data.make_directories()
1027
+
1028
+ configuration_data = ConfigurationData()
1029
+ configuration_data.resolve_paths(
1030
+ root_directory_path=vrpc_root.joinpath(project_name, "configuration"), experiment_name=experiment_name
1031
1031
  )
1032
+ configuration_data.make_directories()
1032
1033
 
1034
+ vrpc_persistent_data = VRPCPersistentData()
1033
1035
  vrpc_persistent_path = vrpc_root.joinpath(project_name, animal_id, "persistent_data")
1036
+ vrpc_persistent_data.resolve_paths(root_directory_path=vrpc_persistent_path)
1037
+ vrpc_persistent_data.make_directories()
1038
+
1039
+ scanimagepc_persistent_data = ScanImagePCPersistentData()
1034
1040
  scanimagepc_persistent_path = mesoscope_root.joinpath(project_name, animal_id, "persistent_data")
1035
- persistent_data = PersistentData(
1036
- vrpc_persistent_path=vrpc_persistent_path, scanimage_persistent_path=scanimagepc_persistent_path
1037
- )
1038
- persistent_data.make_dirs() # Generates all persistent directory trees
1041
+ scanimagepc_persistent_data.resolve_paths(root_directory_path=scanimagepc_persistent_path)
1042
+ scanimagepc_persistent_data.make_directories()
1039
1043
 
1040
- mesoscope_data = MesoscopeData(
1041
- root_data_path=mesoscope_root, session_specific_path=mesoscope_root.joinpath(session_name)
1042
- )
1043
- mesoscope_data.make_dirs() # Generates all Mesoscope directory trees
1044
+ mesoscope_data = MesoscopeData()
1045
+ mesoscope_data.resolve_paths(root_mesoscope_path=mesoscope_root, session_name=session_name)
1046
+ mesoscope_data.make_directories()
1044
1047
 
1045
- destinations = Destinations(
1048
+ destinations = VRPCDestinations()
1049
+ destinations.resolve_paths(
1046
1050
  nas_raw_data_path=nas_root.joinpath(project_name, animal_id, session_name, "raw_data"),
1047
1051
  server_raw_data_path=biohpc_root.joinpath(project_name, animal_id, session_name, "raw_data"),
1048
- server_configuration_path=biohpc_workdir.joinpath(project_name, "configuration"),
1052
+ server_configuration_path=biohpc_root.joinpath(project_name, "configuration"),
1049
1053
  server_processed_data_path=biohpc_workdir.joinpath(project_name, "processed_data"),
1050
1054
  )
1051
- destinations.make_dirs() # Generates all destination directory trees
1055
+ destinations.make_directories()
1052
1056
 
1053
1057
  # Packages the sections generated above into a SessionData instance
1054
1058
  instance = SessionData(
@@ -1057,8 +1061,11 @@ class SessionData(YamlConfig):
1057
1061
  session_name=session_name,
1058
1062
  session_type=session_type,
1059
1063
  raw_data=raw_data,
1064
+ deeplabcut_data=dlc_data,
1065
+ configuration_data=configuration_data,
1060
1066
  processed_data=processed_data,
1061
- persistent_data=persistent_data,
1067
+ vrpc_persistent_data=vrpc_persistent_data,
1068
+ scanimagepc_persistent_data=scanimagepc_persistent_data,
1062
1069
  mesoscope_data=mesoscope_data,
1063
1070
  destinations=destinations,
1064
1071
  experiment_name=experiment_name,
@@ -1072,19 +1079,16 @@ class SessionData(YamlConfig):
1072
1079
  # classes is not exhaustive. More classes are saved as part of the session runtime management class start() and
1073
1080
  # __init__() method runtimes:
1074
1081
 
1075
- # Resolves the path to the project configuration folder
1076
- vrpc_configuration_path = vrpc_root.joinpath(project_name, "configuration")
1077
-
1078
1082
  # Discovers and saves the necessary configuration class instances to the raw_data folder of the managed session:
1079
1083
  # Project Configuration
1080
1084
  sh.copy2(
1081
- src=vrpc_configuration_path.joinpath("project_configuration.yaml"),
1085
+ src=instance.configuration_data.project_configuration_path,
1082
1086
  dst=instance.raw_data.project_configuration_path,
1083
1087
  )
1084
1088
  # Experiment Configuration, if the session type is Experiment.
1085
1089
  if experiment_name is not None:
1086
1090
  sh.copy2(
1087
- src=vrpc_configuration_path.joinpath(f"{experiment_name}.yaml"),
1091
+ src=instance.configuration_data.experiment_configuration_path,
1088
1092
  dst=instance.raw_data.experiment_configuration_path,
1089
1093
  )
1090
1094
 
@@ -1108,9 +1112,10 @@ class SessionData(YamlConfig):
1108
1112
 
1109
1113
  Args:
1110
1114
  session_path: The path to the root directory of an existing session, e.g.: vrpc_root/project/animal/session.
1111
- on_server: Determines whether the method is used to initialize an existing session on the VRPC or the
1112
- BioHPC server. Note, VRPC runtimes use the same 'root' directory to store raw_data and processed_data
1113
- subfolders. BioHPC server runtimes use different volumes (drives) to store these subfolders.
1115
+ on_server: Determines whether the method is used to initialize an existing session on the BioHPC server or
1116
+ a non-server machine. Note, non-server runtimes use the same 'root' directory to store raw_data and
1117
+ processed_data subfolders. BioHPC server runtimes use different volumes (drives) to store these
1118
+ subfolders.
1114
1119
 
1115
1120
  Returns:
1116
1121
  An initialized SessionData instance for the session whose data is stored at the provided path.
@@ -1130,48 +1135,100 @@ class SessionData(YamlConfig):
1130
1135
  )
1131
1136
  console.error(message=message, error=FileNotFoundError)
1132
1137
 
1133
- # Loads class data from .yaml
1138
+ # Loads class data from .yaml file
1134
1139
  instance: SessionData = cls.from_yaml(file_path=session_data_path) # type: ignore
1135
1140
 
1136
1141
  # The method assumes that the 'donor' .yaml file is always stored inside the raw_data directory of the session
1137
1142
  # to be processed. Since the directory itself might have moved (between or even within the same PC) relative to
1138
1143
  # where it was when the SessionData snapshot was generated, reconfigures the paths to all raw_data files using
1139
1144
  # the root from above.
1140
- instance.raw_data.switch_root(new_root=session_path.parents[2])
1145
+ local_root = session_path.parents[2]
1146
+
1147
+ # RAW DATA
1148
+ new_root = local_root.joinpath(instance.project_name, instance.animal_id, instance.session_name, "raw_data")
1149
+ instance.raw_data.resolve_paths(root_directory_path=new_root)
1150
+
1151
+ # Uses the adjusted raw_data section to load the ProjectConfiguration instance. This is used below to resolve
1152
+ # all other SessionData sections, as it stores various required root directories.
1153
+ project_configuration: ProjectConfiguration = ProjectConfiguration.load(
1154
+ project_name=instance.project_name,
1155
+ configuration_path=Path(instance.raw_data.project_configuration_path),
1156
+ )
1157
+
1158
+ # Resolves the new roots for all sections that use the same root as the raw_data directory:
1159
+
1160
+ # CONFIGURATION
1161
+ new_root = local_root.joinpath(instance.project_name, "configuration")
1162
+ instance.configuration_data.resolve_paths(
1163
+ root_directory_path=new_root, experiment_name=instance.experiment_name
1164
+ )
1165
+
1166
+ # DEEPLABCUT
1167
+ new_root = local_root.joinpath(instance.project_name, "deeplabcut")
1168
+ instance.deeplabcut_data.resolve_paths(root_directory_path=new_root)
1169
+
1170
+ # Resolves the roots for all VRPC-specific sections that use the data from the ProjectConfiguration instance:
1171
+
1172
+ # VRPC PERSISTENT DATA
1173
+ new_root = Path(project_configuration.local_root_directory).joinpath(
1174
+ instance.project_name, instance.animal_id, "persistent_data"
1175
+ )
1176
+ instance.vrpc_persistent_data.resolve_paths(root_directory_path=new_root)
1177
+
1178
+ # SCANIMAGEPC PERSISTENT DATA
1179
+ new_root = Path(project_configuration.local_mesoscope_directory).joinpath(
1180
+ instance.project_name, instance.animal_id, "persistent_data"
1181
+ )
1182
+ instance.scanimagepc_persistent_data.resolve_paths(root_directory_path=new_root)
1183
+
1184
+ # MESOSCOPE DATA
1185
+ instance.mesoscope_data.resolve_paths(
1186
+ root_mesoscope_path=Path(project_configuration.local_mesoscope_directory),
1187
+ session_name=instance.session_name,
1188
+ )
1189
+
1190
+ # DESTINATIONS
1191
+ instance.destinations.resolve_paths(
1192
+ nas_raw_data_path=Path(project_configuration.local_nas_directory).joinpath(
1193
+ instance.project_name, instance.animal_id, instance.session_name, "raw_data"
1194
+ ),
1195
+ server_raw_data_path=Path(project_configuration.local_server_directory).joinpath(
1196
+ instance.project_name, instance.animal_id, instance.session_name, "raw_data"
1197
+ ),
1198
+ server_configuration_path=Path(project_configuration.local_server_directory).joinpath(
1199
+ instance.project_name, "configuration"
1200
+ ),
1201
+ server_processed_data_path=Path(project_configuration.local_server_working_directory).joinpath(
1202
+ instance.project_name, instance.animal_id, instance.session_name, "processed_data"
1203
+ ),
1204
+ )
1141
1205
 
1142
1206
  # Resolves the paths to the processed_data directories. The resolution strategy depends on whether the method is
1143
1207
  # called on the VRPC (locally) or the BioHPC server (remotely).
1144
- new_root: Path
1145
1208
  if not on_server:
1146
1209
  # Local runtimes use the same root session directory for both raw_data and processed_data. This stems from
1147
1210
  # the assumption that most local machines in the lab only use NVME (fast) volumes and, therefore, do not
1148
1211
  # need to separate 'storage' and 'working' data directories.
1149
-
1150
- # Regenerates the processed_data paths using the information loaded from the session_data.yaml file and
1151
- # static path definition rules used in the lab.
1152
- new_root = session_path.parents[2] # Reuses the local root for non-server runtimes
1212
+ new_root = local_root # Reuses the local root for non-server runtimes
1153
1213
 
1154
1214
  else:
1155
1215
  # The BioHPC server stores raw_data on slow volume and processed_data on fast (NVME) volume. Therefore, to
1156
1216
  # configure processed_data paths, the method first needs to load the fast volume root path from the
1157
1217
  # project_configuration.yaml file stored in the raw_data folder.
1158
- project_configuration: ProjectConfiguration = ProjectConfiguration.load(
1159
- project_name=instance.project_name,
1160
- configuration_path=Path(instance.raw_data.project_configuration_path),
1161
- )
1162
1218
  new_root = Path(project_configuration.remote_working_directory)
1163
1219
 
1164
1220
  # Regenerates the processed_data path depending on the root resolution above
1165
- instance.processed_data = ProcessedData(
1166
- processed_data_path=new_root.joinpath(
1221
+ instance.processed_data.resolve_paths(
1222
+ root_directory_path=new_root.joinpath(
1167
1223
  instance.project_name, instance.animal_id, instance.session_name, "processed_data"
1168
- ),
1169
- configuration_path=new_root.joinpath(instance.project_name, instance.animal_id, "configuration"),
1170
- deeplabcut_path=new_root.joinpath(instance.project_name, "deeplabcut"),
1224
+ )
1171
1225
  )
1172
1226
 
1173
- # Generates processed_data directories
1174
- instance.processed_data.make_dirs()
1227
+ # Generates data directory hierarchies that may be missing on the local machine
1228
+ instance.raw_data.make_directories()
1229
+ instance.configuration_data.make_directories()
1230
+ instance.deeplabcut_data.make_directories()
1231
+ instance.processed_data.make_directories()
1175
1232
 
1176
1233
  # Returns the initialized SessionData instance to caller
1177
1234
  return instance
@@ -1184,21 +1241,8 @@ class SessionData(YamlConfig):
1184
1241
  create() method runtime.
1185
1242
  """
1186
1243
 
1187
- # Copies instance data to prevent it from being modified by reference when executing the steps below
1188
- original = copy.deepcopy(self)
1189
-
1190
- # Extracts the target file path before it is converted to a string.
1191
- file_path: Path = copy.copy(self.raw_data.session_data_path) # type: ignore
1192
-
1193
- # Converts all Paths objects to strings before dumping the data to YAML.
1194
- original.raw_data.make_string()
1195
- original.processed_data.make_string()
1196
- original.persistent_data.make_string()
1197
- original.mesoscope_data.make_string()
1198
- original.destinations.make_string()
1199
-
1200
1244
  # Saves instance data as a .YAML file
1201
- original.to_yaml(file_path=file_path)
1245
+ self.to_yaml(file_path=self.raw_data.session_data_path)
1202
1246
 
1203
1247
 
1204
1248
  @dataclass()