sl-shared-assets 1.0.0rc12__py3-none-any.whl → 1.0.0rc14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

@@ -7,7 +7,6 @@ import re
7
7
  import copy
8
8
  import shutil as sh
9
9
  from pathlib import Path
10
- import warnings
11
10
  from dataclasses import field, dataclass
12
11
 
13
12
  import appdirs
@@ -15,10 +14,6 @@ from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
15
14
  from ataraxis_data_structures import YamlConfig
16
15
  from ataraxis_time.time_helpers import get_timestamp
17
16
 
18
- # Ensures console is enabled when this file is imported
19
- if not console.enabled:
20
- console.enable()
21
-
22
17
 
23
18
  def replace_root_path(path: Path) -> None:
24
19
  """Replaces the path to the local root directory used to store all Sun lab projects with the provided path.
@@ -104,8 +99,8 @@ class ProjectConfiguration(YamlConfig):
104
99
  this field is configured automatically each time the class is instantiated through any method, so overwriting it
105
100
  manually will not be respected."""
106
101
  local_server_directory: str | Path = Path("/home/cybermouse/server/storage/sun_data")
107
- """The absolute path to the directory where all projects are stored on the BioHPC server. This directory should be
108
- locally accessible (mounted) using a network sharing protocol, such as SMB."""
102
+ """The absolute path to the directory where the raw data portion of all projects is stored on the BioHPC server.
103
+ This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
109
104
  local_nas_directory: str | Path = Path("/home/cybermouse/nas/rawdata")
110
105
  """The absolute path to the directory where all projects are stored on the Synology NAS. This directory should be
111
106
  locally accessible (mounted) using a network sharing protocol, such as SMB."""
@@ -113,6 +108,9 @@ class ProjectConfiguration(YamlConfig):
113
108
  """The absolute path to the root mesoscope (ScanImagePC) directory where all mesoscope-acquired data is aggregated
114
109
  during acquisition runtime. This directory should be locally accessible (mounted) using a network sharing
115
110
  protocol, such as SMB."""
111
+ local_server_working_directory: str | Path = Path("/home/cybermouse/server/workdir/sun_data")
112
+ """The absolute path to the directory where the processed data portion of all projects is stored on the BioHPC
113
+ server. This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
116
114
  remote_storage_directory: str | Path = Path("/storage/sun_data")
117
115
  """The absolute path, relative to the BioHPC server root, to the directory where all projects are stored on the
118
116
  slow (SSD) volume of the server. This path is used when running remote (server-side) jobs and, therefore, has to
@@ -257,6 +255,7 @@ class ProjectConfiguration(YamlConfig):
257
255
  instance.local_mesoscope_directory = Path(instance.local_mesoscope_directory)
258
256
  instance.local_nas_directory = Path(instance.local_nas_directory)
259
257
  instance.local_server_directory = Path(instance.local_server_directory)
258
+ instance.local_server_working_directory = Path(instance.local_server_working_directory)
260
259
  instance.remote_storage_directory = Path(instance.remote_storage_directory)
261
260
  instance.remote_working_directory = Path(instance.remote_working_directory)
262
261
  instance.google_credentials_path = Path(instance.google_credentials_path)
@@ -301,6 +300,7 @@ class ProjectConfiguration(YamlConfig):
301
300
  original.local_mesoscope_directory = str(original.local_mesoscope_directory)
302
301
  original.local_nas_directory = str(original.local_nas_directory)
303
302
  original.local_server_directory = str(original.local_server_directory)
303
+ original.local_server_working_directory = str(original.local_server_working_directory)
304
304
  original.remote_storage_directory = str(original.remote_storage_directory)
305
305
  original.remote_working_directory = str(original.remote_working_directory)
306
306
  original.google_credentials_path = str(original.google_credentials_path)
@@ -358,472 +358,510 @@ class ProjectConfiguration(YamlConfig):
358
358
 
359
359
  @dataclass()
360
360
  class RawData:
361
- """Stores the paths to the directories and files that make up the 'raw_data' session directory.
361
+ """Stores the paths to the directories and files that make up the 'raw_data' session-specific directory.
362
362
 
363
363
  The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
364
364
  preprocessing does not alter the data, any data in that folder is considered 'raw'. The raw_data folder is initially
365
365
  created on the VRPC and, after preprocessing, is copied to the BioHPC server and the Synology NAS for long-term
366
366
  storage and further processing.
367
-
368
- Notes:
369
- The overall structure of the raw_data directory remains fixed for the entire lifetime of the data. It is reused
370
- across all destinations.
371
367
  """
372
368
 
373
- raw_data_path: str | Path
369
+ raw_data_path: Path = Path()
374
370
  """Stores the path to the root raw_data directory of the session. This directory stores all raw data during
375
371
  acquisition and preprocessing. Note, preprocessing does not alter raw data, so at any point in time all data inside
376
372
  the folder is considered 'raw'."""
377
- camera_data_path: str | Path = "null"
373
+ camera_data_path: Path = Path()
378
374
  """Stores the path to the directory that contains all camera data acquired during the session. Primarily, this
379
375
  includes .mp4 video files from each recorded camera."""
380
- mesoscope_data_path: str | Path = "null"
376
+ mesoscope_data_path: Path = Path()
381
377
  """Stores the path to the directory that contains all Mesoscope data acquired during the session. Primarily, this
382
378
  includes the mesoscope-acquired .tiff files (brain activity data) and the motion estimation data."""
383
- behavior_data_path: str | Path = "null"
379
+ behavior_data_path: Path = Path()
384
380
  """Stores the path to the directory that contains all behavior data acquired during the session. Primarily, this
385
381
  includes the .npz log files used by data-acquisition libraries to store all acquired data. The data stored in this
386
382
  way includes the camera and mesoscope frame timestamps and the states of Mesoscope-VR components, such as lick
387
383
  sensors, rotary encoders, and other modules."""
388
- zaber_positions_path: str | Path = "null"
384
+ zaber_positions_path: Path = Path()
389
385
  """Stores the path to the zaber_positions.yaml file. This file contains the snapshot of all Zaber motor positions
390
386
  at the end of the session. Zaber motors are used to position the LickPort and the HeadBar manipulators, which is
391
387
  essential for supporting proper brain imaging and animal's running behavior during the session."""
392
- session_descriptor_path: str | Path = "null"
388
+ session_descriptor_path: Path = Path()
393
389
  """Stores the path to the session_descriptor.yaml file. This file is partially filled by the system during runtime
394
390
  and partially by the experimenter after the runtime. It contains session-specific information, such as the specific
395
391
  training parameters, the positions of the Mesoscope objective and the notes made by the experimenter during
396
392
  runtime."""
397
- hardware_configuration_path: str | Path = "null"
393
+ hardware_configuration_path: Path = Path()
398
394
  """Stores the path to the hardware_configuration.yaml file. This file contains the partial snapshot of the
399
395
  calibration parameters used by the Mesoscope-VR system components during runtime. Primarily, this is used during
400
396
  data processing to read the .npz data log files generated during runtime."""
401
- surgery_metadata_path: str | Path = "null"
397
+ surgery_metadata_path: Path = Path()
402
398
  """Stores the path to the surgery_metadata.yaml file. This file contains the most actual information about the
403
399
  surgical intervention(s) performed on the animal prior to the session."""
404
- project_configuration_path: str | Path = "null"
400
+ project_configuration_path: Path = Path()
405
401
  """Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
406
402
  parameters for the session's project."""
407
- session_data_path: str | Path = "null"
403
+ session_data_path: Path = Path()
408
404
  """Stores the path to the session_data.yaml file. This path is used b y the SessionData instance to save itself to
409
405
  disk as a .yaml file. The file contains all paths used during data acquisition and processing on both the VRPC and
410
406
  the BioHPC server."""
411
- experiment_configuration_path: str | Path = "null"
407
+ experiment_configuration_path: Path = Path()
412
408
  """Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
413
409
  experiment runtime configuration used by the session. This file is only created for experiment session. It does not
414
410
  exist for behavior training sessions."""
415
- mesoscope_positions_path: str | Path = "null"
411
+ mesoscope_positions_path: Path = Path()
416
412
  """Stores the path to the mesoscope_positions.yaml file. This file contains the snapshot of the positions used
417
413
  by the Mesoscope at the end of the session. This includes both the physical position of the mesoscope objective and
418
414
  the 'virtual' tip, tilt, and fastZ positions set via ScanImage software. This file is only created for experiment
419
415
  sessions that use the mesoscope, it is omitted for behavior training sessions."""
420
- window_screenshot_path: str | Path = "null"
416
+ window_screenshot_path: Path = Path()
421
417
  """Stores the path to the .png screenshot of the ScanImagePC screen. The screenshot should contain the image of the
422
418
  cranial window and the red-dot alignment windows. This is used to generate a visual snapshot of the cranial window
423
419
  alignment and appearance for each experiment session. This file is only created for experiment sessions that use
424
420
  the mesoscope, it is omitted for behavior training sessions."""
425
- telomere_path: str | Path = "null"
421
+ telomere_path: Path = Path()
426
422
  """Stores the path to the telomere.bin file. This file is created by the data processing pipelines running on the
427
- BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission. This
428
- path is used by the BioHPC server pipeline. The VRPC uses the path stored in the 'destinations' section to check
429
- for the existence of the file when it purges redundant data."""
430
- checksum_path: str | Path = "null"
423
+ BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission."""
424
+ checksum_path: Path = Path()
431
425
  """Stores the path to the ax_checksum.txt file. This file is generated as part of packaging the data for
432
426
  transmission and stores the xxHash-128 checksum of the data. It is used to verify that the transmission did not
433
- damage or otherwise alter the data.."""
427
+ damage or otherwise alter the data."""
434
428
 
435
- def __post_init__(self) -> None:
436
- """This method is automatically called after class instantiation and ensures that all path fields of the class
437
- are converted to Path objects.
438
- """
429
+ def resolve_paths(self, root_directory_path: Path) -> None:
430
+ """Resolves all paths managed by the class instance based on the input root directory path.
431
+
432
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
433
+ machine that instantiates the class.
439
434
 
440
- # Translates the root raw_data path to the Path object before using it to determine other Path objects.
441
- self.raw_data_path = Path(self.raw_data_path)
442
-
443
- # When the class is instantiated for the first time, resolves all 'null' path placeholders
444
- if self.camera_data_path == "null":
445
- self.camera_data_path = self.raw_data_path.joinpath("camera_data")
446
- if self.mesoscope_data_path == "null":
447
- self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
448
- if self.behavior_data_path == "null":
449
- self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
450
- if self.zaber_positions_path == "null":
451
- self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
452
- if self.session_descriptor_path == "null":
453
- self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
454
- if self.hardware_configuration_path == "null":
455
- self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
456
- if self.surgery_metadata_path == "null":
457
- self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
458
- if self.project_configuration_path == "null":
459
- self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
460
- if self.session_data_path == "null":
461
- self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
462
- if self.experiment_configuration_path == "null":
463
- self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
464
- if self.mesoscope_positions_path == "null":
465
- self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
466
- if self.window_screenshot_path == "null":
467
- self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
468
- if self.telomere_path == "null":
469
- self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
470
- if self.checksum_path == "null":
471
- self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
472
-
473
- def make_string(self) -> None:
474
- """Converts all Path objects stored inside the class to strings.
475
-
476
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
477
- on disk.
435
+ Args:
436
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
437
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
438
+ the managed session.
478
439
  """
479
- self.raw_data_path = str(self.raw_data_path)
480
- self.camera_data_path = str(self.camera_data_path)
481
- self.mesoscope_data_path = str(self.mesoscope_data_path)
482
- self.behavior_data_path = str(self.behavior_data_path)
483
- self.zaber_positions_path = str(self.zaber_positions_path)
484
- self.session_descriptor_path = str(self.session_descriptor_path)
485
- self.hardware_configuration_path = str(self.hardware_configuration_path)
486
- self.surgery_metadata_path = str(self.surgery_metadata_path)
487
- self.project_configuration_path = str(self.project_configuration_path)
488
- self.session_data_path = str(self.session_data_path)
489
- self.experiment_configuration_path = str(self.experiment_configuration_path)
490
- self.mesoscope_positions_path = str(self.mesoscope_positions_path)
491
- self.window_screenshot_path = str(self.window_screenshot_path)
492
- self.telomere_path = str(self.telomere_path)
493
- self.checksum_path = str(self.checksum_path)
494
-
495
- def make_dirs(self) -> None:
496
- """Ensures that all major subdirectories and the root raw_data directory exist.
497
-
498
- This method is used by the VRPC to generate the raw_data directory when it creates a new session.
440
+
441
+ # Generates the managed paths
442
+ self.raw_data_path = root_directory_path
443
+ self.camera_data_path = self.raw_data_path.joinpath("camera_data")
444
+ self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
445
+ self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
446
+ self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
447
+ self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
448
+ self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
449
+ self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
450
+ self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
451
+ self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
452
+ self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
453
+ self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
454
+ self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
455
+ self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
456
+ self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
457
+
458
+ def make_directories(self) -> None:
459
+ """Ensures that all major subdirectories and the root directory exist."""
460
+ ensure_directory_exists(self.raw_data_path)
461
+ ensure_directory_exists(self.camera_data_path)
462
+ ensure_directory_exists(self.mesoscope_data_path)
463
+ ensure_directory_exists(self.behavior_data_path)
464
+
465
+
466
+ @dataclass()
467
+ class DeepLabCutData:
468
+ """Stores the paths to the directories and files that make up the 'deeplabcut' project-specific directory.
469
+
470
+ DeepLabCut (DLC) is used to track animal body parts and poses in video data acquired during experiment and training
471
+ sessions. Since DLC is designed to work with projects, rather than single animals or sessions, each Sun lab
472
+ project data hierarchy contains a dedicated 'deeplabcut' directory under the root project directory. The contents of
473
+ that directory are largely managed by the DLC itself. Therefore, each session of a given project refers to and
474
+ uses the same 'deeplabcut' directory.
475
+ """
476
+
477
+ deeplabcut_path: Path = Path()
478
+ """Stores the path to the project-specific DeepLabCut directory. This folder stores all DeepLabCut data specific to
479
+ a single project, which is reused during the processing of all sessions of the project."""
480
+
481
+ def resolve_paths(self, root_directory_path: Path) -> None:
482
+ """Resolves all paths managed by the class instance based on the input root directory path.
483
+
484
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
485
+ machine that instantiates the class.
486
+
487
+ Args:
488
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
489
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
490
+ the managed session.
499
491
  """
500
- ensure_directory_exists(Path(self.raw_data_path))
501
- ensure_directory_exists(Path(self.camera_data_path))
502
- ensure_directory_exists(Path(self.mesoscope_data_path))
503
- ensure_directory_exists(Path(self.behavior_data_path))
504
492
 
505
- def switch_root(self, new_root: Path) -> None:
506
- """Changes the root of the managed raw_data directory to the provided root path.
493
+ # Generates the managed paths
494
+ self.deeplabcut_path = root_directory_path
495
+
496
+ def make_directories(self) -> None:
497
+ """Ensures that all major subdirectories and the root directory exist."""
498
+ ensure_directory_exists(self.deeplabcut_path)
499
+
500
+
501
+ @dataclass()
502
+ class ConfigurationData:
503
+ """Stores the paths to the directories and files that make up the 'configuration' project-specific directory.
507
504
 
508
- This service method is used by the SessionData class to convert all paths in this class to be relative to the
509
- new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
510
- (another root). This method is only implemented for subclasses intended to be used both locally and on the
511
- BioHPC server.
505
+ The configuration directory contains various configuration files and settings used by data acquisition,
506
+ preprocessing, and processing pipelines in the lab. Generally, all configuration settings are defined once for each
507
+ project and are reused for every session within the project. Therefore, this directory is created under each main
508
+ project directory.
512
509
 
513
- Note:
514
- This is the only class that contains this method. Other classes are re-initialized using the data loaded
515
- from the RawData section, if necessary.
510
+ Notes:
511
+ Some attribute names inside this section match the names in the RawData section. This is intentional, as some
512
+ configuration files are copied into the raw_data session directories to allow reinstating the session data
513
+ hierarchy across machines.
514
+ """
515
+
516
+ configuration_path: Path = Path()
517
+ """Stores the path to the project-specific configuration directory. This directory is used by all animals
518
+ and sessions of the project to store all pan-project configuration files. The configuration data is reused by all
519
+ sessions in the project."""
520
+ experiment_configuration_path: Path = Path()
521
+ """Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
522
+ experiment runtime configuration used by the session. This file is only created for experiment session. It does not
523
+ exist for behavior training sessions."""
524
+ project_configuration_path: Path = Path()
525
+ """Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
526
+ parameters for the session's project."""
527
+ suite2p_configuration_path: Path = Path()
528
+ """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
529
+ the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
530
+ registration pipeline, which is applied to each session that generates brain activity data."""
531
+ multiday_configuration_path: Path = Path()
532
+ """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
533
+ on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
534
+ sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
535
+
536
+ def resolve_paths(self, root_directory_path: Path, experiment_name: str | None = None) -> None:
537
+ """Resolves all paths managed by the class instance based on the input root directory path.
538
+
539
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
540
+ machine that instantiates the class.
516
541
 
517
542
  Args:
518
- new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
519
- directory that stores all Sun lab projects on the target machine.
543
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
544
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
545
+ the managed session.
546
+ experiment_name: Optionally specifies the name of the experiment executed as part of the managed session's
547
+ runtime. This is used to correctly configure the path to the specific ExperimentConfiguration data file.
548
+ If the managed session is not an Experiment session, this parameter should be set to None.
520
549
  """
521
- # Gets current root from the raw_data_path.
522
- old_root = Path(self.raw_data_path).parents[3]
523
-
524
- # Updates all paths by replacing old_root with new_root
525
- self.raw_data_path = new_root.joinpath(Path(self.raw_data_path).relative_to(old_root))
526
- self.camera_data_path = new_root.joinpath(Path(self.camera_data_path).relative_to(old_root))
527
- self.mesoscope_data_path = new_root.joinpath(Path(self.mesoscope_data_path).relative_to(old_root))
528
- self.behavior_data_path = new_root.joinpath(Path(self.behavior_data_path).relative_to(old_root))
529
- self.zaber_positions_path = new_root.joinpath(Path(self.zaber_positions_path).relative_to(old_root))
530
- self.session_descriptor_path = new_root.joinpath(Path(self.session_descriptor_path).relative_to(old_root))
531
- self.hardware_configuration_path = new_root.joinpath(
532
- Path(self.hardware_configuration_path).relative_to(old_root)
533
- )
534
- self.surgery_metadata_path = new_root.joinpath(Path(self.surgery_metadata_path).relative_to(old_root))
535
- self.project_configuration_path = new_root.joinpath(Path(self.project_configuration_path).relative_to(old_root))
536
- self.session_data_path = new_root.joinpath(Path(self.session_data_path).relative_to(old_root))
537
- self.experiment_configuration_path = new_root.joinpath(
538
- Path(self.experiment_configuration_path).relative_to(old_root)
539
- )
540
- self.mesoscope_positions_path = new_root.joinpath(Path(self.mesoscope_positions_path).relative_to(old_root))
541
- self.window_screenshot_path = new_root.joinpath(Path(self.window_screenshot_path).relative_to(old_root))
542
- self.telomere_path = new_root.joinpath(Path(self.telomere_path).relative_to(old_root))
543
- self.checksum_path = new_root.joinpath(Path(self.checksum_path).relative_to(old_root))
550
+
551
+ # Generates the managed paths
552
+ self.configuration_path = root_directory_path
553
+ if experiment_name is None:
554
+ self.experiment_configuration_path = self.configuration_path.joinpath("null")
555
+ else:
556
+ self.experiment_configuration_path = self.configuration_path.joinpath(f"{experiment_name}.yaml")
557
+ self.project_configuration_path = self.configuration_path.joinpath("project_configuration.yaml")
558
+ self.suite2p_configuration_path = self.configuration_path.joinpath("suite2p_configuration.yaml")
559
+ self.multiday_configuration_path = self.configuration_path.joinpath("multiday_configuration.yaml")
560
+
561
+ def make_directories(self) -> None:
562
+ """Ensures that all major subdirectories and the root directory exist."""
563
+ ensure_directory_exists(self.configuration_path)
544
564
 
545
565
 
546
566
  @dataclass()
547
567
  class ProcessedData:
548
- """Stores the paths to the directories and files that make up the 'processed_data' session directory.
568
+ """Stores the paths to the directories and files that make up the 'processed_data' session-specific directory.
549
569
 
550
- The processed_data directory stores the data generated by various processing pipelines from the raw data. Processed
551
- data represents an intermediate step between raw data and the dataset used in the data analysis, but is not itself
552
- designed to be analyzed.
570
+ The processed_data directory stores the data generated by various processing pipelines from the raw data (contents
571
+ of the raw_data directory). Processed data represents an intermediate step between raw data and the dataset used in
572
+ the data analysis, but is not itself designed to be analyzed.
553
573
 
554
574
  Notes:
555
575
  The paths from this section are typically used only on the BioHPC server. This is because most data processing
556
576
  in the lab is performed using the processing server's resources. On the server, processed data is stored on
557
- the fast volume, in contrast to raw data, which is stored on the slow volume. However, to support local testing,
558
- the class resolves the paths in this section both locally and globally (on the server).
577
+ the fast (NVME) drive volume, in contrast to raw data, which is stored on the slow (SSD) drive volume.
559
578
 
579
+ When this class is instantiated on a machine other than BioHPC server, for example, to test processing
580
+ pipelines, it uses the same drive as the raw_data folder to create the processed_data folder. This relies on the
581
+ assumption that non-server machines in the lab only use fast NVME drives, so there is no need to separate
582
+ storage and processing volumes.
560
583
  """
561
584
 
562
- processed_data_path: str | Path
585
+ processed_data_path: Path = Path()
563
586
  """Stores the path to the root processed_data directory of the session. This directory stores the processed data
564
587
  as it is generated by various data processing pipelines."""
565
- deeplabcut_path: str | Path
566
- """Stores the path to the root DeepLabCut project directory. Since DeepLabCut adopts a project-based directory
567
- management hierarchy, it is easier to have a single DLC folder shared by all animals and sessions of a given
568
- project. This root folder is typically stored under the main project directory on the fast BioHPC server volume."""
569
- configuration_path: str | Path
570
- """Stores the path to the project's processed data configuration directory. This directory is used by all animals
571
- and sessions of the project to store data processing configuration files. Since processing configuration is
572
- typically reused for all data in the project, this allows updating a single file to control the processing for all
573
- data. This directory is typically stored on the fast BioHPC server volume."""
574
- camera_data_path: str | Path = "null"
575
- """Stores the output of the DeepLabCut pose estimation pipeline."""
576
- mesoscope_data_path: str | Path = "null"
577
- """Stores the output of the suite2p cell registration pipeline."""
578
- behavior_data_path: str | Path = "null"
579
- """Stores the output of the Sun lab behavior data extraction pipeline."""
580
- suite2p_configuration_path: str | Path = "null"
581
- """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
582
- the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
583
- registration pipeline, which is applied to each session that generates brain activity data."""
584
- processing_tracker_path: str | Path = "null"
585
- """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
586
- the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
587
- session and the status (success / failure) of each processing step.
588
+ camera_data_path: Path = Path()
589
+ """Stores the path to the directory that contains video tracking data generated by our DeepLabCut-based video
590
+ processing pipelines."""
591
+ mesoscope_data_path: Path = Path()
592
+ """Stores path to the directory that contains processed brain activity (cell) data generated by our suite2p-based
593
+ photometry processing pipelines (single day and multi day)."""
594
+ behavior_data_path: Path = Path()
595
+ """Stores the path to the directory that contains the non-video behavior and system runtime data extracted from
596
+ .npz log files by our in-house log parsing pipeline."""
597
+ job_logs_path: Path = Path()
598
+ """Stores the path to the directory that stores the standard output and standard error data collected during
599
+ server-side data processing pipeline runtimes. Since we use SLURM job manager to execute multiple compute jobs on
600
+ the BioHPC server, all information sent to the terminal during runtime is redirected to text files stored in this
601
+ directory."""
602
+ processing_tracker_path: Path = Path()
603
+ """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory.
604
+ This file is used to track which processing pipelines need to be applied to the target session and the status
605
+ (success / failure) each of these pipelines. Primarily, this is used to optimize data processing to avoid redundant
606
+ operations.
588
607
  """
589
- multiday_configuration_path: str | Path = "null"
590
- """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
591
- on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
592
- sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
593
608
 
594
- def __post_init__(self) -> None:
595
- """This method is automatically called after class instantiation and ensures that all path fields of the class
596
- are converted to Path objects.
597
- """
609
+ def resolve_paths(self, root_directory_path: Path) -> None:
610
+ """Resolves all paths managed by the class instance based on the input root directory path.
598
611
 
599
- # Ensures all objects that have to be provided externally are transformed to Path objects. Unlike 'null' fields,
600
- # these paths cannot be resolved automatically
601
- self.processed_data_path = Path(self.processed_data_path)
602
- self.deeplabcut_path = Path(self.deeplabcut_path)
603
- self.configuration_path = Path(self.configuration_path)
604
-
605
- # When the class is instantiated for the first time, resolves all 'null' path placeholders
606
- if self.camera_data_path == "null":
607
- self.camera_data_path = self.processed_data_path.joinpath("camera_data")
608
- if self.mesoscope_data_path == "null":
609
- self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
610
- if self.behavior_data_path == "null":
611
- self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
612
- if self.suite2p_configuration_path == "null":
613
- self.suite2p_configuration_path = self.processed_data_path.joinpath("suite2p_configuration.yaml")
614
- if self.processing_tracker_path == "null":
615
- self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
616
- if self.multiday_configuration_path == "null":
617
- self.multiday_configuration_path = self.processed_data_path.joinpath("multiday_configuration.yaml")
618
-
619
- def make_string(self) -> None:
620
- """Converts all Path objects stored inside the class to strings.
621
-
622
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
623
- on disk.
624
- """
625
- self.processed_data_path = str(self.processed_data_path)
626
- self.deeplabcut_path = str(self.deeplabcut_path)
627
- self.configuration_path = str(self.configuration_path)
628
- self.camera_data_path = str(self.camera_data_path)
629
- self.mesoscope_data_path = str(self.mesoscope_data_path)
630
- self.behavior_data_path = str(self.behavior_data_path)
631
- self.suite2p_configuration_path = str(self.suite2p_configuration_path)
632
- self.processing_tracker_path = str(self.processing_tracker_path)
633
- self.multiday_configuration_path = str(self.multiday_configuration_path)
634
-
635
- def make_dirs(self) -> None:
636
- """Ensures that all major subdirectories of the processed_data directory exist.
637
-
638
- This method is used by the BioHPC server to generate the processed_data directory as part of the sl-forgery
639
- library runtime.
612
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
613
+ machine that instantiates the class.
614
+
615
+ Args:
616
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
617
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
618
+ the managed session.
640
619
  """
641
- ensure_directory_exists(Path(self.processed_data_path))
642
- ensure_directory_exists(Path(self.deeplabcut_path))
643
- ensure_directory_exists(Path(self.configuration_path))
644
- ensure_directory_exists(Path(self.camera_data_path))
645
- ensure_directory_exists(Path(self.mesoscope_data_path))
646
- ensure_directory_exists(Path(self.behavior_data_path))
620
+ # Generates the managed paths
621
+ self.processed_data_path = root_directory_path
622
+ self.camera_data_path = self.processed_data_path.joinpath("camera_data")
623
+ self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
624
+ self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
625
+ self.job_logs_path = self.processed_data_path.joinpath("job_logs")
626
+ self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
627
+
628
+ def make_directories(self) -> None:
629
+ """Ensures that all major subdirectories and the root directory exist."""
630
+
631
+ ensure_directory_exists(self.processed_data_path)
632
+ ensure_directory_exists(self.camera_data_path)
633
+ ensure_directory_exists(self.mesoscope_data_path)
634
+ ensure_directory_exists(self.behavior_data_path)
635
+ ensure_directory_exists(self.job_logs_path)
647
636
 
648
637
 
649
638
  @dataclass()
650
- class PersistentData:
651
- """Stores the paths to the directories and files that make up the 'persistent_data' directories of the VRPC and
652
- the ScanImagePC.
639
+ class VRPCPersistentData:
640
+ """Stores the paths to the directories and files that make up the 'persistent_data' directory on the VRPC.
653
641
 
654
- Persistent data directories are used to keep certain files on the VRPC and the ScanImagePC. Typically, this data
655
- is reused during the following sessions. For example, a copy of Zaber motor positions is persisted on the VRPC for
656
- each animal after every session to support automatically restoring Zaber motors to the positions used during the
657
- previous session.
642
+ Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
643
+ are purposefully designed for specific PCs that participate in data acquisition. This section manages the
644
+ animal-specific persistent_data directory stored on the VRPC.
658
645
 
659
- Notes:
660
- Persistent data includes the project and experiment configuration data. Some persistent data is overwritten
661
- after each session, other data is generated once and kept through the animal's lifetime. Primarily, this data is
662
- only used internally by the sl-experiment or sl-forgery libraries and is not intended for end-users.
646
+ VRPC persistent data directory is used to preserve configuration data, such as the positions of Zaber motors and
647
+ Meososcope objective, so that they can be reused across sessions of the same animals. The data in this directory
648
+ is read at the beginning of each session and replaced at the end of each session.
663
649
  """
664
650
 
665
- scanimage_persistent_path: str | Path
651
+ persistent_data_path: Path = Path()
666
652
  """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
667
- belongs, relative to the ScanImagePC root. This directory is primarily used to store reference motion registration
668
- data acquired during the first experiment session for each animal."""
669
- vrpc_persistent_path: str | Path
670
- """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
671
- belongs, relative to the VRPC root. This directory is primarily used to back up the Zaber and Mesoscope positions,
672
- so that they can be restored between sessions of the same animals."""
673
- zaber_positions_path: str | Path = "null"
653
+ belongs, relative to the VRPC root. This directory is exclusively used on the VRPC."""
654
+ zaber_positions_path: Path = Path()
674
655
  """Stores the path to the Zaber motor positions snapshot generated at the end of the previous session runtime. This
675
656
  is used to automatically restore all Zaber motors to the same position across all sessions."""
676
- mesoscope_positions_path: str | Path = "null"
657
+ mesoscope_positions_path: Path = Path()
677
658
  """Stores the path to the Mesoscope positions snapshot generated at the end of the previous session runtime. This
678
659
  is used to help the user to (manually) restore the Mesoscope to the same position across all sessions."""
679
- motion_estimator_path: str | Path = "null"
680
- """Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
681
- file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
682
660
 
683
- def __post_init__(self) -> None:
684
- """This method is automatically called after class instantiation and ensures that all path fields of the class
685
- are converted to Path objects.
661
+ def resolve_paths(self, root_directory_path: Path) -> None:
662
+ """Resolves all paths managed by the class instance based on the input root directory path.
663
+
664
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
665
+ machine that instantiates the class.
666
+
667
+ Args:
668
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
669
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
670
+ the managed session.
686
671
  """
687
672
 
688
- # Converts 'anchor' paths that have to be provided at class instantiation
689
- self.scanimage_persistent_path = Path(self.scanimage_persistent_path)
690
- self.vrpc_persistent_path = Path(self.vrpc_persistent_path)
673
+ # Generates the managed paths
674
+ self.persistent_data_path = root_directory_path
675
+ self.zaber_positions_path = self.persistent_data_path.joinpath("zaber_positions.yaml")
676
+ self.mesoscope_positions_path = self.persistent_data_path.joinpath("mesoscope_positions.yaml")
677
+
678
+ def make_directories(self) -> None:
679
+ """Ensures that all major subdirectories and the root directory exist."""
691
680
 
692
- # Replaces 'null' defaults with automatically resolved paths
693
- if self.zaber_positions_path == "null":
694
- self.zaber_positions_path = self.vrpc_persistent_path.joinpath("zaber_positions.yaml")
695
- if self.mesoscope_positions_path == "null":
696
- self.mesoscope_positions_path = self.vrpc_persistent_path.joinpath("mesoscope_positions.yaml")
697
- if self.motion_estimator_path == "null":
698
- self.motion_estimator_path = self.scanimage_persistent_path.joinpath("MotionEstimator.me")
681
+ ensure_directory_exists(self.persistent_data_path)
699
682
 
700
- def make_string(self) -> None:
701
- """Converts all Path objects stored inside the class to strings.
702
683
 
703
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
704
- on disk.
684
+ @dataclass()
685
+ class ScanImagePCPersistentData:
686
+ """Stores the paths to the directories and files that make up the 'persistent_data' directory on the ScanImagePC.
687
+
688
+ Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
689
+ are purposefully designed for specific PCs that participate in data acquisition. This section manages the
690
+ animal-specific persistent_data directory stored on the ScanImagePC (Mesoscope PC).
691
+
692
+ ScanImagePC persistent data directory is used to preserve the motion estimation snapshot, generated during the first
693
+ experiment session. This is necessary to align the brain recording field of view across sessions. In turn, this
694
+ is used to carry out 'online' motion and z-drift correction, improving the accuracy of across-day (multi-day)
695
+ cell tracking.
696
+ """
697
+
698
+ persistent_data_path: Path = Path()
699
+ """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
700
+ belongs, relative to the ScanImagePC root. This directory is exclusively used on the ScanImagePC (Mesoscope PC)."""
701
+ motion_estimator_path: Path = Path()
702
+ """Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
703
+ file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
704
+
705
+ def resolve_paths(self, root_directory_path: Path) -> None:
706
+ """Resolves all paths managed by the class instance based on the input root directory path.
707
+
708
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
709
+ machine that instantiates the class.
710
+
711
+ Args:
712
+ root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
713
+ hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
714
+ the managed session.
705
715
  """
706
- self.scanimage_persistent_path = str(self.scanimage_persistent_path)
707
- self.vrpc_persistent_path = str(self.vrpc_persistent_path)
708
- self.zaber_positions_path = str(self.zaber_positions_path)
709
- self.mesoscope_positions_path = str(self.mesoscope_positions_path)
710
- self.motion_estimator_path = str(self.motion_estimator_path)
711
716
 
712
- def make_dirs(self) -> None:
713
- """Ensures that the VRPC and the ScanImagePC persistent_data directories exist."""
717
+ # Generates the managed paths
718
+ self.persistent_data_path = root_directory_path
719
+ self.motion_estimator_path = self.persistent_data_path.joinpath("MotionEstimator.me")
714
720
 
715
- # We need to call ensure_directory_exists one for each unique directory tree
716
- ensure_directory_exists(Path(self.vrpc_persistent_path)) # vrpc_root/project/animal/persistent_data
717
- ensure_directory_exists(Path(self.scanimage_persistent_path)) # scanimagepc_root/project/animal/persistent_data
721
+ def make_directories(self) -> None:
722
+ """Ensures that all major subdirectories and the root directory exist."""
723
+
724
+ ensure_directory_exists(self.persistent_data_path)
718
725
 
719
726
 
720
727
  @dataclass()
721
728
  class MesoscopeData:
722
- """Stores the paths to the directories used by the ScanImagePC to save mesoscope-generated data during session
723
- runtime.
729
+ """Stores the paths to the directories and files that make up the 'meso_data' directory on the ScanImagePC.
730
+
731
+ The meso_data directory is the root directory where all mesoscope-generated data is stored on the ScanImagePC. The
732
+ path to this directory should be given relative to the VRPC root and be mounted to the VRPC filesystem via the
733
+ SMB or equivalent protocol.
724
734
 
725
- The ScanImagePC is largely isolated from the VRPC during runtime. For the VRPC to pull the data acquired by the
726
- ScanImagePC, it has to use the predefined directory structure to save the data. This class stores the predefined
727
- path to various directories where ScanImagePC is expected to save the data and store it after acquisition.sers.
735
+ During runtime, the ScanImagePC should organize all collected data under this root directory. During preprocessing,
736
+ the VRPC uses SMB to access the data in this directory and merge it into the 'raw_data' session directory. The paths
737
+ in this section, therefore, are specific to the VRPC and are not used on other PCs.
728
738
  """
729
739
 
730
- root_data_path: str | Path
740
+ meso_data_path: Path = Path()
731
741
  """Stores the path to the root ScanImagePC data directory, mounted to the VRPC filesystem via the SMB or equivalent
732
- protocol. This path is used during experiment session runtimes to discover the cranial window screenshots
733
- taken by the user before starting the experiment."""
734
- session_specific_path: str | Path
742
+ protocol. All mesoscope-generated data is stored under this root directory before it is merged into the VRPC-managed
743
+ raw_data directory of each session."""
744
+ mesoscope_data_path: Path = Path()
745
+ """Stores the path to the 'default' mesoscope_data directory. All experiment sessions across all animals and
746
+ projects use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
747
+ software. This simplifies ScanImagePC configuration process during runtime, as all data is always saved in the same
748
+ directory. During preprocessing, the data is moved from the default directory first into a session-specific
749
+ ScanImagePC directory and then into the VRPC raw_data session directory."""
750
+ session_specific_path: Path = Path()
735
751
  """Stores the path to the session-specific data directory. This directory is generated at the end of each experiment
736
- runtime to prepare mesoscope data for further processing and to reset the 'shared' folder for the next session's
737
- runtime."""
738
- mesoscope_data_path: str | Path = "null"
739
- """Stores the path to the 'general' mesoscope_data directory. All experiment sessions (across all animals and
740
- projects) use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
741
- software. This simplifies ScanImagePC configuration process during runtime. The data is moved into a
742
- session-specific directory during preprocessing."""
743
- ubiquitin_path: str | Path = "null"
752
+ runtime to prepare mesoscope data for being moved to the VRPC-managed raw_data directory and to reset the 'default'
753
+ mesoscope_data directory for the next session's runtime."""
754
+ ubiquitin_path: Path = Path()
744
755
  """Stores the path to the 'ubiquitin.bin' file. This file is automatically generated inside the session-specific
745
- data folder after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
746
- removal, the VRPC searches for directories marked with ubiquitin.bin, so that they can be safely removed from the
756
+ data directory after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
757
+ removal step of preprocessing, the VRPC searches for directories marked with ubiquitin.bin and deletes them from the
747
758
  ScanImagePC filesystem."""
748
759
 
749
- def __post_init__(self) -> None:
750
- """This method is automatically called after class instantiation and ensures that all path fields of the class
751
- are converted to Path objects.
752
- """
753
- # Converts the paths that have to be provided at class instantiation to the Path objects.
754
- self.root_data_path = Path(self.root_data_path)
755
- self.session_specific_path = Path(self.session_specific_path)
756
-
757
- # Resolves the 'null' placeholders using the 'anchor' paths above
758
- if self.mesoscope_data_path == "null":
759
- self.mesoscope_data_path = self.root_data_path.joinpath("mesoscope_data")
760
- if self.ubiquitin_path == "null":
761
- self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
760
+ def resolve_paths(self, root_mesoscope_path: Path, session_name: str) -> None:
761
+ """Resolves all paths managed by the class instance based on the input root directory path.
762
762
 
763
- def make_string(self) -> None:
764
- """Converts all Path objects stored inside the class to strings.
763
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
764
+ machine that instantiates the class.
765
765
 
766
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
767
- on disk.
766
+ Args:
767
+ root_mesoscope_path: The path to the top-level directory of the ScanImagePC data hierarchy mounted to the
768
+ VRPC via the SMB or equivalent protocol.
769
+ session_name: The name of the session for which this subclass is initialized.
768
770
  """
769
- self.root_data_path = str(self.root_data_path)
770
- self.session_specific_path = str(self.session_specific_path)
771
- self.mesoscope_data_path = str(self.mesoscope_data_path)
772
- self.ubiquitin_path = str(self.ubiquitin_path)
773
771
 
774
- def make_dirs(self) -> None:
775
- """Ensures that the ScanImagePC data acquisition directories exist."""
772
+ # Generates the managed paths
773
+ self.meso_data_path = root_mesoscope_path
774
+ self.session_specific_path = self.meso_data_path.joinpath(session_name)
775
+ self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
776
+ self.mesoscope_data_path = self.meso_data_path.joinpath("mesoscope_data")
776
777
 
777
- # Does not create the session-specific directory. This is on purpose, as the session-specific directory
778
- # is generated during runtime by renaming the 'general' mesoscope_data directory. The 'general' directory is
779
- # then recreated from scratch. This ensures that the general directory is empty (ready for the next session)
780
- # with minimal I/O overhead.
781
- ensure_directory_exists(Path(self.mesoscope_data_path))
778
+ def make_directories(self) -> None:
779
+ """Ensures that all major subdirectories and the root directory exist."""
780
+
781
+ ensure_directory_exists(self.meso_data_path)
782
782
 
783
783
 
784
784
  @dataclass()
785
- class Destinations:
786
- """Stores the paths to the VRPC filesystem-mounted Synology NAS and BioHPC server directories.
785
+ class VRPCDestinations:
786
+ """Stores the paths to the VRPC filesystem-mounted directories of the Synology NAS and BioHPC server.
787
+
788
+ The paths from this section are primarily used to transfer preprocessed data to the long-term storage destinations.
789
+ Additionally, they allow VRPC to interface with the configuration directory of the BioHPC server to start data
790
+ processing jobs and to read the data from the processed_data directory to remove redundant data from the VRPC
791
+ filesystem.
787
792
 
788
- These directories are used during data preprocessing to transfer the preprocessed raw_data directory from the
789
- VRPC to the long-term storage destinations.
793
+ Overall, this section is intended solely for the VRPC and should not be used on other PCs.
790
794
  """
791
795
 
792
- nas_raw_data_path: str | Path
796
+ nas_raw_data_path: Path = Path()
793
797
  """Stores the path to the session's raw_data directory on the Synology NAS, which is mounted to the VRPC via the
794
798
  SMB or equivalent protocol."""
795
- server_raw_data_path: str | Path
799
+ server_raw_data_path: Path = Path()
796
800
  """Stores the path to the session's raw_data directory on the BioHPC server, which is mounted to the VRPC via the
797
801
  SMB or equivalent protocol."""
798
- server_telomere_path: str | Path = "null"
799
- """Stores the path to the session's telomere.bin marker. This marker is generated as part of BioHPC-side data
800
- processing pipeline to notify he VRPC that the server received the data intact. it is used to determine which
801
- data can be safely removed from the VRPC filesystem."""
802
-
803
- def __post_init__(self) -> None:
804
- """This method is automatically called after class instantiation and ensures that all path fields of the class
805
- are converted to Path objects.
806
- """
807
- self.nas_raw_data_path = Path(self.nas_raw_data_path)
808
- self.server_raw_data_path = Path(self.server_raw_data_path)
802
+ server_processed_data_path: Path = Path()
803
+ """Stores the path to the session's processed_data directory on the BioHPC server, which is mounted to the VRPC via
804
+ the SMB or equivalent protocol."""
805
+ server_configuration_path: Path = Path()
806
+ """Stores the path to the project-specific 'configuration' directory on the BioHPC server, which is mounted to the
807
+ VRPC via the SMB or equivalent protocol."""
808
+ telomere_path: Path = Path()
809
+ """Stores the path to the session's telomere.bin marker. This marker is generated as part of data processing on the
810
+ BioHPC server to notify the VRPC that the server received preprocessed data intact. The presence of this marker is
811
+ used by the VRPC to determine which locally stored raw_data is safe to delete from the filesystem."""
812
+ suite2p_configuration_path: Path = Path()
813
+ """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
814
+ the BioHPC server. This configuration file specifies the parameters for the 'single day' sl-suite2p registration
815
+ pipeline, which is applied to each session that generates brain activity data."""
816
+ processing_tracker_path: Path = Path()
817
+ """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
818
+ the BioHPC server. This file tracks which processing pipelines need to be applied the target session and the status
819
+ (success / failure) of each applied pipeline.
820
+ """
821
+ multiday_configuration_path: Path = Path()
822
+ """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
823
+ on the BioHPC server. This configuration file specifies the parameters for the 'multiday' sl-suite2p registration
824
+ pipeline used to track brain cells across multiple sessions."""
809
825
 
810
- if self.server_telomere_path == "null":
811
- self.server_telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
826
+ def resolve_paths(
827
+ self,
828
+ nas_raw_data_path: Path,
829
+ server_raw_data_path: Path,
830
+ server_processed_data_path: Path,
831
+ server_configuration_path: Path,
832
+ ) -> None:
833
+ """Resolves all paths managed by the class instance based on the input root directory paths.
812
834
 
813
- def make_string(self) -> None:
814
- """Converts all Path objects stored inside the class to strings.
835
+ This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
836
+ machine that instantiates the class.
815
837
 
816
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
817
- on disk.
838
+ Args:
839
+ nas_raw_data_path: The path to the session's raw_data directory on the Synology NAS, relative to the VRPC
840
+ filesystem root.
841
+ server_raw_data_path: The path to the session's raw_data directory on the BioHPC server, relative to the
842
+ VRPC filesystem root.
843
+ server_processed_data_path: The path to the session's processed_data directory on the BioHPC server,
844
+ relative to the VRPC filesystem root.
845
+ server_configuration_path: The path to the project-specific 'configuration' directory on the BioHPC server,
846
+ relative to the VRPC filesystem root.
818
847
  """
819
- self.nas_raw_data_path = str(self.nas_raw_data_path)
820
- self.server_raw_data_path = str(self.server_raw_data_path)
821
- self.server_telomere_path = str(self.server_telomere_path)
822
848
 
823
- def make_dirs(self) -> None:
824
- """Ensures that all destination directories exist."""
825
- ensure_directory_exists(Path(self.nas_raw_data_path))
826
- ensure_directory_exists(Path(self.server_raw_data_path))
849
+ # Generates the managed paths
850
+ self.nas_raw_data_path = nas_raw_data_path
851
+ self.server_raw_data_path = server_raw_data_path
852
+ self.server_processed_data_path = server_processed_data_path
853
+ self.server_configuration_path = server_configuration_path
854
+ self.telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
855
+ self.suite2p_configuration_path = self.server_configuration_path.joinpath("suite2p_configuration.yaml")
856
+ self.processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
857
+ self.multiday_configuration_path = self.server_configuration_path.joinpath("multiday_configuration.yaml")
858
+
859
+ def make_directories(self) -> None:
860
+ """Ensures that all major subdirectories and the root directory exist."""
861
+ ensure_directory_exists(self.nas_raw_data_path)
862
+ ensure_directory_exists(self.server_raw_data_path)
863
+ ensure_directory_exists(self.server_configuration_path)
864
+ ensure_directory_exists(self.server_processed_data_path)
827
865
 
828
866
 
829
867
  @dataclass
@@ -869,32 +907,30 @@ class SessionData(YamlConfig):
869
907
  field is not None (null), it communicates the specific experiment configuration used by the session. During runtime,
870
908
  the name stored here is used to load the specific experiment configuration data stored in a .yaml file with the
871
909
  same name. If the session is not an experiment session, this field is ignored."""
872
- raw_data: RawData
873
- """This section stores the paths to various directories and files that make up the raw_data subfolder. This
874
- subfolder stores all data acquired during training or experiment runtimes before and after preprocessing. Note, the
875
- preprocessing does not change the raw data in any way other than lossless compression and minor format
876
- reorganization. Therefore, the data is considered 'raw' both before and after preprocessing."""
877
- processed_data: ProcessedData
878
- """This section stores the paths to various directories used to store processed session data. Processed data is
879
- generated from raw data by running various processing pipelines, such as suite2p, DeepLabCut and Sun lab's behavior
880
- parsing pipelines. Typically, all data is processed on the BioHPC server and may be stored on a filesystem volume
881
- different from the one that stores the raw data."""
882
- persistent_data: PersistentData
883
- """This section stores the paths to various files and directories that are held back on the VRPC and ScanImagePC
884
- after the session data is transferred to long-term storage destinations as part of preprocessing. Typically, this
885
- data is reused during the acquisition of future runtime session data. This section is not used during data
886
- processing."""
887
- mesoscope_data: MesoscopeData
888
- """This section stores the paths to various directories used by the ScanImagePC when acquiring mesoscope-related
889
- data. During runtime, the VRPC (behavior data and experiment control) and the ScanImagePC (brain activity data and
890
- Mesoscope control) operate mostly independently of each-other. During preprocessing, the VRPC pulls the data from
891
- the ScanImagePC, using the paths in this section to find the data to be transferred. This section is not used
892
- during data processing."""
893
- destinations: Destinations
894
- """This section stores the paths to the destination directories on the BioHPC server and Synology NAS, to which the
895
- data is copied as part of preprocessing for long-term storage and further processing. Both of these directories
896
- should be mapped (mounted) to the VRPC's filesystem via the SMB or equivalent protocol. This section is not used
897
- during data processing."""
910
+ raw_data: RawData = field(default_factory=lambda: RawData())
911
+ """Stores the paths to all subfolders and files found under the /project/animal/session/raw_data directory of any
912
+ PC used to work with Sun lab data."""
913
+ processed_data: ProcessedData = field(default_factory=lambda: ProcessedData())
914
+ """Stores the paths to all subfolders and files found under the /project/animal/session/processed_data directory of
915
+ any PC used to work with Sun lab data."""
916
+ deeplabcut_data: DeepLabCutData = field(default_factory=lambda: DeepLabCutData())
917
+ """Stores the paths to all subfolders and files found under the /project/deeplabcut directory of any PC used to
918
+ work with Sun lab data."""
919
+ configuration_data: ConfigurationData = field(default_factory=lambda: ConfigurationData())
920
+ """Stores the paths to all subfolders and files found under the /project/configuration directory of any PC used to
921
+ work with Sun lab data."""
922
+ vrpc_persistent_data: VRPCPersistentData = field(default_factory=lambda: VRPCPersistentData())
923
+ """Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
924
+ the VRPC used in the Sun lab to acquire behavior data."""
925
+ scanimagepc_persistent_data: ScanImagePCPersistentData = field(default_factory=lambda: ScanImagePCPersistentData())
926
+ """Stores the paths to all subfolders and files found under the /project/animal/persistent_data directory of
927
+ the ScanImagePC used in the Sun lab to acquire brain activity data."""
928
+ mesoscope_data: MesoscopeData = field(default_factory=lambda: MesoscopeData())
929
+ """Stores the paths to all subfolders and files found under the /meso_data (root mesoscope data) directory of
930
+ the ScanImagePC used in the Sun lab to acquire brain activity data."""
931
+ destinations: VRPCDestinations = field(default_factory=lambda: VRPCDestinations())
932
+ """Stores the paths to all subfolders and files under various VRPC-filesystem-mounted directories of other machines
933
+ used in the Sun lab for long-term data storage."""
898
934
 
899
935
  @classmethod
900
936
  def create(
@@ -943,17 +979,18 @@ class SessionData(YamlConfig):
943
979
  if session_name is None:
944
980
  session_name = str(get_timestamp(time_separator="-"))
945
981
 
946
- # Extracts the root directory paths stored inside the project configuration file. All roots are expected to be
947
- # mounted on the local (VRPC) via SMB or equivalent protocol and be relative to the VRPC root.
982
+ # Extracts the root directory paths stored inside the project configuration file. These roots are then used to
983
+ # initialize this class instance.
948
984
  vrpc_root = Path(project_configuration.local_root_directory)
949
985
  mesoscope_root = Path(project_configuration.local_mesoscope_directory)
950
986
  biohpc_root = Path(project_configuration.local_server_directory)
987
+ biohpc_workdir = Path(project_configuration.local_server_working_directory)
951
988
  nas_root = Path(project_configuration.local_nas_directory)
952
989
 
953
990
  # Extracts the name of the project stored inside the project configuration file.
954
991
  project_name = project_configuration.project_name
955
992
 
956
- # Constructs the session directory path and generates the directory
993
+ # Constructs the session directory path
957
994
  session_path = vrpc_root.joinpath(project_name, animal_id, session_name)
958
995
 
959
996
  # Handles potential session name conflicts
@@ -971,38 +1008,51 @@ class SessionData(YamlConfig):
971
1008
  f"already exists. The newly created session directory uses a '_{counter}' postfix to distinguish "
972
1009
  f"itself from the already existing session directory."
973
1010
  )
974
- warnings.warn(message=message)
1011
+ console.echo(message=message, level=LogLevel.ERROR)
975
1012
 
976
1013
  # Generates subclasses stored inside the main class instance based on the data resolved above. Note; most fields
977
- # of these classes are initialized automatically, using the 'root' path arguments.
978
- raw_data = RawData(
979
- raw_data_path=session_path.joinpath("raw_data"),
980
- )
981
- raw_data.make_dirs() # Generates the local directory tree
982
-
983
- processed_data = ProcessedData(
984
- processed_data_path=session_path.joinpath("processed_data"),
985
- deeplabcut_path=vrpc_root.joinpath(project_name, "deeplabcut"),
986
- configuration_path=vrpc_root.joinpath(project_name, "configuration"),
1014
+ # of these classes are resolved automatically, based on one or more 'root' paths provided to the 'resolve_paths'
1015
+ # method.
1016
+ raw_data = RawData()
1017
+ raw_data.resolve_paths(root_directory_path=session_path.joinpath("raw_data"))
1018
+ raw_data.make_directories() # Generates the local directory tree
1019
+
1020
+ processed_data = ProcessedData()
1021
+ processed_data.resolve_paths(root_directory_path=session_path.joinpath("processed_data"))
1022
+ processed_data.make_directories()
1023
+
1024
+ dlc_data = DeepLabCutData()
1025
+ dlc_data.resolve_paths(root_directory_path=vrpc_root.joinpath(project_name, "deeplabcut"))
1026
+ dlc_data.make_directories()
1027
+
1028
+ configuration_data = ConfigurationData()
1029
+ configuration_data.resolve_paths(
1030
+ root_directory_path=vrpc_root.joinpath(project_name, "configuration"), experiment_name=experiment_name
987
1031
  )
1032
+ configuration_data.make_directories()
988
1033
 
1034
+ vrpc_persistent_data = VRPCPersistentData()
989
1035
  vrpc_persistent_path = vrpc_root.joinpath(project_name, animal_id, "persistent_data")
1036
+ vrpc_persistent_data.resolve_paths(root_directory_path=vrpc_persistent_path)
1037
+ vrpc_persistent_data.make_directories()
1038
+
1039
+ scanimagepc_persistent_data = ScanImagePCPersistentData()
990
1040
  scanimagepc_persistent_path = mesoscope_root.joinpath(project_name, animal_id, "persistent_data")
991
- persistent_data = PersistentData(
992
- vrpc_persistent_path=vrpc_persistent_path, scanimage_persistent_path=scanimagepc_persistent_path
993
- )
994
- persistent_data.make_dirs() # Generates all persistent directory trees
1041
+ scanimagepc_persistent_data.resolve_paths(root_directory_path=scanimagepc_persistent_path)
1042
+ scanimagepc_persistent_data.make_directories()
995
1043
 
996
- mesoscope_data = MesoscopeData(
997
- root_data_path=mesoscope_root, session_specific_path=mesoscope_root.joinpath(session_name)
998
- )
999
- mesoscope_data.make_dirs() # Generates all Mesoscope directory trees
1044
+ mesoscope_data = MesoscopeData()
1045
+ mesoscope_data.resolve_paths(root_mesoscope_path=mesoscope_root, session_name=session_name)
1046
+ mesoscope_data.make_directories()
1000
1047
 
1001
- destinations = Destinations(
1048
+ destinations = VRPCDestinations()
1049
+ destinations.resolve_paths(
1002
1050
  nas_raw_data_path=nas_root.joinpath(project_name, animal_id, session_name, "raw_data"),
1003
1051
  server_raw_data_path=biohpc_root.joinpath(project_name, animal_id, session_name, "raw_data"),
1052
+ server_configuration_path=biohpc_root.joinpath(project_name, "configuration"),
1053
+ server_processed_data_path=biohpc_workdir.joinpath(project_name, "processed_data"),
1004
1054
  )
1005
- destinations.make_dirs() # Generates all destination directory trees
1055
+ destinations.make_directories()
1006
1056
 
1007
1057
  # Packages the sections generated above into a SessionData instance
1008
1058
  instance = SessionData(
@@ -1011,8 +1061,11 @@ class SessionData(YamlConfig):
1011
1061
  session_name=session_name,
1012
1062
  session_type=session_type,
1013
1063
  raw_data=raw_data,
1064
+ deeplabcut_data=dlc_data,
1065
+ configuration_data=configuration_data,
1014
1066
  processed_data=processed_data,
1015
- persistent_data=persistent_data,
1067
+ vrpc_persistent_data=vrpc_persistent_data,
1068
+ scanimagepc_persistent_data=scanimagepc_persistent_data,
1016
1069
  mesoscope_data=mesoscope_data,
1017
1070
  destinations=destinations,
1018
1071
  experiment_name=experiment_name,
@@ -1026,19 +1079,16 @@ class SessionData(YamlConfig):
1026
1079
  # classes is not exhaustive. More classes are saved as part of the session runtime management class start() and
1027
1080
  # __init__() method runtimes:
1028
1081
 
1029
- # Resolves the path to the project configuration folder
1030
- vrpc_configuration_path = vrpc_root.joinpath(project_name, "configuration")
1031
-
1032
1082
  # Discovers and saves the necessary configuration class instances to the raw_data folder of the managed session:
1033
1083
  # Project Configuration
1034
1084
  sh.copy2(
1035
- src=vrpc_configuration_path.joinpath("project_configuration.yaml"),
1085
+ src=instance.configuration_data.project_configuration_path,
1036
1086
  dst=instance.raw_data.project_configuration_path,
1037
1087
  )
1038
1088
  # Experiment Configuration, if the session type is Experiment.
1039
1089
  if experiment_name is not None:
1040
1090
  sh.copy2(
1041
- src=vrpc_configuration_path.joinpath(f"{experiment_name}.yaml"),
1091
+ src=instance.configuration_data.experiment_configuration_path,
1042
1092
  dst=instance.raw_data.experiment_configuration_path,
1043
1093
  )
1044
1094
 
@@ -1062,9 +1112,10 @@ class SessionData(YamlConfig):
1062
1112
 
1063
1113
  Args:
1064
1114
  session_path: The path to the root directory of an existing session, e.g.: vrpc_root/project/animal/session.
1065
- on_server: Determines whether the method is used to initialize an existing session on the VRPC or the
1066
- BioHPC server. Note, VRPC runtimes use the same 'root' directory to store raw_data and processed_data
1067
- subfolders. BioHPC server runtimes use different volumes (drives) to store these subfolders.
1115
+ on_server: Determines whether the method is used to initialize an existing session on the BioHPC server or
1116
+ a non-server machine. Note, non-server runtimes use the same 'root' directory to store raw_data and
1117
+ processed_data subfolders. BioHPC server runtimes use different volumes (drives) to store these
1118
+ subfolders.
1068
1119
 
1069
1120
  Returns:
1070
1121
  An initialized SessionData instance for the session whose data is stored at the provided path.
@@ -1084,48 +1135,100 @@ class SessionData(YamlConfig):
1084
1135
  )
1085
1136
  console.error(message=message, error=FileNotFoundError)
1086
1137
 
1087
- # Loads class data from .yaml
1138
+ # Loads class data from .yaml file
1088
1139
  instance: SessionData = cls.from_yaml(file_path=session_data_path) # type: ignore
1089
1140
 
1090
1141
  # The method assumes that the 'donor' .yaml file is always stored inside the raw_data directory of the session
1091
1142
  # to be processed. Since the directory itself might have moved (between or even within the same PC) relative to
1092
1143
  # where it was when the SessionData snapshot was generated, reconfigures the paths to all raw_data files using
1093
1144
  # the root from above.
1094
- instance.raw_data.switch_root(new_root=session_path.parents[2])
1145
+ local_root = session_path.parents[2]
1146
+
1147
+ # RAW DATA
1148
+ new_root = local_root.joinpath(instance.project_name, instance.animal_id, instance.session_name, "raw_data")
1149
+ instance.raw_data.resolve_paths(root_directory_path=new_root)
1150
+
1151
+ # Uses the adjusted raw_data section to load the ProjectConfiguration instance. This is used below to resolve
1152
+ # all other SessionData sections, as it stores various required root directories.
1153
+ project_configuration: ProjectConfiguration = ProjectConfiguration.load(
1154
+ project_name=instance.project_name,
1155
+ configuration_path=Path(instance.raw_data.project_configuration_path),
1156
+ )
1157
+
1158
+ # Resolves the new roots for all sections that use the same root as the raw_data directory:
1159
+
1160
+ # CONFIGURATION
1161
+ new_root = local_root.joinpath(instance.project_name, "configuration")
1162
+ instance.configuration_data.resolve_paths(
1163
+ root_directory_path=new_root, experiment_name=instance.experiment_name
1164
+ )
1165
+
1166
+ # DEEPLABCUT
1167
+ new_root = local_root.joinpath(instance.project_name, "deeplabcut")
1168
+ instance.deeplabcut_data.resolve_paths(root_directory_path=new_root)
1169
+
1170
+ # Resolves the roots for all VRPC-specific sections that use the data from the ProjectConfiguration instance:
1171
+
1172
+ # VRPC PERSISTENT DATA
1173
+ new_root = Path(project_configuration.local_root_directory).joinpath(
1174
+ instance.project_name, instance.animal_id, "persistent_data"
1175
+ )
1176
+ instance.vrpc_persistent_data.resolve_paths(root_directory_path=new_root)
1177
+
1178
+ # SCANIMAGEPC PERSISTENT DATA
1179
+ new_root = Path(project_configuration.local_mesoscope_directory).joinpath(
1180
+ instance.project_name, instance.animal_id, "persistent_data"
1181
+ )
1182
+ instance.scanimagepc_persistent_data.resolve_paths(root_directory_path=new_root)
1183
+
1184
+ # MESOSCOPE DATA
1185
+ instance.mesoscope_data.resolve_paths(
1186
+ root_mesoscope_path=Path(project_configuration.local_mesoscope_directory),
1187
+ session_name=instance.session_name,
1188
+ )
1189
+
1190
+ # DESTINATIONS
1191
+ instance.destinations.resolve_paths(
1192
+ nas_raw_data_path=Path(project_configuration.local_nas_directory).joinpath(
1193
+ instance.project_name, instance.animal_id, instance.session_name, "raw_data"
1194
+ ),
1195
+ server_raw_data_path=Path(project_configuration.local_server_directory).joinpath(
1196
+ instance.project_name, instance.animal_id, instance.session_name, "raw_data"
1197
+ ),
1198
+ server_configuration_path=Path(project_configuration.local_server_directory).joinpath(
1199
+ instance.project_name, "configuration"
1200
+ ),
1201
+ server_processed_data_path=Path(project_configuration.local_server_working_directory).joinpath(
1202
+ instance.project_name, instance.animal_id, instance.session_name, "processed_data"
1203
+ ),
1204
+ )
1095
1205
 
1096
1206
  # Resolves the paths to the processed_data directories. The resolution strategy depends on whether the method is
1097
1207
  # called on the VRPC (locally) or the BioHPC server (remotely).
1098
- new_root: Path
1099
1208
  if not on_server:
1100
1209
  # Local runtimes use the same root session directory for both raw_data and processed_data. This stems from
1101
1210
  # the assumption that most local machines in the lab only use NVME (fast) volumes and, therefore, do not
1102
1211
  # need to separate 'storage' and 'working' data directories.
1103
-
1104
- # Regenerates the processed_data paths using the information loaded from the session_data.yaml file and
1105
- # static path definition rules used in the lab.
1106
- new_root = session_path.parents[2] # Reuses the local root for non-server runtimes
1212
+ new_root = local_root # Reuses the local root for non-server runtimes
1107
1213
 
1108
1214
  else:
1109
1215
  # The BioHPC server stores raw_data on slow volume and processed_data on fast (NVME) volume. Therefore, to
1110
1216
  # configure processed_data paths, the method first needs to load the fast volume root path from the
1111
1217
  # project_configuration.yaml file stored in the raw_data folder.
1112
- project_configuration: ProjectConfiguration = ProjectConfiguration.load(
1113
- project_name=instance.project_name,
1114
- configuration_path=Path(instance.raw_data.project_configuration_path),
1115
- )
1116
1218
  new_root = Path(project_configuration.remote_working_directory)
1117
1219
 
1118
1220
  # Regenerates the processed_data path depending on the root resolution above
1119
- instance.processed_data = ProcessedData(
1120
- processed_data_path=new_root.joinpath(
1221
+ instance.processed_data.resolve_paths(
1222
+ root_directory_path=new_root.joinpath(
1121
1223
  instance.project_name, instance.animal_id, instance.session_name, "processed_data"
1122
- ),
1123
- configuration_path=new_root.joinpath(instance.project_name, instance.animal_id, "configuration"),
1124
- deeplabcut_path=new_root.joinpath(instance.project_name, "deeplabcut"),
1224
+ )
1125
1225
  )
1126
1226
 
1127
- # Generates processed_data directories
1128
- instance.processed_data.make_dirs()
1227
+ # Generates data directory hierarchies that may be missing on the local machine
1228
+ instance.raw_data.make_directories()
1229
+ instance.configuration_data.make_directories()
1230
+ instance.deeplabcut_data.make_directories()
1231
+ instance.processed_data.make_directories()
1129
1232
 
1130
1233
  # Returns the initialized SessionData instance to caller
1131
1234
  return instance
@@ -1138,21 +1241,8 @@ class SessionData(YamlConfig):
1138
1241
  create() method runtime.
1139
1242
  """
1140
1243
 
1141
- # Copies instance data to prevent it from being modified by reference when executing the steps below
1142
- original = copy.deepcopy(self)
1143
-
1144
- # Extracts the target file path before it is converted to a string.
1145
- file_path: Path = copy.copy(self.raw_data.session_data_path) # type: ignore
1146
-
1147
- # Converts all Paths objects to strings before dumping the data to YAML.
1148
- original.raw_data.make_string()
1149
- original.processed_data.make_string()
1150
- original.persistent_data.make_string()
1151
- original.mesoscope_data.make_string()
1152
- original.destinations.make_string()
1153
-
1154
1244
  # Saves instance data as a .YAML file
1155
- original.to_yaml(file_path=file_path)
1245
+ self.to_yaml(file_path=self.raw_data.session_data_path)
1156
1246
 
1157
1247
 
1158
1248
  @dataclass()
@@ -1582,7 +1672,7 @@ class SurgeryData(YamlConfig):
1582
1672
 
1583
1673
  @dataclass()
1584
1674
  class ProcessingTracker(YamlConfig):
1585
- """ Tracks the data processing status for a single session.
1675
+ """Tracks the data processing status for a single session.
1586
1676
 
1587
1677
  This class is used during BioHPC-server data processing runtimes to track which processing steps are enabled and
1588
1678
  have been successfully applied to a given session. This is used to optimize data processing and avoid unnecessary