sl-shared-assets 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

sl_shared_assets/cli.py

@@ -10,6 +10,7 @@ from .server import generate_server_credentials
 from .data_classes import (
     SessionData,
     ExperimentState,
+    ProcessingTracker,
     ProjectConfiguration,
     MesoscopeSystemConfiguration,
     MesoscopeExperimentConfiguration,
@@ -33,7 +34,7 @@ from .data_classes import (
     show_default=True,
     default=False,
     help=(
-        "Determines whether to created the processed data hierarchy. This flag should be disabled for most runtimes. "
+        "Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes. "
         "Primarily, it is used by lab acquisition system code to generate processed data directories on the remote "
         "compute servers as part of the data preprocessing pipeline."
     ),
@@ -65,6 +66,7 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     processed session. This use case is fully automated and should not be triggered manually by the user.
     """
     session = Path(session_path)
+    session_data = SessionData.load(session_path=session)
 
     # Runs the verification process
     verify_session_checksum(
@@ -72,10 +74,12 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     )
 
     # Checks the outcome of the verification process
-    session_data = SessionData.load(session_path=session)
-    if session_data.raw_data.verified_bin_path.exists():
+    tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
+    if tracker.is_complete:
+        # noinspection PyTypeChecker
         console.echo(message=f"Session {session.stem} raw data integrity: Verified.", level=LogLevel.SUCCESS)
     else:
+        # noinspection PyTypeChecker
         console.echo(message=f"Session {session.stem} raw data integrity: Compromised!", level=LogLevel.ERROR)
 
 
@@ -120,6 +124,7 @@ def generate_project_manifest_file(
         output_directory=Path(output_directory),
         processed_project_directory=Path(project_processed_path) if project_processed_path else None,
     )
+    # noinspection PyTypeChecker
     console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)
 
 
@@ -177,6 +182,7 @@ def generate_system_configuration_file(output_directory: str, acquisition_system
             f"Mesoscope-VR system configuration file: generated. Edit the configuration parameters stored inside the "
             f"{file_name} file to match the state of the acquisition system and use context."
         )
+        # noinspection PyTypeChecker
         console.echo(message=message, level=LogLevel.SUCCESS)
 
     # For unsupported system types, raises an error message
@@ -234,6 +240,7 @@ def generate_server_credentials_file(output_directory: str, host: str, username:
         f"Server access credentials file: generated. If necessary, remember to edit the data acquisition system "
         f"configuration file to include the path to the credentials file generated via this CLI."
     )
+    # noinspection PyTypeChecker
     console.echo(message=message, level=LogLevel.SUCCESS)
 
 
@@ -284,6 +291,7 @@ def generate_project_configuration_file(project: str, surgery_log_id: str, water
         project_name=project, surgery_sheet_id=surgery_log_id, water_log_sheet_id=water_restriction_log_id
     )
     configuration.save(path=file_path.joinpath())
+    # noinspection PyTypeChecker
     console.echo(message=f"Project {project} data structure and configuration file: generated.", level=LogLevel.SUCCESS)
 
 
@@ -360,6 +368,7 @@ def generate_experiment_configuration_file(project: str, experiment: str, state_
         raise ValueError(message)  # Fall-back to appease mypy, should not be reachable
 
     experiment_configuration.to_yaml(file_path=file_path)
+    # noinspection PyTypeChecker
     console.echo(message=f"Experiment {experiment} configuration file: generated.", level=LogLevel.SUCCESS)
 
 

sl_shared_assets/cli.pyi

@@ -9,6 +9,7 @@ from .server import generate_server_credentials as generate_server_credentials
 from .data_classes import (
     SessionData as SessionData,
     ExperimentState as ExperimentState,
+    ProcessingTracker as ProcessingTracker,
     ProjectConfiguration as ProjectConfiguration,
     MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
     MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,

sl_shared_assets/data_classes/__init__.py

@@ -15,6 +15,7 @@ from .session_data import (
     RawData,
     SessionData,
     ProcessedData,
+    ProcessingTracker,
     ProjectConfiguration,
 )
 from .surgery_data import (
@@ -63,4 +64,5 @@ __all__ = [
     "MesoscopeCameras",
     "MesoscopeMicroControllers",
     "MesoscopeAdditionalFirmware",
+    "ProcessingTracker",
 ]

sl_shared_assets/data_classes/__init__.pyi

@@ -10,6 +10,7 @@ from .session_data import (
     RawData as RawData,
     SessionData as SessionData,
     ProcessedData as ProcessedData,
+    ProcessingTracker as ProcessingTracker,
     ProjectConfiguration as ProjectConfiguration,
 )
 from .surgery_data import (
@@ -58,4 +59,5 @@ __all__ = [
     "MesoscopeCameras",
     "MesoscopeMicroControllers",
     "MesoscopeAdditionalFirmware",
+    "ProcessingTracker",
 ]

sl_shared_assets/data_classes/session_data.py

@@ -11,6 +11,7 @@ import shutil as sh
 from pathlib import Path
 from dataclasses import field, dataclass
 
+from filelock import Timeout, FileLock
 from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
 from ataraxis_data_structures import YamlConfig
 from ataraxis_time.time_helpers import get_timestamp
@@ -217,7 +218,7 @@ class RawData:
     the long-term storage destinations (NAS and Server) and the integrity of the moved data is verified on at least one 
     destination. During 'purge' sl-experiment runtimes, the library discovers and removes all session data marked with 
     'ubiquitin.bin' files from the machine that runs the code."""
-    verified_bin_path: Path = Path()
+    integrity_verification_tracker_path: Path = Path()
     """Stores the path to the verified.bin file. This marker file is created (or removed) by the sl-shared-assets 
     'verify-session' CLI command to indicate whether the session data inside the folder marked by the file has been 
     verified for integrity. Primarily, this is used when the data is moved to the long-term storage destination (BioHPC
@@ -253,7 +254,7 @@ class RawData:
         self.system_configuration_path = self.raw_data_path.joinpath("system_configuration.yaml")
         self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
         self.ubiquitin_path = self.raw_data_path.joinpath("ubiquitin.bin")
-        self.verified_bin_path = self.raw_data_path.joinpath("verified.bin")
+        self.integrity_verification_tracker_path = self.raw_data_path.joinpath("integrity_verification_tracker.yaml")
 
     def make_directories(self) -> None:
         """Ensures that all major subdirectories and the root directory exist, creating any missing directories."""
@@ -290,20 +291,20 @@ class ProcessedData:
     server-side data processing pipeline runtimes. This directory is primarily used when running data processing jobs 
     on the remote server. However, it is possible to configure local runtimes to also redirect log data to files 
     stored in this directory (by editing ataraxis-base-utilities 'console' variable)."""
-    single_day_suite2p_bin_path: Path = Path()
+    suite2p_processing_tracker_path: Path = Path()
     """Stores the path to the single_day_suite2p.bin file. This file is created by our single-day suite2p data 
     processing pipeline to mark sessions that have been successfully processed with the single-day sl-suite2p library 
     pipeline. Note, the file is removed at the beginning of the suite2p pipeline, so its presence always indicates 
     successful processing runtime completion."""
-    multi_day_suite2p_bin_path: Path = Path()
+    dataset_formation_tracker_path: Path = Path()
     """Same as single_day_suite2p_bin_path, but tracks whether the session has been successfully processed with the 
     multi-day suite2p pipeline."""
-    behavior_bin_path: Path = Path()
+    behavior_processing_tracker_path: Path = Path()
     """Stores the path to the behavior.bin file. This file is created by our behavior data extraction pipeline
     to mark sessions that have been successfully processed with the sl-behavior library pipeline. Note, the 
     file is removed at the beginning of the behavior data extraction pipeline, so its presence always indicates 
     successful processing runtime completion."""
-    dlc_bin_path: Path = Path()
+    video_processing_tracker_path: Path = Path()
     """Stores the path to the dlc.bin file. This file is created by our DeepLabCut-based pose tracking pipeline
     to mark sessions that have been successfully processed with the sl-dlc library pipeline. Note, the 
     file is removed at the beginning of the DeepLabCut pipeline, so its presence always indicates successful processing 
@@ -326,10 +327,10 @@ class ProcessedData:
         self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
         self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
         self.job_logs_path = self.processed_data_path.joinpath("job_logs")
-        self.single_day_suite2p_bin_path = self.processed_data_path.joinpath("single_day_suite2p.bin")
-        self.multi_day_suite2p_bin_path = self.processed_data_path.joinpath("multi_day_suite2p.bin")
-        self.behavior_bin_path = self.processed_data_path.joinpath("behavior.bin")
-        self.dlc_bin_path = self.processed_data_path.joinpath("dlc.bin")
+        self.suite2p_processing_tracker_path = self.processed_data_path.joinpath("suite2p_processing_tracker.yaml")
+        self.dataset_formation_tracker_path = self.processed_data_path.joinpath("dataset_formation_tracker.yaml")
+        self.behavior_processing_tracker_path = self.processed_data_path.joinpath("behavior_processing_tracker.yaml")
+        self.video_processing_tracker_path = self.processed_data_path.joinpath("video_processing_tracker.yaml")
 
     def make_directories(self) -> None:
         """Ensures that all major subdirectories and the root directory exist, creating any missing directories."""
@@ -477,6 +478,7 @@ class SessionData(YamlConfig):
                 f"already exists. The newly created session directory uses a '_{counter}' postfix to distinguish "
                 f"itself from the already existing session directory."
             )
+            # noinspection PyTypeChecker
             console.echo(message=message, level=LogLevel.ERROR)
 
         # Generates subclasses stored inside the main class instance based on the data resolved above.
@@ -490,6 +492,7 @@ class SessionData(YamlConfig):
         processed_data.resolve_paths(root_directory_path=session_path.joinpath("processed_data"))
 
         # Packages the sections generated above into a SessionData instance
+        # noinspection PyArgumentList
         instance = SessionData(
             project_name=project_name,
             animal_id=animal_id,
@@ -587,7 +590,7 @@ class SessionData(YamlConfig):
 
         # Unless a different root is provided for processed data, it uses the same root as raw_data.
         if processed_data_root is None:
-            processed_data_root = new_root
+            processed_data_root = local_root
 
         # Regenerates the processed_data path depending on the root resolution above
         instance.processed_data.resolve_paths(
@@ -623,3 +626,252 @@ class SessionData(YamlConfig):
 
         # Saves instance data as a .YAML file
         origin.to_yaml(file_path=self.raw_data.session_data_path)
+
+
+@dataclass()
+class ProcessingTracker(YamlConfig):
+    """Wraps the .yaml file that tracks the state of a data processing runtime and provides tools for communicating the
+    state between multiple processes in a thread-safe manner.
+
+    Primarily, this tracker class is used by all remote data processing pipelines in the lab to prevent race conditions
+    and make it impossible to run multiple processing runtimes at the same time.
+    """
+
+    file_path: Path
+    """Stores the path to the .yaml file used to save the tracker data between runtimes. The class instance functions as
+    a wrapper around the data stored inside the specified .yaml file."""
+    _is_complete: bool = False
+    """Tracks whether the processing runtime managed by this tracker has been successfully carried out for the session 
+    that calls the tracker."""
+    _encountered_error: bool = False
+    """Tracks whether the processing runtime managed by this tracker has encountered an error while running for the 
+    session that calls the tracker."""
+    _is_running: bool = False
+    """Tracks whether the processing runtime managed by this tracker is currently running for the session that calls 
+    the tracker."""
+    _lock_path: str = field(init=False)
+    """Stores the path to the .lock file for the target tracker .yaml file. This file is used to ensure that only one 
+    process can simultaneously read from or write to the wrapped .yaml file."""
+
+    def __post_init__(self) -> None:
+        # Generates the lock file for the target .yaml file path.
+        if self.file_path is not None:
+            self._lock_path = str(self.file_path.with_suffix(self.file_path.suffix + ".lock"))
+        else:
+            self._lock_path = ""
+
+    def _load_state(self) -> None:
+        """Reads the current processing state from the wrapped .YAML file."""
+        if self.file_path.exists():
+            # Loads the data for the state values, but does not replace the file path or lock attributes.
+            instance: ProcessingTracker = self.from_yaml(self.file_path)  # type: ignore
+            self._is_complete = instance._is_complete
+            self._encountered_error = instance._encountered_error
+            self._is_running = instance._is_running
+        else:
+            # Otherwise, if the tracker file does not exist, generates a new .yaml file using default instance values.
+            self._save_state()
+
+    def _save_state(self) -> None:
+        """Saves the current processing state stored inside instance attributes to the specified .YAML file."""
+        # Resets the _lock and file_path to None before dumping the data to .YAML to avoid issues with loading it
+        # back.
+        console.echo(message=f"{self.file_path}")
+        original = copy.deepcopy(self)
+        original.file_path = None  # type: ignore
+        original._lock_path = None  # type: ignore
+        original.to_yaml(file_path=self.file_path)
+
+    def start(self) -> None:
+        """Configures the tracker file to indicate that the tracked processing runtime is currently running.
+
+        All further attempts to start the same processing runtime for the same session's data will automatically abort
+        with an error.
+
+        Raises:
+            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+        """
+        try:
+            # Acquires the lock
+            lock = FileLock(self._lock_path)
+            with lock.acquire(timeout=10.0):
+                # Loads tracker state from .yaml file
+                self._load_state()
+
+                # If the runtime is already running, aborts with an error
+                if self._is_running:
+                    message = (
+                        f"Unable to start the processing runtime. The {self.file_path.name} tracker file indicates "
+                        f"that the runtime is currently running from a different process. Only a single runtime "
+                        f"instance is allowed to run at the same time."
+                    )
+                    console.error(message=message, error=RuntimeError)
+                    raise RuntimeError(message)  # Fallback to appease mypy, should not be reachable
+
+                # Otherwise, marks the runtime as running and saves the state back to the .yaml file.
+                self._is_running = True
+                self._is_complete = False
+                self._encountered_error = False
+                self._save_state()
+
+        # If lock acquisition fails for any reason, aborts with an error
+        except Timeout:
+            message = (
+                f"Unable to interface with the ProcessingTracker instance data cached inside the target .yaml file "
+                f"{self.file_path.stem}. Specifically, unable to acquire the file lock before the timeout duration of "
+                f"10 minutes has passed."
+            )
+            console.error(message=message, error=Timeout)
+            raise Timeout(message)  # Fallback to appease mypy, should not be reachable
+
+    def error(self) -> None:
+        """Configures the tracker file to indicate that the tracked processing runtime encountered an error and failed
+        to complete.
+
+        This method will only work for an active runtime. When called for an active runtime, it expects the runtime to
+        be aborted with an error after the method returns. It configures the target tracker to allow other processes
+        to restart the runtime at any point after this method returns, so it is UNSAFE to do any further processing
+        from the process that calls this method.
+
+        Raises:
+            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+        """
+
+        try:
+            # Acquires the lock
+            lock = FileLock(self._lock_path)
+            with lock.acquire(timeout=10.0):
+                # Loads tracker state from .yaml file
+                self._load_state()
+
+                # If the runtime is not running, aborts with an error
+                if not self._is_running:
+                    message = (
+                        f"Unable to report that the processing runtime encountered an error. The {self.file_path.name} "
+                        f"tracker file indicates that the runtime is currently NOT running. A runtime has to be "
+                        f"actively running to set the tracker to an error state."
+                    )
+                    console.error(message=message, error=RuntimeError)
+                    raise RuntimeError(message)  # Fallback to appease mypy, should not be reachable
+
+                # Otherwise, indicates that the runtime aborted with an error
+                self._is_running = False
+                self._is_complete = False
+                self._encountered_error = True
+                self._save_state()
+
+        # If lock acquisition fails for any reason, aborts with an error
+        except Timeout:
+            message = (
+                f"Unable to interface with the ProcessingTracker instance data cached inside the target .yaml file "
+                f"{self.file_path.stem}. Specifically, unable to acquire the file lock before the timeout duration of "
+                f"10 minutes has passed."
+            )
+            console.error(message=message, error=Timeout)
+            raise Timeout(message)  # Fallback to appease mypy, should not be reachable
+
+    def stop(self) -> None:
+        """Mark processing as started.
+
+        Raises:
+            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+        """
+
+        try:
+            # Acquires the lock
+            lock = FileLock(self._lock_path)
+            with lock.acquire(timeout=10.0):
+                # Loads tracker state from .yaml file
+                self._load_state()
+
+                # If the runtime is not running, aborts with an error
+                if not self._is_running:
+                    message = (
+                        f"Unable to stop (complete) the processing runtime. The {self.file_path.name} tracker file "
+                        f"indicates that the runtime is currently NOT running. A runtime has to be actively running to "
+                        f"mark it as complete and stop the runtime."
+                    )
+                    console.error(message=message, error=RuntimeError)
+                    raise RuntimeError(message)  # Fallback to appease mypy, should not be reachable
+
+                # Otherwise, marks the runtime as complete (stopped)
+                self._is_running = False
+                self._is_complete = True
+                self._encountered_error = False
+                self._save_state()
+
+        # If lock acquisition fails for any reason, aborts with an error
+        except Timeout:
+            message = (
+                f"Unable to interface with the ProcessingTracker instance data cached inside the target .yaml file "
+                f"{self.file_path.stem}. Specifically, unable to acquire the file lock before the timeout duration of "
+                f"10 minutes has passed."
+            )
+            console.error(message=message, error=Timeout)
+            raise Timeout(message)  # Fallback to appease mypy, should not be reachable
+
+    @property
+    def is_complete(self) -> bool:
+        """Returns True if the tracker wrapped by the instance indicates that the processing runtime has been completed
+        successfully and False otherwise."""
+        try:
+            # Acquires the lock
+            lock = FileLock(self._lock_path)
+            with lock.acquire(timeout=10.0):
+                # Loads tracker state from .yaml file
+                self._load_state()
+                return self._is_complete
+
+            # If lock acquisition fails for any reason, aborts with an error
+        except Timeout:
+            message = (
+                f"Unable to interface with the ProcessingTracker instance data cached inside the target .yaml file "
+                f"{self.file_path.stem}. Specifically, unable to acquire the file lock before the timeout duration of "
+                f"10 minutes has passed."
+            )
+            console.error(message=message, error=Timeout)
+            raise Timeout(message)  # Fallback to appease mypy, should not be reachable
+
+    @property
+    def encountered_error(self) -> bool:
+        """Returns True if the tracker wrapped by the instance indicates that the processing runtime aborted due to
+        encountering an error and False otherwise."""
+        try:
+            # Acquires the lock
+            lock = FileLock(self._lock_path)
+            with lock.acquire(timeout=10.0):
+                # Loads tracker state from .yaml file
+                self._load_state()
+                return self._encountered_error
+
+            # If lock acquisition fails for any reason, aborts with an error
+        except Timeout:
+            message = (
+                f"Unable to interface with the ProcessingTracker instance data cached inside the target .yaml file "
+                f"{self.file_path.stem}. Specifically, unable to acquire the file lock before the timeout duration of "
+                f"10 minutes has passed."
+            )
+            console.error(message=message, error=Timeout)
+            raise Timeout(message)  # Fallback to appease mypy, should not be reachable
+
+    @property
+    def is_running(self) -> bool:
+        """Returns True if the tracker wrapped by the instance indicates that the processing runtime is currently
+        running and False otherwise."""
+        try:
+            # Acquires the lock
+            lock = FileLock(self._lock_path)
+            with lock.acquire(timeout=10.0):
+                # Loads tracker state from .yaml file
+                self._load_state()
+                return self._is_running
+
+            # If lock acquisition fails for any reason, aborts with an error
+        except Timeout:
+            message = (
+                f"Unable to interface with the ProcessingTracker instance data cached inside the target .yaml file "
+                f"{self.file_path.stem}. Specifically, unable to acquire the file lock before the timeout duration of "
+                f"10 minutes has passed."
+            )
+            console.error(message=message, error=Timeout)
+            raise Timeout(message)  # Fallback to appease mypy, should not be reachable

sl_shared_assets/data_classes/session_data.pyi

@@ -96,7 +96,7 @@ class RawData:
     checksum_path: Path = ...
     telomere_path: Path = ...
     ubiquitin_path: Path = ...
-    verified_bin_path: Path = ...
+    integrity_verification_tracker_path: Path = ...
     def resolve_paths(self, root_directory_path: Path) -> None:
         """Resolves all paths managed by the class instance based on the input root directory path.
 
@@ -125,10 +125,10 @@ class ProcessedData:
     mesoscope_data_path: Path = ...
     behavior_data_path: Path = ...
     job_logs_path: Path = ...
-    single_day_suite2p_bin_path: Path = ...
-    multi_day_suite2p_bin_path: Path = ...
-    behavior_bin_path: Path = ...
-    dlc_bin_path: Path = ...
+    suite2p_processing_tracker_path: Path = ...
+    dataset_formation_tracker_path: Path = ...
+    behavior_processing_tracker_path: Path = ...
+    video_processing_tracker_path: Path = ...
     def resolve_paths(self, root_directory_path: Path) -> None:
         """Resolves all paths managed by the class instance based on the input root directory path.
 
@@ -250,3 +250,62 @@ class SessionData(YamlConfig):
         data processing. The method is intended to only be used by the SessionData instance itself during its
         create() method runtime.
         """
+
+@dataclass()
+class ProcessingTracker(YamlConfig):
+    """Wraps the .yaml file that tracks the state of a data processing runtime and provides tools for communicating the
+    state between multiple processes in a thread-safe manner.
+
+    Primarily, this tracker class is used by all remote data processing pipelines in the lab to prevent race conditions
+    and make it impossible to run multiple processing runtimes at the same time.
+    """
+
+    file_path: Path
+    _is_complete: bool = ...
+    _encountered_error: bool = ...
+    _is_running: bool = ...
+    _lock_path: str = field(init=False)
+    def __post_init__(self) -> None: ...
+    def _load_state(self) -> None:
+        """Reads the current processing state from the wrapped .YAML file."""
+    def _save_state(self) -> None:
+        """Saves the current processing state stored inside instance attributes to the specified .YAML file."""
+    def start(self) -> None:
+        """Configures the tracker file to indicate that the tracked processing runtime is currently running.
+
+        All further attempts to start the same processing runtime for the same session's data will automatically abort
+        with an error.
+
+        Raises:
+            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+        """
+    def error(self) -> None:
+        """Configures the tracker file to indicate that the tracked processing runtime encountered an error and failed
+        to complete.
+
+        This method will only work for an active runtime. When called for an active runtime, it expects the runtime to
+        be aborted with an error after the method returns. It configures the target tracker to allow other processes
+        to restart the runtime at any point after this method returns, so it is UNSAFE to do any further processing
+        from the process that calls this method.
+
+        Raises:
+            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+        """
+    def stop(self) -> None:
+        """Mark processing as started.
+
+        Raises:
+            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+        """
+    @property
+    def is_complete(self) -> bool:
+        """Returns True if the tracker wrapped by the instance indicates that the processing runtime has been completed
+        successfully and False otherwise."""
+    @property
+    def encountered_error(self) -> bool:
+        """Returns True if the tracker wrapped by the instance indicates that the processing runtime aborted due to
+        encountering an error and False otherwise."""
+    @property
+    def is_running(self) -> bool:
+        """Returns True if the tracker wrapped by the instance indicates that the processing runtime is currently
+        running and False otherwise."""

sl_shared_assets/server/server.py

@@ -204,6 +204,28 @@ class Server:
         else:
             return False
 
+    def pull_file(self, local_file_path: Path, remote_file_path: Path) -> None:
+        """Moves the specified file from the remote server to the local machine.
+
+        Args:
+            local_file_path: The path to the local instance of the file (where to copy the file).
+            remote_file_path: The path to the target file on the remote server (the file to be copied).
+        """
+        sftp = self._client.open_sftp()
+        sftp.get(localpath=local_file_path, remotepath=str(remote_file_path))
+        sftp.close()
+
+    def push_file(self, local_file_path: Path, remote_file_path: Path) -> None:
+        """Moves the specified file from the remote server to the local machine.
+
+        Args:
+            local_file_path: The path to the file that needs to be copied to the remote server.
+            remote_file_path: The path to the file on the remote server (where to copy the file).
+        """
+        sftp = self._client.open_sftp()
+        sftp.put(localpath=local_file_path, remotepath=str(remote_file_path))
+        sftp.close()
+
     def close(self) -> None:
         """Closes the SSH connection to the server.
 

sl_shared_assets/server/server.pyi

@@ -88,6 +88,20 @@ class Server:
             ValueError: If the input Job object does not contain a valid job_id, suggesting that it has not been
                 submitted to the server.
         """
+    def pull_file(self, local_file_path: Path, remote_file_path: Path) -> None:
+        """Moves the specified file from the remote server to the local machine.
+
+        Args:
+            local_file_path: The path to the local instance of the file (where to copy the file).
+            remote_file_path: The path to the target file on the remote server (the file to be copied).
+        """
+    def push_file(self, local_file_path: Path, remote_file_path: Path) -> None:
+        """Moves the specified file from the remote server to the local machine.
+
+        Args:
+            local_file_path: The path to the file that needs to be copied to the remote server.
+            remote_file_path: The path to the file on the remote server (where to copy the file).
+        """
     def close(self) -> None:
         """Closes the SSH connection to the server.
 

sl_shared_assets/tools/packaging_tools.py

@@ -17,11 +17,16 @@ _excluded_files = {
     "ax_checksum.txt",
     "ubiquitin.bin",
     "telomere.bin",
-    "single_day_suite2p.bin",
-    "multi_day_suite2p.bin",
-    "behavior.bin",
-    "dlc.bin",
-    "verified.bin",
+    "suite2p_processing_tracker.yaml",
+    "dataset_formation_tracker.yaml",
+    "behavior_processing_tracker.yaml",
+    "video_processing_tracker.yaml",
+    "integrity_verification_tracker.yaml",
+    "suite2p_processing_tracker.yaml.lock",
+    "dataset_formation_tracker.yaml.lock",
+    "behavior_processing_tracker.yaml.lock",
+    "video_processing_tracker.yaml.lock",
+    "integrity_verification_tracker.yaml.lock",
 }
 
 

sl_shared_assets/tools/project_management_tools.py

@@ -7,7 +7,7 @@ from pathlib import Path
 import polars as pl
 from ataraxis_base_utilities import console
 
-from ..data_classes import SessionData
+from ..data_classes import SessionData, ProcessingTracker
 from .packaging_tools import calculate_directory_checksum
 
 
@@ -60,12 +60,15 @@ def generate_project_manifest(
         "type": [],  # Type of the session (e.g., Experiment, Training, etc.).
         "raw_data": [],  # Server-side raw_data folder path.
         "processed_data": [],  # Server-side processed_data folder path.
-        "complete": [],  # Determines if the session data is complete. Incomplete sessions are excluded from processing.
-        "verified": [],  # Determines if the session data integrity has been verified upon transfer to storage machine.
-        "single_day_suite2p": [],  # Determines whether the session has been processed with the single-day s2p pipeline.
-        "multi_day_suite2p": [],  # Determines whether the session has been processed with the multi-day s2p pipeline.
-        "behavior": [],  # Determines whether the session has been processed with the behavior extraction pipeline.
-        "dlc": [],  # Determines whether the session has been processed with the DeepLabCut pipeline.
+        # Determines whether the session data is complete. Incomplete sessions are excluded from processing.
+        "complete": [],
+        # Determines whether the session data integrity has been verified upon transfer to storage machine.
+        "integrity_verification": [],
+        "suite2p_processing": [],  # Determines whether the session has been processed with the single-day s2p pipeline.
+        "dataset_formation": [],  # Determines whether the session's data has been integrated into a dataset.
+        # Determines whether the session has been processed with the behavior extraction pipeline.
+        "behavior_processing": [],
+        "video_processing": [],  # Determines whether the session has been processed with the DeepLabCut pipeline.
     }
 
     # Loops over each session of every animal in the project and extracts session ID information and information
@@ -88,34 +91,35 @@ def generate_project_manifest(
         # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
         manifest["complete"].append(session_data.raw_data.telomere_path.exists())
 
-        # If the session raw_data folder contains the verified.bin file, marks the session as verified.
-        manifest["verified"].append(session_data.raw_data.verified_bin_path.exists())
+        # Data verification status
+        tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
+        manifest["integrity_verification"].append(tracker.is_complete)
 
         # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing is
         # disabled for incomplete sessions. If the session unverified, the case is even more severe, as its data may be
         # corrupted.
         if not manifest["complete"][-1] or not not manifest["verified"][-1]:
-            manifest["single_day_suite2p"].append(False)
-            manifest["multi_day_suite2p"].append(False)
-            manifest["behavior"].append(False)
-            manifest["dlc"].append(False)
+            manifest["suite2p_processing"].append(False)
+            manifest["dataset_formation"].append(False)
+            manifest["behavior_processing"].append(False)
+            manifest["video_processing"].append(False)
             continue  # Cycles to the next session
 
-        # If the session processed_data folder contains the single-day suite2p.bin file, marks the single-day suite2p
-        # processing step as complete.
-        manifest["single_day_suite2p"].append(session_data.processed_data.single_day_suite2p_bin_path.exists())
+        # Suite2p (single-day) status
+        tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
+        manifest["suite2p_processing"].append(tracker.is_complete)
 
-        # If the session processed_data folder contains the multi-day suite2p.bin file, marks the multi-day suite2p
-        # processing step as complete.
-        manifest["multi_day_suite2p"].append(session_data.processed_data.multi_day_suite2p_bin_path.exists())
+        # Dataset formation (integration) status. Tracks whether the session has been added to any dataset(s).
+        tracker = ProcessingTracker(file_path=session_data.processed_data.dataset_formation_tracker_path)
+        manifest["dataset_formation"].append(tracker.is_complete)
 
-        # If the session processed_data folder contains the behavior.bin file, marks the behavior processing step as
-        # complete.
-        manifest["behavior"].append(session_data.processed_data.behavior_data_path.exists())
+        # Dataset formation (integration) status. Tracks whether the session has been added to any dataset(s).
+        tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
+        manifest["behavior_processing"].append(tracker.is_complete)
 
-        # If the session processed_data folder contains the dlc.bin file, marks the dlc processing step as
-        # complete.
-        manifest["dlc"].append(session_data.processed_data.dlc_bin_path.exists())
+        # DeepLabCut (video) processing status.
+        tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
+        manifest["video_processing"].append(tracker.is_complete)
 
     # Converts the manifest dictionary to a Polars Dataframe
     schema = {
@@ -125,11 +129,11 @@ def generate_project_manifest(
         "processed_data": pl.String,
         "type": pl.String,
         "complete": pl.Boolean,
-        "verified": pl.Boolean,
-        "single_day_suite2p": pl.Boolean,
-        "multi_day_suite2p": pl.Boolean,
-        "behavior": pl.Boolean,
-        "dlc": pl.Boolean,
+        "integrity_verification": pl.Boolean,
+        "suite2p_processing": pl.Boolean,
+        "dataset_formation": pl.Boolean,
+        "behavior_processing": pl.Boolean,
+        "video_processing": pl.Boolean,
     }
     df = pl.DataFrame(manifest, schema=schema)
 
@@ -176,26 +180,38 @@ def verify_session_checksum(
         make_processed_data_directory=create_processed_data_directory,
     )
 
-    # Unlinks the verified.bin marker if it exists. The presence or absence of the marker is used as the
-    # primary heuristic for determining if the session data passed verification. Unlinking it early helps in the case
-    # the verification procedure aborts unexpectedly for any reason.
-    session_data.raw_data.verified_bin_path.unlink(missing_ok=True)
+    # Initializes the ProcessingTracker instance for the verification tracker file
+    tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
 
-    # Re-calculates the checksum for the raw_data directory
-    calculated_checksum = calculate_directory_checksum(
-        directory=session_data.raw_data.raw_data_path, batch=False, save_checksum=False
-    )
-
-    # Loads the checksum stored inside the ax_checksum.txt file
-    with open(session_data.raw_data.checksum_path, "r") as f:
-        stored_checksum = f.read().strip()
+    # Updates the tracker data to communicate that the verification process has started. This automatically clears
+    # the previous 'completed' status.
+    tracker.start()
 
-    # If the two checksums do not match, this likely indicates data corruption.
-    if stored_checksum != calculated_checksum:
-        # If the telomere.bin file exists, removes this file. This automatically marks the session as incomplete for
-        # all other Sun lab runtimes.
-        session_data.raw_data.telomere_path.unlink(missing_ok=True)
+    # Try starts here to allow for proper error-driven 'start' terminations of the tracker cannot acquire the lock for
+    # a long time, or if another runtime is already underway.
+    try:
+        # Re-calculates the checksum for the raw_data directory
+        calculated_checksum = calculate_directory_checksum(
+            directory=session_data.raw_data.raw_data_path, batch=False, save_checksum=False
+        )
 
-    # Otherwise, ensures that the session is marked with the verified.bin marker file.
-    else:
-        session_data.raw_data.verified_bin_path.touch(exist_ok=True)
+        # Loads the checksum stored inside the ax_checksum.txt file
+        with open(session_data.raw_data.checksum_path, "r") as f:
+            stored_checksum = f.read().strip()
+
+        # If the two checksums do not match, this likely indicates data corruption.
+        if stored_checksum != calculated_checksum:
+            # If the telomere.bin file exists, removes this file. This automatically marks the session as incomplete for
+            # all other Sun lab runtimes.
+            session_data.raw_data.telomere_path.unlink(missing_ok=True)
+
+        else:
+            # Sets the tracker to indicate that the verification runtime completed successfully.
+            tracker.stop()
+
+    finally:
+        # If the code reaches this section while the tracker indicates that the processing is still running,
+        # this means that the verification runtime encountered an error. Configures the tracker to indicate that this
+        # runtime finished with an error to prevent deadlocking the runtime.
+        if tracker.is_running:
+            tracker.error()

sl_shared_assets/tools/project_management_tools.pyi

@@ -1,6 +1,9 @@
 from pathlib import Path
 
-from ..data_classes import SessionData as SessionData
+from ..data_classes import (
+    SessionData as SessionData,
+    ProcessingTracker as ProcessingTracker,
+)
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 
 def generate_project_manifest(

{sl_shared_assets-1.0.1.dist-info → sl_shared_assets-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.1
+Version: 1.1.0
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -685,7 +685,7 @@ Keywords: acquisition,assets,data,processing,sunlab
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
@@ -695,6 +695,7 @@ Requires-Dist: ataraxis-base-utilities<4,>=3
 Requires-Dist: ataraxis-data-structures<4,>=3.1.1
 Requires-Dist: ataraxis-time<4,>=3
 Requires-Dist: click<9,>=8
+Requires-Dist: filelock<4,>=3
 Requires-Dist: natsort<9,>=8
 Requires-Dist: paramiko<4,>=3.5.1
 Requires-Dist: polars<2,>=1
@@ -714,18 +715,20 @@ Requires-Dist: sphinx-rtd-theme<4,>=3; extra == 'conda'
 Requires-Dist: sphinx<9,>=8; extra == 'conda'
 Requires-Dist: twine<7,>=6; extra == 'conda'
 Requires-Dist: types-appdirs<2,>=1; extra == 'conda'
+Requires-Dist: types-filelock<4,>=3; extra == 'conda'
 Requires-Dist: types-paramiko<4,>=3; extra == 'conda'
 Requires-Dist: types-tqdm<5,>=4; extra == 'conda'
 Provides-Extra: condarun
 Requires-Dist: appdirs<2,>=1; extra == 'condarun'
 Requires-Dist: click<9,>=8; extra == 'condarun'
+Requires-Dist: filelock<4,>=3; extra == 'condarun'
 Requires-Dist: natsort<9,>=8; extra == 'condarun'
 Requires-Dist: paramiko<4,>=3.5.1; extra == 'condarun'
 Requires-Dist: polars<2,>=1; extra == 'condarun'
 Requires-Dist: pyarrow<21,>=20; extra == 'condarun'
 Requires-Dist: tqdm<5,>=4; extra == 'condarun'
 Provides-Extra: dev
-Requires-Dist: ataraxis-automation<5,>=4; extra == 'dev'
+Requires-Dist: ataraxis-automation<6,>=5; extra == 'dev'
 Requires-Dist: build<2,>=1; extra == 'dev'
 Requires-Dist: grayskull<3,>=2; extra == 'dev'
 Requires-Dist: hatchling<2,>=1; extra == 'dev'
@@ -741,11 +744,12 @@ Requires-Dist: tox-uv<2,>=1; extra == 'dev'
 Requires-Dist: tox<5,>=4; extra == 'dev'
 Requires-Dist: twine<7,>=6; extra == 'dev'
 Requires-Dist: types-appdirs<2,>=1; extra == 'dev'
+Requires-Dist: types-filelock<4,>=3; extra == 'dev'
 Requires-Dist: types-paramiko<4,>=3; extra == 'dev'
 Requires-Dist: types-tqdm<5,>=4; extra == 'dev'
 Requires-Dist: uv<1,>=0; extra == 'dev'
 Provides-Extra: noconda
-Requires-Dist: ataraxis-automation<5,>=4; extra == 'noconda'
+Requires-Dist: ataraxis-automation<6,>=5; extra == 'noconda'
 Requires-Dist: build<2,>=1; extra == 'noconda'
 Requires-Dist: sphinx-rtd-dark-mode<2,>=1; extra == 'noconda'
 Requires-Dist: tox-uv<2,>=1; extra == 'noconda'

{sl_shared_assets-1.0.1.dist-info → sl_shared_assets-1.1.0.dist-info}/RECORD

@@ -1,36 +1,36 @@
 sl_shared_assets/__init__.py,sha256=rscR353jiyUQ-wpguTLOM-A5Lqr1ftQtuTan1D0AtR0,2196
 sl_shared_assets/__init__.pyi,sha256=Ye6eY_y_l9CTqwbCf-OxXOphxXKfn9UJnuw09DdLEtc,2405
-sl_shared_assets/cli.py,sha256=9PXEfRgKb9-6xb1icmtr_ijdWur-AIsdMwvgA0aAeto,17434
-sl_shared_assets/cli.pyi,sha256=p1tUflODySXx1_2IrLZ_uqixopbQPTearuJvqJ4jKI4,6075
+sl_shared_assets/cli.py,sha256=2HAgnD7hHnFp3R7_tJAfWBI_jRbhSuyDBFK3TGIHYsw,17771
+sl_shared_assets/cli.pyi,sha256=Fh8GZBSQzII_Iz6k5nLQOsVMbp7q1R5mp4KNZjdGflY,6119
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sl_shared_assets/data_classes/__init__.py,sha256=b0BwTAJCD1zbwjd2UdpXheq61q1sgBuYhtAL-GHA2h4,1835
-sl_shared_assets/data_classes/__init__.pyi,sha256=e2082sm5pSw3bxNZGqwcRhuKLH8T7hcEo6kWtOZNDBU,1968
+sl_shared_assets/data_classes/__init__.py,sha256=S1-3mYop1MhI7HRusPE5sfu30hmIgH9BfmB5-r6Bbm8,1883
+sl_shared_assets/data_classes/__init__.pyi,sha256=jB5Y9QQ-XL_auk7Ot0DA87lIbfn1j3AuWOk2LcOyxvk,2037
 sl_shared_assets/data_classes/configuration_data.py,sha256=ZVk1ynk25CfVuQfMofoH90BUaOPqk7zW8ukY6ls_Pp0,30360
 sl_shared_assets/data_classes/configuration_data.pyi,sha256=h7AV3z73SC2ITXWcnsShczuezC1Is7L4WSMnEWGSLPQ,9617
 sl_shared_assets/data_classes/runtime_data.py,sha256=Q7Ykf9hgrw1jYKXa53mn_LW8G2cPmLLuxgGkP6qQcc4,15483
 sl_shared_assets/data_classes/runtime_data.pyi,sha256=PxaCbeF9COR4ri91pdzh7zVrqaz2KEDYB1EoLhZQC_c,6618
-sl_shared_assets/data_classes/session_data.py,sha256=oAiTNqNlQi6NQHv5WsaQb3KglBrDGAmBv1EVPPuM6Y4,38329
-sl_shared_assets/data_classes/session_data.pyi,sha256=eLemAagVD6TQ-9pzMIc0eXDgnh-quJwEidKHYitvsjQ,13243
+sl_shared_assets/data_classes/session_data.py,sha256=RWqHHaJU1pnkxVXCuFjAYbWN50HEO7oEjEsT9mmKQic,51112
+sl_shared_assets/data_classes/session_data.pyi,sha256=mVy1UJyZE6sh-hV-qFDyw8UiWGA_UCYNR2iCNbNeh4o,16190
 sl_shared_assets/data_classes/surgery_data.py,sha256=qsMj3NkjhylAT9b_wHBY-1XwTu2xsZcZatdECmkA7Bs,7437
 sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
 sl_shared_assets/server/__init__.py,sha256=nyX6-9ACcrQeRQOCNvBVrWSTHGjRPANIG_u0aq7HPTg,426
 sl_shared_assets/server/__init__.pyi,sha256=7o99f8uf6NuBjMZjNAM1FX69Qbu5uBluRSAyaUWbXOU,263
 sl_shared_assets/server/job.py,sha256=GB31yYPEqXR6MgwNmnQrSQuHRJqUHFXcd6p7hb38q_c,7928
 sl_shared_assets/server/job.pyi,sha256=cxgHMpuwHsJGf_ZcTSSa2tZNzeR_GxqlICOsYGV_oy0,5655
-sl_shared_assets/server/server.py,sha256=SkFJA8eYlZrm2dag7sF_CyPiBphFgoq32zV10kqnedc,9414
-sl_shared_assets/server/server.pyi,sha256=SoTRqSPJ1GQ5i5gV3N57rDT9xzeT47RjMRH1FYKsOkg,4143
+sl_shared_assets/server/server.py,sha256=KQUxLFp2TKn2N9FoMLqk56rkxOKEhMEO_AeyI0fDAwE,10466
+sl_shared_assets/server/server.pyi,sha256=UAt0NksgEtreJ8pAGOmicvozSa0Szj6nlW4GTbroExk,4915
 sl_shared_assets/tools/__init__.py,sha256=N95ZPMz-_HdNPrbVieCFza-QSVS6BV2KRB4K1OLRttc,636
 sl_shared_assets/tools/__init__.pyi,sha256=xeDF8itMc0JRgLrO_IN_9gW7cp_Ld-Gf-rjtrgWvQ2I,551
 sl_shared_assets/tools/ascension_tools.py,sha256=kIqYGX9F8lRao_LaVOacIiT8J9SypTvarb9mgaI9ZPs,15387
 sl_shared_assets/tools/ascension_tools.pyi,sha256=tQCDdWZ20ZjUjpMs8aGIN0yBg5ff3j6spi62b3Han4o,3755
-sl_shared_assets/tools/packaging_tools.py,sha256=2yaIysL11g5m-j0i4I2MRVzHA6_37WdyLYqXMl2FPAc,7289
+sl_shared_assets/tools/packaging_tools.py,sha256=c9U0bKB6Btj7sfyeU7xx2Jiqv930qTnXbm0ZbNR-o2k,7594
 sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=aMOCq3nCR2xOTDMQpXbXEh90IuKOvSbIfeKh2VaZIXw,11215
-sl_shared_assets/tools/project_management_tools.pyi,sha256=HiU1qqumscmXtAXNX6eaSrkwanGHEvk-lromwZ2ijD8,3445
+sl_shared_assets/tools/project_management_tools.py,sha256=DgMKd6i3iLG4lwVgcCgQeO8jZEfVoWFKU6882JrDvL4,11993
+sl_shared_assets/tools/project_management_tools.pyi,sha256=f_3O8UjnfHRMEe2iZpQxKK9Vb0_lJB2yI1WcJPUqGEU,3498
 sl_shared_assets/tools/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
 sl_shared_assets/tools/transfer_tools.pyi,sha256=FoH7eYZe7guGHfPr0MK5ggO62uXKwD2aJ7h1Bu7PaEE,3294
-sl_shared_assets-1.0.1.dist-info/METADATA,sha256=eGEvjbrv_aOueU-LBq4Z-fC6A_PEDK7pTkiilWZTaOw,48890
-sl_shared_assets-1.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-1.0.1.dist-info/entry_points.txt,sha256=76c00fRS4IuXBP2xOBdvycT15Zen-lHiDg2FaSt-HB4,547
-sl_shared_assets-1.0.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-1.0.1.dist-info/RECORD,,
+sl_shared_assets-1.1.0.dist-info/METADATA,sha256=VlvFc5wSDMdDrxHRtY6SRXe_cni75TDXJ4LQJqNEPL0,49077
+sl_shared_assets-1.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-1.1.0.dist-info/entry_points.txt,sha256=76c00fRS4IuXBP2xOBdvycT15Zen-lHiDg2FaSt-HB4,547
+sl_shared_assets-1.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-1.1.0.dist-info/RECORD,,