sl-shared-assets 3.1.2__py3-none-any.whl → 3.1.3__py3-none-any.whl

sl_shared_assets/cli.py

@@ -26,7 +26,7 @@ from .data_classes import SessionData, ProcessingTracker
     default=False,
     help=(
         "Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes. "
-        "Primarily, it is used by lab acquisition system code to generate processed data directories on the remote "
+        "Primarily, it is used by acquisition systems to generate processed data directories on the remote "
         "compute servers as part of the data preprocessing pipeline."
     ),
 )
@@ -105,18 +105,21 @@ def verify_session_integrity(
     help="The absolute path to the directory where to store the generated project manifest file.",
 )
 @click.option(
-    "-ppp",
-    "--project_processed_path",
+    "-pdr",
+    "--processed_data_root",
     type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
     required=False,
     help=(
-        "The absolute path to the project directory where processed session data is stored, if different from the "
-        "directory used to store raw session data. Typically, this extra argument is only used when processing data "
-        "stored on remote compute server(s)."
+        "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
+        "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
+        "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
+        "automatically modified to include the project name. Note, if the system cannot properly resolve the path to "
+        "the processed data, the generated manifest will indicate that no data processing has been performed for the "
+        "project."
     ),
 )
 def generate_project_manifest_file(
-    project_path: Path, output_directory: Path, project_processed_path: Path | None
+    project_path: Path, output_directory: Path, processed_data_root: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -128,7 +131,7 @@ def generate_project_manifest_file(
     generate_project_manifest(
         raw_project_directory=Path(project_path),
         output_directory=Path(output_directory),
-        processed_data_root=Path(project_processed_path) if project_processed_path else None,
+        processed_data_root=Path(processed_data_root) if processed_data_root else None,
     )
     # noinspection PyTypeChecker
     console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)
@@ -428,14 +431,15 @@ def start_jupyter_server(
     help="Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes.",
 )
 @click.option(
-    "-ppp",
-    "--project_processed_path",
+    "-pdr",
+    "--processed_data_root",
     type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
     required=False,
     help=(
-        "The absolute path to the project directory where processed session data is stored, if different from the "
-        "directory used to store raw session data. Typically, this extra argument is only used when processing data "
-        "stored on remote compute server(s)."
+        "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
+        "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
+        "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
+        "automatically modified to include the project name."
     ),
 )
 @click.option(
@@ -463,7 +467,7 @@ def start_jupyter_server(
 def resolve_dataset_marker(
     session_path: Path,
     create_processed_directories: bool,
-    project_processed_path: Path | None,
+    processed_data_root: Path | None,
     remove: bool,
     update_manifest: bool,
 ) -> None:
@@ -477,7 +481,7 @@ def resolve_dataset_marker(
     resolve_p53_marker(
         session_path=session_path,
         create_processed_data_directory=create_processed_directories,
-        processed_data_root=project_processed_path,
+        processed_data_root=processed_data_root,
         remove=remove,
         update_manifest=update_manifest,
     )

sl_shared_assets/cli.pyi

@@ -33,7 +33,7 @@ def verify_session_integrity(
     """
 
 def generate_project_manifest_file(
-    project_path: Path, output_directory: Path, project_processed_path: Path | None
+    project_path: Path, output_directory: Path, processed_data_root: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -88,7 +88,7 @@ def start_jupyter_server(
 def resolve_dataset_marker(
     session_path: Path,
     create_processed_directories: bool,
-    project_processed_path: Path | None,
+    processed_data_root: Path | None,
     remove: bool,
     update_manifest: bool,
 ) -> None:

sl_shared_assets/data_classes/session_data.py

@@ -264,8 +264,8 @@ class SessionData(YamlConfig):
     Notes:
         This class is specifically designed for working with the data from a single session, performed by a single
         animal under the specific experiment. The class is used to manage both raw and processed data. It follows the
-        data through acquisition, preprocessing and processing stages of the Sun lab data workflow. This class serves as
-        an entry point for all interactions with the managed session's data.
+        data through acquisition, preprocessing, and processing stages of the Sun lab data workflow. This class serves
+        as an entry point for all interactions with the managed session's data.
     """
 
     project_name: str

sl_shared_assets/data_classes/session_data.pyi

@@ -125,8 +125,8 @@ class SessionData(YamlConfig):
     Notes:
         This class is specifically designed for working with the data from a single session, performed by a single
         animal under the specific experiment. The class is used to manage both raw and processed data. It follows the
-        data through acquisition, preprocessing and processing stages of the Sun lab data workflow. This class serves as
-        an entry point for all interactions with the managed session's data.
+        data through acquisition, preprocessing, and processing stages of the Sun lab data workflow. This class serves
+        as an entry point for all interactions with the managed session's data.
     """
 
     project_name: str

sl_shared_assets/server/server.py

@@ -552,22 +552,59 @@ class Server:
         finally:
             sftp.close()
 
-    def remove(self, remote_path: Path, is_dir: bool) -> None:
+    def remove(self, remote_path: Path, is_dir: bool, recursive: bool = False) -> None:
         """Removes the specified file or directory from the remote server.
 
         Args:
             remote_path: The path to the file or directory on the remote server to be removed.
             is_dir: Determines whether the input path represents a directory or a file.
+            recursive: If True and is_dir is True, recursively deletes all contents of the directory
+                before removing it. If False, only removes empty directories (standard rmdir behavior).
         """
         sftp = self._client.open_sftp()
         try:
             if is_dir:
-                sftp.rmdir(path=str(remote_path))
+                if recursive:
+                    # Recursively deletes all contents first and then removes the top-level (now empty) directory
+                    self._recursive_remove(sftp, remote_path)
+                else:
+                    # Only removes empty directories
+                    sftp.rmdir(path=str(remote_path))
             else:
                 sftp.unlink(path=str(remote_path))
         finally:
             sftp.close()
 
+    def _recursive_remove(self, sftp: paramiko.SFTPClient, remote_path: Path) -> None:
+        """Recursively removes a directory and all its contents.
+
+        This worker method is used by the user-facing remove() method to recursively remove non-empty directories.
+
+        Args:
+            sftp: The SFTP client instance to use for remove operations.
+            remote_path: The path to the remote directory to recursively remove.
+        """
+        try:
+            # Lists all items in the directory
+            items = sftp.listdir_attr(str(remote_path))
+
+            for item in items:
+                item_path = remote_path / item.filename
+
+                # Checks if the item is a directory
+                if stat.S_ISDIR(item.st_mode):  # type: ignore
+                    # Recursively removes subdirectories
+                    self._recursive_remove(sftp, item_path)
+                else:
+                    # Recursively removes files
+                    sftp.unlink(str(item_path))
+
+            # After all contents are removed, removes the empty directory
+            sftp.rmdir(str(remote_path))
+
+        except Exception as e:
+            console.echo(f"Unable to remove the specified directory {remote_path}: {str(e)}", level=LogLevel.WARNING)
+
     def create_directory(self, remote_path: Path, parents: bool = True) -> None:
         """Creates the specified directory tree on the managed remote server via SFTP.
 

sl_shared_assets/server/server.pyi

@@ -1,6 +1,7 @@
 from pathlib import Path
 from dataclasses import field, dataclass
 
+import paramiko
 from _typeshed import Incomplete
 from simple_slurm import Slurm as Slurm
 from paramiko.client import SSHClient as SSHClient
@@ -230,12 +231,23 @@ class Server:
             local_directory_path: The path to the local directory to be uploaded.
             remote_directory_path: The path on the remote server where the directory will be copied.
         """
-    def remove(self, remote_path: Path, is_dir: bool) -> None:
+    def remove(self, remote_path: Path, is_dir: bool, recursive: bool = False) -> None:
         """Removes the specified file or directory from the remote server.
 
         Args:
             remote_path: The path to the file or directory on the remote server to be removed.
             is_dir: Determines whether the input path represents a directory or a file.
+            recursive: If True and is_dir is True, recursively deletes all contents of the directory
+                before removing it. If False, only removes empty directories (standard rmdir behavior).
+        """
+    def _recursive_remove(self, sftp: paramiko.SFTPClient, remote_path: Path) -> None:
+        """Recursively removes a directory and all its contents.
+
+        This worker method is used by the user-facing remove() method to recursively remove non-empty directories.
+
+        Args:
+            sftp: The SFTP client instance to use for remove operations.
+            remote_path: The path to the remote directory to recursively remove.
         """
     def create_directory(self, remote_path: Path, parents: bool = True) -> None:
         """Creates the specified directory tree on the managed remote server via SFTP.

sl_shared_assets/tools/project_management_tools.py

@@ -8,7 +8,7 @@ from datetime import datetime
 import pytz
 import polars as pl
 from filelock import FileLock
-from ataraxis_base_utilities import console
+from ataraxis_base_utilities import LogLevel, console
 
 from ..data_classes import (
     SessionData,
@@ -362,6 +362,8 @@ def generate_project_manifest(
                     manifest["notes"].append(descriptor.experimenter_notes)
                 except Exception:
                     manifest["notes"].append("N/A")
+            else:
+                manifest["notes"].append("N/A")
 
             # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
             manifest["complete"].append(session_data.raw_data.telomere_path.exists())
@@ -573,12 +575,27 @@ def resolve_p53_marker(
     if session_data.processed_data.p53_path.exists():
         if remove:
             session_data.processed_data.p53_path.unlink()
+            message = (
+                f"Dataset marker for the session {session_data.session_name} acquired for the animal "
+                f"{session_data.animal_id} and {session_data.project_name} project: Removed."
+            )
+            console.echo(message=message, level=LogLevel.SUCCESS)
             return  # Ends remove runtime
 
+        message = (
+            f"Dataset marker for the session {session_data.session_name} acquired for the animal "
+            f"{session_data.animal_id} and {session_data.project_name} project: Already exists. No actions taken."
+        )
+        console.echo(message=message, level=LogLevel.SUCCESS)
         return  # Ends create runtime
 
     # If the marker does not exist and the function is called in 'remove' mode, aborts the runtime
     elif remove:
+        message = (
+            f"Dataset marker for the session {session_data.session_name} acquired for the animal "
+            f"{session_data.animal_id} and {session_data.project_name} project: Does not exist. No actions taken."
+        )
+        console.echo(message=message, level=LogLevel.SUCCESS)
         return  # Ends remove runtime
 
     # The rest of the runtime deals with determining whether it is safe to create the marker file.
@@ -588,6 +605,13 @@ def resolve_p53_marker(
     # Window checking sessions are not designed to be integrated into datasets, so they cannot be marked with the
     # p53.bin file. Similarly, any incomplete session is automatically excluded from dataset formation.
     if session_type == SessionTypes.WINDOW_CHECKING or not session_data.raw_data.telomere_path.exists():
+        message = (
+            f"Unable to generate the dataset marker for the session {session_data.session_name} acquired for the "
+            f"animal {session_data.animal_id} and {session_data.project_name} project, as the session is incomplete or "
+            f"is of Window Checking type. These sessions must be manually evaluated and marked for dataset inclusion "
+            f"by the experimenter. "
+        )
+        console.echo(message=message, level=LogLevel.ERROR)
         return
 
     # Training sessions collect similar data and share processing pipeline requirements
@@ -597,6 +621,13 @@ def resolve_p53_marker(
         video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
         if behavior_tracker.is_running or video_tracker.is_running:
             # Note, training runtimes do not require suite2p processing.
+            message = (
+                f"Unable to generate the dataset marker for the session {session_data.session_name} acquired for the "
+                f"animal {session_data.animal_id} and {session_data.project_name} project, as it is currently being "
+                f"processed by one of the data processing pipelines. Wait until the session is fully processed by all "
+                f"pipelines and repeat the command that encountered this error."
+            )
+            console.echo(message=message, level=LogLevel.ERROR)
             return
 
     # Mesoscope experiment sessions require additional processing with suite2p
@@ -607,11 +638,23 @@ def resolve_p53_marker(
 
         # Similar to the above, ensures that the session is not being processed with one of the supported pipelines.
         if behavior_tracker.is_running or suite2p_tracker.is_running or video_tracker.is_running:
+            message = (
+                f"Unable to generate the dataset marker for the session {session_data.session_name} acquired for the "
+                f"animal {session_data.animal_id} and {session_data.project_name} project, as it is currently being "
+                f"processed by one of the data processing pipelines. Wait until the session is fully processed by all "
+                f"pipelines and repeat the command that encountered this error."
+            )
+            console.echo(message=message, level=LogLevel.ERROR)
             return
 
     # If the runtime reached this point, the session is eligible for dataset integration. Creates the p53.bin marker
     # file, preventing the session from being processed again as long as the marker exists.
     session_data.processed_data.p53_path.touch()
+    message = (
+        f"Dataset marker for the session {session_data.session_name} acquired for the animal "
+        f"{session_data.animal_id} and {session_data.project_name} project: Created."
+    )
+    console.echo(message=message, level=LogLevel.SUCCESS)
 
     # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
     # existing manifest file for the target project.

{sl_shared_assets-3.1.2.dist-info → sl_shared_assets-3.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 3.1.2
+Version: 3.1.3
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -700,7 +700,7 @@ Requires-Dist: natsort==8.4.0
 Requires-Dist: numpy==2.2.6
 Requires-Dist: paramiko==3.5.1
 Requires-Dist: polars==1.31.0
-Requires-Dist: pyarrow==20.0.0
+Requires-Dist: pyarrow==21.0.0
 Requires-Dist: pytz==2025.2
 Requires-Dist: simple-slurm==0.3.6
 Requires-Dist: tqdm==4.67.1

{sl_shared_assets-3.1.2.dist-info → sl_shared_assets-3.1.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 sl_shared_assets/__init__.py,sha256=ybThh0XDtijjwahKkSEnnQ44rxrN2SVyjB5dHaXts0E,2391
 sl_shared_assets/__init__.pyi,sha256=Cb-umRqvnynk2udbgqAJ6h5_tiJyvVtWmx0kLKrL2Yg,2678
-sl_shared_assets/cli.py,sha256=ERSU9iTLZxL0eIseQA-5d7eTBjlmfjHhEozWTg0dkJA,19079
-sl_shared_assets/cli.pyi,sha256=5hEbOnYaH4q5qdqJ-zhM9-ElzgcaBeMAX34tuHaUDos,5328
+sl_shared_assets/cli.py,sha256=w7FJW93LkhWbLckonFKj8DFN4yAJLPxPLe0kaMBqFMk,19513
+sl_shared_assets/cli.pyi,sha256=-2-rAGqkF3-EeLLijKsp1sWNxEhZ2zVmaAO4VdvGW6k,5322
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sl_shared_assets/data_classes/__init__.py,sha256=mP__bBIIjMf0EETM4PgQzKy1ZKsjp6paRPNDWWbPRV4,1962
 sl_shared_assets/data_classes/__init__.pyi,sha256=J7ZCH9qQ4qz-3Wq9ILdihlmK9zFR3iU1cpLcSaN45Y8,2238
@@ -9,28 +9,28 @@ sl_shared_assets/data_classes/configuration_data.py,sha256=npMYu9WQMJnoanzC2_5rh
 sl_shared_assets/data_classes/configuration_data.pyi,sha256=1_kmBDPGkHmVwXEGYR_3uERBSsenQOTuquMBtjKVTA8,11068
 sl_shared_assets/data_classes/runtime_data.py,sha256=kmXTUk5rDAJBN3XrYLYgusJRfVJ5WiBBk0RPNiSk2pE,16725
 sl_shared_assets/data_classes/runtime_data.pyi,sha256=Hyc-dBePM0xIGgkSIoKmwwUUmdOokm1LUwy1OHHalyU,6771
-sl_shared_assets/data_classes/session_data.py,sha256=YKXako1sNB87LDkGEXx9WZFs6lG3aD619lga5g4L4Ks,49172
-sl_shared_assets/data_classes/session_data.pyi,sha256=CgB4nIDBl4bY1JvcIILfFTlos3ukl3WK4AOaku4CL3Y,15959
+sl_shared_assets/data_classes/session_data.py,sha256=lni8TAAxkFukdRSkMSfcrLYgfYcKpPfMAqmM8EDGVYs,49173
+sl_shared_assets/data_classes/session_data.pyi,sha256=5pk7PtZLooj4zmwLf1MImf5WfTIyTURgQVM9rRNoLJA,15960
 sl_shared_assets/data_classes/surgery_data.py,sha256=5B1OPKFq4bnzbAoe-_c5dFV3kbSD5YFzXbX2zXmfGs8,7485
 sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
 sl_shared_assets/server/__init__.py,sha256=GOQ7wWjiS5Xg_WgTqeEqCTRF9ms9GXx0nffCr-BmKsA,453
 sl_shared_assets/server/__init__.pyi,sha256=Zc12G90fZdgEMwaVZbFzrRVV1wH_LEj3sxaV3lhk1Cw,316
 sl_shared_assets/server/job.py,sha256=wZbppMrv6fqch79bKLjOGQ9AYfjiDKDnTyUe7xgAT44,19461
 sl_shared_assets/server/job.pyi,sha256=wop4ulVY2u6eb3twajeA9MS0EAtNb89aA56pPoGF1Xc,11673
-sl_shared_assets/server/server.py,sha256=oEwdXisyel72Hdk7ZpEwTPq3Lu64UbQWfGHArV8Y6nI,32978
-sl_shared_assets/server/server.pyi,sha256=84XFtqU9fYbxu6Ldf-OMB2nFe6wdGneZM1MFtR9rz4s,15133
+sl_shared_assets/server/server.py,sha256=AuoPy4LUPG19oYOnQ61z8s4S3TSXix_DjF7eTtVjDSY,34764
+sl_shared_assets/server/server.pyi,sha256=K42zwzaZk13LH964fex3sK2kCL_Ulis_GualZkZG-NA,15828
 sl_shared_assets/tools/__init__.py,sha256=i-oUVw_un3lzyyII4Sc75s4BnUfZh_aUbQe6dP2Vrbc,743
 sl_shared_assets/tools/__init__.pyi,sha256=pi-5AJyQYeuqIFGWpJ_HhUpXLq6P_nItIqDhsdaIJFU,686
 sl_shared_assets/tools/ascension_tools.py,sha256=xI-hrkR9NIgb7lyhj-ntc8tCYQvDEv6YgYJXl1yvxCs,14639
 sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
 sl_shared_assets/tools/packaging_tools.py,sha256=VxQoluGPDUWjPj1ftEt2dvUcdmj0g7T1frGZhZPM8NE,7541
 sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=vGCysu-mvK1JebDOnmyb7IvseLwpHY4T3XhAxcA-bJI,32153
+sl_shared_assets/tools/project_management_tools.py,sha256=Gyp_LrYi0H09-sSl-LenimEw-4lyXydlOE47wMEuUQg,34965
 sl_shared_assets/tools/project_management_tools.pyi,sha256=r45nLPP51mrtn0ajm9iSVq-aR37CS71DGZuRXqd29Zc,11377
 sl_shared_assets/tools/transfer_tools.py,sha256=vqYO4sERZV0W1DFNFnTpJA6QBZ4QJA94a2TyUhZW2Qk,6605
 sl_shared_assets/tools/transfer_tools.pyi,sha256=WtUGfaKV9FP_CnhBg_UvclpuDvOlEESOSMlEDtWpOLg,3293
-sl_shared_assets-3.1.2.dist-info/METADATA,sha256=TOn08frNpw_CDhAGpYrjtIrxdcJ4TwMloOT4kqllvLU,56944
-sl_shared_assets-3.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-3.1.2.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
-sl_shared_assets-3.1.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-3.1.2.dist-info/RECORD,,
+sl_shared_assets-3.1.3.dist-info/METADATA,sha256=OW1w-PlNKnEDMaVh-mAANdk0cO_KZhZGxn_0-stSd2g,56944
+sl_shared_assets-3.1.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-3.1.3.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
+sl_shared_assets-3.1.3.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-3.1.3.dist-info/RECORD,,