sl-shared-assets 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -8,6 +8,7 @@ Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Natalie Yeung
 from ataraxis_base_utilities import console
 
 from .tools import (
+    ProjectManifest,
     resolve_p53_marker,
     transfer_directory,
     calculate_directory_checksum,
@@ -86,6 +87,7 @@ __all__ = [
     "AcquisitionSystems",
     "WindowCheckingDescriptor",
     # Tools package
+    "ProjectManifest",
     "resolve_p53_marker",
     "transfer_directory",
     "calculate_directory_checksum",

sl_shared_assets/__init__.pyi

@@ -1,4 +1,5 @@
 from .tools import (
+    ProjectManifest as ProjectManifest,
     resolve_p53_marker as resolve_p53_marker,
     transfer_directory as transfer_directory,
     calculate_directory_checksum as calculate_directory_checksum,
@@ -75,6 +76,7 @@ __all__ = [
     "SessionTypes",
     "AcquisitionSystems",
     "WindowCheckingDescriptor",
+    "ProjectManifest",
     "resolve_p53_marker",
     "transfer_directory",
     "calculate_directory_checksum",

sl_shared_assets/cli.py

@@ -43,8 +43,18 @@ from .data_classes import SessionData, ProcessingTracker
         "used if 'create_processed_directories' flag is True."
     ),
 )
+@click.option(
+    "-um",
+    "--update_manifest",
+    is_flag=True,
+    help=(
+        "Determines whether to (re)generate the manifest file for the processed session's project. This flag "
+        "should always be enabled when this CLI is executed on the remote compute server(s) to ensure that the "
+        "manifest file always reflects the most actual state of each project."
+    ),
+)
 def verify_session_integrity(
-    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None, update_manifest: bool
 ) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
@@ -63,7 +73,10 @@ def verify_session_integrity(
 
     # Runs the verification process
     verify_session_checksum(
-        session, create_processed_data_directory=create_processed_directories, processed_data_root=processed_data_root
+        session,
+        create_processed_data_directory=create_processed_directories,
+        processed_data_root=processed_data_root,
+        update_manifest=update_manifest,
     )
 
     # Checks the outcome of the verification process
@@ -437,8 +450,22 @@ def start_jupyter_server(
         "being integrated into any datasets."
     ),
 )
+@click.option(
+    "-um",
+    "--update_manifest",
+    is_flag=True,
+    help=(
+        "Determines whether to (re)generate the manifest file for the processed session's project. This flag "
+        "should always be enabled when this CLI is executed on the remote compute server(s) to ensure that the "
+        "manifest file always reflects the most actual state of each project."
+    ),
+)
 def resolve_dataset_marker(
-    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+    session_path: Path,
+    create_processed_directories: bool,
+    project_processed_path: Path | None,
+    remove: bool,
+    update_manifest: bool,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker from the target session.
 
@@ -452,4 +479,5 @@ def resolve_dataset_marker(
         create_processed_data_directory=create_processed_directories,
         processed_data_root=project_processed_path,
         remove=remove,
+        update_manifest=update_manifest,
     )

sl_shared_assets/cli.pyi

@@ -17,7 +17,7 @@ from .data_classes import (
 )
 
 def verify_session_integrity(
-    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None, update_manifest: bool
 ) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
@@ -86,7 +86,11 @@ def start_jupyter_server(
     """
 
 def resolve_dataset_marker(
-    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+    session_path: Path,
+    create_processed_directories: bool,
+    project_processed_path: Path | None,
+    remove: bool,
+    update_manifest: bool,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker from the target session.
 

sl_shared_assets/data_classes/session_data.pyi

@@ -249,7 +249,7 @@ class ProcessingTracker(YamlConfig):
     def __post_init__(self) -> None: ...
     def __del__(self) -> None:
         """If the instance as used to start a runtime, ensures that the instance properly marks the runtime as completed
-        or erred before beign garbage-collected.
+        or erred before being garbage-collected.
 
         This is a security mechanism to prevent deadlocking the processed session and pipeline for future runtimes.
         """

sl_shared_assets/tools/__init__.py

@@ -4,9 +4,15 @@ integrity of the data. The tools from this package are used by most other data p
 from .transfer_tools import transfer_directory
 from .ascension_tools import ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum
-from .project_management_tools import resolve_p53_marker, verify_session_checksum, generate_project_manifest
+from .project_management_tools import (
+    ProjectManifest,
+    resolve_p53_marker,
+    verify_session_checksum,
+    generate_project_manifest,
+)
 
 __all__ = [
+    "ProjectManifest",
     "transfer_directory",
     "calculate_directory_checksum",
     "ascend_tyche_data",

sl_shared_assets/tools/__init__.pyi

@@ -2,12 +2,14 @@ from .transfer_tools import transfer_directory as transfer_directory
 from .ascension_tools import ascend_tyche_data as ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 from .project_management_tools import (
+    ProjectManifest as ProjectManifest,
     resolve_p53_marker as resolve_p53_marker,
     verify_session_checksum as verify_session_checksum,
     generate_project_manifest as generate_project_manifest,
 )
 
 __all__ = [
+    "ProjectManifest",
     "transfer_directory",
     "calculate_directory_checksum",
     "ascend_tyche_data",

sl_shared_assets/tools/project_management_tools.py

@@ -424,7 +424,10 @@ def generate_project_manifest(
 
 
 def verify_session_checksum(
-    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    update_manifest: bool = False,
 ) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
@@ -440,6 +443,9 @@ def verify_session_checksum(
         This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
         part of the data preprocessing runtime performed by a data acquisition system.
 
+        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
@@ -447,6 +453,9 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -492,12 +501,33 @@ def verify_session_checksum(
         if tracker.is_running:
             tracker.error()
 
+        # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
+        # existing manifest file for the target project.
+        if update_manifest:
+            # All sessions are stored under root/project/animal/session. Therefore, the grandparent of the session is
+            # the raw project directory.
+            raw_directory = session_path.parents[1]
+
+            # Depending on the processed_data_root configuration, determines the path for the project's processed
+            # data directory.
+            processed_directory: Path | None = None
+            if processed_data_root is not None:
+                processed_directory = processed_data_root.joinpath(session_data.project_name)
+
+            # Generates the manifest file inside the root raw data project directory
+            generate_project_manifest(
+                raw_project_directory=session_path.parents[1],
+                processed_project_directory=processed_directory,
+                output_directory=raw_directory,
+            )
+
 
 def resolve_p53_marker(
     session_path: Path,
     create_processed_data_directory: bool = True,
     processed_data_root: None | Path = None,
     remove: bool = False,
+    update_manifest: bool = False,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
 
@@ -511,7 +541,10 @@ def resolve_p53_marker(
 
         For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
         p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
-        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab compute server.
+
+        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -521,6 +554,9 @@ def resolve_p53_marker(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
         remove: Determines whether this function is called to create or remove the p53.bin marker.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -574,3 +610,23 @@ def resolve_p53_marker(
     # If the runtime reached this point, the session is eligible for dataset integration. Creates the p53.bin marker
     # file, preventing the session from being processed again as long as the marker exists.
     session_data.processed_data.p53_path.touch()
+
+    # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
+    # existing manifest file for the target project.
+    if update_manifest:
+        # All sessions are stored under root/project/animal/session. Therefore, the grandparent of the session is
+        # the raw project directory.
+        raw_directory = session_path.parents[1]
+
+        # Depending on the processed_data_root configuration, determines the path for the project's processed
+        # data directory.
+        processed_directory: Path | None = None
+        if processed_data_root is not None:
+            processed_directory = processed_data_root.joinpath(session_data.project_name)
+
+        # Generates the manifest file inside the root raw data project directory
+        generate_project_manifest(
+            raw_project_directory=session_path.parents[1],
+            processed_project_directory=processed_directory,
+            output_directory=raw_directory,
+        )

sl_shared_assets/tools/project_management_tools.pyi

@@ -131,7 +131,10 @@ def generate_project_manifest(
     """
 
 def verify_session_checksum(
-    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    update_manifest: bool = False,
 ) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
@@ -147,6 +150,9 @@ def verify_session_checksum(
         This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
         part of the data preprocessing runtime performed by a data acquisition system.
 
+        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
@@ -154,6 +160,9 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
 def resolve_p53_marker(
@@ -161,6 +170,7 @@ def resolve_p53_marker(
     create_processed_data_directory: bool = True,
     processed_data_root: None | Path = None,
     remove: bool = False,
+    update_manifest: bool = False,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
 
@@ -174,7 +184,10 @@ def resolve_p53_marker(
 
         For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
         p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
-        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab compute server.
+
+        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -184,4 +197,7 @@ def resolve_p53_marker(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
         remove: Determines whether this function is called to create or remove the p53.bin marker.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """

{sl_shared_assets-3.0.1.dist-info → sl_shared_assets-3.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 3.0.1
+Version: 3.1.0
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-3.0.1.dist-info → sl_shared_assets-3.1.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-sl_shared_assets/__init__.py,sha256=rCu1VYs2Lc1l0jqHO3UtfuymU0uY2ccxEn4UyscIut8,2347
-sl_shared_assets/__init__.pyi,sha256=WCWIS-I3ToP4XybNZAi3fA7j2CZ48dl9D-fmd7oZKCo,2615
-sl_shared_assets/cli.py,sha256=Af2yRujqZ01IzmXBqsMQmuz2yvRm4raKZ-y58j49fUI,18141
-sl_shared_assets/cli.pyi,sha256=hVxU1YlhWDz41GHaaH7rkbZz2slTMBmG83-7rg_geYk,5265
+sl_shared_assets/__init__.py,sha256=ybThh0XDtijjwahKkSEnnQ44rxrN2SVyjB5dHaXts0E,2391
+sl_shared_assets/__init__.pyi,sha256=Cb-umRqvnynk2udbgqAJ6h5_tiJyvVtWmx0kLKrL2Yg,2678
+sl_shared_assets/cli.py,sha256=OIwXf6pNPnzqzUPL7mSmEw17KIa3yAOpP0Mpo1Zpf88,19087
+sl_shared_assets/cli.pyi,sha256=5hEbOnYaH4q5qdqJ-zhM9-ElzgcaBeMAX34tuHaUDos,5328
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sl_shared_assets/data_classes/__init__.py,sha256=mP__bBIIjMf0EETM4PgQzKy1ZKsjp6paRPNDWWbPRV4,1962
 sl_shared_assets/data_classes/__init__.pyi,sha256=J7ZCH9qQ4qz-3Wq9ILdihlmK9zFR3iU1cpLcSaN45Y8,2238
@@ -10,7 +10,7 @@ sl_shared_assets/data_classes/configuration_data.pyi,sha256=1_kmBDPGkHmVwXEGYR_3
 sl_shared_assets/data_classes/runtime_data.py,sha256=kmXTUk5rDAJBN3XrYLYgusJRfVJ5WiBBk0RPNiSk2pE,16725
 sl_shared_assets/data_classes/runtime_data.pyi,sha256=Hyc-dBePM0xIGgkSIoKmwwUUmdOokm1LUwy1OHHalyU,6771
 sl_shared_assets/data_classes/session_data.py,sha256=YKXako1sNB87LDkGEXx9WZFs6lG3aD619lga5g4L4Ks,49172
-sl_shared_assets/data_classes/session_data.pyi,sha256=F_6UAzEiPJjxfHQaOt_pttjSUvSggq1FT48GAHmmcbA,15959
+sl_shared_assets/data_classes/session_data.pyi,sha256=CgB4nIDBl4bY1JvcIILfFTlos3ukl3WK4AOaku4CL3Y,15959
 sl_shared_assets/data_classes/surgery_data.py,sha256=5B1OPKFq4bnzbAoe-_c5dFV3kbSD5YFzXbX2zXmfGs8,7485
 sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
 sl_shared_assets/server/__init__.py,sha256=GOQ7wWjiS5Xg_WgTqeEqCTRF9ms9GXx0nffCr-BmKsA,453
@@ -19,18 +19,18 @@ sl_shared_assets/server/job.py,sha256=wZbppMrv6fqch79bKLjOGQ9AYfjiDKDnTyUe7xgAT4
 sl_shared_assets/server/job.pyi,sha256=wop4ulVY2u6eb3twajeA9MS0EAtNb89aA56pPoGF1Xc,11673
 sl_shared_assets/server/server.py,sha256=oEwdXisyel72Hdk7ZpEwTPq3Lu64UbQWfGHArV8Y6nI,32978
 sl_shared_assets/server/server.pyi,sha256=84XFtqU9fYbxu6Ldf-OMB2nFe6wdGneZM1MFtR9rz4s,15133
-sl_shared_assets/tools/__init__.py,sha256=NktXk62E_HHOrO_93z_MVmSd6-Oir3mE4xE9Yr8Qa7U,682
-sl_shared_assets/tools/__init__.pyi,sha256=0UXorfCXXmHQOP5z7hODpsqEX0DAkOta5VZqN6FSS-w,623
+sl_shared_assets/tools/__init__.py,sha256=i-oUVw_un3lzyyII4Sc75s4BnUfZh_aUbQe6dP2Vrbc,743
+sl_shared_assets/tools/__init__.pyi,sha256=pi-5AJyQYeuqIFGWpJ_HhUpXLq6P_nItIqDhsdaIJFU,686
 sl_shared_assets/tools/ascension_tools.py,sha256=xI-hrkR9NIgb7lyhj-ntc8tCYQvDEv6YgYJXl1yvxCs,14639
 sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
 sl_shared_assets/tools/packaging_tools.py,sha256=VxQoluGPDUWjPj1ftEt2dvUcdmj0g7T1frGZhZPM8NE,7541
 sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=_AQFLKqK22pqzMDPqNY5Wde0bRVvZLcTO2gYrhKD89M,28744
-sl_shared_assets/tools/project_management_tools.pyi,sha256=AeBG-8XUygiJndfsBCKACKIZdnvk0avQRibWO24ahtM,10238
+sl_shared_assets/tools/project_management_tools.py,sha256=vutKi0pdQn5dxBk1OcxPB4XspzQyJwzerNhGi4Vg4iw,31935
+sl_shared_assets/tools/project_management_tools.pyi,sha256=hdn0U9e3_j9McJH75Dzoas-FxcB9nVCTHEFHPofdLtg,11361
 sl_shared_assets/tools/transfer_tools.py,sha256=vqYO4sERZV0W1DFNFnTpJA6QBZ4QJA94a2TyUhZW2Qk,6605
 sl_shared_assets/tools/transfer_tools.pyi,sha256=WtUGfaKV9FP_CnhBg_UvclpuDvOlEESOSMlEDtWpOLg,3293
-sl_shared_assets-3.0.1.dist-info/METADATA,sha256=78zdy2sXUtOmroilVGfI7k2LmVnFKjkyYpVXP0LesMA,56944
-sl_shared_assets-3.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-3.0.1.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
-sl_shared_assets-3.0.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-3.0.1.dist-info/RECORD,,
+sl_shared_assets-3.1.0.dist-info/METADATA,sha256=SbnWSGHffTfwIaQGGP04zsSZ-T2yFga1jL79eLLoib8,56944
+sl_shared_assets-3.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-3.1.0.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
+sl_shared_assets-3.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-3.1.0.dist-info/RECORD,,