sl-shared-assets 3.0.1__py3-none-any.whl → 3.1.1__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -8,6 +8,7 @@ Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Natalie Yeung
 from ataraxis_base_utilities import console
 
 from .tools import (
+    ProjectManifest,
     resolve_p53_marker,
     transfer_directory,
     calculate_directory_checksum,
@@ -86,6 +87,7 @@ __all__ = [
     "AcquisitionSystems",
     "WindowCheckingDescriptor",
     # Tools package
+    "ProjectManifest",
     "resolve_p53_marker",
     "transfer_directory",
     "calculate_directory_checksum",

sl_shared_assets/__init__.pyi

@@ -1,4 +1,5 @@
 from .tools import (
+    ProjectManifest as ProjectManifest,
     resolve_p53_marker as resolve_p53_marker,
     transfer_directory as transfer_directory,
     calculate_directory_checksum as calculate_directory_checksum,
@@ -75,6 +76,7 @@ __all__ = [
     "SessionTypes",
     "AcquisitionSystems",
     "WindowCheckingDescriptor",
+    "ProjectManifest",
     "resolve_p53_marker",
     "transfer_directory",
     "calculate_directory_checksum",

sl_shared_assets/cli.py

@@ -43,8 +43,18 @@ from .data_classes import SessionData, ProcessingTracker
         "used if 'create_processed_directories' flag is True."
     ),
 )
+@click.option(
+    "-um",
+    "--update_manifest",
+    is_flag=True,
+    help=(
+        "Determines whether to (re)generate the manifest file for the processed session's project. This flag "
+        "should always be enabled when this CLI is executed on the remote compute server(s) to ensure that the "
+        "manifest file always reflects the most actual state of each project."
+    ),
+)
 def verify_session_integrity(
-    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None, update_manifest: bool
 ) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
@@ -63,7 +73,10 @@ def verify_session_integrity(
 
     # Runs the verification process
     verify_session_checksum(
-        session, create_processed_data_directory=create_processed_directories, processed_data_root=processed_data_root
+        session,
+        create_processed_data_directory=create_processed_directories,
+        processed_data_root=processed_data_root,
+        update_manifest=update_manifest,
     )
 
     # Checks the outcome of the verification process
@@ -437,8 +450,22 @@ def start_jupyter_server(
         "being integrated into any datasets."
     ),
 )
+@click.option(
+    "-um",
+    "--update_manifest",
+    is_flag=True,
+    help=(
+        "Determines whether to (re)generate the manifest file for the processed session's project. This flag "
+        "should always be enabled when this CLI is executed on the remote compute server(s) to ensure that the "
+        "manifest file always reflects the most actual state of each project."
+    ),
+)
 def resolve_dataset_marker(
-    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+    session_path: Path,
+    create_processed_directories: bool,
+    project_processed_path: Path | None,
+    remove: bool,
+    update_manifest: bool,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker from the target session.
 
@@ -452,4 +479,5 @@ def resolve_dataset_marker(
         create_processed_data_directory=create_processed_directories,
         processed_data_root=project_processed_path,
         remove=remove,
+        update_manifest=update_manifest,
     )

sl_shared_assets/cli.pyi

@@ -17,7 +17,7 @@ from .data_classes import (
 )
 
 def verify_session_integrity(
-    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None, update_manifest: bool
 ) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
@@ -86,7 +86,11 @@ def start_jupyter_server(
     """
 
 def resolve_dataset_marker(
-    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+    session_path: Path,
+    create_processed_directories: bool,
+    project_processed_path: Path | None,
+    remove: bool,
+    update_manifest: bool,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker from the target session.
 

sl_shared_assets/data_classes/session_data.pyi

@@ -249,7 +249,7 @@ class ProcessingTracker(YamlConfig):
     def __post_init__(self) -> None: ...
     def __del__(self) -> None:
         """If the instance as used to start a runtime, ensures that the instance properly marks the runtime as completed
-        or erred before beign garbage-collected.
+        or erred before being garbage-collected.
 
         This is a security mechanism to prevent deadlocking the processed session and pipeline for future runtimes.
         """

sl_shared_assets/tools/__init__.py

@@ -4,9 +4,15 @@ integrity of the data. The tools from this package are used by most other data p
 from .transfer_tools import transfer_directory
 from .ascension_tools import ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum
-from .project_management_tools import resolve_p53_marker, verify_session_checksum, generate_project_manifest
+from .project_management_tools import (
+    ProjectManifest,
+    resolve_p53_marker,
+    verify_session_checksum,
+    generate_project_manifest,
+)
 
 __all__ = [
+    "ProjectManifest",
     "transfer_directory",
     "calculate_directory_checksum",
     "ascend_tyche_data",

sl_shared_assets/tools/__init__.pyi

@@ -2,12 +2,14 @@ from .transfer_tools import transfer_directory as transfer_directory
 from .ascension_tools import ascend_tyche_data as ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 from .project_management_tools import (
+    ProjectManifest as ProjectManifest,
     resolve_p53_marker as resolve_p53_marker,
     verify_session_checksum as verify_session_checksum,
     generate_project_manifest as generate_project_manifest,
 )
 
 __all__ = [
+    "ProjectManifest",
     "transfer_directory",
     "calculate_directory_checksum",
     "ascend_tyche_data",

sl_shared_assets/tools/project_management_tools.py

@@ -7,6 +7,7 @@ from datetime import datetime
 
 import pytz
 import polars as pl
+from filelock import FileLock
 from ataraxis_base_utilities import console
 
 from ..data_classes import (
@@ -288,143 +289,153 @@ def generate_project_manifest(
         "dataset": [],  # Determines whether the session's data is ready to be integrated into a dataset.
     }
 
-    # Loops over each session of every animal in the project and extracts session ID information and information
-    # about which processing steps have been successfully applied to the session.
-    for directory in session_directories:
-        # Skips processing directories without files (sessions with empty raw-data directories)
-        if len([file for file in directory.joinpath("raw_data").glob("*")]) == 0:
-            continue
-
-        # Instantiates the SessionData instance to resolve the paths to all session's data files and locations.
-        session_data = SessionData.load(
-            session_path=directory,
-            processed_data_root=processed_project_directory,
-            make_processed_data_directory=False,
-        )
+    # Resolves the path to the manifest .feather file to be created and the .lock file for the generated manifest
+    manifest_path = output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather")
+    manifest_lock = manifest_path.with_suffix(manifest_path.suffix + ".lock")
+
+    # Acquires the lock
+    lock = FileLock(str(manifest_lock))
+    with lock.acquire(timeout=20.0):
+        # Loops over each session of every animal in the project and extracts session ID information and information
+        # about which processing steps have been successfully applied to the session.
+        for directory in session_directories:
+            # Skips processing directories without files (sessions with empty raw-data directories)
+            if len([file for file in directory.joinpath("raw_data").glob("*")]) == 0:
+                continue
+
+            # Instantiates the SessionData instance to resolve the paths to all session's data files and locations.
+            session_data = SessionData.load(
+                session_path=directory,
+                processed_data_root=processed_project_directory,
+                make_processed_data_directory=False,
+            )
 
-        # Fills the manifest dictionary with data for the processed session:
-
-        # Extracts ID and data path information from the SessionData instance
-        manifest["animal"].append(session_data.animal_id)
-        manifest["session"].append(session_data.session_name)
-        manifest["type"].append(session_data.session_type)
-
-        # Parses session name into the date-time object to simplify working with date-time data in the future
-        date_time_components = session_data.session_name.split("-")
-        date_time = datetime(
-            year=int(date_time_components[0]),
-            month=int(date_time_components[1]),
-            day=int(date_time_components[2]),
-            hour=int(date_time_components[3]),
-            minute=int(date_time_components[4]),
-            second=int(date_time_components[5]),
-            microsecond=int(date_time_components[6]),
-            tzinfo=pytz.UTC,
-        )
+            # Fills the manifest dictionary with data for the processed session:
+
+            # Extracts ID and data path information from the SessionData instance
+            manifest["animal"].append(session_data.animal_id)
+            manifest["session"].append(session_data.session_name)
+            manifest["type"].append(session_data.session_type)
+
+            # Parses session name into the date-time object to simplify working with date-time data in the future
+            date_time_components = session_data.session_name.split("-")
+            date_time = datetime(
+                year=int(date_time_components[0]),
+                month=int(date_time_components[1]),
+                day=int(date_time_components[2]),
+                hour=int(date_time_components[3]),
+                minute=int(date_time_components[4]),
+                second=int(date_time_components[5]),
+                microsecond=int(date_time_components[6]),
+                tzinfo=pytz.UTC,
+            )
 
-        # Converts from UTC to EST / EDT for user convenience
-        eastern = pytz.timezone("America/New_York")
-        date_time = date_time.astimezone(eastern)
-        manifest["date"].append(date_time)
+            # Converts from UTC to EST / EDT for user convenience
+            eastern = pytz.timezone("America/New_York")
+            date_time = date_time.astimezone(eastern)
+            manifest["date"].append(date_time)
 
-        # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
-        # experimenter notes
-        if session_data.session_type == SessionTypes.LICK_TRAINING:
-            descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
-                file_path=session_data.raw_data.session_descriptor_path
-            )
-            manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == SessionTypes.RUN_TRAINING:
-            descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
-                file_path=session_data.raw_data.session_descriptor_path
-            )
-            manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
-            descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
-                file_path=session_data.raw_data.session_descriptor_path
-            )
-            manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
-            # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file does not
-            # exist in prior versions, this section is written to statically handle the discrepancy.
-            try:
-                descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+            # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
+            # experimenter notes
+            if session_data.session_type == SessionTypes.LICK_TRAINING:
+                descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
                     file_path=session_data.raw_data.session_descriptor_path
                 )
                 manifest["notes"].append(descriptor.experimenter_notes)
-            except Exception:
-                manifest["notes"].append("N/A")
-
-        # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
-        manifest["complete"].append(session_data.raw_data.telomere_path.exists())
-
-        # Data verification status
-        tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
-        manifest["integrity"].append(tracker.is_complete)
-
-        # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing is
-        # disabled for incomplete sessions. If the session is unverified, the case is even more severe, as its data may
-        # be corrupted.
-        if not manifest["complete"][-1] or not manifest["integrity"][-1]:
-            manifest["suite2p"].append(False)
-            manifest["dataset"].append(False)
-            manifest["behavior"].append(False)
-            manifest["video"].append(False)
-            continue  # Cycles to the next session
-
-        # Suite2p (single-day) processing status.
-        tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
-        manifest["suite2p"].append(tracker.is_complete)
-
-        # Behavior data processing status.
-        tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
-        manifest["behavior"].append(tracker.is_complete)
-
-        # DeepLabCut (video) processing status.
-        tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
-        manifest["video"].append(tracker.is_complete)
-
-        # Tracks whether the session's data is currently in the processing or dataset integration mode.
-        manifest["dataset"].append(session_data.processed_data.p53_path.exists())
-
-    # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
-    # them as strings. The latter options are primarily kept for compatibility with Tyche data
-    animal_type: type[pl.UInt64] | type[pl.String]
-    if all([str(animal).isdigit() for animal in manifest["animal"]]):
-        # Converts all strings to integers
-        manifest["animal"] = [int(animal) for animal in manifest["animal"]]  # type: ignore
-        animal_type = pl.UInt64  # Uint64 for future proofing
-    else:
-        animal_type = pl.String
-
-    # Converts the manifest dictionary to a Polars Dataframe.
-    schema = {
-        "animal": animal_type,
-        "date": pl.Datetime,
-        "session": pl.String,
-        "type": pl.String,
-        "notes": pl.String,
-        "complete": pl.UInt8,
-        "integrity": pl.UInt8,
-        "suite2p": pl.UInt8,
-        "dataset": pl.UInt8,
-        "behavior": pl.UInt8,
-        "video": pl.UInt8,
-    }
-    df = pl.DataFrame(manifest, schema=schema, strict=False)
-
-    # Sorts the DataFrame by animal and then session. Since we assign animal IDs sequentially and 'name' sessions based
-    # on acquisition timestamps, the sort order is chronological.
-    sorted_df = df.sort(["animal", "session"])
-
-    # Saves the generated manifest to the project-specific manifest .feather file for further processing.
-    sorted_df.write_ipc(
-        file=output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather"), compression="lz4"
-    )
+            elif session_data.session_type == SessionTypes.RUN_TRAINING:
+                descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
+                descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
+                # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file
+                # does not exist in prior versions, this section is written to statically handle the discrepancy.
+                try:
+                    descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+                        file_path=session_data.raw_data.session_descriptor_path
+                    )
+                    manifest["notes"].append(descriptor.experimenter_notes)
+                except Exception:
+                    manifest["notes"].append("N/A")
+
+            # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
+            manifest["complete"].append(session_data.raw_data.telomere_path.exists())
+
+            # Data verification status
+            tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
+            manifest["integrity"].append(tracker.is_complete)
+
+            # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing
+            # is disabled for incomplete sessions. If the session is unverified, the case is even more severe, as its
+            # data may be corrupted.
+            if not manifest["complete"][-1] or not manifest["integrity"][-1]:
+                manifest["suite2p"].append(False)
+                manifest["dataset"].append(False)
+                manifest["behavior"].append(False)
+                manifest["video"].append(False)
+                continue  # Cycles to the next session
+
+            # Suite2p (single-day) processing status.
+            tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
+            manifest["suite2p"].append(tracker.is_complete)
+
+            # Behavior data processing status.
+            tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
+            manifest["behavior"].append(tracker.is_complete)
+
+            # DeepLabCut (video) processing status.
+            tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
+            manifest["video"].append(tracker.is_complete)
+
+            # Tracks whether the session's data is currently in the processing or dataset integration mode.
+            manifest["dataset"].append(session_data.processed_data.p53_path.exists())
+
+        # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
+        # them as strings. The latter options are primarily kept for compatibility with Tyche data
+        animal_type: type[pl.UInt64] | type[pl.String]
+        if all([str(animal).isdigit() for animal in manifest["animal"]]):
+            # Converts all strings to integers
+            manifest["animal"] = [int(animal) for animal in manifest["animal"]]  # type: ignore
+            animal_type = pl.UInt64  # Uint64 for future proofing
+        else:
+            animal_type = pl.String
+
+        # Converts the manifest dictionary to a Polars Dataframe.
+        schema = {
+            "animal": animal_type,
+            "date": pl.Datetime,
+            "session": pl.String,
+            "type": pl.String,
+            "notes": pl.String,
+            "complete": pl.UInt8,
+            "integrity": pl.UInt8,
+            "suite2p": pl.UInt8,
+            "dataset": pl.UInt8,
+            "behavior": pl.UInt8,
+            "video": pl.UInt8,
+        }
+        df = pl.DataFrame(manifest, schema=schema, strict=False)
+
+        # Sorts the DataFrame by animal and then session. Since we assign animal IDs sequentially and 'name' sessions
+        # based on acquisition timestamps, the sort order is chronological.
+        sorted_df = df.sort(["animal", "session"])
+
+        # Saves the generated manifest to the project-specific manifest .feather file for further processing.
+        sorted_df.write_ipc(
+            file=output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather"), compression="lz4"
+        )
 
 
 def verify_session_checksum(
-    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    update_manifest: bool = False,
 ) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
@@ -440,6 +451,9 @@ def verify_session_checksum(
         This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
         part of the data preprocessing runtime performed by a data acquisition system.
 
+        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
@@ -447,6 +461,9 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -492,12 +509,27 @@ def verify_session_checksum(
         if tracker.is_running:
             tracker.error()
 
+        # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
+        # existing manifest file for the target project.
+        if update_manifest:
+            # All sessions are stored under root/project/animal/session. Therefore, the grandparent of the session is
+            # the raw project directory.
+            raw_directory = session_path.parents[1]
+
+            # Generates the manifest file inside the root raw data project directory
+            generate_project_manifest(
+                raw_project_directory=session_path.parents[1],
+                processed_project_directory=processed_data_root,
+                output_directory=raw_directory,
+            )
+
 
 def resolve_p53_marker(
     session_path: Path,
     create_processed_data_directory: bool = True,
     processed_data_root: None | Path = None,
     remove: bool = False,
+    update_manifest: bool = False,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
 
@@ -511,7 +543,10 @@ def resolve_p53_marker(
 
         For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
         p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
-        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab compute server.
+
+        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -521,6 +556,9 @@ def resolve_p53_marker(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
         remove: Determines whether this function is called to create or remove the p53.bin marker.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -574,3 +612,17 @@ def resolve_p53_marker(
     # If the runtime reached this point, the session is eligible for dataset integration. Creates the p53.bin marker
     # file, preventing the session from being processed again as long as the marker exists.
     session_data.processed_data.p53_path.touch()
+
+    # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
+    # existing manifest file for the target project.
+    if update_manifest:
+        # All sessions are stored under root/project/animal/session. Therefore, the grandparent of the session is
+        # the raw project directory.
+        raw_directory = session_path.parents[1]
+
+        # Generates the manifest file inside the root raw data project directory
+        generate_project_manifest(
+            raw_project_directory=session_path.parents[1],
+            processed_project_directory=processed_data_root,
+            output_directory=raw_directory,
+        )

sl_shared_assets/tools/project_management_tools.pyi

@@ -131,7 +131,10 @@ def generate_project_manifest(
     """
 
 def verify_session_checksum(
-    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    update_manifest: bool = False,
 ) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
@@ -147,6 +150,9 @@ def verify_session_checksum(
         This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
         part of the data preprocessing runtime performed by a data acquisition system.
 
+        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
@@ -154,6 +160,9 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
 def resolve_p53_marker(
@@ -161,6 +170,7 @@ def resolve_p53_marker(
     create_processed_data_directory: bool = True,
     processed_data_root: None | Path = None,
     remove: bool = False,
+    update_manifest: bool = False,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
 
@@ -174,7 +184,10 @@ def resolve_p53_marker(
 
         For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
         p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
-        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab compute server.
+
+        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -184,4 +197,7 @@ def resolve_p53_marker(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
         remove: Determines whether this function is called to create or remove the p53.bin marker.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """

{sl_shared_assets-3.0.1.dist-info → sl_shared_assets-3.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 3.0.1
+Version: 3.1.1
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-3.0.1.dist-info → sl_shared_assets-3.1.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-sl_shared_assets/__init__.py,sha256=rCu1VYs2Lc1l0jqHO3UtfuymU0uY2ccxEn4UyscIut8,2347
-sl_shared_assets/__init__.pyi,sha256=WCWIS-I3ToP4XybNZAi3fA7j2CZ48dl9D-fmd7oZKCo,2615
-sl_shared_assets/cli.py,sha256=Af2yRujqZ01IzmXBqsMQmuz2yvRm4raKZ-y58j49fUI,18141
-sl_shared_assets/cli.pyi,sha256=hVxU1YlhWDz41GHaaH7rkbZz2slTMBmG83-7rg_geYk,5265
+sl_shared_assets/__init__.py,sha256=ybThh0XDtijjwahKkSEnnQ44rxrN2SVyjB5dHaXts0E,2391
+sl_shared_assets/__init__.pyi,sha256=Cb-umRqvnynk2udbgqAJ6h5_tiJyvVtWmx0kLKrL2Yg,2678
+sl_shared_assets/cli.py,sha256=OIwXf6pNPnzqzUPL7mSmEw17KIa3yAOpP0Mpo1Zpf88,19087
+sl_shared_assets/cli.pyi,sha256=5hEbOnYaH4q5qdqJ-zhM9-ElzgcaBeMAX34tuHaUDos,5328
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sl_shared_assets/data_classes/__init__.py,sha256=mP__bBIIjMf0EETM4PgQzKy1ZKsjp6paRPNDWWbPRV4,1962
 sl_shared_assets/data_classes/__init__.pyi,sha256=J7ZCH9qQ4qz-3Wq9ILdihlmK9zFR3iU1cpLcSaN45Y8,2238
@@ -10,7 +10,7 @@ sl_shared_assets/data_classes/configuration_data.pyi,sha256=1_kmBDPGkHmVwXEGYR_3
 sl_shared_assets/data_classes/runtime_data.py,sha256=kmXTUk5rDAJBN3XrYLYgusJRfVJ5WiBBk0RPNiSk2pE,16725
 sl_shared_assets/data_classes/runtime_data.pyi,sha256=Hyc-dBePM0xIGgkSIoKmwwUUmdOokm1LUwy1OHHalyU,6771
 sl_shared_assets/data_classes/session_data.py,sha256=YKXako1sNB87LDkGEXx9WZFs6lG3aD619lga5g4L4Ks,49172
-sl_shared_assets/data_classes/session_data.pyi,sha256=F_6UAzEiPJjxfHQaOt_pttjSUvSggq1FT48GAHmmcbA,15959
+sl_shared_assets/data_classes/session_data.pyi,sha256=CgB4nIDBl4bY1JvcIILfFTlos3ukl3WK4AOaku4CL3Y,15959
 sl_shared_assets/data_classes/surgery_data.py,sha256=5B1OPKFq4bnzbAoe-_c5dFV3kbSD5YFzXbX2zXmfGs8,7485
 sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
 sl_shared_assets/server/__init__.py,sha256=GOQ7wWjiS5Xg_WgTqeEqCTRF9ms9GXx0nffCr-BmKsA,453
@@ -19,18 +19,18 @@ sl_shared_assets/server/job.py,sha256=wZbppMrv6fqch79bKLjOGQ9AYfjiDKDnTyUe7xgAT4
 sl_shared_assets/server/job.pyi,sha256=wop4ulVY2u6eb3twajeA9MS0EAtNb89aA56pPoGF1Xc,11673
 sl_shared_assets/server/server.py,sha256=oEwdXisyel72Hdk7ZpEwTPq3Lu64UbQWfGHArV8Y6nI,32978
 sl_shared_assets/server/server.pyi,sha256=84XFtqU9fYbxu6Ldf-OMB2nFe6wdGneZM1MFtR9rz4s,15133
-sl_shared_assets/tools/__init__.py,sha256=NktXk62E_HHOrO_93z_MVmSd6-Oir3mE4xE9Yr8Qa7U,682
-sl_shared_assets/tools/__init__.pyi,sha256=0UXorfCXXmHQOP5z7hODpsqEX0DAkOta5VZqN6FSS-w,623
+sl_shared_assets/tools/__init__.py,sha256=i-oUVw_un3lzyyII4Sc75s4BnUfZh_aUbQe6dP2Vrbc,743
+sl_shared_assets/tools/__init__.pyi,sha256=pi-5AJyQYeuqIFGWpJ_HhUpXLq6P_nItIqDhsdaIJFU,686
 sl_shared_assets/tools/ascension_tools.py,sha256=xI-hrkR9NIgb7lyhj-ntc8tCYQvDEv6YgYJXl1yvxCs,14639
 sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
 sl_shared_assets/tools/packaging_tools.py,sha256=VxQoluGPDUWjPj1ftEt2dvUcdmj0g7T1frGZhZPM8NE,7541
 sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=_AQFLKqK22pqzMDPqNY5Wde0bRVvZLcTO2gYrhKD89M,28744
-sl_shared_assets/tools/project_management_tools.pyi,sha256=AeBG-8XUygiJndfsBCKACKIZdnvk0avQRibWO24ahtM,10238
+sl_shared_assets/tools/project_management_tools.py,sha256=9G9jdeUxs80yLkcFJsWUdVcAcuXDB8reqnTfxVa-FAw,32161
+sl_shared_assets/tools/project_management_tools.pyi,sha256=hdn0U9e3_j9McJH75Dzoas-FxcB9nVCTHEFHPofdLtg,11361
 sl_shared_assets/tools/transfer_tools.py,sha256=vqYO4sERZV0W1DFNFnTpJA6QBZ4QJA94a2TyUhZW2Qk,6605
 sl_shared_assets/tools/transfer_tools.pyi,sha256=WtUGfaKV9FP_CnhBg_UvclpuDvOlEESOSMlEDtWpOLg,3293
-sl_shared_assets-3.0.1.dist-info/METADATA,sha256=78zdy2sXUtOmroilVGfI7k2LmVnFKjkyYpVXP0LesMA,56944
-sl_shared_assets-3.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-3.0.1.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
-sl_shared_assets-3.0.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-3.0.1.dist-info/RECORD,,
+sl_shared_assets-3.1.1.dist-info/METADATA,sha256=eBMBDc_ahWT4kEFQH0X_i3U1cthEtn5O23x5VBr3EOQ,56944
+sl_shared_assets-3.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-3.1.1.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
+sl_shared_assets-3.1.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-3.1.1.dist-info/RECORD,,