sl-shared-assets 3.1.0__py3-none-any.whl → 3.1.2__py3-none-any.whl

sl_shared_assets/cli.py

@@ -128,7 +128,7 @@ def generate_project_manifest_file(
     generate_project_manifest(
         raw_project_directory=Path(project_path),
         output_directory=Path(output_directory),
-        processed_project_directory=Path(project_processed_path) if project_processed_path else None,
+        processed_data_root=Path(project_processed_path) if project_processed_path else None,
     )
     # noinspection PyTypeChecker
     console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)

sl_shared_assets/tools/project_management_tools.py

@@ -7,6 +7,7 @@ from datetime import datetime
 
 import pytz
 import polars as pl
+from filelock import FileLock
 from ataraxis_base_utilities import console
 
 from ..data_classes import (
@@ -229,7 +230,7 @@ class ProjectManifest:
 
 
 def generate_project_manifest(
-    raw_project_directory: Path, output_directory: Path, processed_project_directory: Path | None = None
+    raw_project_directory: Path, output_directory: Path, processed_data_root: Path | None = None
 ) -> None:
     """Builds and saves the project manifest .feather file under the specified output directory.
 
@@ -247,9 +248,9 @@ def generate_project_manifest(
     Args:
         raw_project_directory: The path to the root project directory used to store raw session data.
         output_directory: The path to the directory where to save the generated manifest file.
-        processed_project_directory: The path to the root project directory used to store processed session data if it
-            is different from the 'raw_project_directory'. Typically, this would be the case on remote compute server(s)
-            and not on local machines.
+        processed_data_root: The path to the root directory (volume) used to store processed data for all Sun lab
+            projects if it is different from the parent of the 'raw_project_directory'. Typically, this would be the
+            case on remote compute server(s) and not on local machines.
     """
 
     if not raw_project_directory.exists():
@@ -288,139 +289,146 @@ def generate_project_manifest(
         "dataset": [],  # Determines whether the session's data is ready to be integrated into a dataset.
     }
 
-    # Loops over each session of every animal in the project and extracts session ID information and information
-    # about which processing steps have been successfully applied to the session.
-    for directory in session_directories:
-        # Skips processing directories without files (sessions with empty raw-data directories)
-        if len([file for file in directory.joinpath("raw_data").glob("*")]) == 0:
-            continue
-
-        # Instantiates the SessionData instance to resolve the paths to all session's data files and locations.
-        session_data = SessionData.load(
-            session_path=directory,
-            processed_data_root=processed_project_directory,
-            make_processed_data_directory=False,
-        )
+    # Resolves the path to the manifest .feather file to be created and the .lock file for the generated manifest
+    manifest_path = output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather")
+    manifest_lock = manifest_path.with_suffix(manifest_path.suffix + ".lock")
+
+    # Acquires the lock
+    lock = FileLock(str(manifest_lock))
+    with lock.acquire(timeout=20.0):
+        # Loops over each session of every animal in the project and extracts session ID information and information
+        # about which processing steps have been successfully applied to the session.
+        for directory in session_directories:
+            # Skips processing directories without files (sessions with empty raw-data directories)
+            if len([file for file in directory.joinpath("raw_data").glob("*")]) == 0:
+                continue
+
+            # Instantiates the SessionData instance to resolve the paths to all session's data files and locations.
+            session_data = SessionData.load(
+                session_path=directory,
+                processed_data_root=processed_data_root,
+                make_processed_data_directory=False,
+            )
 
-        # Fills the manifest dictionary with data for the processed session:
-
-        # Extracts ID and data path information from the SessionData instance
-        manifest["animal"].append(session_data.animal_id)
-        manifest["session"].append(session_data.session_name)
-        manifest["type"].append(session_data.session_type)
-
-        # Parses session name into the date-time object to simplify working with date-time data in the future
-        date_time_components = session_data.session_name.split("-")
-        date_time = datetime(
-            year=int(date_time_components[0]),
-            month=int(date_time_components[1]),
-            day=int(date_time_components[2]),
-            hour=int(date_time_components[3]),
-            minute=int(date_time_components[4]),
-            second=int(date_time_components[5]),
-            microsecond=int(date_time_components[6]),
-            tzinfo=pytz.UTC,
-        )
+            # Fills the manifest dictionary with data for the processed session:
+
+            # Extracts ID and data path information from the SessionData instance
+            manifest["animal"].append(session_data.animal_id)
+            manifest["session"].append(session_data.session_name)
+            manifest["type"].append(session_data.session_type)
+
+            # Parses session name into the date-time object to simplify working with date-time data in the future
+            date_time_components = session_data.session_name.split("-")
+            date_time = datetime(
+                year=int(date_time_components[0]),
+                month=int(date_time_components[1]),
+                day=int(date_time_components[2]),
+                hour=int(date_time_components[3]),
+                minute=int(date_time_components[4]),
+                second=int(date_time_components[5]),
+                microsecond=int(date_time_components[6]),
+                tzinfo=pytz.UTC,
+            )
 
-        # Converts from UTC to EST / EDT for user convenience
-        eastern = pytz.timezone("America/New_York")
-        date_time = date_time.astimezone(eastern)
-        manifest["date"].append(date_time)
+            # Converts from UTC to EST / EDT for user convenience
+            eastern = pytz.timezone("America/New_York")
+            date_time = date_time.astimezone(eastern)
+            manifest["date"].append(date_time)
 
-        # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
-        # experimenter notes
-        if session_data.session_type == SessionTypes.LICK_TRAINING:
-            descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
-                file_path=session_data.raw_data.session_descriptor_path
-            )
-            manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == SessionTypes.RUN_TRAINING:
-            descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
-                file_path=session_data.raw_data.session_descriptor_path
-            )
-            manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
-            descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
-                file_path=session_data.raw_data.session_descriptor_path
-            )
-            manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
-            # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file does not
-            # exist in prior versions, this section is written to statically handle the discrepancy.
-            try:
-                descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+            # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
+            # experimenter notes
+            if session_data.session_type == SessionTypes.LICK_TRAINING:
+                descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
                     file_path=session_data.raw_data.session_descriptor_path
                 )
                 manifest["notes"].append(descriptor.experimenter_notes)
-            except Exception:
-                manifest["notes"].append("N/A")
-
-        # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
-        manifest["complete"].append(session_data.raw_data.telomere_path.exists())
-
-        # Data verification status
-        tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
-        manifest["integrity"].append(tracker.is_complete)
-
-        # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing is
-        # disabled for incomplete sessions. If the session is unverified, the case is even more severe, as its data may
-        # be corrupted.
-        if not manifest["complete"][-1] or not manifest["integrity"][-1]:
-            manifest["suite2p"].append(False)
-            manifest["dataset"].append(False)
-            manifest["behavior"].append(False)
-            manifest["video"].append(False)
-            continue  # Cycles to the next session
-
-        # Suite2p (single-day) processing status.
-        tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
-        manifest["suite2p"].append(tracker.is_complete)
-
-        # Behavior data processing status.
-        tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
-        manifest["behavior"].append(tracker.is_complete)
-
-        # DeepLabCut (video) processing status.
-        tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
-        manifest["video"].append(tracker.is_complete)
-
-        # Tracks whether the session's data is currently in the processing or dataset integration mode.
-        manifest["dataset"].append(session_data.processed_data.p53_path.exists())
-
-    # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
-    # them as strings. The latter options are primarily kept for compatibility with Tyche data
-    animal_type: type[pl.UInt64] | type[pl.String]
-    if all([str(animal).isdigit() for animal in manifest["animal"]]):
-        # Converts all strings to integers
-        manifest["animal"] = [int(animal) for animal in manifest["animal"]]  # type: ignore
-        animal_type = pl.UInt64  # Uint64 for future proofing
-    else:
-        animal_type = pl.String
-
-    # Converts the manifest dictionary to a Polars Dataframe.
-    schema = {
-        "animal": animal_type,
-        "date": pl.Datetime,
-        "session": pl.String,
-        "type": pl.String,
-        "notes": pl.String,
-        "complete": pl.UInt8,
-        "integrity": pl.UInt8,
-        "suite2p": pl.UInt8,
-        "dataset": pl.UInt8,
-        "behavior": pl.UInt8,
-        "video": pl.UInt8,
-    }
-    df = pl.DataFrame(manifest, schema=schema, strict=False)
-
-    # Sorts the DataFrame by animal and then session. Since we assign animal IDs sequentially and 'name' sessions based
-    # on acquisition timestamps, the sort order is chronological.
-    sorted_df = df.sort(["animal", "session"])
-
-    # Saves the generated manifest to the project-specific manifest .feather file for further processing.
-    sorted_df.write_ipc(
-        file=output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather"), compression="lz4"
-    )
+            elif session_data.session_type == SessionTypes.RUN_TRAINING:
+                descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
+                descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
+                # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file
+                # does not exist in prior versions, this section is written to statically handle the discrepancy.
+                try:
+                    descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+                        file_path=session_data.raw_data.session_descriptor_path
+                    )
+                    manifest["notes"].append(descriptor.experimenter_notes)
+                except Exception:
+                    manifest["notes"].append("N/A")
+
+            # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
+            manifest["complete"].append(session_data.raw_data.telomere_path.exists())
+
+            # Data verification status
+            tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
+            manifest["integrity"].append(tracker.is_complete)
+
+            # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing
+            # is disabled for incomplete sessions. If the session is unverified, the case is even more severe, as its
+            # data may be corrupted.
+            if not manifest["complete"][-1] or not manifest["integrity"][-1]:
+                manifest["suite2p"].append(False)
+                manifest["dataset"].append(False)
+                manifest["behavior"].append(False)
+                manifest["video"].append(False)
+                continue  # Cycles to the next session
+
+            # Suite2p (single-day) processing status.
+            tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
+            manifest["suite2p"].append(tracker.is_complete)
+
+            # Behavior data processing status.
+            tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
+            manifest["behavior"].append(tracker.is_complete)
+
+            # DeepLabCut (video) processing status.
+            tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
+            manifest["video"].append(tracker.is_complete)
+
+            # Tracks whether the session's data is currently in the processing or dataset integration mode.
+            manifest["dataset"].append(session_data.processed_data.p53_path.exists())
+
+        # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
+        # them as strings. The latter options are primarily kept for compatibility with Tyche data
+        animal_type: type[pl.UInt64] | type[pl.String]
+        if all([str(animal).isdigit() for animal in manifest["animal"]]):
+            # Converts all strings to integers
+            manifest["animal"] = [int(animal) for animal in manifest["animal"]]  # type: ignore
+            animal_type = pl.UInt64  # Uint64 for future proofing
+        else:
+            animal_type = pl.String
+
+        # Converts the manifest dictionary to a Polars Dataframe.
+        schema = {
+            "animal": animal_type,
+            "date": pl.Datetime,
+            "session": pl.String,
+            "type": pl.String,
+            "notes": pl.String,
+            "complete": pl.UInt8,
+            "integrity": pl.UInt8,
+            "suite2p": pl.UInt8,
+            "dataset": pl.UInt8,
+            "behavior": pl.UInt8,
+            "video": pl.UInt8,
+        }
+        df = pl.DataFrame(manifest, schema=schema, strict=False)
+
+        # Sorts the DataFrame by animal and then session. Since we assign animal IDs sequentially and 'name' sessions
+        # based on acquisition timestamps, the sort order is chronological.
+        sorted_df = df.sort(["animal", "session"])
+
+        # Saves the generated manifest to the project-specific manifest .feather file for further processing.
+        sorted_df.write_ipc(
+            file=output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather"), compression="lz4"
+        )
 
 
 def verify_session_checksum(
@@ -508,16 +516,10 @@ def verify_session_checksum(
             # the raw project directory.
             raw_directory = session_path.parents[1]
 
-            # Depending on the processed_data_root configuration, determines the path for the project's processed
-            # data directory.
-            processed_directory: Path | None = None
-            if processed_data_root is not None:
-                processed_directory = processed_data_root.joinpath(session_data.project_name)
-
             # Generates the manifest file inside the root raw data project directory
             generate_project_manifest(
                 raw_project_directory=session_path.parents[1],
-                processed_project_directory=processed_directory,
+                processed_data_root=processed_data_root,
                 output_directory=raw_directory,
             )
 
@@ -618,15 +620,9 @@ def resolve_p53_marker(
         # the raw project directory.
         raw_directory = session_path.parents[1]
 
-        # Depending on the processed_data_root configuration, determines the path for the project's processed
-        # data directory.
-        processed_directory: Path | None = None
-        if processed_data_root is not None:
-            processed_directory = processed_data_root.joinpath(session_data.project_name)
-
         # Generates the manifest file inside the root raw data project directory
         generate_project_manifest(
             raw_project_directory=session_path.parents[1],
-            processed_project_directory=processed_directory,
+            processed_data_root=processed_data_root,
             output_directory=raw_directory,
         )

sl_shared_assets/tools/project_management_tools.pyi

@@ -107,7 +107,7 @@ class ProjectManifest:
         """
 
 def generate_project_manifest(
-    raw_project_directory: Path, output_directory: Path, processed_project_directory: Path | None = None
+    raw_project_directory: Path, output_directory: Path, processed_data_root: Path | None = None
 ) -> None:
     """Builds and saves the project manifest .feather file under the specified output directory.
 
@@ -125,9 +125,9 @@ def generate_project_manifest(
     Args:
         raw_project_directory: The path to the root project directory used to store raw session data.
         output_directory: The path to the directory where to save the generated manifest file.
-        processed_project_directory: The path to the root project directory used to store processed session data if it
-            is different from the 'raw_project_directory'. Typically, this would be the case on remote compute server(s)
-            and not on local machines.
+        processed_data_root: The path to the root directory (volume) used to store processed data for all Sun lab
+            projects if it is different from the parent of the 'raw_project_directory'. Typically, this would be the
+            case on remote compute server(s) and not on local machines.
     """
 
 def verify_session_checksum(

{sl_shared_assets-3.1.0.dist-info → sl_shared_assets-3.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 3.1.0
+Version: 3.1.2
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-3.1.0.dist-info → sl_shared_assets-3.1.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 sl_shared_assets/__init__.py,sha256=ybThh0XDtijjwahKkSEnnQ44rxrN2SVyjB5dHaXts0E,2391
 sl_shared_assets/__init__.pyi,sha256=Cb-umRqvnynk2udbgqAJ6h5_tiJyvVtWmx0kLKrL2Yg,2678
-sl_shared_assets/cli.py,sha256=OIwXf6pNPnzqzUPL7mSmEw17KIa3yAOpP0Mpo1Zpf88,19087
+sl_shared_assets/cli.py,sha256=ERSU9iTLZxL0eIseQA-5d7eTBjlmfjHhEozWTg0dkJA,19079
 sl_shared_assets/cli.pyi,sha256=5hEbOnYaH4q5qdqJ-zhM9-ElzgcaBeMAX34tuHaUDos,5328
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sl_shared_assets/data_classes/__init__.py,sha256=mP__bBIIjMf0EETM4PgQzKy1ZKsjp6paRPNDWWbPRV4,1962
@@ -25,12 +25,12 @@ sl_shared_assets/tools/ascension_tools.py,sha256=xI-hrkR9NIgb7lyhj-ntc8tCYQvDEv6
 sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
 sl_shared_assets/tools/packaging_tools.py,sha256=VxQoluGPDUWjPj1ftEt2dvUcdmj0g7T1frGZhZPM8NE,7541
 sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=vutKi0pdQn5dxBk1OcxPB4XspzQyJwzerNhGi4Vg4iw,31935
-sl_shared_assets/tools/project_management_tools.pyi,sha256=hdn0U9e3_j9McJH75Dzoas-FxcB9nVCTHEFHPofdLtg,11361
+sl_shared_assets/tools/project_management_tools.py,sha256=vGCysu-mvK1JebDOnmyb7IvseLwpHY4T3XhAxcA-bJI,32153
+sl_shared_assets/tools/project_management_tools.pyi,sha256=r45nLPP51mrtn0ajm9iSVq-aR37CS71DGZuRXqd29Zc,11377
 sl_shared_assets/tools/transfer_tools.py,sha256=vqYO4sERZV0W1DFNFnTpJA6QBZ4QJA94a2TyUhZW2Qk,6605
 sl_shared_assets/tools/transfer_tools.pyi,sha256=WtUGfaKV9FP_CnhBg_UvclpuDvOlEESOSMlEDtWpOLg,3293
-sl_shared_assets-3.1.0.dist-info/METADATA,sha256=SbnWSGHffTfwIaQGGP04zsSZ-T2yFga1jL79eLLoib8,56944
-sl_shared_assets-3.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-3.1.0.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
-sl_shared_assets-3.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-3.1.0.dist-info/RECORD,,
+sl_shared_assets-3.1.2.dist-info/METADATA,sha256=TOn08frNpw_CDhAGpYrjtIrxdcJ4TwMloOT4kqllvLU,56944
+sl_shared_assets-3.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-3.1.2.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
+sl_shared_assets-3.1.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-3.1.2.dist-info/RECORD,,