sl-shared-assets 5.1.1__py3-none-any.whl → 6.0.0__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -8,7 +8,6 @@ Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Natalie Yeung
 from ataraxis_base_utilities import console
 
 from .tools import (
-    ProjectManifest,
     acquire_lock,
     release_lock,
     delete_directory,
@@ -90,7 +89,6 @@ __all__ = [
     "ProcessingPipelines",
     "ProcessingStatus",
     "ProcessingTracker",
-    "ProjectManifest",
     "RawData",
     "RunTrainingDescriptor",
     "Server",

sl_shared_assets/command_line_interfaces/configure.py

@@ -15,7 +15,7 @@ from ..data_classes import (
 )
 
 # Ensures that displayed CLICK help messages are formatted according to the lab standard.
-CONTEXT_SETTINGS = dict(max_content_width=120)  # or any width you want
+CONTEXT_SETTINGS = dict(max_content_width=120)
 
 
 @click.group("manage", context_settings=CONTEXT_SETTINGS)

sl_shared_assets/server/pipeline.py

@@ -342,6 +342,9 @@ class ProcessingTracker(YamlConfig):
                 self._complete = True
                 self._encountered_error = False
                 self._save_state()
+            else:
+                # Otherwise, updates the completed job counter, but does not change any other state variables.
+                self._save_state()
 
     def abort(self) -> None:
         """Resets the pipeline tracker file to the default state.
@@ -524,11 +527,22 @@ class ProcessingPipeline:
         # to the server.
         elif tracker.is_running:
             self._pipeline_stage += 1
-            self._submit_jobs()
+
+            # If the incremented stage is not a valid stage, the pipeline has actually been aborted and the tracker file
+            # does not properly reflect this state. Sets the internal state tracker appropriately and resets (removes)
+            # the tracker file from the server to prevent deadlocking further runtimes
+            if self._pipeline_stage not in self.jobs.keys():
+                sh.rmtree(self.local_tracker_path.parent)  # Removes local temporary data
+                self.pipeline_status = ProcessingStatus.ABORTED
+                self.server.remove(remote_path=self.remote_tracker_path, is_dir=False)
+            else:
+                # Otherwise, submits the next batch of jobs to the server.
+                self._submit_jobs()
 
         # The final and the rarest state: the pipeline was aborted before it finished the runtime. Generally, this state
         # should not be encountered during most runtimes.
         else:
+            sh.rmtree(self.local_tracker_path.parent)  # Removes local temporary data
             self.pipeline_status = ProcessingStatus.ABORTED
 
     def _submit_jobs(self) -> None:

sl_shared_assets/tools/__init__.py

@@ -4,7 +4,6 @@ integrity of the data. The tools from this package are used by most other data p
 from .transfer_tools import delete_directory, transfer_directory
 from .packaging_tools import calculate_directory_checksum
 from .project_management_tools import (
-    ProjectManifest,
     acquire_lock,
     release_lock,
     archive_session,
@@ -14,7 +13,6 @@ from .project_management_tools import (
 )
 
 __all__ = [
-    "ProjectManifest",
     "acquire_lock",
     "archive_session",
     "calculate_directory_checksum",

sl_shared_assets/tools/packaging_tools.py

@@ -115,7 +115,7 @@ def calculate_directory_checksum(
         results = []
         if not batch:
             with tqdm(
-                total=len(files), desc=f"Calculating checksum for {Path(*directory.parts[-6:])}", unit="files"
+                total=len(files), desc=f"Calculating checksum for {Path(*directory.parts[-6:])}", unit="file"
             ) as pbar:
                 for future in as_completed(future_to_path):
                     results.append(future.result())

sl_shared_assets/tools/project_management_tools.py

@@ -661,236 +661,3 @@ def generate_project_manifest(
             # If the tracker indicates that the processing is still running, the runtime has encountered an error.
             if runtime_tracker.is_running:
                 tracker.error(manager_id=manager_id)
-
-
-class ProjectManifest:
-    """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
-    working with the data stored inside the file.
-
-    This class functions as a high-level API for working with Sun lab projects. It is used both to visualize the
-    current state of various projects and during automated data processing to determine which processing steps to
-    apply to different sessions.
-
-    Args:
-        manifest_file: The path to the .feather manifest file that stores the target project's state data.
-
-    Attributes:
-        _data: Stores the manifest data as a Polars DataFrame.
-        _animal_string: Determines whether animal IDs are stored as strings or unsigned integers.
-    """
-
-    def __init__(self, manifest_file: Path):
-        # Reads the data from the target manifest file into the class attribute
-        self._data: pl.DataFrame = pl.read_ipc(source=manifest_file, use_pyarrow=True)
-
-        # Determines whether animal IDs are stored as strings or as numbers
-        self._animal_string = False
-        schema = self._data.collect_schema()
-        if isinstance(schema["animal"], pl.String):
-            self._animal_string = True
-
-    def print_data(self) -> None:
-        """Prints the entire contents of the manifest file to the terminal."""
-        with pl.Config(
-            set_tbl_rows=-1,  # Displays all rows (-1 means unlimited)
-            set_tbl_cols=-1,  # Displays all columns (-1 means unlimited)
-            set_tbl_hide_column_data_types=True,
-            set_tbl_cell_alignment="LEFT",
-            set_tbl_width_chars=250,  # Sets table width to 200 characters
-            set_fmt_str_lengths=600,  # Allows longer strings to display properly (default is 32)
-        ):
-            print(self._data)
-
-    def print_summary(self, animal: str | int | None = None) -> None:
-        """Prints a summary view of the manifest file to the terminal, excluding the 'experimenter notes' data for
-        each session.
-
-        This data view is optimized for tracking which processing steps have been applied to each session inside the
-        project.
-
-        Args:
-            animal: The ID of the animal for which to display the data. If an ID is provided, this method will only
-                display the data for that animal. Otherwise, it will display the data for all animals.
-        """
-        summary_cols = [
-            "animal",
-            "date",
-            "session",
-            "type",
-            "system",
-            "complete",
-            "integrity",
-            "prepared",
-            "suite2p",
-            "behavior",
-            "video",
-            "archived",
-        ]
-
-        # Retrieves the data
-        df = self._data.select(summary_cols)
-
-        # Optionally filters the data for the target animal
-        if animal is not None:
-            # Ensures that the 'animal' argument has the same type as the data inside the DataFrame.
-            if self._animal_string:
-                animal = str(animal)
-            else:
-                animal = int(animal)
-            df = df.filter(pl.col("animal") == animal)
-
-        # Ensures the data displays properly
-        with pl.Config(
-            set_tbl_rows=-1,
-            set_tbl_cols=-1,
-            set_tbl_width_chars=250,
-            set_tbl_hide_column_data_types=True,
-            set_tbl_cell_alignment="CENTER",
-        ):
-            print(df)
-
-    def print_notes(self, animal: str | int | None = None) -> None:
-        """Prints only animal, session, and notes data from the manifest file.
-
-        This data view is optimized for experimenters to check what sessions have been recorded for each animal in the
-        project and refresh their memory on the outcomes of each session using experimenter notes.
-
-        Args:
-            animal: The ID of the animal for which to display the data. If an ID is provided, this method will only
-                display the data for that animal. Otherwise, it will display the data for all animals.
-        """
-
-        # Pre-selects the columns to display
-        df = self._data.select(["animal", "date", "session", "type", "system", "notes"])
-
-        # Optionally filters the data for the target animal
-        if animal is not None:
-            # Ensures that the 'animal' argument has the same type as the data inside the DataFrame.
-            if self._animal_string:
-                animal = str(animal)
-            else:
-                animal = int(animal)
-
-            df = df.filter(pl.col("animal") == animal)
-
-        #  Prints the extracted data
-        with pl.Config(
-            set_tbl_rows=-1,
-            set_tbl_cols=-1,
-            set_tbl_hide_column_data_types=True,
-            set_tbl_cell_alignment="LEFT",
-            set_tbl_width_chars=100,  # Wider columns for notes
-            set_fmt_str_lengths=800,  # Allows very long strings for notes
-        ):
-            print(df)
-
-    @property
-    def animals(self) -> tuple[str, ...]:
-        """Returns all unique animal IDs stored inside the manifest file.
-
-        This provides a tuple of all animal IDs participating in the target project.
-        """
-
-        # If animal IDs are stored as integers, converts them to string to support consistent return types.
-        return tuple(
-            [str(animal) for animal in self._data.select("animal").unique().sort("animal").to_series().to_list()]
-        )
-
-    def _get_filtered_sessions(
-        self,
-        animal: str | int | None = None,
-        exclude_incomplete: bool = True,
-    ) -> tuple[str, ...]:
-        """This worker method is used to get a list of sessions with optional filtering.
-
-        User-facing methods call this worker under-the-hood to fetch the filtered tuple of sessions.
-
-        Args:
-            animal: An optional animal ID to filter the sessions. If set to None, the method returns sessions for all
-                animals.
-            exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
-                list.
-
-        Returns:
-            The tuple of session IDs matching the filter criteria.
-
-        Raises:
-            ValueError: If the specified animal is not found in the manifest file.
-        """
-        data = self._data
-
-        # Filter by animal if specified
-        if animal is not None:
-            # Ensures that the 'animal' argument has the same type as the data inside the DataFrame.
-            if self._animal_string:
-                animal = str(animal)
-            else:
-                animal = int(animal)
-
-            if animal not in self.animals:
-                message = f"Animal ID '{animal}' not found in the project manifest. Available animals: {self.animals}."
-                console.error(message=message, error=ValueError)
-
-            data = data.filter(pl.col("animal") == animal)
-
-        # Optionally filters out incomplete sessions
-        if exclude_incomplete:
-            data = data.filter(pl.col("complete") == 1)
-
-        # Formats and returns session IDs to the caller
-        sessions = data.select("session").sort("session").to_series().to_list()
-        return tuple(sessions)
-
-    @property
-    def sessions(self) -> tuple[str, ...]:
-        """Returns all session IDs stored inside the manifest file.
-
-        This property provides a tuple of all sessions, independent of the participating animal, that were recorded as
-        part of the target project. Use the get_sessions() method to get the list of session tuples with filtering.
-        """
-        return self._get_filtered_sessions(animal=None, exclude_incomplete=False)
-
-    def get_sessions(
-        self,
-        animal: str | int | None = None,
-        exclude_incomplete: bool = True,
-    ) -> tuple[str, ...]:
-        """Returns requested session IDs based on selected filtering criteria.
-
-        This method provides a tuple of sessions based on the specified filters. If no animal is specified, returns
-        sessions for all animals in the project.
-
-        Args:
-            animal: An optional animal ID to filter the sessions. If set to None, the method returns sessions for all
-                animals.
-            exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
-                list.
-
-        Returns:
-            The tuple of session IDs matching the filter criteria.
-
-        Raises:
-            ValueError: If the specified animal is not found in the manifest file.
-        """
-        return self._get_filtered_sessions(
-            animal=animal,
-            exclude_incomplete=exclude_incomplete,
-        )
-
-    def get_session_info(self, session: str) -> pl.DataFrame:
-        """Returns a Polars DataFrame that stores detailed information for the specified session.
-
-        Since session IDs are unique, it is expected that filtering by session ID is enough to get the requested
-        information.
-
-        Args:
-            session: The ID of the session for which to retrieve the data.
-
-        Returns:
-            A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'system',
-            'complete', 'integrity', 'suite2p', 'behavior', 'video', 'archived'.
-        """
-
-        df = self._data
-        df = df.filter(pl.col("session").eq(session))
-        return df

{sl_shared_assets-5.1.1.dist-info → sl_shared_assets-6.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 5.1.1
+Version: 6.0.0
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -686,12 +686,10 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Typing :: Typed
-Requires-Python: >=3.11
+Requires-Python: <3.14,>=3.13
 Requires-Dist: appdirs<2,>=1
 Requires-Dist: ataraxis-base-utilities<4,>=3
 Requires-Dist: ataraxis-data-structures<4,>=3

{sl_shared_assets-5.1.1.dist-info → sl_shared_assets-6.0.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-sl_shared_assets/__init__.py,sha256=t3naIsPh7nKh4kFKBTOvEUHt3Wke3zH1KYSF6ON91x0,2830
+sl_shared_assets/__init__.py,sha256=rFNpMl0SFSQhBnCnmEYWsqf_4v0zmwt1xvSFFblUic0,2786
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sl_shared_assets/command_line_interfaces/__init__.py,sha256=dbcCiuBCe1uqz6d1D-dFpSkRwYP603tEGkc1RbZcI6w,126
-sl_shared_assets/command_line_interfaces/configure.py,sha256=lAWhLzJ7UmdvUZlges9Hg5pnblQuXSDlgzr12y41108,6140
+sl_shared_assets/command_line_interfaces/configure.py,sha256=ebzej7m2gZtJW_lH6m7ERIv1BcvYfqXNBxlUf0vXFq4,6115
 sl_shared_assets/command_line_interfaces/manage.py,sha256=-Ac5AufjiIpIYs2SSSstIAEHryZWsDHPt_cgWrNp8O4,10786
 sl_shared_assets/data_classes/__init__.py,sha256=7Cslz_z0K2VmgXjOR1u-NBySoWYfUNawBHh_1xLiFZ0,2092
 sl_shared_assets/data_classes/configuration_data.py,sha256=9gVo7bQShCMqAuau8d9BML6RrQyA4wg8ZFol1RJfJ7g,41334
@@ -10,14 +10,14 @@ sl_shared_assets/data_classes/session_data.py,sha256=gfzdFnBXwawWXQWiwZ3UXQeYaja
 sl_shared_assets/data_classes/surgery_data.py,sha256=-rENeSjAPmoIc-fp_skI4-EW_uxbK0Ps9aTLPWLkTSQ,7624
 sl_shared_assets/server/__init__.py,sha256=cIZFNE8pPPujp_FxIt6yT22L-4V4XBb1e_x4cnTrtSI,685
 sl_shared_assets/server/job.py,sha256=_u27gXwSfg3ztCemAtiskC9KnDfqF-RvWlTYYk0wHvQ,18704
-sl_shared_assets/server/pipeline.py,sha256=Uqg210L6N71OHTy4--8iocRz_bV5cBSUOVZnOk_u1zg,32157
+sl_shared_assets/server/pipeline.py,sha256=OtpF28Nd8590RoKkyGUBwLdOxVD692cLMX6mM-Rhicw,33134
 sl_shared_assets/server/server.py,sha256=FEBx9isTswndAKFaZ6MM4qyFp75FcE2ZWUiwc_uycMI,32825
-sl_shared_assets/tools/__init__.py,sha256=gMyKO3ZA70pmG7Hh8fIA5CLrkOXtN5pQk2o4NrKUjXM,813
-sl_shared_assets/tools/packaging_tools.py,sha256=9kwUSQQRCOycpNA7ovthCTXYX1HCC7p1iDnCYz2K2B8,6978
-sl_shared_assets/tools/project_management_tools.py,sha256=I1EqSapoQU2zspoMc_OxOrUMw_iYIqX6uIfmZZ3paGU,43205
+sl_shared_assets/tools/__init__.py,sha256=mz0YCAKm4UMYtLErBLXIRJ-QxkHV9S9j5BHrDZMqVWg,769
+sl_shared_assets/tools/packaging_tools.py,sha256=4MRYvXQ6PfW1bB_uB5ngDHR0NbgJQW2kCwZw_7U-224,6977
+sl_shared_assets/tools/project_management_tools.py,sha256=7oqR5NSA0pLHZyS-gKIJwajfGkdz1ENgJpuKyxBdN_8,33836
 sl_shared_assets/tools/transfer_tools.py,sha256=6lwk4zyHWWB0QeWQageiQ1pPIGUV0IhBNNzsbgQ_qH4,9026
-sl_shared_assets-5.1.1.dist-info/METADATA,sha256=pjm-qXxbdiIPJoOmiTUbOuQS0KnXIMMh5u1nJ6NNVYQ,46884
-sl_shared_assets-5.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-5.1.1.dist-info/entry_points.txt,sha256=19NzFPG5CcW-4RhEacTcX3J2TbrvmjOE4tlLCH2O5wI,161
-sl_shared_assets-5.1.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-5.1.1.dist-info/RECORD,,
+sl_shared_assets-6.0.0.dist-info/METADATA,sha256=mnepeiK1pc5pX8autsg1mDaNBHMmbyf2MtfI0CodsgM,46788
+sl_shared_assets-6.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-6.0.0.dist-info/entry_points.txt,sha256=19NzFPG5CcW-4RhEacTcX3J2TbrvmjOE4tlLCH2O5wI,161
+sl_shared_assets-6.0.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-6.0.0.dist-info/RECORD,,