sl-shared-assets 5.1.1__tar.gz → 6.0.0__tar.gz

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 5.1.1
+Version: 6.0.0
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -686,12 +686,10 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Typing :: Typed
-Requires-Python: >=3.11
+Requires-Python: <3.14,>=3.13
 Requires-Dist: appdirs<2,>=1
 Requires-Dist: ataraxis-base-utilities<4,>=3
 Requires-Dist: ataraxis-data-structures<4,>=3

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/pyproject.toml

@@ -1,16 +1,16 @@
 # Defines the build system used for the project.
 [build-system]
-requires = ["hatchling==1.27.0"]
+requires = ["hatchling>=1,<2"]
 build-backend = "hatchling.build"
 
 # Project metdata section. Provides the genral ID information about the project.
 [project]
 name = "sl-shared-assets"
-version = "5.1.1"
+version = "6.0.0"
 description = "Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries."
 readme = "README.md"
 license = { file = "LICENSE" }
-requires-python = ">=3.11"
+requires-python = ">=3.13,<3.14"
 authors = [
     { name = "Ivan Kondratyev" },
     { name = "Kushaan Gupta" },
@@ -31,8 +31,6 @@ classifiers = [
     # License
     "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
     # Supported Python Versions
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     # Supported OS systems
     "Operating System :: OS Independent",
@@ -101,7 +99,7 @@ packages = ["src/sl_shared_assets"]
 [tool.ruff]
 line-length = 120         # The maximum column length is set to 120 for this project.
 indent-width = 4          # Same as black, indents are 4 spaces
-target-version = "py311"  # Targets the lowest supported version of python
+target-version = "py313"  # Targets the lowest supported version of python
 src = ["src"]             # The name of the source directory
 
 # Excludes 'service' .py files, such as the sphinx configuration file, from linting.

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/src/sl_shared_assets/__init__.py

@@ -8,7 +8,6 @@ Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Natalie Yeung
 from ataraxis_base_utilities import console
 
 from .tools import (
-    ProjectManifest,
     acquire_lock,
     release_lock,
     delete_directory,
@@ -90,7 +89,6 @@ __all__ = [
     "ProcessingPipelines",
     "ProcessingStatus",
     "ProcessingTracker",
-    "ProjectManifest",
     "RawData",
     "RunTrainingDescriptor",
     "Server",

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/src/sl_shared_assets/command_line_interfaces/configure.py

@@ -15,7 +15,7 @@ from ..data_classes import (
 )
 
 # Ensures that displayed CLICK help messages are formatted according to the lab standard.
-CONTEXT_SETTINGS = dict(max_content_width=120)  # or any width you want
+CONTEXT_SETTINGS = dict(max_content_width=120)
 
 
 @click.group("manage", context_settings=CONTEXT_SETTINGS)

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/src/sl_shared_assets/server/pipeline.py

@@ -342,6 +342,9 @@ class ProcessingTracker(YamlConfig):
                 self._complete = True
                 self._encountered_error = False
                 self._save_state()
+            else:
+                # Otherwise, updates the completed job counter, but does not change any other state variables.
+                self._save_state()
 
     def abort(self) -> None:
         """Resets the pipeline tracker file to the default state.
@@ -524,11 +527,22 @@ class ProcessingPipeline:
         # to the server.
         elif tracker.is_running:
             self._pipeline_stage += 1
-            self._submit_jobs()
+
+            # If the incremented stage is not a valid stage, the pipeline has actually been aborted and the tracker file
+            # does not properly reflect this state. Sets the internal state tracker appropriately and resets (removes)
+            # the tracker file from the server to prevent deadlocking further runtimes
+            if self._pipeline_stage not in self.jobs.keys():
+                sh.rmtree(self.local_tracker_path.parent)  # Removes local temporary data
+                self.pipeline_status = ProcessingStatus.ABORTED
+                self.server.remove(remote_path=self.remote_tracker_path, is_dir=False)
+            else:
+                # Otherwise, submits the next batch of jobs to the server.
+                self._submit_jobs()
 
         # The final and the rarest state: the pipeline was aborted before it finished the runtime. Generally, this state
         # should not be encountered during most runtimes.
         else:
+            sh.rmtree(self.local_tracker_path.parent)  # Removes local temporary data
             self.pipeline_status = ProcessingStatus.ABORTED
 
     def _submit_jobs(self) -> None:

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/src/sl_shared_assets/tools/__init__.py

@@ -4,7 +4,6 @@ integrity of the data. The tools from this package are used by most other data p
 from .transfer_tools import delete_directory, transfer_directory
 from .packaging_tools import calculate_directory_checksum
 from .project_management_tools import (
-    ProjectManifest,
     acquire_lock,
     release_lock,
     archive_session,
@@ -14,7 +13,6 @@ from .project_management_tools import (
 )
 
 __all__ = [
-    "ProjectManifest",
     "acquire_lock",
     "archive_session",
     "calculate_directory_checksum",

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/src/sl_shared_assets/tools/packaging_tools.py

@@ -115,7 +115,7 @@ def calculate_directory_checksum(
         results = []
         if not batch:
             with tqdm(
-                total=len(files), desc=f"Calculating checksum for {Path(*directory.parts[-6:])}", unit="files"
+                total=len(files), desc=f"Calculating checksum for {Path(*directory.parts[-6:])}", unit="file"
             ) as pbar:
                 for future in as_completed(future_to_path):
                     results.append(future.result())

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/src/sl_shared_assets/tools/project_management_tools.py

@@ -661,236 +661,3 @@ def generate_project_manifest(
             # If the tracker indicates that the processing is still running, the runtime has encountered an error.
             if runtime_tracker.is_running:
                 tracker.error(manager_id=manager_id)
-
-
-class ProjectManifest:
-    """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
-    working with the data stored inside the file.
-
-    This class functions as a high-level API for working with Sun lab projects. It is used both to visualize the
-    current state of various projects and during automated data processing to determine which processing steps to
-    apply to different sessions.
-
-    Args:
-        manifest_file: The path to the .feather manifest file that stores the target project's state data.
-
-    Attributes:
-        _data: Stores the manifest data as a Polars DataFrame.
-        _animal_string: Determines whether animal IDs are stored as strings or unsigned integers.
-    """
-
-    def __init__(self, manifest_file: Path):
-        # Reads the data from the target manifest file into the class attribute
-        self._data: pl.DataFrame = pl.read_ipc(source=manifest_file, use_pyarrow=True)
-
-        # Determines whether animal IDs are stored as strings or as numbers
-        self._animal_string = False
-        schema = self._data.collect_schema()
-        if isinstance(schema["animal"], pl.String):
-            self._animal_string = True
-
-    def print_data(self) -> None:
-        """Prints the entire contents of the manifest file to the terminal."""
-        with pl.Config(
-            set_tbl_rows=-1,  # Displays all rows (-1 means unlimited)
-            set_tbl_cols=-1,  # Displays all columns (-1 means unlimited)
-            set_tbl_hide_column_data_types=True,
-            set_tbl_cell_alignment="LEFT",
-            set_tbl_width_chars=250,  # Sets table width to 200 characters
-            set_fmt_str_lengths=600,  # Allows longer strings to display properly (default is 32)
-        ):
-            print(self._data)
-
-    def print_summary(self, animal: str | int | None = None) -> None:
-        """Prints a summary view of the manifest file to the terminal, excluding the 'experimenter notes' data for
-        each session.
-
-        This data view is optimized for tracking which processing steps have been applied to each session inside the
-        project.
-
-        Args:
-            animal: The ID of the animal for which to display the data. If an ID is provided, this method will only
-                display the data for that animal. Otherwise, it will display the data for all animals.
-        """
-        summary_cols = [
-            "animal",
-            "date",
-            "session",
-            "type",
-            "system",
-            "complete",
-            "integrity",
-            "prepared",
-            "suite2p",
-            "behavior",
-            "video",
-            "archived",
-        ]
-
-        # Retrieves the data
-        df = self._data.select(summary_cols)
-
-        # Optionally filters the data for the target animal
-        if animal is not None:
-            # Ensures that the 'animal' argument has the same type as the data inside the DataFrame.
-            if self._animal_string:
-                animal = str(animal)
-            else:
-                animal = int(animal)
-            df = df.filter(pl.col("animal") == animal)
-
-        # Ensures the data displays properly
-        with pl.Config(
-            set_tbl_rows=-1,
-            set_tbl_cols=-1,
-            set_tbl_width_chars=250,
-            set_tbl_hide_column_data_types=True,
-            set_tbl_cell_alignment="CENTER",
-        ):
-            print(df)
-
-    def print_notes(self, animal: str | int | None = None) -> None:
-        """Prints only animal, session, and notes data from the manifest file.
-
-        This data view is optimized for experimenters to check what sessions have been recorded for each animal in the
-        project and refresh their memory on the outcomes of each session using experimenter notes.
-
-        Args:
-            animal: The ID of the animal for which to display the data. If an ID is provided, this method will only
-                display the data for that animal. Otherwise, it will display the data for all animals.
-        """
-
-        # Pre-selects the columns to display
-        df = self._data.select(["animal", "date", "session", "type", "system", "notes"])
-
-        # Optionally filters the data for the target animal
-        if animal is not None:
-            # Ensures that the 'animal' argument has the same type as the data inside the DataFrame.
-            if self._animal_string:
-                animal = str(animal)
-            else:
-                animal = int(animal)
-
-            df = df.filter(pl.col("animal") == animal)
-
-        #  Prints the extracted data
-        with pl.Config(
-            set_tbl_rows=-1,
-            set_tbl_cols=-1,
-            set_tbl_hide_column_data_types=True,
-            set_tbl_cell_alignment="LEFT",
-            set_tbl_width_chars=100,  # Wider columns for notes
-            set_fmt_str_lengths=800,  # Allows very long strings for notes
-        ):
-            print(df)
-
-    @property
-    def animals(self) -> tuple[str, ...]:
-        """Returns all unique animal IDs stored inside the manifest file.
-
-        This provides a tuple of all animal IDs participating in the target project.
-        """
-
-        # If animal IDs are stored as integers, converts them to string to support consistent return types.
-        return tuple(
-            [str(animal) for animal in self._data.select("animal").unique().sort("animal").to_series().to_list()]
-        )
-
-    def _get_filtered_sessions(
-        self,
-        animal: str | int | None = None,
-        exclude_incomplete: bool = True,
-    ) -> tuple[str, ...]:
-        """This worker method is used to get a list of sessions with optional filtering.
-
-        User-facing methods call this worker under-the-hood to fetch the filtered tuple of sessions.
-
-        Args:
-            animal: An optional animal ID to filter the sessions. If set to None, the method returns sessions for all
-                animals.
-            exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
-                list.
-
-        Returns:
-            The tuple of session IDs matching the filter criteria.
-
-        Raises:
-            ValueError: If the specified animal is not found in the manifest file.
-        """
-        data = self._data
-
-        # Filter by animal if specified
-        if animal is not None:
-            # Ensures that the 'animal' argument has the same type as the data inside the DataFrame.
-            if self._animal_string:
-                animal = str(animal)
-            else:
-                animal = int(animal)
-
-            if animal not in self.animals:
-                message = f"Animal ID '{animal}' not found in the project manifest. Available animals: {self.animals}."
-                console.error(message=message, error=ValueError)
-
-            data = data.filter(pl.col("animal") == animal)
-
-        # Optionally filters out incomplete sessions
-        if exclude_incomplete:
-            data = data.filter(pl.col("complete") == 1)
-
-        # Formats and returns session IDs to the caller
-        sessions = data.select("session").sort("session").to_series().to_list()
-        return tuple(sessions)
-
-    @property
-    def sessions(self) -> tuple[str, ...]:
-        """Returns all session IDs stored inside the manifest file.
-
-        This property provides a tuple of all sessions, independent of the participating animal, that were recorded as
-        part of the target project. Use the get_sessions() method to get the list of session tuples with filtering.
-        """
-        return self._get_filtered_sessions(animal=None, exclude_incomplete=False)
-
-    def get_sessions(
-        self,
-        animal: str | int | None = None,
-        exclude_incomplete: bool = True,
-    ) -> tuple[str, ...]:
-        """Returns requested session IDs based on selected filtering criteria.
-
-        This method provides a tuple of sessions based on the specified filters. If no animal is specified, returns
-        sessions for all animals in the project.
-
-        Args:
-            animal: An optional animal ID to filter the sessions. If set to None, the method returns sessions for all
-                animals.
-            exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
-                list.
-
-        Returns:
-            The tuple of session IDs matching the filter criteria.
-
-        Raises:
-            ValueError: If the specified animal is not found in the manifest file.
-        """
-        return self._get_filtered_sessions(
-            animal=animal,
-            exclude_incomplete=exclude_incomplete,
-        )
-
-    def get_session_info(self, session: str) -> pl.DataFrame:
-        """Returns a Polars DataFrame that stores detailed information for the specified session.
-
-        Since session IDs are unique, it is expected that filtering by session ID is enough to get the requested
-        information.
-
-        Args:
-            session: The ID of the session for which to retrieve the data.
-
-        Returns:
-            A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'system',
-            'complete', 'integrity', 'suite2p', 'behavior', 'video', 'archived'.
-        """
-
-        df = self._data
-        df = df.filter(pl.col("session").eq(session))
-        return df

{sl_shared_assets-5.1.1 → sl_shared_assets-6.0.0}/tox.ini

@@ -29,7 +29,7 @@ description =
     Runs static code formatting, style, and typing checkers. Mypy may not always work properly until py.typed marker is
     added the first time the 'stubs' task is executed.
 extras = dev
-basepython = py311
+basepython = py313
 commands =
     automation-cli purge-stubs
     ruff check --select I,NPY,FLY,ERA,PTH,N,FURB,TRY,RUF,UP,ARG --fix