sl-shared-assets 1.0.0rc26__py3-none-any.whl → 1.0.0rc28__py3-none-any.whl

sl_shared_assets/cli.py

@@ -8,6 +8,7 @@ from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
 from .tools import ascend_tyche_data, verify_session_checksum, generate_project_manifest
 from .server import generate_server_credentials
 from .data_classes import (
+    SessionData,
     ExperimentState,
     ProjectConfiguration,
     MesoscopeSystemConfiguration,
@@ -57,18 +58,25 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     that stores the data checksum generated before transferring the data to long-term storage destination. This function
     always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any other
     directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the session as
-    'incomplete' and automatically excluding it from all further automated processing runtimes.
+    'incomplete' and automatically excluding it from all further automated processing runtimes. if the session data
+    is intact, generates a 'verified.bin' marker file inside the session's raw_data folder.
 
     The command is also used by Sun lab data acquisition systems to generate the processed data hierarchy for each
     processed session. This use case is fully automated and should not be triggered manually by the user.
     """
     session = Path(session_path)
-    if verify_session_checksum(
+
+    # Runs the verification process
+    verify_session_checksum(
         session, create_processed_data_directory=create_processed_directories, processed_data_root=processed_data_root
-    ):
-        console.echo(message=f"Session {session.stem} raw data integrity: verified.", level=LogLevel.SUCCESS)
+    )
+
+    # Checks the outcome of the verification process
+    session_data = SessionData.load(session_path=session)
+    if session_data.raw_data.verified_bin_path.exists():
+        console.echo(message=f"Session {session.stem} raw data integrity: Verified.", level=LogLevel.SUCCESS)
     else:
-        console.echo(message=f"Session {session.stem} raw data integrity: compromised!", level=LogLevel.ERROR)
+        console.echo(message=f"Session {session.stem} raw data integrity: Compromised!", level=LogLevel.ERROR)
 
 
 @click.command()

sl_shared_assets/cli.pyi

@@ -7,6 +7,7 @@ from .tools import (
 )
 from .server import generate_server_credentials as generate_server_credentials
 from .data_classes import (
+    SessionData as SessionData,
     ExperimentState as ExperimentState,
     ProjectConfiguration as ProjectConfiguration,
     MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
@@ -22,7 +23,8 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     that stores the data checksum generated before transferring the data to long-term storage destination. This function
     always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any other
     directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the session as
-    'incomplete' and automatically excluding it from all further automated processing runtimes.
+    'incomplete' and automatically excluding it from all further automated processing runtimes. if the session data
+    is intact, generates a 'verified.bin' marker file inside the session's raw_data folder.
 
     The command is also used by Sun lab data acquisition systems to generate the processed data hierarchy for each
     processed session. This use case is fully automated and should not be triggered manually by the user.

sl_shared_assets/data_classes/session_data.py

@@ -217,6 +217,11 @@ class RawData:
     the long-term storage destinations (NAS and Server) and the integrity of the moved data is verified on at least one 
     destination. During 'purge' sl-experiment runtimes, the library discovers and removes all session data marked with 
     'ubiquitin.bin' files from the machine that runs the code."""
+    verified_bin_path: Path = Path()
+    """Stores the path to the verified.bin file. This marker file is created (or removed) by the sl-shared-assets 
+    'verify-session' CLI command to indicate whether the session data inside the folder marked by the file has been 
+    verified for integrity. Primarily, this is used when the data is moved to the long-term storage destination (BioHPC
+    server) to ensure it is safe to remove any local copies of the data stored on the acquisition system machine(s)."""
 
     def resolve_paths(self, root_directory_path: Path) -> None:
         """Resolves all paths managed by the class instance based on the input root directory path.
@@ -248,6 +253,7 @@ class RawData:
         self.system_configuration_path = self.raw_data_path.joinpath("system_configuration.yaml")
         self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
         self.ubiquitin_path = self.raw_data_path.joinpath("ubiquitin.bin")
+        self.verified_bin_path = self.raw_data_path.joinpath("verified.bin")
 
     def make_directories(self) -> None:
         """Ensures that all major subdirectories and the root directory exist, creating any missing directories."""

sl_shared_assets/data_classes/session_data.pyi

@@ -96,6 +96,7 @@ class RawData:
     checksum_path: Path = ...
     telomere_path: Path = ...
     ubiquitin_path: Path = ...
+    verified_bin_path: Path = ...
     def resolve_paths(self, root_directory_path: Path) -> None:
         """Resolves all paths managed by the class instance based on the input root directory path.
 

sl_shared_assets/server/job.py

@@ -99,7 +99,7 @@ class Job:
         self._command.add_cmd("conda init bash")
 
         # Activates the target conda environment for the command.
-        self._command.add_cmd(f"conda activate {conda_environment}")
+        self._command.add_cmd(f"source activate {conda_environment}")  # Need to use old syntax for our server.
 
     def __repr__(self) -> str:
         """Returns the string representation of the Job instance."""

sl_shared_assets/server/server.py

@@ -131,7 +131,7 @@ class Server:
             job: The Job object that contains all job data.
 
         Returns:
-            The job object whose 'job_id' attribute had been modified with the job ID, if the job was successfully
+            The job object whose 'job_id' attribute had been modified with the job ID if the job was successfully
             submitted.
 
         Raises:

sl_shared_assets/server/server.pyi

@@ -69,7 +69,7 @@ class Server:
             job: The Job object that contains all job data.
 
         Returns:
-            The job object whose 'job_id' attribute had been modified with the job ID, if the job was successfully
+            The job object whose 'job_id' attribute had been modified with the job ID if the job was successfully
             submitted.
 
         Raises:

sl_shared_assets/tools/project_management_tools.py

@@ -61,6 +61,7 @@ def generate_project_manifest(
         "raw_data": [],  # Server-side raw_data folder path.
         "processed_data": [],  # Server-side processed_data folder path.
         "complete": [],  # Determines if the session data is complete. Incomplete sessions are excluded from processing.
+        "verified": [],  # Determines if the session data integrity has been verified upon transfer to storage machine.
         "single_day_suite2p": [],  # Determines whether the session has been processed with the single-day s2p pipeline.
         "multi_day_suite2p": [],  # Determines whether the session has been processed with the multi-day s2p pipeline.
         "behavior": [],  # Determines whether the session has been processed with the behavior extraction pipeline.
@@ -87,9 +88,13 @@ def generate_project_manifest(
         # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
         manifest["complete"].append(session_data.raw_data.telomere_path.exists())
 
-        # If the session is incomplete, marks all processing steps as FALSE, as automatic processing is disabled for
-        # incomplete sessions.
-        if not manifest["complete"][-1]:
+        # If the session raw_data folder contains the verified.bin file, marks the session as verified.
+        manifest["verified"].append(session_data.raw_data.verified_bin_path.exists())
+
+        # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing is
+        # disabled for incomplete sessions. If the session unverified, the case is even more severe, as its data may be
+        # corrupted.
+        if not manifest["complete"][-1] or not not manifest["verified"][-1]:
             manifest["single_day_suite2p"].append(False)
             manifest["multi_day_suite2p"].append(False)
             manifest["behavior"].append(False)
@@ -120,6 +125,7 @@ def generate_project_manifest(
         "processed_data": pl.String,
         "type": pl.String,
         "complete": pl.Boolean,
+        "verified": pl.Boolean,
         "single_day_suite2p": pl.Boolean,
         "multi_day_suite2p": pl.Boolean,
         "behavior": pl.Boolean,
@@ -139,13 +145,13 @@ def generate_project_manifest(
 
 def verify_session_checksum(
     session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
-) -> bool:
+) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
 
     Primarily, this function is used to verify data integrity after transferring it from a local PC to the remote
-    server for long-term storage. This function is designed to do nothing if the checksum matches and to remove the
-    'telomere.bin' marker file if it does not.
+    server for long-term storage. This function is designed to create the 'verified.bin' marker file if the checksum
+    matches and to remove the 'telomere.bin' and 'verified.bin' marker files if it does not.
 
     Notes:
         Removing the telomere.bin marker file from session's raw_data folder marks the session as incomplete, excluding
@@ -161,9 +167,6 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
-
-    Returns:
-        True if the checksum matches, False otherwise.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -185,10 +188,13 @@ def verify_session_checksum(
     # If the two checksums do not match, this likely indicates data corruption.
     if stored_checksum != calculated_checksum:
         # If the telomere.bin file exists, removes this file. This automatically marks the session as incomplete for
-        # all other Sun lab runtimes. The presence of the telomere.bin file after integrity verification is used as a
-        # heuristic for determining whether the session has passed the verification process.
-        if session_data.raw_data.telomere_path.exists():
-            session_data.raw_data.telomere_path.unlink()
-        return False
+        # all other Sun lab runtimes.
+        session_data.raw_data.telomere_path.unlink(missing_ok=True)
+
+        # Also unlinks the verified.bin marker if it exists. The presence or absence of the marker is used as the
+        # primary heuristic for determining if the session data passed verification.
+        session_data.raw_data.verified_bin_path.unlink(missing_ok=True)
 
-    return True
+    # Otherwise, ensures that the session is marked with the verified.bin marker file.
+    else:
+        session_data.raw_data.verified_bin_path.touch(exist_ok=True)

sl_shared_assets/tools/project_management_tools.pyi

@@ -29,13 +29,13 @@ def generate_project_manifest(
 
 def verify_session_checksum(
     session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
-) -> bool:
+) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
 
     Primarily, this function is used to verify data integrity after transferring it from a local PC to the remote
-    server for long-term storage. This function is designed to do nothing if the checksum matches and to remove the
-    'telomere.bin' marker file if it does not.
+    server for long-term storage. This function is designed to create the 'verified.bin' marker file if the checksum
+    matches and to remove the 'telomere.bin' and 'verified.bin' marker files if it does not.
 
     Notes:
         Removing the telomere.bin marker file from session's raw_data folder marks the session as incomplete, excluding
@@ -51,7 +51,4 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
-
-    Returns:
-        True if the checksum matches, False otherwise.
     """

{sl_shared_assets-1.0.0rc26.dist-info → sl_shared_assets-1.0.0rc28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc26
+Version: 1.0.0rc28
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-1.0.0rc26.dist-info → sl_shared_assets-1.0.0rc28.dist-info}/RECORD

@@ -1,7 +1,7 @@
 sl_shared_assets/__init__.py,sha256=rscR353jiyUQ-wpguTLOM-A5Lqr1ftQtuTan1D0AtR0,2196
 sl_shared_assets/__init__.pyi,sha256=Ye6eY_y_l9CTqwbCf-OxXOphxXKfn9UJnuw09DdLEtc,2405
-sl_shared_assets/cli.py,sha256=-w9RAbHEcMspp6UDiXHM0fYSUwPSjJpZZcX3T9hLzUc,17103
-sl_shared_assets/cli.pyi,sha256=DuhJhqAXB7e5uOdzCI2c29FwsVwpQefR2eLUfW0gWyQ,5931
+sl_shared_assets/cli.py,sha256=9PXEfRgKb9-6xb1icmtr_ijdWur-AIsdMwvgA0aAeto,17434
+sl_shared_assets/cli.pyi,sha256=p1tUflODySXx1_2IrLZ_uqixopbQPTearuJvqJ4jKI4,6075
 sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sl_shared_assets/data_classes/__init__.py,sha256=b0BwTAJCD1zbwjd2UdpXheq61q1sgBuYhtAL-GHA2h4,1835
 sl_shared_assets/data_classes/__init__.pyi,sha256=e2082sm5pSw3bxNZGqwcRhuKLH8T7hcEo6kWtOZNDBU,1968
@@ -9,28 +9,28 @@ sl_shared_assets/data_classes/configuration_data.py,sha256=wuPFz3zx1DN12KimPhH-f
 sl_shared_assets/data_classes/configuration_data.pyi,sha256=FWQG6LlF6gSDV5bU0KvMvIE6wf0Jfrz5wRbuHwyCKHw,9600
 sl_shared_assets/data_classes/runtime_data.py,sha256=Q7Ykf9hgrw1jYKXa53mn_LW8G2cPmLLuxgGkP6qQcc4,15483
 sl_shared_assets/data_classes/runtime_data.pyi,sha256=PxaCbeF9COR4ri91pdzh7zVrqaz2KEDYB1EoLhZQC_c,6618
-sl_shared_assets/data_classes/session_data.py,sha256=YJCZi10amiRZxuGHv2UYB4NaRnorywBUNxZnRHlQsMs,37743
-sl_shared_assets/data_classes/session_data.pyi,sha256=FtqkYDOufLmdPZWzz4J8U5KGOkuTSC1cjg05q-QEopM,13209
+sl_shared_assets/data_classes/session_data.py,sha256=oAiTNqNlQi6NQHv5WsaQb3KglBrDGAmBv1EVPPuM6Y4,38329
+sl_shared_assets/data_classes/session_data.pyi,sha256=eLemAagVD6TQ-9pzMIc0eXDgnh-quJwEidKHYitvsjQ,13243
 sl_shared_assets/data_classes/surgery_data.py,sha256=qsMj3NkjhylAT9b_wHBY-1XwTu2xsZcZatdECmkA7Bs,7437
 sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
 sl_shared_assets/server/__init__.py,sha256=nyX6-9ACcrQeRQOCNvBVrWSTHGjRPANIG_u0aq7HPTg,426
 sl_shared_assets/server/__init__.pyi,sha256=7o99f8uf6NuBjMZjNAM1FX69Qbu5uBluRSAyaUWbXOU,263
-sl_shared_assets/server/job.py,sha256=M7ZytqhluEV-YQPM9VQV3SqK-F9egQ3UcLc1SLBH4rA,7885
+sl_shared_assets/server/job.py,sha256=GB31yYPEqXR6MgwNmnQrSQuHRJqUHFXcd6p7hb38q_c,7928
 sl_shared_assets/server/job.pyi,sha256=cxgHMpuwHsJGf_ZcTSSa2tZNzeR_GxqlICOsYGV_oy0,5655
-sl_shared_assets/server/server.py,sha256=lgHkg0_pHo0CJA6A6FGb22FC8PKVaKakeaJn4tfWk6I,9382
-sl_shared_assets/server/server.pyi,sha256=uu-bs5LBuOF-AL7qroYFbGc1aWgBalm-uHUXzZNxheU,4144
+sl_shared_assets/server/server.py,sha256=Rsc5rSmthFcKMsY3fH0la2GRCZsmB1j7fnz7T0RkSok,9381
+sl_shared_assets/server/server.pyi,sha256=SoTRqSPJ1GQ5i5gV3N57rDT9xzeT47RjMRH1FYKsOkg,4143
 sl_shared_assets/tools/__init__.py,sha256=N95ZPMz-_HdNPrbVieCFza-QSVS6BV2KRB4K1OLRttc,636
 sl_shared_assets/tools/__init__.pyi,sha256=xeDF8itMc0JRgLrO_IN_9gW7cp_Ld-Gf-rjtrgWvQ2I,551
 sl_shared_assets/tools/ascension_tools.py,sha256=kIqYGX9F8lRao_LaVOacIiT8J9SypTvarb9mgaI9ZPs,15387
 sl_shared_assets/tools/ascension_tools.pyi,sha256=tQCDdWZ20ZjUjpMs8aGIN0yBg5ff3j6spi62b3Han4o,3755
 sl_shared_assets/tools/packaging_tools.py,sha256=oY-EWuTiMfWobYllVZy0piGlVnTHCpPY1GF-WmqQdj4,7269
 sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=E50xhjJZlq6zQXnj8DHm1KkVcTDK7lr61m1ge-u25TY,10476
-sl_shared_assets/tools/project_management_tools.pyi,sha256=bRwohpGa98LDIy1ntLAPSKEGRB4S7ZmnIfyDy097c94,3467
+sl_shared_assets/tools/project_management_tools.py,sha256=Z1wEo0ei6xuV-8_DZQPejHwJGPFvE3AGKwvASW6cevY,11126
+sl_shared_assets/tools/project_management_tools.pyi,sha256=HiU1qqumscmXtAXNX6eaSrkwanGHEvk-lromwZ2ijD8,3445
 sl_shared_assets/tools/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
 sl_shared_assets/tools/transfer_tools.pyi,sha256=FoH7eYZe7guGHfPr0MK5ggO62uXKwD2aJ7h1Bu7PaEE,3294
-sl_shared_assets-1.0.0rc26.dist-info/METADATA,sha256=62dj0yu8PMgJEHQeqv6ntd-PQDpLEVW-dTJxhELrcKI,48613
-sl_shared_assets-1.0.0rc26.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-1.0.0rc26.dist-info/entry_points.txt,sha256=76c00fRS4IuXBP2xOBdvycT15Zen-lHiDg2FaSt-HB4,547
-sl_shared_assets-1.0.0rc26.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-1.0.0rc26.dist-info/RECORD,,
+sl_shared_assets-1.0.0rc28.dist-info/METADATA,sha256=3XV5KTtWCVfvMZnFkPbVEgTC6WUqZcu7n2xqFPvhxMw,48613
+sl_shared_assets-1.0.0rc28.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-1.0.0rc28.dist-info/entry_points.txt,sha256=76c00fRS4IuXBP2xOBdvycT15Zen-lHiDg2FaSt-HB4,547
+sl_shared_assets-1.0.0rc28.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-1.0.0rc28.dist-info/RECORD,,