sl-shared-assets 1.0.0rc8__py3-none-any.whl → 1.0.0rc9__py3-none-any.whl

sl_shared_assets/cli.py

@@ -8,6 +8,7 @@ import click
 
 from .server import generate_server_credentials
 from .data_classes import replace_root_path
+from .legacy_tools import ascend_tyche_data
 
 
 @click.command()
@@ -70,3 +71,45 @@ def generate_server_credentials_file(output_directory: str, host: str, username:
     generate_server_credentials(
         output_directory=Path(output_directory), username=username, password=password, host=host
     )
+
+
+@click.command()
+@click.option(
+    "-p",
+    "--path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    prompt="Enter the absolute path to the root directory storing Tyche animal folders to ascend (modernize): ",
+    help="The path to the root directory storing Tyche animal folders to ascend (modernize).",
+)
+@click.option(
+    "-o",
+    "--output_directory",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    prompt="Enter the path to the local directory where to create the ascended Tyche project hierarchy: ",
+    help="The path to the local directory where to create the ascended Tyche project hierarchy.",
+)
+@click.option(
+    "-s",
+    "--server_directory",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    prompt="Enter the path to the SMB-mounted BioHPC server directory that will be used to store the ascended data: ",
+    help="The path to the SMB-mounted BioHPC server directory that will be used to store the ascended data.",
+)
+def ascend_tyche_directory(path: str, output_directory: str, server_directory: str) -> None:
+    """Restructures all original Tyche folders to use the modern Sun lab data structure.
+
+    This CLI is used to convert the old Tyche data to make it compatible with modern Sun lab processing pipelines and
+    data management workflows. This process is commonly referred to as 'ascension' amongst lab engineers. After
+    ascension, the data can be processed and analyzed using all modern Sun lab (sl-) tools and libraries.
+
+    Note! This CLi does NOT move the data to the BioHPC server. The data has to be manually transferred to the server
+    before it can be processed using our server-side pipelines.
+    """
+    ascend_tyche_data(
+        root_directory=Path(path),
+        output_root_directory=Path(output_directory),
+        server_root_directory=Path(server_directory),
+    )

sl_shared_assets/data_classes.py

@@ -13,7 +13,6 @@ import appdirs
 from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
 from ataraxis_data_structures import YamlConfig
 from ataraxis_time.time_helpers import get_timestamp
-import copy
 
 
 def replace_root_path(path: Path) -> None:
@@ -230,7 +229,8 @@ class ProjectConfiguration(YamlConfig):
             # this process, the class generates the correct 'local_root_path' based on the path provided by the
             # user.
             precursor = ProjectConfiguration(local_root_directory=Path(str(configuration_path.parents[2])))
-            precursor._to_path(path=configuration_path)
+            precursor.project_name = project_name
+            precursor.to_path(path=configuration_path)
 
             # Waits for the user to manually configure the newly created file.
             input(f"Enter anything to continue: ")
@@ -263,7 +263,7 @@ class ProjectConfiguration(YamlConfig):
         # Returns the initialized class instance to caller
         return instance
 
-    def _to_path(self, path: Path) -> None:
+    def to_path(self, path: Path) -> None:
         """Saves the instance data to disk as a project_configuration.yaml file.
 
         This method is automatically called when the project is created. All future runtimes should use the load()
@@ -797,6 +797,7 @@ class SessionData(YamlConfig):
         session_type: str,
         project_configuration: ProjectConfiguration,
         experiment_name: str | None = None,
+        session_name: str | None = None,
     ) -> "SessionData":
         """Creates a new SessionData object and uses it to generate the session's data structure.
 
@@ -821,13 +822,17 @@ class SessionData(YamlConfig):
                 copy it into the session's raw_data directory.
             project_configuration: The initialized ProjectConfiguration instance that stores the data for the session's
                 project. This is used to determine the root directory paths for all PCs used in the data workflow.
+            session_name: An optional session_name override. Generally, this argument should not be provided for most
+                use cases. When provided, the method uses this name instead of generating a new timestamp-based name.
+                This is only used when reformatting other data structures to follow Sun lab structure.
 
         Returns:
             An initialized SessionData instance for the newly created session.
         """
 
         # Acquires the UTC timestamp to use as the session name
-        session_name = str(get_timestamp(time_separator="-"))
+        if session_name is None:
+            session_name = str(get_timestamp(time_separator="-"))
 
         # Extracts the root directory paths stored inside the project configuration file. All roots are expected to be
         # mounted on the local (VRPC) via SMB or equivalent protocol and be relative to the VRPC root.

sl_shared_assets/legacy_tools.py

@@ -0,0 +1,262 @@
+"""This module provides tools for working with old (legacy) data and data formats. Primarily, they are used to reformat
+old Tyche project data to make it compatible with modern Sun lab pipelines."""
+
+from pathlib import Path
+import datetime
+import tempfile
+
+import numpy as np
+from ataraxis_base_utilities import LogLevel, console
+from ataraxis_time.time_helpers import extract_timestamp_from_bytes
+
+from .data_classes import SessionData, ProjectConfiguration
+from .packaging_tools import calculate_directory_checksum
+
+
+def _generate_session_name(acquisition_path: Path) -> str:
+    """Generates a session name using the last modification time of a zstack.mat or MotionEstimator.me file.
+
+    This worker function uses one of the motion estimation files stored in each 'acquisition' subfolder of the original
+    Tyche session data structure to generate a compatible timestamp-based session name. This is used to translate the
+    original Tyche data hierarchy into the hierarchy used by all modern Sun lab projects and pipelines.
+
+    Args:
+        acquisition_path: The absolute path to the target acquisition folder. These folders are found under the 'day'
+            folders for each animal, e.g.: Tyche-A7/2022_01_03/1.
+
+    Returns:
+        The generated session name for that acquisition.
+    """
+
+    # All well-formed sessions are expected to contain both the zstack.mat and the MotionEstimator.me file.
+    # We use the last modification time from one of these files to infer when the session was carried out. This allows
+    # us to gather the time information, which is missing from the original session naming pattern.
+    source: Path
+    if acquisition_path.joinpath("zstack.mat").exists():
+        source = acquisition_path.joinpath("zstack.mat")
+    elif acquisition_path.joinpath("MotionEstimator.me").exists():
+        source = acquisition_path.joinpath("MotionEstimator.me")
+    else:
+        message = (
+            f"Unable to find zstack.mat or MotionEstimator.me file in the target acquisition subfolder "
+            f"{acquisition_path} of the session {acquisition_path.parent}. Manual intervention is required to ascend "
+            f"the target session folder to the latest Sun lab data format."
+        )
+        console.error(message=message, error=FileNotFoundError)
+        raise FileNotFoundError(message)  # Fall-back to appease mypy
+
+    # Gets last modified time (available on all platforms) and converts it to a UTC timestamp object.
+    mod_time = source.stat().st_mtime
+    mod_datetime = datetime.datetime.fromtimestamp(mod_time)
+
+    # Converts the timestamp to microseconds as uint64, then to an array of 8 uint8 bytes. The array is then reformatted
+    # to match the session name pattern used in the modern Sun lab data pipelines.
+    timestamp_microseconds = np.uint64(int(mod_datetime.timestamp() * 1_000_000))
+    timestamp_bytes = np.array([(timestamp_microseconds >> (8 * i)) & 0xFF for i in range(8)], dtype=np.uint8)
+    stamp = extract_timestamp_from_bytes(timestamp_bytes=timestamp_bytes)
+
+    # Returns the generated session name to caller.
+    return stamp
+
+
+def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
+    """Reorganizes and moves the session data from the source acquisition folder to the newly generated session
+    hierarchy.
+
+    This worker function is used to physically rearrange the data from the original Tyche acquisition folder to the
+    newly created session hierarchy. It both moves the existing files to their new destinations and renames certain
+    files to match the latest naming convention used in the Sun lab.
+
+    Args:
+        session_data: The initialized SessionData instance managing the newly created session data hierarchy.
+        source_root: The absolute path to the source Tyche acquisition folder.
+
+    Returns:
+        True if all expected data was found and moved. False, if any expected file was not found inside the folder, or
+        the reorganization process otherwise behaved unexpectedly. If this function returns False, the folder is
+        statically flagged for manual user intervention to investigate the issue.
+    """
+
+    # Resolves expected data targets:
+
+    # These files should be present in all well-formed session data folders. While not all session folders are
+    # well-formed, we will likely exclude any non-well-formed folders from processing.
+    zstack_path = source_root.joinpath("zstack.mat")
+    motion_estimator_path = source_root.joinpath("MotionEstimator.me")
+    ops_path = source_root.joinpath("ops.json")
+    mesoscope_frames_path = source_root.joinpath("mesoscope_frames")
+    ax_checksum_path = source_root.joinpath("ax_checksum.txt")
+
+    # These two file types are present for some, but not all folders. They are not as important as the group of files
+    # above though, as, currently, the data stored in these files is not used during processing.
+    frame_metadata_path = source_root.joinpath("frame_metadata.npz")
+    metadata_path = source_root.joinpath("metadata.json")
+
+    # This tracker is used to mark the session for manual intervention if any expected data is missing from the source
+    # session folder. At the end of this function's runtime, it determines whether the function returns True or False
+    data_missing = False
+
+    # First, moves the mesoscope TIFF stacks to the newly created session data hierarchy as mesoscope_data subfolder
+    if mesoscope_frames_path.exists():
+        mesoscope_frames_path.rename(session_data.raw_data.mesoscope_data_path)
+    else:
+        data_missing = True
+
+    # Then, moves 'loose' mesoscope-related data files to the mesoscope_data folder.
+    if zstack_path.exists():
+        zstack_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("zstack.mat"))
+    else:
+        data_missing = True
+
+    if motion_estimator_path.exists():
+        motion_estimator_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("MotionEstimator.me"))
+    else:
+        data_missing = True
+
+    if ops_path.exists():
+        ops_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("ops.json"))
+    else:
+        data_missing = True
+
+    # If variant and invariant metadata files exist, also moves them to the mesoscope data folder and renames the
+    # files to use the latest naming convention. Missing any of these files is not considered a user-intervention-worthy
+    # situation.
+    if frame_metadata_path.exists():
+        frame_metadata_path.rename(
+            Path(session_data.raw_data.mesoscope_data_path).joinpath("frame_variant_metadata.npz")
+        )
+    if metadata_path.exists():
+        metadata_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("frame_invariant_metadata.json"))
+
+    # Loops over all camera video files (using the .avi extension) and moves them to the camera_data folder.
+    videos_found = 0
+    for video in source_root.glob("*.avi"):
+        videos_found += 1
+        video.rename(Path(session_data.raw_data.camera_data_path).joinpath(video.name))
+    if videos_found == 0:
+        data_missing = True
+
+    # Loops over all behavior log files (old GIMBL format) and moves them to the behavior_data folder.
+    logs_found = 0
+    for log in source_root.glob("Log Tyche-* ????-??-?? session *.json"):
+        logs_found += 1
+        log.rename(Path(session_data.raw_data.behavior_data_path).joinpath(log.name))
+    if logs_found == 0:
+        data_missing = True
+
+    # Removes the checksum file if it exists. Due to file name and location changes, the session data folder has to
+    # be re-checksummed after the reorganization anyway, so there is no need to keep the original file.
+    ax_checksum_path.unlink(missing_ok=True)
+
+    # Loops over all remaining contents of the directory.
+    for path in source_root.glob("*"):
+        # At this point, there should be no more subfolders left inside the root directory. If there are more
+        # subfolders, this case requires user intervention
+        if path.is_dir():
+            data_missing = True
+
+        # All non-subfolder files are moved to the root raw_data directory of the newly created session.
+        else:
+            path.rename(Path(session_data.raw_data.raw_data_path).joinpath(path.name))
+
+    # Session data has been fully reorganized. Depending on whether there was any missing data during processing,
+    # returns the boolean flag for whether user intervention is required
+    if data_missing:
+        return False
+    else:
+        return True
+
+
+def ascend_tyche_data(root_directory: Path, output_root_directory: Path, server_root_directory: Path) -> None:
+    """Converts raw data from the Tyche project to use the modern Sun lab layout.
+
+    This function is used to convert old data to the modern data management standard. In turn, this allows using all
+    modern data processing pipelines on this data.
+
+    Notes:
+        This function is statically written to work with the raw Tyche dataset featured in the OSM manuscript:
+        https://www.nature.com/articles/s41586-024-08548-w. Additionally, it assumes that the dataset has been
+        preprocessed with the early Sun lab mesoscope compression pipeline. The function will not work for any other
+        project or data hierarchy.
+
+        This function does not automatically transfer the data to the Server. It only creates the necessary root
+        hierarchy on the server and writes the necessary configuration to process the data on the Server, once it is
+        manually transferred.
+
+    Args:
+        root_directory: The root 'project' directory that stores individual Tyche animal folders to process.
+        output_root_directory: The path to the root directory where to generate the converted Tyche project hierarchy.
+        server_root_directory: The path to the SMB-mounted BioHPC server storage root directory. The Tyche project
+            hierarchy is generated both locally and on the Server.
+    """
+    # Generates a (shared) project configuration file.
+    project_configuration = ProjectConfiguration()
+
+    # Generates a temporary directory for NAS and Mesoscope paths. Since Tyche data is already backed up on the NAS and
+    # we are not generating new data, these root paths are not needed, but have to be created as part of the pipeline.
+    # Redirecting them to local temporary directories allows avoiding extra steps to manually remove these redundant
+    # directories after runtime.
+    temp_nas_dir = Path(tempfile.mkdtemp(prefix="nas_temp_"))
+    temp_mesoscope_dir = Path(tempfile.mkdtemp(prefix="mesoscope_temp_"))
+
+    # Statically defines project name and local root paths
+    project_configuration.project_name = "Tyche"
+    project_configuration.local_root_directory = output_root_directory
+    project_configuration.local_server_directory = server_root_directory
+    project_configuration.local_nas_directory = temp_nas_dir
+    project_configuration.local_mesoscope_directory = temp_mesoscope_dir
+
+    # Uses nonsensical google sheet IDs. Tyche project did not use Google Sheet processing like our modern projects do.
+    project_configuration.water_log_sheet_id = "1xFh9Q2zT7pL3mVkJdR8bN6yXoE4wS5aG0cHu2Kf7D3v"
+    project_configuration.surgery_sheet_id = "1xFh9Q2zT7pL3mVkJdR8bN6yXoE4wS5aG0cHu2Kf7D3v"
+
+    # Dumps project configuration into the 'configuration' subfolder of the Tyche project.
+    configuration_path = output_root_directory.joinpath("Tyche", "configuration", "project_configuration.yaml")
+    project_configuration.to_path(path=configuration_path)
+
+    # Assumes that root directory stores all animal folders to be processed
+    for animal_folder in root_directory.iterdir():
+        # Each animal folder is named to include project name and a static animal ID, e.g.: Tyche-A7. This extracts each
+        # animal ID.
+        animal_name = animal_folder.name.split(sep="-")[1]
+
+        # Under each animal root folder, there are day folders that use YYYY-MM-DD timestamps
+        for session_folder in animal_folder.iterdir():
+            # Inside each day folder, there are one or more acquisitions (sessions)
+            for acquisition_folder in session_folder.iterdir():
+                # For each session, we extract the modification time from either (preferentially) zstack.mat or
+                # MotionEstimator.me file. Any session without these files is flagged for additional user intervention.
+                # This procedure generates timestamp-based session names, analogous to how our modern pipeline does it.
+                session_name = _generate_session_name(acquisition_path=acquisition_folder)
+
+                # Uses derived session name and the statically created project configuration file to create the
+                # session data hierarchy using the output root. This generates a 'standard' Sun lab directory structure
+                # for the Tyche data.
+                session_data = SessionData.create_session(
+                    session_name=session_name,
+                    animal_id=animal_name,
+                    project_configuration=project_configuration,
+                    session_type="Experiment",
+                    experiment_name=None,  # Has to be none, otherwise the system tries to copy a configuration file.
+                )
+
+                # Moves the data from the old hierarchy to the new hierarchy. If the process runs as expected, and
+                # fully empties the source acquisition folder, destroys the folder. Otherwise, notifies the user that
+                # the runtime did not fully process the session data and requests intervention.
+                success = _reorganize_data(session_data, acquisition_folder)
+                if not success:
+                    message = (
+                        f"Encountered issues when reorganizing {animal_name} session {session_name}. "
+                        f"User intervention is required to finish data reorganization process for this session."
+                    )
+                    console.echo(message=message, level=LogLevel.WARNING)
+                else:
+                    # If the transfer process was successful, generates a new checksum for the moved data and removes
+                    # the now-empty acquisition folder.
+                    calculate_directory_checksum(directory=Path(session_data.raw_data.raw_data_path))
+                    acquisition_folder.rmdir()
+
+            # If the loop above removed all acquisition folders, all data for that day has been successfully converted
+            # to use the new session format. Removes the now-empty 'day' folder from the target animal
+            if len([folder for folder in session_folder.iterdir()]) == 0:
+                session_folder.rmdir()

{sl_shared_assets-1.0.0rc8.dist-info → sl_shared_assets-1.0.0rc9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc8
+Version: 1.0.0rc9
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

sl_shared_assets-1.0.0rc9.dist-info/RECORD

@@ -0,0 +1,14 @@
+sl_shared_assets/__init__.py,sha256=V7EvTTSB_GhetCbyYPg2RoiG1etDVeML5EBWgGvUo7E,2227
+sl_shared_assets/cli.py,sha256=1OzQUYZS741ca2x8LUCTOOTsC1leZGuwqEt0Q2qUNUQ,4676
+sl_shared_assets/data_classes.py,sha256=wRh493DHBfd2GuuWtHT0vKC7AVWhsE-wyQosZEpHvg0,87529
+sl_shared_assets/legacy_tools.py,sha256=x9Vl_BHUQf_-E6t2xx1r2autzqnmyWY3CNuKzMH97TE,14763
+sl_shared_assets/packaging_tools.py,sha256=3kAXFK37Lv4JA1YhjcoBz1x2Ell8ObCqe9pwxAts4m4,6709
+sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sl_shared_assets/server.py,sha256=VtFwS4PEy24n_pGz9W56zufkZEf_PKxIllP2ZnF5Zgc,13269
+sl_shared_assets/suite2p.py,sha256=sQ5Zj0TJFD-gUHqtWnRvapBpr8QgmaiVil123cWxGxc,20511
+sl_shared_assets/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
+sl_shared_assets-1.0.0rc9.dist-info/METADATA,sha256=TVcnp1yuSKs3by_NONUTGY9XncBQRtLVleYviPtrjcg,47806
+sl_shared_assets-1.0.0rc9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-1.0.0rc9.dist-info/entry_points.txt,sha256=3VPr5RkWBkusNN9OhWXtC-DN0utu7uMrUulazIK2VNA,166
+sl_shared_assets-1.0.0rc9.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-1.0.0rc9.dist-info/RECORD,,

sl_shared_assets/__init__.pyi

@@ -1,87 +0,0 @@
-from .server import (
-    Server as Server,
-    ServerCredentials as ServerCredentials,
-)
-from .suite2p import (
-    Main as Main,
-    FileIO as FileIO,
-    Output as Output,
-    Channel2 as Channel2,
-    NonRigid as NonRigid,
-    ROIDetection as ROIDetection,
-    Registration as Registration,
-    Classification as Classification,
-    OnePRegistration as OnePRegistration,
-    SignalExtraction as SignalExtraction,
-    CellposeDetection as CellposeDetection,
-    SpikeDeconvolution as SpikeDeconvolution,
-    Suite2PConfiguration as Suite2PConfiguration,
-)
-from .data_classes import (
-    RawData as RawData,
-    DrugData as DrugData,
-    ImplantData as ImplantData,
-    SessionData as SessionData,
-    SubjectData as SubjectData,
-    SurgeryData as SurgeryData,
-    Destinations as Destinations,
-    InjectionData as InjectionData,
-    MesoscopeData as MesoscopeData,
-    ProcedureData as ProcedureData,
-    ProcessedData as ProcessedData,
-    PersistentData as PersistentData,
-    ZaberPositions as ZaberPositions,
-    ExperimentState as ExperimentState,
-    MesoscopePositions as MesoscopePositions,
-    ProjectConfiguration as ProjectConfiguration,
-    HardwareConfiguration as HardwareConfiguration,
-    RunTrainingDescriptor as RunTrainingDescriptor,
-    LickTrainingDescriptor as LickTrainingDescriptor,
-    ExperimentConfiguration as ExperimentConfiguration,
-    MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
-    replace_root_path as replace_root_path,
-)
-from .transfer_tools import transfer_directory as transfer_directory
-from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
-
-__all__ = [
-    "Server",
-    "ServerCredentials",
-    "Main",
-    "FileIO",
-    "Output",
-    "Channel2",
-    "NonRigid",
-    "ROIDetection",
-    "Registration",
-    "Classification",
-    "OnePRegistration",
-    "SignalExtraction",
-    "CellposeDetection",
-    "SpikeDeconvolution",
-    "Suite2PConfiguration",
-    "RawData",
-    "DrugData",
-    "ImplantData",
-    "SessionData",
-    "SubjectData",
-    "SurgeryData",
-    "Destinations",
-    "InjectionData",
-    "MesoscopeData",
-    "ProcedureData",
-    "ProcessedData",
-    "PersistentData",
-    "ZaberPositions",
-    "ExperimentState",
-    "MesoscopePositions",
-    "ProjectConfiguration",
-    "HardwareConfiguration",
-    "RunTrainingDescriptor",
-    "LickTrainingDescriptor",
-    "ExperimentConfiguration",
-    "MesoscopeExperimentDescriptor",
-    "replace_root_path",
-    "transfer_directory",
-    "calculate_directory_checksum",
-]

sl_shared_assets/cli.pyi

@@ -1,17 +0,0 @@
-from .server import generate_server_credentials as generate_server_credentials
-from .data_classes import replace_root_path as replace_root_path
-
-def replace_local_root_directory(path: str) -> None:
-    """Replaces the root directory used to store all lab projects on the local PC with the specified directory.
-
-    To ensure all projects are saved in the same location, this library resolves and saves the absolute path to the
-    project directory when it is used for the first time. All future projects reuse the same 'root' path. Since this
-    information is stored in a typically hidden user directory, this CLI can be used to replace the local directory
-    path, if necessary.
-    """
-
-def generate_server_credentials_file(output_directory: str, host: str, username: str, password: str) -> None:
-    """Generates a new server_credentials.yaml file under the specified directory, using input information.
-
-    This CLI is used during the initial PC setup (typically, VRPC) to allow it to access the lab BioHPC server.
-    """