np_codeocean 0.2.1__tar.gz → 0.3.1__tar.gz

{np_codeocean-0.2.1 → np_codeocean-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: np_codeocean
-Version: 0.2.1
+Version: 0.3.1
 Summary: Tools for uploading and interacting with Mindscope Neuropixels experiments on Code Ocean
 Author-Email: Ben Hardcastle <ben.hardcastle@alleninstitute.org>, Chris Mochizuki <chrism@alleninstitute.org>, Arjun Sridhar <arjun.sridhar@alleninstitute.org>
 License: MIT
@@ -20,15 +20,18 @@ Requires-Dist: np-config>=0.4.33
 Requires-Dist: requests>=2.31.0
 Requires-Dist: npc-session>=0.1.34
 Requires-Dist: polars>=0.20.16
-Requires-Dist: aind-data-transfer-models>=0.5.1
 Requires-Dist: npc-lims>=0.1.168
+Requires-Dist: npc-ephys>=0.1.28
+Provides-Extra: dev
 Requires-Dist: bump>=1.3.2; extra == "dev"
 Requires-Dist: pdm>=2.4.9; extra == "dev"
-Requires-Dist: np-aind-metadata==0.1.16; extra == "dynamic-routing-metadata"
-Requires-Dist: npc-lims>=0.1.154; extra == "dynamic-routing-metadata"
-Requires-Dist: npc-sessions>=0.0.226; extra == "dynamic-routing-metadata"
-Provides-Extra: dev
 Provides-Extra: dynamic-routing-metadata
+Requires-Dist: np-aind-metadata>=0.1.17; extra == "dynamic-routing-metadata"
+Requires-Dist: npc-lims>=0.1.154; extra == "dynamic-routing-metadata"
+Requires-Dist: npc-sessions>=0.0.253; extra == "dynamic-routing-metadata"
+Requires-Dist: aind-data-transfer-models>=0.13.1; extra == "dynamic-routing-metadata"
+Requires-Dist: aind-codeocean-pipeline-monitor[full]>=0.5.0; extra == "dynamic-routing-metadata"
+Requires-Dist: aind-metadata-mapper==0.18.2; extra == "dynamic-routing-metadata"
 Description-Content-Type: text/markdown
 
 # np_codeocean

{np_codeocean-0.2.1 → np_codeocean-0.3.1}/pyproject.toml

@@ -40,7 +40,7 @@ composite = [
 
 [project]
 name = "np_codeocean"
-version = "0.2.1"
+version = "0.3.1"
 description = "Tools for uploading and interacting with Mindscope Neuropixels experiments on Code Ocean"
 authors = [
     { name = "Ben Hardcastle", email = "ben.hardcastle@alleninstitute.org" },
@@ -54,8 +54,8 @@ dependencies = [
     "requests>=2.31.0",
     "npc-session>=0.1.34",
     "polars>=0.20.16",
-    "aind-data-transfer-models>=0.5.1",
     "npc-lims>=0.1.168",
+    "npc-ephys>=0.1.28",
 ]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -87,9 +87,12 @@ dev = [
     "pdm>=2.4.9",
 ]
 dynamic-routing-metadata = [
-    "np-aind-metadata==0.1.16",
+    "np-aind-metadata>=0.1.17",
     "npc-lims>=0.1.154",
-    "npc-sessions>=0.0.226",
+    "npc-sessions>=0.0.253",
+    "aind-data-transfer-models>=0.13.1",
+    "aind-codeocean-pipeline-monitor[full]>=0.5.0",
+    "aind-metadata-mapper==0.18.2",
 ]
 
 [build-system]

{np_codeocean-0.2.1 → np_codeocean-0.3.1}/src/np_codeocean/np_session_utils.py

@@ -7,6 +7,7 @@ import pathlib
 from collections.abc import Iterable
 import shutil
 
+import aind_data_transfer_models.core
 import np_config
 import np_logging
 import np_session
@@ -82,7 +83,7 @@ def create_aind_metadata_symlinks(upload: CodeOceanUpload) -> bool:
     has_metadata_files = False
     for src in upload.session.npexp_path.glob('*'):
         if src.stem in utils.AIND_METADATA_NAMES:
-            np_tools.symlink(utils.ensure_posix(src), upload.aind_metadata / src.name)
+            np_tools.symlink(src, upload.aind_metadata / src.name)
             has_metadata_files = True
     if has_metadata_files:
         logger.debug(f'Finished creating symlinks to aind metadata files in {upload.session.npexp_path}')
@@ -116,7 +117,7 @@ def create_ephys_symlinks(session: np_session.Session, dest: pathlib.Path,
         root_path, specific_recording_dir_names=recording_dirs
         ):
         if not abs_path.is_dir():
-            np_tools.symlink(utils.ensure_posix(abs_path), dest / rel_path)
+            np_tools.symlink(abs_path, dest / rel_path)
     logger.debug(f'Finished creating symlinks to raw ephys data files in {root_path}')
     utils.cleanup_ephys_symlinks(dest)
 
@@ -132,7 +133,7 @@ def create_behavior_symlinks(session: np_session.Session, dest: pathlib.Path | N
     logger.info(f'Creating symlinks in {dest} to files in {session.npexp_path}...')
     for src in session.npexp_path.glob('*'):
         if not src.is_dir() and not utils.is_behavior_video_file(src):
-            np_tools.symlink(utils.ensure_posix(src), dest / src.relative_to(session.npexp_path))
+            np_tools.symlink(src, dest / src.relative_to(session.npexp_path))
     logger.debug(f'Finished creating symlinks to top-level files in {session.npexp_path}')
 
     for name in subfolder_names:
@@ -141,7 +142,7 @@ def create_behavior_symlinks(session: np_session.Session, dest: pathlib.Path | N
             continue
         for src in subfolder.rglob('*'):
             if not src.is_dir():
-                np_tools.symlink(utils.ensure_posix(src), dest / src.relative_to(session.npexp_path))
+                np_tools.symlink(src, dest / src.relative_to(session.npexp_path))
         logger.debug(f'Finished creating symlinks to {name!r} files')
 
 
@@ -155,7 +156,7 @@ def create_behavior_videos_symlinks(session: np_session.Session, dest: pathlib.P
     logger.info(f'Creating symlinks in {dest} to files in {session.npexp_path}...')
     for src in session.npexp_path.glob('*'):
         if utils.is_behavior_video_file(src):
-            np_tools.symlink(utils.ensure_posix(src), dest / src.relative_to(session.npexp_path))
+            np_tools.symlink(src, dest / src.relative_to(session.npexp_path))
     logger.debug(f'Finished creating symlinks to behavior video files in {session.npexp_path}')
 
 
@@ -299,6 +300,8 @@ def upload_session(
     test: bool = False,
     hpc_upload_job_email: str = utils.HPC_UPLOAD_JOB_EMAIL,
     regenerate_symlinks: bool = True,
+    adjust_ephys_timestamps: bool = True,
+    codeocean_configs: aind_data_transfer_models.core.CodeOceanPipelineMonitorConfigs | None = None,
 ) -> None:
     codeocean_root = np_session.NPEXP_PATH / ('codeocean-dev' if test else 'codeocean')
     logger.debug(f'{codeocean_root = }')
@@ -310,7 +313,7 @@ def upload_session(
     )
     if regenerate_symlinks and upload.root.exists():
         logger.debug(f'Removing existing {upload.root = }')
-        shutil.rmtree(upload.root.as_posix())
+        shutil.rmtree(upload.root.as_posix(), ignore_errors=True)
     if upload.aind_metadata:
         create_aind_metadata_symlinks(upload)
     if upload.ephys:
@@ -319,11 +322,28 @@ def upload_session(
         create_behavior_symlinks(upload.session, upload.behavior)
     if upload.behavior_videos:
         create_behavior_videos_symlinks(upload.session, upload.behavior_videos)
+    if adjust_ephys_timestamps and upload.ephys:
+        if not upload.behavior: # includes surface channel recordings
+            logger.warning(f"Cannot adjust ephys timestamps for {upload.session} - no behavior folder supplied for upload")
+        else:
+            try:
+                utils.write_corrected_ephys_timestamps(ephys_dir=upload.ephys, behavior_dir=upload.behavior)
+            except utils.SyncFileNotFoundError:
+                raise FileNotFoundError(
+                    (
+                        f"Cannot adjust timestamps - no sync file found in {upload.behavior}. "
+                        "If the session doesn't have one, run with "
+                        "`adjust_ephys_timestamps=False` or `--no-sync` flag in CLI"
+                    )
+                ) from None
+    for path in (upload.ephys, upload.behavior, upload.behavior_videos, upload.aind_metadata):
+        if path is not None and path.exists():
+            utils.convert_symlinks_to_posix(path)
     csv_content: dict = get_upload_csv_for_session(upload)
     utils.write_upload_csv(csv_content, upload.job)
     np_logging.web('np_codeocean').info(f'Submitting {upload.session} to hpc upload queue')
     utils.put_jobs_for_hpc_upload(
-        utils.get_job_models_from_csv(upload.job),
+        utils.get_job_models_from_csv(upload.job, codeocean_configs=codeocean_configs),
         upload_service_url=utils.DEV_SERVICE if test else utils.AIND_DATA_TRANSFER_SERVICE,
         user_email=hpc_upload_job_email,
         dry_run=dry_run,

{np_codeocean-0.2.1 → np_codeocean-0.3.1}/src/np_codeocean/scripts/upload_dynamic_routing_behavior.py

@@ -43,7 +43,8 @@ def reset_log_file() -> None:
         log.unlink(missing_ok=True)
     
 def get_log_file() -> pathlib.Path:
-    folder = pathlib.Path(".").resolve() / "logs"
+    folder = pathlib.Path("//allen/programs/mindscope/workgroups/np-exp") / "codeocean-logs"
+    folder.mkdir(exist_ok=True)
     return folder / f"{pathlib.Path(__file__).stem}_{datetime.datetime.now().strftime('%Y-%m-%d')}.log"
 
 logging.basicConfig(

{np_codeocean-0.2.1 → np_codeocean-0.3.1}/src/np_codeocean/scripts/upload_dynamic_routing_ecephys.py

@@ -1,4 +1,5 @@
 import argparse
+import contextlib
 import datetime
 import logging
 import pathlib
@@ -10,14 +11,18 @@ import npc_session
 import npc_sessions
 from aind_data_schema.core.rig import Rig
 from np_aind_metadata.integrations import dynamic_routing_task
-
+import aind_codeocean_pipeline_monitor.models 
+import aind_data_transfer_models.core
+import codeocean.capsule
+import codeocean.data_asset
+import codeocean.computation 
 import np_codeocean
 
 # Disable divide by zero or NaN warnings
 warnings.filterwarnings("ignore", category=RuntimeWarning)
 
 logging.basicConfig(
-    filename=f"logs/{pathlib.Path(__file__).stem}_{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log",
+    filename=f"//allen/programs/mindscope/workgroups/np-exp/codeocean-logs/{pathlib.Path(__file__).stem}_{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log",
     level=logging.DEBUG, 
     format="%(asctime)s | %(name)s | %(levelname)s | %(message)s", 
     datefmt="%Y-%d-%m %H:%M:%S",
@@ -123,6 +128,7 @@ def write_metadata_and_upload(
     hpc_upload_job_email: str = np_codeocean.HPC_UPLOAD_JOB_EMAIL,
     regenerate_metadata: bool = False,
     regenerate_symlinks: bool = True,
+    adjust_ephys_timestamps: bool = False,
 ) -> None:
     """Writes and updates aind-data-schema to the session directory
      associated with the `session`. The aind-data-schema session model is
@@ -134,16 +140,35 @@ def write_metadata_and_upload(
     """
     # session = np_session.Session(session) #! this doesn't work for surface_channels
     session = np_codeocean.get_np_session(session_path_or_folder_name)
-    add_metadata(
-        session_directory=session.npexp_path,
-        session_datetime=(
-            session.start 
-            if not np_codeocean.is_surface_channel_recording(session.npexp_path.name)
-            else np_codeocean.get_surface_channel_start_time(session)
+    with contextlib.suppress(Exception):
+        add_metadata(
+            session_directory=session.npexp_path,
+            session_datetime=(
+                session.start 
+                if not np_codeocean.is_surface_channel_recording(session.npexp_path.name)
+                else np_codeocean.get_surface_channel_start_time(session)
+            ),
+            rig_storage_directory=pathlib.Path(np_codeocean.get_project_config()["rig_metadata_dir"]),
+            ignore_errors=True,
+            skip_existing=not regenerate_metadata,
+        )
+    pipelines = [
+        aind_codeocean_pipeline_monitor.models.PipelineMonitorSettings(
+            run_params=codeocean.computation.RunParams(
+                capsule_id="287db808-74ce-4e44-b14b-fde1471eba45",
+            ),
+            data_assets=[
+                codeocean.data_asset.DataAsset(
+                    mount="ecephys",
+                    id="", # ID of new raw data asset will be inserted here by airflow
+                ),
+            ],
+            computation_polling_interval=15 * 60,
+            computation_timeout=48 * 3600,
         ),
-        rig_storage_directory=pathlib.Path(np_codeocean.get_project_config()["rig_metadata_dir"]),
-        ignore_errors=True,
-        skip_existing=not regenerate_metadata,
+    ]
+    codeocean_configs = aind_data_transfer_models.core.CodeOceanPipelineMonitorConfigs(
+        pipeline_monitor_capsule_settings=pipelines,
     )
     return np_codeocean.upload_session(
         session_path_or_folder_name,
@@ -153,6 +178,8 @@ def write_metadata_and_upload(
         test=test,
         hpc_upload_job_email=hpc_upload_job_email,
         regenerate_symlinks=regenerate_symlinks,
+        adjust_ephys_timestamps=adjust_ephys_timestamps,
+        codeocean_configs=codeocean_configs,
     )
 
 def parse_args() -> argparse.Namespace:
@@ -165,11 +192,15 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument('--dry-run', action='store_true', help="Create upload job but do not submit to hpc upload queue.")
     parser.add_argument('--preserve-symlinks', dest='regenerate_symlinks', action='store_false', help="Existing symlink folders will not be deleted and regenerated - may result in additional data being uploaded")
     parser.add_argument('--regenerate-metadata', action='store_true', help="Regenerate metadata files (session.json and rig.json) even if they already exist")
+    parser.add_argument('--no-sync', dest="adjust_ephys_timestamps", action='store_false', help="(disabled) Skip adjustment of ephys timestamps.npy using sync data for sessions where no sync data is available")
     return parser.parse_args()
 
 def main() -> None:
     args = parse_args()
-    write_metadata_and_upload(**vars(args))
+    kwargs = vars(args)
+    kwargs |= {'adjust_ephys_timestamps':  False} # unnecessary while we have machinery in place for adjusting in npc_sessions (adds 5 GB of timestamps files for each upload)
+    np_codeocean.utils.set_npc_lims_credentials()
+    write_metadata_and_upload(**kwargs)
 
 
 if __name__ == '__main__':

{np_codeocean-0.2.1 → np_codeocean-0.3.1}/src/np_codeocean/utils.py

@@ -4,6 +4,7 @@ import contextlib
 import csv
 import datetime
 import functools
+import itertools
 import json
 import logging
 import os
@@ -15,6 +16,8 @@ import aind_data_transfer_models.core
 import aind_slurm_rest.models
 import np_config
 import np_tools
+import npc_ephys
+import npc_sync
 import npc_session
 import numpy as np
 import polars as pl
@@ -38,6 +41,9 @@ DEFAULT_EPHYS_SLURM_SETTINGS = aind_slurm_rest.models.V0036JobProperties(
 )
 """Increased timelimit and cpus for running ephys compression on the hpc"""
 
+class SyncFileNotFoundError(FileNotFoundError):
+    pass
+
 @functools.cache
 def get_project_config() -> dict[str, Any]:
     """Config for this project"""
@@ -202,6 +208,34 @@ def cleanup_ephys_metadata(toplevel_dir: pathlib.Path) -> None:
             oebin_path.write_text(json.dumps(oebin_obj, indent=4))
             logger.debug('Overwrote symlink to structure.oebin with corrected structure.oebin')
 
+def write_corrected_ephys_timestamps(
+    ephys_dir: pathlib.Path,
+    behavior_dir: pathlib.Path,
+) -> None:
+    for path in itertools.chain(behavior_dir.glob('*.h5'), behavior_dir.glob('*.sync')):
+        with contextlib.suppress(Exception):
+            sync_dataset = npc_sync.SyncDataset(path)
+            _ = sync_dataset.line_labels
+            logger.info(f'Found valid sync file {path.as_posix()}')
+            break
+    else:
+        raise SyncFileNotFoundError(f'No valid sync file found in {behavior_dir.as_posix()}')
+    
+    timing_on_pxi = (
+        timing
+        for timing in npc_ephys.get_ephys_timing_on_pxi(
+            ephys_dir.glob("**/experiment*/recording*"),
+        )
+    )
+    timing_on_sync = (
+        npc_ephys.get_ephys_timing_on_sync(
+            sync=sync_dataset,
+            devices=timing_on_pxi,
+        )
+    )
+    npc_ephys.overwrite_timestamps(timing_on_sync)
+    logger.info(f'Corrected timestamps in {ephys_dir}')
+    
 def decode_symlink_path(oebin_path: pathlib.Path) -> pathlib.Path:
     if not oebin_path.is_symlink():
         return oebin_path
@@ -266,6 +300,7 @@ def get_job_models_from_csv(
     path: pathlib.Path,
     ephys_slurm_settings: aind_slurm_rest.models.V0036JobProperties = DEFAULT_EPHYS_SLURM_SETTINGS,
     user_email: str = HPC_UPLOAD_JOB_EMAIL,
+    **extra_BasicUploadJobConfigs_params: Any,
 ) -> tuple[aind_data_transfer_models.core.BasicUploadJobConfigs, ...]:
     jobs = pl.read_csv(path, eol_char='\r').with_columns(
         pl.col('subject-id').cast(str),
@@ -293,6 +328,7 @@ def get_job_models_from_csv(
                 **{k.replace('-', '_'): v for k,v in job.items()},
                 modalities=modalities,
                 user_email=user_email,
+                **extra_BasicUploadJobConfigs_params,
             )
         )
     return tuple(models)
@@ -401,6 +437,16 @@ def ensure_posix(path: str | pathlib.Path) -> str:
     return posix
 
 
+def convert_symlinks_to_posix(toplevel_dir: str | pathlib.Path) -> None:
+    """Convert all symlinks in `root_dir` (recursively) to POSIX paths. This is a
+    necessary last step before submitting uploads to run on the HPC.
+    """
+    for path in pathlib.Path(toplevel_dir).rglob('*'):
+        if path.is_symlink():
+            posix_target = path.readlink().as_posix().removeprefix('//?/UNC')
+            path.unlink()
+            np_tools.symlink(src=ensure_posix(posix_target), dest=path)
+            
 if __name__ == '__main__':
     import doctest
     doctest.testmod(optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.IGNORE_EXCEPTION_DETAIL)