np_codeocean 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl

np_codeocean/upload.py

@@ -4,9 +4,7 @@ import argparse
 import contextlib
 import csv
 import json
-from multiprocessing import context
 import pathlib
-import sys
 import datetime
 from pathlib import Path
 from typing import NamedTuple
@@ -20,9 +18,8 @@ import np_tools
 import doctest
 import numpy as np
 import polars as pl
-
 import requests
-
+from pydantic import ValidationError # may be returned from aind-data-transfer-service
 
 logger = np_logging.get_logger(__name__)
 
@@ -46,13 +43,16 @@ class CodeOceanUpload(NamedTuple):
     """Directory of symlinks to behavior video files and video info jsons in
     top-level of session folder on np-exp."""
     
-    ephys: Path
+    ephys: Path | None
     """Directory of symlinks to raw ephys data files on np-exp, with only one
     `recording` per `Record Node` folder."""
 
     job: Path
     """File containing job parameters for `aind-data-transfer`"""
 
+    force_cloud_sync: bool = False
+    """If True, re-upload and re-make raw asset even if data exists on S3."""
+    
 def as_posix(path: pathlib.Path) -> str:
     return path.as_posix()[1:]
 
@@ -83,6 +83,37 @@ def create_ephys_symlinks(session: np_session.Session, dest: Path,
         if not abs_path.is_dir():
             np_tools.symlink(as_posix(abs_path), dest / rel_path)
     logger.debug(f'Finished creating symlinks to raw ephys data files in {root_path}')
+    correct_structure(dest)
+
+def correct_structure(dest: Path) -> None:
+    """
+    In case some probes are missing, remove device entries from structure.oebin
+    files for devices with folders that have not been preserved.
+    """
+    logger.debug('Checking structure.oebin for missing folders...')
+    recording_dirs = dest.rglob('recording[0-9]')
+    for recording_dir in recording_dirs:
+        if not recording_dir.is_dir():
+            continue
+        oebin_path = recording_dir / 'structure.oebin'
+        if not (oebin_path.is_symlink() or oebin_path.exists()):
+            logger.warning(f'No structure.oebin found in {recording_dir}')
+            continue
+        logger.debug(f'Examining oebin: {oebin_path} for correction')
+        oebin_obj = np_tools.read_oebin(np_config.normalize_path(oebin_path.readlink()))
+        any_removed = False
+        for subdir_name in ('events', 'continuous'):    
+            subdir = oebin_path.parent / subdir_name
+            # iterate over copy of list so as to not disrupt iteration when elements are removed
+            for device in [device for device in oebin_obj[subdir_name]]:
+                if not (subdir / device['folder_name']).exists():
+                    logger.info(f'{device["folder_name"]} not found in {subdir}, removing from structure.oebin')
+                    oebin_obj[subdir_name].remove(device)
+                    any_removed = True
+        if any_removed:
+            oebin_path.unlink()
+            oebin_path.write_text(json.dumps(oebin_obj, indent=4))
+            logger.debug('Overwrote symlink to structure.oebin with corrected strcuture.oebin')
 
 def is_behavior_video_file(path: Path) -> bool:
     if path.is_dir() or path.suffix not in ('.mp4', '.avi', '.json'):
@@ -92,13 +123,10 @@ def is_behavior_video_file(path: Path) -> bool:
         return True
     return False
 
-def create_behavior_symlinks(session: np_session.Session, dest: Path | None) -> None:
+def create_behavior_symlinks(session: np_session.Session, dest: Path) -> None:
     """Create symlinks in `dest` pointing to files in top-level of session
     folder on np-exp, plus all files in `exp` subfolder, if present.
     """
-    if dest is None: 
-        logger.debug(f"No behavior folder supplied for {session}")
-        return
     subfolder_names = ('exp', 'qc')
     logger.info(f'Creating symlinks in {dest} to files in {session.npexp_path}...')
     for src in session.npexp_path.glob('*'):
@@ -115,13 +143,10 @@ def create_behavior_symlinks(session: np_session.Session, dest: Path | None) ->
                 np_tools.symlink(as_posix(src), dest / src.relative_to(session.npexp_path))
         logger.debug(f'Finished creating symlinks to {name!r} files')
 
-def create_behavior_videos_symlinks(session: np_session.Session, dest: Path | None) -> None:
+def create_behavior_videos_symlinks(session: np_session.Session, dest: Path) -> None:
     """Create symlinks in `dest` pointing to MVR video files and info jsons in top-level of session
     folder on np-exp.
     """
-    if dest is None: 
-        logger.debug(f"No behavior_videos folder supplied for {session}")
-        return
     logger.info(f'Creating symlinks in {dest} to files in {session.npexp_path}...')
     for src in session.npexp_path.glob('*'):
         if is_behavior_video_file(src):
@@ -130,7 +155,7 @@ def create_behavior_videos_symlinks(session: np_session.Session, dest: Path | No
     
 def is_surface_channel_recording(path_name: str) -> bool:
     """
-    >>> session = np_session.Session("//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_660023_20230808_surface_channels")
+    >>> session = np_session.Session("//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_690706_20231129_surface_channels")
     >>> is_surface_channel_recording(session.npexp_path.as_posix())
     True
     """
@@ -138,9 +163,9 @@ def is_surface_channel_recording(path_name: str) -> bool:
 
 def get_surface_channel_start_time(session: np_session.Session) -> datetime.datetime:
     """
-    >>> session = np_session.Session("//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_660023_20230808_surface_channels")
+    >>> session = np_session.Session("//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_690706_20231129_surface_channels")
     >>> get_surface_channel_start_time(session)
-    datetime.datetime(2023, 8, 8, 15, 11, 14, 240000)
+    datetime.datetime(2023, 11, 29, 14, 56, 25, 219000)
     """
     sync_messages_paths = tuple(session.npexp_path.glob('*/*/*/sync_messages.txt'))
     if not sync_messages_paths:
@@ -154,43 +179,42 @@ def get_surface_channel_start_time(session: np_session.Session) -> datetime.date
     timestamp = datetime.datetime.fromtimestamp(timestamp_value / 1e3)
     return timestamp
 
-def get_ephys_upload_csv_for_session(upload: CodeOceanUpload) -> dict[str, str | int]:
+def get_upload_csv_for_session(upload: CodeOceanUpload) -> dict[str, str | int | bool]:
     """
-    >>> path = "//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_660023_20230808_surface_channels"
+    >>> path = "//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_690706_20231129_surface_channels"
     >>> is_surface_channel_recording(path)
     True
     >>> upload = create_codeocean_upload(path)
-    >>> ephys_upload_csv = get_ephys_upload_csv_for_session(upload)
+    >>> ephys_upload_csv = get_upload_csv_for_session(upload)
     >>> ephys_upload_csv['modality0.source']
-    '//allen/programs/mindscope/workgroups/np-exp/codeocean/DRpilot_660023_20230808_surface_channels/ephys'
+    '//allen/programs/mindscope/workgroups/np-exp/codeocean/DRpilot_690706_20231129_surface_channels/ephys'
     >>> ephys_upload_csv.keys()
-    dict_keys(['modality0.source', 'modality0', 's3-bucket', 'subject-id', 'platform', 'acq-datetime'])
+    dict_keys(['platform', 'subject-id', 'force_cloud_sync', 'modality0', 'modality0.source', 'acq-datetime'])
     """
-    
-    ephys_upload = {
-        'modality0.source': np_config.normalize_path(upload.ephys).as_posix(),
-        'modality0': 'ecephys',
-        's3-bucket': CONFIG['s3-bucket'],
-        'subject-id': str(upload.session.mouse),
+    params = {
         'platform': 'ecephys',
+        'subject-id': str(upload.session.mouse),
+        'force_cloud_sync': upload.force_cloud_sync,
     }
-
-    if upload.behavior is not None:
-        ephys_upload['modality1.source'] = np_config.normalize_path(upload.behavior).as_posix()
-        ephys_upload['modality1'] = 'behavior'
-    
-    if upload.behavior_videos is not None:
-        ephys_upload['modality2.source'] = np_config.normalize_path(upload.behavior_videos).as_posix()
-        ephys_upload['modality2'] = 'behavior-videos'
-    
+    idx = 0
+    for modality_name, attr_name in {
+        'ecephys': 'ephys',
+        'behavior': 'behavior',
+        'behavior-videos': 'behavior_videos',
+    }.items():
+        if getattr(upload, attr_name) is not None:
+            params[f'modality{idx}'] = modality_name
+            params[f'modality{idx}.source'] = np_config.normalize_path(getattr(upload, attr_name)).as_posix()
+            idx += 1
+            
     if is_surface_channel_recording(upload.session.npexp_path.as_posix()):
         date = datetime.datetime(upload.session.date.year, upload.session.date.month, upload.session.date.day)
         session_date_time = date.combine(upload.session.date, get_surface_channel_start_time(upload.session).time())
-        ephys_upload['acq-datetime'] = f'{session_date_time.strftime("%Y-%m-%d %H:%M:%S")}'
+        params['acq-datetime'] = f'{session_date_time.strftime("%Y-%m-%d %H:%M:%S")}'
     else:
-        ephys_upload['acq-datetime'] = f'{upload.session.start.strftime("%Y-%m-%d %H:%M:%S")}'
+        params['acq-datetime'] = f'{upload.session.start.strftime("%Y-%m-%d %H:%M:%S")}'
     
-    return ephys_upload
+    return params
 
 
 def is_in_hpc_upload_queue(csv_path: pathlib.Path) -> bool:
@@ -230,7 +254,8 @@ def put_csv_for_hpc_upload(csv_path: pathlib.Path) -> None:
         to get the real error class + message."""
         if response.status_code != 200:
             try:
-                raise eval(response.json()['data']['errors'][0])
+                x = response.json()['data']['errors']
+                import pdb; pdb.set_trace()
             except (KeyError, IndexError, requests.exceptions.JSONDecodeError, SyntaxError) as exc1:
                 try:
                     response.raise_for_status()
@@ -261,10 +286,13 @@ def put_csv_for_hpc_upload(csv_path: pathlib.Path) -> None:
         ),
     )
     _raise_for_status(post_csv_response)
-    
+
+def is_ephys_session(session: np_session.Session) -> bool:
+    return bool(next(session.npexp_path.rglob('settings.xml'), None))
+
 def create_upload_job(upload: CodeOceanUpload) -> None:
     logger.info(f'Creating upload job file {upload.job} for session {upload.session}...')
-    job: dict = get_ephys_upload_csv_for_session(upload)
+    job: dict = get_upload_csv_for_session(upload)
     with open(upload.job, 'w') as f:
         w = csv.writer(f, lineterminator='')
         w.writerow(job.keys())
@@ -273,28 +301,33 @@ def create_upload_job(upload: CodeOceanUpload) -> None:
         w.writerow(job.values()) 
 
 def create_codeocean_upload(session: str | int | np_session.Session, 
-                            recording_dirs: Iterable[str] | None = None) -> CodeOceanUpload:
-    """
-    >>> upload = create_codeocean_upload("//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_660023_20230808_surface_channels")
-    >>> upload.behavior is None
-    True
-    >>> upload.ephys.exists()
-    True
-    """
+                            recording_dirs: Iterable[str] | None = None,
+                            force_cloud_sync: bool = False,
+                            ) -> CodeOceanUpload:
     """Create directories of symlinks to np-exp files with correct structure
     for upload to CodeOcean.
     
     - only one `recording` per `Record Node` folder (largest if multiple found)
     - job file for feeding into `aind-data-transfer`
-    """
     
-    session = np_session.Session(session)
+    >>> upload = create_codeocean_upload("//allen/programs/mindscope/workgroups/dynamicrouting/PilotEphys/Task 2 pilot/DRpilot_690706_20231129_surface_channels")
+    >>> upload.behavior is None
+    True
+    >>> upload.ephys.exists()
+    True
+    """
 
-    if is_surface_channel_recording(session.npexp_path.as_posix()):
+    if is_surface_channel_recording(str(session)):
+        session = np_session.Session(session)
+        if not is_surface_channel_recording(session.npexp_path.name):
+            # manually assign surface channel path 
+            session = np_session.Session(session.npexp_path.parent / f'{session.folder}_surface_channels')
+            assert session.npexp_path.exists(), f"Surface channel path {session.npexp_path} does not exist in same folder as main session recording"
         root = np_session.NPEXP_PATH / 'codeocean' / f'{session.folder}_surface_channels'
         behavior = None
         behavior_videos = None
     else:
+        session = np_session.Session(session)
         root = np_session.NPEXP_PATH / 'codeocean' / session.folder
         behavior = np_config.normalize_path(root / 'behavior')
         behavior_videos = behavior.with_name('behavior-videos')
@@ -305,19 +338,24 @@ def create_codeocean_upload(session: str | int | np_session.Session,
         session = session, 
         behavior = behavior,
         behavior_videos = behavior_videos,
-        ephys = np_config.normalize_path(root / 'ephys'),
+        ephys = np_config.normalize_path(root / 'ephys') if is_ephys_session(session) else None,
         job = np_config.normalize_path(root / 'upload.csv'),
+        force_cloud_sync=force_cloud_sync,
         )
-
-    create_ephys_symlinks(upload.session, upload.ephys, recording_dirs=recording_dirs)
-    create_behavior_symlinks(upload.session, upload.behavior)
-    create_behavior_videos_symlinks(upload.session, upload.behavior_videos)
+    if upload.ephys:
+        create_ephys_symlinks(upload.session, upload.ephys, recording_dirs=recording_dirs)
+    if upload.behavior:
+        create_behavior_symlinks(upload.session, upload.behavior)
+    if upload.behavior_videos:
+        create_behavior_videos_symlinks(upload.session, upload.behavior_videos)
     create_upload_job(upload)    
     return upload
 
 def upload_session(session: str | int | pathlib.Path | np_session.Session, 
-                   recording_dirs: Iterable[str] | None = None) -> None:
-    upload = create_codeocean_upload(str(session), recording_dirs=recording_dirs)
+                   recording_dirs: Iterable[str] | None = None,
+                   force: bool = False,
+                   ) -> None:
+    upload = create_codeocean_upload(str(session), recording_dirs=recording_dirs, force_cloud_sync=force)
     np_logging.web('np_codeocean').info(f'Submitting {upload.session} to hpc upload queue')
     put_csv_for_hpc_upload(upload.job)
     logger.debug(f'Submitted {upload.session} to hpc upload queue')
@@ -336,6 +374,7 @@ def main() -> None:
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Upload a session to CodeOcean")
     parser.add_argument('session', help="session ID (lims or np-exp foldername) or path to session folder")
+    parser.add_argument('--force', action='store_true', help="enable `force_cloud_sync` option, re-uploading and re-making raw asset even if data exists on S3")
     parser.add_argument('recording_dirs', nargs='*', type=list, help="[optional] specific recording directories to upload - for use with split recordings only.")
     return parser.parse_args()
 
@@ -343,6 +382,5 @@ if __name__ == '__main__':
     import doctest
 
     doctest.testmod(
-        optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE)
-    )
-    main()
+        optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE),
+    )

{np_codeocean-0.1.6.dist-info → np_codeocean-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: np_codeocean
-Version: 0.1.6
+Version: 0.1.8
 Summary: Tools for uploading and interacting with Mindscope Neuropixels experiments on Code Ocean
 Author-Email: Ben Hardcastle <ben.hardcastle@alleninstitute.org>
 License: MIT
@@ -19,7 +19,7 @@ Requires-Dist: np-tools>=0.1.21
 Requires-Dist: np-config>=0.4.24
 Requires-Dist: requests>=2.31.0
 Requires-Dist: npc-session>=0.1.34
-Requires-Dist: polars>=0.19.19
+Requires-Dist: polars>=0.20.16
 Requires-Dist: bump>=1.3.2; extra == "dev"
 Requires-Dist: pdm>=2.4.9; extra == "dev"
 Provides-Extra: dev

{np_codeocean-0.1.6.dist-info → np_codeocean-0.1.8.dist-info}/RECORD

@@ -1,9 +1,9 @@
-np_codeocean-0.1.6.dist-info/METADATA,sha256=sjvrCBR2236bHRQuxEHoI92Vq9eW0RrPaaadQwbur2o,2488
-np_codeocean-0.1.6.dist-info/WHEEL,sha256=N2J68yzZqJh3mI_Wg92rwhw0rtJDFpZj9bwQIMJgaVg,90
-np_codeocean-0.1.6.dist-info/entry_points.txt,sha256=T3Is83nShuWFYg7bTLxVhRWi15OVxO99WYcUg3-xURM,113
+np_codeocean-0.1.8.dist-info/METADATA,sha256=hvXhhe5Fv5FEzz17K1RK6wtpJcyKhJKL_qS9JT_OeZM,2488
+np_codeocean-0.1.8.dist-info/WHEEL,sha256=N2J68yzZqJh3mI_Wg92rwhw0rtJDFpZj9bwQIMJgaVg,90
+np_codeocean-0.1.8.dist-info/entry_points.txt,sha256=T3Is83nShuWFYg7bTLxVhRWi15OVxO99WYcUg3-xURM,113
 np_codeocean/__init__.py,sha256=BYXXoFDa1J_Lv-YG52Ch6k5L4DMCEPXtfHsrMmMeST4,66
 np_codeocean/scripts/upload_sessions.py,sha256=1_aqoBxAkB_VpRKYqyPsEQBDGvgyAHXAkIJA0ZT2Vb0,1490
-np_codeocean/upload.py,sha256=nFnPD-eRc6f3eWZMkr6Qe_D7-77nDZookTVzrjwZkfo,14940
+np_codeocean/upload.py,sha256=qSxujYOMjuaz_NMJa_v9ZhuuTj9IiSs2qIxy8zWXeB4,17487
 np_codeocean/upload_one.py,sha256=-egSjXvA0bBfshbY3D2TZ0M0GfLokFBZ3mSCm_gOGXE,7367
 np_codeocean/utils.py,sha256=p0pmljaH4j7RjRsc4TPYXPpLhq-2ScvnfyXOYFSFBTM,3375
-np_codeocean-0.1.6.dist-info/RECORD,,
+np_codeocean-0.1.8.dist-info/RECORD,,