np_codeocean 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

np_codeocean/np_session_utils.py

@@ -6,6 +6,8 @@ import doctest
 import pathlib
 from collections.abc import Iterable
 import shutil
+import time
+from typing import Any
 
 import aind_data_transfer_models.core
 import np_config
@@ -302,6 +304,7 @@ def upload_session(
     regenerate_symlinks: bool = True,
     adjust_ephys_timestamps: bool = True,
     codeocean_configs: aind_data_transfer_models.core.CodeOceanPipelineMonitorConfigs | None = None,
+    extra_BasicUploadJobConfigs_params: dict[str, Any] | None = None,
 ) -> None:
     codeocean_root = np_session.NPEXP_PATH / ('codeocean-dev' if test else 'codeocean')
     logger.debug(f'{codeocean_root = }')
@@ -322,6 +325,7 @@ def upload_session(
         create_behavior_symlinks(upload.session, upload.behavior)
     if upload.behavior_videos:
         create_behavior_videos_symlinks(upload.session, upload.behavior_videos)
+    timestamps_adjusted = False
     if adjust_ephys_timestamps and upload.ephys:
         if not upload.behavior: # includes surface channel recordings
             logger.warning(f"Cannot adjust ephys timestamps for {upload.session} - no behavior folder supplied for upload")
@@ -336,14 +340,22 @@ def upload_session(
                         "`adjust_ephys_timestamps=False` or `--no-sync` flag in CLI"
                     )
                 ) from None
+            else:
+                timestamps_adjusted = True
     for path in (upload.ephys, upload.behavior, upload.behavior_videos, upload.aind_metadata):
         if path is not None and path.exists():
             utils.convert_symlinks_to_posix(path)
     csv_content: dict = get_upload_csv_for_session(upload)
     utils.write_upload_csv(csv_content, upload.job)
     np_logging.web('np_codeocean').info(f'Submitting {upload.session} to hpc upload queue')
+    if extra_BasicUploadJobConfigs_params is None:
+        extra_BasicUploadJobConfigs_params = {}
+    if codeocean_configs is not None:
+        if 'codeocean_configs' in extra_BasicUploadJobConfigs_params:
+            raise ValueError("Cannot pass `codeocean_configs` as a parameter to `extra_BasicUploadJobConfigs_params`")
+        extra_BasicUploadJobConfigs_params['codeocean_configs'] = codeocean_configs
     utils.put_jobs_for_hpc_upload(
-        utils.get_job_models_from_csv(upload.job, codeocean_configs=codeocean_configs),
+        utils.get_job_models_from_csv(upload.job, check_timestamps=timestamps_adjusted, **extra_BasicUploadJobConfigs_params),
         upload_service_url=utils.DEV_SERVICE if test else utils.AIND_DATA_TRANSFER_SERVICE,
         user_email=hpc_upload_job_email,
         dry_run=dry_run,

np_codeocean/scripts/upload_dynamic_routing_ecephys.py

@@ -153,6 +153,7 @@ def write_metadata_and_upload(
             ignore_errors=True,
             skip_existing=not regenerate_metadata,
         )
+        
     pipelines = [
         aind_codeocean_pipeline_monitor.models.PipelineMonitorSettings(
             run_params=codeocean.computation.RunParams(
@@ -182,6 +183,7 @@ def write_metadata_and_upload(
     codeocean_configs = aind_data_transfer_models.core.CodeOceanPipelineMonitorConfigs(
         pipeline_monitor_capsule_settings=pipelines,
     )
+    
     return np_codeocean.upload_session(
         session_path_or_folder_name,
         recording_dirs=recording_dirs,
@@ -191,7 +193,9 @@ def write_metadata_and_upload(
         hpc_upload_job_email=hpc_upload_job_email,
         regenerate_symlinks=regenerate_symlinks,
         adjust_ephys_timestamps=adjust_ephys_timestamps,
-        codeocean_configs=codeocean_configs,
+        extra_BasicUploadJobConfigs_params={
+            'codeocean_configs': codeocean_configs,
+        },
     )
 
 def parse_args() -> argparse.Namespace:
@@ -204,13 +208,12 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument('--dry-run', action='store_true', help="Create upload job but do not submit to hpc upload queue.")
     parser.add_argument('--preserve-symlinks', dest='regenerate_symlinks', action='store_false', help="Existing symlink folders will not be deleted and regenerated - may result in additional data being uploaded")
     parser.add_argument('--regenerate-metadata', action='store_true', help="Regenerate metadata files (session.json and rig.json) even if they already exist")
-    parser.add_argument('--no-sync', dest="adjust_ephys_timestamps", action='store_false', help="(disabled) Skip adjustment of ephys timestamps.npy using sync data for sessions where no sync data is available")
+    parser.add_argument('--sync', dest="adjust_ephys_timestamps", action='store_true', help="Adjust ephys timestamps.npy prior to upload using sync data (if available)")
     return parser.parse_args()
 
 def main() -> None:
     args = parse_args()
     kwargs = vars(args)
-    kwargs |= {'adjust_ephys_timestamps':  False} # unnecessary while we have machinery in place for adjusting in npc_sessions (adds 5 GB of timestamps files for each upload)
     np_codeocean.utils.set_npc_lims_credentials()
     write_metadata_and_upload(**kwargs)
 

np_codeocean/utils.py

@@ -307,6 +307,7 @@ def write_upload_csv(
 def get_job_models_from_csv(
     path: pathlib.Path,
     ephys_slurm_settings: aind_slurm_rest.models.V0036JobProperties = DEFAULT_EPHYS_SLURM_SETTINGS,
+    check_timestamps: bool = True, # default in transfer service is True: checks timestamps have been corrected via flag file
     user_email: str = HPC_UPLOAD_JOB_EMAIL,
     **extra_BasicUploadJobConfigs_params: Any,
 ) -> tuple[aind_data_transfer_models.core.BasicUploadJobConfigs, ...]:
@@ -317,15 +318,18 @@ def get_job_models_from_csv(
     models = []
     for job in jobs.copy():
         modalities = []
+        if 'modalities' in extra_BasicUploadJobConfigs_params:
+            raise ValueError('modalities should not be passed as a parameter in extra_BasicUploadJobConfigs_params')
         for modality_column in (k for k in job.keys() if k.startswith('modality') and ".source" not in k):
             modality_name = job[modality_column]
             modalities.append(
                 aind_data_transfer_models.core.ModalityConfigs(
                     modality=modality_name,
                     source=job[f"{modality_column}.source"],
-                    slurm_settings = ephys_slurm_settings if modality_name == 'ecephys' else None,
-                    ),
-                )
+                    slurm_settings=ephys_slurm_settings if modality_name == 'ecephys' else None,
+                    job_settings={'check_timestamps': False} if modality_name == 'ecephys' and not check_timestamps else None,
+                ),
+            )
         for k in (k for k in job.copy().keys() if k.startswith('modality')):
             del job[k]
         for k, v in job.items():

{np_codeocean-0.3.2.dist-info → np_codeocean-0.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: np_codeocean
-Version: 0.3.2
+Version: 0.3.4
 Summary: Tools for uploading and interacting with Mindscope Neuropixels experiments on Code Ocean
 Author-Email: Ben Hardcastle <ben.hardcastle@alleninstitute.org>, Chris Mochizuki <chrism@alleninstitute.org>, Arjun Sridhar <arjun.sridhar@alleninstitute.org>
 License: MIT

{np_codeocean-0.3.2.dist-info → np_codeocean-0.3.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
-np_codeocean-0.3.2.dist-info/METADATA,sha256=PiG3ljcIx1Pkg4T7H9KYjsXBw2XTF8ljgOSGWQSduNM,3159
-np_codeocean-0.3.2.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-np_codeocean-0.3.2.dist-info/entry_points.txt,sha256=p32aRkIjrFa4KtUbq2E6ZMYBVNRUw3U8ZIarvwNkK1E,250
+np_codeocean-0.3.4.dist-info/METADATA,sha256=cgyWIHnxtfuDzsm7FSp7DMjKCChhq95V7-QXYplZRW8,3159
+np_codeocean-0.3.4.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
+np_codeocean-0.3.4.dist-info/entry_points.txt,sha256=p32aRkIjrFa4KtUbq2E6ZMYBVNRUw3U8ZIarvwNkK1E,250
 np_codeocean/__init__.py,sha256=ED7YOu-3AIQIEML9jPT9bQ690lfhYZNiOB4QhnJ8r8U,76
 np_codeocean/metadata/__init__.py,sha256=jayhwvrb4fZYODtiVkJBXEYYkLJ_5SNNQ5WW21bsx2s,72
 np_codeocean/metadata/common.py,sha256=ZOAe8MJJwNtBt9Voqm4MsA-xZCOrzCKkc_rku5A2yNA,584
@@ -14,7 +14,7 @@ np_codeocean/metadata/rigs.py,sha256=Xq3BWvz0pxbkVA1UN9EFprbHA4hYaiSAFk82pXUfdLQ
 np_codeocean/metadata/storage.py,sha256=b7vivpO1E42xoqfLdomAOU-FER4VB3-cUmjTwUQNxNE,2123
 np_codeocean/metadata/update.py,sha256=x2hploIM7AXmX5DaapaNsojJ7E8s_UmTDfVuGRWiyHY,5211
 np_codeocean/metadata/utils.py,sha256=X6CVxgemeCaXiGnt35AHqvSCz0p5keJHHGkIgvYDq5E,4318
-np_codeocean/np_session_utils.py,sha256=yRz62ybX7k192Qr_viRPOL3saIPh5EH60KakAva5Wck,16659
+np_codeocean/np_session_utils.py,sha256=yfLiTUElkgA7M8EFMIu7T6sTP-gB8X0ZVaGHHngjVwg,17318
 np_codeocean/scripts/.mypy_cache/.gitignore,sha256=_7s5RkPBEvI1a7XzGE6V4z_XMhRokvTu6tEml1Vw_zk,36
 np_codeocean/scripts/.mypy_cache/3.11/@plugins_snapshot.json,sha256=RBNvo1WzZ4oRRq0W9-hknpT7T8If536DEMBg9hyq_4o,2
 np_codeocean/scripts/.mypy_cache/3.11/IPython/__init__.data.json,sha256=XIcapuTlKIvgnYu5wffxS8aovyHcg02vUtjEfA_eBDw,5427
@@ -3395,7 +3395,7 @@ np_codeocean/scripts/.mypy_cache/3.11/zoneinfo/__init__.data.json,sha256=g3oqeS6
 np_codeocean/scripts/.mypy_cache/3.11/zoneinfo/__init__.meta.json,sha256=AvLhG4cE-92NWoQ-6ZIX546e4YT57YVtwE2rTyRA7rg,1826
 np_codeocean/scripts/.mypy_cache/CACHEDIR.TAG,sha256=cOu1T6y3b32rMFgmHu-0OiMV0o0Q_PkKhVOMBacrjR4,193
 np_codeocean/scripts/upload_dynamic_routing_behavior.py,sha256=A7FapFuVECdPZeRmeaOtH708DzWh44IkXHTMHSVzHys,17169
-np_codeocean/scripts/upload_dynamic_routing_ecephys.py,sha256=8-mwX06PspPwfLqS9DrJTPvHKZ1B9thSdq7fCH3iNn8,10664
+np_codeocean/scripts/upload_dynamic_routing_ecephys.py,sha256=6dzumt57OrDl4hsqqEY2EZbGzuxX-jZdbsqVBBfw82o,10528
 np_codeocean/scripts/upload_split_recordings_example.py,sha256=1_aqoBxAkB_VpRKYqyPsEQBDGvgyAHXAkIJA0ZT2Vb0,1490
-np_codeocean/utils.py,sha256=AstOldFmunAb_O75cOrjsyATQnaEWzKnSYShv3tzVCk,20648
-np_codeocean-0.3.2.dist-info/RECORD,,
+np_codeocean/utils.py,sha256=xOlDmZhF0m57t0bOS6S9niIakg2EDuz3gjvKZIfUPDk,21075
+np_codeocean-0.3.4.dist-info/RECORD,,

{np_codeocean-0.3.2.dist-info → np_codeocean-0.3.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.4.3)
+Generator: pdm-backend (2.4.4)
 Root-Is-Purelib: true
 Tag: py3-none-any