np-workflows 1.6.86__py3-none-any.whl → 1.6.88__py3-none-any.whl

np_workflows/experiments/openscope_v2/__init__.py

@@ -0,0 +1,2 @@
+from .main_V2_pilot import new_experiment, Hab, Ephys, validate_selected_workflow
+from .V2_workflow_widget import V2_workflow_widget

np_workflows/experiments/openscope_v2/main_v2_pilot.py

@@ -0,0 +1,217 @@
+import configparser
+import contextlib
+import copy
+import dataclasses
+import datetime
+import enum
+import functools
+import pathlib
+import platform
+import shutil
+import threading
+import time
+import zlib
+from typing import ClassVar, Literal, NamedTuple, NoReturn, Optional, TypedDict
+
+import IPython
+import IPython.display
+import ipywidgets as ipw
+import np_config
+import np_logging
+import np_services
+import np_session
+import np_workflows
+import PIL.Image
+import pydantic
+from pyparsing import Any
+from np_services import (
+    Service,
+    Finalizable,
+    ScriptCamstim, SessionCamstim, 
+    SessionCamstim,
+    OpenEphys,
+    Sync,
+    VideoMVR,
+    NewScaleCoordinateRecorder,
+    MouseDirector,
+)
+
+logger = np_logging.getLogger(__name__)
+
+
+class V2Session(enum.Enum):
+    """Enum for the different sessions available, each with different param sets."""
+
+    PRETEST = "pretest"
+    HAB = "hab"
+    EPHYS = "ephys"
+
+
+class V2Mixin:
+    """Provides project-specific methods and attributes, mainly related to camstim scripts."""
+    
+    workflow: V2Session
+    """Enum for particular workflow/session, e.g. PRETEST, HAB_60, HAB_90,
+    EPHYS."""
+    
+    session: np_session.PipelineSession
+    mouse: np_session.Mouse
+    user: np_session.User
+    platform_json: np_session.PlatformJson
+    
+    @property
+    def recorders(self) -> tuple[Service, ...]:
+        """Services to be started before stimuli run, and stopped after. Session-dependent."""
+        match self.workflow:
+            case V2Session.PRETEST | V2Session.EPHYS:
+                return (Sync, VideoMVR, OpenEphys)
+            case V2Session.HAB:
+                return (Sync, VideoMVR)
+
+    @property
+    def stims(self) -> tuple[Service, ...]:
+        return (SessionCamstim, )
+    
+    def initialize_and_test_services(self) -> None:
+        """Configure, initialize (ie. reset), then test all services."""
+        
+        MouseDirector.user = self.user.id
+        MouseDirector.mouse = self.mouse.id
+
+        OpenEphys.folder = self.session.folder
+
+        NewScaleCoordinateRecorder.log_root = self.session.npexp_path
+        NewScaleCoordinateRecorder.log_name = self.platform_json.path.name
+
+        SessionCamstim.labtracks_mouse_id = self.mouse.id
+        SessionCamstim.lims_user_id = self.user.id
+
+        self.configure_services()
+
+        super().initialize_and_test_services()
+
+    def update_state(self) -> None:
+        "Store useful but non-essential info."
+        self.mouse.state['last_session'] = self.session.id
+        self.mouse.state['last_V2_session'] = str(self.workflow)
+        if self.mouse == 366122:
+            return
+        match self.workflow:
+            case V2Session.PRETEST:
+                return
+            case V2Session.HAB:
+                self.session.project.state['latest_hab'] = self.session.id
+            case V2Session.EPHYS:
+                self.session.project.state['latest_ephys'] = self.session.id
+                self.session.project.state['sessions'] = self.session.project.state.get('sessions', []) + [self.session.id]
+                
+    def run_stim(self) -> None:
+
+        self.update_state()
+        
+        if not SessionCamstim.is_ready_to_start():
+            raise RuntimeError("SessionCamstim is not ready to start.")
+        
+        np_logging.web(f'V2_{self.workflow.name.lower()}').info(f"Started session {self.mouse.mtrain.stage['name']}")
+        SessionCamstim.start()
+        
+        with contextlib.suppress(Exception):
+            while not SessionCamstim.is_ready_to_start():
+                time.sleep(2.5)
+            
+        if isinstance(SessionCamstim, Finalizable):
+            SessionCamstim.finalize()
+
+        with contextlib.suppress(Exception):
+            np_logging.web(f'V2_{self.workflow.name.lower()}').info(f"Finished session {self.mouse.mtrain.stage['name']}")
+    
+    
+    def copy_data_files(self) -> None: 
+        super().copy_data_files()
+        
+        # When all processing completes, camstim Agent class passes data and uuid to
+        # /camstim/lims BehaviorSession class, and write_behavior_data() writes a
+        # final .pkl with default name YYYYMMDDSSSS_mouseID_foragingID.pkl
+        # - if we have a foraging ID, we can search for that
+        if None == (stim_pkl := next(self.session.npexp_path.glob(f'{self.session.date:%y%m%d}*_{self.session.mouse}_*.pkl'), None)):
+            logger.warning('Did not find stim file on npexp matching the format `YYYYMMDDSSSS_mouseID_foragingID.pkl`')
+            return
+        assert stim_pkl
+        if not self.session.platform_json.foraging_id:
+            self.session.platform_json.foraging_id = stim_pkl.stem.split('_')[-1]
+        new_stem = f'{self.session.folder}.stim'
+        logger.debug(f'Renaming stim file copied to npexp: {stim_pkl} -> {new_stem}')
+        stim_pkl = stim_pkl.rename(stim_pkl.with_stem(new_stem))
+        
+        # remove other stim pkl, which is nearly identical, if it was also copied
+        for pkl in self.session.npexp_path.glob('*.pkl'):
+            if (
+                self.session.folder not in pkl.stem
+                and 
+                abs(pkl.stat().st_size - stim_pkl.stat().st_size) < 1e6
+            ):
+                logger.debug(f'Deleting extra stim pkl copied to npexp: {pkl.stem}')
+                pkl.unlink()
+        
+        
+def validate_selected_workflow(session: V2Session, mouse: np_session.Mouse) -> None:
+    for workflow in ('hab', 'ephys'):
+        if (
+            workflow in session.value.lower()
+            and workflow not in mouse.mtrain.stage['name'].lower()
+        ) or (
+            session.value.lower() == 'ephys' and 'hab' in mouse.mtrain.stage['name'].lower()
+        ):
+            raise ValueError(f"Workflow selected ({session.value}) does not match MTrain stage ({mouse.mtrain.stage['name']}): please check cells above.")
+
+    
+class Hab(V2Mixin, np_workflows.PipelineHab):
+    def __init__(self, *args, **kwargs):
+        self.services = (
+            MouseDirector,
+            Sync,
+            VideoMVR,
+            self.imager,
+            NewScaleCoordinateRecorder,
+            SessionCamstim,
+        )
+        super().__init__(*args, **kwargs)
+
+
+class Ephys(V2Mixin, np_workflows.PipelineEphys):
+    def __init__(self, *args, **kwargs):
+        self.services = (
+            MouseDirector,
+            Sync,
+            VideoMVR,
+            self.imager,
+            NewScaleCoordinateRecorder,
+            SessionCamstim,
+            OpenEphys,
+        )
+        super().__init__(*args, **kwargs)
+
+
+# --------------------------------------------------------------------------------------
+
+
+def new_experiment(
+    mouse: int | str | np_session.Mouse,
+    user: str | np_session.User,
+    workflow: V2Session,
+) -> Ephys | Hab:
+    """Create a new experiment for the given mouse and user."""
+    match workflow:
+        case V2Session.PRETEST | V2Session.EPHYS:
+            experiment = Ephys(mouse, user)
+        case V2Session.HAB:
+            experiment = Hab(mouse, user)
+        case _:
+            raise ValueError(f"Invalid workflow type: {workflow}")
+    experiment.workflow = workflow
+    
+    with contextlib.suppress(Exception):
+        np_logging.web(f'V2_{experiment.workflow.name.lower()}').info(f"{experiment} created")
+            
+    return experiment
+

np_workflows/experiments/openscope_v2/v2_workflow_widget.py

@@ -0,0 +1,83 @@
+import configparser
+import contextlib
+import copy
+import enum
+import functools
+from typing import ClassVar, Literal, NamedTuple, NoReturn, Optional, TypedDict
+
+import IPython.display
+import ipywidgets as ipw
+import np_config
+import np_logging
+import np_session
+import np_workflows
+from pyparsing import Any
+
+from np_workflows.experiments.openscope_V2.main_V2_pilot import V2Session
+
+global_state = {}
+"""Global variable for persisting widget states."""
+
+# for widget, before creating a experiment --------------------------------------------- #
+
+class SelectedSession:
+    def __init__(self, session: str | V2Session, mouse: str | int | np_session.Mouse):
+        if isinstance(session, str):
+            session = V2Session(session)
+        self.session = session
+        self.mouse = str(mouse)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.session}, {self.mouse})"
+
+
+def V2_workflow_widget(
+    mouse: str | int | np_session.Mouse,
+) -> SelectedSession:
+    """Select a stimulus session (hab, pretest, ephys) to run.
+
+    An object with mutable attributes is returned, so the selected session can be
+    updated along with the GUI selection. (Preference would be to return an enum
+    directly, and change it's value, but that doesn't seem possible.)
+
+    """
+
+    selection = SelectedSession(V2Session.PRETEST, mouse)
+
+    session_dropdown = ipw.Select(
+        options=tuple(_.value for _ in V2Session),
+        description="Session",
+    )
+    
+    def update_selection():
+        selection.__init__(str(session_dropdown.value), str(mouse))
+        
+    if (previously_selected_value := global_state.get('selected_session')):
+        session_dropdown.value = previously_selected_value
+        update_selection()
+        
+    console = ipw.Output()
+    with console:
+        if last_session := np_session.Mouse(selection.mouse).state.get('last_V2_session'):
+            print(f"{mouse} last session: {last_session}")
+        print(f"Selected: {selection.session}")
+
+    def update(change):
+        if change["name"] != "value":
+            return
+        if (options := getattr(change["owner"], "options", None)) and change[
+            "new"
+        ] not in options:
+            return
+        if change["new"] == change["old"]:
+            return
+        update_selection()
+        with console:
+            print(f"Selected: {selection.session}")
+        global_state['selected_session'] = selection.session.value
+        
+    session_dropdown.observe(update, names='value')
+
+    IPython.display.display(ipw.VBox([session_dropdown, console]))
+
+    return selection

np_workflows/shared/base_experiments.py

@@ -8,7 +8,7 @@ import pathlib
 import re
 import shutil
 import time
-from typing import Any, ClassVar, Iterable, Literal, Optional, Protocol, Sequence, Type
+from typing import Any, ClassVar, Iterable, Literal, Mapping, Optional, Protocol, Sequence, Type
 
 import fabric
 import invoke
@@ -712,7 +712,13 @@ class DynamicRoutingExperiment(WithSession):
         else:
             np_services.ScriptCamstim.script = self.camstim_script.as_posix()
         
-        np_services.ScriptCamstim.params = params
+        if (script_override_params := getattr(self, 'script_override_params', None)) is not None:
+            if not isinstance(script_override_params, Mapping):
+                raise TypeError(f"script_override_params must be a dict/Mapping, got {type(script_override_params)}")
+        else:
+            script_override_params = {}
+            
+        np_services.ScriptCamstim.params = params | script_override_params
         
         self.update_state()
         self.log(f"{stim} started")

{np_workflows-1.6.86.dist-info → np_workflows-1.6.88.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: np-workflows
-Version: 1.6.86
+Version: 1.6.88
 Summary: Ecephys and behavior workflows for the Mindscope Neuropixels team.
 Author-Email: Ben Hardcastle <ben.hardcastle@alleninstitute.org>
 License: MIT

{np_workflows-1.6.86.dist-info → np_workflows-1.6.88.dist-info}/RECORD

@@ -1,6 +1,6 @@
-np_workflows-1.6.86.dist-info/METADATA,sha256=ghO62M3597HuIb5S3o_7XAAjGeCp4smhtUVawkXXz_I,3991
-np_workflows-1.6.86.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-np_workflows-1.6.86.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+np_workflows-1.6.88.dist-info/METADATA,sha256=IhX7bzLQy2osL9vRBRVOZ2piKwCVCtitJPIkfQQ5jmw,3991
+np_workflows-1.6.88.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
+np_workflows-1.6.88.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
 np_workflows/__init__.py,sha256=QWnSRZ3THmQPF6HO8rFC4EkcjGd5ypap7BssqKqva64,232
 np_workflows/assets/images/logo_np_hab.png,sha256=N1skdSq4lQv0liucargHoulzfE-zeTYoud-wAePD2VY,10071
 np_workflows/assets/images/logo_np_vis.png,sha256=yDaQ3dQcX1OXPN8Rsne9SvaevUlgvVRcw_tWs02cyTU,19349
@@ -21,6 +21,9 @@ np_workflows/experiments/openscope_loop/main_loop_pilot.py,sha256=ye9qPdZPy5_Em7
 np_workflows/experiments/openscope_psycode/__init__.py,sha256=FYX6cNbpueIFqKU5g2l3IeFAgXv8Ikzuv6dC0SMpusQ,148
 np_workflows/experiments/openscope_psycode/main_psycode_pilot.py,sha256=gpYAw4JKU71iveRIu4O_L3gAw4KodCVYcPGhW7lHrCg,7404
 np_workflows/experiments/openscope_psycode/psycode_workflow_widget.py,sha256=LZPRfi3H0z4_zbVezmAPhVUfInpZyaKBKx0tEqkPx4I,2654
+np_workflows/experiments/openscope_v2/__init__.py,sha256=Tmzui2YozeY_a1fiB8dYnlWZdpK_1gEV7MaVtwprP-0,133
+np_workflows/experiments/openscope_v2/main_v2_pilot.py,sha256=O0LThYX8ahKLTgKKcW0rkaqM9bmQkXaDi6l13Wzh-t0,7304
+np_workflows/experiments/openscope_v2/v2_workflow_widget.py,sha256=VI3gohtX5lWXG08x5Xf1a98o-QXXWV25TQpADIlVc2I,2609
 np_workflows/experiments/openscope_vippo/__init__.py,sha256=LTrsyImW6JjUFsu6qoIjdb3aM7YTkQovyLEDHWFv2aA,142
 np_workflows/experiments/openscope_vippo/main_vippo_pilot.py,sha256=q859hjqaFqlMUl6phmOH8PnVYg6iCyG6H67o_HAvDMA,7364
 np_workflows/experiments/openscope_vippo/vippo_workflow_widget.py,sha256=B5kiU-kXypoA7lSonaS7OWZecV4HdmgOdPZq9INbpHE,2636
@@ -62,9 +65,9 @@ np_workflows/experiments/templeton/__init__.py,sha256=LrTnP6DnkTSB-sdadvSOJvLZMZ
 np_workflows/experiments/templeton/main.py,sha256=5JDBN4fQbBy2PfiUh6OQr0NVZo-h58la3C-mgfLsU3Y,2906
 np_workflows/experiments/templeton/widgets.py,sha256=e0t7zhD0N-i6Q5_8EWW4lYrESuAb7hIz_BEL4NPaywo,2884
 np_workflows/shared/__init__.py,sha256=L9h5R3dioT8m0KcwjlQV5_eoEFoneJHEzDTGaH4_ySg,131
-np_workflows/shared/base_experiments.py,sha256=suPXvb_qQOSDEX0ELk70-4COFg7IF9D84FhK84S_x_0,34119
+np_workflows/shared/base_experiments.py,sha256=K0G4XSMkpP2s6c9XZAtxaj9niRd8GLpZQ6FVX3YDPuA,34500
 np_workflows/shared/camstim_scripts/flash_250ms.stim,sha256=ARwNQPJRRQwEaD_k0oXR1lafdafWlf_Kvfbq6TVEkDU,408
 np_workflows/shared/camstim_scripts/gabor_20_deg_250ms.stim,sha256=xvuRMORZiN9IuOLWZbNTWuxmhfHefGnZtIpeYcsQXpE,718
 np_workflows/shared/npxc.py,sha256=NWi826e14EEt3vKlsLwsQ2wYcsLk9Ea_Hrb8hn2iA3Y,6689
 np_workflows/shared/widgets.py,sha256=aehptvxluwfyRs7Sa4ZSvCMhOwGCxhUM7STBR4wFG5k,26880
-np_workflows-1.6.86.dist-info/RECORD,,
+np_workflows-1.6.88.dist-info/RECORD,,

{np_workflows-1.6.86.dist-info → np_workflows-1.6.88.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.4.3)
+Generator: pdm-backend (2.4.4)
 Root-Is-Purelib: true
 Tag: py3-none-any