np-workflows 1.6.89__py3-none-any.whl

np_workflows/experiments/task_trained_network/main_ttn_pilot.py

@@ -0,0 +1,263 @@
+import configparser
+import contextlib
+import copy
+import dataclasses
+import datetime
+import enum
+import functools
+import pathlib
+import platform
+import shutil
+import threading
+import time
+import zlib
+from typing import ClassVar, Literal, NamedTuple, NoReturn, Optional, TypedDict
+
+import IPython
+import IPython.display
+import ipywidgets as ipw
+import np_config
+import np_logging
+import np_services
+import np_session
+import np_workflows
+import PIL.Image
+import pydantic
+from pyparsing import Any
+from np_services import (
+    Service,
+    Finalizable,
+    ScriptCamstim,
+    OpenEphys,
+    Sync,
+    VideoMVR,
+    NewScaleCoordinateRecorder,
+    MouseDirector,
+)
+
+from .ttn_stim_config import (
+    TTNSession,
+    camstim_defaults,
+    per_session_main_stim_params,
+    per_session_mapping_params,
+    per_session_opto_params,
+    default_ttn_params as DEFAULT_STIM_PARAMS,
+)
+
+logger = np_logging.getLogger(__name__)
+
+
+class TTNMixin:
+    """Provides TTN-specific methods and attributes, mainly related to camstim scripts."""
+    
+    ttn_session: TTNSession
+    """Enum for session type, e.g. PRETEST, HAB_60, HAB_90, EPHYS."""
+
+    @property
+    def script_root_on_stim(self) -> pathlib.Path:
+        "Path to local copy on Stim, from Stim."
+        return pathlib.Path('C:/ProgramData/StimulusFiles/dev')
+    
+    @property
+    def script_root_on_local(self) -> pathlib.Path:
+        "Path to version controlled scripts on local machine."
+        return (pathlib.Path(__file__).parent / 'camstim_scripts')
+    
+    @property
+    def script_names(self) -> dict[Literal["main", "mapping", "opto"], str]:
+        return {
+            label: f"ttn_{label}_script.py" for label in ("main", "mapping", "opto")
+        }
+        
+    @property
+    def stim_root_on_stim(self) -> pathlib.Path:
+        "Path to dev folder on Stim computer, as seen from local machine."
+        return np_config.local_to_unc(self.rig.stim, self.script_root_on_stim)
+    
+    @property
+    def stim_root_on_local(self) -> pathlib.Path:
+        "Path to version controlled stim files on local machine."
+        return self.script_root_on_local / 'stims'
+        
+    @property
+    def recorders(self) -> tuple[Service, ...]:
+        """Services to be started before stimuli run, and stopped after. Session-dependent."""
+        match self.ttn_session:
+            case TTNSession.PRETEST | TTNSession.EPHYS:
+                return (Sync, VideoMVR, OpenEphys)
+            case TTNSession.HAB_60 | TTNSession.HAB_90 | TTNSession.HAB_120:
+                return (Sync, VideoMVR)
+
+    @property
+    def stims(self) -> tuple[Service, ...]:
+        return (ScriptCamstim,)
+    
+    def initialize_and_test_services(self) -> None:
+        """Configure, initialize (ie. reset), then test all services."""
+        
+        MouseDirector.user = self.user.id
+        MouseDirector.mouse = self.mouse.id
+
+        OpenEphys.folder = self.session.folder
+
+        NewScaleCoordinateRecorder.log_root = self.session.npexp_path
+        NewScaleCoordinateRecorder.log_name = self.platform_json.path.name
+
+        self.configure_services()
+
+        super().initialize_and_test_services()
+
+
+    def update_state(self) -> None:
+        "Store useful but non-essential info."
+        self.mouse.state['last_session'] = self.session.id
+        self.mouse.state['last_ttn_session'] = str(self.ttn_session)
+        if self.mouse == 366122:
+            return
+        match self.ttn_session:
+            case TTNSession.PRETEST:
+                return
+            case TTNSession.HAB_60 | TTNSession.HAB_90 | TTNSession.HAB_120:
+                self.session.project.state['latest_hab'] = self.session.id
+            case TTNSession.EPHYS:
+                self.session.project.state['latest_ephys'] = self.session.id
+                self.session.project.state['sessions'] = self.session.project.state.get('sessions', []) + [self.session.id]
+                
+    def run_stim_scripts(self) -> None:
+        self.validate_or_copy_stim_files()
+        self.update_state()
+        
+        for stim in ('mapping', 'main', 'opto'):
+            
+            if not (params := self.params[stim]):
+                logger.info("%s script skipped this session: %r", stim, self.ttn_session)
+                continue
+            
+            ScriptCamstim.params = params
+            ScriptCamstim.script = self.scripts[stim]
+            
+            logger.debug("Starting %s script", stim)
+            
+            ScriptCamstim.start()
+            
+            with contextlib.suppress(Exception):
+                np_logging.web(f'ttn_{self.ttn_session.name.lower()}').info(f"{stim.capitalize()} stim started")
+            
+            with contextlib.suppress(Exception):
+                while not ScriptCamstim.is_ready_to_start():
+                    time.sleep(2.5)
+                
+            if isinstance(ScriptCamstim, Finalizable):
+                ScriptCamstim.finalize()
+
+            with contextlib.suppress(Exception):
+                np_logging.web(f'ttn_{self.ttn_session.name.lower()}').info(f"{stim.capitalize()} stim finished")
+    
+    def validate_or_copy_stim_files(self):
+        for vc_copy in self.stim_root_on_local.iterdir():
+            stim_copy = self.stim_root_on_stim / vc_copy.name
+            validate_or_overwrite(validate=stim_copy, src=vc_copy)
+            
+    @property
+    def params(self) -> dict[Literal["main", "mapping", "opto", "system"], dict[str, Any]]:
+        params = copy.deepcopy(DEFAULT_STIM_PARAMS)
+        params["mouse_id"] = str(self.mouse)
+        params["user_id"] = str(self.user)
+        if system := self.system_camstim_params:
+            params["system"] = system
+        params["main"] = per_session_main_stim_params(self.ttn_session)
+        params["mapping"] = per_session_mapping_params(self.ttn_session)
+        params["opto"] =  per_session_opto_params(self.ttn_session, self.mouse)
+        return params
+
+    @functools.cached_property
+    def scripts(self) -> dict[Literal["main", "mapping", "opto"], str]:
+        """Local path on Stim computer to each script.
+        
+        Verifies Stim copy matches v.c., or overwrites on Stim.
+        """
+        for label in ("main", "mapping", "opto"):
+            script = self.script_names[label]
+            vc_copy = self.script_root_on_local / script
+            stim_copy = np_config.local_to_unc(
+                self.rig.stim, self.script_root_on_stim / script,
+            )
+            
+            validate_or_overwrite(validate=stim_copy, src=vc_copy)
+            
+        return {
+            label: str(self.script_root_on_stim / script) 
+            for label, script in self.script_names.items()
+        }
+
+    @functools.cached_property
+    def system_camstim_params(self) -> dict[str, Any]:
+        "System config on Stim computer, if accessible."
+        return camstim_defaults()
+    
+    
+class Hab(TTNMixin, np_workflows.PipelineHab):
+    def __init__(self, *args, **kwargs):
+        self.services = (
+            MouseDirector,
+            Sync,
+            VideoMVR,
+            self.imager,
+            NewScaleCoordinateRecorder,
+            ScriptCamstim,
+        )
+        super().__init__(*args, **kwargs)
+
+
+class Ephys(TTNMixin, np_workflows.PipelineEphys):
+    def __init__(self, *args, **kwargs):
+        self.services = (
+            MouseDirector,
+            Sync,
+            VideoMVR,
+            self.imager,
+            NewScaleCoordinateRecorder,
+            ScriptCamstim,
+            OpenEphys,
+        )
+        super().__init__(*args, **kwargs)
+
+
+# --------------------------------------------------------------------------------------
+
+
+def new_experiment(
+    mouse: int | str | np_session.Mouse,
+    user: str | np_session.User,
+    session: TTNSession,
+) -> Ephys | Hab:
+    """Create a new experiment for the given mouse and user."""
+    match session:
+        case TTNSession.PRETEST | TTNSession.EPHYS:
+            experiment = Ephys(mouse, user)
+        case TTNSession.HAB_60 | TTNSession.HAB_90 | TTNSession.HAB_120:
+            experiment = Hab(mouse, user)
+        case _:
+            raise ValueError(f"Invalid session type: {session}")
+    experiment.ttn_session = session
+    
+    with contextlib.suppress(Exception):
+        np_logging.web(f'ttn_{experiment.ttn_session.name.lower()}').info(f"{experiment} created")
+            
+    return experiment
+
+
+# --------------------------------------------------------------------------------------
+
+def validate_or_overwrite(validate: str | pathlib.Path, src: str | pathlib.Path):
+    "Checksum validate against `src`, (over)write `validate` as `src` if different."
+    validate, src = pathlib.Path(validate), pathlib.Path(src)
+    def copy():
+        logger.debug("Copying %s to %s", src, validate)
+        shutil.copy2(src, validate)
+    while (
+        validate.exists() == False
+        or (v := zlib.crc32(validate.read_bytes())) != (c := zlib.crc32(pathlib.Path(src).read_bytes()))
+        ):
+        copy()
+    logger.debug("Validated %s CRC32: %08X", validate, (v & 0xFFFFFFFF) )

np_workflows/experiments/task_trained_network/ttn_session_widget.py

@@ -0,0 +1,83 @@
+import configparser
+import contextlib
+import copy
+import enum
+import functools
+from typing import ClassVar, Literal, NamedTuple, NoReturn, Optional, TypedDict
+
+import IPython.display
+import ipywidgets as ipw
+import np_config
+import np_logging
+import np_session
+import np_workflows
+from pyparsing import Any
+
+from .ttn_stim_config import TTNSession
+
+global_state = {}
+"""Global variable for persisting widget states."""
+
+# for widget, before creating a experiment --------------------------------------------- #
+
+class TTNSelectedSession:
+    def __init__(self, session: str | TTNSession, mouse: str | int | np_session.Mouse):
+        if isinstance(session, str):
+            session = TTNSession(session)
+        self.session = session
+        self.mouse = str(mouse)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.session}, {self.mouse})"
+
+
+def stim_session_select_widget(
+    mouse: str | int | np_session.Mouse,
+) -> TTNSelectedSession:
+    """Select a stimulus session (hab, pretest, ephys) to run.
+
+    An object with mutable attributes is returned, so the selected session can be
+    updated along with the GUI selection. (Preference would be to return an enum
+    directly, and change it's value, but that doesn't seem possible.)
+
+    """
+
+    selection = TTNSelectedSession(TTNSession.PRETEST, mouse)
+
+    session_dropdown = ipw.Select(
+        options=tuple(_.value for _ in TTNSession),
+        description="Session",
+    )
+        
+    def update_selection():
+        selection.__init__(str(session_dropdown.value), str(mouse))
+        
+    if (previously_selected_value := global_state.get('selected_session')):
+        session_dropdown.value = previously_selected_value
+        update_selection()
+        
+    console = ipw.Output()
+    with console:
+        if last_session := np_session.Mouse(selection.mouse).state.get('last_ttn_session'):
+            print(f"{mouse} last session: {last_session}")
+        print(f"Selected: {selection.session}")
+
+    def update(change):
+        if change["name"] != "value":
+            return
+        if (options := getattr(change["owner"], "options", None)) and change[
+            "new"
+        ] not in options:
+            return
+        if change["new"] == change["old"]:
+            return
+        update_selection()
+        with console:
+            print(f"Selected: {selection.session}")
+        global_state['selected_session'] = selection.session.value
+        
+    session_dropdown.observe(update, names='value')
+
+    IPython.display.display(ipw.VBox([session_dropdown, console]))
+
+    return selection

np_workflows/experiments/task_trained_network/ttn_stim_config.py

@@ -0,0 +1,213 @@
+import configparser
+import contextlib
+import copy
+import enum
+import functools
+from typing import ClassVar, Literal, NamedTuple, NoReturn, Optional, TypedDict
+
+import IPython.display
+import ipywidgets as ipw
+import np_config
+import np_logging
+import np_session
+import np_workflows
+from typing import Any
+
+logger = np_logging.getLogger(__name__)
+
+
+class TTNSession(enum.Enum):
+    """Enum for the different TTN sessions available, each with different param sets."""
+
+    PRETEST = "pretest"
+    HAB_60 = "hab 60"
+    HAB_90 = "hab 90"
+    HAB_120 = "hab 120"
+    EPHYS = "ephys"
+
+
+# setup parameters ---------------------------------------------------------------------
+default_ttn_params = {}
+
+def camstim_defaults() -> dict:
+    """Try to load defaults from camstim config file on the Stim computer.
+    
+    May encounter permission error if not running as svc_neuropix.
+    """
+    with contextlib.suppress(OSError):
+        parser = configparser.RawConfigParser()
+        parser.read(
+            (np_config.Rig().paths["Camstim"].parent / "config" / "stim.cfg").as_posix()
+        )
+
+        camstim_default_config = {}
+        for section in parser.sections():
+            camstim_default_config[section] = {}
+            for k, v in parser[section].items():
+                try:
+                    value = eval(
+                        v
+                    )  # this removes comments in config and converts values to expected datatype
+                except:
+                    continue
+                else:
+                    camstim_default_config[section][k] = value
+        return camstim_default_config
+    logger.warning("Could not load camstim defaults from config file on Stim computer.")
+    return {}
+
+## no longer added to default_ttn_params:
+# default_ttn_params.update(**camstim_defaults())
+
+
+# main stimulus defaults ---------------------------------------------------------------
+default_ttn_params["main"] = {}
+
+default_ttn_params["main"]["sweepstim"] = {
+    "trigger_delay_sec": 5.0,           #! does it matter?
+    'sync_sqr_loc': (870, 525),         # for Window.warp=Warp.Disabled 
+    'sync_sqr_loc_warp': (540, 329),    # for Window.warp=Warp.Spherical
+}
+# default_ttn_params["main"]["movie_path"] = "C:/ProgramData/StimulusFiles/dev/"
+default_ttn_params["main"][
+    "monitor"
+] = 'Gamma1.Luminance50'
+
+# other parameters that vary depending on session type (pretest, hab, ephys):
+def per_session_main_stim_params(session: TTNSession) -> dict[str, Any]:
+    def build_session_stim_params(
+        key: str, old: int, reversed: int, annotated: int
+    ) -> dict[str, dict[str, int]]:
+        return {key: dict(old=old, reversed=reversed, annotated=annotated)}
+
+    def session_stim_repeats(session: TTNSession) -> dict[str, dict[str, int]]:
+        repeats = functools.partial(build_session_stim_params, "stim_repeats")
+        match session:
+            case TTNSession.PRETEST:
+                return repeats(1, 1, 1)
+            case TTNSession.HAB_60:
+                return repeats(15, 5, 1)
+            case TTNSession.HAB_90:
+                return repeats(20, 7, 2)
+            case TTNSession.HAB_120 | TTNSession.EPHYS:
+                return repeats(25, 8, 2)
+            case _:
+                raise ValueError(f"Stim repeats not implemented for {session}")
+
+    def session_stim_lengths(session: TTNSession) -> dict[str, dict[str, int]]:
+        lengths_sec = functools.partial(build_session_stim_params, "stim_lengths_sec")
+        match session:
+            case TTNSession.PRETEST:
+                return lengths_sec(1, 1, 1)
+            case _:
+                return lengths_sec(40, 40, 60)
+
+    def main_blank_screen(session: TTNSession) -> dict[str, float | int]:
+        match session:
+            case TTNSession.PRETEST:
+                return {"pre_blank_screen_sec": .5, "post_blank_screen_sec": .5}
+            case _:
+                return {"pre_blank_screen_sec": 2, "post_blank_screen_sec": 2}
+
+    params = copy.deepcopy(default_ttn_params["main"])
+    params.update(session_stim_repeats(session))
+    params.update(session_stim_lengths(session))
+    params.update(main_blank_screen(session))
+    return params
+
+
+# optotagging defaults -----------------------------------------------------------------
+
+default_ttn_params["opto"] = {}
+
+# all parameters depend on session type (pretest, hab, ephys):
+
+def per_session_opto_params(
+    session: TTNSession, mouse: str | int | np_session.Mouse
+) -> dict[str, dict[str, str | list[float] | Literal["pretest", "experiment"]]]:
+    "All params for opto depending on session (e.g. will be empty for habs)."
+
+    def opto_mouse_id(mouse_id: str | int | np_session.Mouse) -> dict[str, str]:
+        return {"mouseID": str(mouse_id)}
+
+    def opto_levels(session: TTNSession) -> dict[str, list[float]]:
+        default_opto_levels: list[float] = camstim_defaults()["Optogenetics"][
+            "level_list"
+        ]
+        match session:
+            case TTNSession.PRETEST | TTNSession.EPHYS:
+                return {"level_list": sorted(default_opto_levels)[-2:]}
+            case _:
+                raise ValueError(f"Opto levels not implemented for {session}")
+
+    def opto_operation_mode(
+        session: TTNSession,
+    ) -> dict[str, Literal["pretest", "experiment"]]:
+        match session:
+            case TTNSession.PRETEST:
+                return {"operation_mode": "pretest"}
+            case TTNSession.EPHYS:
+                return {"operation_mode": "experiment"}
+            case _:
+                raise ValueError(f"Opto levels not implemented for {session}")
+
+    match session:
+        case TTNSession.PRETEST | TTNSession.EPHYS:
+            params = copy.deepcopy(default_ttn_params["opto"])
+            params.update(opto_mouse_id(mouse))
+            params.update(opto_levels(session))
+            params.update(opto_operation_mode(session))
+            return params
+        case TTNSession.HAB_60 | TTNSession.HAB_90 | TTNSession.HAB_120:
+            return {}
+        case _:
+            raise ValueError(f"Opto params not implemented for {session}")
+
+
+# mapping defaults ---------------------------------------------------------------------
+
+default_ttn_params["mapping"] = {}
+
+default_ttn_params["mapping"]["monitor"] = "Gamma1.Luminance50"
+default_ttn_params["mapping"]["gabor_path"] = "gabor_20_deg_250ms.stim" 
+default_ttn_params["mapping"]["flash_path"] = "flash_250ms.stim" # relpath in StimulusFiles dir (for _v2 objects)
+default_ttn_params["mapping"]["default_gabor_duration_seconds"] = 1200
+default_ttn_params["mapping"]["default_flash_duration_seconds"] = 300 # may be overriden by 'max_total_duration_minutes'
+
+default_ttn_params["mapping"]["sweepstim"] = {
+    'sync_sqr_loc_warp': (540, 329),     # for Window.warp=Warp.Spherical
+} 
+# trigger_delay_sec not specified
+# all stim parameters depend on session type (pretest, hab, ephys):
+def per_session_mapping_params(session: TTNSession) -> dict[str, dict[str, int]]:
+    "`'mapping'` key in params dict should be updated with the returned dict (which will be empty for habs)."
+
+    def mapping_duration(session: TTNSession) -> dict[str, float]:
+        # 0 = full length = gabor_duration + flash_duration = maximum possible
+        match session:
+            case TTNSession.PRETEST:
+                return {"max_total_duration_minutes": 0.1}
+            case TTNSession.EPHYS | TTNSession.HAB_120:
+                return {"max_total_duration_minutes": 10}
+            case _:
+                raise ValueError(f"Mapping params not implemented for {session}")
+
+    def mapping_blank_screen(session: TTNSession) -> dict[str, float | int]:
+        match session:
+            case TTNSession.PRETEST:
+                return {"pre_blank_screen_sec": .5, "post_blank_screen_sec": .5}
+            case TTNSession.EPHYS | TTNSession.HAB_120:
+                return {"pre_blank_screen_sec": 2, "post_blank_screen_sec": 2}
+            case _:
+                raise ValueError(f"Mapping params not implemented for {session}")
+
+    match session:
+        case TTNSession.PRETEST | TTNSession.EPHYS | TTNSession.HAB_120:
+            params = copy.deepcopy(default_ttn_params["mapping"])
+            params.update(mapping_duration(session))
+            params.update(mapping_blank_screen(session))
+            return params
+        case TTNSession.HAB_60 | TTNSession.HAB_90:
+            return {}
+        case _:
+            raise ValueError(f"Mapping params not implemented for {session}")

np_workflows/experiments/templeton/__init__.py

@@ -0,0 +1,2 @@
+from np_workflows.experiments.templeton.main import *
+from np_workflows.experiments.templeton.widgets import *

np_workflows/experiments/templeton/main.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import configparser
+import contextlib
+import copy
+import dataclasses
+import datetime
+import enum
+import functools
+import pathlib
+import platform
+import shutil
+import threading
+import time
+import zlib
+from typing import ClassVar, Literal, NamedTuple, NoReturn, Optional, Type, TypedDict
+
+import IPython
+import IPython.display
+import ipywidgets as ipw
+import np_config
+import np_logging
+import np_services
+import np_session
+import np_workflows
+import fabric
+import PIL.Image
+import pydantic
+from pyparsing import Any
+from np_services import (
+    Service,
+    Finalizable,
+    ScriptCamstim,
+    OpenEphys,
+    Sync,
+    ImageMVR,
+    VideoMVR,
+    NewScaleCoordinateRecorder,
+    MouseDirector,
+)
+
+from np_workflows.shared import base_experiments
+
+logger = np_logging.getLogger(__name__)
+
+class TempletonPilot(base_experiments.DynamicRoutingExperiment):
+    """Provides project-specific methods and attributes, mainly related to camstim scripts."""
+    
+    default_session_subclass = np_session.TempletonPilotSession
+    
+    workflow: base_experiments.DynamicRoutingExperiment.Workflow
+    """Enum for workflow type, e.g. PRETEST, HAB_AUD, HAB_VIS, EPHYS_ etc."""
+
+    @property
+    def task_name(self) -> str:
+        task_name = super().task_name
+        return f'templeton {task_name}' if 'templeton' not in task_name else task_name
+    
+    @task_name.setter
+    def task_name(self, value: str):
+        super().task_name = value
+       
+def new_experiment(
+    mouse: int | str | np_session.Mouse,
+    user: str | np_session.User,
+    workflow: base_experiments.DynamicRoutingExperiment.Workflow,
+) -> TempletonPilot:
+    """Create a new experiment for the given mouse and user."""
+    experiment: Ephys | Hab
+    if any(tag in workflow.name for tag in ('EPHYS', 'PRETEST')):
+        experiment = Ephys(mouse, user)
+    elif 'HAB' in workflow.name:
+        experiment = Hab(mouse, user)
+    else:
+        raise ValueError(f"Unknown {workflow = }. Create an experiment with e.g.\n\n\texperiment = Ephys(mouse, user)\nexperiment.session.npexp_path.mkdir()")
+    experiment.workflow = workflow
+    experiment.log(f"{experiment} created")
+    experiment.session.npexp_path.mkdir(parents=True, exist_ok=True) 
+    return experiment
+
+class Hab(TempletonPilot):
+    def __init__(self, *args, **kwargs):
+        self.services = (
+            MouseDirector,
+            Sync,
+            VideoMVR,
+            self.imager,
+            ScriptCamstim,
+            NewScaleCoordinateRecorder,
+        )
+        super().__init__(*args, **kwargs)
+
+
+class Ephys(TempletonPilot):
+    def __init__(self, *args, **kwargs):
+        self.services = (
+            MouseDirector,
+            Sync,
+            VideoMVR,
+            self.imager,
+            ScriptCamstim,
+            OpenEphys,
+            NewScaleCoordinateRecorder,
+        )
+        super().__init__(*args, **kwargs)