deriva-ml 1.17.9__py3-none-any.whl → 1.17.11__py3-none-any.whl

deriva_ml/execution/find_caller.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import inspect
+import sys
+from pathlib import Path
+from types import FrameType
+from typing import Optional
+
+try:  # optional imports — used only when running in notebooks
+    from IPython.core.getipython import get_ipython  # type: ignore
+except Exception:  # pragma: no cover - optional
+
+    def get_ipython():  # type: ignore
+        return None
+
+
+try:  # optional — only available when inside a kernel
+    from ipykernel.connect import get_connection_file as _get_kernel_connection
+except Exception:  # pragma: no cover - optional
+    _get_kernel_connection = None  # type: ignore
+
+try:  # optional — only available when jupyter-server is installed
+    from jupyter_server.serverapp import list_running_servers as _list_running_servers  # type: ignore
+except Exception:  # pragma: no cover - optional
+    _list_running_servers = None  # type: ignore
+
+try:  # optional — HTTP call to Jupyter server API
+    import requests  # type: ignore
+    from requests import RequestException  # type: ignore
+except Exception:  # pragma: no cover - optional
+    requests = None  # type: ignore
+    RequestException = Exception  # type: ignore
+
+
+def _norm(p: str) -> str:
+    """Normalize a path string using pathlib.
+
+    - Expands ~
+    - Resolves to absolute path
+    - Returns a string path
+    Note: We no longer apply os.path.normcase explicitly; pathlib's resolve
+    provides a consistent absolute path. This should be sufficient for our
+    use-cases across platforms.
+    """
+    try:
+        return str(Path(p).expanduser().resolve())
+    except Exception:
+        # As a very last resort, return the original string
+        return p
+
+
+# Treat certain pseudo filenames from IPython/Jupyter as user code so they
+# can be selected as the calling location when appropriate (e.g., in REPL).
+def _is_pseudo_user_filename(filename: str) -> bool:
+    """Return True if filename looks like an IPython/Jupyter pseudo file.
+
+    Examples that should return True:
+    - "<ipython-input-7-abcdef>"
+    - "<jupyter-input-3-123456>"
+    - "<ipykernel_12345>"
+
+    Other pseudo files like "<stdin>" or "<string>" should return False here
+    so they can be treated by the generic pseudo-file handling below.
+    """
+    if not (filename.startswith("<") and filename.endswith(">")):
+        return False
+    lower = filename.lower()
+    return lower.startswith("<ipython-input-") or lower.startswith("<jupyter-input-") or lower.startswith("<ipykernel_")
+
+
+# Names that frequently represent "system/tooling" frames rather than user code
+_SYSTEM_MODULE_PREFIXES = (
+    # pytest + plugin stack
+    "pytest",
+    "_pytest",
+    "pluggy",
+    # IPython/Jupyter stack
+    "IPython",
+    "traitlets",
+    "tornado",
+    "jupyter_client",
+    "jupyter_core",
+    "ipykernel",
+    # IDE/debugger stack (PyCharm)
+    "pydevd",
+    "_pydevd_bundle",
+    "_pydev_bundle",
+    # Python internals
+    "importlib",
+    "runpy",
+    "inspect",
+    "traceback",
+    "contextlib",
+    "asyncio",
+    "threading",
+    # DerivaML CLI runners - skip to find user's model code
+    "deriva_ml.run_model",
+    "deriva_ml.run_notebook",
+    # Hydra/hydra-zen internals
+    "hydra",
+    "hydra_zen",
+    "omegaconf",
+)
+
+
+# --- Helpers focused on determining the current "python model" (file) ---
+
+
+def _top_user_frame() -> Optional[FrameType]:
+    """Return the outermost (top-level) non-tooling frame from the current stack.
+
+    This function traverses the call stack from the current execution point
+    back to the entry point, filtering out known tooling (pytest, IDE helpers,
+    Jupyter internals) and returns the highest-level frame that belongs to
+    user code.
+    """
+    tooling_prefixes = _SYSTEM_MODULE_PREFIXES
+    tooling_filename_parts = (
+        "pydevconsole.py",  # PyCharm REPL console
+        "/pydev/",  # PyCharm helpers path segment
+        "/_pydevd_bundle/",
+        "/_pydev_bundle/",
+        "_pytest",
+        "/pycharm/",
+        # DerivaML CLI entry points - skip to find user's model code
+        "/deriva_ml/run_model.py",
+        "/deriva_ml/run_notebook.py",
+        # Hydra/hydra-zen internals
+        "/hydra/",
+        "/hydra_zen/",
+        "/omegaconf/",
+    )
+
+    f = inspect.currentframe()
+    last_user_frame = None
+
+    if f is not None:
+        f = f.f_back  # Skip the _top_user_frame itself
+
+    while f is not None:
+        filename = f.f_code.co_filename or ""
+        mod_name = f.f_globals.get("__name__", "") or ""
+
+        # 1. Treat IPython cell as user code
+        if _is_pseudo_user_filename(filename):
+            last_user_frame = f
+            f = f.f_back
+            continue
+
+        # 2. Skip other pseudo files like <stdin>, <string>, etc., unless __main__
+        if filename.startswith("<") and filename.endswith(">") and mod_name not in ("__main__", "__mp_main__"):
+            f = f.f_back
+            continue
+
+        # 3. Skip known tooling frames by module prefix
+        if any(mod_name == p or mod_name.startswith(p + ".") for p in tooling_prefixes):
+            f = f.f_back
+            continue
+
+        # 4. Skip known tooling frames by filename patterns
+        if any(part in filename for part in tooling_filename_parts):
+            f = f.f_back
+            continue
+
+        # 5. Skip frames that belong to this helper module (find_caller.py)
+        try:
+            cur = str(Path(filename).resolve())
+            this = str(Path(__file__).resolve())
+            if cur == this:
+                f = f.f_back
+                continue
+        except Exception:
+            pass
+
+        # If it passed all filters, it is a user frame.
+        # We record it and keep going back to find an even "higher" one.
+        last_user_frame = f
+        f = f.f_back
+
+    return last_user_frame
+
+
+def _get_notebook_path() -> Optional[str]:
+    """Best‑effort to obtain the current Jupyter notebook path.
+
+    Returns absolute path string if discoverable, else None.
+    """
+    ip = get_ipython()
+    if ip is None:
+        return None
+
+    # Must be running inside a kernel with a connection file
+    if _get_kernel_connection is None:
+        return None
+    try:
+        connection_file = Path(_get_kernel_connection()).name  # type: ignore[operator]
+    except Exception:
+        return None
+
+    # Need jupyter-server and requests to query sessions
+    if _list_running_servers is None or requests is None:
+        return None
+
+    # Extract kernel ID from connection filename.
+    # Standard Jupyter format: "kernel-<kernel_id>.json"
+    # PyCharm/other formats may vary: "<kernel_id>.json" or other patterns
+    kernel_id = None
+    if connection_file.startswith("kernel-") and "-" in connection_file:
+        # Standard format: kernel-<uuid>.json
+        parts = connection_file.split("-", 1)
+        if len(parts) > 1:
+            kernel_id = parts[1].rsplit(".", 1)[0]
+    else:
+        # Fallback: assume filename (without extension) is the kernel ID
+        kernel_id = connection_file.rsplit(".", 1)[0]
+
+    if not kernel_id:
+        return None
+
+    try:
+        servers = list(_list_running_servers())  # type: ignore[func-returns-value]
+    except Exception:
+        return None
+
+    for server in servers:
+        try:
+            token = server.get("token", "")
+            headers = {"Authorization": f"token {token}"} if token else {}
+            url = server["url"] + "api/sessions"
+            resp = requests.get(url, headers=headers, timeout=3)  # type: ignore[attr-defined]
+            resp.raise_for_status()
+            for sess in resp.json():
+                if sess.get("kernel", {}).get("id") == kernel_id:
+                    rel = sess.get("notebook", {}).get("path")
+                    if rel:
+                        root_dir = server.get("root_dir") or server.get("notebook_dir")
+                        if root_dir:
+                            return str(Path(root_dir) / rel)
+        except RequestException:
+            continue
+        except Exception:
+            continue
+    return None
+
+
+def _get_calling_module() -> str:
+    """Return the relevant source filename for the current execution context.
+
+    Behavior:
+    1) In Jupyter Notebook/Hub: returns the .ipynb file path.
+    2) In a script: returns the script filename.
+    3) In pytest or any REPL (PyCharm or regular): returns the filename that
+       contains the function currently executing (nearest user frame).
+    4) If executing code from an installed package in a venv, still returns that
+       package module file (we do NOT exclude site-packages).
+    """
+    # 1) Jupyter notebook
+    nb = _get_notebook_path()
+    if nb:
+        return str(Path(nb))
+
+    # 2) If running as a script (python myscript.py), prefer __main__.__file__ or argv[0]
+    def _is_tooling_script_path(p: str) -> bool:
+        # Normalize path to forward slashes and lowercase for robust substring checks
+        pn = p.replace("\\", "/").casefold()
+        # Detect common IDE/console helper scripts and CLI runners
+        tooling_markers = (
+            "pydevconsole.py",
+            "/pydev/",
+            "/_pydevd_bundle/",
+            "/_pydev_bundle/",
+            # DerivaML CLI entry points - skip to find user's model code
+            "/deriva_ml/run_model.py",
+            "/deriva_ml/run_notebook.py",
+        )
+        return any(m in pn for m in tooling_markers)
+
+    f = _top_user_frame()
+    if f is not None:
+        return _norm(f.f_code.co_filename)
+    main_mod = sys.modules.get("__main__")
+    main_file = getattr(main_mod, "__file__", None)
+
+    if isinstance(main_file, str) and main_file:
+        if not _is_tooling_script_path(main_file):
+            return _norm(main_file)
+    if sys.argv and sys.argv[0] and sys.argv[0] != "-c":
+        if not _is_tooling_script_path(sys.argv[0]):
+            return _norm(sys.argv[0])
+
+    # 3) Pytest/REPL/IDE: use nearest user frame
+    f = _top_user_frame()
+
+    if f is not None:
+        return _norm(f.f_code.co_filename)
+
+    # Fallback: <stdin> or current working directory marker
+    return str(Path.cwd() / "REPL")

deriva_ml/execution/model_protocol.py

@@ -0,0 +1,175 @@
+"""
+DerivaML Model Protocol
+=======================
+
+This module defines the protocol (interface) that model functions must follow
+to work with DerivaML's execution framework and the run_model() function.
+
+The DerivaMLModel protocol specifies that models must accept two special
+keyword arguments that are injected at runtime:
+
+- ml_instance: The DerivaML (or subclass) instance for catalog operations
+- execution: The Execution context for managing inputs, outputs, and provenance
+
+All other parameters are model-specific and configured via Hydra.
+
+Example
+-------
+A compliant model function:
+
+    def train_classifier(
+        # Model-specific parameters (configured via Hydra)
+        epochs: int = 10,
+        learning_rate: float = 0.001,
+        batch_size: int = 32,
+        # Runtime parameters (injected by run_model)
+        ml_instance: DerivaML = None,
+        execution: Execution = None,
+    ) -> None:
+        '''Train a classifier within the DerivaML execution context.'''
+
+        # Download input datasets
+        for dataset_spec in execution.datasets:
+            bag = execution.download_dataset_bag(dataset_spec)
+            images = load_images_from_bag(bag)
+
+        # Train the model
+        model = MyClassifier()
+        for epoch in range(epochs):
+            train_one_epoch(model, images, learning_rate, batch_size)
+
+        # Save outputs (will be uploaded to catalog)
+        model_path = execution.asset_file_path("Model", "model.pt")
+        torch.save(model.state_dict(), model_path)
+
+        metrics_path = execution.asset_file_path("Execution_Metadata", "metrics.json")
+        with open(metrics_path, "w") as f:
+            json.dump({"final_accuracy": 0.95}, f)
+
+Registering with Hydra-Zen
+--------------------------
+Wrap your model with builds() and zen_partial=True:
+
+    from hydra_zen import builds, store
+
+    TrainClassifierConfig = builds(
+        train_classifier,
+        epochs=10,
+        learning_rate=0.001,
+        batch_size=32,
+        zen_partial=True,  # Creates a partial function
+    )
+
+    # Register in the model_config group
+    store(TrainClassifierConfig, group="model_config", name="default_model")
+
+    # Create variants with different defaults
+    store(TrainClassifierConfig, epochs=50, group="model_config", name="extended")
+    store(TrainClassifierConfig, epochs=5, group="model_config", name="quick")
+
+Type Checking
+-------------
+Use the DerivaMLModel protocol for type hints in utilities:
+
+    from deriva_ml.execution.model_protocol import DerivaMLModel
+
+    def validate_model(model: DerivaMLModel) -> bool:
+        '''Check if a callable conforms to the model protocol.'''
+        return isinstance(model, DerivaMLModel)
+
+The protocol uses @runtime_checkable, so isinstance() checks work at runtime.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, Any, runtime_checkable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from deriva_ml import DerivaML
+    from deriva_ml.execution.execution import Execution
+
+
+@runtime_checkable
+class DerivaMLModel(Protocol):
+    """Protocol for model functions compatible with DerivaML's run_model().
+
+    A model function must accept keyword arguments `ml_instance` and `execution`
+    that are injected at runtime by run_model(). All other parameters are
+    configured via Hydra and passed through the model_config.
+
+    The model function is responsible for:
+    1. Downloading input datasets via execution.download_dataset_bag()
+    2. Performing the ML computation (training, inference, etc.)
+    3. Registering output files via execution.asset_file_path()
+
+    Output files registered with asset_file_path() are automatically uploaded
+    to the catalog after the model completes.
+
+    Attributes
+    ----------
+    This protocol defines a callable signature, not attributes.
+
+    Examples
+    --------
+    Basic model function:
+
+        def my_model(
+            epochs: int = 10,
+            ml_instance: DerivaML = None,
+            execution: Execution = None,
+        ) -> None:
+            # Training logic here
+            pass
+
+    With domain-specific DerivaML subclass:
+
+        def eyeai_model(
+            threshold: float = 0.5,
+            ml_instance: EyeAI = None,  # EyeAI is a DerivaML subclass
+            execution: Execution = None,
+        ) -> None:
+            # Can use EyeAI-specific methods
+            ml_instance.some_eyeai_method()
+
+    Checking protocol compliance:
+
+        >>> from deriva_ml.execution.model_protocol import DerivaMLModel
+        >>> isinstance(my_model, DerivaMLModel)
+        True
+    """
+
+    def __call__(
+        self,
+        *args: Any,
+        ml_instance: "DerivaML",
+        execution: "Execution",
+        **kwargs: Any,
+    ) -> None:
+        """Execute the model within a DerivaML execution context.
+
+        Parameters
+        ----------
+        *args : Any
+            Positional arguments (typically not used; prefer keyword args).
+        ml_instance : DerivaML
+            The DerivaML instance (or subclass like EyeAI) connected to the
+            catalog. Use this for catalog operations not available through
+            the execution context.
+        execution : Execution
+            The execution context manager. Provides:
+            - execution.datasets: List of input DatasetSpec objects
+            - execution.download_dataset_bag(): Download dataset as BDBag
+            - execution.asset_file_path(): Register output file for upload
+            - execution.working_dir: Path to local working directory
+        **kwargs : Any
+            Model-specific parameters configured via Hydra.
+
+        Returns
+        -------
+        None
+            Models should not return values. Results are captured through:
+            - Files registered with asset_file_path() (uploaded to catalog)
+            - Datasets created with execution.create_dataset()
+            - Status updates via execution.update_status()
+        """
+        ...

deriva_ml/execution/multirun_config.py

@@ -0,0 +1,153 @@
+"""Multirun configuration for DerivaML experiments.
+
+This module provides a way to define named multirun configurations that bundle
+together Hydra overrides and a description. This allows you to document complex
+experiment sweeps in code rather than on the command line.
+
+Usage:
+    # In configs/multiruns.py
+    from deriva_ml.execution import multirun_config
+
+    multirun_config(
+        "quick_vs_extended",
+        overrides=[
+            "+experiment=cifar10_quick,cifar10_extended",
+        ],
+        description="## Quick vs Extended Comparison\\n\\nComparing training configs...",
+    )
+
+    multirun_config(
+        "lr_sweep",
+        overrides=[
+            "+experiment=cifar10_lr_sweep",
+            "model_config.learning_rate=0.0001,0.001,0.01,0.1",
+        ],
+        description="## Learning Rate Sweep\\n\\nExploring optimal learning rates...",
+    )
+
+Then run with:
+    deriva-ml-run +multirun=quick_vs_extended
+    deriva-ml-run +multirun=lr_sweep model_config.epochs=5  # Can still override
+
+Benefits:
+    - Explicit declaration of multirun experiments
+    - Rich markdown descriptions for parent executions
+    - Reproducible sweeps documented in code
+    - Same Hydra override syntax as command line
+"""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class MultirunSpec:
+    """Specification for a multirun experiment.
+
+    Attributes:
+        name: Unique identifier for this multirun configuration.
+        overrides: List of Hydra override strings (same syntax as command line).
+            Examples:
+            - "+experiment=cifar10_quick,cifar10_extended"
+            - "model_config.learning_rate=0.0001,0.001,0.01"
+            - "model_config.epochs=5,10,25,50"
+        description: Rich description for the parent execution. Supports full
+            markdown formatting (headers, tables, bold, etc.).
+    """
+    name: str
+    overrides: list[str] = field(default_factory=list)
+    description: str = ""
+
+
+# Global registry of multirun configurations
+_multirun_registry: dict[str, MultirunSpec] = {}
+
+
+def multirun_config(
+    name: str,
+    overrides: list[str],
+    description: str = "",
+) -> MultirunSpec:
+    """Register a named multirun configuration.
+
+    This function registers a multirun specification that can be invoked with
+    `deriva-ml-run +multirun=<name>`. The overrides use the same syntax as
+    Hydra command-line overrides.
+
+    Args:
+        name: Unique name for this multirun configuration. Used to invoke it
+            via `+multirun=<name>`.
+        overrides: List of Hydra override strings. These are the same overrides
+            you would pass on the command line after `--multirun`. Examples:
+            - "+experiment=cifar10_quick,cifar10_extended" - run multiple experiments
+            - "model_config.learning_rate=0.0001,0.001,0.01" - sweep a parameter
+            - "datasets=small,medium,large" - sweep datasets
+        description: Rich description for the parent execution. This supports
+            full markdown formatting since it's defined in Python, not on the
+            command line. Use this to document:
+            - What experiments are being compared and why
+            - Expected outcomes
+            - Methodology and metrics to analyze
+
+    Returns:
+        The registered MultirunSpec instance.
+
+    Example:
+        >>> from deriva_ml.execution import multirun_config
+        >>>
+        >>> multirun_config(
+        ...     "lr_sweep",
+        ...     overrides=[
+        ...         "+experiment=cifar10_lr_sweep",
+        ...         "model_config.learning_rate=0.0001,0.001,0.01,0.1",
+        ...     ],
+        ...     description='''## Learning Rate Sweep
+        ...
+        ... **Objective:** Find optimal learning rate for CIFAR-10 CNN.
+        ...
+        ... | Learning Rate | Expected Behavior |
+        ... |--------------|-------------------|
+        ... | 0.0001 | Slow convergence |
+        ... | 0.001 | Standard baseline |
+        ... | 0.01 | Fast, may overshoot |
+        ... | 0.1 | Likely unstable |
+        ... ''',
+        ... )
+    """
+    spec = MultirunSpec(
+        name=name,
+        overrides=overrides,
+        description=description,
+    )
+    _multirun_registry[name] = spec
+    return spec
+
+
+def get_multirun_config(name: str) -> MultirunSpec | None:
+    """Look up a registered multirun configuration by name.
+
+    Args:
+        name: The name of the multirun configuration.
+
+    Returns:
+        The MultirunSpec if found, None otherwise.
+    """
+    return _multirun_registry.get(name)
+
+
+def list_multirun_configs() -> list[str]:
+    """List all registered multirun configuration names.
+
+    Returns:
+        List of registered multirun config names.
+    """
+    return list(_multirun_registry.keys())
+
+
+def get_all_multirun_configs() -> dict[str, MultirunSpec]:
+    """Get all registered multirun configurations.
+
+    Returns:
+        Dictionary mapping names to MultirunSpec instances.
+    """
+    return dict(_multirun_registry)