bioimageflow-core 0.1.0__tar.gz

bioimageflow_core-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+bif_data/
+data/
+.pytest_cache/
+.claude/
+.venv/
+__pycache__/
+IMPLEMENTATION_PLAN.md
+docs/build/

bioimageflow_core-0.1.0/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: bioimageflow-core
+Version: 0.1.0
+Summary: Core types and tool base classes for BioImageFlow — zero dependencies
+Requires-Python: >=3.10

bioimageflow_core-0.1.0/bioimageflow_core/__init__.py

@@ -0,0 +1,14 @@
+from bioimageflow_core.types import (
+    GUIMeta,
+    ImagePath,
+    ImageShared,
+    ImageSpec,
+    Layout,
+    Semantic,
+    SharedArray,
+    check_compatibility,
+    extract_gui_meta,
+)
+from bioimageflow_core.environment import EnvironmentSpec, EnvironmentMismatchError, ResourceSpec
+from bioimageflow_core.tool import BaseTool, IOModel, ProcessingTool
+from bioimageflow_core.arguments import Arguments

bioimageflow_core-0.1.0/bioimageflow_core/arguments.py

@@ -0,0 +1,36 @@
+"""Arguments namespace and index lineage helpers."""
+
+from difflib import get_close_matches as _get_close_matches
+from typing import Any
+
+
+class Arguments:
+    """
+    Lightweight namespace for passing resolved values to tool methods.
+    Supports attribute access with helpful error messages on typos.
+    """
+    def __init__(self, **kwargs: Any) -> None:
+        self.__dict__.update(kwargs)
+
+    def __getattr__(self, name: str) -> Any:
+        if name.startswith('_'):
+            raise AttributeError(name)
+        available = [k for k in self.__dict__ if not k.startswith('_')]
+        close = _get_close_matches(name, available, n=3, cutoff=0.6)
+        msg = f"Arguments has no field '{name}'."
+        if close:
+            msg += f" Did you mean: {', '.join(close)}?"
+        else:
+            msg += f" Available fields: {', '.join(sorted(available))}"
+        raise AttributeError(msg)
+
+
+def parse_index_lineage(index: str) -> list[str]:
+    """Split an exploded index into its lineage components."""
+    return index.split("::")
+
+
+def parent_index(index: str) -> str:
+    """Return the parent index (strip last explosion level)."""
+    parts = index.split("::")
+    return "::".join(parts[:-1]) if len(parts) > 1 else index

bioimageflow_core-0.1.0/bioimageflow_core/environment.py

@@ -0,0 +1,26 @@
+"""Environment and resource specifications."""
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class EnvironmentSpec:
+    """Defines a reusable Wetlands environment specification."""
+    name: str
+    dependencies: dict[str, str | list[str]]
+
+
+@dataclass(frozen=True)
+class ResourceSpec:
+    """Resource requirements for a processing tool."""
+    cpu: int = 1
+    gpu: int = 0
+    gpu_memory: str | None = None
+    max_concurrent: int = 0
+    memory: str | None = None
+
+
+class EnvironmentMismatchError(Exception):
+    """Raised when two EnvironmentSpecs share a name but differ in dependencies."""
+    pass

bioimageflow_core-0.1.0/bioimageflow_core/io.py

@@ -0,0 +1,33 @@
+"""I/O dispatch — zero declared dependencies. Uses numpy at runtime."""
+
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from multiprocessing.shared_memory import SharedMemory
+from pathlib import Path
+from typing import Any
+
+from bioimageflow_core.types import SharedArray
+
+
+@contextmanager
+def load_image(source: Any, *, file_reader: Callable[[Path], Any]) -> Generator[Any, None, None]:
+    """
+    Dispatch between file and shared memory sources.
+    - SharedArray: attaches to shared memory, yields numpy view.
+    - Path or str: delegates to file_reader, yields result.
+    """
+    if isinstance(source, SharedArray):
+        import numpy as np
+        shm = SharedMemory(name=source.name)
+        try:
+            arr = np.ndarray(source.shape, dtype=source.dtype, buffer=shm.buf)
+            yield arr
+        finally:
+            shm.close()
+    else:
+        yield file_reader(Path(source))
+
+
+def save_image(destination: str | Path, data: Any, *, file_writer: Callable[[Path, Any], None]) -> None:
+    """Save image data to disk using the provided writer."""
+    file_writer(Path(destination), data)

bioimageflow_core-0.1.0/bioimageflow_core/shm.py

@@ -0,0 +1,48 @@
+"""Shared memory helpers — zero declared dependencies. Uses numpy at runtime."""
+
+import uuid
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import Any
+
+from bioimageflow_core.types import SharedArray
+
+
+@contextmanager
+def create_shared_output(data: Any, name: str | None = None) -> Generator[SharedArray, None, None]:
+    """
+    Create a shared memory segment, copy data into it, and yield a SharedArray.
+    Closes the local handle on exit but does NOT unlink (data persists).
+    """
+    import numpy as np
+    from multiprocessing.shared_memory import SharedMemory
+
+    arr = np.asarray(data)
+    if name is None:
+        name = f"bif_{uuid.uuid4().hex[:16]}"
+
+    shm = SharedMemory(name=name, create=True, size=arr.nbytes)
+    try:
+        shared_arr = np.ndarray(arr.shape, dtype=arr.dtype, buffer=shm.buf)
+        shared_arr[:] = arr[:]
+        ref = SharedArray(name=shm.name, shape=arr.shape, dtype=str(arr.dtype))
+        yield ref
+    finally:
+        shm.close()
+
+
+@contextmanager
+def open_shared_array(ref: SharedArray) -> Generator[Any, None, None]:
+    """
+    Attach to an existing shared memory segment.
+    Yields a zero-copy numpy array. Closes handle on exit.
+    """
+    import numpy as np
+    from multiprocessing.shared_memory import SharedMemory
+
+    shm = SharedMemory(name=ref.name)
+    try:
+        arr = np.ndarray(ref.shape, dtype=ref.dtype, buffer=shm.buf)
+        yield arr
+    finally:
+        shm.close()

bioimageflow_core-0.1.0/bioimageflow_core/tool.py

@@ -0,0 +1,91 @@
+"""Tool base classes — zero external dependencies."""
+
+from typing import Any, ClassVar
+
+
+class IOModel:
+    """Lightweight declarative base for tool Inputs/Outputs."""
+
+    @classmethod
+    def _get_all_annotations(cls) -> dict[str, Any]:
+        """Walk the MRO to collect annotations from all ancestor classes."""
+        annotations: dict[str, Any] = {}
+        for klass in reversed(cls.__mro__):
+            annotations.update(getattr(klass, '__annotations__', {}))
+        return annotations
+
+    def __init__(self, **kwargs: Any) -> None:
+        all_annotations = self._get_all_annotations()
+        unknown = set(kwargs) - set(all_annotations)
+        if unknown:
+            raise TypeError(f"Unknown fields: {unknown}")
+        for name in all_annotations:
+            if name in kwargs:
+                setattr(self, name, kwargs[name])
+            elif hasattr(self.__class__, name):
+                setattr(self, name, getattr(self.__class__, name))
+            else:
+                raise TypeError(f"Missing required field: '{name}'")
+
+    def __repr__(self) -> str:
+        fields = {k: getattr(self, k) for k in self._get_all_annotations()}
+        return f"{self.__class__.__name__}({fields})"
+
+
+class BaseTool:
+    """
+    Common base for all tools. Provides identity and Inputs.
+    __call__ is NOT defined here — each subclass defines its own.
+    """
+    name: ClassVar[str]
+    documentation: ClassVar[str] = ""
+    tags: ClassVar[list[str]] = []
+    Inputs: ClassVar[type[IOModel]] = IOModel
+    Outputs: ClassVar[type[IOModel] | None] = None
+
+    def __init__(self) -> None:
+        pass
+
+
+class ProcessingTool(BaseTool):
+    """Tool that processes data in an isolated Wetlands environment."""
+    environment: ClassVar[Any]
+    Outputs: ClassVar[type[IOModel] | None]
+    resources: ClassVar[Any] = None
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        # Only validate leaf concrete classes that define BOTH name and Outputs on themselves
+        has_own_name = 'name' in cls.__dict__ and isinstance(cls.__dict__['name'], str)
+        has_own_outputs = 'Outputs' in cls.__dict__
+        if not has_own_name or not has_own_outputs:
+            return
+        # Check that at least one of process_row or process_batch is overridden
+        has_process_row = cls.process_row is not ProcessingTool.process_row
+        has_process_batch = cls.process_batch is not ProcessingTool.process_batch
+        if not has_process_row and not has_process_batch:
+            raise TypeError(
+                f"{cls.__name__} must implement process_row or process_batch"
+            )
+
+    def __call__(self, *, name: str | None = None, **kwargs: Any) -> Any:
+        """Create a graph node. No computation occurs."""
+        try:
+            from bioimageflow.node import Node
+        except ImportError:
+            raise RuntimeError(
+                f"{type(self).__name__}.__call__() requires the bioimageflow "
+                f"orchestrator package. This method is not available in worker "
+                f"environments — use process_row/process_batch instead."
+            )
+        return Node(tool=self, kwargs=kwargs, name=name)
+
+    def process_row(self, arguments: Any) -> Any:
+        """Process a single row. Override in subclasses."""
+        raise NotImplementedError(
+            f"{type(self).__name__} must implement process_row or process_batch."
+        )
+
+    def process_batch(self, arguments_list: list[Any]) -> Any:
+        """Process all rows at once. Override for batch processing."""
+        raise NotImplementedError

bioimageflow_core-0.1.0/bioimageflow_core/types.py

@@ -0,0 +1,142 @@
+"""BioImageFlow type system — zero external dependencies."""
+
+import warnings
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, Set, Tuple, get_args, get_origin
+
+
+class Semantic(str, Enum):
+    """What the pixel values represent."""
+    BINARY = "binary"
+    LABEL = "label"
+    INTENSITY = "intensity"
+    PROBABILITY = "probability"
+    DISPLACEMENT = "displacement"
+    FEATURE = "feature"
+
+
+class Layout(str, Enum):
+    """Axis ordering of the image data."""
+    PLANAR = "YX"
+    PLANAR_CHANNEL = "CYX"
+    PLANAR_TIME = "TYX"
+    PLANAR_TIME_CHANNEL = "TCYX"
+    VOLUMETRIC = "ZYX"
+    VOLUMETRIC_CHANNEL = "CZYX"
+    VOLUMETRIC_TIME = "TZYX"
+    VOLUMETRIC_TIME_CHANNEL = "TCZYX"
+
+    @property
+    def ndim(self) -> int:
+        return len(self.value)
+
+
+@dataclass(frozen=True)
+class ImageSpec:
+    """Defines type constraints. Empty sets mean 'any' (wildcard)."""
+    semantics: Set[Semantic] = field(default_factory=set)
+    layouts: Set[Layout] = field(default_factory=set)
+    dtypes: Set[str] = field(default_factory=set)
+    formats: Set[str] = field(default_factory=set)
+
+
+@dataclass(frozen=True)
+class SharedArray:
+    """A reference to data in shared memory. Picklable."""
+    name: str
+    shape: Tuple[int, ...]
+    dtype: str
+
+
+def _normalize_param(value: Any) -> set[Any]:
+    """Convert single value or None to a set."""
+    if value is None:
+        return set()
+    if isinstance(value, set):
+        return value
+    if isinstance(value, (list, tuple, frozenset)):
+        return set(value)
+    return {value}
+
+
+def ImagePath(
+    semantics: Any = None,
+    layouts: Any = None,
+    dtypes: Any = None,
+    formats: Any = None,
+) -> Any:
+    """Returns Annotated[Path, ImageSpec(...)]. Used for file-based image data."""
+    spec = ImageSpec(
+        semantics=_normalize_param(semantics),
+        layouts=_normalize_param(layouts),
+        dtypes=_normalize_param(dtypes),
+        formats=_normalize_param(formats),
+    )
+    return Annotated[Path, spec]
+
+
+def ImageShared(
+    semantics: Any = None,
+    layouts: Any = None,
+    dtypes: Any = None,
+) -> Any:
+    """Returns Annotated[SharedArray, ImageSpec(...)]. Formats is implicitly {'memory'}."""
+    spec = ImageSpec(
+        semantics=_normalize_param(semantics),
+        layouts=_normalize_param(layouts),
+        dtypes=_normalize_param(dtypes),
+        formats={"memory"},
+    )
+    return Annotated[SharedArray, spec]
+
+
+@dataclass(frozen=True)
+class GUIMeta:
+    """Declarative GUI hints for a tool input field.
+
+    Attach to an ``Inputs`` annotation via ``Annotated`` to control how a
+    GUI renders the field.
+
+    Parameters
+    ----------
+    connectable : bool
+        Whether this input can be bound to an upstream column.  Defaults to
+        ``True``.  Set to ``False`` for pure user-parameters (e.g. a
+        threshold slider).
+    min : float | None
+        Minimum allowed value (numeric fields only).
+    max : float | None
+        Maximum allowed value (numeric fields only).
+    step : float | None
+        Step increment for spinbox / slider widgets (numeric fields only).
+    """
+    connectable: bool = True
+    min: float | None = None
+    max: float | None = None
+    step: float | None = None
+
+
+def extract_gui_meta(annotation: Any) -> GUIMeta | None:
+    """Extract :class:`GUIMeta` from an ``Annotated`` type, or return ``None``."""
+    if get_origin(annotation) is Annotated:
+        for arg in get_args(annotation):
+            if isinstance(arg, GUIMeta):
+                return arg
+    return None
+
+
+def check_compatibility(producer_spec: ImageSpec, consumer_spec: ImageSpec) -> bool:
+    """Returns True if the producer's output is acceptable for the consumer's input."""
+    for attr in ["semantics", "layouts", "dtypes", "formats"]:
+        producer_values: set[Any] = getattr(producer_spec, attr)
+        consumer_values: set[Any] = getattr(consumer_spec, attr)
+        if not consumer_values:
+            continue
+        if not producer_values:
+            warnings.warn(f"Producer does not declare '{attr}'; cannot verify.")
+            continue
+        if not producer_values.intersection(consumer_values):
+            return False
+    return True

bioimageflow_core-0.1.0/bioimageflow_core/worker.py

@@ -0,0 +1,90 @@
+"""Worker-side dispatcher for Wetlands environments.
+
+This module is imported by the orchestrator via env.import_module()
+inside isolated Conda environments. It discovers tool classes in a
+given module and dispatches process_row/process_batch calls.
+
+All functions accept and return only picklable types (dicts, lists,
+strings, numbers) to cross the Wetlands serialization boundary.
+"""
+
+import importlib
+import inspect
+from pathlib import Path
+
+from bioimageflow_core.arguments import Arguments
+from bioimageflow_core.tool import BaseTool
+
+
+# Per-module registries: module_path -> {class_name -> class}
+_tool_registries: dict[str, dict[str, type]] = {}
+# Per-module instances: module_path -> {class_name -> instance}
+_instances: dict[str, dict[str, BaseTool]] = {}
+
+
+def _discover_tools(module: object) -> dict[str, type]:
+    """Build a name->class registry from all BaseTool subclasses in the module."""
+    registry: dict[str, type] = {}
+    for name, obj in inspect.getmembers(module, inspect.isclass):
+        if issubclass(obj, BaseTool) and obj is not BaseTool and hasattr(obj, 'name'):
+            registry[obj.__name__] = obj
+    return registry
+
+
+def _get_instance(module_path: str, tool_class_name: str) -> BaseTool:
+    """Get or create a cached tool instance for the given module and class."""
+    if module_path not in _tool_registries:
+        mod = importlib.import_module(module_path)
+        _tool_registries[module_path] = _discover_tools(mod)
+        _instances[module_path] = {}
+    registry = _tool_registries[module_path]
+    instances = _instances[module_path]
+    if tool_class_name not in instances:
+        if tool_class_name not in registry:
+            raise ValueError(
+                f"Tool class '{tool_class_name}' not found in module '{module_path}'. "
+                f"Available: {list(registry.keys())}"
+            )
+        instances[tool_class_name] = registry[tool_class_name]()
+    return instances[tool_class_name]
+
+
+def _outputs_to_dict(outputs: object) -> dict:
+    """Convert an Outputs instance to a plain dict with picklable values."""
+    if hasattr(outputs, '_get_all_annotations'):
+        d: dict = {}
+        for k in outputs._get_all_annotations():
+            v = getattr(outputs, k)
+            if isinstance(v, Path):
+                v = str(v)
+            d[k] = v
+        return d
+    return {k: str(v) if isinstance(v, Path) else v for k, v in vars(outputs).items()}
+
+
+def run_process_row(module_path: str, tool_class_name: str, arguments_dict: dict) -> list[dict]:
+    """Dispatch a single-row call to a tool's process_row method.
+
+    Returns a list of output dicts (one per output row, usually one).
+    """
+    tool = _get_instance(module_path, tool_class_name)
+    args = Arguments(**arguments_dict)
+    result = tool.process_row(args)
+    outputs = result if isinstance(result, list) else [result]
+    return [_outputs_to_dict(out) for out in outputs]
+
+
+def run_process_batch(
+    module_path: str, tool_class_name: str, arguments_dicts: list[dict],
+) -> list[list[dict]]:
+    """Dispatch a batch call to a tool's process_batch method.
+
+    Returns a list of lists of output dicts (one inner list per input row).
+    """
+    tool = _get_instance(module_path, tool_class_name)
+    args_list = [Arguments(**d) for d in arguments_dicts]
+    results = tool.process_batch(args_list)
+    # Auto-wrap list[Outputs] -> list[list[Outputs]] for 1-to-1 batch tools
+    if results and not isinstance(results[0], list):
+        results = [[r] for r in results]
+    return [[_outputs_to_dict(out) for out in row_outputs] for row_outputs in results]

bioimageflow_core-0.1.0/pyproject.toml

@@ -0,0 +1,10 @@
+[project]
+name = "bioimageflow-core"
+version = "0.1.0"
+description = "Core types and tool base classes for BioImageFlow — zero dependencies"
+requires-python = ">=3.10"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"