taskferry 0.2.0__py3-none-any.whl

taskferry/core/delivery.py

@@ -0,0 +1,36 @@
+"""Delivery-semantics vocabulary (sections 21-22, ADR-0010).
+
+Taskferry **never** promises exactly-once. Distributed systems produce
+duplicates; the whole family assumes at-least-once with possible duplicates and
+promotes idempotency. These enums let adapters *declare* their real semantics so
+callers can reason about them instead of guessing.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class DeliveryGuarantee(StrEnum):
+    """How many times a message may be delivered."""
+
+    AT_MOST_ONCE = "at_most_once"
+    """Delivered zero or one time. Losses possible, duplicates impossible."""
+
+    AT_LEAST_ONCE = "at_least_once"
+    """Delivered one or more times. Losses impossible, duplicates possible."""
+
+    # Note: EXACTLY_ONCE is intentionally absent. See ADR-0010.
+
+
+class Ordering(StrEnum):
+    """Ordering guarantee across a stream/queue/topic."""
+
+    UNORDERED = "unordered"
+    PER_KEY = "per_key"
+    """Ordered within a partition/group key (e.g. FIFO message group)."""
+
+    TOTAL = "total"
+
+
+__all__ = ["DeliveryGuarantee", "Ordering"]

taskferry/core/errors.py

@@ -0,0 +1,55 @@
+"""The Taskferry error hierarchy root.
+
+Adapter distributions define their own leaf errors that subclass
+:class:`TaskferryError`, so a caller can catch the whole family or one adapter.
+The execution-layer hierarchy built on this root lives in :mod:`taskferry.errors`.
+"""
+
+from __future__ import annotations
+
+
+class TaskferryError(Exception):
+    """Base class for every error raised anywhere in the Taskferry family."""
+
+
+class ConfigurationError(TaskferryError):
+    """Raised when configuration is missing, malformed, or contradictory."""
+
+
+class ProviderError(TaskferryError):
+    """Wraps an error originating from an underlying provider/SDK.
+
+    Adapters should raise this (chaining the original with ``from``) so callers
+    can depend on a stable Taskferry type instead of provider-specific exceptions.
+    """
+
+    def __init__(self, message: str, *, provider: str | None = None) -> None:
+        super().__init__(message)
+        self.provider = provider
+
+
+class UnsupportedCapabilityError(TaskferryError):
+    """Raised when an operation requires a capability the provider lacks.
+
+    Taskferry never silently simulates a missing capability (ADR-0005). Adapters
+    fail loudly with this error, naming the capability and provider.
+    """
+
+    def __init__(self, capability: str, *, provider: str | None = None) -> None:
+        provider_suffix = f" (provider={provider!r})" if provider else ""
+        super().__init__(f"Unsupported capability: {capability!r}{provider_suffix}")
+        self.capability = capability
+        self.provider = provider
+
+
+class SerializationError(TaskferryError):
+    """Raised when a payload cannot be serialized to / from the wire format."""
+
+
+__all__ = [
+    "ConfigurationError",
+    "ProviderError",
+    "SerializationError",
+    "TaskferryError",
+    "UnsupportedCapabilityError",
+]

taskferry/core/ids.py

@@ -0,0 +1,37 @@
+"""Portable Taskferry identifiers.
+
+A Taskferry id is always Taskferry-owned and never *is* the provider id. Adapters
+keep the provider id alongside it in :class:`~taskferry.core.provider.ProviderMetadata`
+so a flow can be followed across systems (sections 20, 48).
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import NewType
+
+# A distinct type so a Taskferry id is never accidentally confused with a provider
+# id or an arbitrary string in signatures.
+TaskferryId = NewType("TaskferryId", str)
+
+_ID_PREFIXES = frozenset({"tp", "task", "job", "evt", "sch", "run", "corr"})
+
+
+def new_id(prefix: str = "tp") -> TaskferryId:
+    """Return a fresh, URL-safe, sortable-enough Taskferry id.
+
+    The prefix is advisory and only aids human debugging (e.g. ``job_9f2c...``).
+    Uses uuid4 for collision resistance without coordination.
+    """
+    if not prefix or not prefix.isidentifier():
+        raise ValueError(f"id prefix must be a valid identifier, got {prefix!r}")
+    return TaskferryId(f"{prefix}_{uuid.uuid4().hex}")
+
+
+def is_taskferry_id(value: str) -> bool:
+    """Best-effort check that ``value`` looks like a Taskferry id."""
+    prefix, _, rest = value.partition("_")
+    return bool(rest) and prefix.isidentifier()
+
+
+__all__ = ["TaskferryId", "is_taskferry_id", "new_id"]

taskferry/core/observability.py

@@ -0,0 +1,136 @@
+"""Observability hooks — designed in from v1, mandatory on no one (ADR-0012).
+
+Taskferry propagates trace context and correlation and emits spans at consistent
+points, but it does **not** depend on the OpenTelemetry SDK. Out of the box a
+:class:`NoopTracer` is used; installing the ``otel`` extra and calling
+:func:`set_tracer` with an OTel-backed tracer lights everything up.
+
+Span names and attribute keys are centralized here so every package
+(``taskferry.task.enqueue``, ``taskferry.job.submit`` ...) stays coherent
+(section 47).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from types import TracebackType
+from typing import Protocol, runtime_checkable
+
+# --- Semantic span names (section 47) -------------------------------------- #
+SPAN_TASK_ENQUEUE = "taskferry.task.enqueue"
+SPAN_TASK_EXECUTE = "taskferry.task.execute"
+SPAN_JOB_SUBMIT = "taskferry.job.submit"
+SPAN_JOB_POLL = "taskferry.job.poll"
+SPAN_EVENT_PUBLISH = "taskferry.event.publish"
+SPAN_EVENT_CONSUME = "taskferry.event.consume"
+SPAN_SCHEDULE_CREATE = "taskferry.schedule.create"
+
+# --- Semantic attribute keys (section 18) ---------------------------------- #
+ATTR_PROVIDER = "taskferry.provider"
+ATTR_TASK_ID = "taskferry.task_id"
+ATTR_JOB_ID = "taskferry.job_id"
+ATTR_EVENT_ID = "taskferry.event_id"
+ATTR_SCHEDULE_ID = "taskferry.schedule_id"
+ATTR_CORRELATION_ID = "taskferry.correlation_id"
+ATTR_ATTEMPT = "taskferry.attempt"
+ATTR_PROVIDER_ID = "taskferry.provider_id"
+
+AttributeValue = str | int | float | bool
+Attributes = Mapping[str, AttributeValue]
+
+
+@runtime_checkable
+class Span(Protocol):
+    """Minimal span surface adapters use inside a ``with`` block."""
+
+    def set_attribute(self, key: str, value: AttributeValue) -> None: ...
+
+    def record_exception(self, exc: BaseException) -> None: ...
+
+    def __enter__(self) -> Span: ...
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None: ...
+
+
+@runtime_checkable
+class Tracer(Protocol):
+    """A factory of spans. The OTel bridge and the no-op both satisfy this."""
+
+    def start_span(self, name: str, attributes: Attributes | None = None) -> Span: ...
+
+
+class _NoopSpan:
+    __slots__ = ()
+
+    def set_attribute(self, key: str, value: AttributeValue) -> None:
+        return None
+
+    def record_exception(self, exc: BaseException) -> None:
+        return None
+
+    def __enter__(self) -> _NoopSpan:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        return None
+
+
+class NoopTracer:
+    """Default tracer: correct, cheap, does nothing."""
+
+    def start_span(self, name: str, attributes: Attributes | None = None) -> Span:
+        return _NoopSpan()
+
+
+_tracer: Tracer = NoopTracer()
+
+
+def set_tracer(tracer: Tracer) -> None:
+    """Install the process-wide tracer (e.g. an OpenTelemetry-backed one)."""
+    global _tracer
+    _tracer = tracer
+
+
+def get_tracer() -> Tracer:
+    """Return the process-wide tracer (a :class:`NoopTracer` until set)."""
+    return _tracer
+
+
+@contextmanager
+def span(name: str, attributes: Attributes | None = None) -> Iterator[Span]:
+    """Convenience wrapper: ``with span("taskferry.job.submit", {...}) as s:``."""
+    started = get_tracer().start_span(name, attributes)
+    with started as active:
+        yield active
+
+
+__all__ = [
+    "ATTR_ATTEMPT",
+    "ATTR_CORRELATION_ID",
+    "ATTR_EVENT_ID",
+    "ATTR_JOB_ID",
+    "ATTR_PROVIDER",
+    "ATTR_PROVIDER_ID",
+    "ATTR_SCHEDULE_ID",
+    "ATTR_TASK_ID",
+    "SPAN_EVENT_CONSUME",
+    "SPAN_EVENT_PUBLISH",
+    "SPAN_JOB_POLL",
+    "SPAN_JOB_SUBMIT",
+    "SPAN_SCHEDULE_CREATE",
+    "SPAN_TASK_ENQUEUE",
+    "SPAN_TASK_EXECUTE",
+    "Attributes",
+    "NoopTracer",
+    "Span",
+    "Tracer",
+    "get_tracer",
+    "set_tracer",
+    "span",
+]

taskferry/core/otel.py

@@ -0,0 +1,83 @@
+"""OpenTelemetry bridge (ADR-0012).
+
+Installing the ``otel`` extra and calling :func:`configure_opentelemetry` swaps
+the process-wide :class:`~taskferry.core.observability.NoopTracer` for one backed
+by the OpenTelemetry API, so every ``taskferry.*`` span becomes a real OTel span.
+
+OpenTelemetry is imported lazily here, so ``import taskferry.core`` (which imports
+this module for the public re-exports) never requires the OTel packages — only
+calling :func:`configure_opentelemetry` does.
+"""
+
+from __future__ import annotations
+
+from types import TracebackType
+from typing import Any
+
+from .observability import Attributes, Span, Tracer, set_tracer
+
+
+class _OTelSpan:
+    """Adapts an OpenTelemetry ``start_as_current_span`` context manager to :class:`Span`."""
+
+    __slots__ = ("_cm", "_span")
+
+    def __init__(self, cm: Any) -> None:
+        self._cm = cm
+        self._span: Any = None
+
+    def __enter__(self) -> _OTelSpan:
+        self._span = self._cm.__enter__()
+        return self
+
+    def set_attribute(self, key: str, value: object) -> None:
+        if self._span is not None:
+            self._span.set_attribute(key, value)
+
+    def record_exception(self, exc: BaseException) -> None:
+        if self._span is not None:
+            self._span.record_exception(exc)
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if exc is not None and self._span is not None:
+            self._span.record_exception(exc)
+        self._cm.__exit__(exc_type, exc, tb)
+
+
+class OTelTracer:
+    """A :class:`Tracer` backed by an OpenTelemetry tracer."""
+
+    def __init__(self, otel_tracer: Any) -> None:
+        self._tracer = otel_tracer
+
+    def start_span(self, name: str, attributes: Attributes | None = None) -> Span:
+        cm = self._tracer.start_as_current_span(
+            name, attributes=dict(attributes) if attributes else None
+        )
+        return _OTelSpan(cm)
+
+
+def configure_opentelemetry(instrumenting_module_name: str = "taskferry") -> Tracer:
+    """Install an OpenTelemetry-backed tracer process-wide and return it.
+
+    Requires the ``otel`` extra (``opentelemetry-api``). The application is still
+    responsible for configuring the OTel ``TracerProvider`` and exporters.
+    """
+    try:
+        from opentelemetry import trace
+    except ImportError as exc:  # pragma: no cover - env-specific
+        raise RuntimeError(
+            "OpenTelemetry is required for configure_opentelemetry; "
+            "install taskferry-otel (which pulls in opentelemetry-api)"
+        ) from exc
+    tracer = OTelTracer(trace.get_tracer(instrumenting_module_name))
+    set_tracer(tracer)
+    return tracer
+
+
+__all__ = ["OTelTracer", "configure_opentelemetry"]

taskferry/core/provider.py

@@ -0,0 +1,50 @@
+"""Provider metadata — the escape hatch for provider-specific facts.
+
+Portable contracts stay clean; provider-specific truth (the real provider id,
+region, resource name, arbitrary labels) lives here, attached to handles/results.
+This is how Taskferry avoids "contaminating the common models with hundreds of
+cloud options" (section 17) while still exposing them when needed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field, replace
+from types import MappingProxyType
+
+
+@dataclass(frozen=True, slots=True)
+class ProviderMetadata:
+    """Immutable description of where/how a resource actually lives.
+
+    Attributes:
+        provider: Short provider key, e.g. ``"local"``, ``"gcp"``, ``"aws"``,
+            ``"azure"``, ``"kubernetes"``.
+        provider_id: The provider's own identifier (task name, execution name,
+            message id, schedule name). May be ``None`` before submission.
+        region: Provider region/location when meaningful.
+        resource: Fully-qualified provider resource name when meaningful.
+        labels: Arbitrary provider-specific key/value metadata.
+    """
+
+    provider: str
+    provider_id: str | None = None
+    region: str | None = None
+    resource: str | None = None
+    labels: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        # Freeze the mapping so a frozen dataclass is actually immutable.
+        object.__setattr__(self, "labels", MappingProxyType(dict(self.labels)))
+
+    def with_provider_id(self, provider_id: str) -> ProviderMetadata:
+        """Return a copy carrying the resolved provider id."""
+        return replace(self, provider_id=provider_id)
+
+    def with_labels(self, **labels: str) -> ProviderMetadata:
+        """Return a copy with additional labels merged in."""
+        merged = {**self.labels, **labels}
+        return replace(self, labels=merged)
+
+
+__all__ = ["ProviderMetadata"]

taskferry/core/registry.py

@@ -0,0 +1,92 @@
+"""A tiny lazy registry used by every domain (sections 35, 31).
+
+``runners["default"]``, ``publishers["default"]``, ``schedulers["default"]`` are
+all backed by this. Providers are built **lazily** on first access and cached, so
+importing an adapter package never imports ``boto3``/``google.cloud`` for a
+provider you don't touch.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator, Mapping
+
+from .config import resolve_factory
+from .errors import ConfigurationError
+
+
+class LazyRegistry[T]:
+    """Maps an alias to a zero-arg factory, instantiating on first access.
+
+    ``kind`` is a human label used in error messages (e.g. ``"job runner"``).
+    """
+
+    def __init__(self, kind: str) -> None:
+        self._kind = kind
+        self._factories: dict[str, Callable[[], T]] = {}
+        self._instances: dict[str, T] = {}
+
+    def register(self, alias: str, factory: Callable[[], T], *, replace: bool = False) -> None:
+        """Register a zero-arg ``factory`` under ``alias``."""
+        if alias in self._factories and not replace:
+            raise ConfigurationError(f"{self._kind} alias {alias!r} is already registered")
+        self._factories[alias] = factory
+        self._instances.pop(alias, None)
+
+    def register_spec(self, alias: str, spec: str, /, **kwargs: object) -> None:
+        """Register by import string, resolved lazily on first access.
+
+        ``spec`` is ``"module.path:Factory"``; ``kwargs`` are passed to it.
+        """
+
+        def _factory() -> T:
+            target = resolve_factory(spec)
+            return target(**kwargs)  # type: ignore[return-value]
+
+        self.register(alias, _factory, replace=True)
+
+    def configure(self, config: Mapping[str, Mapping[str, object]]) -> None:
+        """Bulk-register from a settings-style mapping.
+
+        Each entry maps an alias to ``{"factory": "mod:Cls", ...kwargs}``. This is
+        the shape an adapter registry consumes.
+        """
+        for alias, entry in config.items():
+            params = dict(entry)
+            factory_spec = params.pop("factory", None)
+            if not isinstance(factory_spec, str):
+                raise ConfigurationError(
+                    f"{self._kind} {alias!r} config needs a 'factory' import string"
+                )
+            self.register_spec(alias, factory_spec, **params)
+
+    def __getitem__(self, alias: str) -> T:
+        if alias not in self._instances:
+            factory = self._factories.get(alias)
+            if factory is None:
+                known = ", ".join(sorted(self._factories)) or "<none>"
+                raise ConfigurationError(
+                    f"no {self._kind} registered under {alias!r} (known: {known})"
+                )
+            self._instances[alias] = factory()
+        return self._instances[alias]
+
+    def __contains__(self, alias: object) -> bool:
+        return alias in self._factories
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._factories)
+
+    def aliases(self) -> tuple[str, ...]:
+        return tuple(self._factories)
+
+    def clear(self) -> None:
+        """Drop all registrations and cached instances (useful in tests)."""
+        self._factories.clear()
+        self._instances.clear()
+
+    def reset_instances(self) -> None:
+        """Drop cached instances but keep registrations (re-build on next access)."""
+        self._instances.clear()
+
+
+__all__ = ["LazyRegistry"]

taskferry/core/serialization.py

@@ -0,0 +1,79 @@
+"""JSON-only serialization for Taskferry payloads (section 23, ADR-0010).
+
+Rules Taskferry enforces so payloads stay portable and safe:
+
+* JSON is the default and only built-in transport format.
+* **Never** pickle — it is unsafe for cloud/untrusted messages.
+* Do not serialize ORM instances or arbitrary Python objects. Pass identifiers
+  (``process_resource(resource_id)``), not objects.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Protocol, runtime_checkable
+
+from .errors import SerializationError
+from .typing import JSONValue
+
+
+@runtime_checkable
+class Serializer(Protocol):
+    """Minimal serializer contract adapters can depend on."""
+
+    content_type: str
+
+    def dumps(self, value: JSONValue) -> bytes: ...
+
+    def loads(self, data: bytes | str) -> JSONValue: ...
+
+
+class JsonSerializer:
+    """Strict JSON serializer.
+
+    ``dumps`` rejects non-JSON-serializable input eagerly with
+    :class:`SerializationError` instead of producing a partial/garbage payload.
+    """
+
+    content_type = "application/json"
+
+    def __init__(self, *, sort_keys: bool = True) -> None:
+        self._sort_keys = sort_keys
+
+    def dumps(self, value: JSONValue) -> bytes:
+        try:
+            text = json.dumps(
+                value, sort_keys=self._sort_keys, separators=(",", ":"), allow_nan=False
+            )
+        except (TypeError, ValueError) as exc:
+            raise SerializationError(f"value is not JSON-serializable: {exc}") from exc
+        return text.encode("utf-8")
+
+    def loads(self, data: bytes | str) -> JSONValue:
+        try:
+            payload = data.decode("utf-8") if isinstance(data, bytes) else data
+            return json.loads(payload)  # type: ignore[no-any-return]
+        except (ValueError, UnicodeDecodeError) as exc:
+            raise SerializationError(f"payload is not valid JSON: {exc}") from exc
+
+
+def ensure_json_serializable(value: object) -> JSONValue:
+    """Return ``value`` unchanged if it round-trips through JSON, else raise.
+
+    A cheap boundary guard: call it where application data enters Taskferry so
+    failures surface at enqueue time, not on a remote worker.
+    """
+    try:
+        json.dumps(value, allow_nan=False)
+    except (TypeError, ValueError) as exc:
+        raise SerializationError(
+            f"value of type {type(value).__name__!r} is not JSON-serializable: {exc}"
+        ) from exc
+    return value  # type: ignore[return-value]
+
+
+__all__ = [
+    "JsonSerializer",
+    "Serializer",
+    "ensure_json_serializable",
+]

taskferry/core/typing.py

@@ -0,0 +1,16 @@
+"""Typing helpers shared across the Taskferry family.
+
+Kept deliberately tiny. Only genuinely cross-cutting aliases live here.
+"""
+
+from __future__ import annotations
+
+# JSON is the only transport payload Taskferry promises to serialize (ADR-0010,
+# section 23). These PEP 695 aliases document that contract at the type level;
+# lazy evaluation lets the recursive definition reference itself without quotes.
+type JSONScalar = str | int | float | bool | None
+type JSONValue = JSONScalar | JSONArray | JSONObject
+type JSONArray = list[JSONValue]
+type JSONObject = dict[str, JSONValue]
+
+__all__ = ["JSONArray", "JSONObject", "JSONScalar", "JSONValue"]