taskferry 0.2.0__py3-none-any.whl

taskferry/contract/job.py

@@ -0,0 +1,91 @@
+"""The contract every :class:`~taskferry.ports.JobBackend` must satisfy."""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+
+import pytest
+
+from ..capabilities import Capability
+from ..errors import UnsupportedCapability
+from ..execution import ExecutionKind
+from ..ports import ExecutionBackend
+from ..specs import JobSpec, Resources
+from .base import ExecutionBackendContract
+
+
+class JobBackendContract(ExecutionBackendContract):
+    """Subclass in an adapter's test module and implement the two hooks.
+
+    ::
+
+        class TestCloudRun(JobBackendContract):
+            def make_backend(self):
+                return CloudRunJobBackend(project="p", location="eu", jobs_client=FakeJobs())
+
+            def success_spec(self):
+                return JobSpec(job="build-cog", image="gdal:latest")
+    """
+
+    @abstractmethod
+    def make_backend(self) -> ExecutionBackend: ...
+
+    @abstractmethod
+    def success_spec(self) -> JobSpec: ...
+
+    def test_is_a_job_backend(self) -> None:
+        assert self.make_backend().kind is ExecutionKind.JOB
+
+    def test_gpu_capability_matches_gpu_requests(self) -> None:
+        """The assertion that matters most: never run a GPU job on the CPU quietly."""
+        backend = self.make_backend()
+        spec = self.success_spec().evolve(resources=Resources(gpu=1, gpu_type="nvidia-tesla-t4"))
+        if Capability.GPU in backend.capabilities:
+            assert backend.submit(spec).id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(spec)
+
+    def test_cpu_and_memory_capabilities_match_resource_requests(self) -> None:
+        backend = self.make_backend()
+        caps = backend.capabilities
+        spec = self.success_spec().evolve(resources=Resources(cpu="1000m", memory="512Mi"))
+        if Capability.CPU in caps and Capability.MEMORY in caps:
+            assert backend.submit(spec).id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(spec)
+
+    def test_parallelism_capability_matches_array_jobs(self) -> None:
+        backend = self.make_backend()
+        spec = self.success_spec().evolve(parallelism=4)
+        if Capability.PARALLELISM in backend.capabilities:
+            assert backend.submit(spec).id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(spec)
+
+    def test_timeout_capability_matches_timeout_requests(self) -> None:
+        from ..retry import TimeoutPolicy
+
+        backend = self.make_backend()
+        spec = self.success_spec().evolve(timeout=TimeoutPolicy(seconds=60))
+        if Capability.TIMEOUT in backend.capabilities:
+            assert backend.submit(spec).id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(spec)
+
+    def test_exit_code_is_reported_when_results_are_supported(self) -> None:
+        if not self.reaches_terminal_state:
+            pytest.skip("backend does not execute work in this test environment")
+        backend = self.make_backend()
+        if Capability.RESULT not in backend.capabilities:
+            pytest.skip("backend does not report results")
+        execution = backend.submit(self.success_spec())
+        final = self.wait_for_terminal(backend, execution)
+        assert final.result is not None
+        assert final.result.exit_code == 0, "a successful job exits zero"
+
+
+__all__ = ["JobBackendContract"]

taskferry/contract/task.py

@@ -0,0 +1,91 @@
+"""The contract every :class:`~taskferry.ports.TaskBackend` must satisfy."""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from datetime import timedelta
+
+import pytest
+
+from ..capabilities import Capability
+from ..errors import UnsupportedCapability
+from ..execution import ExecutionKind
+from ..ports import ExecutionBackend
+from ..specs import TaskSpec
+from .base import ExecutionBackendContract
+
+
+class TaskBackendContract(ExecutionBackendContract):
+    """Subclass in an adapter's test module and implement the two hooks.
+
+    ::
+
+        class TestProcrastinate(TaskBackendContract):
+            def make_backend(self):
+                return ProcrastinateTaskBackend(app=FakeApp())
+
+            def success_spec(self):
+                return TaskSpec(task="tests.tasks:ok", args=(1, 2))
+    """
+
+    @abstractmethod
+    def make_backend(self) -> ExecutionBackend: ...
+
+    @abstractmethod
+    def success_spec(self) -> TaskSpec: ...
+
+    def test_is_a_task_backend(self) -> None:
+        assert self.make_backend().kind is ExecutionKind.TASK
+
+    def test_delay_capability_matches_deferred_specs(self) -> None:
+        """A backend must either honour ``delay`` or refuse it outright."""
+        backend = self.make_backend()
+        deferred = self.success_spec().evolve(delay=timedelta(seconds=30))
+        if Capability.DELAY in backend.capabilities:
+            execution = backend.submit(deferred)
+            assert execution.id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(deferred)
+
+    def test_priority_capability_matches_prioritised_specs(self) -> None:
+        backend = self.make_backend()
+        prioritised = self.success_spec().evolve(priority=5)
+        if Capability.PRIORITY in backend.capabilities:
+            assert backend.submit(prioritised).id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(prioritised)
+
+    def test_deduplication_capability_matches_idempotency_keys(self) -> None:
+        """``idempotency_key`` is a request, and a backend that cannot honour it says so."""
+        backend = self.make_backend()
+        keyed = self.success_spec().evolve(idempotency_key="contract-key-1")
+        if Capability.DEDUPLICATION in backend.capabilities:
+            assert backend.submit(keyed).id
+        else:
+            with pytest.raises(UnsupportedCapability):
+                backend.submit(keyed)
+
+    def test_backend_options_for_other_backends_are_ignored(self) -> None:
+        """One spec must travel unchanged across engines.
+
+        A spec carrying options for a *different* backend has to submit cleanly —
+        otherwise moving a queue from Procrastinate to Cloud Tasks would mean
+        editing every call site, which is the coupling Taskferry exists to remove.
+        """
+        from ..specs import BackendOptions
+
+        backend = self.make_backend()
+        spec = self.success_spec().evolve(
+            backend_options=BackendOptions({"some-other-engine": {"nonsense": True}})
+        )
+        assert backend.submit(spec).id
+
+    def test_arguments_survive_the_spec_unchanged(self) -> None:
+        spec = self.success_spec()
+        assert spec.args == tuple(spec.args)
+        assert dict(spec.kwargs) == dict(spec.kwargs)
+
+
+__all__ = ["TaskBackendContract"]

taskferry/core/__init__.py

@@ -0,0 +1,130 @@
+"""``taskferry.core`` — the tiny shared substrate for the Taskferry family.
+
+This package deliberately stays small (ADR-0004). It contains only what is
+*genuinely* transversal across Tasks, Jobs, Events and Schedules: identifiers,
+correlation, provider metadata, the capability model, configuration primitives,
+serialization, observability hooks, the lazy registry, delivery vocabulary and
+the root error hierarchy.
+
+Domain types (``JobSpec``, ``Event``, ``Schedule`` ...) live in their own
+packages, not here — centralization is not a reason to promote a type.
+
+End users normally do not depend on ``taskferry-core`` directly; they install
+``taskferry-django`` / ``taskferry-jobs`` / ``taskferry-events`` /
+``taskferry-scheduler``, which depend on it.
+"""
+
+from __future__ import annotations
+
+from .capabilities import Capability, CapabilitySet
+from .config import (
+    ProviderOptions,
+    env_bool,
+    env_int,
+    env_str,
+    require,
+    resolve_factory,
+)
+from .correlation import (
+    Correlation,
+    current_correlation,
+    ensure_correlation,
+    use_correlation,
+)
+from .delivery import DeliveryGuarantee, Ordering
+from .errors import (
+    ConfigurationError,
+    ProviderError,
+    SerializationError,
+    TaskferryError,
+    UnsupportedCapabilityError,
+)
+from .ids import TaskferryId, is_taskferry_id, new_id
+from .observability import (
+    ATTR_ATTEMPT,
+    ATTR_CORRELATION_ID,
+    ATTR_EVENT_ID,
+    ATTR_JOB_ID,
+    ATTR_PROVIDER,
+    ATTR_PROVIDER_ID,
+    ATTR_SCHEDULE_ID,
+    ATTR_TASK_ID,
+    SPAN_EVENT_CONSUME,
+    SPAN_EVENT_PUBLISH,
+    SPAN_JOB_POLL,
+    SPAN_JOB_SUBMIT,
+    SPAN_SCHEDULE_CREATE,
+    SPAN_TASK_ENQUEUE,
+    SPAN_TASK_EXECUTE,
+    NoopTracer,
+    Span,
+    Tracer,
+    get_tracer,
+    set_tracer,
+    span,
+)
+from .otel import OTelTracer, configure_opentelemetry
+from .provider import ProviderMetadata
+from .registry import LazyRegistry
+from .serialization import JsonSerializer, Serializer, ensure_json_serializable
+from .typing import JSONArray, JSONObject, JSONScalar, JSONValue
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ATTR_ATTEMPT",
+    "ATTR_CORRELATION_ID",
+    "ATTR_EVENT_ID",
+    "ATTR_JOB_ID",
+    "ATTR_PROVIDER",
+    "ATTR_PROVIDER_ID",
+    "ATTR_SCHEDULE_ID",
+    "ATTR_TASK_ID",
+    "SPAN_EVENT_CONSUME",
+    "SPAN_EVENT_PUBLISH",
+    "SPAN_JOB_POLL",
+    "SPAN_JOB_SUBMIT",
+    "SPAN_SCHEDULE_CREATE",
+    "SPAN_TASK_ENQUEUE",
+    "SPAN_TASK_EXECUTE",
+    "Capability",
+    "CapabilitySet",
+    "ConfigurationError",
+    "Correlation",
+    "DeliveryGuarantee",
+    "JSONArray",
+    "JSONObject",
+    "JSONScalar",
+    "JSONValue",
+    "JsonSerializer",
+    "LazyRegistry",
+    "NoopTracer",
+    "OTelTracer",
+    "Ordering",
+    "ProviderError",
+    "ProviderMetadata",
+    "ProviderOptions",
+    "SerializationError",
+    "Serializer",
+    "Span",
+    "TaskferryError",
+    "TaskferryId",
+    "Tracer",
+    "UnsupportedCapabilityError",
+    "__version__",
+    "configure_opentelemetry",
+    "current_correlation",
+    "ensure_correlation",
+    "ensure_json_serializable",
+    "env_bool",
+    "env_int",
+    "env_str",
+    "get_tracer",
+    "is_taskferry_id",
+    "new_id",
+    "require",
+    "resolve_factory",
+    "set_tracer",
+    "span",
+    "use_correlation",
+]

taskferry/core/capabilities.py

@@ -0,0 +1,89 @@
+"""The capability model shared by every Taskferry domain (ADR-0005).
+
+Providers do not support the same things. Taskferry refuses to lie about this.
+Each domain defines its own capability enum (a :class:`Capability` subclass) and
+every provider exposes an immutable :class:`CapabilitySet`. Callers can:
+
+* **feature-detect** — ``if cap in provider.capabilities: ...``
+* **assert** — ``provider.capabilities.require(cap)`` raises
+  :class:`~taskferry.core.errors.UnsupportedCapabilityError` when absent.
+
+A capability that does not exist is never ignored and never faked.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator
+from enum import StrEnum
+
+from .errors import UnsupportedCapabilityError
+
+
+class Capability(StrEnum):
+    """Base class for all domain capability enums.
+
+    Subclassed per domain (``TaskCapability``, ``JobCapability``, ...). Using a
+    ``StrEnum`` keeps capabilities serializable and comparable to plain strings,
+    which matters for logging, wire formats, and cross-package checks.
+    """
+
+
+class CapabilitySet:
+    """An immutable, hashable set of capabilities advertised by a provider."""
+
+    __slots__ = ("_caps", "_provider")
+
+    def __init__(self, capabilities: Iterable[Capability], *, provider: str | None = None) -> None:
+        self._caps: frozenset[Capability] = frozenset(capabilities)
+        self._provider = provider
+
+    @property
+    def provider(self) -> str | None:
+        return self._provider
+
+    def __contains__(self, capability: object) -> bool:
+        return capability in self._caps
+
+    def __iter__(self) -> Iterator[Capability]:
+        return iter(self._caps)
+
+    def __len__(self) -> int:
+        return len(self._caps)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, CapabilitySet):
+            return self._caps == other._caps
+        if isinstance(other, (set, frozenset)):
+            return self._caps == frozenset(other)
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self._caps)
+
+    def __repr__(self) -> str:
+        names = ", ".join(sorted(c.value for c in self._caps))
+        provider = f" provider={self._provider!r}" if self._provider else ""
+        return f"CapabilitySet({{{names}}}{provider})"
+
+    def supports(self, capability: Capability) -> bool:
+        """Return whether ``capability`` is advertised."""
+        return capability in self._caps
+
+    def supports_all(self, capabilities: Iterable[Capability]) -> bool:
+        return frozenset(capabilities) <= self._caps
+
+    def require(self, capability: Capability) -> None:
+        """Raise :class:`UnsupportedCapabilityError` unless ``capability`` is present."""
+        if capability not in self._caps:
+            raise UnsupportedCapabilityError(str(capability), provider=self._provider)
+
+    def require_all(self, capabilities: Iterable[Capability]) -> None:
+        for capability in capabilities:
+            self.require(capability)
+
+    def missing(self, capabilities: Iterable[Capability]) -> frozenset[Capability]:
+        """Return the subset of ``capabilities`` that is *not* supported."""
+        return frozenset(capabilities) - self._caps
+
+
+__all__ = ["Capability", "CapabilitySet"]

taskferry/core/config.py

@@ -0,0 +1,167 @@
+"""Configuration primitives (sections 14-17, ADR-0007).
+
+Principle: platform config / secrets / env flow into *application* settings,
+which are then handed to Taskferry. Taskferry libraries must **not** reach into
+``os.environ`` arbitrarily from deep inside adapters.
+
+These helpers are therefore:
+
+* **Opt-in** — an app may configure Taskferry from Django settings, plain Python,
+  a dict, ``.env``, Vault, Secret Manager, etc. Env vars are one option, never
+  the only one.
+* **Injectable** — env readers take the environment mapping as an argument
+  (defaulting to ``os.environ``) so they are testable and never hidden.
+* **Convention-friendly** — they read standard names (``REDIS_URL``,
+  ``DATABASE_URL``, ``GOOGLE_CLOUD_PROJECT`` ...) rather than forcing a
+  ``TASKFERRY_`` prefix on variables the ecosystem already standardizes.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+
+from .errors import ConfigurationError
+from .typing import JSONObject
+
+_MISSING = object()
+
+# Truthy/falsey spellings accepted by env_bool, matching common 12-factor usage.
+_TRUE = frozenset({"1", "true", "t", "yes", "y", "on"})
+_FALSE = frozenset({"0", "false", "f", "no", "n", "off", ""})
+
+
+def env_str(
+    key: str,
+    default: str | None = None,
+    *,
+    required: bool = False,
+    environ: Mapping[str, str] | None = None,
+) -> str | None:
+    """Read a string from the environment mapping.
+
+    Raises :class:`ConfigurationError` when ``required`` and absent, so missing
+    configuration fails fast at startup (section 15).
+    """
+    source = environ if environ is not None else os.environ
+    value = source.get(key)
+    if value is None or value == "":
+        if required:
+            raise ConfigurationError(f"required environment variable {key!r} is not set")
+        return default
+    return value
+
+
+def env_int(
+    key: str,
+    default: int | None = None,
+    *,
+    required: bool = False,
+    environ: Mapping[str, str] | None = None,
+) -> int | None:
+    raw = env_str(key, None, required=required, environ=environ)
+    if raw is None:
+        return default
+    try:
+        return int(raw)
+    except ValueError as exc:
+        raise ConfigurationError(f"{key!r} must be an integer, got {raw!r}") from exc
+
+
+def env_bool(
+    key: str,
+    default: bool = False,
+    *,
+    environ: Mapping[str, str] | None = None,
+) -> bool:
+    raw = env_str(key, None, environ=environ)
+    if raw is None:
+        return default
+    lowered = raw.strip().lower()
+    if lowered in _TRUE:
+        return True
+    if lowered in _FALSE:
+        return False
+    raise ConfigurationError(f"{key!r} must be a boolean-like value, got {raw!r}")
+
+
+def require[T](value: T | None, name: str) -> T:
+    """Return ``value`` or raise a clear :class:`ConfigurationError` if ``None``."""
+    if value is None:
+        raise ConfigurationError(f"{name} is required but was not provided")
+    return value
+
+
+@dataclass(frozen=True, slots=True)
+class ProviderOptions:
+    """Provider-specific escape hatch (section 17).
+
+    Portable config lives on the common models; anything genuinely
+    provider-specific goes here, namespaced per provider::
+
+        ProviderOptions({"gcp": {"http_target": {...}}, "aws": {"MessageGroupId": "x"}})
+
+    Adapters read only their own namespace and ignore the rest, so one
+    configuration object travels unchanged across providers.
+    """
+
+    by_provider: Mapping[str, JSONObject] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        frozen = {k: MappingProxyType(dict(v)) for k, v in self.by_provider.items()}
+        object.__setattr__(self, "by_provider", MappingProxyType(frozen))
+
+    def for_provider(self, provider: str) -> JSONObject:
+        """Return this provider's options (an empty mapping if none supplied)."""
+        return dict(self.by_provider.get(provider, {}))
+
+    def option(
+        self,
+        provider: str,
+        key: str,
+        default: object = _MISSING,
+    ) -> object:
+        """Read a single option, raising if absent and no default is given."""
+        opts = self.by_provider.get(provider, {})
+        if key in opts:
+            return opts[key]
+        if default is _MISSING:
+            raise ConfigurationError(
+                f"provider option {provider!r}.{key!r} is required but missing"
+            )
+        return default
+
+
+def resolve_factory(spec: str) -> Callable[..., object]:
+    """Resolve a ``"module.path:attr"`` (or ``"module.path.attr"``) string to a callable.
+
+    Used by registries (ADR and section 35) to build providers lazily from
+    configuration without importing every adapter eagerly.
+    """
+    import importlib
+
+    module_path, sep, attr = spec.partition(":")
+    if not sep:
+        module_path, _, attr = spec.rpartition(".")
+    if not module_path or not attr:
+        raise ConfigurationError(f"invalid factory spec {spec!r}")
+    try:
+        module = importlib.import_module(module_path)
+        target = getattr(module, attr)
+    except (ImportError, AttributeError) as exc:
+        raise ConfigurationError(f"could not import factory {spec!r}: {exc}") from exc
+    if not callable(target):
+        raise ConfigurationError(f"factory {spec!r} is not callable")
+    return target  # type: ignore[no-any-return]
+
+
+__all__ = [
+    "ProviderOptions",
+    "env_bool",
+    "env_int",
+    "env_str",
+    "require",
+    "resolve_factory",
+]

taskferry/core/correlation.py

@@ -0,0 +1,120 @@
+"""Correlation metadata for following a logical flow across boundaries.
+
+Lets you determine that an HTTP request → Task → Event → Job all belong to the
+same logical flow (section 19) **without** turning Taskferry into a workflow
+engine. It is just metadata that adapters propagate.
+
+The ``trace_context`` mapping carries W3C Trace Context (``traceparent`` /
+``tracestate``) so distributed tracing works even without the OpenTelemetry SDK
+installed (ADR-0012). Taskferry only moves the strings around.
+"""
+
+from __future__ import annotations
+
+import contextvars
+from collections.abc import Mapping
+from dataclasses import dataclass, field, replace
+from types import MappingProxyType
+
+from .ids import TaskferryId, new_id
+
+
+@dataclass(frozen=True, slots=True)
+class Correlation:
+    """Immutable correlation envelope propagated with every operation.
+
+    Attributes:
+        correlation_id: Stable id shared by every step of one logical flow.
+        causation_id: Id of the *immediate* cause (the step that triggered this
+            one). ``None`` at the root of a flow.
+        trace_context: W3C Trace Context carrier (``traceparent``/``tracestate``).
+    """
+
+    correlation_id: TaskferryId
+    causation_id: TaskferryId | None = None
+    trace_context: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "trace_context", MappingProxyType(dict(self.trace_context)))
+
+    @classmethod
+    def start(cls) -> Correlation:
+        """Begin a fresh flow with a new correlation id and no cause."""
+        return cls(correlation_id=new_id("corr"))
+
+    def caused(self, cause_id: TaskferryId) -> Correlation:
+        """Return a child correlation for a step caused by ``cause_id``.
+
+        The correlation id is preserved (same flow); the causation id advances.
+        """
+        return replace(self, causation_id=cause_id)
+
+    def to_headers(self) -> dict[str, str]:
+        """Serialize to portable transport headers/attributes."""
+        headers = {
+            "taskferry-correlation-id": self.correlation_id,
+            **dict(self.trace_context),
+        }
+        if self.causation_id is not None:
+            headers["taskferry-causation-id"] = self.causation_id
+        return headers
+
+    @classmethod
+    def from_headers(cls, headers: Mapping[str, str]) -> Correlation:
+        """Reconstruct correlation from transport headers/attributes.
+
+        Falls back to starting a fresh flow when no correlation id is present.
+        """
+        correlation_id = headers.get("taskferry-correlation-id")
+        if not correlation_id:
+            return cls.start()
+        trace_context = {k: v for k, v in headers.items() if k in ("traceparent", "tracestate")}
+        causation = headers.get("taskferry-causation-id")
+        return cls(
+            correlation_id=TaskferryId(correlation_id),
+            causation_id=TaskferryId(causation) if causation else None,
+            trace_context=trace_context,
+        )
+
+
+_current: contextvars.ContextVar[Correlation | None] = contextvars.ContextVar(
+    "taskferry_correlation", default=None
+)
+
+
+def current_correlation() -> Correlation | None:
+    """Return the correlation bound to the current context, if any."""
+    return _current.get()
+
+
+def ensure_correlation() -> Correlation:
+    """Return the current correlation, starting a fresh flow if none is bound."""
+    existing = _current.get()
+    return existing if existing is not None else Correlation.start()
+
+
+class use_correlation:
+    """Context manager binding ``correlation`` to the current execution context.
+
+    Restores the previous value on exit, so it nests cleanly.
+    """
+
+    def __init__(self, correlation: Correlation) -> None:
+        self._correlation = correlation
+        self._token: contextvars.Token[Correlation | None] | None = None
+
+    def __enter__(self) -> Correlation:
+        self._token = _current.set(self._correlation)
+        return self._correlation
+
+    def __exit__(self, *exc: object) -> None:
+        assert self._token is not None
+        _current.reset(self._token)
+
+
+__all__ = [
+    "Correlation",
+    "current_correlation",
+    "ensure_correlation",
+    "use_correlation",
+]