overture-serverless 0.1.0__py3-none-any.whl

overture_serverless/airflow_compat.py

@@ -0,0 +1,249 @@
+"""Airflow 2 / 3 compatibility shim.
+
+Import Airflow symbols from this module instead of directly from ``airflow.sdk``
+(3.x) or the scattered 2.x paths. Works transparently on both versions.
+
+Usage::
+
+    from overture_serverless.airflow_compat import DAG, task, Variable, chain
+
+Two paths to remove this file:
+
+Upgrade path when the running Airflow ships Airflow 3:
+  1. Delete this file.
+  2. Global find-and-replace: ``from overture_serverless.airflow_compat import``
+     -> ``from airflow.sdk import``
+  3. Done -- one clean commit.
+
+Upgrade path when Airflow v2.11.1+ is the floor:
+  1. Delete this file.
+  2. Global find-and-replace: ``from overture_serverless.airflow_compat import``
+     -> ``from airflow.providers.common.compat.sdk import``
+  3. Done -- one clean commit.
+  4. Optional -- follow upstream and remove the common-compat shim package from
+     your codebase entirely once Airflow 3 is supported.
+
+Implementation mirrors the lazy-__getattr__ approach used by the official
+``airflow.providers.common.compat.sdk`` shim so symbols are only imported on
+first access, keeping DAG parse time fast.
+
+Inspired by upstream:
+https://github.com/apache/airflow/blob/main/providers/common/compat/src/airflow/providers/common/compat/sdk.py
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Rename map: symbols that changed name between Airflow 2.x and 3.x
+# Format: new_name -> (v3_module, v2_module, v2_name)
+# ---------------------------------------------------------------------------
+_RENAME_MAP: dict[str, tuple[str, str, str]] = {
+    "Asset": ("airflow.sdk", "airflow.datasets", "Dataset"),
+    "AssetAlias": ("airflow.sdk", "airflow.datasets", "DatasetAlias"),
+    "AssetAll": ("airflow.sdk", "airflow.datasets", "DatasetAll"),
+    "AssetAny": ("airflow.sdk", "airflow.datasets", "DatasetAny"),
+}
+
+# ---------------------------------------------------------------------------
+# Import map: symbol name -> (v3_module, v2_module)
+# None as v2_module means the symbol is Airflow 3-only (no 2.x equivalent).
+# ---------------------------------------------------------------------------
+_IMPORT_MAP: dict[str, tuple[str, str | None]] = {
+    # -- DAG / decorators -----------------------------------------------------
+    "DAG": ("airflow.sdk", "airflow.models.dag"),
+    "dag": ("airflow.sdk", "airflow.decorators"),
+    "task": ("airflow.sdk", "airflow.decorators"),
+    "task_group": ("airflow.sdk", "airflow.decorators"),
+    "setup": ("airflow.sdk", "airflow.decorators"),
+    "teardown": ("airflow.sdk", "airflow.decorators"),
+    "chain": ("airflow.sdk", "airflow.models.baseoperator"),
+    "chain_linear": ("airflow.sdk", "airflow.models.baseoperator"),
+    "cross_downstream": ("airflow.sdk", "airflow.models.baseoperator"),
+    # -- Models ----------------------------------------------------------------
+    # Variable is handled separately below (see `Variable` class) -- Variable.get()'s
+    # `default_var` kwarg was renamed to `default` in 3.x, so a straight symbol swap
+    # isn't enough; every call site would need touching otherwise.
+    "Connection": ("airflow.sdk", "airflow.models.connection"),
+    "Param": ("airflow.sdk", "airflow.models.param"),
+    "ParamsDict": ("airflow.sdk", "airflow.models.param"),
+    "XComArg": ("airflow.sdk", "airflow.models.xcom_arg"),
+    "XCom": ("airflow.sdk.execution_time.xcom", "airflow.models.xcom"),
+    "MappedOperator": (
+        "airflow.sdk.definitions.mappedoperator",
+        "airflow.models.mappedoperator",
+    ),
+    # -- Operators / base classes ----------------------------------------------
+    "BaseOperator": ("airflow.sdk", "airflow.models.baseoperator"),
+    "BaseOperatorLink": ("airflow.sdk", "airflow.models.baseoperatorlink"),
+    "BaseSensorOperator": ("airflow.sdk", "airflow.sensors.base"),
+    "PokeReturnValue": ("airflow.sdk", "airflow.sensors.base"),
+    "poke_mode_only": ("airflow.sdk.bases.sensor", "airflow.sensors.base"),
+    "BaseHook": ("airflow.sdk", "airflow.hooks.base"),
+    "BaseBranchOperator": (
+        "airflow.sdk.bases.branch",
+        "airflow.providers.standard.operators.branch",
+    ),
+    "BranchMixIn": (
+        "airflow.sdk.bases.branch",
+        "airflow.providers.standard.operators.branch",
+    ),
+    "SkipMixin": ("airflow.sdk.bases.skipmixin", "airflow.models.skipmixin"),
+    # -- Decorator internals ---------------------------------------------------
+    "TaskDecorator": ("airflow.sdk.bases.decorator", "airflow.decorators.base"),
+    "DecoratedOperator": ("airflow.sdk.bases.decorator", "airflow.decorators.base"),
+    "DecoratedMappedOperator": (
+        "airflow.sdk.bases.decorator",
+        "airflow.decorators.base",
+    ),
+    "task_decorator_factory": (
+        "airflow.sdk.bases.decorator",
+        "airflow.decorators.base",
+    ),
+    "get_unique_task_id": ("airflow.sdk.bases.decorator", "airflow.decorators.base"),
+    # -- Groups & modifiers ----------------------------------------------------
+    "TaskGroup": ("airflow.sdk", "airflow.utils.task_group"),
+    "EdgeModifier": ("airflow.sdk", "airflow.utils.edgemodifier"),
+    "Label": ("airflow.sdk", "airflow.utils.edgemodifier"),
+    # -- State / enums ---------------------------------------------------------
+    "TriggerRule": ("airflow.sdk", "airflow.utils.trigger_rule"),
+    "DagRunState": ("airflow.sdk", "airflow.utils.state"),
+    "TaskInstanceState": ("airflow.sdk", "airflow.utils.state"),
+    "WeightRule": ("airflow.sdk", "airflow.utils.weight_rule"),
+    # -- Context & utilities ---------------------------------------------------
+    "Context": ("airflow.sdk", "airflow.utils.context"),
+    "context_merge": ("airflow.sdk.definitions.context", "airflow.utils.context"),
+    "get_current_context": ("airflow.sdk", "airflow.operators.python"),
+    "get_parsing_context": ("airflow.sdk", "airflow.utils.dag_parsing_context"),
+    "TaskInstanceKey": ("airflow.sdk.types", "airflow.models.taskinstancekey"),
+    # -- Assets ----------------------------------------------------------------
+    "Metadata": ("airflow.sdk", "airflow.datasets.metadata"),
+    "ObjectStoragePath": ("airflow.sdk", "airflow.io.path"),
+    # -- Exceptions ------------------------------------------------------------
+    "AirflowException": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "AirflowConfigException": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "AirflowFailException": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "AirflowNotFoundException": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "AirflowOptionalProviderFeatureException": (
+        "airflow.sdk.exceptions",
+        "airflow.exceptions",
+    ),
+    "AirflowSensorTimeout": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "AirflowSkipException": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "AirflowTaskTimeout": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "ParamValidationError": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "TaskDeferred": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    "XComNotFound": ("airflow.sdk.exceptions", "airflow.exceptions"),
+    # Airflow 3-only -- no 2.x equivalent
+    "DagRunTriggerException": ("airflow.sdk.exceptions", None),
+    "DownstreamTasksSkipped": ("airflow.sdk.exceptions", None),
+    # -- Observability ---------------------------------------------------------
+    "Stats": ("airflow.sdk._shared.observability.metrics.stats", "airflow.stats"),
+    # -- Configuration ---------------------------------------------------------
+    "conf": ("airflow.sdk.configuration", "airflow.configuration"),
+    # -- Lineage ---------------------------------------------------------------
+    "HookLineage": ("airflow.sdk.lineage", "airflow.lineage.hook"),
+    "HookLineageCollector": ("airflow.sdk.lineage", "airflow.lineage.hook"),
+    "HookLineageReader": ("airflow.sdk.lineage", "airflow.lineage.hook"),
+    "NoOpCollector": ("airflow.sdk.lineage", "airflow.lineage.hook"),
+    "get_hook_lineage_collector": ("airflow.sdk.lineage", "airflow.lineage.hook"),
+    # -- Plugins ---------------------------------------------------------------
+    "AirflowPlugin": ("airflow.sdk.plugins_manager", "airflow.plugins_manager"),
+    # -- BaseNotifier ----------------------------------------------------------
+    "BaseNotifier": ("airflow.sdk", "airflow.notifications.basenotifier"),
+    # -- Secrets -----------------------------------------------------------------
+    # airflow.sdk.log (not the _shared.secrets_masker it wraps) is the intended
+    # public entry point on 3.x -- it also notifies the supervisor process.
+    "mask_secret": ("airflow.sdk.log", "airflow.utils.log.secrets_masker"),
+}
+
+# ---------------------------------------------------------------------------
+# Module map: symbols that are entire modules (imported with `import X`)
+# Format: name -> (v3_module_path, v2_module_path)
+# ---------------------------------------------------------------------------
+_MODULE_MAP: dict[str, tuple[str, str]] = {
+    "timezone": ("airflow.sdk.timezone", "airflow.utils.timezone"),
+    "io": ("airflow.sdk.io", "airflow.io"),
+}
+
+
+def _load(module_path: str, attr: str) -> Any:
+    mod = importlib.import_module(module_path)
+    return getattr(mod, attr)
+
+
+def _real_variable() -> Any:
+    try:
+        return _load("airflow.sdk", "Variable")
+    except (ImportError, AttributeError):
+        return _load("airflow.models.variable", "Variable")
+
+
+_NOTSET = object()
+
+
+class _VariableMeta(type):
+    """Delegates any attribute not defined on `Variable` (`.set`, `.delete`, ...)
+    straight to the real Variable class for the running Airflow version."""
+
+    def __getattr__(cls, name: str) -> Any:
+        return getattr(_real_variable(), name)
+
+
+class Variable(metaclass=_VariableMeta):
+    """Variable proxy that normalizes `Variable.get()`'s `default_var` (2.x) /
+    `default` (3.x) kwarg rename, so call sites can keep using `default_var=`
+    on both versions. Everything else delegates to the real class via `_VariableMeta`.
+    """
+
+    @staticmethod
+    def get(
+        key: str, default_var: Any = _NOTSET, deserialize_json: bool = False
+    ) -> Any:
+        real = _real_variable()
+        if default_var is _NOTSET:
+            return real.get(key, deserialize_json=deserialize_json)
+        try:
+            return real.get(key, default=default_var, deserialize_json=deserialize_json)
+        except TypeError:
+            return real.get(
+                key, default_var=default_var, deserialize_json=deserialize_json
+            )
+
+
+def __getattr__(name: str) -> Any:
+    # Renamed symbols (e.g. Dataset -> Asset)
+    if name in _RENAME_MAP:
+        v3_mod, v2_mod, v2_name = _RENAME_MAP[name]
+        try:
+            return _load(v3_mod, name)
+        except (ImportError, AttributeError):
+            return _load(v2_mod, v2_name)
+
+    # Whole-module symbols
+    if name in _MODULE_MAP:
+        v3_path, v2_path = _MODULE_MAP[name]
+        for path in (v3_path, v2_path):
+            try:
+                return importlib.import_module(path)
+            except ImportError:
+                continue
+        raise ImportError(f"airflow_compat: cannot import module {name!r}")
+
+    # Regular symbols
+    if name in _IMPORT_MAP:
+        v3_mod, v2_mod = _IMPORT_MAP[name]
+        try:
+            return _load(v3_mod, name)
+        except (ImportError, AttributeError):
+            pass
+        if v2_mod is not None:
+            return _load(v2_mod, name)
+        raise ImportError(f"airflow_compat: {name!r} requires Airflow 3+")
+
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["Variable", *_RENAME_MAP, *_IMPORT_MAP, *_MODULE_MAP]

overture_serverless/backends/__init__.py

@@ -0,0 +1,19 @@
+"""Backend contract implementations for ``serverless_python_task_group``.
+
+A backend is an Airflow-facing function that provisions managed compute,
+injects the runner-contract env vars (``MODULE_NAME``, ``CLASS_NAME``,
+``PYTHON_PACKAGES``, ``PIP_INDEX_URL``, ``PARAMS``), and runs the container.
+:mod:`overture_serverless.backends.fargate` is the AWS ECS Fargate
+implementation shipped today.
+
+Every cloud- or account-specific value (network configuration, IAM role,
+container image URI, ECS task-definition composition) is a parameter the
+caller supplies rather than something a backend resolves for itself, so this
+package stays free of any one deployment's account IDs, VPC layout, or naming
+conventions.
+
+Import from the specific backend module (e.g. ``.fargate``) rather than a
+top-level re-export: this keeps ``overture_serverless``'s base install free of
+the ``apache-airflow`` / ``apache-airflow-providers-amazon`` dependencies that
+only backend users need (see the ``fargate`` extra).
+"""

overture_serverless/backends/fargate.py

@@ -0,0 +1,421 @@
+"""AWS ECS Fargate backend for the ``serverless_python`` framework.
+
+One implementation of the ``serverless_python_task_group`` contract. It
+launches the reference runner container (``overture-python-runner``) on
+Fargate; at run time the container installs the requested Python packages
+from a pip index and dispatches to the caller's job class.
+
+The runner container is cloud-agnostic -- it speaks a simple runner contract
+(``MODULE_NAME``, ``CLASS_NAME``, ``PYTHON_PACKAGES``, ``PIP_INDEX_URL``,
+``PARAMS`` env vars) and takes a single ``PIP_INDEX_URL``. This backend
+assembles that URL from CodeArtifact credentials before passing it in; a
+hypothetical GCP Cloud Run or Azure Container Apps backend would live
+alongside this file and assemble the URL from its own artifact registry.
+
+Every value specific to a deployment's AWS account -- VPC/subnet layout, IAM
+role ARNs, container image URI, and how the underlying ECS task definition is
+registered/run/torn down -- is supplied by the caller rather than resolved
+here, so this module has no dependency on any one deployment's private
+helpers. See ``network_config``, ``task_role_arn``, ``image_uri``, and
+``ecs_task_builder_factory`` on :func:`serverless_python_task_group`.
+
+Task pipeline::
+
+    register_task_definition -> execute_job -> deregister_task_definition
+
+- ``register_task_definition`` / ``deregister_task_definition`` come from the
+  injected ``ecs_task_builder_factory`` and manage the ephemeral Fargate task
+  definition.
+- ``execute_job`` fetches a short-lived CodeArtifact auth token inside the
+  operator's ``execute()`` and injects a fully-formed ``PIP_INDEX_URL`` into
+  ``containerOverrides`` in memory before calling ECS ``RunTask``. The token
+  never lands in XCom, task logs, or the metadata DB.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Literal, Protocol
+
+from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
+
+from overture_serverless.airflow_compat import TaskGroup, chain, mask_secret
+
+_DEFAULT_ECS_CLUSTER = "overture-python-runner"
+_DEFAULT_LOG_GROUP = "/ecs/overture-python-runner"
+
+log = logging.getLogger(__name__)
+
+_CONTAINER_NAME = "runner"
+_CODEARTIFACT_TOKEN_DURATION_SECONDS = (
+    3600  # 1 hour, plenty for Fargate cold start + pip install
+)
+
+# T-shirt-sized presets. All combos are valid Fargate CPU/memory pairings.
+_SIZE_PRESETS: dict[str, tuple[str, str]] = {
+    "xs": ("256", "512"),
+    "s": ("512", "1024"),
+    "m": ("1024", "2048"),
+    "l": ("2048", "4096"),
+}
+
+
+class EcsTaskBuilderLike(Protocol):
+    """Duck-typed contract for the ECS register -> run -> teardown builder.
+
+    Matches ``EcsTaskBuilder`` from Overture's internal ``ecs_helper`` module
+    (and any equivalent a caller wants to substitute) without this package
+    importing that class directly.
+    """
+
+    def register(self, task_id: str = ...) -> Any: ...
+
+    def run(
+        self,
+        overrides: dict,
+        task_id: str = ...,
+        awslogs_stream_prefix: str = ...,
+        retries: int = ...,
+        reattach: bool = ...,
+    ) -> Any: ...
+
+    def teardown(
+        self,
+        container_name: str,
+        run_op: Any | None = ...,
+        task_id: str = ...,
+    ) -> list: ...
+
+
+def _build_fargate_combos() -> set[tuple[str, str]]:
+    """Enumerate every valid Fargate (cpu_units, memory_mib) combination.
+
+    Sourced from
+    https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html - no enum in botocore model
+    Used to fail fast at DAG-parse time when a caller passes an invalid
+    exact override, instead of surfacing an ECS ``InvalidParameterException``
+    at task-run time.
+    """
+    combos: set[tuple[str, str]] = set()
+    combos.update((("256", m) for m in ("512", "1024", "2048")))
+    combos.update((("512", m) for m in ("1024", "2048", "3072", "4096")))
+    combos.update((("1024", str(m)) for m in range(2048, 8193, 1024)))
+    combos.update((("2048", str(m)) for m in range(4096, 16385, 1024)))
+    combos.update((("4096", str(m)) for m in range(8192, 30721, 1024)))
+    combos.update((("8192", str(m)) for m in range(16384, 61441, 4096)))
+    combos.update((("16384", str(m)) for m in range(32768, 122881, 8192)))
+    return combos
+
+
+_FARGATE_VALID_COMBOS = _build_fargate_combos()
+
+
+def _resolve_cpu_memory(
+    size: str | None, cpu: str | None, memory: str | None
+) -> tuple[str, str]:
+    if size is not None and (cpu is not None or memory is not None):
+        raise ValueError("Set either `size` or (`cpu` and `memory`), not both.")
+    if cpu is None and memory is None:
+        preset = size or "xs"
+        if preset not in _SIZE_PRESETS:
+            raise ValueError(
+                f"Unknown size preset: {preset!r}. Valid: {sorted(_SIZE_PRESETS)}. "
+                "If you think this is a valid preset, check "
+                "https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html "
+                "and update _build_fargate_combos() accordingly."
+            )
+        return _SIZE_PRESETS[preset]
+    if cpu is None or memory is None:
+        raise ValueError(
+            "Both `cpu` and `memory` must be set when using an exact override."
+        )
+    if (cpu, memory) not in _FARGATE_VALID_COMBOS:
+        raise ValueError(
+            f"cpu={cpu!r}, memory={memory!r} is not a valid Fargate combo. "
+            "See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html "
+            "and update _build_fargate_combos() accordingly."
+        )
+    return cpu, memory
+
+
+class _CodeArtifactPipIndexUrlEcsOperator(EcsRunTaskOperator):
+    """``EcsRunTaskOperator`` that mints a CodeArtifact-backed
+    ``PIP_INDEX_URL`` inside ``execute()`` and injects it into
+    ``containerOverrides`` in memory just before submitting to ECS.
+
+    The token is registered with Airflow's secrets masker so it is redacted
+    from task logs. It never lands in XCom, the Airflow metadata DB, or any
+    templated field.
+    """
+
+    def __init__(
+        self,
+        *,
+        codeartifact_domain: str,
+        codeartifact_owner: str,
+        codeartifact_repo: str,
+        codeartifact_region: str,
+        target_container_name: str,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self._ca_domain = codeartifact_domain
+        self._ca_owner = codeartifact_owner
+        self._ca_repo = codeartifact_repo
+        self._ca_region = codeartifact_region
+        self._target_container_name = target_container_name
+
+    def execute(self, context):
+        import boto3
+
+        ca = boto3.client("codeartifact", region_name=self._ca_region)
+        token = ca.get_authorization_token(
+            domain=self._ca_domain,
+            domainOwner=self._ca_owner,
+            durationSeconds=_CODEARTIFACT_TOKEN_DURATION_SECONDS,
+        )["authorizationToken"]
+        mask_secret(token)
+        pip_index_url = (
+            f"https://aws:{token}"
+            f"@{self._ca_domain}-{self._ca_owner}"
+            f".d.codeartifact.{self._ca_region}.amazonaws.com"
+            f"/pypi/{self._ca_repo}/simple/"
+        )
+        mask_secret(pip_index_url)
+        containers = self.overrides.get("containerOverrides", [])
+        target = next(
+            (c for c in containers if c.get("name") == self._target_container_name),
+            None,
+        )
+        if target is None:
+            raise ValueError(
+                f"target_container_name={self._target_container_name!r} not found "
+                f"in containerOverrides; cannot inject PIP_INDEX_URL"
+            )
+        # NativeEnvironment can literal_eval a JSON string back to a dict.
+        environment = target.setdefault("environment", [])
+        for env in environment:
+            if not isinstance(env.get("value"), str):
+                env["value"] = json.dumps(env["value"])
+        # Retries re-run execute() on the same operator instance, so drop any
+        # PIP_INDEX_URL this method injected on a prior attempt before adding
+        # the freshly minted one -- otherwise the container sees duplicate,
+        # increasingly stale entries.
+        environment[:] = [e for e in environment if e.get("name") != "PIP_INDEX_URL"]
+        environment.append({"name": "PIP_INDEX_URL", "value": pip_index_url})
+        return super().execute(context)
+
+    def _start_task(self):
+        # Superclass populates self.arn after the RunTask call. Emit a
+        # clickable console URL as soon as the task ID is known so
+        # engineers can jump straight from Airflow logs to the running
+        # job. Label stays platform-neutral ("Job console") so future
+        # backends can supply their own URL template without a rename.
+        result = super()._start_task()
+        if self.arn:
+            # arn:aws:ecs:<region>:<account>:task/<cluster>/<task-id>
+            region = self.arn.split(":")[3]
+            task_id = self.arn.rsplit("/", 1)[-1]
+            self.log.info(
+                "Job console: https://%s.console.aws.amazon.com/ecs/v2/clusters/%s/tasks/%s?region=%s",
+                region,
+                self.cluster,
+                task_id,
+                region,
+            )
+        return result
+
+
+def serverless_python_task_group(
+    group_id: str,
+    *,
+    module_name: str,
+    class_name: str,
+    python_packages: str,
+    task_role_arn: str,
+    network_config: dict,
+    image_uri: str,
+    ecs_task_builder_factory: "type[EcsTaskBuilderLike] | Any",
+    parameters: str = "{}",
+    size: Literal["xs", "s", "m", "l"] | None = None,
+    cpu: str | None = None,
+    memory: str | None = None,
+    ephemeral_storage_gib: int = 21,
+    retries: int = 0,
+    job_name: str = "",
+    region: str = "us-west-2",
+    log_region: str | None = None,
+    ecs_cluster: str = _DEFAULT_ECS_CLUSTER,
+    log_group: str = _DEFAULT_LOG_GROUP,
+    codeartifact_domain: str = "overture-pypi",
+    codeartifact_owner: str = "505071440022",
+    codeartifact_repo: str = "overture",
+    output_path: str | None = None,
+):
+    """Run a Python job on the AWS ECS Fargate serverless backend.
+
+    Callers implement a class exposing ``run(params: str) -> None`` --
+    typically a ``ServerlessPythonJob`` subclass, but any object with that
+    shape works. The class is cloud-agnostic; only this factory ties the
+    invocation to Fargate.
+
+    Args:
+        group_id: Airflow task group ID shown in the UI.
+        module_name: Dotted Python module containing the job class,
+            e.g. ``"overture_addresses.collect"``.
+        class_name: Name of the class to instantiate, e.g. ``"CollectionJob"``.
+        python_packages: Space-separated pip specs installed at run time.
+        task_role_arn: Full ARN of the IAM role used as both ``taskRoleArn``
+            and ``executionRoleArn``. Required -- resolve it (e.g. from an
+            IAM role name via STS) before calling in; this module has no
+            opinion on how role names map to ARNs in your account.
+        network_config: ECS ``networkConfiguration`` dict (subnets, security
+            groups, ``assignPublicIp``) as accepted by the ECS ``RunTask``
+            API. Required -- resolve it from wherever your deployment stores
+            VPC/subnet/security-group configuration before calling in.
+        image_uri: Fully resolved runner container image URI (registry,
+            repository, and tag). Required -- build it however your
+            deployment maps environments/versions to image tags (e.g. via an
+            ECR-URI builder keyed by account and region) before calling in.
+        ecs_task_builder_factory: Callable with the same call signature as
+            ``EcsTaskBuilder.__init__`` (``family``, ``container_definitions``,
+            ``role_arn``, ``network_configuration``, ``cpu``, ``memory``,
+            ``ephemeral_storage_gib``, ``cluster``, ``awslogs_group``,
+            ``awslogs_region``, ``launch_type``, ``network_mode``,
+            ``output_path``) that returns an object satisfying
+            :class:`EcsTaskBuilderLike` (``register()``/``run()``/``teardown()``).
+            Required -- pass your deployment's ECS task-definition builder
+            (e.g. Overture's internal ``EcsTaskBuilder``); this module never
+            constructs ECS task definitions on its own.
+        parameters: JSON string forwarded to ``cls().run(params)``.
+        size: T-shirt preset. ``xs`` (default) covers streaming I/O, API
+            calls, and lightweight transforms. Presets:
+
+            +------+-----------+------------+-----------+---------+
+            | size | CPU units | Memory MiB | real vCPU | RAM     |
+            +======+===========+============+===========+=========+
+            | xs   | 256       | 512        | 0.25      | 0.5 GB  |
+            | s    | 512       | 1024       | 0.5       | 1 GB    |
+            | m    | 1024      | 2048       | 1         | 2 GB    |
+            | l    | 2048      | 4096       | 2         | 4 GB    |
+            +------+-----------+------------+-----------+---------+
+
+            If ``l`` is not enough, pass ``cpu``/``memory`` explicitly.
+        cpu, memory: Escape hatch for exact Fargate CPU units / memory MiB.
+            Must form a valid Fargate combo; the wrapper validates at parse
+            time. Mutually exclusive with ``size``.
+        ephemeral_storage_gib: Fargate ephemeral storage in GiB (minimum 21
+            when set explicitly, max 200).
+        retries: Airflow-level retries on the ECS run task.
+        job_name: Optional display suffix appended to the task definition
+            family name in the ECS console.
+        region: AWS region for Fargate, ECS, and CodeArtifact.
+        log_region: AWS region for CloudWatch Logs. Defaults to ``region`` --
+            override when logs live in a different region than compute.
+        ecs_cluster: ECS cluster name. Defaults to Overture's reference
+            runner cluster (``"overture-python-runner"``); override for any
+            other deployment.
+        log_group: CloudWatch Logs group for the task's container logs.
+            Defaults to Overture's reference runner log group; override for
+            any other deployment.
+        codeartifact_domain, codeartifact_owner, codeartifact_repo:
+            CodeArtifact coordinates. This backend fetches an auth token
+            and assembles a ``PIP_INDEX_URL`` before invoking the runner;
+            the container itself is not CodeArtifact-aware.
+        output_path: Forwarded to ``ecs_task_builder_factory`` so an
+            injected builder can record container image provenance; this
+            module does not interpret it itself.
+    """
+    resolved_cpu, resolved_memory = _resolve_cpu_memory(size, cpu, memory)
+
+    if not 21 <= ephemeral_storage_gib <= 200:
+        raise ValueError(
+            f"ephemeral_storage_gib={ephemeral_storage_gib} is invalid; Fargate requires 21..200 GiB"
+        )
+    resolved_log_region = log_region or region
+
+    family_suffix = "-".join(
+        part.replace(".", "-") for part in (module_name, class_name, job_name) if part
+    )
+    family = f"serverless-python-{family_suffix}"
+
+    with TaskGroup(group_id=group_id) as tg:
+        ecs = ecs_task_builder_factory(
+            family=family,
+            cluster=ecs_cluster,
+            awslogs_group=log_group,
+            container_definitions=[
+                {
+                    "name": _CONTAINER_NAME,
+                    "image": image_uri,
+                    "essential": True,
+                    "logConfiguration": {
+                        "logDriver": "awslogs",
+                        "options": {
+                            "awslogs-group": log_group,
+                            "awslogs-region": resolved_log_region,
+                            "awslogs-stream-prefix": f"serverless-python/{group_id}",
+                        },
+                    },
+                }
+            ],
+            role_arn=task_role_arn,
+            network_configuration=network_config,
+            cpu=resolved_cpu,
+            memory=resolved_memory,
+            ephemeral_storage_gib=ephemeral_storage_gib,
+            output_path=output_path,
+        )
+
+        # Task IDs use platform-agnostic names (setup / execute_job / cleanup)
+        # rather than the ECS-specific "register_task_definition" /
+        # "deregister_task_definition" defaults, so future backends
+        # (k8s Jobs, Cloud Run, Azure Container Apps) can slot into the
+        # same UI shape without leaking cloud vocabulary.
+        register = ecs.register(task_id="setup")
+        # Build the run operator directly rather than via ``ecs.run()`` so we
+        # can use the custom subclass that mints and injects PIP_INDEX_URL at
+        # execute time. Mirrors ``EcsTaskBuilder.run()`` internals; register()
+        # must have been called to expose the task-definition XCom.
+        run = _CodeArtifactPipIndexUrlEcsOperator(
+            task_id="execute_job",
+            cluster=ecs_cluster,
+            task_definition=str(register.output),
+            launch_type="FARGATE",
+            overrides={
+                "containerOverrides": [
+                    {
+                        "name": _CONTAINER_NAME,
+                        "environment": [
+                            {"name": "MODULE_NAME", "value": module_name},
+                            {"name": "CLASS_NAME", "value": class_name},
+                            {"name": "PYTHON_PACKAGES", "value": python_packages},
+                            {"name": "PARAMS", "value": parameters},
+                        ],
+                    }
+                ]
+            },
+            network_configuration=network_config,
+            awslogs_group=log_group,
+            awslogs_region=resolved_log_region,
+            awslogs_stream_prefix=f"serverless-python/{group_id}",
+            do_xcom_push=True,
+            retries=retries,
+            codeartifact_domain=codeartifact_domain,
+            codeartifact_owner=codeartifact_owner,
+            codeartifact_repo=codeartifact_repo,
+            codeartifact_region=region,
+            target_container_name=_CONTAINER_NAME,
+        )
+
+        chain(
+            register,
+            run,
+            ecs.teardown(_CONTAINER_NAME, run_op=run, task_id="cleanup"),
+        )
+
+    return tg
+
+
+__all__ = ["serverless_python_task_group", "EcsTaskBuilderLike"]

overture_serverless/job.py

@@ -0,0 +1,42 @@
+"""Lightweight base class for a portable "serverless job" contract.
+
+Subclass and implement ``execute_job()``; a runner (container, script, cron
+job, whatever) instantiates the subclass and calls ``run()``.
+
+Plain Python by design — no cloud SDKs, no framework dependencies. The same
+subclass runs identically on any backend or on a developer laptop for local
+iteration.
+"""
+
+import json
+from abc import ABC, abstractmethod
+
+
+class ServerlessPythonJob(ABC):
+    def __init__(self) -> None:
+        self._params: dict = {}
+
+    def run(self, params: "str | dict" = "{}") -> None:
+        print(f"Starting job {self.__class__.__name__}")
+        self._params = json.loads(params) if isinstance(params, str) else params
+        if not isinstance(self._params, dict):
+            raise TypeError(
+                f"params must decode to a JSON object/dict, got {type(self._params).__name__}"
+            )
+        print(f"Received params: {sorted(self._params.keys())}")
+        self.execute_job()
+        print(f"Job {self.__class__.__name__} completed successfully")
+
+    def get_param(self, name: str, default=None, is_required: bool = True):
+        value = self._params.get(name, default)
+        if isinstance(value, str):
+            value = value.strip() or None
+        if is_required and value is None:
+            raise ValueError(f"Missing required parameter: {name}")
+        return default if value is None else value
+
+    def log(self, msg: str) -> None:
+        print(msg)
+
+    @abstractmethod
+    def execute_job(self) -> None: ...

overture_serverless-0.1.0.dist-info/METADATA

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: overture-serverless
+Version: 0.1.0
+Summary: Base classes for portable, framework-agnostic serverless Python jobs.
+Author: Overture Maps Foundation
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/OvertureMaps/overture-core
+Project-URL: Source, https://github.com/OvertureMaps/overture-core/tree/main/packages/overture_serverless
+Project-URL: Issues, https://github.com/OvertureMaps/overture-core/issues
+Keywords: overture,overturemaps,serverless,jobs
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: fargate
+Requires-Dist: apache-airflow<3,>=2.9; extra == "fargate"
+Requires-Dist: apache-airflow-providers-amazon<9,>=8.13.0; extra == "fargate"
+Provides-Extra: dev
+Requires-Dist: pytest>=9.1.1; extra == "dev"
+Requires-Dist: pytest-cov>=7.0.0; extra == "dev"
+Requires-Dist: apache-airflow<3,>=2.9; extra == "dev"
+Requires-Dist: apache-airflow-providers-amazon<9,>=8.13.0; extra == "dev"
+Dynamic: license-file
+
+# overture-serverless
+
+[![PyPI](https://img.shields.io/pypi/v/overture-serverless.svg)](https://pypi.org/project/overture-serverless/)
+[![Python versions](https://img.shields.io/pypi/pyversions/overture-serverless.svg)](https://pypi.org/project/overture-serverless/)
+
+Base class for portable, framework-agnostic "job" classes, plus the Airflow-facing backends that launch them. A published package that gets installed at run time wherever a job actually executes, and provides the contract your job subclasses.
+
+It contains:
+- `ServerlessPythonJob` — abstract base class your jobs subclass (`execute_job()`), with parameter parsing (`get_param()`) and logging (`log()`). Plain Python, no cloud dependencies — runs identically on any backend or on your laptop.
+- `backends.fargate.serverless_python_task_group` — an Airflow `TaskGroup` factory that runs a job on AWS ECS Fargate. Requires the `fargate` extra (`pip install overture-serverless[fargate]`), which pulls in `apache-airflow` and `apache-airflow-providers-amazon`; the base install stays dependency-free since `ServerlessPythonJob` runs inside the job container, not inside Airflow.
+
+## Writing a job
+
+```python
+from overture_serverless.job import ServerlessPythonJob
+
+
+class CollectionJob(ServerlessPythonJob):
+    def execute_job(self) -> None:
+        path = self.get_param("input_path")
+        self.log(f"Processing {path}")
+        # ... your logic
+```
+
+## Testing locally
+
+```bash
+cd packages/overture_serverless
+uv run python -c "
+from overture_serverless.job import ServerlessPythonJob
+
+class ExampleJob(ServerlessPythonJob):
+    def execute_job(self) -> None:
+        self.log(self.get_param('input_path'))
+
+ExampleJob().run('{\"input_path\": \"s3://...\"}')
+"
+```
+
+## Launching a job on Fargate
+
+`serverless_python_task_group` builds an Airflow `TaskGroup` that provisions an ECS Fargate task, runs your job's runner container, and tears the task definition down afterward. It has no opinion on your AWS account's VPC layout, IAM roles, or container registry — you resolve those and pass them in:
+
+```python
+from overture_serverless.backends.fargate import serverless_python_task_group
+
+collect = serverless_python_task_group(
+    group_id="collection_job",
+    module_name="overture_addresses.collect",
+    class_name="CollectionJob",
+    python_packages="overture-addresses",
+    task_role_arn=my_resolved_role_arn,  # e.g. via STS in your own DAG code
+    network_config=my_ecs_network_config,  # ECS `networkConfiguration` dict
+    image_uri=my_resolved_runner_image_uri,  # e.g. from your own ECR-URI builder
+    ecs_task_builder_factory=MyEcsTaskBuilder,  # your register/run/teardown builder
+)
+```
+
+See the `serverless_python_task_group` docstring for the full parameter list (sizing presets, CodeArtifact coordinates, retries, etc.).
+
+## Publishing
+
+See [`PACKAGE_VERSIONING.md`](../PACKAGE_VERSIONING.md) for how a version bump here turns into a PyPI release.

overture_serverless-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+overture_serverless/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+overture_serverless/airflow_compat.py,sha256=gpb4cd4K20KylY189IR-fCGjCNZlBIKqEKusrQvTvOE,11779
+overture_serverless/job.py,sha256=TzgJ-soiMNIHGGcHH7JbXBggwU6wljOE2C7khN4ThEg,1547
+overture_serverless/backends/__init__.py,sha256=0WcREfe4kH82HyKanfYuEuHIloYI_ewESJ0umnCKHAU,998
+overture_serverless/backends/fargate.py,sha256=mpLH0aaBtgHqQoRvMJH7J6M7UmQWTU0cp_0VClshutE,18378
+overture_serverless-0.1.0.dist-info/licenses/LICENSE,sha256=24ta8OgpjROjE-HE1FWF9T3w46CiGzvcFzxjuEOFIUI,1070
+overture_serverless-0.1.0.dist-info/METADATA,sha256=hrvCxAvqNXwgL4aDu6kTIBAW1JWkc0_5yU3IdFhLZHo,4239
+overture_serverless-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+overture_serverless-0.1.0.dist-info/top_level.txt,sha256=UlY7jr7QD9r-mrw3G3cvlkb2OVmiFwWC-i6_p91Rbpo,20
+overture_serverless-0.1.0.dist-info/RECORD,,

overture_serverless-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

overture_serverless-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Overture Maps
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

overture_serverless-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+overture_serverless