phlo-airbyte 0.15.1__tar.gz

phlo_airbyte-0.15.1/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: phlo-airbyte
+Version: 0.15.1
+Summary: Airbyte control-plane integration plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo<0.16,>=0.15.1
+Requires-Dist: requests>=2.32.5
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+Airbyte control-plane integration plugin for Phlo.

phlo_airbyte-0.15.1/README.md

@@ -0,0 +1,60 @@
+# phlo-airbyte
+
+Airbyte control-plane integration plugin for Phlo.
+
+## Description
+
+`phlo-airbyte` treats Airbyte as a **connector execution engine, not a second
+orchestrator**. Dagster remains the scheduler: a Phlo asset
+(`AirbyteConnectionAsset`) starts one sync on a pre-existing Airbyte
+connection, polls its job, fails closed on unknown or ambiguous terminal
+states, and only then emits a materialization carrying the job id, connection
+id, output tables, and timestamps as lineage evidence.
+
+Source credentials live in Airbyte's secret store; Phlo config never stores
+them.
+
+> **Boundary note:** connector execution requires the full self-managed
+> Airbyte stack (workers, temporal, webapp), deployed via `abctl` or an
+> external installation. The pinned `airbyte/server` service here provides
+> the control-plane API Phlo integrates with. The Iceberg destination
+> contract is under an active compatibility spike against the pinned
+> release; if it cannot land schema-evolving output in the required Iceberg
+> layout, the approved fallback keeps Airbyte for extraction and delegates
+> the final Iceberg write to Phlo's existing dlt path.
+
+## Installation
+
+```bash
+pip install phlo-airbyte
+# or
+phlo plugin install airbyte
+```
+
+## Configuration
+
+| Variable | Default | Description |
+| -------- | ------- | ----------- |
+| `AIRBYTE_PORT` | `10020` | Airbyte server API host port |
+| `AIRBYTE_WORKSPACE_ID` | _(empty)_ | Airbyte workspace id for connection lookups |
+| `AIRBYTE_POLL_INTERVAL_SECONDS` | `10` | Seconds between sync job status polls |
+| `AIRBYTE_SYNC_TIMEOUT_SECONDS` | `3600` | Max seconds to wait for one sync |
+
+## Usage
+
+```python
+from phlo_airbyte.assets import phlo_airbyte_connection
+
+phlo_airbyte_connection(
+    connection_id="<airbyte-connection-uuid>",
+    tables=["bronze.postgres_users"],
+    group="ingestion",
+    name="postgres_users",
+)
+```
+
+```bash
+phlo airbyte status
+phlo airbyte connections
+phlo airbyte sync <connection-id>
+```

phlo_airbyte-0.15.1/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [
+    "setuptools>=45",
+    "wheel",
+]
+
+[project]
+dependencies = [
+    "phlo>=0.15.1,<0.16",
+    "requests>=2.32.5",
+]
+description = "Airbyte control-plane integration plugin for Phlo"
+name = "phlo-airbyte"
+requires-python = ">=3.11"
+version = "0.15.1"
+
+[[project.authors]]
+email = "team@phlo.dev"
+name = "Phlo Team"
+
+[project.entry-points."phlo.plugins.assets"]
+airbyte = "phlo_airbyte.plugin:AirbyteAssetProvider"
+
+[project.entry-points."phlo.plugins.cli"]
+airbyte = "phlo_airbyte.cli_plugin:AirbyteCliPlugin"
+
+[project.entry-points."phlo.plugins.ingestion_providers"]
+airbyte = "phlo_airbyte.plugin:AirbyteIngestionProvider"
+
+[project.entry-points."phlo.plugins.resources"]
+airbyte = "phlo_airbyte.resource_provider:AirbyteResourceProvider"
+
+[project.entry-points."phlo.plugins.services"]
+airbyte = "phlo_airbyte.plugin:AirbyteServicePlugin"
+
+[project.license]
+text = "MIT"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.readme]
+content-type = "text/plain"
+text = "Airbyte control-plane integration plugin for Phlo."
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+phlo_airbyte = ["service.yaml"]
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools.packages.find]
+where = ["src"]

phlo_airbyte-0.15.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_airbyte-0.15.1/src/phlo_airbyte/__init__.py

@@ -0,0 +1,13 @@
+"""Phlo Airbyte integration package.
+
+Provides the self-managed Airbyte service descriptor, the Configuration API
+client, and the ``AirbyteConnectionAsset`` that runs one Airbyte sync inside
+Phlo's Dagster-owned lifecycle with job-state evidence.
+"""
+
+from importlib.metadata import version
+
+from phlo_airbyte.plugin import AirbyteServicePlugin
+
+__all__ = ["AirbyteServicePlugin"]
+__version__ = version("phlo-airbyte")

phlo_airbyte-0.15.1/src/phlo_airbyte/assets.py

@@ -0,0 +1,153 @@
+"""Airbyte connection assets.
+
+``phlo_airbyte_connection`` declares a named, pre-existing Airbyte connection
+plus its expected output tables. Dagster owns scheduling: the asset starts one
+sync, polls it to a verified terminal state, and only then emits a
+materialization carrying the job id, connection id, output tables, and
+timestamps as lineage evidence. Source credentials stay in Airbyte's secret
+store; Phlo never stores them.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+
+from phlo.capabilities import AssetSpec, MaterializeResult, PartitionSpec, RunSpec
+from phlo.capabilities.runtime import RuntimeContext
+from phlo.exceptions import PhloConfigError
+from phlo.logging import log_event
+
+from phlo_airbyte.client import AirbyteClient
+
+# Populated at decoration time and kept for the process lifetime. Cleared only
+# by clear_airbyte_assets(), which tests and plugin reloads use to reset state.
+_AIRBYTE_ASSETS: list[AssetSpec] = []
+
+
+def get_airbyte_assets() -> list[AssetSpec]:
+    """Return registered Airbyte connection asset specifications."""
+    return list(_AIRBYTE_ASSETS)
+
+
+def clear_airbyte_assets() -> None:
+    """Clear all registered Airbyte assets (for tests and plugin reloads)."""
+    _AIRBYTE_ASSETS.clear()
+
+
+def _validate_connection_config(connection_id: str, tables: list[str]) -> None:
+    if not connection_id or not connection_id.strip():
+        raise PhloConfigError(
+            message="Airbyte connection assets require a connection_id",
+            suggestions=["Copy the connection id from the Airbyte workspace."],
+        )
+    if not tables:
+        raise PhloConfigError(
+            message="Airbyte connection assets must declare their expected output tables",
+            suggestions=[
+                "List the tables the connection writes so downstream assets can depend on them."
+            ],
+        )
+
+
+def _build_asset_run(
+    *,
+    connection_id: str,
+    tables: list[str],
+    client_factory: Callable[[], AirbyteClient] | None,
+) -> Callable[[RuntimeContext], Iterator[MaterializeResult]]:
+    """Build the run callable that triggers and polls one Airbyte sync."""
+
+    def run(runtime: RuntimeContext) -> Iterator[MaterializeResult]:
+        logger = runtime.logger
+        log_event(logger, "info", "starting_airbyte_sync", connection_id=connection_id)
+
+        if client_factory is not None:
+            client = client_factory()
+        else:
+            client = AirbyteClient()
+        evidence = client.run_sync(connection_id)
+
+        log_event(
+            logger,
+            "info",
+            "airbyte_sync_completed",
+            job_id=evidence["job_id"],
+            status=evidence["status"],
+        )
+        yield MaterializeResult(
+            metadata={
+                "airbyte_job_id": evidence["job_id"],
+                "airbyte_connection_id": evidence["connection_id"],
+                "airbyte_status": evidence["status"],
+                "output_tables": tables,
+                "airbyte_started_at": evidence.get("started_at"),
+                "airbyte_ended_at": evidence.get("ended_at"),
+                "airbyte_elapsed_seconds": evidence.get("elapsed_seconds"),
+                "source_state": {"job_id": evidence["job_id"], "status": evidence["status"]},
+            },
+            status="ok",
+        )
+
+    return run
+
+
+def phlo_airbyte_connection(
+    connection_id: str,
+    tables: list[str],
+    group: str,
+    *,
+    name: str | None = None,
+    destination: str = "iceberg",
+    description: str | None = None,
+    max_runtime_seconds: int = 3600,
+    max_retries: int = 1,
+    retry_delay_seconds: int = 60,
+    cron: str | None = None,
+    client_factory: Callable[[], AirbyteClient] | None = None,
+) -> AssetSpec:
+    """Register an Airbyte connection asset and return its specification.
+
+    ``connection_id`` must reference a connection that already exists in the
+    Airbyte workspace; Phlo triggers and observes the sync but never mutates
+    Airbyte configuration.
+    """
+    _validate_connection_config(connection_id, tables)
+    asset_name = name or connection_id[:8]
+    asset_key = f"{group}.{asset_name}"
+    spec = AssetSpec(
+        key=asset_key,
+        group=group,
+        description=description
+        or f"Airbyte sync for connection {connection_id} landing {', '.join(tables)}",
+        kinds={"airbyte", "ingestion"},
+        tags={
+            "provider": "airbyte",
+            "asset_type": "ingestion",
+            "source": "airbyte",
+            "destination": destination,
+        },
+        metadata={
+            "provider": "airbyte",
+            "airbyte_connection_id": connection_id,
+            "output_tables": tables,
+            "destination": destination,
+            "group": group,
+        },
+        partitions=PartitionSpec(kind="daily"),
+        resources=set(),
+        run=RunSpec(
+            fn=_build_asset_run(
+                connection_id=connection_id,
+                tables=tables,
+                client_factory=client_factory,
+            ),
+            max_runtime_seconds=max_runtime_seconds,
+            max_retries=max_retries,
+            retry_delay_seconds=retry_delay_seconds,
+            cron=cron,
+            freshness_hours=None,
+        ),
+        checks=[],
+    )
+    _AIRBYTE_ASSETS.append(spec)
+    return spec

phlo_airbyte-0.15.1/src/phlo_airbyte/cli.py

@@ -0,0 +1,63 @@
+"""Airbyte CLI commands: status, connections, and manual sync."""
+
+from __future__ import annotations
+
+import click
+
+from phlo.cli.output import command_failed_error
+
+
+@click.command(name="airbyte")
+@click.argument("airbyte_args", nargs=-1, type=click.UNPROCESSED)
+@click.pass_context
+def airbyte_group(ctx: click.Context, airbyte_args: tuple[str, ...]) -> None:
+    """Interact with the Airbyte control plane (status, connections, sync)."""
+    args = list(airbyte_args)
+    if not args or args[0] in {"-h", "--help", "help"}:
+        click.echo(ctx.get_help())
+        return
+    command = args.pop(0)
+    if command == "status":
+        _status()
+        return
+    if command == "connections":
+        _connections()
+        return
+    if command == "sync":
+        if not args:
+            command_failed_error("sync requires a connection id")
+        _sync(args[0])
+        return
+    click.echo(f"Unknown airbyte command: {command}", err=True)
+    ctx.exit(2)
+
+
+def _status() -> None:
+    from phlo_airbyte.client import AirbyteClient
+
+    client = AirbyteClient()
+    healthy = client.health_check()
+    click.echo(f"Airbyte health: {'ok' if healthy else 'unavailable'}")
+
+
+def _connections() -> None:
+    from phlo_airbyte.client import AirbyteClient
+
+    client = AirbyteClient()
+    try:
+        connections = client.list_connections()
+    except Exception as exc:
+        command_failed_error(f"Could not list Airbyte connections: {exc}")
+    for connection in connections:
+        click.echo(f"  - {connection.get('connectionId', '?')}: {connection.get('name', '')}")
+
+
+def _sync(connection_id: str) -> None:
+    from phlo_airbyte.client import AirbyteClient
+
+    client = AirbyteClient()
+    try:
+        evidence = client.run_sync(connection_id)
+    except Exception as exc:
+        command_failed_error(f"Airbyte sync failed: {exc}")
+    click.echo(f"Sync {evidence['job_id']}: {evidence['status']}")

phlo_airbyte-0.15.1/src/phlo_airbyte/cli_plugin.py

@@ -0,0 +1,15 @@
+"""CLI plugin registration for the Airbyte package."""
+
+from __future__ import annotations
+
+from phlo.plugins.base import cli_command_plugin_class
+
+from phlo_airbyte.cli import airbyte_group
+
+AirbyteCliPlugin = cli_command_plugin_class(
+    "AirbyteCliPlugin",
+    name="airbyte",
+    version="0.1.0",
+    description="Airbyte connection and sync commands",
+    commands=[airbyte_group],
+)

phlo_airbyte-0.15.1/src/phlo_airbyte/client.py

@@ -0,0 +1,157 @@
+"""Airbyte Configuration API client with fail-closed job-state handling.
+
+Dagster owns scheduling; Airbyte owns connector execution and state. This
+client starts one sync, polls its job, and only returns a verdict when the
+pinned Airbyte release reports a known terminal state. Unknown or ambiguous
+states raise :class:`AmbiguousJobStateError` so the Phlo asset fails closed
+instead of guessing.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+import requests
+
+from phlo.logging import get_logger
+from phlo_airbyte.settings import AirbyteSettings, get_settings
+
+logger = get_logger(__name__)
+
+REQUEST_TIMEOUT_SECONDS = 30
+
+KNOWN_TERMINAL_STATUSES: dict[str, str] = {
+    "succeeded": "succeeded",
+    "failed": "failed",
+    "cancelled": "cancelled",
+}
+
+
+class AmbiguousJobStateError(RuntimeError):
+    """Raised when a sync job ends in an unknown or ambiguous state."""
+
+
+class AirbyteClient:
+    """HTTP client for the Airbyte Configuration API."""
+
+    def __init__(self, settings: AirbyteSettings | None = None) -> None:
+        self._settings = settings
+
+    @property
+    def settings(self) -> AirbyteSettings:
+        if self._settings is None:
+            self._settings = get_settings()
+        return self._settings
+
+    def _request(self, method: str, path: str, *, json_body: dict[str, Any] | None = None) -> Any:
+        url = f"{self.settings.airbyte_api_uri()}{path}"
+        response = requests.request(method, url, json=json_body, timeout=REQUEST_TIMEOUT_SECONDS)
+        response.raise_for_status()
+        payload = response.json()
+        return payload if payload is not None else {}
+
+    def health_check(self) -> bool:
+        """Return whether the Airbyte server responds on its health endpoint."""
+        try:
+            response = requests.get(
+                f"{self.settings.airbyte_api_uri()}/api/v1/health",
+                timeout=REQUEST_TIMEOUT_SECONDS,
+            )
+        except requests.RequestException:
+            logger.warning("airbyte_health_check_failed", exc_info=True)
+            return False
+        return response.status_code == 200
+
+    def list_connections(self) -> list[dict[str, Any]]:
+        """List connections in the configured workspace."""
+        body: dict[str, Any] = {}
+        if self.settings.airbyte_workspace_id:
+            body["workspaceId"] = self.settings.airbyte_workspace_id
+        payload = self._request("POST", "/api/v1/connections/list", json_body=body)
+        return list(payload.get("connections", []))
+
+    def trigger_sync(self, connection_id: str) -> dict[str, Any]:
+        """Start one sync for a pre-existing Airbyte connection."""
+        return dict(
+            self._request(
+                "POST",
+                "/api/v1/jobs/run",
+                json_body={"connectionId": connection_id, "jobType": "sync"},
+            )
+        )
+
+    def get_job(self, job_id: str) -> dict[str, Any]:
+        """Return the current job record for one sync."""
+        return dict(self._request("POST", "/api/v1/jobs/get", json_body={"id": int(job_id)}))
+
+    @staticmethod
+    def classify_status(status: str | None) -> str | None:
+        """Map an Airbyte job status to a Phlo terminal verdict, else None.
+
+        Raises AmbiguousJobStateError for any status that is neither a known
+        terminal state nor a known in-progress state, so callers fail closed.
+        """
+        if status is None:
+            raise AmbiguousJobStateError("Airbyte job returned no status")
+        normalized = str(status).strip().lower()
+        if normalized in KNOWN_TERMINAL_STATUSES:
+            return KNOWN_TERMINAL_STATUSES[normalized]
+        if normalized in {"pending", "running", "incomplete", "incomplete_retrying"}:
+            return None
+        raise AmbiguousJobStateError(
+            f"Unknown Airbyte job status {status!r}; refusing to guess the outcome"
+        )
+
+    def run_sync(
+        self,
+        connection_id: str,
+        *,
+        poll_interval_seconds: int | None = None,
+        timeout_seconds: int | None = None,
+        clock: Any = None,
+    ) -> dict[str, Any]:
+        """Run one sync to a verified terminal state and return its evidence.
+
+        The returned evidence carries the job id, connection id, terminal
+        status, and timestamps for lineage. Unknown states and timeouts raise
+        rather than reporting success.
+        """
+        settings = self.settings
+        poll_interval = poll_interval_seconds or settings.airbyte_poll_interval_seconds
+        timeout = timeout_seconds or settings.airbyte_sync_timeout_seconds
+        sleep = clock.sleep if clock is not None else time.sleep
+
+        started = time.time()
+        job_payload = self.trigger_sync(connection_id)
+        job = job_payload.get("job", job_payload)
+        job_id = str(job.get("id"))
+        logger.info("airbyte_sync_started", connection_id=connection_id, job_id=job_id)
+
+        while True:
+            record = self.get_job(job_id).get("job", {})
+            verdict = self.classify_status(record.get("status"))
+            if verdict is not None:
+                evidence = {
+                    "job_id": job_id,
+                    "connection_id": connection_id,
+                    "status": verdict,
+                    "started_at": job.get("createdAt"),
+                    "ended_at": record.get("updatedAt"),
+                    "elapsed_seconds": round(time.time() - started, 2),
+                    "records_synced": record.get("recordsSynced")
+                    if isinstance(record, dict)
+                    else None,
+                }
+                if verdict != "succeeded":
+                    raise RuntimeError(
+                        f"Airbyte sync {job_id} for connection {connection_id} "
+                        f"ended with status {verdict!r}"
+                    )
+                return evidence
+            if time.time() - started > timeout:
+                raise TimeoutError(
+                    f"Airbyte sync {job_id} for connection {connection_id} did not reach a "
+                    f"terminal state within {timeout}s"
+                )
+            sleep(poll_interval)

phlo_airbyte-0.15.1/src/phlo_airbyte/plugin.py

@@ -0,0 +1,77 @@
+"""Airbyte plugin registrations: service, assets, and ingestion provider."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from phlo.capabilities import AssetCheckSpec, AssetSpec
+from phlo.plugins.base import (
+    AssetProviderPlugin,
+    IngestionProviderPlugin,
+    PluginMetadata,
+    service_plugin_class,
+)
+
+AirbyteServicePlugin = service_plugin_class(
+    "AirbyteServicePlugin",
+    name="airbyte",
+    version="0.1.0",
+    description="Self-managed Airbyte control plane for connector-managed ingestion",
+    author="Phlo Team",
+    tags=["ingestion", "airbyte", "connectors"],
+)
+
+
+class AirbyteAssetProvider(AssetProviderPlugin):
+    """Expose registered Airbyte connection assets to the orchestrator."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the Airbyte asset provider."""
+        return PluginMetadata(
+            name="airbyte",
+            version="0.1.0",
+            description="Airbyte connection assets for Phlo",
+        )
+
+    def get_assets(self) -> Iterable[AssetSpec]:
+        """Return Airbyte assets registered via the decorator."""
+        from phlo_airbyte.assets import get_airbyte_assets
+
+        return get_airbyte_assets()
+
+    def get_checks(self) -> Iterable[AssetCheckSpec]:
+        """Airbyte connections do not register checks directly."""
+        return []
+
+    def clear_registries(self) -> None:
+        """Reset the asset registry (tests and plugin reloads)."""
+        from phlo_airbyte.assets import clear_airbyte_assets
+
+        clear_airbyte_assets()
+
+
+class AirbyteIngestionProvider(IngestionProviderPlugin):
+    """Expose the Airbyte connection decorator to workflow authoring."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the Airbyte ingestion provider."""
+        return PluginMetadata(
+            name="airbyte",
+            version="0.1.0",
+            description="Airbyte connector-managed ingestion provider",
+        )
+
+    def get_decorator(self) -> Callable[..., Any]:
+        """Return the Airbyte connection decorator function."""
+        from phlo_airbyte.assets import phlo_airbyte_connection
+
+        return phlo_airbyte_connection
+
+    def get_asset_retriever(self) -> Callable[[], list[Any]]:
+        """Return the function retrieving registered Airbyte assets."""
+        from phlo_airbyte.assets import get_airbyte_assets
+
+        return get_airbyte_assets

phlo_airbyte-0.15.1/src/phlo_airbyte/resource_provider.py

@@ -0,0 +1,25 @@
+"""Capability provider exposing the Airbyte client as a runtime resource."""
+
+from __future__ import annotations
+
+from phlo.capabilities import ResourceSpec
+from phlo.plugins.base import PluginMetadata, ResourceProviderPlugin
+
+from phlo_airbyte.client import AirbyteClient
+
+
+class AirbyteResourceProvider(ResourceProviderPlugin):
+    """Expose the Airbyte Configuration API client as a capability resource."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for the Airbyte resource provider."""
+        return PluginMetadata(
+            name="airbyte",
+            version="0.1.0",
+            description="Airbyte API client resource",
+        )
+
+    def get_resources(self) -> list[ResourceSpec]:
+        """Expose the raw Airbyte client as a runtime resource."""
+        return [ResourceSpec(name="airbyte", resource=AirbyteClient())]

phlo_airbyte-0.15.1/src/phlo_airbyte/service.yaml

@@ -0,0 +1,60 @@
+name: airbyte
+description: Self-managed Airbyte control plane for connector-managed ingestion
+category: ingestion
+default: false
+
+image: airbyte/server:2.2.0@sha256:70e125498a1c110bc3e9444ccf9b373cc82517a3cfc68b8d4c54dbc553a4f4eb
+
+depends_on:
+  - postgres
+  - airbyte-temporal
+  - airbyte-manifest
+compose:
+  restart: unless-stopped
+  labels:
+    phlo.metrics.enabled: "false"
+  environment:
+    AIRBYTE_VERSION: "2.2.0"
+    MANIFEST_SERVER_API_HOST: http://airbyte-manifest:8080
+    AIRBYTE_API_HOST: http://localhost:8001/api/public
+    AIRBYTE_URL: ${AIRBYTE_URL:-http://localhost:8001}
+    CONFIG_ROOT: /data
+    WORKSPACE_ROOT: /data
+    DATABASE_USER: ${POSTGRES_USER:-phlo}
+    DATABASE_PASSWORD: ${POSTGRES_PASSWORD:-phlo}
+    DATABASE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-phlo}
+    CONFIG_DATABASE_REPLICA_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-phlo}
+    CONFIG_DATABASE_REPLICA_USER: ${POSTGRES_USER:-phlo}
+    CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.35.15.001"
+    JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.29.15.001"
+    ENTERPRISE_SOURCE_STUBS_URL: https://connectors.airbyte.com/files/resources/connector_stubs/v1/connector_stubs.json
+    STORAGE_TYPE: minio
+    STORAGE_BUCKET_LOG: airbyte-bucket
+    STORAGE_BUCKET_STATE: airbyte-bucket
+    STORAGE_BUCKET_AUDIT_LOGGING: airbyte-bucket
+    STORAGE_BUCKET_WORKLOAD_OUTPUT: airbyte-bucket
+    STORAGE_BUCKET_ACTIVITY_PAYLOAD: airbyte-bucket
+    AWS_ACCESS_KEY_ID: ${ICEBERG_S3_ACCESS_KEY:-minio}
+    AWS_SECRET_ACCESS_KEY: ${ICEBERG_S3_SECRET_KEY:-minio123}
+    AWS_DEFAULT_REGION: ${ICEBERG_S3_REGION:-us-east-1}
+    MINIO_ENDPOINT: http://minio:9000
+    S3_PATH_STYLE_ACCESS: "true"
+    WORKSPACE_ID: ${AIRBYTE_WORKSPACE_ID:-}
+  ports:
+    - "${AIRBYTE_PORT:-10020}:8001"
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:8001/api/v1/health"]
+    interval: 15s
+    timeout: 5s
+    retries: 12
+    start_period: 60s
+
+env_vars:
+  AIRBYTE_PORT:
+    default: 10020
+    description: Airbyte server API host port
+  AIRBYTE_WORKSPACE_ID:
+    default: ""
+    description: Airbyte workspace id used for connection lookups
+
+files:

phlo_airbyte-0.15.1/src/phlo_airbyte/settings.py

@@ -0,0 +1,49 @@
+"""Airbyte settings resolved from the project environment."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from phlo.config.base import BaseConfig
+from phlo.config.cache import project_root_cached
+from phlo.config.network import resolve_host
+from pydantic import Field
+
+
+class AirbyteSettings(BaseConfig):
+    """Settings for the Airbyte control-plane integration."""
+
+    airbyte_host: str = Field(default="airbyte-server", description="Airbyte API host")
+    airbyte_port: int = Field(default=8001, description="Airbyte server API port")
+    airbyte_workspace_id: str | None = Field(
+        default=None, description="Airbyte workspace id for connection lookups"
+    )
+    airbyte_client_id: str | None = Field(
+        default=None, description="Airbyte API client id for token auth"
+    )
+    airbyte_client_secret: str | None = Field(
+        default=None,
+        description="Airbyte API client secret for token auth",
+    )
+    airbyte_poll_interval_seconds: int = Field(
+        default=10, description="Seconds between sync job status polls"
+    )
+    airbyte_sync_timeout_seconds: int = Field(
+        default=3600, description="Maximum seconds to wait for one sync to reach a terminal state"
+    )
+
+    def model_post_init(self, __context: Any) -> None:
+        host, port = resolve_host(self.airbyte_host, self.airbyte_port, port_env_var="AIRBYTE_PORT")
+        object.__setattr__(self, "airbyte_host", host)
+        object.__setattr__(self, "airbyte_port", port)
+
+    def airbyte_api_uri(self) -> str:
+        """Return the Airbyte server API base URI."""
+        return f"http://{self.airbyte_host}:{self.airbyte_port}"
+
+
+@project_root_cached
+def get_settings(project_root: Path) -> AirbyteSettings:
+    """Return cached Airbyte settings for the selected project root."""
+    return AirbyteSettings()

phlo_airbyte-0.15.1/src/phlo_airbyte.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: phlo-airbyte
+Version: 0.15.1
+Summary: Airbyte control-plane integration plugin for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo<0.16,>=0.15.1
+Requires-Dist: requests>=2.32.5
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+Airbyte control-plane integration plugin for Phlo.

phlo_airbyte-0.15.1/src/phlo_airbyte.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+README.md
+pyproject.toml
+src/phlo_airbyte/__init__.py
+src/phlo_airbyte/assets.py
+src/phlo_airbyte/cli.py
+src/phlo_airbyte/cli_plugin.py
+src/phlo_airbyte/client.py
+src/phlo_airbyte/plugin.py
+src/phlo_airbyte/resource_provider.py
+src/phlo_airbyte/service.yaml
+src/phlo_airbyte/settings.py
+src/phlo_airbyte.egg-info/PKG-INFO
+src/phlo_airbyte.egg-info/SOURCES.txt
+src/phlo_airbyte.egg-info/dependency_links.txt
+src/phlo_airbyte.egg-info/entry_points.txt
+src/phlo_airbyte.egg-info/requires.txt
+src/phlo_airbyte.egg-info/top_level.txt
+tests/test_airbyte_assets.py
+tests/test_airbyte_client.py
+tests/test_airbyte_plugin.py

phlo_airbyte-0.15.1/src/phlo_airbyte.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

phlo_airbyte-0.15.1/src/phlo_airbyte.egg-info/entry_points.txt

@@ -0,0 +1,14 @@
+[phlo.plugins.assets]
+airbyte = phlo_airbyte.plugin:AirbyteAssetProvider
+
+[phlo.plugins.cli]
+airbyte = phlo_airbyte.cli_plugin:AirbyteCliPlugin
+
+[phlo.plugins.ingestion_providers]
+airbyte = phlo_airbyte.plugin:AirbyteIngestionProvider
+
+[phlo.plugins.resources]
+airbyte = phlo_airbyte.resource_provider:AirbyteResourceProvider
+
+[phlo.plugins.services]
+airbyte = phlo_airbyte.plugin:AirbyteServicePlugin

phlo_airbyte-0.15.1/src/phlo_airbyte.egg-info/requires.txt

@@ -0,0 +1,6 @@
+phlo<0.16,>=0.15.1
+requests>=2.32.5
+
+[dev]
+pytest>=7.0
+ruff>=0.1.0

phlo_airbyte-0.15.1/src/phlo_airbyte.egg-info/top_level.txt

@@ -0,0 +1 @@
+phlo_airbyte

phlo_airbyte-0.15.1/tests/test_airbyte_assets.py

@@ -0,0 +1,93 @@
+"""Tests for Airbyte connection asset registration and execution."""
+
+from __future__ import annotations
+
+import pytest
+from phlo.helpers.testing import FakeRuntimeContext
+from phlo_airbyte.assets import (
+    clear_airbyte_assets,
+    get_airbyte_assets,
+    phlo_airbyte_connection,
+)
+
+
+class FakeClient:
+    def __init__(self, evidence: dict | None = None, error: Exception | None = None) -> None:
+        self.evidence = evidence or {
+            "job_id": "42",
+            "connection_id": "conn-1",
+            "status": "succeeded",
+            "started_at": 1000,
+            "ended_at": 2000,
+            "elapsed_seconds": 1.5,
+        }
+        self.error = error
+        self.ran: list[str] = []
+
+    def run_sync(self, connection_id: str, **kwargs) -> dict:
+        self.ran.append(connection_id)
+        if self.error is not None:
+            raise self.error
+        return self.evidence
+
+
+@pytest.fixture(autouse=True)
+def _clean_registry():
+    clear_airbyte_assets()
+    yield
+    clear_airbyte_assets()
+
+
+def _runtime() -> FakeRuntimeContext:
+    return FakeRuntimeContext(partition_key="2026-09-01", run_id="run-1")
+
+
+def test_connection_asset_requires_tables_and_connection_id() -> None:
+    with pytest.raises(Exception, match="connection_id"):
+        phlo_airbyte_connection(connection_id="  ", tables=["t"], group="ingestion")
+    with pytest.raises(Exception, match="output tables"):
+        phlo_airbyte_connection(connection_id="conn-1", tables=[], group="ingestion")
+
+
+def test_registered_asset_carries_airbyte_metadata() -> None:
+    spec = phlo_airbyte_connection(
+        connection_id="conn-1",
+        tables=["bronze.users"],
+        group="ingestion",
+        name="users",
+    )
+    assert spec.key == "ingestion.users"
+    assert spec.tags["provider"] == "airbyte"
+    assert spec.metadata["airbyte_connection_id"] == "conn-1"
+    assert spec.metadata["output_tables"] == ["bronze.users"]
+    assert get_airbyte_assets() == [spec]
+
+
+def test_asset_run_emits_job_evidence_metadata() -> None:
+    client = FakeClient()
+    spec = phlo_airbyte_connection(
+        connection_id="conn-1",
+        tables=["bronze.users"],
+        group="ingestion",
+        client_factory=lambda: client,
+    )
+    results = list(spec.run.fn(_runtime()))
+    assert len(results) == 1
+    metadata = results[0].metadata
+    assert metadata["airbyte_job_id"] == "42"
+    assert metadata["airbyte_connection_id"] == "conn-1"
+    assert metadata["output_tables"] == ["bronze.users"]
+    assert metadata["source_state"] == {"job_id": "42", "status": "succeeded"}
+    assert client.ran == ["conn-1"]
+
+
+def test_asset_run_fails_closed_on_sync_failure() -> None:
+    client = FakeClient(error=RuntimeError("Airbyte sync ended with status 'failed'"))
+    spec = phlo_airbyte_connection(
+        connection_id="conn-1",
+        tables=["bronze.users"],
+        group="ingestion",
+        client_factory=lambda: client,
+    )
+    with pytest.raises(RuntimeError, match="failed"):
+        list(spec.run.fn(_runtime()))

phlo_airbyte-0.15.1/tests/test_airbyte_client.py

@@ -0,0 +1,106 @@
+"""Tests for the Airbyte client's fail-closed job-state handling."""
+
+from __future__ import annotations
+
+import pytest
+from phlo_airbyte.client import AirbyteClient, AmbiguousJobStateError
+from phlo_airbyte.settings import AirbyteSettings
+
+
+class FakeTransport:
+    """Scripted HTTP transport returning queued payloads per path."""
+
+    def __init__(self, responses: dict[str, list[dict]]) -> None:
+        self.responses = {key: list(value) for key, value in responses.items()}
+        self.calls: list[tuple[str, dict]] = []
+
+    def post(self, path: str, body: dict) -> dict:
+        self.calls.append((path, body))
+        queue = self.responses.get(path, [])
+        if not queue:
+            raise AssertionError(f"unexpected request to {path}")
+        item = queue.pop(0)
+        if isinstance(item, Exception):
+            raise item
+        return item
+
+
+def _client(transport: FakeTransport) -> AirbyteClient:
+    settings = AirbyteSettings(airbyte_poll_interval_seconds=0)
+    client = AirbyteClient(settings=settings)
+    client._request = lambda method, path, json_body=None: transport.post(path, json_body or {})
+    return client
+
+
+def test_trigger_sync_sends_connection_and_job_type() -> None:
+    transport = FakeTransport({"/api/v1/jobs/run": [{"job": {"id": 42, "status": "pending"}}]})
+    client = _client(transport)
+    job = client.trigger_sync("conn-1")
+    assert job["job"]["id"] == 42
+    assert transport.calls == [("/api/v1/jobs/run", {"connectionId": "conn-1", "jobType": "sync"})]
+
+
+def test_run_sync_returns_evidence_after_success() -> None:
+    transport = FakeTransport(
+        {
+            "/api/v1/jobs/run": [{"job": {"id": 42, "status": "pending", "createdAt": 1000}}],
+            "/api/v1/jobs/get": [
+                {"job": {"id": 42, "status": "running"}},
+                {"job": {"id": 42, "status": "succeeded", "updatedAt": 2000}},
+            ],
+        }
+    )
+    sleeps: list[int] = []
+    client = _client(transport)
+
+    evidence = client.run_sync(
+        "conn-1", poll_interval_seconds=1, timeout_seconds=30, clock=SimpleClock(sleeps)
+    )
+
+    assert evidence["status"] == "succeeded"
+    assert evidence["job_id"] == "42"
+    assert evidence["connection_id"] == "conn-1"
+    assert sleeps == [1]
+
+
+class SimpleClock:
+    def __init__(self, sleeps: list[int]) -> None:
+        self.sleeps = sleeps
+
+    def sleep(self, seconds: int) -> None:
+        self.sleeps.append(seconds)
+
+
+def test_run_sync_raises_for_failed_terminal_state() -> None:
+    transport = FakeTransport(
+        {
+            "/api/v1/jobs/run": [{"job": {"id": 7, "status": "running"}}],
+            "/api/v1/jobs/get": [{"job": {"id": 7, "status": "failed"}}],
+        }
+    )
+    client = _client(transport)
+    with pytest.raises(RuntimeError, match="ended with status 'failed'"):
+        client.run_sync("conn-1", poll_interval_seconds=1, timeout_seconds=10)
+
+
+def test_unknown_job_status_fails_closed() -> None:
+    client = AirbyteClient()
+    assert client.classify_status("succeeded") == "succeeded"
+    assert client.classify_status("running") is None
+    assert client.classify_status("incomplete_retrying") is None
+    with pytest.raises(AmbiguousJobStateError, match="refusing to guess"):
+        client.classify_status("mystery-state")
+    with pytest.raises(AmbiguousJobStateError):
+        client.classify_status(None)
+
+
+def test_run_sync_times_out_without_terminal_state() -> None:
+    transport = FakeTransport(
+        {
+            "/api/v1/jobs/run": [{"job": {"id": 9, "status": "running"}}],
+            "/api/v1/jobs/get": [{"job": {"id": 9, "status": "running"}}],
+        }
+    )
+    client = _client(transport)
+    with pytest.raises(TimeoutError, match="did not reach a terminal state"):
+        client.run_sync("conn-1", poll_interval_seconds=1, timeout_seconds=-1)

phlo_airbyte-0.15.1/tests/test_airbyte_plugin.py

@@ -0,0 +1,37 @@
+"""Tests for the Airbyte service definition and plugin registrations."""
+
+from __future__ import annotations
+
+from phlo_airbyte.plugin import (
+    AirbyteAssetProvider,
+    AirbyteServicePlugin,
+    AirbyteIngestionProvider,
+)
+from phlo_airbyte.resource_provider import AirbyteResourceProvider
+
+
+def test_service_definition_is_digest_pinned_and_opt_in() -> None:
+    plugin = AirbyteServicePlugin()
+    definition = plugin.service_definition
+    assert definition["name"] == "airbyte"
+    assert definition["default"] is False
+    assert definition["image"].startswith("airbyte/server:2.2.0@sha256:")
+
+
+def test_asset_provider_exposes_registered_assets() -> None:
+    provider = AirbyteAssetProvider()
+    provider.clear_registries()
+    assert list(provider.get_assets()) == []
+    provider.clear_registries()
+
+
+def test_resource_provider_exposes_airbyte_client() -> None:
+    provider = AirbyteResourceProvider()
+    resources = provider.get_resources()
+    assert resources[0].name == "airbyte"
+
+
+def test_ingestion_provider_wires_decorator_and_retriever() -> None:
+    provider = AirbyteIngestionProvider()
+    assert callable(provider.get_decorator())
+    assert callable(provider.get_asset_retriever())