phlo-observe-plugin 0.2.0__py3-none-any.whl

phlo_observe_plugin/__init__.py

@@ -0,0 +1,42 @@
+"""phlo-observe integration for Phlo.
+
+- ``ObserveHookPlugin`` translates hook-bus events into canonical
+  ``phlo-observe`` events.
+- ``ObserveDagsterExtension`` contributes a run-status sensor that closes each
+  physical Dagster run with a terminal ``pipeline.run`` event.
+- ``PhloObserverServicePlugin`` / ``PhloObserverDbSetupPlugin`` declare the
+  observer compose service and its database provisioning step.
+- ``presentation`` carries Phlo's ``PrettyRenderer`` rules and the ``pretty``
+  drain — imported lazily since it needs a PrettyRenderer-capable SDK.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "ObserveDagsterExtension",
+    "ObserveHookPlugin",
+    "PhloObserverDbSetupPlugin",
+    "PhloObserverServicePlugin",
+    "PrettyDrain",
+    "pretty_renderer",
+]
+
+
+def __getattr__(name: str) -> object:
+    if name == "ObserveHookPlugin":
+        from phlo_observe_plugin.hooks_plugin import ObserveHookPlugin
+
+        return ObserveHookPlugin
+    if name == "ObserveDagsterExtension":
+        from phlo_observe_plugin.dagster_ext import ObserveDagsterExtension
+
+        return ObserveDagsterExtension
+    if name in {"PhloObserverServicePlugin", "PhloObserverDbSetupPlugin"}:
+        from phlo_observe_plugin import plugin as _plugin
+
+        return getattr(_plugin, name)
+    if name in {"PrettyDrain", "pretty_renderer"}:
+        from phlo_observe_plugin import presentation as _presentation
+
+        return getattr(_presentation, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

phlo_observe_plugin/dagster_ext.py

@@ -0,0 +1,161 @@
+"""Dagster extension emitting terminal ``pipeline.run`` events per run.
+
+Asset/step/check events are emitted inside the run worker by the adapter
+instrumentation; this extension adds the run-boundary event. Run-status
+sensors evaluate in the Dagster daemon, so the terminal ``pipeline.run`` is
+emitted with explicit correlation rather than ambient context.
+
+Run identity is the *physical* Dagster ``run_id`` — one observer run per
+attempt. The logical Phlo run id (``phlo/run_id`` tag) and attempt number are
+carried as attributes so retries group logically without violating the
+observer's monotonic run-status precedence (a failed attempt must not pin a
+later successful attempt to "failed").
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+import phlo.telemetry as phlo_observe
+from phlo.logging import get_logger
+from phlo.plugins.base import PluginMetadata
+from phlo_dagster.dagster_ext import DagsterExtensionPlugin
+
+logger = get_logger(__name__)
+
+_WAP_RUN_ID_TAG = "phlo/run_id"
+_WAP_ATTEMPT_TAG = "phlo/attempt"
+_WAP_BRANCH_TAG = "phlo/wap_branch"
+_WAP_CATALOG_SYSTEM_TAG = "phlo/catalog_system"
+_PARTITION_TAG = "dagster/partition"
+
+_STATUS_OUTCOME = {
+    "SUCCESS": "success",
+    "FAILURE": "failure",
+    "CANCELED": "cancelled",
+}
+
+
+def _ts(epoch: Any) -> datetime | None:
+    try:
+        return datetime.fromtimestamp(float(epoch), UTC)
+    except (TypeError, ValueError, OverflowError):
+        return None
+
+
+def _emit_pipeline_run(context: Any, dagster_status: Any) -> None:
+    """Emit the terminal ``pipeline.run`` event for one Dagster run."""
+    # enabled(), not available(): a disabled runtime should skip the run-stats
+    # query entirely, not do the work and drop the event at emit().
+    if not phlo_observe.enabled():
+        return
+    try:
+        run = context.dagster_run
+        tags = dict(run.tags or {})
+        run_id = run.run_id
+        partition_key = tags.get(_PARTITION_TAG)
+        branch = tags.get(_WAP_BRANCH_TAG)
+        # DagsterRunStatus is a plain Enum: str() gives 'DagsterRunStatus.X',
+        # so normalize through .value to the bare status name.
+        status_key = str(getattr(dagster_status, "value", dagster_status))
+
+        entities: dict[str, Any] = {"run": phlo_observe.run_entity_for("dagster", run_id)}
+        if branch:
+            # The staging ref is owned by the catalog the launch resolved —
+            # Nessie for the branch strategy, the snapshot catalog (e.g.
+            # polaris) for the snapshot strategy.
+            entities["branch"] = phlo_observe.branch_entity_id(
+                branch, system=tags.get(_WAP_CATALOG_SYSTEM_TAG) or "nessie"
+            )
+
+        started_at = None
+        ended_at = None
+        duration_ms = None
+        try:
+            # DagsterEvent carries no timestamp; run stats hold the real
+            # launch/start/end times for the physical attempt.
+            stats = context.instance.get_run_stats(run_id)
+            started_at = _ts(getattr(stats, "start_time", None)) or _ts(
+                getattr(stats, "launch_time", None)
+            )
+            ended_at = _ts(getattr(stats, "end_time", None))
+            if started_at and ended_at:
+                duration_ms = (ended_at - started_at).total_seconds() * 1000.0
+        except Exception:  # noqa: BLE001 - timing is best-effort
+            pass
+
+        attributes: dict[str, Any] = {
+            "job_name": run.job_name,
+            "dagster_status": status_key,
+            "phlo_run_id": tags.get(_WAP_RUN_ID_TAG),
+            "root_run_id": run.root_run_id,
+            "parent_run_id": run.parent_run_id,
+        }
+        attempt = tags.get(_WAP_ATTEMPT_TAG)
+        if attempt:
+            attributes["attempt"] = attempt
+        selection = getattr(run, "asset_selection", None)
+        if selection:
+            attributes["asset_keys"] = sorted(str(key) for key in selection)
+
+        phlo_observe.emit(
+            "pipeline.run",
+            category="pipeline",
+            outcome=_STATUS_OUTCOME.get(status_key, "unknown"),
+            severity="error" if status_key == "FAILURE" else "info",
+            attributes=attributes,
+            correlation={
+                "run_id": run_id,
+                "job_id": run.job_name,
+                "partition_key": partition_key,
+                "branch": branch,
+            },
+            entities=entities or None,
+            producer="dagster",
+            started_at=started_at,
+            ended_at=ended_at,
+            duration_ms=duration_ms,
+        )
+    except Exception:  # noqa: BLE001 - telemetry must never fail a sensor tick
+        logger.debug("observe_run_status_emit_failed", exc_info=True)
+
+
+class ObserveDagsterExtension(DagsterExtensionPlugin):
+    """Contribute run-status sensors that close observer runs terminally."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for discovery and identification."""
+        return PluginMetadata(
+            name="observe",
+            version="0.1.0",
+            description="phlo-observe run-boundary sensors for Dagster",
+        )
+
+    def get_definitions(self) -> Any:
+        """Return the run-status sensors as Dagster definitions."""
+        import dagster as dg
+
+        def _sensor(status: dg.DagsterRunStatus, suffix: str) -> Any:
+            @dg.run_status_sensor(
+                run_status=status,
+                name=f"observe_run_{suffix}",
+                monitor_all_code_locations=True,
+                # run_status_sensor defaults to STOPPED: without RUNNING the
+                # terminal pipeline.run events this extension exists to emit
+                # would never fire in a deployed stack.
+                default_status=dg.DefaultSensorStatus.RUNNING,
+            )
+            def _observe_run(context: Any) -> None:
+                _emit_pipeline_run(context, status)
+
+            return _observe_run
+
+        return dg.Definitions(
+            sensors=[
+                _sensor(dg.DagsterRunStatus.SUCCESS, "success"),
+                _sensor(dg.DagsterRunStatus.FAILURE, "failure"),
+                _sensor(dg.DagsterRunStatus.CANCELED, "canceled"),
+            ]
+        )

phlo_observe_plugin/db-setup.yaml

@@ -0,0 +1,37 @@
+name: phlo-observer-db-setup
+description: Create the phlo-observer database on the shared PostgreSQL instance
+category: observability
+default: false
+profile: observability
+
+# Same pinned image as the postgres service so no extra pull is needed.
+image: postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15
+
+depends_on:
+  - postgres
+
+compose:
+  restart: "no"
+  entrypoint: /bin/sh
+  environment:
+    PGPASSWORD: ${POSTGRES_PASSWORD:-phlo}
+    PHLO_OBSERVER_DB: ${PHLO_OBSERVER_DB:-phlo_observer}
+  command:
+    # $$PHLO_OBSERVER_DB expands at container runtime, not compose-render
+    # time, so the value can never inject into this script's text; the case
+    # guard additionally restricts it to a safe SQL identifier.
+    - -c
+    - |
+      case "$$PHLO_OBSERVER_DB" in
+        *[!A-Za-z0-9_]*)
+          echo "phlo-observer-db-setup: PHLO_OBSERVER_DB must match [A-Za-z0-9_]+" >&2
+          exit 1
+          ;;
+      esac
+      psql "postgresql://${POSTGRES_USER:-phlo}@postgres:5432/postgres" \
+        -v ON_ERROR_STOP=1 -tc \
+        "SELECT 1 FROM pg_database WHERE datname = '$$PHLO_OBSERVER_DB'" \
+        | grep -q 1 \
+        || psql "postgresql://${POSTGRES_USER:-phlo}@postgres:5432/postgres" \
+          -v ON_ERROR_STOP=1 \
+          -c "CREATE DATABASE \"$$PHLO_OBSERVER_DB\""