phlo-observe-plugin 0.2.0__tar.gz

phlo_observe_plugin-0.2.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: phlo-observe-plugin
+Version: 0.2.0
+Summary: phlo-observe SDK integration and phlo-observer service for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/plain
+Requires-Dist: phlo<0.18,>=0.17.0
+Requires-Dist: phlo-dagster>=0.3.2
+Requires-Dist: phlo-observe<0.3,>=0.2.3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+Canonical event emission and observer service wiring for Phlo.

phlo_observe_plugin-0.2.0/README.md

@@ -0,0 +1,50 @@
+# phlo-observe-plugin
+
+Canonical `phlo-observe` integration for Phlo.
+
+The package installs the published `phlo-observe` SDK and its core and query
+dependencies from PyPI.
+
+## What it provides
+
+- A hook plugin that translates pipeline events into canonical logs, metrics,
+  and operation events while preserving correlation.
+- A Dagster extension that keeps physical execution IDs distinct from logical
+  retry-chain IDs.
+- An opt-in `phlo-observer` service for ingest and query. ClickStack remains
+  Phlo's default observability backend.
+
+Observability is fail-open: unavailable drains or observer services never stop
+pipeline work.
+
+## Use
+
+Install the package, then enable the observability profile when a standalone
+observer is wanted:
+
+```bash
+pip install phlo-observe-plugin
+phlo services init --profile observability
+phlo services start
+phlo services start --profile observability
+```
+
+Set `OBSERVE_HTTP_ENDPOINT` to the observer's `/v1/events` endpoint. For local
+development, the service defaults to unauthenticated ingest. For hardened
+deployments, configure matching `OBSERVE_HTTP_TOKEN` and
+`PHLO_OBSERVER_INGEST_TOKENS` values.
+
+The observability profile enables the concise terminal drain. Set
+`PHLO_OBSERVE_PRETTY_VERBOSE=true` to include secondary diagnostic events and
+the full framework log stream. Set `PHLO_OBSERVE_PRETTY=false` to keep
+canonical events out of the terminal. Store either setting in the top-level
+`env` block in `phlo.yaml`, then regenerate the service configuration:
+
+```yaml
+env:
+  PHLO_OBSERVE_PRETTY: "true"
+  PHLO_OBSERVE_PRETTY_VERBOSE: "false"
+```
+
+The existing hook bus API remains compatible. The plugin translates its
+events rather than replacing them.

phlo_observe_plugin-0.2.0/pyproject.toml

@@ -0,0 +1,60 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [
+    "setuptools>=45",
+    "wheel",
+]
+
+[project]
+dependencies = [
+    "phlo>=0.17.0,<0.18",
+    "phlo-dagster>=0.3.2",
+    "phlo-observe>=0.2.3,<0.3",
+]
+description = "phlo-observe SDK integration and phlo-observer service for Phlo"
+name = "phlo-observe-plugin"
+requires-python = ">=3.12"
+version = "0.2.0"
+
+[[project.authors]]
+email = "team@phlo.dev"
+name = "Phlo Team"
+
+[project.entry-points."phlo.plugins.dagster"]
+observe = "phlo_observe_plugin.dagster_ext:ObserveDagsterExtension"
+
+[project.entry-points."phlo.plugins.hooks"]
+observe = "phlo_observe_plugin.hooks_plugin:ObserveHookPlugin"
+
+[project.entry-points."phlo.plugins.services"]
+phlo-observer = "phlo_observe_plugin.plugin:PhloObserverServicePlugin"
+phlo-observer-db-setup = "phlo_observe_plugin.plugin:PhloObserverDbSetupPlugin"
+
+[project.license]
+text = "MIT"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.readme]
+content-type = "text/plain"
+text = "Canonical event emission and observer service wiring for Phlo."
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.setuptools.package-data]
+phlo_observe_plugin = [
+    "service.yaml",
+    "db-setup.yaml",
+]
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools.packages.find]
+where = ["src"]

phlo_observe_plugin-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_observe_plugin-0.2.0/src/phlo_observe_plugin/__init__.py

@@ -0,0 +1,42 @@
+"""phlo-observe integration for Phlo.
+
+- ``ObserveHookPlugin`` translates hook-bus events into canonical
+  ``phlo-observe`` events.
+- ``ObserveDagsterExtension`` contributes a run-status sensor that closes each
+  physical Dagster run with a terminal ``pipeline.run`` event.
+- ``PhloObserverServicePlugin`` / ``PhloObserverDbSetupPlugin`` declare the
+  observer compose service and its database provisioning step.
+- ``presentation`` carries Phlo's ``PrettyRenderer`` rules and the ``pretty``
+  drain — imported lazily since it needs a PrettyRenderer-capable SDK.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "ObserveDagsterExtension",
+    "ObserveHookPlugin",
+    "PhloObserverDbSetupPlugin",
+    "PhloObserverServicePlugin",
+    "PrettyDrain",
+    "pretty_renderer",
+]
+
+
+def __getattr__(name: str) -> object:
+    if name == "ObserveHookPlugin":
+        from phlo_observe_plugin.hooks_plugin import ObserveHookPlugin
+
+        return ObserveHookPlugin
+    if name == "ObserveDagsterExtension":
+        from phlo_observe_plugin.dagster_ext import ObserveDagsterExtension
+
+        return ObserveDagsterExtension
+    if name in {"PhloObserverServicePlugin", "PhloObserverDbSetupPlugin"}:
+        from phlo_observe_plugin import plugin as _plugin
+
+        return getattr(_plugin, name)
+    if name in {"PrettyDrain", "pretty_renderer"}:
+        from phlo_observe_plugin import presentation as _presentation
+
+        return getattr(_presentation, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

phlo_observe_plugin-0.2.0/src/phlo_observe_plugin/dagster_ext.py

@@ -0,0 +1,161 @@
+"""Dagster extension emitting terminal ``pipeline.run`` events per run.
+
+Asset/step/check events are emitted inside the run worker by the adapter
+instrumentation; this extension adds the run-boundary event. Run-status
+sensors evaluate in the Dagster daemon, so the terminal ``pipeline.run`` is
+emitted with explicit correlation rather than ambient context.
+
+Run identity is the *physical* Dagster ``run_id`` — one observer run per
+attempt. The logical Phlo run id (``phlo/run_id`` tag) and attempt number are
+carried as attributes so retries group logically without violating the
+observer's monotonic run-status precedence (a failed attempt must not pin a
+later successful attempt to "failed").
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+import phlo.telemetry as phlo_observe
+from phlo.logging import get_logger
+from phlo.plugins.base import PluginMetadata
+from phlo_dagster.dagster_ext import DagsterExtensionPlugin
+
+logger = get_logger(__name__)
+
+_WAP_RUN_ID_TAG = "phlo/run_id"
+_WAP_ATTEMPT_TAG = "phlo/attempt"
+_WAP_BRANCH_TAG = "phlo/wap_branch"
+_WAP_CATALOG_SYSTEM_TAG = "phlo/catalog_system"
+_PARTITION_TAG = "dagster/partition"
+
+_STATUS_OUTCOME = {
+    "SUCCESS": "success",
+    "FAILURE": "failure",
+    "CANCELED": "cancelled",
+}
+
+
+def _ts(epoch: Any) -> datetime | None:
+    try:
+        return datetime.fromtimestamp(float(epoch), UTC)
+    except (TypeError, ValueError, OverflowError):
+        return None
+
+
+def _emit_pipeline_run(context: Any, dagster_status: Any) -> None:
+    """Emit the terminal ``pipeline.run`` event for one Dagster run."""
+    # enabled(), not available(): a disabled runtime should skip the run-stats
+    # query entirely, not do the work and drop the event at emit().
+    if not phlo_observe.enabled():
+        return
+    try:
+        run = context.dagster_run
+        tags = dict(run.tags or {})
+        run_id = run.run_id
+        partition_key = tags.get(_PARTITION_TAG)
+        branch = tags.get(_WAP_BRANCH_TAG)
+        # DagsterRunStatus is a plain Enum: str() gives 'DagsterRunStatus.X',
+        # so normalize through .value to the bare status name.
+        status_key = str(getattr(dagster_status, "value", dagster_status))
+
+        entities: dict[str, Any] = {"run": phlo_observe.run_entity_for("dagster", run_id)}
+        if branch:
+            # The staging ref is owned by the catalog the launch resolved —
+            # Nessie for the branch strategy, the snapshot catalog (e.g.
+            # polaris) for the snapshot strategy.
+            entities["branch"] = phlo_observe.branch_entity_id(
+                branch, system=tags.get(_WAP_CATALOG_SYSTEM_TAG) or "nessie"
+            )
+
+        started_at = None
+        ended_at = None
+        duration_ms = None
+        try:
+            # DagsterEvent carries no timestamp; run stats hold the real
+            # launch/start/end times for the physical attempt.
+            stats = context.instance.get_run_stats(run_id)
+            started_at = _ts(getattr(stats, "start_time", None)) or _ts(
+                getattr(stats, "launch_time", None)
+            )
+            ended_at = _ts(getattr(stats, "end_time", None))
+            if started_at and ended_at:
+                duration_ms = (ended_at - started_at).total_seconds() * 1000.0
+        except Exception:  # noqa: BLE001 - timing is best-effort
+            pass
+
+        attributes: dict[str, Any] = {
+            "job_name": run.job_name,
+            "dagster_status": status_key,
+            "phlo_run_id": tags.get(_WAP_RUN_ID_TAG),
+            "root_run_id": run.root_run_id,
+            "parent_run_id": run.parent_run_id,
+        }
+        attempt = tags.get(_WAP_ATTEMPT_TAG)
+        if attempt:
+            attributes["attempt"] = attempt
+        selection = getattr(run, "asset_selection", None)
+        if selection:
+            attributes["asset_keys"] = sorted(str(key) for key in selection)
+
+        phlo_observe.emit(
+            "pipeline.run",
+            category="pipeline",
+            outcome=_STATUS_OUTCOME.get(status_key, "unknown"),
+            severity="error" if status_key == "FAILURE" else "info",
+            attributes=attributes,
+            correlation={
+                "run_id": run_id,
+                "job_id": run.job_name,
+                "partition_key": partition_key,
+                "branch": branch,
+            },
+            entities=entities or None,
+            producer="dagster",
+            started_at=started_at,
+            ended_at=ended_at,
+            duration_ms=duration_ms,
+        )
+    except Exception:  # noqa: BLE001 - telemetry must never fail a sensor tick
+        logger.debug("observe_run_status_emit_failed", exc_info=True)
+
+
+class ObserveDagsterExtension(DagsterExtensionPlugin):
+    """Contribute run-status sensors that close observer runs terminally."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for discovery and identification."""
+        return PluginMetadata(
+            name="observe",
+            version="0.1.0",
+            description="phlo-observe run-boundary sensors for Dagster",
+        )
+
+    def get_definitions(self) -> Any:
+        """Return the run-status sensors as Dagster definitions."""
+        import dagster as dg
+
+        def _sensor(status: dg.DagsterRunStatus, suffix: str) -> Any:
+            @dg.run_status_sensor(
+                run_status=status,
+                name=f"observe_run_{suffix}",
+                monitor_all_code_locations=True,
+                # run_status_sensor defaults to STOPPED: without RUNNING the
+                # terminal pipeline.run events this extension exists to emit
+                # would never fire in a deployed stack.
+                default_status=dg.DefaultSensorStatus.RUNNING,
+            )
+            def _observe_run(context: Any) -> None:
+                _emit_pipeline_run(context, status)
+
+            return _observe_run
+
+        return dg.Definitions(
+            sensors=[
+                _sensor(dg.DagsterRunStatus.SUCCESS, "success"),
+                _sensor(dg.DagsterRunStatus.FAILURE, "failure"),
+                _sensor(dg.DagsterRunStatus.CANCELED, "canceled"),
+            ]
+        )

phlo_observe_plugin-0.2.0/src/phlo_observe_plugin/db-setup.yaml

@@ -0,0 +1,37 @@
+name: phlo-observer-db-setup
+description: Create the phlo-observer database on the shared PostgreSQL instance
+category: observability
+default: false
+profile: observability
+
+# Same pinned image as the postgres service so no extra pull is needed.
+image: postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15
+
+depends_on:
+  - postgres
+
+compose:
+  restart: "no"
+  entrypoint: /bin/sh
+  environment:
+    PGPASSWORD: ${POSTGRES_PASSWORD:-phlo}
+    PHLO_OBSERVER_DB: ${PHLO_OBSERVER_DB:-phlo_observer}
+  command:
+    # $$PHLO_OBSERVER_DB expands at container runtime, not compose-render
+    # time, so the value can never inject into this script's text; the case
+    # guard additionally restricts it to a safe SQL identifier.
+    - -c
+    - |
+      case "$$PHLO_OBSERVER_DB" in
+        *[!A-Za-z0-9_]*)
+          echo "phlo-observer-db-setup: PHLO_OBSERVER_DB must match [A-Za-z0-9_]+" >&2
+          exit 1
+          ;;
+      esac
+      psql "postgresql://${POSTGRES_USER:-phlo}@postgres:5432/postgres" \
+        -v ON_ERROR_STOP=1 -tc \
+        "SELECT 1 FROM pg_database WHERE datname = '$$PHLO_OBSERVER_DB'" \
+        | grep -q 1 \
+        || psql "postgresql://${POSTGRES_USER:-phlo}@postgres:5432/postgres" \
+          -v ON_ERROR_STOP=1 \
+          -c "CREATE DATABASE \"$$PHLO_OBSERVER_DB\""