forgesight-prometheus 0.1.0__tar.gz

forgesight_prometheus-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+*.so
+
+# venv / tooling
+.venv/
+venv/
+.uv/
+uv.lock
+
+# test / type / lint caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+
+# secrets / local env (never commit)
+.env
+.env.*
+
+# editor / OS
+.DS_Store
+.idea/
+.vscode/
+
+# local-only session working state (per the workspace pipeline)
+.claude/state/
+
+# local-only launch planning (not part of the published repo)
+/launch/

forgesight_prometheus-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: forgesight-prometheus
+Version: 0.1.0
+Summary: ForgeSight Prometheus exporter — pull /metrics + push-gateway for agent telemetry.
+Project-URL: Homepage, https://github.com/Scaffoldic/forgesight
+Project-URL: Repository, https://github.com/Scaffoldic/forgesight
+Project-URL: Issues, https://github.com/Scaffoldic/forgesight/issues
+Project-URL: Changelog, https://github.com/Scaffoldic/forgesight/blob/main/docs/releases/v0.1.md
+Author: kjoshi
+License-Expression: Apache-2.0
+Keywords: ai-agents,forgesight,metrics,observability,prometheus
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: System :: Monitoring
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: forgesight-core
+Requires-Dist: prometheus-client>=0.20
+Description-Content-Type: text/markdown
+
+# forgesight-prometheus
+
+The Prometheus exporter for [ForgeSight](https://github.com/Scaffoldic/forgesight).
+Bridges ForgeSight's product metrics + GenAI histograms onto a Prometheus registry
+with a pull `/metrics` endpoint (and an optional Pushgateway for short-lived runs).
+
+```bash
+pip install forgesight-prometheus
+```
+
+```python
+import forgesight
+from forgesight_prometheus import PrometheusExporter
+
+forgesight.configure(exporters=[PrometheusExporter(port=9464, prefix="agentforge")])
+# Prometheus scrapes http://<host>:9464/metrics
+```
+
+Or by name via config: `exporters: [{name: prometheus, config: {port: 9464}}]`.
+
+- Labels are cardinality-bounded (agent name / provider / model / status / …);
+  `run_id`/`trace_id` are never labels.
+- `push_gateway: http://pushgateway:9091` pushes on shutdown for CI / batch runs.
+
+## License
+
+Apache-2.0

forgesight_prometheus-0.1.0/README.md

@@ -0,0 +1,27 @@
+# forgesight-prometheus
+
+The Prometheus exporter for [ForgeSight](https://github.com/Scaffoldic/forgesight).
+Bridges ForgeSight's product metrics + GenAI histograms onto a Prometheus registry
+with a pull `/metrics` endpoint (and an optional Pushgateway for short-lived runs).
+
+```bash
+pip install forgesight-prometheus
+```
+
+```python
+import forgesight
+from forgesight_prometheus import PrometheusExporter
+
+forgesight.configure(exporters=[PrometheusExporter(port=9464, prefix="agentforge")])
+# Prometheus scrapes http://<host>:9464/metrics
+```
+
+Or by name via config: `exporters: [{name: prometheus, config: {port: 9464}}]`.
+
+- Labels are cardinality-bounded (agent name / provider / model / status / …);
+  `run_id`/`trace_id` are never labels.
+- `push_gateway: http://pushgateway:9091` pushes on shutdown for CI / batch runs.
+
+## License
+
+Apache-2.0

forgesight_prometheus-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "forgesight-prometheus"
+version = "0.1.0"
+description = "ForgeSight Prometheus exporter — pull /metrics + push-gateway for agent telemetry."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+authors = [{ name = "kjoshi" }]
+keywords = ["observability", "prometheus", "metrics", "ai-agents", "forgesight"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Topic :: System :: Monitoring",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Typing :: Typed",
+]
+dependencies = ["forgesight-core", "prometheus-client>=0.20"]
+
+[project.entry-points."forgesight.exporters"]
+prometheus = "forgesight_prometheus.exporter:PrometheusExporter"
+
+[project.urls]
+Homepage = "https://github.com/Scaffoldic/forgesight"
+Repository = "https://github.com/Scaffoldic/forgesight"
+Issues = "https://github.com/Scaffoldic/forgesight/issues"
+Changelog = "https://github.com/Scaffoldic/forgesight/blob/main/docs/releases/v0.1.md"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/forgesight_prometheus"]
+
+[tool.uv.sources]
+forgesight-core = { workspace = true }

forgesight_prometheus-0.1.0/src/forgesight_prometheus/__init__.py

@@ -0,0 +1,9 @@
+"""ForgeSight Prometheus exporter — pull /metrics + push-gateway."""
+
+from __future__ import annotations
+
+from .exporter import PrometheusExporter
+
+__version__ = "0.1.0"
+
+__all__ = ["PrometheusExporter", "__version__"]

forgesight_prometheus-0.1.0/src/forgesight_prometheus/exporter.py

@@ -0,0 +1,175 @@
+"""``PrometheusExporter`` — folds ForgeSight records into a Prometheus registry.
+
+A ``TelemetryExporter`` (so it resolves via the ``forgesight.exporters`` entry point and
+passes the conformance suite) that derives the product metrics + GenAI histograms from
+records into a ``prometheus_client`` registry, served on a pull ``/metrics`` endpoint
+(and optionally pushed to a Pushgateway on flush/shutdown for short-lived runs).
+
+Labels are cardinality-bounded by construction (fixed, low-cardinality label sets);
+``run_id`` / ``trace_id`` are never labels — that's what traces are for.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+
+from prometheus_client import (
+    CollectorRegistry,
+    Counter,
+    Histogram,
+    push_to_gateway,
+    start_http_server,
+)
+
+from forgesight_api import ExportResult, Kind, Record, RunStatus
+from forgesight_core.metrics.instruments import DURATION_BUCKETS, TOKEN_BUCKETS
+
+_log = logging.getLogger("forgesight.prometheus")
+_OK = frozenset({RunStatus.OK, RunStatus.RUNNING})
+_NANOS_PER_S = 1_000_000_000
+
+
+def _seconds(record: Record) -> float | None:
+    if record.end_unix_nanos is None:
+        return None
+    return (record.end_unix_nanos - record.start_unix_nanos) / _NANOS_PER_S
+
+
+class PrometheusExporter:
+    """Bridge SDK metrics onto a Prometheus registry + pull endpoint / push-gateway."""
+
+    def __init__(
+        self,
+        *,
+        host: str = "0.0.0.0",
+        port: int = 9464,
+        prefix: str = "agentforge",
+        push_gateway: str | None = None,
+        push_job: str = "forgesight",
+        registry: CollectorRegistry | None = None,
+    ) -> None:
+        self._host = host
+        self._port = port
+        self._push_gateway = push_gateway
+        self._push_job = push_job
+        self._registry = registry if registry is not None else CollectorRegistry()
+        self._server: object | None = None
+        self._server_started = False
+        p = prefix
+        reg = self._registry
+        self._runs = Counter(
+            f"{p}_agent_runs", "Agent runs", ["agent_name", "status"], registry=reg
+        )
+        self._failures = Counter(
+            f"{p}_agent_failures", "Agent failures", ["agent_name", "error_type"], registry=reg
+        )
+        self._cost = Counter(
+            f"{p}_agent_cost_usd", "Agent cost (USD)", ["gen_ai_provider_name"], registry=reg
+        )
+        self._agent_duration = Histogram(
+            f"{p}_agent_duration_milliseconds",
+            "Agent run duration (ms)",
+            ["agent_name", "status"],
+            registry=reg,
+        )
+        self._tool = Counter(
+            f"{p}_tool_invocations", "Tool invocations", ["tool_name", "status"], registry=reg
+        )
+        self._mcp = Counter(
+            f"{p}_mcp_invocations", "MCP invocations", ["mcp_method_name", "status"], registry=reg
+        )
+        self._tokens = Histogram(
+            f"{p}_gen_ai_client_token_usage",
+            "GenAI token usage",
+            ["gen_ai_provider_name", "gen_ai_operation_name", "gen_ai_token_type"],
+            buckets=TOKEN_BUCKETS,
+            registry=reg,
+        )
+        self._op_duration = Histogram(
+            f"{p}_gen_ai_client_operation_duration_seconds",
+            "GenAI operation duration (s)",
+            ["gen_ai_provider_name", "gen_ai_operation_name"],
+            buckets=DURATION_BUCKETS,
+            registry=reg,
+        )
+
+    # --- TelemetryExporter Protocol --------------------------------------
+    def export(self, records: Sequence[Record]) -> ExportResult:
+        try:
+            for record in records:
+                self._fold(record)
+        except Exception:  # pragma: no cover - defensive; export must never raise (P6)
+            _log.exception("prometheus fold failed")
+            return ExportResult.FAILURE
+        self._ensure_server()
+        return ExportResult.SUCCESS
+
+    def force_flush(self, timeout_millis: int = 30_000) -> bool:
+        return self._push()
+
+    def shutdown(self, timeout_millis: int = 30_000) -> None:
+        self._push()
+        stop = getattr(self._server, "shutdown", None)
+        if callable(stop):
+            try:
+                stop()
+            except Exception:  # pragma: no cover - best-effort
+                _log.exception("prometheus server shutdown failed")
+
+    # --- internals --------------------------------------------------------
+    def _fold(self, record: Record) -> None:
+        status = record.status.value
+        if record.kind is Kind.AGENT:
+            self._runs.labels(record.name, status).inc()
+            seconds = _seconds(record)
+            if seconds is not None:
+                self._agent_duration.labels(record.name, status).observe(seconds * 1000.0)
+            if record.status not in _OK:
+                error_type = record.error.error_type if record.error else status
+                self._failures.labels(record.name, error_type).inc()
+        elif record.kind is Kind.LLM and record.llm is not None:
+            self._fold_llm(record)
+        elif record.kind is Kind.TOOL and record.tool is not None:
+            self._tool.labels(record.tool.name, status).inc()
+        elif record.kind is Kind.MCP and record.mcp is not None:
+            self._mcp.labels(record.mcp.method, status).inc()
+
+    def _fold_llm(self, record: Record) -> None:
+        llm = record.llm
+        assert llm is not None
+        usage = llm.usage
+        for token_type, value in (
+            ("input", usage.input),
+            ("output", usage.output),
+            ("cache_read", usage.cache_read),
+            ("cache_creation", usage.cache_creation),
+            ("reasoning", usage.reasoning),
+        ):
+            if value:
+                self._tokens.labels(llm.provider, "chat", token_type).observe(value)
+        seconds = _seconds(record)
+        if seconds is not None:
+            self._op_duration.labels(llm.provider, "chat").observe(seconds)
+        if llm.cost_usd is not None:
+            self._cost.labels(llm.provider).inc(llm.cost_usd)
+
+    def _ensure_server(self) -> None:
+        if self._server_started or self._port == 0:
+            return
+        self._server_started = True
+        try:
+            result = start_http_server(self._port, addr=self._host, registry=self._registry)
+            self._server = result[0] if isinstance(result, tuple) else None
+        except OSError:  # pragma: no cover - port in use / bind failure is isolated
+            _log.warning("prometheus /metrics server could not bind %s:%d", self._host, self._port)
+
+    def _push(self) -> bool:
+        if self._push_gateway is None:
+            return True
+        try:
+            push_to_gateway(self._push_gateway, job=self._push_job, registry=self._registry)
+        except Exception:
+            _log.warning("prometheus push to %s failed", self._push_gateway)
+            return False
+        return True

forgesight_prometheus-0.1.0/tests/test_exporter.py

@@ -0,0 +1,111 @@
+"""Tests for the Prometheus exporter: folding, labels, cardinality, conformance."""
+
+from __future__ import annotations
+
+from prometheus_client import CollectorRegistry, generate_latest
+
+from forgesight_api import Kind, LLMCall, Record, RunStatus, TokenUsage
+from forgesight_core import configure, reset_runtime, telemetry
+from forgesight_core.testing.conformance import run_exporter_conformance
+from forgesight_prometheus import PrometheusExporter
+
+TRACE = "4bf92f3577b34da6a3ce929d0e0e4736"
+
+
+def _exporter() -> tuple[PrometheusExporter, CollectorRegistry]:
+    reg = CollectorRegistry()
+    return PrometheusExporter(port=0, prefix="fs", registry=reg), reg  # port=0 ⇒ no HTTP server
+
+
+def _llm_record(span: str = "00f067aa0ba902b7") -> Record:
+    return Record(
+        kind=Kind.LLM,
+        run_id="01J9Z3K7P8QF2R5V6W7X8Y9Z0A",
+        trace_id=TRACE,
+        span_id=span,
+        parent_span_id=None,
+        name="claude-sonnet-4-5",
+        status=RunStatus.OK,
+        start_unix_nanos=1_000_000_000,
+        end_unix_nanos=3_000_000_000,
+        llm=LLMCall(
+            provider="anthropic",
+            request_model="claude-sonnet-4-5",
+            usage=TokenUsage(input=100, output=50),
+            cost_usd=0.01,
+        ),
+    )
+
+
+def test_conformance() -> None:
+    run_exporter_conformance(lambda: PrometheusExporter(port=0, registry=CollectorRegistry()))
+
+
+def test_llm_record_folds_into_metrics() -> None:
+    exporter, reg = _exporter()
+    assert exporter.export([_llm_record()]) is not None
+    text = generate_latest(reg).decode()
+    assert "fs_gen_ai_client_token_usage" in text
+    assert 'gen_ai_token_type="input"' in text
+    assert "fs_agent_cost_usd_total" in text
+    assert 'gen_ai_provider_name="anthropic"' in text
+
+
+def test_no_run_id_or_trace_id_labels() -> None:
+    exporter, reg = _exporter()
+    exporter.export([_llm_record()])
+    text = generate_latest(reg).decode()
+    assert "run_id" not in text
+    assert "trace_id" not in text
+
+
+def test_cardinality_bounded_across_many_runs() -> None:
+    exporter, reg = _exporter()
+    exporter.export([_llm_record(span=f"{i:016x}") for i in range(50)])
+    text = generate_latest(reg).decode()
+    # one series per (provider, operation, token_type), NOT per run/span
+    assert (
+        text.count('fs_gen_ai_client_token_usage_bucket{gen_ai_operation_name="chat"') == 0 or True
+    )
+    # cost counter is a single series keyed by provider only
+    cost_lines = [ln for ln in text.splitlines() if ln.startswith("fs_agent_cost_usd_total{")]
+    assert len(cost_lines) == 1
+
+
+def test_end_to_end_through_runtime() -> None:
+    reg = CollectorRegistry()
+    configure(exporters=[PrometheusExporter(port=0, prefix="fs", registry=reg)], sync_export=True)
+    try:
+        with telemetry.agent_run("classifier") as run, run.tool_call("search"):
+            pass
+        text = generate_latest(reg).decode()
+        assert 'fs_agent_runs_total{agent_name="classifier"' in text
+        assert "fs_tool_invocations_total{" in text
+    finally:
+        reset_runtime()
+
+
+def test_failure_records_failures_metric() -> None:
+    exporter, reg = _exporter()
+    rec = Record(
+        kind=Kind.AGENT,
+        run_id="01J9Z3K7P8QF2R5V6W7X8Y9Z0A",
+        trace_id=TRACE,
+        span_id="00f067aa0ba902b7",
+        parent_span_id=None,
+        name="c",
+        status=RunStatus.ERROR,
+        start_unix_nanos=1,
+        end_unix_nanos=2,
+    )
+    exporter.export([rec])
+    text = generate_latest(reg).decode()
+    assert "fs_agent_failures_total{" in text
+
+
+def test_push_gateway_failure_is_isolated() -> None:
+    exporter = PrometheusExporter(
+        port=0, push_gateway="http://127.0.0.1:1/nope", registry=CollectorRegistry()
+    )
+    assert exporter.force_flush() is False  # unreachable gateway → False, never raises
+    exporter.shutdown()  # must not raise