agentship-observability 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentship_observability-0.0.1/.gitignore +27 -0
- agentship_observability-0.0.1/PKG-INFO +56 -0
- agentship_observability-0.0.1/README.md +40 -0
- agentship_observability-0.0.1/pyproject.toml +37 -0
- agentship_observability-0.0.1/src/agentship_observability/__init__.py +14 -0
- agentship_observability-0.0.1/src/agentship_observability/config.py +57 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/__init__.py +13 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/console.py +24 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/factory.py +38 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/langfuse.py +41 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/langsmith.py +32 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/opik.py +43 -0
- agentship_observability-0.0.1/src/agentship_observability/exporters/phoenix.py +32 -0
- agentship_observability-0.0.1/src/agentship_observability/factory.py +122 -0
- agentship_observability-0.0.1/src/agentship_observability/litellm_logger.py +152 -0
- agentship_observability-0.0.1/src/agentship_observability/otel/__init__.py +8 -0
- agentship_observability-0.0.1/src/agentship_observability/otel/kinds.py +26 -0
- agentship_observability-0.0.1/src/agentship_observability/otel/observer.py +224 -0
- agentship_observability-0.0.1/tests/test_config.py +50 -0
- agentship_observability-0.0.1/tests/test_exporters.py +92 -0
- agentship_observability-0.0.1/tests/test_factory.py +92 -0
- agentship_observability-0.0.1/tests/test_litellm_logger.py +119 -0
- agentship_observability-0.0.1/tests/test_observer_parity.py +162 -0
- agentship_observability-0.0.1/tests/test_otel_observer.py +144 -0
- agentship_observability-0.0.1/tests/test_replay_capture.py +87 -0
- agentship_observability-0.0.1/tests/test_semconv_upstream.py +49 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
.pytest_cache/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
.mypy_cache/
|
|
13
|
+
.coverage
|
|
14
|
+
htmlcov/
|
|
15
|
+
|
|
16
|
+
# Env / secrets — never commit
|
|
17
|
+
.env
|
|
18
|
+
.env.*
|
|
19
|
+
!.env.example
|
|
20
|
+
|
|
21
|
+
# Editor / OS
|
|
22
|
+
.DS_Store
|
|
23
|
+
.idea/
|
|
24
|
+
.vscode/
|
|
25
|
+
|
|
26
|
+
# Docs build
|
|
27
|
+
site/
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agentship-observability
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: AgentShip observability — the OpenTelemetry implementation of the Observer port: span-tree exporter pipeline (Phoenix/Langfuse/LangSmith), the one-per-process LiteLLM cost/token callback, and the studio launcher. The vendor-free contract lives in agentship-core.
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.13
|
|
7
|
+
Requires-Dist: agentship-core==0.0.1
|
|
8
|
+
Requires-Dist: litellm>=1.74
|
|
9
|
+
Requires-Dist: opentelemetry-api>=1.27
|
|
10
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27
|
|
11
|
+
Requires-Dist: opentelemetry-sdk>=1.27
|
|
12
|
+
Provides-Extra: phoenix
|
|
13
|
+
Requires-Dist: arize-phoenix-otel>=0.6; extra == 'phoenix'
|
|
14
|
+
Requires-Dist: openinference-semantic-conventions>=0.1; extra == 'phoenix'
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# agentship-observability
|
|
18
|
+
|
|
19
|
+
The OpenTelemetry implementation of AgentShip's `Observer` port (Phase 07).
|
|
20
|
+
|
|
21
|
+
The **contract** — the `Observer`/`Span` port, `SpanKind`/`Usage`, the frozen SEMCONV span-name and
|
|
22
|
+
attribute-key constants, and the `TraceView` read-port — lives in **`agentship-core`** with no OTel
|
|
23
|
+
dependency, so an engine or eval hook can read a trace with only the kernel installed (design §4.6).
|
|
24
|
+
|
|
25
|
+
This package holds only the **pipeline**:
|
|
26
|
+
|
|
27
|
+
- `otel/observer.py` — `OTelObserver`, the concrete port over a `TracerProvider`: it builds the
|
|
28
|
+
canonical span tree (§4.1), sets GenAI attributes (§4.2), and rolls up cost/tokens/latency.
|
|
29
|
+
- `exporters/` — one `SpanProcessor` per backend (`console`, `phoenix`, `langfuse`, `langsmith`),
|
|
30
|
+
all reached over OTLP/HTTP (no per-vendor SDK lock-in).
|
|
31
|
+
- `litellm_callback.py` — one process-global `CustomLogger` that stamps cost/tokens/latency onto the
|
|
32
|
+
current `model` span, so every LiteLLM caller is covered without per-call wiring.
|
|
33
|
+
- `config.py` + `factory.py` — the `observability:` config surface and the process-global
|
|
34
|
+
`TracerProvider` the factory memoises; `provider: otel` (default) → `OTelObserver`, `none` → NoOp.
|
|
35
|
+
- `capture.py` — the record/replay hook that stamps `agentship.replay.request_hash` (+ gated
|
|
36
|
+
request/response) so P12 can build deterministic cassettes.
|
|
37
|
+
- `studio.py` — `generate_langgraph_json` + the loopback-only, dev-token studio launcher.
|
|
38
|
+
|
|
39
|
+
Everything a price table, trace store, or UI would provide is **consumed** (Phoenix/Langfuse/
|
|
40
|
+
LangSmith + LangGraph Studio); we build only the composition.
|
|
41
|
+
|
|
42
|
+
## Conformance guards (we keep thin, but prove it conforms)
|
|
43
|
+
|
|
44
|
+
Two pieces are deliberately vendor-free — the semantic-convention key constants and the
|
|
45
|
+
`RecordingObserver` — because the kernel cannot import OpenTelemetry (design §4.6) and the base
|
|
46
|
+
install ships without a collector. To keep them from silently drifting from the real thing, each is
|
|
47
|
+
continuously proven against upstream:
|
|
48
|
+
|
|
49
|
+
- `tests/test_semconv_upstream.py` asserts every `gen_ai.*` / `llm.*` key equals the OTel-GenAI /
|
|
50
|
+
OpenInference string it claims to speak.
|
|
51
|
+
- `tests/test_observer_parity.py` runs one interaction through both the in-memory `RecordingObserver`
|
|
52
|
+
and the real `OTelObserver`, and asserts the two produce the same span tree and model-span
|
|
53
|
+
attributes.
|
|
54
|
+
|
|
55
|
+
See [`docs/decisions/0001-integrate-not-invent.md`](../../docs/decisions/0001-integrate-not-invent.md)
|
|
56
|
+
for the full rationale.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# agentship-observability
|
|
2
|
+
|
|
3
|
+
The OpenTelemetry implementation of AgentShip's `Observer` port (Phase 07).
|
|
4
|
+
|
|
5
|
+
The **contract** — the `Observer`/`Span` port, `SpanKind`/`Usage`, the frozen SEMCONV span-name and
|
|
6
|
+
attribute-key constants, and the `TraceView` read-port — lives in **`agentship-core`** with no OTel
|
|
7
|
+
dependency, so an engine or eval hook can read a trace with only the kernel installed (design §4.6).
|
|
8
|
+
|
|
9
|
+
This package holds only the **pipeline**:
|
|
10
|
+
|
|
11
|
+
- `otel/observer.py` — `OTelObserver`, the concrete port over a `TracerProvider`: it builds the
|
|
12
|
+
canonical span tree (§4.1), sets GenAI attributes (§4.2), and rolls up cost/tokens/latency.
|
|
13
|
+
- `exporters/` — one `SpanProcessor` per backend (`console`, `phoenix`, `langfuse`, `langsmith`),
|
|
14
|
+
all reached over OTLP/HTTP (no per-vendor SDK lock-in).
|
|
15
|
+
- `litellm_callback.py` — one process-global `CustomLogger` that stamps cost/tokens/latency onto the
|
|
16
|
+
current `model` span, so every LiteLLM caller is covered without per-call wiring.
|
|
17
|
+
- `config.py` + `factory.py` — the `observability:` config surface and the process-global
|
|
18
|
+
`TracerProvider` the factory memoises; `provider: otel` (default) → `OTelObserver`, `none` → NoOp.
|
|
19
|
+
- `capture.py` — the record/replay hook that stamps `agentship.replay.request_hash` (+ gated
|
|
20
|
+
request/response) so P12 can build deterministic cassettes.
|
|
21
|
+
- `studio.py` — `generate_langgraph_json` + the loopback-only, dev-token studio launcher.
|
|
22
|
+
|
|
23
|
+
Everything a price table, trace store, or UI would provide is **consumed** (Phoenix/Langfuse/
|
|
24
|
+
LangSmith + LangGraph Studio); we build only the composition.
|
|
25
|
+
|
|
26
|
+
## Conformance guards (we keep thin, but prove it conforms)
|
|
27
|
+
|
|
28
|
+
Two pieces are deliberately vendor-free — the semantic-convention key constants and the
|
|
29
|
+
`RecordingObserver` — because the kernel cannot import OpenTelemetry (design §4.6) and the base
|
|
30
|
+
install ships without a collector. To keep them from silently drifting from the real thing, each is
|
|
31
|
+
continuously proven against upstream:
|
|
32
|
+
|
|
33
|
+
- `tests/test_semconv_upstream.py` asserts every `gen_ai.*` / `llm.*` key equals the OTel-GenAI /
|
|
34
|
+
OpenInference string it claims to speak.
|
|
35
|
+
- `tests/test_observer_parity.py` runs one interaction through both the in-memory `RecordingObserver`
|
|
36
|
+
and the real `OTelObserver`, and asserts the two produce the same span tree and model-span
|
|
37
|
+
attributes.
|
|
38
|
+
|
|
39
|
+
See [`docs/decisions/0001-integrate-not-invent.md`](../../docs/decisions/0001-integrate-not-invent.md)
|
|
40
|
+
for the full rationale.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "agentship-observability"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
description = "AgentShip observability — the OpenTelemetry implementation of the Observer port: span-tree exporter pipeline (Phoenix/Langfuse/LangSmith), the one-per-process LiteLLM cost/token callback, and the studio launcher. The vendor-free contract lives in agentship-core."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.13"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
# The tracing SDK + OTLP/HTTP export are the only hard deps: every backend
|
|
13
|
+
# (Phoenix, Langfuse, LangSmith) is reached over OTLP/HTTP, so there is no
|
|
14
|
+
# per-vendor SDK lock-in here (design §5 — we consume, we do not reinvent).
|
|
15
|
+
dependencies = [
|
|
16
|
+
"agentship-core==0.0.1",
|
|
17
|
+
"opentelemetry-api>=1.27",
|
|
18
|
+
"opentelemetry-sdk>=1.27",
|
|
19
|
+
"opentelemetry-exporter-otlp-proto-http>=1.27",
|
|
20
|
+
# The one success callback that turns each model call into span usage runs off
|
|
21
|
+
# LiteLLM's CustomLogger; every model call in AgentShip already routes through it.
|
|
22
|
+
"litellm>=1.74",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
# The OpenInference semantic-convention keys + Phoenix's one-command OTel wiring.
|
|
27
|
+
# The exporter works over plain OTLP without these; they add the attribute mirrors
|
|
28
|
+
# Phoenix's cost panel reads. Optional so a Langfuse/LangSmith user need not pull them.
|
|
29
|
+
phoenix = ["arize-phoenix-otel>=0.6", "openinference-semantic-conventions>=0.1"]
|
|
30
|
+
|
|
31
|
+
# The default OTel observer registers here so the factory discovers it by name with
|
|
32
|
+
# no core edit — mirroring how engines and auth providers are found (entry points).
|
|
33
|
+
[project.entry-points."agentship.observers"]
|
|
34
|
+
otel = "agentship_observability.factory:build_otel_observer"
|
|
35
|
+
|
|
36
|
+
[tool.hatch.build.targets.wheel]
|
|
37
|
+
packages = ["src/agentship_observability"]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""AgentShip observability — the OpenTelemetry pipeline behind the vendor-free ``Observer`` port.
|
|
2
|
+
|
|
3
|
+
The contract (port, ``SpanKind``/``Usage``, SEMCONV, ``TraceView``) lives in ``agentship-core``;
|
|
4
|
+
this package supplies the concrete OTel ``OTelObserver``, the
|
|
5
|
+
exporter pipeline, the one-per-process LiteLLM cost/token callback, the record/replay capture hook,
|
|
6
|
+
and the studio launcher (design §4.6). Import ``OTelObserver`` here; build a configured one with
|
|
7
|
+
:func:`~agentship_observability.factory.build_otel_observer`.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .otel import OTelObserver
|
|
13
|
+
|
|
14
|
+
__all__ = ["OTelObserver"]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""``ObservabilityConfig`` — the ``observability:`` config surface (C5).
|
|
2
|
+
|
|
3
|
+
One small Pydantic model parses the ``observability:`` block from an agent's YAML (secrets come from
|
|
4
|
+
the environment, never the file). Defaults are the safe ones: tracing on, the OTel provider, the
|
|
5
|
+
``console`` exporter (which needs no running backend), content capture **off** (the PHI gate), user
|
|
6
|
+
ids hashed, full sampling, and SaaS exporters blocked. Phoenix is the recommended production OSS
|
|
7
|
+
exporter — a one-line ``exporters: [phoenix]`` swap — but is not the zero-config default because it
|
|
8
|
+
needs a collector to point at.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, Field, field_validator
|
|
16
|
+
|
|
17
|
+
#: The exporter names the factory knows how to build. Extend here + in ``exporters/factory.py``.
|
|
18
|
+
ExporterName = Literal["console", "phoenix", "langfuse", "langsmith", "opik"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ObservabilityConfig(BaseModel):
|
|
22
|
+
"""Declarative tracing config for one agent; the factory turns it into an ``Observer``.
|
|
23
|
+
|
|
24
|
+
``provider`` selects the implementation: ``otel`` (the default) builds an ``OTelObserver`` over
|
|
25
|
+
a process-global ``TracerProvider``; ``none`` yields the no-op observer so a run is untraced but
|
|
26
|
+
unchanged. ``exporters`` is a list so a span tree can fan out to several backends at once.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
model_config = {"extra": "forbid"}
|
|
30
|
+
|
|
31
|
+
#: Master switch. When false the factory yields the no-op observer regardless of ``provider``.
|
|
32
|
+
enabled: bool = True
|
|
33
|
+
#: The observer implementation: ``otel`` (default) or ``none`` (no-op).
|
|
34
|
+
provider: Literal["otel", "none"] = "otel"
|
|
35
|
+
#: Backends to export to; every span tree is sent to each. Default console (needs no backend).
|
|
36
|
+
exporters: list[ExporterName] = Field(default_factory=lambda: ["console"])
|
|
37
|
+
#: PHI gate — when false, prompt/response content is never put on a span (§4.6). Default off.
|
|
38
|
+
capture_content: bool = False
|
|
39
|
+
#: Hash the caller's user id on spans so a raw id never lands in a trace store. Default on.
|
|
40
|
+
hash_user_id: bool = True
|
|
41
|
+
#: Parent-based sampling ratio, 0.0–1.0. 1.0 keeps every trace; lower drops a fraction.
|
|
42
|
+
sample_ratio: float = 1.0
|
|
43
|
+
#: Allow a SaaS exporter (LangSmith) to receive content-bearing spans. Off in the PHI profile.
|
|
44
|
+
allow_saas_exporter: bool = False
|
|
45
|
+
|
|
46
|
+
@field_validator("sample_ratio")
|
|
47
|
+
@classmethod
|
|
48
|
+
def _ratio_in_range(cls, value: float) -> float:
|
|
49
|
+
"""Reject a sampling ratio outside 0.0–1.0 — a typo here silently loses or floods traces."""
|
|
50
|
+
if not 0.0 <= value <= 1.0:
|
|
51
|
+
raise ValueError(f"sample_ratio must be between 0.0 and 1.0, got {value}")
|
|
52
|
+
return value
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def uses_saas_exporter(self) -> bool:
|
|
56
|
+
"""Whether any configured exporter ships spans to a SaaS backend (currently LangSmith)."""
|
|
57
|
+
return "langsmith" in self.exporters
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Exporter builders — each turns a configured backend name into an OTel ``SpanProcessor``.
|
|
2
|
+
|
|
3
|
+
One module per backend (console, phoenix, langfuse, langsmith, opik); :func:`build_processor` in
|
|
4
|
+
``factory`` dispatches a config's ``exporters`` list to them. Every builder returns a ready
|
|
5
|
+
``SpanProcessor`` the tracer provider can register, so adding a backend is a one-file change plus a
|
|
6
|
+
line in the dispatch table.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from .factory import build_processor
|
|
12
|
+
|
|
13
|
+
__all__ = ["build_processor"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Console exporter — prints finished spans to stderr. The zero-backend default (§C4).
|
|
2
|
+
|
|
3
|
+
Needs nothing running, so it is the safe default: a dev can see the span tree immediately and an
|
|
4
|
+
offline CI run never fails on a missing collector. Uses a ``SimpleSpanProcessor`` (synchronous
|
|
5
|
+
export) because console output is cheap and immediate feedback beats batching here.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor, SpanProcessor
|
|
13
|
+
|
|
14
|
+
from ..config import ObservabilityConfig
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build(config: ObservabilityConfig) -> SpanProcessor:
|
|
18
|
+
"""Build a synchronous processor that prints each finished span to stderr.
|
|
19
|
+
|
|
20
|
+
Spans go to **stderr**, not stdout: the CLI reserves stdout for the agent's answer so a piped
|
|
21
|
+
run is never polluted by trace output. ``ConsoleSpanExporter`` defaults to stdout, so the target
|
|
22
|
+
stream is set explicitly here.
|
|
23
|
+
"""
|
|
24
|
+
return SimpleSpanProcessor(ConsoleSpanExporter(out=sys.stderr))
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Dispatch a config's ``exporters`` list to per-backend processor builders (§C4).
|
|
2
|
+
|
|
3
|
+
One tiny lookup table maps each exporter name to its module's ``build(config)``. Adding a backend is
|
|
4
|
+
a one-line change here plus its module. An unknown name is a fail-fast :class:`CapabilityError` with
|
|
5
|
+
the known names listed, so a typo in YAML surfaces at build time, not as silent missing traces.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
|
|
12
|
+
from agentship.errors import CapabilityError
|
|
13
|
+
from opentelemetry.sdk.trace.export import SpanProcessor
|
|
14
|
+
|
|
15
|
+
from ..config import ObservabilityConfig
|
|
16
|
+
from . import console, langfuse, langsmith, opik, phoenix
|
|
17
|
+
|
|
18
|
+
#: Exporter name → builder. Keys must match ``config.ExporterName``.
|
|
19
|
+
_BUILDERS: dict[str, Callable[[ObservabilityConfig], SpanProcessor]] = {
|
|
20
|
+
"console": console.build,
|
|
21
|
+
"phoenix": phoenix.build,
|
|
22
|
+
"langfuse": langfuse.build,
|
|
23
|
+
"langsmith": langsmith.build,
|
|
24
|
+
"opik": opik.build,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_processor(name: str, config: ObservabilityConfig) -> SpanProcessor:
|
|
29
|
+
"""Build the ``SpanProcessor`` for one exporter ``name``.
|
|
30
|
+
|
|
31
|
+
Raises :class:`CapabilityError` (listing the known names) when ``name`` is not a registered
|
|
32
|
+
exporter, so a misconfigured ``exporters:`` entry fails loudly at build time.
|
|
33
|
+
"""
|
|
34
|
+
builder = _BUILDERS.get(name)
|
|
35
|
+
if builder is None:
|
|
36
|
+
known = ", ".join(sorted(_BUILDERS))
|
|
37
|
+
raise CapabilityError(f"unknown exporter {name!r}; known exporters: {known}")
|
|
38
|
+
return builder(config)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Langfuse exporter — OTLP/HTTP to a Langfuse project (§C4).
|
|
2
|
+
|
|
3
|
+
Langfuse (self-hostable or SaaS) accepts OTLP at ``${LANGFUSE_HOST}/api/public/otel/v1/traces`` and
|
|
4
|
+
authenticates with HTTP Basic over the project's public/secret key pair. Keys and host come from the
|
|
5
|
+
environment (never the config file) so a secret never lands in checked-in YAML.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
14
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanProcessor
|
|
15
|
+
|
|
16
|
+
from ..config import ObservabilityConfig
|
|
17
|
+
|
|
18
|
+
#: Langfuse Cloud host used when ``LANGFUSE_HOST`` is unset; self-hosters override it.
|
|
19
|
+
DEFAULT_HOST = "https://cloud.langfuse.com"
|
|
20
|
+
#: OTLP traces path appended to the host.
|
|
21
|
+
OTEL_PATH = "/api/public/otel/v1/traces"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _basic_auth_header(public_key: str, secret_key: str) -> str:
|
|
25
|
+
"""Return the ``Basic <b64>`` value for Langfuse's public/secret key pair."""
|
|
26
|
+
token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
|
|
27
|
+
return f"Basic {token}"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def build(config: ObservabilityConfig) -> SpanProcessor:
|
|
31
|
+
"""Build a batched OTLP/HTTP processor pointed at Langfuse with Basic-auth headers.
|
|
32
|
+
|
|
33
|
+
Reads ``LANGFUSE_HOST`` (optional), ``LANGFUSE_PUBLIC_KEY`` and ``LANGFUSE_SECRET_KEY`` from the
|
|
34
|
+
environment. Missing keys still build a processor (fail-open) — the backend simply rejects the
|
|
35
|
+
unauthenticated export rather than breaking the agent run.
|
|
36
|
+
"""
|
|
37
|
+
host = os.getenv("LANGFUSE_HOST", DEFAULT_HOST).rstrip("/")
|
|
38
|
+
public_key = os.getenv("LANGFUSE_PUBLIC_KEY", "")
|
|
39
|
+
secret_key = os.getenv("LANGFUSE_SECRET_KEY", "")
|
|
40
|
+
headers = {"Authorization": _basic_auth_header(public_key, secret_key)}
|
|
41
|
+
return BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{host}{OTEL_PATH}", headers=headers))
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""LangSmith exporter — OTLP/HTTP to LangSmith. The one SaaS backend, gated (§C4, §4.6).
|
|
2
|
+
|
|
3
|
+
LangSmith is a hosted service, so spans leave the boundary. The factory only wires it in when the
|
|
4
|
+
config's ``allow_saas_exporter`` is set — this module just builds the processor. Endpoint and
|
|
5
|
+
``x-api-key`` come from the environment; an optional ``LANGSMITH_PROJECT`` tags the traces.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
13
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanProcessor
|
|
14
|
+
|
|
15
|
+
from ..config import ObservabilityConfig
|
|
16
|
+
|
|
17
|
+
#: LangSmith's hosted OTLP traces endpoint when ``LANGSMITH_OTEL_ENDPOINT`` is unset.
|
|
18
|
+
DEFAULT_ENDPOINT = "https://api.smith.langchain.com/otel/v1/traces"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build(config: ObservabilityConfig) -> SpanProcessor:
|
|
22
|
+
"""Build a batched OTLP/HTTP processor pointed at LangSmith with its ``x-api-key`` header.
|
|
23
|
+
|
|
24
|
+
Reads ``LANGSMITH_OTEL_ENDPOINT`` (optional), ``LANGSMITH_API_KEY`` and ``LANGSMITH_PROJECT``
|
|
25
|
+
(optional) from the environment. The SaaS gate lives in the factory, not here.
|
|
26
|
+
"""
|
|
27
|
+
endpoint = os.getenv("LANGSMITH_OTEL_ENDPOINT", DEFAULT_ENDPOINT)
|
|
28
|
+
headers = {"x-api-key": os.getenv("LANGSMITH_API_KEY", "")}
|
|
29
|
+
project = os.getenv("LANGSMITH_PROJECT")
|
|
30
|
+
if project:
|
|
31
|
+
headers["Langsmith-Project"] = project
|
|
32
|
+
return BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, headers=headers))
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Opik exporter — OTLP/HTTP to an Opik backend (Comet's open-source LLM tracing tool) (§C4).
|
|
2
|
+
|
|
3
|
+
Opik is open-source and self-hostable, so the default endpoint is the local Opik server; it also
|
|
4
|
+
reads the GenAI/OpenInference attributes the observer stamps. As with Phoenix we reach it over the
|
|
5
|
+
**standard OTLP/HTTP exporter** rather than Opik's SDK, keeping the pipeline vendor-neutral — Opik
|
|
6
|
+
is just an OTLP endpoint here (integrate, don't reinvent).
|
|
7
|
+
|
|
8
|
+
The endpoint comes from ``OPIK_OTEL_ENDPOINT`` (default the local server). Auth is optional so the
|
|
9
|
+
local, keyless case just works: when ``OPIK_API_KEY`` is set an ``Authorization`` header is sent,
|
|
10
|
+
and ``OPIK_WORKSPACE`` / ``OPIK_PROJECT_NAME`` add the workspace and project headers Opik Cloud
|
|
11
|
+
expects. Pointing this at Opik Cloud sends spans off-box — the same SaaS consideration the
|
|
12
|
+
factory's ``allow_saas_exporter`` gate covers when the endpoint is a hosted one.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
|
|
19
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
20
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanProcessor
|
|
21
|
+
|
|
22
|
+
from ..config import ObservabilityConfig
|
|
23
|
+
|
|
24
|
+
#: Where a local self-hosted Opik server accepts OTLP/HTTP traces when nothing else is configured.
|
|
25
|
+
DEFAULT_ENDPOINT = "http://localhost:5173/api/v1/private/otel/v1/traces"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build(config: ObservabilityConfig) -> SpanProcessor:
|
|
29
|
+
"""Build a batched OTLP/HTTP processor pointed at Opik, with optional auth/workspace headers.
|
|
30
|
+
|
|
31
|
+
Reads ``OPIK_OTEL_ENDPOINT`` (optional) and, for Opik Cloud, the optional ``OPIK_API_KEY``,
|
|
32
|
+
``OPIK_WORKSPACE`` and ``OPIK_PROJECT_NAME``. With none of them set it targets a local Opik
|
|
33
|
+
server with no auth, so the offline/self-hosted path needs no configuration.
|
|
34
|
+
"""
|
|
35
|
+
endpoint = os.getenv("OPIK_OTEL_ENDPOINT", DEFAULT_ENDPOINT)
|
|
36
|
+
headers: dict[str, str] = {}
|
|
37
|
+
if api_key := os.getenv("OPIK_API_KEY"):
|
|
38
|
+
headers["Authorization"] = api_key
|
|
39
|
+
if workspace := os.getenv("OPIK_WORKSPACE"):
|
|
40
|
+
headers["Comet-Workspace"] = workspace
|
|
41
|
+
if project := os.getenv("OPIK_PROJECT_NAME"):
|
|
42
|
+
headers["projectName"] = project
|
|
43
|
+
return BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, headers=headers))
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Phoenix exporter — OTLP/HTTP to an Arize Phoenix collector. The recommended OSS backend (§C4).
|
|
2
|
+
|
|
3
|
+
Phoenix is self-hosted and open-source, reads the GenAI + OpenInference attributes the observer
|
|
4
|
+
stamps, and needs no API key — just a running collector. The endpoint comes from
|
|
5
|
+
``PHOENIX_COLLECTOR_ENDPOINT`` (default the local dev collector), so pointing at a shared Phoenix is
|
|
6
|
+
an env-var change, not a code change.
|
|
7
|
+
|
|
8
|
+
We deliberately reach Phoenix over the **standard OTLP/HTTP exporter**, not ``arize-phoenix-otel``'s
|
|
9
|
+
``register()`` convenience. ``register()`` installs a *global* tracer provider; our observer
|
|
10
|
+
composes this processor into its own provider (multiple exporters, one pipeline), so that global
|
|
11
|
+
grab would fight our factory. Using plain OTLP keeps us vendor-neutral with zero Phoenix-SDK
|
|
12
|
+
lock-in — Phoenix
|
|
13
|
+
is just an OTLP endpoint here — which is exactly the integrate-don't-reinvent contract.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
|
|
20
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
21
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanProcessor
|
|
22
|
+
|
|
23
|
+
from ..config import ObservabilityConfig
|
|
24
|
+
|
|
25
|
+
#: Where Phoenix listens for OTLP/HTTP traces when nothing else is configured (local dev collector).
|
|
26
|
+
DEFAULT_ENDPOINT = "http://localhost:6006/v1/traces"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def build(config: ObservabilityConfig) -> SpanProcessor:
|
|
30
|
+
"""Build a batched OTLP/HTTP processor pointed at the Phoenix collector."""
|
|
31
|
+
endpoint = os.getenv("PHOENIX_COLLECTOR_ENDPOINT", DEFAULT_ENDPOINT)
|
|
32
|
+
return BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Build an :class:`Observer` from an :class:`ObservabilityConfig` (§C5).
|
|
2
|
+
|
|
3
|
+
The public seam the rest of AgentShip calls. :func:`build_observer` maps a config to a concrete
|
|
4
|
+
observer: ``provider: none`` (or ``enabled: false``) yields the no-op observer so a run is untraced
|
|
5
|
+
but unchanged; ``provider: otel`` builds an :class:`OTelObserver` over a **process-global**
|
|
6
|
+
``TracerProvider``.
|
|
7
|
+
|
|
8
|
+
The provider is memoized (§DoD "one process-global"): every agent in a process shares one provider,
|
|
9
|
+
so exporters and their batching threads are created once, not per agent. The memo key is the config
|
|
10
|
+
signature that affects the pipeline (service name, exporters, sampling), so two agents with the same
|
|
11
|
+
observability block reuse the same provider and two with different blocks each get their own.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
|
|
18
|
+
from agentship.errors import CapabilityError
|
|
19
|
+
from agentship.observability import NoOpObserver, Observer
|
|
20
|
+
from agentship.spec import ObservabilitySpec
|
|
21
|
+
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
|
|
22
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
23
|
+
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
|
24
|
+
|
|
25
|
+
from .config import ObservabilityConfig
|
|
26
|
+
from .exporters import build_processor
|
|
27
|
+
from .otel import OTelObserver
|
|
28
|
+
|
|
29
|
+
#: The OTel ``service.name`` for every AgentShip span, overridable via env for multi-service setups.
|
|
30
|
+
DEFAULT_SERVICE_NAME = "agentship"
|
|
31
|
+
|
|
32
|
+
#: Process-global cache of built providers, keyed by :func:`_provider_key`.
|
|
33
|
+
_PROVIDERS: dict[tuple, TracerProvider] = {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _service_name() -> str:
|
|
37
|
+
"""Resolve the ``service.name`` resource attribute (env override, then default)."""
|
|
38
|
+
return os.getenv("AGENTSHIP_SERVICE_NAME", DEFAULT_SERVICE_NAME)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _resolved_exporters(config: ObservabilityConfig) -> list[str]:
|
|
42
|
+
"""Return the exporter names to wire, enforcing the SaaS gate (§4.6).
|
|
43
|
+
|
|
44
|
+
A SaaS exporter (LangSmith) ships spans off-box, so it is only allowed when the config opts in
|
|
45
|
+
with ``allow_saas_exporter``. Otherwise it is a fail-fast :class:`CapabilityError` rather than a
|
|
46
|
+
silent data-egress the operator did not ask for.
|
|
47
|
+
"""
|
|
48
|
+
if config.uses_saas_exporter and not config.allow_saas_exporter:
|
|
49
|
+
raise CapabilityError(
|
|
50
|
+
"exporter 'langsmith' ships spans to a SaaS backend; set allow_saas_exporter: true "
|
|
51
|
+
"to enable it"
|
|
52
|
+
)
|
|
53
|
+
return list(config.exporters)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _provider_key(config: ObservabilityConfig) -> tuple:
|
|
57
|
+
"""Build the memo key: the config fields that change the provider's pipeline."""
|
|
58
|
+
return (_service_name(), tuple(config.exporters), config.sample_ratio)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def build_tracer_provider(config: ObservabilityConfig) -> TracerProvider:
|
|
62
|
+
"""Return the process-global ``TracerProvider`` for ``config``, building it once.
|
|
63
|
+
|
|
64
|
+
The provider carries the ``service.name`` resource and a ``ParentBased(TraceIdRatioBased)``
|
|
65
|
+
sampler (so a child inherits its root's keep/drop decision) and registers one span processor per
|
|
66
|
+
configured exporter.
|
|
67
|
+
"""
|
|
68
|
+
key = _provider_key(config)
|
|
69
|
+
provider = _PROVIDERS.get(key)
|
|
70
|
+
if provider is not None:
|
|
71
|
+
return provider
|
|
72
|
+
|
|
73
|
+
resource = Resource.create({SERVICE_NAME: _service_name()})
|
|
74
|
+
sampler = ParentBased(TraceIdRatioBased(config.sample_ratio))
|
|
75
|
+
provider = TracerProvider(resource=resource, sampler=sampler)
|
|
76
|
+
for name in _resolved_exporters(config):
|
|
77
|
+
provider.add_span_processor(build_processor(name, config))
|
|
78
|
+
|
|
79
|
+
_PROVIDERS[key] = provider
|
|
80
|
+
return provider
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _as_config(config: ObservabilityConfig | ObservabilitySpec | None) -> ObservabilityConfig:
|
|
84
|
+
"""Coerce the declarative :class:`ObservabilitySpec` (or ``None``) into an adapter config.
|
|
85
|
+
|
|
86
|
+
The kernel's ``observability:`` block is an :class:`~agentship.spec.ObservabilitySpec` — a
|
|
87
|
+
vendor-neutral authoring surface. This maps its fields onto the adapter's
|
|
88
|
+
:class:`ObservabilityConfig`, which is where exporter names are actually validated (against the
|
|
89
|
+
backends this package can build) and env-driven endpoints/keys are resolved. A pass-through
|
|
90
|
+
``ObservabilityConfig`` is returned as-is, and ``None`` yields the console-only default.
|
|
91
|
+
"""
|
|
92
|
+
if isinstance(config, ObservabilitySpec):
|
|
93
|
+
return ObservabilityConfig(**config.model_dump())
|
|
94
|
+
return config or ObservabilityConfig()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def build_otel_observer(
|
|
98
|
+
config: ObservabilityConfig | ObservabilitySpec | None = None,
|
|
99
|
+
) -> OTelObserver:
|
|
100
|
+
"""Build an :class:`OTelObserver` over the process-global provider for ``config``.
|
|
101
|
+
|
|
102
|
+
Called by the ``agentship.observers`` entry point (``otel``) with an agent's
|
|
103
|
+
:class:`~agentship.spec.ObservabilitySpec`, and by tests with an :class:`ObservabilityConfig`
|
|
104
|
+
directly. Either is coerced by :func:`_as_config`; a bare call still yields a working
|
|
105
|
+
console-exporting observer.
|
|
106
|
+
"""
|
|
107
|
+
config = _as_config(config)
|
|
108
|
+
provider = build_tracer_provider(config)
|
|
109
|
+
return OTelObserver(provider, capture_content=config.capture_content)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_observer(config: ObservabilityConfig | None = None) -> Observer:
|
|
113
|
+
"""Map a config to a concrete observer: no-op when disabled/``none``, else the OTel observer."""
|
|
114
|
+
config = config or ObservabilityConfig()
|
|
115
|
+
if not config.enabled or config.provider == "none":
|
|
116
|
+
return NoOpObserver()
|
|
117
|
+
return build_otel_observer(config)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _reset_providers_for_tests() -> None:
|
|
121
|
+
"""Clear the process-global provider cache. For tests that assert build-once behaviour."""
|
|
122
|
+
_PROVIDERS.clear()
|