trodo-python 2.21.0__tar.gz → 2.23.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trodo_python-2.21.0 → trodo_python-2.23.0}/PKG-INFO +1 -1
- {trodo_python-2.21.0 → trodo_python-2.23.0}/pyproject.toml +1 -1
- trodo_python-2.23.0/tests/test_instrumentor_resolution.py +109 -0
- trodo_python-2.23.0/tests/test_prompt_trace.py +261 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_prompts.py +24 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/auto_instrument.py +58 -53
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/prompt_trace.py +65 -3
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/wrap_agent.py +53 -7
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/prompts/compile.py +38 -2
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/prompts/template.py +16 -1
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/prompts/types.py +21 -12
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo_python.egg-info/PKG-INFO +1 -1
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo_python.egg-info/SOURCES.txt +1 -0
- trodo_python-2.21.0/tests/test_prompt_trace.py +0 -107
- {trodo_python-2.21.0 → trodo_python-2.23.0}/README.md +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/setup.cfg +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_anon_distinct_id.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_auto_instrument_fixes.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_cross_process_session.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_datasets.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_end_run.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_error_enrichment.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_experiments.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_get_prompt_version_types.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_llm_usage_cost.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_multi_agent_shape.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_processor_methods.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_register_otel.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_start_run.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_user_traits.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/tests/test_wrap_agent_unchanged.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/api/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/api/async_client.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/api/endpoints.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/api/http_client.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/auto/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/auto/auto_event_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/client.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/dataset_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/experiment_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/group_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/people_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/prompt_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/managers/user_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/context.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/helpers.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/processor.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/register.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/otel/transport.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/prompts/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/queue/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/queue/batch_flusher.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/queue/event_queue.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/session/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/session/server_session.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/session/session_manager.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/types.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/user_context.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/util/__init__.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo/util/lru.py +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo_python.egg-info/dependency_links.txt +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo_python.egg-info/requires.txt +0 -0
- {trodo_python-2.21.0 → trodo_python-2.23.0}/trodo_python.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Instrumentor class resolution must survive upstream renames.
|
|
2
|
+
|
|
3
|
+
Every per-framework instrumentor is loaded by name, and the loader in
|
|
4
|
+
``enable_auto_instrument`` swallows failures so a missing optional package
|
|
5
|
+
never breaks user code. The cost of that tolerance: a RENAMED export is
|
|
6
|
+
indistinguishable from an absent package. The user installs exactly the
|
|
7
|
+
package we documented, gets zero spans, and gets no error explaining why.
|
|
8
|
+
|
|
9
|
+
That is not hypothetical — it happened twice. ``google-generativeai`` renamed
|
|
10
|
+
its class, and ``opentelemetry-instrumentation-langchain`` ships
|
|
11
|
+
``LangchainInstrumentor`` while we imported ``LangChainInstrumentor``, which
|
|
12
|
+
silently disabled LangChain tracing for every Python user on a current version.
|
|
13
|
+
|
|
14
|
+
So resolution tries the known spellings and then falls back to whatever single
|
|
15
|
+
``*Instrumentor`` the module exports.
|
|
16
|
+
"""
|
|
17
|
+
import sys
|
|
18
|
+
import types
|
|
19
|
+
|
|
20
|
+
import pytest
|
|
21
|
+
|
|
22
|
+
from trodo.otel.auto_instrument import _instrument
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _fake_module(name, **attrs):
|
|
26
|
+
mod = types.ModuleType(name)
|
|
27
|
+
for k, v in attrs.items():
|
|
28
|
+
setattr(mod, k, v)
|
|
29
|
+
sys.modules[name] = mod
|
|
30
|
+
return mod
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _Recorder:
|
|
34
|
+
instrumented = False
|
|
35
|
+
|
|
36
|
+
def instrument(self):
|
|
37
|
+
type(self).instrumented = True
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.fixture(autouse=True)
|
|
41
|
+
def _clean():
|
|
42
|
+
made = []
|
|
43
|
+
yield made
|
|
44
|
+
for n in made:
|
|
45
|
+
sys.modules.pop(n, None)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_uses_the_preferred_name_when_present(_clean):
|
|
49
|
+
name = 'fake_instr_preferred'
|
|
50
|
+
_clean.append(name)
|
|
51
|
+
cls = type('WantedInstrumentor', (_Recorder,), {'instrumented': False})
|
|
52
|
+
_fake_module(name, WantedInstrumentor=cls, OtherInstrumentor=type('OtherInstrumentor', (_Recorder,), {}))
|
|
53
|
+
_instrument(name, 'WantedInstrumentor')
|
|
54
|
+
assert cls.instrumented is True
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_tries_each_spelling_in_order(_clean):
|
|
58
|
+
"""The real LangChain case: we ask for both casings, upstream ships one."""
|
|
59
|
+
name = 'fake_instr_langchain'
|
|
60
|
+
_clean.append(name)
|
|
61
|
+
cls = type('LangchainInstrumentor', (_Recorder,), {'instrumented': False})
|
|
62
|
+
_fake_module(name, LangchainInstrumentor=cls)
|
|
63
|
+
# 'LangChainInstrumentor' (capital C) does not exist here — must not raise.
|
|
64
|
+
_instrument(name, 'LangchainInstrumentor', 'LangChainInstrumentor')
|
|
65
|
+
assert cls.instrumented is True
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_falls_back_to_the_only_exported_instrumentor(_clean):
|
|
69
|
+
"""An unforeseen rename still works, which is the whole point."""
|
|
70
|
+
name = 'fake_instr_renamed'
|
|
71
|
+
_clean.append(name)
|
|
72
|
+
cls = type('SomethingWeNeverHeardOfInstrumentor', (_Recorder,), {'instrumented': False})
|
|
73
|
+
_fake_module(name, SomethingWeNeverHeardOfInstrumentor=cls)
|
|
74
|
+
_instrument(name, 'ExpectedInstrumentor')
|
|
75
|
+
assert cls.instrumented is True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_refuses_to_guess_between_several_candidates(_clean):
|
|
79
|
+
"""Picking arbitrarily could instrument the wrong thing — say so instead."""
|
|
80
|
+
name = 'fake_instr_ambiguous'
|
|
81
|
+
_clean.append(name)
|
|
82
|
+
_fake_module(name,
|
|
83
|
+
AInstrumentor=type('AInstrumentor', (_Recorder,), {}),
|
|
84
|
+
BInstrumentor=type('BInstrumentor', (_Recorder,), {}))
|
|
85
|
+
with pytest.raises(ImportError, match='several Instrumentor classes'):
|
|
86
|
+
_instrument(name, 'NopeInstrumentor')
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_reports_when_nothing_is_exported(_clean):
|
|
90
|
+
name = 'fake_instr_empty'
|
|
91
|
+
_clean.append(name)
|
|
92
|
+
_fake_module(name)
|
|
93
|
+
with pytest.raises(ImportError, match='no Instrumentor class'):
|
|
94
|
+
_instrument(name, 'AnyInstrumentor')
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_base_instrumentor_is_never_the_fallback(_clean):
|
|
98
|
+
"""Modules re-export BaseInstrumentor; instrumenting it would be wrong."""
|
|
99
|
+
name = 'fake_instr_baseonly'
|
|
100
|
+
_clean.append(name)
|
|
101
|
+
_fake_module(name, BaseInstrumentor=type('BaseInstrumentor', (_Recorder,), {}))
|
|
102
|
+
with pytest.raises(ImportError, match='no Instrumentor class'):
|
|
103
|
+
_instrument(name, 'AnyInstrumentor')
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_a_missing_module_still_raises_importerror(_clean):
|
|
107
|
+
"""The caller relies on ImportError to mean "skip this framework"."""
|
|
108
|
+
with pytest.raises(ImportError):
|
|
109
|
+
_instrument('module_that_does_not_exist_xyz', 'AnyInstrumentor')
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""Prompt -> span traceability (mirrors the Node promptTrace.test.ts).
|
|
2
|
+
|
|
3
|
+
Compiling a managed prompt must stamp the exact version (by immutable hash)
|
|
4
|
+
onto the spans that used it, and aggregate every prompt used onto the run.
|
|
5
|
+
|
|
6
|
+
Two routes get it there, and both are load-bearing:
|
|
7
|
+
- the RUN SCOPE, when compile() runs inside wrap_agent/span. The only route
|
|
8
|
+
that reaches auto-instrumented spans, whose input we never see.
|
|
9
|
+
- the COMPILED VALUE, when those messages are passed to a span as ``input``.
|
|
10
|
+
The only route that survives compiling outside the run, which is an
|
|
11
|
+
ordinary shape and used to produce no link at all.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from trodo.managers.prompt_manager import PromptManager
|
|
16
|
+
from trodo.otel.wrap_agent import span, wrap_agent
|
|
17
|
+
|
|
18
|
+
PROMPT_WIRE = {
|
|
19
|
+
"name": "refund-agent",
|
|
20
|
+
"version": 3,
|
|
21
|
+
"version_hash": "a3f9c2e8d1b04756",
|
|
22
|
+
"content_hash": "ccccontent1234",
|
|
23
|
+
"parent_hash": "pppparent5678",
|
|
24
|
+
"labels": ["production"],
|
|
25
|
+
"tags": [],
|
|
26
|
+
"messages": [{"role": "system", "content": [{"type": "text", "text": "Help {{q}}"}]}],
|
|
27
|
+
"model": {"provider": "openai", "model": "gpt-4o"},
|
|
28
|
+
"tools": [],
|
|
29
|
+
"response_format": None,
|
|
30
|
+
"variables": [{"name": "q", "type": "string"}],
|
|
31
|
+
"updated_at": "2026-07-01T00:00:00Z",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class PromptHttp:
|
|
36
|
+
def __init__(self, prompt=None, fail=False):
|
|
37
|
+
self._prompt = prompt
|
|
38
|
+
self.fail = fail
|
|
39
|
+
|
|
40
|
+
def get_prompt(self, name, label=None, version=None):
|
|
41
|
+
if self.fail:
|
|
42
|
+
raise RuntimeError("unreachable")
|
|
43
|
+
return {"prompt": dict(self._prompt)}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_stamps_version_hash_on_span_and_aggregates_on_run(processor, http):
|
|
47
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
48
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
49
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
50
|
+
with span("llm", kind="llm"):
|
|
51
|
+
p.compile(q="my order")
|
|
52
|
+
|
|
53
|
+
ingest = http.run_ingest[0]
|
|
54
|
+
spans = ingest["spans"]
|
|
55
|
+
llm = next(s for s in spans if s["name"] == "llm")
|
|
56
|
+
attrs = llm["attributes"]
|
|
57
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
58
|
+
assert attrs["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
59
|
+
assert attrs["trodo.prompt.label"] == "production" # default selector
|
|
60
|
+
assert attrs["trodo.prompt.content_hash"] == "ccccontent1234"
|
|
61
|
+
|
|
62
|
+
used = ingest["run"]["attributes"]["trodo.prompts"]
|
|
63
|
+
assert len(used) == 1
|
|
64
|
+
assert used[0]["version_hash"] == "a3f9c2e8d1b04756"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_records_label_and_omits_it_for_version_pin(processor, http):
|
|
68
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
69
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
70
|
+
staged = mgr.get("refund-agent", label="staging", cache_ttl_seconds=0)
|
|
71
|
+
with span("s1"):
|
|
72
|
+
staged.compile(q="x")
|
|
73
|
+
pinned = mgr.get("refund-agent", version="a3f9c2", cache_ttl_seconds=0)
|
|
74
|
+
with span("s2"):
|
|
75
|
+
pinned.compile(q="y")
|
|
76
|
+
|
|
77
|
+
spans = http.run_ingest[0]["spans"]
|
|
78
|
+
s1 = next(s for s in spans if s["name"] == "s1")["attributes"]
|
|
79
|
+
s2 = next(s for s in spans if s["name"] == "s2")["attributes"]
|
|
80
|
+
assert s1["trodo.prompt.label"] == "staging"
|
|
81
|
+
assert "trodo.prompt.label" not in s2 # pinned by hash -> no label
|
|
82
|
+
assert s2["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_no_op_outside_a_run(processor, http):
|
|
86
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
87
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
88
|
+
# Must not raise despite there being no active run context.
|
|
89
|
+
p.compile(q="z")
|
|
90
|
+
assert len(http.run_ingest) == 0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_fallback_prompt_is_not_traced(processor, http):
|
|
94
|
+
mgr = PromptManager(PromptHttp(fail=True))
|
|
95
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
96
|
+
p = mgr.get(
|
|
97
|
+
"refund-agent",
|
|
98
|
+
cache_ttl_seconds=0,
|
|
99
|
+
max_retries=0,
|
|
100
|
+
fallback={
|
|
101
|
+
"messages": [{"role": "user", "content": [{"type": "text", "text": "hi {{q}}"}]}],
|
|
102
|
+
"variables": [{"name": "q"}],
|
|
103
|
+
},
|
|
104
|
+
)
|
|
105
|
+
assert p.is_fallback is True
|
|
106
|
+
with span("llm"):
|
|
107
|
+
p.compile(q="x")
|
|
108
|
+
|
|
109
|
+
spans = http.run_ingest[0]["spans"]
|
|
110
|
+
attrs = next(s for s in spans if s["name"] == "llm").get("attributes") or {}
|
|
111
|
+
assert "trodo.prompt.name" not in attrs
|
|
112
|
+
# to_dict() drops None, so an untraced run has no 'attributes' key at all.
|
|
113
|
+
assert "attributes" not in http.run_ingest[0]["run"]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# ── compiled OUTSIDE the run ────────────────────────────────────────────────
|
|
117
|
+
#
|
|
118
|
+
# Fetch and compile up front, open the run around just the model call. Under the
|
|
119
|
+
# run-scope mechanism alone this produced no link whatsoever.
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_links_span_whose_input_was_compiled_outside_the_run(processor, http):
|
|
123
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
124
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
125
|
+
compiled = p.compile(q="z") # OUTSIDE any run
|
|
126
|
+
|
|
127
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
128
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
129
|
+
pass
|
|
130
|
+
|
|
131
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm")["attributes"]
|
|
132
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
133
|
+
assert attrs["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_outside_compiled_prompt_reaches_the_run_level_list(processor, http):
|
|
137
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
138
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
139
|
+
compiled = p.compile(q="z")
|
|
140
|
+
|
|
141
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
142
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
143
|
+
pass
|
|
144
|
+
|
|
145
|
+
used = http.run_ingest[0]["run"]["attributes"]["trodo.prompts"]
|
|
146
|
+
assert [r["name"] for r in used] == ["refund-agent"]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_the_compiled_wrapper_works_as_input_too(processor, http):
|
|
150
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
151
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
152
|
+
compiled = p.compile(q="z")
|
|
153
|
+
|
|
154
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
155
|
+
with span("llm", kind="llm", input=compiled):
|
|
156
|
+
pass
|
|
157
|
+
|
|
158
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm")["attributes"]
|
|
159
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_span_input_beats_the_runs_most_recent_prompt(processor, http):
|
|
163
|
+
"""Specificity: a span holding prompt A must say A, not whatever came last."""
|
|
164
|
+
other = dict(PROMPT_WIRE, name="other-agent", version_hash="bbbb2222", content_hash="ccc2")
|
|
165
|
+
|
|
166
|
+
class TwoPrompts:
|
|
167
|
+
def get_prompt(self, name, label=None, version=None):
|
|
168
|
+
return {"prompt": dict(other if name == "other-agent" else PROMPT_WIRE)}
|
|
169
|
+
|
|
170
|
+
mgr = PromptManager(TwoPrompts())
|
|
171
|
+
a = mgr.get("refund-agent", cache_ttl_seconds=0).compile(q="first")
|
|
172
|
+
|
|
173
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
174
|
+
# Compiling B inside the run makes B the run's "current" prompt.
|
|
175
|
+
mgr.get("other-agent", cache_ttl_seconds=0).compile(q="second")
|
|
176
|
+
with span("llm", kind="llm", input=a.messages):
|
|
177
|
+
pass
|
|
178
|
+
|
|
179
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm")["attributes"]
|
|
180
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
181
|
+
assert attrs["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def test_a_span_that_never_saw_a_prompt_is_not_labelled(processor, http):
|
|
185
|
+
"""False-positive guard: attribution must not leak to sibling spans."""
|
|
186
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
187
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
188
|
+
compiled = p.compile(q="z")
|
|
189
|
+
|
|
190
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
191
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
192
|
+
pass
|
|
193
|
+
with span("unrelated", input={"some": "payload"}):
|
|
194
|
+
pass
|
|
195
|
+
|
|
196
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "unrelated").get("attributes") or {}
|
|
197
|
+
assert "trodo.prompt.name" not in attrs
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_hand_written_messages_are_not_carriers(processor, http):
|
|
201
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
202
|
+
with span("llm", kind="llm", input=[{"role": "user", "content": "hand-written"}]):
|
|
203
|
+
pass
|
|
204
|
+
|
|
205
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm").get("attributes") or {}
|
|
206
|
+
assert "trodo.prompt.name" not in attrs
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_a_fallback_is_never_tagged_via_the_input_route(processor, http):
|
|
210
|
+
mgr = PromptManager(PromptHttp(fail=True))
|
|
211
|
+
p = mgr.get(
|
|
212
|
+
"refund-agent",
|
|
213
|
+
fallback={"messages": [{"role": "user", "content": [{"type": "text", "text": "hi {{q}}"}]}],
|
|
214
|
+
"variables": [{"name": "q"}]},
|
|
215
|
+
cache_ttl_seconds=0,
|
|
216
|
+
)
|
|
217
|
+
assert p.is_fallback
|
|
218
|
+
compiled = p.compile(q="x")
|
|
219
|
+
|
|
220
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
221
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
222
|
+
pass
|
|
223
|
+
|
|
224
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm").get("attributes") or {}
|
|
225
|
+
assert "trodo.prompt.name" not in attrs
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def test_carrier_table_is_bounded(processor, http):
|
|
229
|
+
"""It must not grow without limit in a long-lived process."""
|
|
230
|
+
from trodo.otel import prompt_trace
|
|
231
|
+
|
|
232
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
233
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
234
|
+
for _ in range(prompt_trace._MAX_CARRIERS + 200):
|
|
235
|
+
p.compile(q="x")
|
|
236
|
+
|
|
237
|
+
assert len(prompt_trace._carrier_refs) <= prompt_trace._MAX_CARRIERS
|
|
238
|
+
assert len(prompt_trace._carrier_keepalive) <= prompt_trace._MAX_CARRIERS
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def test_run_set_input_records_the_prompt_on_the_run(processor, http):
|
|
242
|
+
"""The shape that still leaves auto-instrumented spans unlabelled: compile
|
|
243
|
+
up front, hand the messages to the RUN, let a framework make the call. The
|
|
244
|
+
run can at least say which prompt it ran."""
|
|
245
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
246
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
247
|
+
compiled = p.compile(q="z")
|
|
248
|
+
|
|
249
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat") as run:
|
|
250
|
+
run.set_input(compiled.messages)
|
|
251
|
+
|
|
252
|
+
used = http.run_ingest[0]["run"]["attributes"]["trodo.prompts"]
|
|
253
|
+
assert [r["name"] for r in used] == ["refund-agent"]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def test_run_set_input_with_ordinary_input_records_nothing(processor, http):
|
|
257
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat") as run:
|
|
258
|
+
run.set_input({"query": "hello"})
|
|
259
|
+
|
|
260
|
+
attrs = http.run_ingest[0]["run"].get("attributes") or {}
|
|
261
|
+
assert "trodo.prompts" not in attrs
|
|
@@ -408,3 +408,27 @@ def test_list_returns_empty_on_error():
|
|
|
408
408
|
return {"__error": True}
|
|
409
409
|
|
|
410
410
|
assert PromptManager(Err()).list() == []
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
# ── Shared vectors: COMPILE layer ═══════════════════════════════════════════
|
|
414
|
+
#
|
|
415
|
+
# Declared-variable semantics (defaults, explicit null, per-type coercion) are
|
|
416
|
+
# where the three engines actually drifted -- THIS engine substituted the
|
|
417
|
+
# default for an explicit ``None`` and coerced string-typed values with
|
|
418
|
+
# ``str()`` instead of JS ``String()``, so the same call rendered different
|
|
419
|
+
# text in Python than in the playground. The backend and Node suites run the
|
|
420
|
+
# SAME cases from the same file.
|
|
421
|
+
|
|
422
|
+
with open(_VECTORS_PATH, encoding="utf-8") as _f:
|
|
423
|
+
_COMPILE_CASES = json.load(_f).get("compileCases", [])
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@pytest.mark.parametrize("case", _COMPILE_CASES, ids=lambda c: c["name"])
|
|
427
|
+
def test_compile_vectors(case):
|
|
428
|
+
prompt = {
|
|
429
|
+
"messages": [{"role": "user", "content": [{"type": "text", "text": case["template"]}]}],
|
|
430
|
+
"model": {}, "tools": [], "response_format": None,
|
|
431
|
+
"variables": case["variables"],
|
|
432
|
+
}
|
|
433
|
+
out = compile_prompt(prompt, case["values"])
|
|
434
|
+
assert out.messages[0]["content"][0]["text"] == case["expect"]
|
|
@@ -401,6 +401,46 @@ class _OtelAdapter(_SpanProcessorBase): # type: ignore[valid-type,misc]
|
|
|
401
401
|
return True
|
|
402
402
|
|
|
403
403
|
|
|
404
|
+
def _instrument(module_name: str, *preferred: str) -> None:
|
|
405
|
+
"""Import ``module_name`` and call ``instrument()`` on its Instrumentor.
|
|
406
|
+
|
|
407
|
+
Upstream renames the exported class often enough that pinning one exact
|
|
408
|
+
spelling is a silent-failure generator: the import raises, the caller's
|
|
409
|
+
``except Exception`` treats it as "package not installed", and the user who
|
|
410
|
+
installed exactly what we told them to gets no spans and no error. That is
|
|
411
|
+
how ``LangChainInstrumentor`` (upstream ships ``LangchainInstrumentor``) and
|
|
412
|
+
``GoogleGenerativeAIInstrumentor`` both went dark.
|
|
413
|
+
|
|
414
|
+
So: try the known spellings in order, then fall back to whatever single
|
|
415
|
+
``*Instrumentor`` the module exports. Mirrors ``instrCtor`` in the Node SDK.
|
|
416
|
+
Raises ImportError if nothing usable is found, which the caller reports.
|
|
417
|
+
"""
|
|
418
|
+
import importlib
|
|
419
|
+
|
|
420
|
+
mod = importlib.import_module(module_name)
|
|
421
|
+
cls = None
|
|
422
|
+
for name in preferred:
|
|
423
|
+
cls = getattr(mod, name, None)
|
|
424
|
+
if cls is not None:
|
|
425
|
+
break
|
|
426
|
+
if cls is None:
|
|
427
|
+
found = [
|
|
428
|
+
n for n in dir(mod)
|
|
429
|
+
if n.endswith("Instrumentor") and n != "BaseInstrumentor"
|
|
430
|
+
and isinstance(getattr(mod, n, None), type)
|
|
431
|
+
]
|
|
432
|
+
if len(found) == 1:
|
|
433
|
+
cls = getattr(mod, found[0])
|
|
434
|
+
elif found:
|
|
435
|
+
raise ImportError(
|
|
436
|
+
f"{module_name} exports several Instrumentor classes {found}; "
|
|
437
|
+
f"none matched the expected names {list(preferred)}"
|
|
438
|
+
)
|
|
439
|
+
if cls is None:
|
|
440
|
+
raise ImportError(f"no Instrumentor class exported by {module_name}")
|
|
441
|
+
cls().instrument()
|
|
442
|
+
|
|
443
|
+
|
|
404
444
|
_INSTRUMENTORS: List[tuple[str, Callable[[], Any]]] = []
|
|
405
445
|
|
|
406
446
|
|
|
@@ -416,84 +456,49 @@ def _register_instrumentors() -> None:
|
|
|
416
456
|
return
|
|
417
457
|
|
|
418
458
|
def _anthropic() -> Any:
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
AnthropicInstrumentor().instrument()
|
|
459
|
+
_instrument("opentelemetry.instrumentation.anthropic", "AnthropicInstrumentor")
|
|
422
460
|
|
|
423
461
|
def _openai() -> Any:
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
OpenAIInstrumentor().instrument()
|
|
462
|
+
_instrument("opentelemetry.instrumentation.openai", "OpenAIInstrumentor")
|
|
427
463
|
|
|
428
464
|
def _openai_v2() -> Any:
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
OpenAIInstrumentor().instrument()
|
|
465
|
+
_instrument("opentelemetry.instrumentation.openai_v2", "OpenAIInstrumentor")
|
|
432
466
|
|
|
433
467
|
def _langchain() -> Any:
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
LangChainInstrumentor().instrument()
|
|
468
|
+
_instrument("opentelemetry.instrumentation.langchain", "LangchainInstrumentor", "LangChainInstrumentor")
|
|
437
469
|
|
|
438
470
|
def _llama_index() -> Any:
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
LlamaIndexInstrumentor().instrument()
|
|
471
|
+
_instrument("opentelemetry.instrumentation.llama_index", "LlamaIndexInstrumentor")
|
|
442
472
|
|
|
443
473
|
def _google_generativeai() -> Any:
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
# SDK, i.e. the `google-genai` package, not legacy google-generativeai.)
|
|
451
|
-
instr = (
|
|
452
|
-
getattr(_m, "GoogleGenerativeAiInstrumentor", None)
|
|
453
|
-
or getattr(_m, "GoogleGenerativeAIInstrumentor", None)
|
|
454
|
-
or getattr(_m, "GenAIInstrumentor", None)
|
|
474
|
+
# Patches the new google-genai SDK, not legacy google-generativeai.
|
|
475
|
+
_instrument(
|
|
476
|
+
"opentelemetry.instrumentation.google_generativeai",
|
|
477
|
+
"GoogleGenerativeAiInstrumentor",
|
|
478
|
+
"GoogleGenerativeAIInstrumentor",
|
|
479
|
+
"GenAIInstrumentor",
|
|
455
480
|
)
|
|
456
|
-
if instr is None:
|
|
457
|
-
raise ImportError(
|
|
458
|
-
"no GoogleGenerativeAi/GenAI Instrumentor in "
|
|
459
|
-
"opentelemetry.instrumentation.google_generativeai"
|
|
460
|
-
)
|
|
461
|
-
instr().instrument()
|
|
462
481
|
|
|
463
482
|
def _vertexai() -> Any:
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
VertexAIInstrumentor().instrument()
|
|
483
|
+
_instrument("opentelemetry.instrumentation.vertexai", "VertexAIInstrumentor")
|
|
467
484
|
|
|
468
485
|
def _bedrock() -> Any:
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
BedrockInstrumentor().instrument()
|
|
486
|
+
_instrument("opentelemetry.instrumentation.bedrock", "BedrockInstrumentor")
|
|
472
487
|
|
|
473
488
|
def _cohere() -> Any:
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
CohereInstrumentor().instrument()
|
|
489
|
+
_instrument("opentelemetry.instrumentation.cohere", "CohereInstrumentor")
|
|
477
490
|
|
|
478
491
|
def _mistralai() -> Any:
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
MistralAiInstrumentor().instrument()
|
|
492
|
+
_instrument("opentelemetry.instrumentation.mistralai", "MistralAiInstrumentor")
|
|
482
493
|
|
|
483
494
|
def _haystack() -> Any:
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
HaystackInstrumentor().instrument()
|
|
495
|
+
_instrument("opentelemetry.instrumentation.haystack", "HaystackInstrumentor")
|
|
487
496
|
|
|
488
497
|
def _httpx() -> Any:
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
HTTPXClientInstrumentor().instrument()
|
|
498
|
+
_instrument("opentelemetry.instrumentation.httpx", "HTTPXClientInstrumentor")
|
|
492
499
|
|
|
493
500
|
def _requests() -> Any:
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
RequestsInstrumentor().instrument()
|
|
501
|
+
_instrument("opentelemetry.instrumentation.requests", "RequestsInstrumentor")
|
|
497
502
|
|
|
498
503
|
_INSTRUMENTORS = [
|
|
499
504
|
("anthropic", _anthropic),
|
|
@@ -14,6 +14,7 @@ Mirrors ``sdks/trodo-node-sdk/src/otel/promptTrace.ts``.
|
|
|
14
14
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
|
+
from collections import OrderedDict
|
|
17
18
|
from typing import Any, Dict, List, Optional
|
|
18
19
|
|
|
19
20
|
from .context import get_active_context
|
|
@@ -27,6 +28,55 @@ def new_prompt_state() -> Dict[str, Any]:
|
|
|
27
28
|
return {"current": None, "all": []}
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
# Compiled prompt (or its messages list) -> the version it came from.
|
|
32
|
+
#
|
|
33
|
+
# Mirrors the Node SDK. The run-scope mechanism below only fires when compile()
|
|
34
|
+
# runs inside a run, which leaves the ordinary "fetch and compile up front, open
|
|
35
|
+
# the run around the model call" shape with no link at all. Tagging the compiled
|
|
36
|
+
# value means a span handed those messages as ``input`` recovers the version
|
|
37
|
+
# wherever compile() happened.
|
|
38
|
+
#
|
|
39
|
+
# Keyed by id() with a weak-ish discipline: lists are unhashable and cannot go
|
|
40
|
+
# in a WeakValueDictionary, so entries are bounded and evicted FIFO rather than
|
|
41
|
+
# held forever. The link is best-effort — losing an old entry costs one span's
|
|
42
|
+
# label, never correctness.
|
|
43
|
+
_MAX_CARRIERS = 4096
|
|
44
|
+
_carrier_refs: "OrderedDict[int, PromptRef]" = OrderedDict()
|
|
45
|
+
_carrier_keepalive: "OrderedDict[int, Any]" = OrderedDict()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def tag_carrier_with_prompt(carrier: Any, ref: PromptRef) -> None:
|
|
49
|
+
"""Attach a prompt identity to a value the caller will pass to a span."""
|
|
50
|
+
if carrier is None or isinstance(carrier, (str, bytes, int, float, bool)):
|
|
51
|
+
return
|
|
52
|
+
key = id(carrier)
|
|
53
|
+
_carrier_refs[key] = ref
|
|
54
|
+
# Hold a reference so the id cannot be recycled by another object while the
|
|
55
|
+
# mapping still names it — that would mislabel an unrelated span.
|
|
56
|
+
_carrier_keepalive[key] = carrier
|
|
57
|
+
_carrier_refs.move_to_end(key)
|
|
58
|
+
_carrier_keepalive.move_to_end(key)
|
|
59
|
+
while len(_carrier_refs) > _MAX_CARRIERS:
|
|
60
|
+
old, _ = _carrier_refs.popitem(last=False)
|
|
61
|
+
_carrier_keepalive.pop(old, None)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def prompt_ref_from_carrier(carrier: Any) -> Optional[PromptRef]:
|
|
65
|
+
"""Recover the prompt identity from a span's input, if it carries one."""
|
|
66
|
+
if carrier is None or isinstance(carrier, (str, bytes, int, float, bool)):
|
|
67
|
+
return None
|
|
68
|
+
return _carrier_refs.get(id(carrier))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def remember_prompt(state: Any, ref: PromptRef) -> None:
|
|
72
|
+
"""Add a ref to the run's deduped set if it isn't already there."""
|
|
73
|
+
if not isinstance(state, dict) or not ref or not ref.get("name"):
|
|
74
|
+
return
|
|
75
|
+
key = _ref_key(ref)
|
|
76
|
+
if not any(_ref_key(r) == key for r in state.get("all", [])):
|
|
77
|
+
state.setdefault("all", []).append(ref)
|
|
78
|
+
|
|
79
|
+
|
|
30
80
|
def _ref_key(ref: PromptRef) -> str:
|
|
31
81
|
return ref.get("version_hash") or ref.get("name") or ""
|
|
32
82
|
|
|
@@ -66,12 +116,24 @@ def prompt_attributes(ref: Optional[PromptRef]) -> Dict[str, str]:
|
|
|
66
116
|
|
|
67
117
|
|
|
68
118
|
def merge_prompt_attrs(
|
|
69
|
-
active: Any, attrs: Optional[Dict[str, Any]]
|
|
119
|
+
active: Any, attrs: Optional[Dict[str, Any]], from_input: Optional[PromptRef] = None
|
|
70
120
|
) -> Optional[Dict[str, Any]]:
|
|
71
|
-
"""Merge the
|
|
121
|
+
"""Merge the prompt this span used into its attributes.
|
|
122
|
+
|
|
123
|
+
The span's own INPUT wins over the run's most-recently-compiled prompt: it
|
|
124
|
+
is more specific (in a run using two prompts, a span holding prompt A's
|
|
125
|
+
messages must report A even if B was compiled later), and it is the only
|
|
126
|
+
signal available when compile() ran outside the run.
|
|
127
|
+
|
|
128
|
+
A prompt found this way is also folded into the run's deduped set so the
|
|
129
|
+
run-level ``trodo.prompts`` list stays complete either way.
|
|
130
|
+
"""
|
|
72
131
|
merged: Dict[str, Any] = dict(attrs or {})
|
|
73
132
|
state = getattr(active, "prompt_state", None) if active is not None else None
|
|
74
|
-
if
|
|
133
|
+
if from_input is not None:
|
|
134
|
+
remember_prompt(state, from_input)
|
|
135
|
+
merged.update(prompt_attributes(from_input))
|
|
136
|
+
elif isinstance(state, dict):
|
|
75
137
|
merged.update(prompt_attributes(state.get("current")))
|
|
76
138
|
return merged or None
|
|
77
139
|
|