trodo-python 2.20.0__tar.gz → 2.22.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trodo_python-2.20.0 → trodo_python-2.22.0}/PKG-INFO +1 -1
- {trodo_python-2.20.0 → trodo_python-2.22.0}/pyproject.toml +1 -1
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_error_enrichment.py +25 -0
- trodo_python-2.22.0/tests/test_instrumentor_resolution.py +109 -0
- trodo_python-2.22.0/tests/test_prompt_trace.py +261 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_prompts.py +64 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/__init__.py +1 -1
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/prompt_manager.py +35 -2
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/auto_instrument.py +58 -53
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/prompt_trace.py +65 -3
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/wrap_agent.py +75 -7
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/prompts/compile.py +16 -3
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/prompts/types.py +21 -12
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo_python.egg-info/PKG-INFO +1 -1
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo_python.egg-info/SOURCES.txt +1 -0
- trodo_python-2.20.0/tests/test_prompt_trace.py +0 -107
- {trodo_python-2.20.0 → trodo_python-2.22.0}/README.md +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/setup.cfg +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_anon_distinct_id.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_auto_instrument_fixes.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_cross_process_session.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_datasets.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_end_run.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_experiments.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_get_prompt_version_types.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_llm_usage_cost.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_multi_agent_shape.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_processor_methods.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_register_otel.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_start_run.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_user_traits.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/tests/test_wrap_agent_unchanged.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/api/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/api/async_client.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/api/endpoints.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/api/http_client.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/auto/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/auto/auto_event_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/client.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/dataset_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/experiment_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/group_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/people_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/managers/user_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/context.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/helpers.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/processor.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/register.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/otel/transport.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/prompts/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/prompts/template.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/queue/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/queue/batch_flusher.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/queue/event_queue.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/session/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/session/server_session.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/session/session_manager.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/types.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/user_context.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/util/__init__.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo/util/lru.py +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo_python.egg-info/dependency_links.txt +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo_python.egg-info/requires.txt +0 -0
- {trodo_python-2.20.0 → trodo_python-2.22.0}/trodo_python.egg-info/top_level.txt +0 -0
|
@@ -70,6 +70,31 @@ def test_errored_span_and_run_carry_rich_error_fields(processor, http):
|
|
|
70
70
|
assert "rate limit exceeded" in run_payload["error_summary"]
|
|
71
71
|
|
|
72
72
|
|
|
73
|
+
def test_raised_exception_carries_the_recorded_run_id(processor, http):
|
|
74
|
+
# The run IS recorded when an agent raises; this is the JOIN -- the
|
|
75
|
+
# exception a developer's error tracker captures must reference it.
|
|
76
|
+
with pytest.raises(RateLimitError) as e:
|
|
77
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
78
|
+
raise RateLimitError("boom")
|
|
79
|
+
|
|
80
|
+
run_payload = http.run_ingest[0]["run"]
|
|
81
|
+
assert e.value.trodo_run_id == run_payload["run_id"]
|
|
82
|
+
assert run_payload["status"] == "error"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_slotted_exception_still_propagates(processor, http):
|
|
86
|
+
# An exception that can't take attributes must not crash the error path --
|
|
87
|
+
# a failure inside error handling is the one unforgivable bug here.
|
|
88
|
+
class Slotted(Exception):
|
|
89
|
+
__slots__ = ()
|
|
90
|
+
|
|
91
|
+
with pytest.raises(Slotted):
|
|
92
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
93
|
+
raise Slotted("rigid")
|
|
94
|
+
|
|
95
|
+
assert http.run_ingest[0]["run"]["status"] == "error"
|
|
96
|
+
|
|
97
|
+
|
|
73
98
|
def test_set_error_records_without_raising(processor, http):
|
|
74
99
|
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat") as run:
|
|
75
100
|
with span("soft-step", kind="tool") as s:
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Instrumentor class resolution must survive upstream renames.
|
|
2
|
+
|
|
3
|
+
Every per-framework instrumentor is loaded by name, and the loader in
|
|
4
|
+
``enable_auto_instrument`` swallows failures so a missing optional package
|
|
5
|
+
never breaks user code. The cost of that tolerance: a RENAMED export is
|
|
6
|
+
indistinguishable from an absent package. The user installs exactly the
|
|
7
|
+
package we documented, gets zero spans, and gets no error explaining why.
|
|
8
|
+
|
|
9
|
+
That is not hypothetical — it happened twice. ``google-generativeai`` renamed
|
|
10
|
+
its class, and ``opentelemetry-instrumentation-langchain`` ships
|
|
11
|
+
``LangchainInstrumentor`` while we imported ``LangChainInstrumentor``, which
|
|
12
|
+
silently disabled LangChain tracing for every Python user on a current version.
|
|
13
|
+
|
|
14
|
+
So resolution tries the known spellings and then falls back to whatever single
|
|
15
|
+
``*Instrumentor`` the module exports.
|
|
16
|
+
"""
|
|
17
|
+
import sys
|
|
18
|
+
import types
|
|
19
|
+
|
|
20
|
+
import pytest
|
|
21
|
+
|
|
22
|
+
from trodo.otel.auto_instrument import _instrument
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _fake_module(name, **attrs):
|
|
26
|
+
mod = types.ModuleType(name)
|
|
27
|
+
for k, v in attrs.items():
|
|
28
|
+
setattr(mod, k, v)
|
|
29
|
+
sys.modules[name] = mod
|
|
30
|
+
return mod
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _Recorder:
|
|
34
|
+
instrumented = False
|
|
35
|
+
|
|
36
|
+
def instrument(self):
|
|
37
|
+
type(self).instrumented = True
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.fixture(autouse=True)
|
|
41
|
+
def _clean():
|
|
42
|
+
made = []
|
|
43
|
+
yield made
|
|
44
|
+
for n in made:
|
|
45
|
+
sys.modules.pop(n, None)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_uses_the_preferred_name_when_present(_clean):
|
|
49
|
+
name = 'fake_instr_preferred'
|
|
50
|
+
_clean.append(name)
|
|
51
|
+
cls = type('WantedInstrumentor', (_Recorder,), {'instrumented': False})
|
|
52
|
+
_fake_module(name, WantedInstrumentor=cls, OtherInstrumentor=type('OtherInstrumentor', (_Recorder,), {}))
|
|
53
|
+
_instrument(name, 'WantedInstrumentor')
|
|
54
|
+
assert cls.instrumented is True
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_tries_each_spelling_in_order(_clean):
|
|
58
|
+
"""The real LangChain case: we ask for both casings, upstream ships one."""
|
|
59
|
+
name = 'fake_instr_langchain'
|
|
60
|
+
_clean.append(name)
|
|
61
|
+
cls = type('LangchainInstrumentor', (_Recorder,), {'instrumented': False})
|
|
62
|
+
_fake_module(name, LangchainInstrumentor=cls)
|
|
63
|
+
# 'LangChainInstrumentor' (capital C) does not exist here — must not raise.
|
|
64
|
+
_instrument(name, 'LangchainInstrumentor', 'LangChainInstrumentor')
|
|
65
|
+
assert cls.instrumented is True
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_falls_back_to_the_only_exported_instrumentor(_clean):
|
|
69
|
+
"""An unforeseen rename still works, which is the whole point."""
|
|
70
|
+
name = 'fake_instr_renamed'
|
|
71
|
+
_clean.append(name)
|
|
72
|
+
cls = type('SomethingWeNeverHeardOfInstrumentor', (_Recorder,), {'instrumented': False})
|
|
73
|
+
_fake_module(name, SomethingWeNeverHeardOfInstrumentor=cls)
|
|
74
|
+
_instrument(name, 'ExpectedInstrumentor')
|
|
75
|
+
assert cls.instrumented is True
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_refuses_to_guess_between_several_candidates(_clean):
|
|
79
|
+
"""Picking arbitrarily could instrument the wrong thing — say so instead."""
|
|
80
|
+
name = 'fake_instr_ambiguous'
|
|
81
|
+
_clean.append(name)
|
|
82
|
+
_fake_module(name,
|
|
83
|
+
AInstrumentor=type('AInstrumentor', (_Recorder,), {}),
|
|
84
|
+
BInstrumentor=type('BInstrumentor', (_Recorder,), {}))
|
|
85
|
+
with pytest.raises(ImportError, match='several Instrumentor classes'):
|
|
86
|
+
_instrument(name, 'NopeInstrumentor')
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_reports_when_nothing_is_exported(_clean):
|
|
90
|
+
name = 'fake_instr_empty'
|
|
91
|
+
_clean.append(name)
|
|
92
|
+
_fake_module(name)
|
|
93
|
+
with pytest.raises(ImportError, match='no Instrumentor class'):
|
|
94
|
+
_instrument(name, 'AnyInstrumentor')
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def test_base_instrumentor_is_never_the_fallback(_clean):
|
|
98
|
+
"""Modules re-export BaseInstrumentor; instrumenting it would be wrong."""
|
|
99
|
+
name = 'fake_instr_baseonly'
|
|
100
|
+
_clean.append(name)
|
|
101
|
+
_fake_module(name, BaseInstrumentor=type('BaseInstrumentor', (_Recorder,), {}))
|
|
102
|
+
with pytest.raises(ImportError, match='no Instrumentor class'):
|
|
103
|
+
_instrument(name, 'AnyInstrumentor')
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_a_missing_module_still_raises_importerror(_clean):
|
|
107
|
+
"""The caller relies on ImportError to mean "skip this framework"."""
|
|
108
|
+
with pytest.raises(ImportError):
|
|
109
|
+
_instrument('module_that_does_not_exist_xyz', 'AnyInstrumentor')
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""Prompt -> span traceability (mirrors the Node promptTrace.test.ts).
|
|
2
|
+
|
|
3
|
+
Compiling a managed prompt must stamp the exact version (by immutable hash)
|
|
4
|
+
onto the spans that used it, and aggregate every prompt used onto the run.
|
|
5
|
+
|
|
6
|
+
Two routes get it there, and both are load-bearing:
|
|
7
|
+
- the RUN SCOPE, when compile() runs inside wrap_agent/span. The only route
|
|
8
|
+
that reaches auto-instrumented spans, whose input we never see.
|
|
9
|
+
- the COMPILED VALUE, when those messages are passed to a span as ``input``.
|
|
10
|
+
The only route that survives compiling outside the run, which is an
|
|
11
|
+
ordinary shape and used to produce no link at all.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from trodo.managers.prompt_manager import PromptManager
|
|
16
|
+
from trodo.otel.wrap_agent import span, wrap_agent
|
|
17
|
+
|
|
18
|
+
PROMPT_WIRE = {
|
|
19
|
+
"name": "refund-agent",
|
|
20
|
+
"version": 3,
|
|
21
|
+
"version_hash": "a3f9c2e8d1b04756",
|
|
22
|
+
"content_hash": "ccccontent1234",
|
|
23
|
+
"parent_hash": "pppparent5678",
|
|
24
|
+
"labels": ["production"],
|
|
25
|
+
"tags": [],
|
|
26
|
+
"messages": [{"role": "system", "content": [{"type": "text", "text": "Help {{q}}"}]}],
|
|
27
|
+
"model": {"provider": "openai", "model": "gpt-4o"},
|
|
28
|
+
"tools": [],
|
|
29
|
+
"response_format": None,
|
|
30
|
+
"variables": [{"name": "q", "type": "string"}],
|
|
31
|
+
"updated_at": "2026-07-01T00:00:00Z",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class PromptHttp:
|
|
36
|
+
def __init__(self, prompt=None, fail=False):
|
|
37
|
+
self._prompt = prompt
|
|
38
|
+
self.fail = fail
|
|
39
|
+
|
|
40
|
+
def get_prompt(self, name, label=None, version=None):
|
|
41
|
+
if self.fail:
|
|
42
|
+
raise RuntimeError("unreachable")
|
|
43
|
+
return {"prompt": dict(self._prompt)}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_stamps_version_hash_on_span_and_aggregates_on_run(processor, http):
|
|
47
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
48
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
49
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
50
|
+
with span("llm", kind="llm"):
|
|
51
|
+
p.compile(q="my order")
|
|
52
|
+
|
|
53
|
+
ingest = http.run_ingest[0]
|
|
54
|
+
spans = ingest["spans"]
|
|
55
|
+
llm = next(s for s in spans if s["name"] == "llm")
|
|
56
|
+
attrs = llm["attributes"]
|
|
57
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
58
|
+
assert attrs["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
59
|
+
assert attrs["trodo.prompt.label"] == "production" # default selector
|
|
60
|
+
assert attrs["trodo.prompt.content_hash"] == "ccccontent1234"
|
|
61
|
+
|
|
62
|
+
used = ingest["run"]["attributes"]["trodo.prompts"]
|
|
63
|
+
assert len(used) == 1
|
|
64
|
+
assert used[0]["version_hash"] == "a3f9c2e8d1b04756"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_records_label_and_omits_it_for_version_pin(processor, http):
|
|
68
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
69
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
70
|
+
staged = mgr.get("refund-agent", label="staging", cache_ttl_seconds=0)
|
|
71
|
+
with span("s1"):
|
|
72
|
+
staged.compile(q="x")
|
|
73
|
+
pinned = mgr.get("refund-agent", version="a3f9c2", cache_ttl_seconds=0)
|
|
74
|
+
with span("s2"):
|
|
75
|
+
pinned.compile(q="y")
|
|
76
|
+
|
|
77
|
+
spans = http.run_ingest[0]["spans"]
|
|
78
|
+
s1 = next(s for s in spans if s["name"] == "s1")["attributes"]
|
|
79
|
+
s2 = next(s for s in spans if s["name"] == "s2")["attributes"]
|
|
80
|
+
assert s1["trodo.prompt.label"] == "staging"
|
|
81
|
+
assert "trodo.prompt.label" not in s2 # pinned by hash -> no label
|
|
82
|
+
assert s2["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_no_op_outside_a_run(processor, http):
|
|
86
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
87
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
88
|
+
# Must not raise despite there being no active run context.
|
|
89
|
+
p.compile(q="z")
|
|
90
|
+
assert len(http.run_ingest) == 0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_fallback_prompt_is_not_traced(processor, http):
|
|
94
|
+
mgr = PromptManager(PromptHttp(fail=True))
|
|
95
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
96
|
+
p = mgr.get(
|
|
97
|
+
"refund-agent",
|
|
98
|
+
cache_ttl_seconds=0,
|
|
99
|
+
max_retries=0,
|
|
100
|
+
fallback={
|
|
101
|
+
"messages": [{"role": "user", "content": [{"type": "text", "text": "hi {{q}}"}]}],
|
|
102
|
+
"variables": [{"name": "q"}],
|
|
103
|
+
},
|
|
104
|
+
)
|
|
105
|
+
assert p.is_fallback is True
|
|
106
|
+
with span("llm"):
|
|
107
|
+
p.compile(q="x")
|
|
108
|
+
|
|
109
|
+
spans = http.run_ingest[0]["spans"]
|
|
110
|
+
attrs = next(s for s in spans if s["name"] == "llm").get("attributes") or {}
|
|
111
|
+
assert "trodo.prompt.name" not in attrs
|
|
112
|
+
# to_dict() drops None, so an untraced run has no 'attributes' key at all.
|
|
113
|
+
assert "attributes" not in http.run_ingest[0]["run"]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# ── compiled OUTSIDE the run ────────────────────────────────────────────────
|
|
117
|
+
#
|
|
118
|
+
# Fetch and compile up front, open the run around just the model call. Under the
|
|
119
|
+
# run-scope mechanism alone this produced no link whatsoever.
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_links_span_whose_input_was_compiled_outside_the_run(processor, http):
|
|
123
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
124
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
125
|
+
compiled = p.compile(q="z") # OUTSIDE any run
|
|
126
|
+
|
|
127
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
128
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
129
|
+
pass
|
|
130
|
+
|
|
131
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm")["attributes"]
|
|
132
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
133
|
+
assert attrs["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_outside_compiled_prompt_reaches_the_run_level_list(processor, http):
|
|
137
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
138
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
139
|
+
compiled = p.compile(q="z")
|
|
140
|
+
|
|
141
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
142
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
143
|
+
pass
|
|
144
|
+
|
|
145
|
+
used = http.run_ingest[0]["run"]["attributes"]["trodo.prompts"]
|
|
146
|
+
assert [r["name"] for r in used] == ["refund-agent"]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_the_compiled_wrapper_works_as_input_too(processor, http):
|
|
150
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
151
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
152
|
+
compiled = p.compile(q="z")
|
|
153
|
+
|
|
154
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
155
|
+
with span("llm", kind="llm", input=compiled):
|
|
156
|
+
pass
|
|
157
|
+
|
|
158
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm")["attributes"]
|
|
159
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_span_input_beats_the_runs_most_recent_prompt(processor, http):
|
|
163
|
+
"""Specificity: a span holding prompt A must say A, not whatever came last."""
|
|
164
|
+
other = dict(PROMPT_WIRE, name="other-agent", version_hash="bbbb2222", content_hash="ccc2")
|
|
165
|
+
|
|
166
|
+
class TwoPrompts:
|
|
167
|
+
def get_prompt(self, name, label=None, version=None):
|
|
168
|
+
return {"prompt": dict(other if name == "other-agent" else PROMPT_WIRE)}
|
|
169
|
+
|
|
170
|
+
mgr = PromptManager(TwoPrompts())
|
|
171
|
+
a = mgr.get("refund-agent", cache_ttl_seconds=0).compile(q="first")
|
|
172
|
+
|
|
173
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
174
|
+
# Compiling B inside the run makes B the run's "current" prompt.
|
|
175
|
+
mgr.get("other-agent", cache_ttl_seconds=0).compile(q="second")
|
|
176
|
+
with span("llm", kind="llm", input=a.messages):
|
|
177
|
+
pass
|
|
178
|
+
|
|
179
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm")["attributes"]
|
|
180
|
+
assert attrs["trodo.prompt.name"] == "refund-agent"
|
|
181
|
+
assert attrs["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def test_a_span_that_never_saw_a_prompt_is_not_labelled(processor, http):
|
|
185
|
+
"""False-positive guard: attribution must not leak to sibling spans."""
|
|
186
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
187
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
188
|
+
compiled = p.compile(q="z")
|
|
189
|
+
|
|
190
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
191
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
192
|
+
pass
|
|
193
|
+
with span("unrelated", input={"some": "payload"}):
|
|
194
|
+
pass
|
|
195
|
+
|
|
196
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "unrelated").get("attributes") or {}
|
|
197
|
+
assert "trodo.prompt.name" not in attrs
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_hand_written_messages_are_not_carriers(processor, http):
|
|
201
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
202
|
+
with span("llm", kind="llm", input=[{"role": "user", "content": "hand-written"}]):
|
|
203
|
+
pass
|
|
204
|
+
|
|
205
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm").get("attributes") or {}
|
|
206
|
+
assert "trodo.prompt.name" not in attrs
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_a_fallback_is_never_tagged_via_the_input_route(processor, http):
|
|
210
|
+
mgr = PromptManager(PromptHttp(fail=True))
|
|
211
|
+
p = mgr.get(
|
|
212
|
+
"refund-agent",
|
|
213
|
+
fallback={"messages": [{"role": "user", "content": [{"type": "text", "text": "hi {{q}}"}]}],
|
|
214
|
+
"variables": [{"name": "q"}]},
|
|
215
|
+
cache_ttl_seconds=0,
|
|
216
|
+
)
|
|
217
|
+
assert p.is_fallback
|
|
218
|
+
compiled = p.compile(q="x")
|
|
219
|
+
|
|
220
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat"):
|
|
221
|
+
with span("llm", kind="llm", input=compiled.messages):
|
|
222
|
+
pass
|
|
223
|
+
|
|
224
|
+
attrs = next(s for s in http.run_ingest[0]["spans"] if s["name"] == "llm").get("attributes") or {}
|
|
225
|
+
assert "trodo.prompt.name" not in attrs
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def test_carrier_table_is_bounded(processor, http):
|
|
229
|
+
"""It must not grow without limit in a long-lived process."""
|
|
230
|
+
from trodo.otel import prompt_trace
|
|
231
|
+
|
|
232
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
233
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
234
|
+
for _ in range(prompt_trace._MAX_CARRIERS + 200):
|
|
235
|
+
p.compile(q="x")
|
|
236
|
+
|
|
237
|
+
assert len(prompt_trace._carrier_refs) <= prompt_trace._MAX_CARRIERS
|
|
238
|
+
assert len(prompt_trace._carrier_keepalive) <= prompt_trace._MAX_CARRIERS
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def test_run_set_input_records_the_prompt_on_the_run(processor, http):
|
|
242
|
+
"""The shape that still leaves auto-instrumented spans unlabelled: compile
|
|
243
|
+
up front, hand the messages to the RUN, let a framework make the call. The
|
|
244
|
+
run can at least say which prompt it ran."""
|
|
245
|
+
mgr = PromptManager(PromptHttp(prompt=PROMPT_WIRE))
|
|
246
|
+
p = mgr.get("refund-agent", cache_ttl_seconds=0)
|
|
247
|
+
compiled = p.compile(q="z")
|
|
248
|
+
|
|
249
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat") as run:
|
|
250
|
+
run.set_input(compiled.messages)
|
|
251
|
+
|
|
252
|
+
used = http.run_ingest[0]["run"]["attributes"]["trodo.prompts"]
|
|
253
|
+
assert [r["name"] for r in used] == ["refund-agent"]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def test_run_set_input_with_ordinary_input_records_nothing(processor, http):
|
|
257
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat") as run:
|
|
258
|
+
run.set_input({"query": "hello"})
|
|
259
|
+
|
|
260
|
+
attrs = http.run_ingest[0]["run"].get("attributes") or {}
|
|
261
|
+
assert "trodo.prompts" not in attrs
|
|
@@ -133,6 +133,16 @@ def test_compile_applies_a_default_when_omitted():
|
|
|
133
133
|
assert out.messages[1]["content"][0] == {"type": "text", "text": "Hello world"}
|
|
134
134
|
|
|
135
135
|
|
|
136
|
+
def test_coercion_error_text_matches_node_byte_for_byte():
|
|
137
|
+
# Node formats the offending value with JSON.stringify; repr's single
|
|
138
|
+
# quotes were the ONE divergence a full cross-SDK parity run found. The
|
|
139
|
+
# double quotes here are the point of the test.
|
|
140
|
+
numeric = {**PROMPT, "variables": [{"name": "who", "type": "number"}]}
|
|
141
|
+
with pytest.raises(CompileError) as e:
|
|
142
|
+
compile_prompt(numeric, {"who": "not-a-number"})
|
|
143
|
+
assert 'variable \'who\': expected a number, got "not-a-number"' in str(e.value)
|
|
144
|
+
|
|
145
|
+
|
|
136
146
|
def test_compile_rejects_undeclared_variable():
|
|
137
147
|
# Passing a variable the prompt doesn't declare is the one variable mistake
|
|
138
148
|
# compile still catches (it's almost always a typo/rename).
|
|
@@ -218,6 +228,8 @@ class FakeHttp:
|
|
|
218
228
|
self.last = {"name": name, "label": label, "version": version}
|
|
219
229
|
if self.fail:
|
|
220
230
|
raise RuntimeError("API down")
|
|
231
|
+
if getattr(self, "error_body", None):
|
|
232
|
+
return dict(self.error_body)
|
|
221
233
|
return {"prompt": self._prompt}
|
|
222
234
|
|
|
223
235
|
def list_prompts(self):
|
|
@@ -289,6 +301,58 @@ def test_stale_cache_survives_an_outage():
|
|
|
289
301
|
assert mgr.get("greeter", cache_ttl_seconds=0.01).version == 2
|
|
290
302
|
|
|
291
303
|
|
|
304
|
+
def test_missing_prompt_with_fallback_serves_fallback():
|
|
305
|
+
# Deleting a prompt mid-deploy is exactly what the fallback exists for --
|
|
306
|
+
# plain prompt_not_found stays an availability failure.
|
|
307
|
+
http = FakeHttp()
|
|
308
|
+
http.error_body = {"__error": True, "status": 404, "error": "prompt_not_found"}
|
|
309
|
+
mgr = PromptManager(http)
|
|
310
|
+
p = mgr.get("gone", cache_ttl_seconds=0, max_retries=0,
|
|
311
|
+
fallback={"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]})
|
|
312
|
+
assert p.is_fallback is True
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def test_bad_version_raises_through_the_fallback():
|
|
316
|
+
# A selector naming nothing is a CONFIG error, not an outage. Being quietly
|
|
317
|
+
# handed the fallback would hide the typo for as long as it ships.
|
|
318
|
+
http = FakeHttp()
|
|
319
|
+
http.error_body = {"__error": True, "status": 404, "error": "version_not_found"}
|
|
320
|
+
mgr = PromptManager(http)
|
|
321
|
+
with pytest.raises(LookupError, match="version 999 does not exist on prompt 'greeter'") as e:
|
|
322
|
+
mgr.get("greeter", version=999, cache_ttl_seconds=0, max_retries=0,
|
|
323
|
+
fallback={"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]})
|
|
324
|
+
assert e.value.code == "version_not_found"
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def test_bad_label_raises_through_the_fallback():
|
|
328
|
+
http = FakeHttp()
|
|
329
|
+
http.error_body = {"__error": True, "status": 404, "error": "label_not_found"}
|
|
330
|
+
mgr = PromptManager(http)
|
|
331
|
+
with pytest.raises(LookupError) as e:
|
|
332
|
+
mgr.get("greeter", label="prod-ue", cache_ttl_seconds=0, max_retries=0,
|
|
333
|
+
fallback={"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]})
|
|
334
|
+
assert e.value.code == "label_not_found"
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def test_versionless_prompt_says_what_to_do():
|
|
338
|
+
http = FakeHttp()
|
|
339
|
+
http.error_body = {"__error": True, "status": 404, "error": "no_versions"}
|
|
340
|
+
mgr = PromptManager(http)
|
|
341
|
+
with pytest.raises(LookupError, match="has no versions yet"):
|
|
342
|
+
mgr.get("empty", cache_ttl_seconds=0, max_retries=0)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def test_config_error_not_masked_by_stale_cache():
|
|
346
|
+
# First fetch succeeds and caches; the label is then deleted server-side.
|
|
347
|
+
http = FakeHttp(prompt=WIRE)
|
|
348
|
+
mgr = PromptManager(http)
|
|
349
|
+
assert mgr.get("greeter", label="production", cache_ttl_seconds=0).version == 2
|
|
350
|
+
http.error_body = {"__error": True, "status": 404, "error": "label_not_found"}
|
|
351
|
+
with pytest.raises(LookupError) as e:
|
|
352
|
+
mgr.get("greeter", label="production", cache_ttl_seconds=0, max_retries=0)
|
|
353
|
+
assert e.value.code == "label_not_found"
|
|
354
|
+
|
|
355
|
+
|
|
292
356
|
def test_fallback_used_when_api_down_and_nothing_cached():
|
|
293
357
|
mgr = PromptManager(FakeHttp(fail=True))
|
|
294
358
|
p = mgr.get(
|
|
@@ -38,6 +38,11 @@ __all__ = [
|
|
|
38
38
|
DEFAULT_TTL_SECONDS = 60.0
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
# 404 bodies that mean "your SELECTOR is wrong", not "the prompt is gone".
|
|
42
|
+
# Mirrors backend/models/prompt.js missReason -- the wire contract's error half.
|
|
43
|
+
_CONFIG_ERROR_CODES = frozenset({"version_not_found", "label_not_found", "no_versions"})
|
|
44
|
+
|
|
45
|
+
|
|
41
46
|
def _cache_key(name: str, version: Optional[Union[int, str]], label: Optional[str]) -> str:
|
|
42
47
|
# Resolution happens server-side on every fetch; the client only caches
|
|
43
48
|
# under whatever selector was asked for. So a label flip propagates within
|
|
@@ -197,7 +202,14 @@ class PromptManager:
|
|
|
197
202
|
Availability ladder — fresh cache -> stale cache -> ``fallback`` ->
|
|
198
203
|
raise. A prompt fetch is on your hot path, so a Trodo outage degrades
|
|
199
204
|
rather than takes your app down. Check ``prompt.is_fallback`` to detect
|
|
200
|
-
the last rung.
|
|
205
|
+
the last rung.
|
|
206
|
+
|
|
207
|
+
The ladder is for AVAILABILITY failures only. A selector that names
|
|
208
|
+
nothing — a ``version`` or ``label`` that doesn't exist on a prompt
|
|
209
|
+
that does — is a config error in your code and raises immediately
|
|
210
|
+
(``e.code`` is ``version_not_found`` | ``label_not_found`` |
|
|
211
|
+
``no_versions``), because being quietly handed the fallback would hide
|
|
212
|
+
the typo for as long as it ships. Pass ``cache_ttl_seconds=0`` to disable caching (handy in
|
|
201
213
|
development).
|
|
202
214
|
|
|
203
215
|
:raises ValueError: if *name* is empty, or both ``label`` and ``version``
|
|
@@ -224,6 +236,22 @@ class PromptManager:
|
|
|
224
236
|
if not res or res.get("__error") or not res.get("prompt"):
|
|
225
237
|
status = res.get("status") if isinstance(res, dict) else None
|
|
226
238
|
detail = res.get("error") if isinstance(res, dict) else None
|
|
239
|
+
# The server distinguishes a selector that names nothing from a
|
|
240
|
+
# prompt that is missing. The first is a CONFIG error -- your
|
|
241
|
+
# code asks for a version or label that does not exist -- and
|
|
242
|
+
# must throw through the availability ladder below rather than
|
|
243
|
+
# be masked by stale content or the fallback.
|
|
244
|
+
if detail in _CONFIG_ERROR_CODES:
|
|
245
|
+
if detail == "version_not_found":
|
|
246
|
+
what = f"version {version!r} does not exist on prompt {name!r}"
|
|
247
|
+
elif detail == "label_not_found":
|
|
248
|
+
what = f"label {label!r} does not exist on prompt {name!r}"
|
|
249
|
+
else:
|
|
250
|
+
what = f"prompt {name!r} has no versions yet -- save one in the dashboard"
|
|
251
|
+
err = LookupError(f"trodo: {what}" + (f" (HTTP {status})" if status else ""))
|
|
252
|
+
err.code = detail # type: ignore[attr-defined]
|
|
253
|
+
err.prompt_config_error = True # type: ignore[attr-defined]
|
|
254
|
+
raise err
|
|
227
255
|
raise LookupError(
|
|
228
256
|
f"trodo: could not fetch prompt {name!r}"
|
|
229
257
|
+ (f" (HTTP {status})" if status else "")
|
|
@@ -248,7 +276,12 @@ class PromptManager:
|
|
|
248
276
|
if ttl > 0:
|
|
249
277
|
self._cache.set(key, raw, ttl)
|
|
250
278
|
return _to_prompt(raw, trace_label=trace_label)
|
|
251
|
-
except Exception:
|
|
279
|
+
except Exception as e:
|
|
280
|
+
# A config error is not an outage: a typo'd label silently serving
|
|
281
|
+
# the fallback forever would hide the mistake for as long as it
|
|
282
|
+
# ships. Config errors surface.
|
|
283
|
+
if getattr(e, "prompt_config_error", False):
|
|
284
|
+
raise
|
|
252
285
|
stale = self._cache.get_stale(key)
|
|
253
286
|
if stale is not None:
|
|
254
287
|
return _to_prompt(stale, trace_label=trace_label)
|