trodo-python 2.23.4__tar.gz → 2.25.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trodo_python-2.23.4 → trodo_python-2.25.0}/PKG-INFO +1 -1
- {trodo_python-2.23.4 → trodo_python-2.25.0}/pyproject.toml +1 -1
- trodo_python-2.25.0/tests/test_prompt_fingerprint.py +198 -0
- trodo_python-2.25.0/tests/test_sender.py +273 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/__init__.py +18 -1
- trodo_python-2.25.0/trodo/_version.py +1 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/api/endpoints.py +3 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/api/http_client.py +33 -1
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/client.py +40 -7
- trodo_python-2.25.0/trodo/delivery/sender.py +500 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/auto_instrument.py +73 -2
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/processor.py +19 -14
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/prompt_trace.py +185 -1
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/register.py +8 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/wrap_agent.py +11 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/prompts/types.py +6 -0
- trodo_python-2.25.0/trodo/session/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo_python.egg-info/PKG-INFO +1 -1
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo_python.egg-info/SOURCES.txt +5 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/README.md +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/setup.cfg +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_anon_distinct_id.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_auto_instrument_fixes.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_cross_process_session.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_datasets.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_end_run.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_error_enrichment.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_experiments.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_framework_deference.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_get_prompt_version_types.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_instrumentor_resolution.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_llm_usage_cost.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_multi_agent_shape.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_processor_methods.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_prompt_trace.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_prompts.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_register_otel.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_start_run.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_user_traits.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/tests/test_wrap_agent_unchanged.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/api/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/api/_retry.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/api/async_client.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/auto/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/auto/auto_event_manager.py +0 -0
- {trodo_python-2.23.4/trodo/managers → trodo_python-2.25.0/trodo/delivery}/__init__.py +0 -0
- {trodo_python-2.23.4/trodo/queue → trodo_python-2.25.0/trodo/managers}/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/managers/dataset_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/managers/experiment_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/managers/group_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/managers/people_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/managers/prompt_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/managers/user_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/context.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/helpers.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/otel/transport.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/prompts/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/prompts/compile.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/prompts/template.py +0 -0
- {trodo_python-2.23.4/trodo/session → trodo_python-2.25.0/trodo/queue}/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/queue/batch_flusher.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/queue/event_queue.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/session/server_session.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/session/session_manager.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/types.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/user_context.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/util/__init__.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo/util/lru.py +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo_python.egg-info/dependency_links.txt +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo_python.egg-info/requires.txt +0 -0
- {trodo_python-2.23.4 → trodo_python-2.25.0}/trodo_python.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Exact-match prompt linking for AUTO-INSTRUMENTED provider spans.
|
|
2
|
+
|
|
3
|
+
Mirrors the Node promptFingerprint.test.ts. compile() before the run plus a
|
|
4
|
+
provider call made by an auto-instrumentor leaves no run context at compile
|
|
5
|
+
time and no carrier on the span's serialised input; compile() therefore
|
|
6
|
+
registers a content fingerprint, and a model span whose input has identical
|
|
7
|
+
(role, text) content is linked. Exact or nothing.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from types import SimpleNamespace
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from trodo.managers.prompt_manager import PromptManager
|
|
16
|
+
from trodo.otel import prompt_trace as pt
|
|
17
|
+
from trodo.otel.auto_instrument import _OtelAdapter
|
|
18
|
+
from trodo.otel.context import ActiveSpanContext, run_with_context
|
|
19
|
+
|
|
20
|
+
RUN_ID = "11111111-1111-1111-1111-111111111111"
|
|
21
|
+
ROOT = "22222222-2222-2222-2222-222222222222"
|
|
22
|
+
|
|
23
|
+
WIRE = {
|
|
24
|
+
"name": "refund-agent",
|
|
25
|
+
"version": 3,
|
|
26
|
+
"version_hash": "a3f9c2e8d1b04756",
|
|
27
|
+
"content_hash": "ccccontent1234",
|
|
28
|
+
"labels": ["production"],
|
|
29
|
+
"tags": [],
|
|
30
|
+
"messages": [
|
|
31
|
+
{"role": "system", "content": [{"type": "text", "text": "You help with {{q}}"}]},
|
|
32
|
+
{"role": "user", "content": [{"type": "text", "text": "Hello {{q}}"}]},
|
|
33
|
+
],
|
|
34
|
+
"model": {"provider": "openai", "model": "gpt-4o"},
|
|
35
|
+
"tools": [],
|
|
36
|
+
"response_format": None,
|
|
37
|
+
"variables": [{"name": "q", "type": "string"}],
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class PromptHttp:
|
|
42
|
+
def __init__(self, fail=False):
|
|
43
|
+
self.fail = fail
|
|
44
|
+
|
|
45
|
+
def get_prompt(self, name, label=None, version=None):
|
|
46
|
+
if self.fail:
|
|
47
|
+
raise RuntimeError("unreachable")
|
|
48
|
+
return {"prompt": dict(WIRE)}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@pytest.fixture(autouse=True)
|
|
52
|
+
def _clean():
|
|
53
|
+
pt._reset_fingerprints_for_tests()
|
|
54
|
+
yield
|
|
55
|
+
pt._reset_fingerprints_for_tests()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def provider_span(messages, extra=None):
|
|
59
|
+
attrs = {
|
|
60
|
+
"trodo.run_id": RUN_ID,
|
|
61
|
+
"trodo.parent_span_id": ROOT,
|
|
62
|
+
"gen_ai.request.model": "gpt-4o",
|
|
63
|
+
"gen_ai.system": "openai",
|
|
64
|
+
**(extra or {}),
|
|
65
|
+
}
|
|
66
|
+
for i, m in enumerate(messages):
|
|
67
|
+
attrs[f"gen_ai.prompt.{i}.role"] = m["role"]
|
|
68
|
+
attrs[f"gen_ai.prompt.{i}.content"] = m["content"]
|
|
69
|
+
span_context = SimpleNamespace(span_id=0xABCD1234ABCD1234)
|
|
70
|
+
return SimpleNamespace(
|
|
71
|
+
name="openai.chat",
|
|
72
|
+
get_span_context=lambda: span_context,
|
|
73
|
+
attributes=attrs,
|
|
74
|
+
start_time=1_000_000_000,
|
|
75
|
+
end_time=2_000_000_000,
|
|
76
|
+
status=None,
|
|
77
|
+
parent=None,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def end_span(otel):
|
|
82
|
+
captured = []
|
|
83
|
+
proc = SimpleNamespace(enqueue_span=captured.append)
|
|
84
|
+
_OtelAdapter(proc).on_end(otel)
|
|
85
|
+
return captured[0] if captured else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_links_after_compile_before_run_with_identical_messages():
|
|
89
|
+
p = PromptManager(PromptHttp()).get("refund-agent", cache_ttl_seconds=0)
|
|
90
|
+
p.compile(q="my order") # no run, no span
|
|
91
|
+
span = end_span(provider_span([
|
|
92
|
+
{"role": "system", "content": "You help with my order"},
|
|
93
|
+
{"role": "user", "content": "Hello my order"},
|
|
94
|
+
]))
|
|
95
|
+
assert span.attributes["trodo.prompt.name"] == "refund-agent"
|
|
96
|
+
assert span.attributes["trodo.prompt.version_hash"] == "a3f9c2e8d1b04756"
|
|
97
|
+
assert span.attributes["trodo.prompt.label"] == "production"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_matches_parts_arrays_and_anthropic_system_shape():
|
|
101
|
+
p = PromptManager(PromptHttp()).get("refund-agent", cache_ttl_seconds=0)
|
|
102
|
+
compiled = p.compile(q="x")
|
|
103
|
+
assert pt.prompt_ref_from_fingerprint(compiled.messages)["name"] == "refund-agent"
|
|
104
|
+
assert pt.prompt_ref_from_fingerprint({
|
|
105
|
+
"system": "You help with x",
|
|
106
|
+
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello x"}]}],
|
|
107
|
+
})["name"] == "refund-agent"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_different_messages_do_not_link():
|
|
111
|
+
p = PromptManager(PromptHttp()).get("refund-agent", cache_ttl_seconds=0)
|
|
112
|
+
p.compile(q="my order")
|
|
113
|
+
span = end_span(provider_span([
|
|
114
|
+
{"role": "system", "content": "You help with my order"},
|
|
115
|
+
{"role": "user", "content": "Hello someone else"},
|
|
116
|
+
]))
|
|
117
|
+
assert "trodo.prompt.name" not in (span.attributes or {})
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_fallback_is_never_linked():
|
|
121
|
+
mgr = PromptManager(PromptHttp(fail=True))
|
|
122
|
+
p = mgr.get(
|
|
123
|
+
"refund-agent",
|
|
124
|
+
cache_ttl_seconds=0,
|
|
125
|
+
fallback={"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], "variables": []},
|
|
126
|
+
)
|
|
127
|
+
assert p.is_fallback
|
|
128
|
+
p.compile()
|
|
129
|
+
assert pt._fingerprint_count_for_tests() == 0
|
|
130
|
+
span = end_span(provider_span([{"role": "user", "content": "hi"}]))
|
|
131
|
+
assert "trodo.prompt.name" not in (span.attributes or {})
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_run_level_list_gets_the_prompt_via_on_start_state():
|
|
135
|
+
p = PromptManager(PromptHttp()).get("refund-agent", cache_ttl_seconds=0)
|
|
136
|
+
p.compile(q="a")
|
|
137
|
+
state = pt.new_prompt_state()
|
|
138
|
+
otel = provider_span([
|
|
139
|
+
{"role": "system", "content": "You help with a"},
|
|
140
|
+
{"role": "user", "content": "Hello a"},
|
|
141
|
+
])
|
|
142
|
+
otel.set_attribute = lambda k, v: None
|
|
143
|
+
captured = []
|
|
144
|
+
adapter = _OtelAdapter(SimpleNamespace(enqueue_span=captured.append))
|
|
145
|
+
ctx = ActiveSpanContext(
|
|
146
|
+
run_id=RUN_ID, span_id=ROOT, parent_span_id=None,
|
|
147
|
+
team_site_id="site-x", processor=None, prompt_state=state,
|
|
148
|
+
)
|
|
149
|
+
with run_with_context(ctx):
|
|
150
|
+
adapter.on_start(otel)
|
|
151
|
+
adapter.on_end(otel)
|
|
152
|
+
assert captured[0].attributes["trodo.prompt.name"] == "refund-agent"
|
|
153
|
+
assert [r["name"] for r in state["all"]] == ["refund-agent"]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_ttl_expiry(monkeypatch):
|
|
157
|
+
t = [1_000_000.0]
|
|
158
|
+
monkeypatch.setattr(pt, "_now", lambda: t[0])
|
|
159
|
+
ref = {"name": "p", "version_hash": "h1"}
|
|
160
|
+
msgs = [{"role": "user", "content": "hello"}]
|
|
161
|
+
pt.register_compiled_messages(msgs, ref)
|
|
162
|
+
assert pt.prompt_ref_from_fingerprint(msgs) == ref
|
|
163
|
+
t[0] += pt.FINGERPRINT_TTL_SECONDS + 1
|
|
164
|
+
assert pt.prompt_ref_from_fingerprint(msgs) is None
|
|
165
|
+
assert pt._fingerprint_count_for_tests() == 0
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def test_lru_eviction_beyond_cap():
|
|
169
|
+
for i in range(pt.FINGERPRINT_MAX_ENTRIES + 1):
|
|
170
|
+
pt.register_compiled_messages(
|
|
171
|
+
[{"role": "user", "content": f"m{i}"}], {"name": f"p{i}", "version_hash": f"h{i}"}
|
|
172
|
+
)
|
|
173
|
+
assert pt._fingerprint_count_for_tests() == pt.FINGERPRINT_MAX_ENTRIES
|
|
174
|
+
assert pt.prompt_ref_from_fingerprint([{"role": "user", "content": "m0"}]) is None
|
|
175
|
+
assert pt.prompt_ref_from_fingerprint([{"role": "user", "content": "m1"}])["name"] == "p1"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_most_recent_compile_wins_on_identical_content():
|
|
179
|
+
msgs = [{"role": "user", "content": "same"}]
|
|
180
|
+
pt.register_compiled_messages(msgs, {"name": "p", "version_hash": "old"})
|
|
181
|
+
pt.register_compiled_messages(msgs, {"name": "p", "version_hash": "new"})
|
|
182
|
+
assert pt.prompt_ref_from_fingerprint(msgs)["version_hash"] == "new"
|
|
183
|
+
assert pt._fingerprint_count_for_tests() == 1
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_no_work_when_nothing_compiled(monkeypatch):
|
|
187
|
+
def boom(_):
|
|
188
|
+
raise AssertionError("should not fingerprint")
|
|
189
|
+
|
|
190
|
+
monkeypatch.setattr(pt, "fingerprint_messages", boom)
|
|
191
|
+
assert pt.prompt_ref_from_fingerprint([{"role": "user", "content": "x"}]) is None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def test_role_and_text_exact():
|
|
195
|
+
a = pt.fingerprint_messages([{"role": "user", "content": "x"}])
|
|
196
|
+
assert a != pt.fingerprint_messages([{"role": "assistant", "content": "x"}])
|
|
197
|
+
assert a != pt.fingerprint_messages([{"role": "user", "content": "x "}])
|
|
198
|
+
assert a == pt.fingerprint_messages('[{"role":"user","content":"x"}]')
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""Background delivery (2.24): the agent's caller never waits on Trodo, and a
|
|
2
|
+
trace is lost only when the queue is full, the server rejects it, or ten
|
|
3
|
+
minutes of retrying did not get it accepted.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
from typing import Any, Dict, List, Optional
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
from trodo.delivery.sender import Sender, detect_mode, parse_retry_after_s
|
|
15
|
+
from trodo.otel.processor import TrodoSpanProcessor
|
|
16
|
+
from trodo.otel.wrap_agent import wrap_agent
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Server:
|
|
20
|
+
"""A fake /api/sdk/batch: a scripted list of replies, then 200 for everything."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, script: Optional[List[Any]] = None, delay_s: float = 0.0) -> None:
|
|
23
|
+
self.script = list(script or [])
|
|
24
|
+
self.posts: List[Dict[str, Any]] = []
|
|
25
|
+
self.delay_s = delay_s
|
|
26
|
+
self.lock = threading.Lock()
|
|
27
|
+
|
|
28
|
+
def post_raw(self, path: str, body: Dict[str, Any], timeout: float = 0, gzip_body: bool = True):
|
|
29
|
+
if self.delay_s:
|
|
30
|
+
time.sleep(self.delay_s)
|
|
31
|
+
with self.lock:
|
|
32
|
+
self.posts.append({"path": path, "body": body, "at": time.time()})
|
|
33
|
+
nxt = self.script.pop(0) if self.script else None
|
|
34
|
+
if isinstance(nxt, Exception):
|
|
35
|
+
raise nxt
|
|
36
|
+
if nxt is not None:
|
|
37
|
+
return nxt
|
|
38
|
+
return 200, None, {"results": [{"status": 200} for _ in body["items"]]}
|
|
39
|
+
|
|
40
|
+
def post_run_ingest(self, p):
|
|
41
|
+
self.posts.append({"path": "legacy", "body": {"items": [p]}})
|
|
42
|
+
return {}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def run(rid: str) -> Dict[str, Any]:
|
|
46
|
+
return {"run": {"run_id": rid, "agent_name": "bot"}, "spans": [{"span_id": f"{rid}-s1"}]}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def fast(**o: Any) -> Dict[str, Any]:
|
|
50
|
+
"""Sender options that make a test run in well under a second."""
|
|
51
|
+
return {"mode": "background", "on_error": lambda e: None, "register_atexit": False, "flush_interval_s": 0.05, **o}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def wait_for(cond, timeout=3.0):
|
|
55
|
+
end = time.time() + timeout
|
|
56
|
+
while time.time() < end:
|
|
57
|
+
if cond():
|
|
58
|
+
return True
|
|
59
|
+
time.sleep(0.01)
|
|
60
|
+
return cond()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_enqueue_returns_at_once_and_the_batch_follows():
|
|
64
|
+
s = Server(delay_s=0.2)
|
|
65
|
+
sender = Sender(s, **fast())
|
|
66
|
+
t0 = time.time()
|
|
67
|
+
sender.enqueue(run("a")); sender.enqueue(run("b"))
|
|
68
|
+
assert time.time() - t0 < 0.05
|
|
69
|
+
assert s.posts == []
|
|
70
|
+
assert wait_for(lambda: len(s.posts) == 1)
|
|
71
|
+
assert s.posts[0]["path"] == "/api/sdk/batch"
|
|
72
|
+
assert s.posts[0]["body"]["items"] == [run("a"), run("b")]
|
|
73
|
+
sender.flush()
|
|
74
|
+
assert sender.stats()["sent"] == 2 and sender.stats()["dropped"] == 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_flush_at_items_sends_without_waiting_for_the_interval():
|
|
78
|
+
s = Server()
|
|
79
|
+
sender = Sender(s, **fast(flush_interval_s=60.0, flush_at=5))
|
|
80
|
+
for i in range(5):
|
|
81
|
+
sender.enqueue(run(f"r{i}"))
|
|
82
|
+
assert wait_for(lambda: len(s.posts) == 1)
|
|
83
|
+
assert len(s.posts[0]["body"]["items"]) == 5
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_429_pauses_the_sender_and_loses_nothing():
|
|
87
|
+
s = Server([(429, "0.3", {"error": "Rate ceiling exceeded"})])
|
|
88
|
+
sender = Sender(s, **fast())
|
|
89
|
+
sender.enqueue(run("a"))
|
|
90
|
+
assert wait_for(lambda: len(s.posts) == 1)
|
|
91
|
+
sender.enqueue(run("b"))
|
|
92
|
+
time.sleep(0.15)
|
|
93
|
+
assert len(s.posts) == 1 # still paused
|
|
94
|
+
assert sender.stats()["paused_until"] > time.time() - 0.01
|
|
95
|
+
assert wait_for(lambda: len(s.posts) == 2)
|
|
96
|
+
assert s.posts[1]["body"]["items"] == [run("a"), run("b")]
|
|
97
|
+
sender.flush()
|
|
98
|
+
assert sender.stats()["sent"] == 2 and sender.stats()["dropped"] == 0
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_outage_is_retried_with_backoff_until_the_server_is_back(monkeypatch):
|
|
102
|
+
import trodo.delivery.sender as m
|
|
103
|
+
monkeypatch.setattr(m, "_MIN_BACKOFF_S", 0.02)
|
|
104
|
+
s = Server([(503, None, None), RuntimeError("ECONNRESET"), RuntimeError("timeout")])
|
|
105
|
+
sender = Sender(s, **fast())
|
|
106
|
+
sender.enqueue(run("a"))
|
|
107
|
+
assert wait_for(lambda: sender.stats()["sent"] == 1, timeout=5)
|
|
108
|
+
assert len(s.posts) == 4
|
|
109
|
+
assert sender.stats()["dropped"] == 0
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_per_item_verdicts_accepted_rejected_deferred():
|
|
113
|
+
s = Server([(200, None, {"results": [{"status": 200}, {"status": 400, "error": "bad run"}, {"status": 429, "deferred": True}], "retry_after_ms": 100})])
|
|
114
|
+
sender = Sender(s, **fast(flush_at=3))
|
|
115
|
+
for rid in ("ok", "bad", "late"):
|
|
116
|
+
sender.enqueue(run(rid))
|
|
117
|
+
assert wait_for(lambda: len(s.posts) == 2)
|
|
118
|
+
assert s.posts[1]["body"]["items"] == [run("late")]
|
|
119
|
+
assert s.posts[1]["body"]["dropped"] == {"count": 1, "by_reason": {"rejected": 1}} # the server is told
|
|
120
|
+
sender.flush()
|
|
121
|
+
st = sender.stats()
|
|
122
|
+
assert st["sent"] == 2 and st["dropped_by_reason"] == {"rejected": 1}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_413_splits_the_batch_and_a_single_oversized_item_is_dropped():
|
|
126
|
+
s = Server([(413, None, {"error": "too large"}), (413, None, None)])
|
|
127
|
+
sender = Sender(s, **fast(flush_at=2))
|
|
128
|
+
sender.enqueue(run("a")); sender.enqueue(run("b"))
|
|
129
|
+
assert wait_for(lambda: sender.stats()["sent"] == 1 and sender.stats()["dropped"] == 1)
|
|
130
|
+
assert [len(p["body"]["items"]) for p in s.posts] == [2, 1, 1]
|
|
131
|
+
assert sender.stats()["dropped_by_reason"] == {"too_large": 1}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_a_full_queue_drops_the_oldest_and_counts_it():
|
|
135
|
+
s = Server([RuntimeError("down")])
|
|
136
|
+
sender = Sender(s, **fast(flush_interval_s=60.0, max_queue_items=3))
|
|
137
|
+
for rid in "abcde":
|
|
138
|
+
sender.enqueue(run(rid))
|
|
139
|
+
st = sender.stats()
|
|
140
|
+
assert st["queued"] == 3 and st["dropped_by_reason"] == {"queue_full": 2}
|
|
141
|
+
sender.flush(timeout_s=2)
|
|
142
|
+
sent = [i["run"]["run_id"] for p in s.posts for i in p["body"]["items"]]
|
|
143
|
+
assert sent[-3:] == ["c", "d", "e"] # the newest survived
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_gives_up_after_max_retry(monkeypatch):
|
|
147
|
+
import trodo.delivery.sender as m
|
|
148
|
+
monkeypatch.setattr(m, "_MIN_BACKOFF_S", 0.01)
|
|
149
|
+
monkeypatch.setattr(m, "_MAX_BACKOFF_S", 0.02)
|
|
150
|
+
s = Server([RuntimeError("down")] * 100)
|
|
151
|
+
sender = Sender(s, **fast(max_retry_s=0.2))
|
|
152
|
+
sender.enqueue(run("a"))
|
|
153
|
+
assert wait_for(lambda: sender.stats()["dropped"] == 1, timeout=5)
|
|
154
|
+
assert sender.stats()["dropped_by_reason"] == {"gave_up": 1}
|
|
155
|
+
assert sender.stats()["queued"] == 0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_shutdown_flushes_then_stops_accepting():
|
|
159
|
+
s = Server()
|
|
160
|
+
sender = Sender(s, **fast(flush_interval_s=60.0))
|
|
161
|
+
sender.enqueue(run("a"))
|
|
162
|
+
sender.shutdown()
|
|
163
|
+
assert sender.stats()["sent"] == 1
|
|
164
|
+
sender.enqueue(run("b"))
|
|
165
|
+
time.sleep(0.1)
|
|
166
|
+
assert len(s.posts) == 1
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def test_legacy_server_without_batch_route_keeps_refused_items():
|
|
170
|
+
s = Server([(404, None, {"error": "Not found"}), (200, None, {"success": True}), (503, None, None)])
|
|
171
|
+
sender = Sender(s, **fast())
|
|
172
|
+
sender.enqueue(run("a")); sender.enqueue({"run_id": "a", "feedback": {"rating": 5}})
|
|
173
|
+
assert wait_for(lambda: sender.stats()["sent"] == 2, timeout=5)
|
|
174
|
+
assert [p["path"] for p in s.posts][:3] == ["/api/sdk/batch", "/api/sdk/runs/ingest", "/api/sdk/runs/a/feedback"]
|
|
175
|
+
assert sender.stats()["dropped"] == 0
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_on_error_sees_drops():
|
|
179
|
+
seen: List[str] = []
|
|
180
|
+
sender = Sender(Server(), mode="background", on_error=lambda e: seen.append(str(e)), register_atexit=False, max_queue_items=1)
|
|
181
|
+
sender.enqueue(run("a")); sender.enqueue(run("b"))
|
|
182
|
+
assert len(seen) == 1 and "queue is full" in seen[0]
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def test_immediate_mode_blocks_only_within_its_budget():
|
|
186
|
+
s = Server([(503, None, None)] * 50)
|
|
187
|
+
slept: List[float] = []
|
|
188
|
+
clock = [0.0]
|
|
189
|
+
|
|
190
|
+
def sleep(x):
|
|
191
|
+
slept.append(x); clock[0] += x
|
|
192
|
+
|
|
193
|
+
sender = Sender(s, mode="immediate", on_error=lambda e: None, immediate_retry_s=15.0, now=lambda: clock[0], sleep=sleep)
|
|
194
|
+
sender.enqueue(run("a"))
|
|
195
|
+
assert sender.stats()["dropped_by_reason"] == {"gave_up": 1}
|
|
196
|
+
assert sum(slept) <= 15.0
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def test_immediate_mode_sends_before_returning():
|
|
200
|
+
s = Server([RuntimeError("blip")])
|
|
201
|
+
sender = Sender(s, mode="immediate", on_error=lambda e: None, sleep=lambda x: None)
|
|
202
|
+
sender.enqueue(run("a"))
|
|
203
|
+
assert len(s.posts) == 2 and sender.stats()["sent"] == 1
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def test_auto_mode_detection():
|
|
207
|
+
assert detect_mode({}) == "background"
|
|
208
|
+
assert detect_mode({"AWS_LAMBDA_FUNCTION_NAME": "fn"}) == "immediate"
|
|
209
|
+
assert detect_mode({"K_SERVICE": "svc"}) == "immediate"
|
|
210
|
+
assert detect_mode({"VERCEL": "1"}) == "immediate"
|
|
211
|
+
assert detect_mode({"KUBERNETES_SERVICE_HOST": "10.0.0.1"}) == "background"
|
|
212
|
+
assert parse_retry_after_s("12", 0) == 12.0
|
|
213
|
+
assert parse_retry_after_s("soon", 0) is None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
# ── through wrap_agent ───────────────────────────────────────────────────────
|
|
217
|
+
|
|
218
|
+
def test_wrap_agent_returns_before_the_send_and_the_run_arrives_whole():
|
|
219
|
+
s = Server(delay_s=0.3)
|
|
220
|
+
sender = Sender(s, **fast())
|
|
221
|
+
processor = TrodoSpanProcessor(http_client=s, sender=sender)
|
|
222
|
+
t0 = time.time()
|
|
223
|
+
with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat", distinct_id="u") as r:
|
|
224
|
+
r.set_input({"q": "hello"}); r.set_output({"a": "world"})
|
|
225
|
+
assert time.time() - t0 < 0.1 # not the server's 300 ms
|
|
226
|
+
sender.flush()
|
|
227
|
+
item = s.posts[0]["body"]["items"][0]
|
|
228
|
+
assert item["run"]["agent_name"] == "chat" and item["run"]["output"] == {"a": "world"}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def test_async_with_wrap_agent_does_not_block_the_event_loop():
|
|
232
|
+
s = Server(delay_s=0.3)
|
|
233
|
+
sender = Sender(s, **fast())
|
|
234
|
+
processor = TrodoSpanProcessor(http_client=s, sender=sender)
|
|
235
|
+
|
|
236
|
+
async def main():
|
|
237
|
+
ticks = 0
|
|
238
|
+
|
|
239
|
+
async def ticker():
|
|
240
|
+
nonlocal ticks
|
|
241
|
+
for _ in range(5):
|
|
242
|
+
await asyncio.sleep(0.01); ticks += 1
|
|
243
|
+
|
|
244
|
+
t = asyncio.create_task(ticker())
|
|
245
|
+
async with wrap_agent(processor=processor, team_site_id="site-x", agent_name="chat") as r:
|
|
246
|
+
await asyncio.sleep(0.02)
|
|
247
|
+
r.set_output("done")
|
|
248
|
+
await t
|
|
249
|
+
return ticks
|
|
250
|
+
|
|
251
|
+
assert asyncio.run(main()) == 5
|
|
252
|
+
sender.flush()
|
|
253
|
+
assert s.posts[0]["body"]["items"][0]["run"]["output"] == "done"
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def test_agent_decorator_on_an_async_function_records_the_awaited_result():
|
|
257
|
+
from trodo.client import TrodoClient
|
|
258
|
+
|
|
259
|
+
s = Server()
|
|
260
|
+
client = TrodoClient(site_id="site-x", auto_instrument=False, delivery="background", delivery_options={"flush_interval_s": 0.05})
|
|
261
|
+
client._sender._http = s # the fake server behind the real sender
|
|
262
|
+
client._sender._o["flush_interval_s"] = 0.05
|
|
263
|
+
|
|
264
|
+
@client.agent("async-bot")
|
|
265
|
+
async def bot(q: str) -> str:
|
|
266
|
+
await asyncio.sleep(0.01)
|
|
267
|
+
return q.upper()
|
|
268
|
+
|
|
269
|
+
assert asyncio.run(bot("hi")) == "HI"
|
|
270
|
+
client.flush(timeout=2)
|
|
271
|
+
item = s.posts[0]["body"]["items"][0]
|
|
272
|
+
assert item["run"]["agent_name"] == "async-bot"
|
|
273
|
+
assert item["run"]["output"] == "HI" # not "<coroutine object ...>"
|
|
@@ -41,7 +41,7 @@ Downstream microservice (join the caller's run instead of making a new one):
|
|
|
41
41
|
|
|
42
42
|
from __future__ import annotations
|
|
43
43
|
|
|
44
|
-
__version__ = "2.
|
|
44
|
+
__version__ = "2.25.0"
|
|
45
45
|
|
|
46
46
|
from typing import Any, Callable, Dict, List, Optional, Union
|
|
47
47
|
|
|
@@ -163,6 +163,8 @@ def init(
|
|
|
163
163
|
auto_instrument: bool = True,
|
|
164
164
|
disable_instrumentations: Optional[List[str]] = None,
|
|
165
165
|
otel_mode: str = "trodo",
|
|
166
|
+
delivery: str = "auto",
|
|
167
|
+
delivery_options: Optional[Dict[str, Any]] = None,
|
|
166
168
|
) -> TrodoClient:
|
|
167
169
|
"""Initialise the singleton SDK instance.
|
|
168
170
|
|
|
@@ -171,6 +173,19 @@ def init(
|
|
|
171
173
|
any other installed opentelemetry-instrumentation-* package, so
|
|
172
174
|
LLM calls emit token/cost spans with no further code.
|
|
173
175
|
|
|
176
|
+
``delivery`` (2.24): how traces reach Trodo.
|
|
177
|
+
'background' — wrap_agent returns at once; a daemon thread posts queued
|
|
178
|
+
runs in batches every second, retries while nobody waits,
|
|
179
|
+
and flushes at exit. The default on servers.
|
|
180
|
+
'immediate' — the run is sent before wrap_agent returns, with short
|
|
181
|
+
retries (≤ 15 s), for platforms that freeze between
|
|
182
|
+
requests: Lambda, Cloud Functions, Cloud Run, Vercel.
|
|
183
|
+
'auto' — 'immediate' when such a platform's environment variables
|
|
184
|
+
are present, otherwise 'background'. Default.
|
|
185
|
+
``delivery_options``: flush_interval_s (1), flush_at (50), max_batch_bytes
|
|
186
|
+
(4 MB), max_queue_items (10,000), max_queue_bytes (64 MB), max_retry_s
|
|
187
|
+
(600), exit_flush_s (5).
|
|
188
|
+
|
|
174
189
|
``disable_instrumentations`` turns OFF specific instrumentors by id
|
|
175
190
|
(e.g. ``['openai']`` when LangChain is your primary framework, to avoid
|
|
176
191
|
double-counting the same model call). Mirrors Node's
|
|
@@ -191,6 +206,8 @@ def init(
|
|
|
191
206
|
auto_instrument=auto_instrument,
|
|
192
207
|
disable_instrumentations=disable_instrumentations,
|
|
193
208
|
otel_mode=otel_mode,
|
|
209
|
+
delivery=delivery,
|
|
210
|
+
delivery_options=delivery_options,
|
|
194
211
|
)
|
|
195
212
|
return _client
|
|
196
213
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
SDK_VERSION = "2.25.0"
|
|
@@ -22,8 +22,11 @@ GROUPS_REMOVE = "/api/sdk/groups/remove_group"
|
|
|
22
22
|
GROUPS_PROFILE = "/api/sdk/groups/profile"
|
|
23
23
|
# Agent run tracking (Lemma-style, run-centric)
|
|
24
24
|
RUNS_INGEST = "/api/sdk/runs/ingest"
|
|
25
|
+
# Many writes in one request — the background sender (delivery/sender.py).
|
|
26
|
+
BATCH = "/api/sdk/batch"
|
|
25
27
|
RUNS_START = "/api/sdk/runs/start"
|
|
26
28
|
RUNS_BASE = "/api/sdk/runs" # /runs/{run_id}/end, /spans, /feedback
|
|
29
|
+
SPANS_APPEND = "/api/sdk/spans/append"
|
|
27
30
|
OTLP_TRACES = "/api/sdk/otel/v1/traces"
|
|
28
31
|
# Datasets & experiments
|
|
29
32
|
DATASETS_BASE = "/api/sdk/datasets" # /datasets/{ref}/items
|
|
@@ -11,6 +11,7 @@ import requests
|
|
|
11
11
|
from ..types import ApiResult, EventPayload
|
|
12
12
|
from . import endpoints
|
|
13
13
|
from ._retry import jittered_backoff, retry_delay
|
|
14
|
+
from .._version import SDK_VERSION
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class HttpClient:
|
|
@@ -82,9 +83,40 @@ class HttpClient:
|
|
|
82
83
|
return self._request(path, body, attempt + 1)
|
|
83
84
|
self._log(f"Error: {exc}")
|
|
84
85
|
if self.on_error:
|
|
85
|
-
|
|
86
|
+
try:
|
|
87
|
+
self.on_error(exc)
|
|
88
|
+
except Exception:
|
|
89
|
+
pass # a handler that raises must not surface into user code
|
|
86
90
|
return {}
|
|
87
91
|
|
|
92
|
+
def post_raw(
|
|
93
|
+
self, path: str, body: Dict[str, Any], timeout: Optional[float] = None, gzip_body: bool = True,
|
|
94
|
+
) -> tuple:
|
|
95
|
+
"""One attempt, the status told: ``(status, retry_after, parsed_body)``.
|
|
96
|
+
|
|
97
|
+
The background sender (delivery/sender.py) decides what to do with a
|
|
98
|
+
429, a 5xx or a timeout, so unlike ``_request`` this neither retries
|
|
99
|
+
nor swallows: it raises only on a network error or a timeout. Bodies
|
|
100
|
+
over 8 KB are gzipped.
|
|
101
|
+
"""
|
|
102
|
+
import gzip as _gzip
|
|
103
|
+
import json as _json
|
|
104
|
+
|
|
105
|
+
url = f"{self.api_base}{path}"
|
|
106
|
+
raw = _json.dumps(body, default=str).encode("utf-8")
|
|
107
|
+
headers: Dict[str, str] = {"User-Agent": f"trodo-python/{SDK_VERSION}"}
|
|
108
|
+
data = raw
|
|
109
|
+
if gzip_body and len(raw) > 8 * 1024:
|
|
110
|
+
data = _gzip.compress(raw)
|
|
111
|
+
headers["Content-Encoding"] = "gzip"
|
|
112
|
+
self._log(f"POST {url} ({len(raw)} bytes{', gzip' if 'Content-Encoding' in headers else ''})")
|
|
113
|
+
resp = self._session.post(url, data=data, headers=headers, timeout=timeout or self.timeout)
|
|
114
|
+
try:
|
|
115
|
+
parsed = resp.json() if resp.text else None
|
|
116
|
+
except Exception:
|
|
117
|
+
parsed = {"body": resp.text[:300]}
|
|
118
|
+
return resp.status_code, resp.headers.get("retry-after"), parsed
|
|
119
|
+
|
|
88
120
|
def _get(
|
|
89
121
|
self, path: str, params: Optional[Dict[str, Any]] = None, attempt: int = 0
|
|
90
122
|
) -> ApiResult:
|