runtime-labs-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runtime_labs/__init__.py +6 -0
- runtime_labs/binding.py +204 -0
- runtime_labs/bootstrap.py +60 -0
- runtime_labs/config.py +42 -0
- runtime_labs/correlation.py +64 -0
- runtime_labs/export.py +109 -0
- runtime_labs/hooks.py +263 -0
- runtime_labs/payloads.py +147 -0
- runtime_labs/propagation.py +47 -0
- runtime_labs/sdk.py +235 -0
- runtime_labs/semconv.py +64 -0
- runtime_labs_sdk-0.1.0.dist-info/METADATA +382 -0
- runtime_labs_sdk-0.1.0.dist-info/RECORD +14 -0
- runtime_labs_sdk-0.1.0.dist-info/WHEEL +4 -0
runtime_labs/hooks.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Narrow adapters for the pinned public Google and LangGraph interfaces.
|
|
2
|
+
|
|
3
|
+
No whole-program tracing, code generation, static parent reconstruction or
|
|
4
|
+
patching of Google transport layers. Streaming methods are deliberately untouched.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import functools
|
|
8
|
+
import inspect
|
|
9
|
+
import sys
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from contextvars import ContextVar
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
from weakref import WeakKeyDictionary
|
|
14
|
+
|
|
15
|
+
from opentelemetry import context
|
|
16
|
+
from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY
|
|
17
|
+
|
|
18
|
+
from . import correlation, payloads, semconv
|
|
19
|
+
|
|
20
|
+
_provider_active = ContextVar("runtime_labs_provider_active", default=False)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Hooks:
|
|
24
|
+
def __init__(self, sdk):
|
|
25
|
+
self.sdk = sdk
|
|
26
|
+
self._patches = []
|
|
27
|
+
self._graphs = WeakKeyDictionary()
|
|
28
|
+
self._compiled = WeakKeyDictionary()
|
|
29
|
+
|
|
30
|
+
def patch(self, owner, name, replacement):
|
|
31
|
+
original = getattr(owner, name)
|
|
32
|
+
own = name in vars(owner)
|
|
33
|
+
setattr(owner, name, replacement)
|
|
34
|
+
self._patches.append((owner, name, original, replacement, own))
|
|
35
|
+
|
|
36
|
+
def uninstall(self):
|
|
37
|
+
for owner, name, original, replacement, own in reversed(self._patches):
|
|
38
|
+
if getattr(owner, name, None) is replacement:
|
|
39
|
+
if own:
|
|
40
|
+
setattr(owner, name, original)
|
|
41
|
+
else:
|
|
42
|
+
delattr(owner, name)
|
|
43
|
+
self._patches.clear()
|
|
44
|
+
self._graphs.clear()
|
|
45
|
+
self._compiled.clear()
|
|
46
|
+
|
|
47
|
+
def install(self):
|
|
48
|
+
for package, supported, installer in (
|
|
49
|
+
("google-genai", "2.27.0", self.google),
|
|
50
|
+
("langgraph", "1.2.18", self.langgraph),
|
|
51
|
+
):
|
|
52
|
+
try:
|
|
53
|
+
if version(package) != supported:
|
|
54
|
+
self.sdk.diagnose(package.replace("-", "_") + "_version_unsupported")
|
|
55
|
+
continue
|
|
56
|
+
installer()
|
|
57
|
+
except PackageNotFoundError:
|
|
58
|
+
self.sdk.diagnose(package.replace("-", "_") + "_unavailable")
|
|
59
|
+
except Exception:
|
|
60
|
+
self.sdk.diagnose("integration_install_failed")
|
|
61
|
+
self.sdk.attempt(self.tools)
|
|
62
|
+
|
|
63
|
+
def tools(self):
|
|
64
|
+
# Only inspect already-loaded modules; manifest names never trigger imports.
|
|
65
|
+
for module in tuple(sys.modules.values()):
|
|
66
|
+
if module is None:
|
|
67
|
+
continue
|
|
68
|
+
for name, function in tuple(vars(module).items()):
|
|
69
|
+
if (inspect.isfunction(function)
|
|
70
|
+
and not getattr(function, "__runtime_labs_wrapper__", False)
|
|
71
|
+
and self.sdk.sources.candidates(function.__code__, function.__globals__,
|
|
72
|
+
"Tool")):
|
|
73
|
+
self.patch(module, name, self.function(function, "Tool"))
|
|
74
|
+
|
|
75
|
+
def function(self, function, kind):
|
|
76
|
+
if getattr(function, "__runtime_labs_wrapper__", False):
|
|
77
|
+
return function
|
|
78
|
+
sdk = self.sdk
|
|
79
|
+
|
|
80
|
+
def spec():
|
|
81
|
+
entity = sdk.attempt(lambda: sdk.sources.function(function, kind))
|
|
82
|
+
return semconv.operation(kind, function.__name__, entity)
|
|
83
|
+
|
|
84
|
+
if inspect.iscoroutinefunction(function):
|
|
85
|
+
@functools.wraps(function)
|
|
86
|
+
async def wrapped(*args, **kwargs):
|
|
87
|
+
with sdk.span(spec()) as span:
|
|
88
|
+
payloads.invocation_input(sdk, span, function, kind, args, kwargs)
|
|
89
|
+
result = await function(*args, **kwargs)
|
|
90
|
+
payloads.invocation_output(sdk, span, kind, result)
|
|
91
|
+
return result
|
|
92
|
+
else:
|
|
93
|
+
@functools.wraps(function)
|
|
94
|
+
def wrapped(*args, **kwargs):
|
|
95
|
+
with sdk.span(spec()) as span:
|
|
96
|
+
payloads.invocation_input(sdk, span, function, kind, args, kwargs)
|
|
97
|
+
result = function(*args, **kwargs)
|
|
98
|
+
payloads.invocation_output(sdk, span, kind, result)
|
|
99
|
+
return result
|
|
100
|
+
|
|
101
|
+
# Frameworks inspect callback signatures/type hints to inject config/runtime.
|
|
102
|
+
# Resolve annotations in the original namespace before moving the wrapper.
|
|
103
|
+
sdk.attempt(lambda: setattr(wrapped, "__annotations__", inspect.get_annotations(
|
|
104
|
+
function, eval_str=True)))
|
|
105
|
+
wrapped.__runtime_labs_wrapper__ = True
|
|
106
|
+
return wrapped
|
|
107
|
+
|
|
108
|
+
def google(self):
|
|
109
|
+
from google.genai.models import AsyncModels, Models
|
|
110
|
+
|
|
111
|
+
sdk = self.sdk
|
|
112
|
+
|
|
113
|
+
def spec(client, kwargs, frame):
|
|
114
|
+
entity = sdk.attempt(lambda: sdk.sources.call_site(frame))
|
|
115
|
+
return sdk.attempt(lambda: semconv.provider_request(
|
|
116
|
+
client, kwargs.get("model"), entity),
|
|
117
|
+
("generate_content", {"gen_ai.operation.name": "generate_content"},
|
|
118
|
+
semconv.SpanKind.CLIENT))
|
|
119
|
+
|
|
120
|
+
def finish(span, response):
|
|
121
|
+
if span is not None:
|
|
122
|
+
sdk.attempt(lambda: span.set_attributes(semconv.provider_response(response)))
|
|
123
|
+
payloads.provider_output(sdk, span, response)
|
|
124
|
+
|
|
125
|
+
def suppressed():
|
|
126
|
+
return _provider_active.get() or context.get_value(_SUPPRESS_INSTRUMENTATION_KEY)
|
|
127
|
+
|
|
128
|
+
@contextmanager
|
|
129
|
+
def provider_context():
|
|
130
|
+
token = _provider_active.set(True)
|
|
131
|
+
otel_token = sdk.attempt(lambda: context.attach(
|
|
132
|
+
context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)
|
|
133
|
+
))
|
|
134
|
+
try:
|
|
135
|
+
yield
|
|
136
|
+
finally:
|
|
137
|
+
if otel_token is not None:
|
|
138
|
+
sdk.attempt(lambda: context.detach(otel_token))
|
|
139
|
+
_provider_active.reset(token)
|
|
140
|
+
|
|
141
|
+
sync_original = Models.generate_content
|
|
142
|
+
|
|
143
|
+
@functools.wraps(sync_original)
|
|
144
|
+
def generate_content(client, *args, **kwargs):
|
|
145
|
+
if sdk.closed or suppressed():
|
|
146
|
+
return sync_original(client, *args, **kwargs)
|
|
147
|
+
frame = inspect.currentframe().f_back
|
|
148
|
+
span_spec = spec(client, kwargs, frame)
|
|
149
|
+
del frame
|
|
150
|
+
with sdk.span(span_spec) as span:
|
|
151
|
+
sdk.attempt(lambda: payloads.provider_input(sdk, span, kwargs))
|
|
152
|
+
with provider_context():
|
|
153
|
+
response = sync_original(client, *args, **kwargs)
|
|
154
|
+
finish(span, response)
|
|
155
|
+
return response
|
|
156
|
+
|
|
157
|
+
async_original = AsyncModels.generate_content
|
|
158
|
+
|
|
159
|
+
@functools.wraps(async_original)
|
|
160
|
+
async def agenerate_content(client, *args, **kwargs):
|
|
161
|
+
if sdk.closed or suppressed():
|
|
162
|
+
return await async_original(client, *args, **kwargs)
|
|
163
|
+
frame = inspect.currentframe().f_back
|
|
164
|
+
span_spec = spec(client, kwargs, frame)
|
|
165
|
+
del frame
|
|
166
|
+
with sdk.span(span_spec) as span:
|
|
167
|
+
sdk.attempt(lambda: payloads.provider_input(sdk, span, kwargs))
|
|
168
|
+
with provider_context():
|
|
169
|
+
response = await async_original(client, *args, **kwargs)
|
|
170
|
+
finish(span, response)
|
|
171
|
+
return response
|
|
172
|
+
|
|
173
|
+
self.patch(Models, "generate_content", generate_content)
|
|
174
|
+
self.patch(AsyncModels, "generate_content", agenerate_content)
|
|
175
|
+
|
|
176
|
+
def langgraph(self):
|
|
177
|
+
from langgraph.graph import StateGraph
|
|
178
|
+
from langgraph.graph.state import CompiledStateGraph
|
|
179
|
+
|
|
180
|
+
sdk = self.sdk
|
|
181
|
+
init_original = StateGraph.__init__
|
|
182
|
+
|
|
183
|
+
@functools.wraps(init_original)
|
|
184
|
+
def initialize(graph, *args, **kwargs):
|
|
185
|
+
frame = inspect.currentframe().f_back
|
|
186
|
+
# Store no frame or application globals: neither source nor graph instances
|
|
187
|
+
# should be retained by the weak instance association.
|
|
188
|
+
from .binding import call_position
|
|
189
|
+
|
|
190
|
+
binding = (frame.f_code, {key: frame.f_globals.get(key)
|
|
191
|
+
for key in ("__name__", "__file__")},
|
|
192
|
+
sdk.attempt(lambda frame=frame: call_position(frame)))
|
|
193
|
+
del frame
|
|
194
|
+
result = init_original(graph, *args, **kwargs)
|
|
195
|
+
sdk.attempt(lambda: self._graphs.__setitem__(graph, binding))
|
|
196
|
+
sdk.attempt(self.tools)
|
|
197
|
+
return result
|
|
198
|
+
|
|
199
|
+
add_original = StateGraph.add_node
|
|
200
|
+
|
|
201
|
+
@functools.wraps(add_original)
|
|
202
|
+
def add_node(graph, node, action=None, **kwargs):
|
|
203
|
+
callback = action if isinstance(node, str) else node
|
|
204
|
+
if (inspect.isfunction(callback) and sdk.attempt(lambda: sdk.sources.candidates(
|
|
205
|
+
callback.__code__, callback.__globals__, "Agent"))):
|
|
206
|
+
callback = sdk.attempt(lambda: self.function(callback, "Agent"), callback)
|
|
207
|
+
if isinstance(node, str):
|
|
208
|
+
action = callback
|
|
209
|
+
else:
|
|
210
|
+
node = callback
|
|
211
|
+
return add_original(graph, node, action, **kwargs)
|
|
212
|
+
|
|
213
|
+
compile_original = StateGraph.compile
|
|
214
|
+
|
|
215
|
+
@functools.wraps(compile_original)
|
|
216
|
+
def compile_graph(graph, *args, **kwargs):
|
|
217
|
+
result = compile_original(graph, *args, **kwargs)
|
|
218
|
+
if graph in self._graphs:
|
|
219
|
+
sdk.attempt(lambda: self._compiled.__setitem__(result, self._graphs[graph]))
|
|
220
|
+
return result
|
|
221
|
+
|
|
222
|
+
def spec(graph):
|
|
223
|
+
binding = self._compiled.get(graph)
|
|
224
|
+
entity = sdk.attempt(lambda: sdk.sources.match(*binding[:2], "Workflow", binding[2])) \
|
|
225
|
+
if binding else None
|
|
226
|
+
name = entity.qualified_symbol.rsplit(".", 1)[-1] if entity else None
|
|
227
|
+
return semconv.operation("Workflow", name, entity)
|
|
228
|
+
|
|
229
|
+
def correlate(span, args, kwargs):
|
|
230
|
+
if span is not None:
|
|
231
|
+
config = args[1] if len(args) > 1 else kwargs.get("config")
|
|
232
|
+
sdk.attempt(lambda: span.set_attributes(correlation.langgraph(
|
|
233
|
+
config, sdk.configuration.correlation_keys)))
|
|
234
|
+
|
|
235
|
+
invoke_original = CompiledStateGraph.invoke
|
|
236
|
+
|
|
237
|
+
@functools.wraps(invoke_original)
|
|
238
|
+
def invoke(graph, *args, **kwargs):
|
|
239
|
+
with sdk.span(sdk.attempt(lambda: spec(graph), semconv.operation("Workflow"))) as span:
|
|
240
|
+
correlate(span, args, kwargs)
|
|
241
|
+
payloads.record(sdk, span, payloads.INPUT,
|
|
242
|
+
lambda: args[0] if args else kwargs.get("input"))
|
|
243
|
+
result = invoke_original(graph, *args, **kwargs)
|
|
244
|
+
payloads.invocation_output(sdk, span, "Workflow", result)
|
|
245
|
+
return result
|
|
246
|
+
|
|
247
|
+
ainvoke_original = CompiledStateGraph.ainvoke
|
|
248
|
+
|
|
249
|
+
@functools.wraps(ainvoke_original)
|
|
250
|
+
async def ainvoke(graph, *args, **kwargs):
|
|
251
|
+
with sdk.span(sdk.attempt(lambda: spec(graph), semconv.operation("Workflow"))) as span:
|
|
252
|
+
correlate(span, args, kwargs)
|
|
253
|
+
payloads.record(sdk, span, payloads.INPUT,
|
|
254
|
+
lambda: args[0] if args else kwargs.get("input"))
|
|
255
|
+
result = await ainvoke_original(graph, *args, **kwargs)
|
|
256
|
+
payloads.invocation_output(sdk, span, "Workflow", result)
|
|
257
|
+
return result
|
|
258
|
+
|
|
259
|
+
self.patch(StateGraph, "__init__", initialize)
|
|
260
|
+
self.patch(StateGraph, "add_node", add_node)
|
|
261
|
+
self.patch(StateGraph, "compile", compile_graph)
|
|
262
|
+
self.patch(CompiledStateGraph, "invoke", invoke)
|
|
263
|
+
self.patch(CompiledStateGraph, "ainvoke", ainvoke)
|
runtime_labs/payloads.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Full application payloads for evals, serialized into OTel string attributes.
|
|
2
|
+
|
|
3
|
+
No redaction or truncation. Transport clients, auth and HTTP envelopes are not
|
|
4
|
+
application payloads. Serialization failures never change application execution.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import base64
|
|
8
|
+
import dataclasses
|
|
9
|
+
import inspect
|
|
10
|
+
import json
|
|
11
|
+
from datetime import date, datetime
|
|
12
|
+
from enum import Enum
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel
|
|
15
|
+
|
|
16
|
+
INPUT = "runtime_labs.input"
|
|
17
|
+
OUTPUT = "runtime_labs.output"
|
|
18
|
+
REQUEST = "runtime_labs.request"
|
|
19
|
+
RESPONSE = "runtime_labs.response"
|
|
20
|
+
INPUT_MESSAGES = "gen_ai.input.messages"
|
|
21
|
+
OUTPUT_MESSAGES = "gen_ai.output.messages"
|
|
22
|
+
SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions"
|
|
23
|
+
TOOL_ARGUMENTS = "gen_ai.tool.call.arguments"
|
|
24
|
+
TOOL_RESULT = "gen_ai.tool.call.result"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _default(value):
|
|
28
|
+
if isinstance(value, BaseModel):
|
|
29
|
+
return value.model_dump(mode="python", exclude_none=True, serialize_as_any=True)
|
|
30
|
+
if isinstance(value, type) and issubclass(value, BaseModel):
|
|
31
|
+
return value.model_json_schema()
|
|
32
|
+
if isinstance(value, Enum):
|
|
33
|
+
return value.value
|
|
34
|
+
if isinstance(value, bytes):
|
|
35
|
+
return base64.b64encode(value).decode("ascii")
|
|
36
|
+
if isinstance(value, date | datetime):
|
|
37
|
+
return value.isoformat()
|
|
38
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
39
|
+
return dataclasses.asdict(value)
|
|
40
|
+
if callable(value):
|
|
41
|
+
return {"name": value.__name__, "qualified_name": value.__qualname__}
|
|
42
|
+
raise TypeError("Payload contains an unsupported value")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def encode(value):
|
|
46
|
+
return json.dumps(value, default=_default, ensure_ascii=False, allow_nan=False)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def record(sdk, span, attribute, value):
|
|
50
|
+
if span is None or not sdk.configuration.capture_payloads:
|
|
51
|
+
return
|
|
52
|
+
try:
|
|
53
|
+
span.set_attribute(attribute, encode(value()))
|
|
54
|
+
except Exception:
|
|
55
|
+
sdk.diagnose("payload_capture_failed")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def invocation_input(sdk, span, function, kind, args, kwargs):
|
|
59
|
+
if kind == "Tool":
|
|
60
|
+
def arguments():
|
|
61
|
+
bound = inspect.signature(function).bind(*args, **kwargs)
|
|
62
|
+
bound.apply_defaults()
|
|
63
|
+
return bound.arguments
|
|
64
|
+
record(sdk, span, TOOL_ARGUMENTS, arguments)
|
|
65
|
+
else:
|
|
66
|
+
def state():
|
|
67
|
+
signature = inspect.signature(function)
|
|
68
|
+
bound = signature.bind(*args, **kwargs)
|
|
69
|
+
return bound.arguments.get(next(iter(signature.parameters), None))
|
|
70
|
+
record(sdk, span, INPUT, state)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def invocation_output(sdk, span, kind, result):
|
|
74
|
+
record(sdk, span, TOOL_RESULT if kind == "Tool" else OUTPUT, lambda: result)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _parts(content):
|
|
78
|
+
parts = []
|
|
79
|
+
for part in content.parts or []:
|
|
80
|
+
if part.text is not None:
|
|
81
|
+
parts.append({"type": "reasoning" if part.thought else "text", "content": part.text})
|
|
82
|
+
elif part.function_call is not None:
|
|
83
|
+
call = part.function_call
|
|
84
|
+
item = {"type": "tool_call", "name": call.name, "arguments": call.args}
|
|
85
|
+
if call.id is not None:
|
|
86
|
+
item["id"] = call.id
|
|
87
|
+
parts.append(item)
|
|
88
|
+
elif part.function_response is not None:
|
|
89
|
+
response = part.function_response
|
|
90
|
+
item = {"type": "tool_call_response", "response": response.response}
|
|
91
|
+
if response.id is not None:
|
|
92
|
+
item["id"] = response.id
|
|
93
|
+
parts.append(item)
|
|
94
|
+
else:
|
|
95
|
+
parts.append({"type": "gcp.gen_ai.part", **part.model_dump(
|
|
96
|
+
exclude_none=True, serialize_as_any=True,
|
|
97
|
+
)})
|
|
98
|
+
return parts
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _message(content):
|
|
102
|
+
return {"role": "assistant" if content.role == "model" else content.role or "user",
|
|
103
|
+
"parts": _parts(content)}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _request(kwargs):
|
|
107
|
+
request = {key: kwargs[key] for key in ("model", "contents", "config") if key in kwargs}
|
|
108
|
+
config = request.get("config")
|
|
109
|
+
if isinstance(config, BaseModel):
|
|
110
|
+
request["config"] = config.model_dump(exclude={"http_options"}, exclude_none=True,
|
|
111
|
+
serialize_as_any=True)
|
|
112
|
+
elif isinstance(config, dict):
|
|
113
|
+
request["config"] = {k: v for k, v in config.items()
|
|
114
|
+
if k not in {"http_options", "httpOptions"}}
|
|
115
|
+
return request
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def provider_input(sdk, span, kwargs):
|
|
119
|
+
if span is None or not sdk.configuration.capture_payloads:
|
|
120
|
+
return
|
|
121
|
+
# Snapshot before Google can mutate contents during automatic function calling.
|
|
122
|
+
record(sdk, span, REQUEST, lambda: _request(kwargs))
|
|
123
|
+
|
|
124
|
+
def messages():
|
|
125
|
+
from google.genai import _transformers
|
|
126
|
+
|
|
127
|
+
return [_message(c) for c in _transformers.t_contents(kwargs.get("contents"))]
|
|
128
|
+
record(sdk, span, INPUT_MESSAGES, messages)
|
|
129
|
+
config = kwargs.get("config")
|
|
130
|
+
instructions = config.get("system_instruction", config.get("systemInstruction")) \
|
|
131
|
+
if isinstance(config, dict) else getattr(config, "system_instruction", None)
|
|
132
|
+
if instructions is not None:
|
|
133
|
+
def system_parts():
|
|
134
|
+
from google.genai import _transformers
|
|
135
|
+
|
|
136
|
+
return [part for c in _transformers.t_contents(instructions) for part in _parts(c)]
|
|
137
|
+
record(sdk, span, SYSTEM_INSTRUCTIONS, system_parts)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def provider_output(sdk, span, response):
|
|
141
|
+
record(sdk, span, RESPONSE, lambda: response.model_dump(
|
|
142
|
+
exclude={"sdk_http_response"}, exclude_none=True, serialize_as_any=True,
|
|
143
|
+
))
|
|
144
|
+
record(sdk, span, OUTPUT_MESSAGES, lambda: [
|
|
145
|
+
_message(candidate.content) for candidate in response.candidates or []
|
|
146
|
+
if candidate.content is not None
|
|
147
|
+
])
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Carry the active trace across process and service boundaries (W3C Trace Context).
|
|
2
|
+
|
|
3
|
+
A trace can only continue in another process if its context travels with the
|
|
4
|
+
work: an HTTP header, a queue message, an outbox row. These helpers use the
|
|
5
|
+
standard OpenTelemetry propagators, so the carrier is a plain ``traceparent``
|
|
6
|
+
(and ``tracestate``) mapping that any other OTel service also understands.
|
|
7
|
+
|
|
8
|
+
# producer, while handling a request
|
|
9
|
+
event.headers = runtime_labs.inject_context()
|
|
10
|
+
|
|
11
|
+
# consumer, in a worker
|
|
12
|
+
with runtime_labs.continue_trace(event.headers):
|
|
13
|
+
handle(event)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
|
|
18
|
+
from opentelemetry import context, propagate
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def inject_context(carrier: dict | None = None) -> dict:
|
|
22
|
+
"""Return ``carrier`` with the current trace context added (empty if no active trace)."""
|
|
23
|
+
carrier = {} if carrier is None else carrier
|
|
24
|
+
try:
|
|
25
|
+
propagate.inject(carrier)
|
|
26
|
+
except Exception:
|
|
27
|
+
pass
|
|
28
|
+
return carrier
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@contextmanager
|
|
32
|
+
def continue_trace(carrier):
|
|
33
|
+
"""Make spans started inside this block children of the carrier's span."""
|
|
34
|
+
token = None
|
|
35
|
+
try:
|
|
36
|
+
if carrier:
|
|
37
|
+
token = context.attach(propagate.extract(dict(carrier)))
|
|
38
|
+
except Exception:
|
|
39
|
+
token = None
|
|
40
|
+
try:
|
|
41
|
+
yield
|
|
42
|
+
finally:
|
|
43
|
+
if token is not None:
|
|
44
|
+
try:
|
|
45
|
+
context.detach(token)
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
runtime_labs/sdk.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Initialization, provider ownership and failure-isolated span lifecycle."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import atexit
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from threading import RLock
|
|
9
|
+
from weakref import WeakKeyDictionary
|
|
10
|
+
|
|
11
|
+
from opentelemetry import trace
|
|
12
|
+
from opentelemetry.sdk.resources import Resource
|
|
13
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
14
|
+
from opentelemetry.trace import Status, StatusCode
|
|
15
|
+
|
|
16
|
+
from .binding import SourceIndex
|
|
17
|
+
from .config import Configuration, configuration
|
|
18
|
+
from .export import SCOPE, LocalExporter, ScopedProcessor
|
|
19
|
+
|
|
20
|
+
_lock = RLock()
|
|
21
|
+
_active = None
|
|
22
|
+
_processors = WeakKeyDictionary()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SDK:
|
|
26
|
+
def __init__(self, config, exporter, provider):
|
|
27
|
+
self.configuration = config
|
|
28
|
+
self._requested_exporter = exporter
|
|
29
|
+
self._requested_provider = provider
|
|
30
|
+
self._diagnostics = set()
|
|
31
|
+
self._diagnostic_lock = RLock()
|
|
32
|
+
self.closed = False
|
|
33
|
+
self.hooks = None
|
|
34
|
+
self.provider = None
|
|
35
|
+
self.processor = None
|
|
36
|
+
self.tracer = trace.NoOpTracer()
|
|
37
|
+
self.owns_provider = False
|
|
38
|
+
self.remote_ingestion_url = None
|
|
39
|
+
remote_manifest = None
|
|
40
|
+
if config.manifest_path is None:
|
|
41
|
+
from .bootstrap import fetch_runtime_manifest
|
|
42
|
+
result = fetch_runtime_manifest(config, self.diagnose)
|
|
43
|
+
if result:
|
|
44
|
+
remote_manifest = result.manifest
|
|
45
|
+
self.remote_ingestion_url = result.ingestion_url
|
|
46
|
+
self.sources = SourceIndex(config.manifest_path, self.diagnose, manifest=remote_manifest)
|
|
47
|
+
try:
|
|
48
|
+
provider = provider or trace.get_tracer_provider()
|
|
49
|
+
if isinstance(provider, trace.ProxyTracerProvider):
|
|
50
|
+
# A private OTel provider avoids stealing the global application slot.
|
|
51
|
+
provider = TracerProvider(
|
|
52
|
+
resource=Resource({"service.name": "runtime-labs-local"}),
|
|
53
|
+
shutdown_on_exit=False,
|
|
54
|
+
)
|
|
55
|
+
self.owns_provider = True
|
|
56
|
+
self.provider = provider
|
|
57
|
+
self.tracer = provider.get_tracer(SCOPE, "0.1.0")
|
|
58
|
+
if exporter is None:
|
|
59
|
+
exporter = self._exporter()
|
|
60
|
+
if isinstance(provider, TracerProvider):
|
|
61
|
+
processor = _processors.get(provider)
|
|
62
|
+
if processor is None:
|
|
63
|
+
processor = ScopedProcessor()
|
|
64
|
+
provider.add_span_processor(processor)
|
|
65
|
+
_processors[provider] = processor
|
|
66
|
+
processor.configure(exporter, self.diagnose)
|
|
67
|
+
self.processor = processor
|
|
68
|
+
else:
|
|
69
|
+
self.diagnose("provider_export_unsupported")
|
|
70
|
+
except Exception:
|
|
71
|
+
self.diagnose("telemetry_setup_failed")
|
|
72
|
+
from .hooks import Hooks
|
|
73
|
+
|
|
74
|
+
self.hooks = Hooks(self)
|
|
75
|
+
self.hooks.install()
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def diagnostics(self):
|
|
79
|
+
with self._diagnostic_lock:
|
|
80
|
+
return tuple(sorted(self._diagnostics))
|
|
81
|
+
|
|
82
|
+
def diagnose(self, code):
|
|
83
|
+
# Callers supply fixed codes only; never interpolate paths or exception text.
|
|
84
|
+
with self._diagnostic_lock:
|
|
85
|
+
self._diagnostics.add(code)
|
|
86
|
+
|
|
87
|
+
def _exporter(self):
|
|
88
|
+
mode = os.environ.get("OTEL_TRACES_EXPORTER", "otlp" if self.remote_ingestion_url else "console")
|
|
89
|
+
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or os.environ.get(
|
|
90
|
+
"OTEL_EXPORTER_OTLP_ENDPOINT"
|
|
91
|
+
) or self.remote_ingestion_url
|
|
92
|
+
if mode == "otlp" and endpoint:
|
|
93
|
+
protocol = os.environ.get(
|
|
94
|
+
"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL",
|
|
95
|
+
os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"),
|
|
96
|
+
)
|
|
97
|
+
if protocol == "http/protobuf":
|
|
98
|
+
try:
|
|
99
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
|
100
|
+
OTLPSpanExporter,
|
|
101
|
+
)
|
|
102
|
+
from opentelemetry.metrics import NoOpMeterProvider
|
|
103
|
+
|
|
104
|
+
exporter = OTLPSpanExporter(
|
|
105
|
+
endpoint=endpoint, timeout=5,
|
|
106
|
+
headers=(
|
|
107
|
+
{"Authorization": f"Bearer {self.configuration.api_key}"}
|
|
108
|
+
if endpoint == self.remote_ingestion_url else None
|
|
109
|
+
),
|
|
110
|
+
meter_provider=NoOpMeterProvider(),
|
|
111
|
+
)
|
|
112
|
+
# The pinned transport logs arbitrary server reasons and
|
|
113
|
+
# exception text. Silence only this exporter's logger;
|
|
114
|
+
# GuardedExporter reports sanitized failure diagnostics.
|
|
115
|
+
exporter._client._logger = logging.Logger(
|
|
116
|
+
"runtime_labs.otlp", level=logging.CRITICAL + 1
|
|
117
|
+
)
|
|
118
|
+
return exporter
|
|
119
|
+
except Exception:
|
|
120
|
+
self.diagnose("otlp_unavailable")
|
|
121
|
+
else:
|
|
122
|
+
self.diagnose("otlp_protocol_unsupported")
|
|
123
|
+
elif mode == "otlp":
|
|
124
|
+
self.diagnose("otlp_endpoint_required")
|
|
125
|
+
if mode == "none":
|
|
126
|
+
from .export import InMemorySpanExporter
|
|
127
|
+
|
|
128
|
+
# Standard no-remote, bounded-by-zero exporter for explicit opt-out.
|
|
129
|
+
class DisabledExporter(InMemorySpanExporter):
|
|
130
|
+
def export(self, spans):
|
|
131
|
+
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
132
|
+
|
|
133
|
+
return SpanExportResult.SUCCESS
|
|
134
|
+
|
|
135
|
+
return DisabledExporter()
|
|
136
|
+
return LocalExporter()
|
|
137
|
+
|
|
138
|
+
def attempt(self, function, fallback=None):
|
|
139
|
+
try:
|
|
140
|
+
return function()
|
|
141
|
+
except Exception:
|
|
142
|
+
self.diagnose("instrumentation_failed")
|
|
143
|
+
return fallback
|
|
144
|
+
|
|
145
|
+
@contextmanager
|
|
146
|
+
def span(self, spec):
|
|
147
|
+
span = None
|
|
148
|
+
manager = None
|
|
149
|
+
try:
|
|
150
|
+
if not self.closed:
|
|
151
|
+
name, attributes, kind = spec
|
|
152
|
+
span = self.tracer.start_span(name, attributes=attributes, kind=kind)
|
|
153
|
+
manager = trace.use_span(span, end_on_exit=False, record_exception=False,
|
|
154
|
+
set_status_on_exception=False)
|
|
155
|
+
manager.__enter__()
|
|
156
|
+
except Exception:
|
|
157
|
+
self.diagnose("span_start_failed")
|
|
158
|
+
manager = None
|
|
159
|
+
try:
|
|
160
|
+
yield span
|
|
161
|
+
except BaseException as error:
|
|
162
|
+
if span is not None:
|
|
163
|
+
classification = "cancelled" if isinstance(error, asyncio.CancelledError) else \
|
|
164
|
+
"timeout" if isinstance(error, TimeoutError) else "_OTHER"
|
|
165
|
+
self.attempt(lambda: span.set_attribute("error.type", classification))
|
|
166
|
+
self.attempt(lambda: span.set_status(Status(StatusCode.ERROR)))
|
|
167
|
+
raise
|
|
168
|
+
finally:
|
|
169
|
+
if manager is not None:
|
|
170
|
+
self.attempt(lambda: manager.__exit__(None, None, None))
|
|
171
|
+
if span is not None:
|
|
172
|
+
self.attempt(span.end)
|
|
173
|
+
|
|
174
|
+
def flush(self, timeout_millis=30000):
|
|
175
|
+
if self.closed:
|
|
176
|
+
return True
|
|
177
|
+
# Flush only our processor. Never invoke application processors or shutdown.
|
|
178
|
+
return self.attempt(lambda: self.processor.force_flush(timeout_millis), False) \
|
|
179
|
+
if self.processor else True
|
|
180
|
+
|
|
181
|
+
def shutdown(self):
|
|
182
|
+
global _active
|
|
183
|
+
with _lock:
|
|
184
|
+
if self.closed:
|
|
185
|
+
return
|
|
186
|
+
self.flush()
|
|
187
|
+
self.closed = True
|
|
188
|
+
if self.hooks:
|
|
189
|
+
self.attempt(self.hooks.uninstall)
|
|
190
|
+
if self.processor:
|
|
191
|
+
self.attempt(self.processor.shutdown)
|
|
192
|
+
if self.owns_provider and self.provider:
|
|
193
|
+
self.attempt(self.provider.shutdown)
|
|
194
|
+
if _active is self:
|
|
195
|
+
_active = None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def init(*, api_key: str, exporter=None, tracer_provider=None,
|
|
199
|
+
capture_payloads: bool = True, api_url: str | None = None,
|
|
200
|
+
correlation_keys=None) -> SDK:
|
|
201
|
+
"""Initialize once before graph/client construction; failures become diagnostics.
|
|
202
|
+
|
|
203
|
+
exporter and tracer_provider are optional standard OTel interfaces for local
|
|
204
|
+
testing/integration. Full payload capture is enabled by default for evals.
|
|
205
|
+
The key authenticates backend requests, never provider calls or span payloads.
|
|
206
|
+
correlation_keys extends the per-framework allowlist of identifiers copied
|
|
207
|
+
verbatim onto spans, for example {"langgraph": ["tenant"]}.
|
|
208
|
+
"""
|
|
209
|
+
global _active
|
|
210
|
+
with _lock:
|
|
211
|
+
config_failed = False
|
|
212
|
+
try:
|
|
213
|
+
config = configuration(api_key, capture_payloads, api_url=api_url,
|
|
214
|
+
correlation_keys=correlation_keys)
|
|
215
|
+
except Exception:
|
|
216
|
+
config = Configuration(api_key, None, capture_payloads)
|
|
217
|
+
config_failed = True
|
|
218
|
+
if _active is not None:
|
|
219
|
+
if (config != _active.configuration
|
|
220
|
+
or exporter is not _active._requested_exporter
|
|
221
|
+
or tracer_provider is not _active._requested_provider):
|
|
222
|
+
_active.diagnose("initialization_conflict")
|
|
223
|
+
return _active
|
|
224
|
+
_active = SDK(config, exporter, tracer_provider)
|
|
225
|
+
if config_failed:
|
|
226
|
+
_active.diagnose("configuration_invalid")
|
|
227
|
+
return _active
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _shutdown():
|
|
231
|
+
if _active is not None:
|
|
232
|
+
_active.shutdown()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
atexit.register(_shutdown)
|