mantisdk 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mantisdk/__init__.py +1 -1
- mantisdk/tracing/__init__.py +57 -0
- mantisdk/tracing/api.py +546 -0
- mantisdk/tracing/attributes.py +191 -0
- mantisdk/tracing/exporters/__init__.py +10 -0
- mantisdk/tracing/exporters/insight.py +202 -0
- mantisdk/tracing/init.py +371 -0
- mantisdk/tracing/instrumentors/__init__.py +15 -0
- mantisdk/tracing/instrumentors/claude_agent_sdk.py +591 -0
- mantisdk/tracing/instrumentors/instrumentation_principles.md +289 -0
- mantisdk/tracing/instrumentors/registry.py +313 -0
- {mantisdk-0.1.1.dist-info → mantisdk-0.1.2.dist-info}/METADATA +1 -1
- {mantisdk-0.1.1.dist-info → mantisdk-0.1.2.dist-info}/RECORD +16 -6
- {mantisdk-0.1.1.dist-info → mantisdk-0.1.2.dist-info}/WHEEL +0 -0
- {mantisdk-0.1.1.dist-info → mantisdk-0.1.2.dist-info}/entry_points.txt +0 -0
- {mantisdk-0.1.1.dist-info → mantisdk-0.1.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
# Instrumentation Principles for MantisDK
|
|
2
|
+
|
|
3
|
+
Lessons learned from building instrumentors for agent SDKs like `claude-agent-sdk`.
|
|
4
|
+
|
|
5
|
+
## 1. Know Your Backend's Attribute Expectations
|
|
6
|
+
|
|
7
|
+
Different tracing backends extract data from different attribute namespaces.
|
|
8
|
+
|
|
9
|
+
**Example:** Mantis Insight's `OtelIngestionProcessor` only looks for:
|
|
10
|
+
```typescript
|
|
11
|
+
// packages/shared/src/server/otel/OtelIngestionProcessor.ts
|
|
12
|
+
input: attributes["langfuse.observation.input"]
|
|
13
|
+
output: attributes["langfuse.observation.output"]
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
**Solution:** Set multiple attribute conventions for maximum compatibility:
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
# Langfuse (for Mantis Insight visualization)
|
|
20
|
+
span.set_attribute("langfuse.observation.input", input_value)
|
|
21
|
+
span.set_attribute("langfuse.observation.output", output_value)
|
|
22
|
+
|
|
23
|
+
# OpenInference (for Phoenix, Arize)
|
|
24
|
+
span.set_attribute("input.value", input_value)
|
|
25
|
+
span.set_attribute("output.value", output_value)
|
|
26
|
+
span.set_attribute("openinference.span.kind", "LLM")
|
|
27
|
+
|
|
28
|
+
# GenAI (widely supported standard)
|
|
29
|
+
span.set_attribute("gen_ai.system", "anthropic")
|
|
30
|
+
span.set_attribute("gen_ai.usage.input_tokens", token_count)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## 2. Context Propagation is Critical
|
|
34
|
+
|
|
35
|
+
For child spans to appear correctly nested and for timing to be accurate, you must **activate** the parent context.
|
|
36
|
+
|
|
37
|
+
**Bad:**
|
|
38
|
+
```python
|
|
39
|
+
parent_span = tracer.start_span("parent")
|
|
40
|
+
# Other instrumentation won't see this as the active parent
|
|
41
|
+
child_span = tracer.start_span("child") # Will be a sibling, not child!
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
**Good:**
|
|
45
|
+
```python
|
|
46
|
+
parent_span = tracer.start_span("parent")
|
|
47
|
+
parent_ctx = trace.set_span_in_context(parent_span)
|
|
48
|
+
context_token = attach(parent_ctx) # Activate!
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
# Now other instrumentation sees parent_span as active
|
|
52
|
+
# Their spans will be children automatically
|
|
53
|
+
result = await some_instrumented_function()
|
|
54
|
+
finally:
|
|
55
|
+
detach(context_token) # Clean up
|
|
56
|
+
parent_span.end()
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
**Even Better (explicit context for child spans):**
|
|
60
|
+
```python
|
|
61
|
+
parent_ctx = trace.set_span_in_context(parent_span)
|
|
62
|
+
context_token = attach(parent_ctx)
|
|
63
|
+
|
|
64
|
+
# Explicitly pass context when creating known children
|
|
65
|
+
child_span = tracer.start_span("child", context=parent_ctx)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## 3. Root Spans Need Fresh Context
|
|
69
|
+
|
|
70
|
+
To create separate traces (not nested), use an empty `Context()`:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
# Creates a NEW trace (root span)
|
|
74
|
+
span = tracer.start_span("operation", context=Context())
|
|
75
|
+
|
|
76
|
+
# Without context=Context(), this would be a child of any active span
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**Use case:** In chat applications, each user message should be its own trace, not nested under previous messages.
|
|
80
|
+
|
|
81
|
+
## 4. Async Generators Require Special Handling
|
|
82
|
+
|
|
83
|
+
Spans must remain open across async generator boundaries.
|
|
84
|
+
|
|
85
|
+
**Pattern:**
|
|
86
|
+
```python
|
|
87
|
+
def _create_instrumented_query(self, original_query):
|
|
88
|
+
async def instrumented_query(client, prompt):
|
|
89
|
+
span = tracer.start_span("conversation")
|
|
90
|
+
span.set_attribute("input", prompt)
|
|
91
|
+
|
|
92
|
+
# Store span for later use
|
|
93
|
+
setattr(client, "_current_span", span)
|
|
94
|
+
|
|
95
|
+
try:
|
|
96
|
+
return await original_query(client, prompt)
|
|
97
|
+
except Exception as e:
|
|
98
|
+
span.set_status(StatusCode.ERROR)
|
|
99
|
+
span.end() # End early on error
|
|
100
|
+
delattr(client, "_current_span")
|
|
101
|
+
raise
|
|
102
|
+
|
|
103
|
+
def _create_instrumented_receive(self, original_receive):
|
|
104
|
+
async def instrumented_receive(client):
|
|
105
|
+
span = getattr(client, "_current_span", None)
|
|
106
|
+
|
|
107
|
+
try:
|
|
108
|
+
async for item in original_receive(client):
|
|
109
|
+
# Process items, set output on span
|
|
110
|
+
yield item
|
|
111
|
+
|
|
112
|
+
if is_final_item(item):
|
|
113
|
+
span.set_attribute("output", extract_output(item))
|
|
114
|
+
span.end()
|
|
115
|
+
delattr(client, "_current_span")
|
|
116
|
+
finally:
|
|
117
|
+
# Cleanup if generator not fully consumed
|
|
118
|
+
if hasattr(client, "_current_span"):
|
|
119
|
+
remaining = getattr(client, "_current_span")
|
|
120
|
+
if remaining.is_recording():
|
|
121
|
+
remaining.end()
|
|
122
|
+
delattr(client, "_current_span")
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## 5. Capture All Available Context
|
|
126
|
+
|
|
127
|
+
Don't just capture the obvious fields - explore the SDK's type definitions for all available data.
|
|
128
|
+
|
|
129
|
+
**Example:** `claude-agent-sdk` provides:
|
|
130
|
+
```python
|
|
131
|
+
AssistantMessage.error # API error type (rate_limit, billing_error, etc.)
|
|
132
|
+
AssistantMessage.parent_tool_use_id # Sub-agent correlation
|
|
133
|
+
ThinkingBlock.signature # Thinking block signature
|
|
134
|
+
ResultMessage.structured_output # Structured response data
|
|
135
|
+
SystemMessage.subtype # System event type (init, mcp_connection, etc.)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Capture them all for maximum observability:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
if message.error:
|
|
142
|
+
span.set_attribute("claude.api_error", message.error)
|
|
143
|
+
|
|
144
|
+
if isinstance(message, SystemMessage):
|
|
145
|
+
span.add_event(
|
|
146
|
+
f"system.{message.subtype}",
|
|
147
|
+
attributes={"system.data": json.dumps(message.data)}
|
|
148
|
+
)
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## 6. Never Truncate Arbitrarily
|
|
152
|
+
|
|
153
|
+
Truncating attributes like `[:500]` or `[:10000]` breaks observability.
|
|
154
|
+
|
|
155
|
+
**Bad:**
|
|
156
|
+
```python
|
|
157
|
+
span.set_attribute("output", long_text[:500]) # Why 500?
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
**Good:**
|
|
161
|
+
```python
|
|
162
|
+
span.set_attribute("output", long_text) # Let OTEL handle limits
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
OpenTelemetry SDKs have configurable limits. Arbitrary truncation in instrumentors:
|
|
166
|
+
- Loses critical debugging data
|
|
167
|
+
- Isn't configurable by users
|
|
168
|
+
- Often removes the exact data needed for diagnosis
|
|
169
|
+
|
|
170
|
+
## 7. Span Kinds: OTel vs. Domain
|
|
171
|
+
|
|
172
|
+
OpenTelemetry has `SpanKind` (CLIENT, SERVER, INTERNAL), but domain conventions like OpenInference use attributes.
|
|
173
|
+
|
|
174
|
+
**Set both:**
|
|
175
|
+
```python
|
|
176
|
+
# OTel SpanKind (for protocol semantics)
|
|
177
|
+
span = tracer.start_span("llm_call", kind=SpanKind.CLIENT)
|
|
178
|
+
|
|
179
|
+
# Domain span kind (for tracing UI semantics)
|
|
180
|
+
span.set_attribute("openinference.span.kind", "LLM")
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Common domain kinds:
|
|
184
|
+
- `LLM` - Language model generation
|
|
185
|
+
- `TOOL` - Tool/function execution
|
|
186
|
+
- `AGENT` - Agent orchestration
|
|
187
|
+
- `CHAIN` - Workflow steps
|
|
188
|
+
|
|
189
|
+
## 8. Timing Accuracy Limitations
|
|
190
|
+
|
|
191
|
+
Be aware that span timing in streaming contexts reflects **when you receive messages**, not when work actually happened.
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
# In claude-agent-sdk:
|
|
195
|
+
ToolUseBlock arrives → start tool span (time T1)
|
|
196
|
+
# ... tool executes in CLI subprocess (actual work happens)
|
|
197
|
+
ToolResultBlock arrives → end tool span (time T2)
|
|
198
|
+
|
|
199
|
+
# T2 - T1 includes:
|
|
200
|
+
# - Actual tool execution time
|
|
201
|
+
# - Stream buffering delay
|
|
202
|
+
# - Network latency
|
|
203
|
+
# - Message parsing overhead
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
**Document this limitation:**
|
|
207
|
+
```python
|
|
208
|
+
# Note: Span timing is based on when we receive ToolUseBlock/ToolResultBlock messages,
|
|
209
|
+
# not when the tool actually executes in Claude CLI. This may cause timing inaccuracies.
|
|
210
|
+
tool_span = tracer.start_span(f"tool.{block.name}", ...)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
**Alternative (if SDK provides timing):**
|
|
214
|
+
```python
|
|
215
|
+
# Some SDKs provide explicit timestamps in result messages
|
|
216
|
+
if hasattr(message, 'started_at') and hasattr(message, 'completed_at'):
|
|
217
|
+
tool_span = tracer.start_span(
|
|
218
|
+
name=f"tool.{name}",
|
|
219
|
+
start_time=parse_timestamp(message.started_at), # Explicit start
|
|
220
|
+
)
|
|
221
|
+
tool_span.end(end_time=parse_timestamp(message.completed_at)) # Explicit end
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## 9. Centralize Attribute Definitions
|
|
225
|
+
|
|
226
|
+
Don't hardcode attribute names throughout your instrumentor.
|
|
227
|
+
|
|
228
|
+
**Bad:**
|
|
229
|
+
```python
|
|
230
|
+
span.set_attribute("langfuse.observation.input", value)
|
|
231
|
+
# ...later in another file...
|
|
232
|
+
span.set_attribute("langfuse.observation.input", value) # Typo risk!
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
**Good:**
|
|
236
|
+
```python
|
|
237
|
+
# attributes.py
|
|
238
|
+
LANGFUSE_OBSERVATION_INPUT = "langfuse.observation.input"
|
|
239
|
+
LANGFUSE_OBSERVATION_OUTPUT = "langfuse.observation.output"
|
|
240
|
+
|
|
241
|
+
# instrumentor.py
|
|
242
|
+
from ..attributes import LANGFUSE_OBSERVATION_INPUT
|
|
243
|
+
span.set_attribute(LANGFUSE_OBSERVATION_INPUT, value)
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Benefits:
|
|
247
|
+
- Type safety
|
|
248
|
+
- Easier refactoring
|
|
249
|
+
- Single source of truth
|
|
250
|
+
- Documentation via constants
|
|
251
|
+
|
|
252
|
+
## 10. Preserve Event Sequence
|
|
253
|
+
|
|
254
|
+
When LLMs interleave text and tool calls, preserve the order:
|
|
255
|
+
|
|
256
|
+
**Bad (loses sequence):**
|
|
257
|
+
```python
|
|
258
|
+
# Concatenate all text, list all tools
|
|
259
|
+
output = {
|
|
260
|
+
"text": text1 + text2,
|
|
261
|
+
"tool_calls": [tool1, tool2]
|
|
262
|
+
}
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
**Good (preserves sequence):**
|
|
266
|
+
```python
|
|
267
|
+
# Preserve: text → tool_call → text
|
|
268
|
+
output_blocks = [
|
|
269
|
+
{"type": "text", "content": text1},
|
|
270
|
+
{"type": "tool_call", "name": "search", "input": {...}, "output": "..."},
|
|
271
|
+
{"type": "text", "content": text2},
|
|
272
|
+
]
|
|
273
|
+
span.set_attribute("output", json.dumps(output_blocks))
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
This helps understand the agent's reasoning flow: "It said X, then called tool Y, then concluded Z."
|
|
277
|
+
|
|
278
|
+
## Summary
|
|
279
|
+
|
|
280
|
+
Building robust instrumentors requires:
|
|
281
|
+
1. ✅ Understanding backend attribute expectations
|
|
282
|
+
2. ✅ Proper context propagation with `attach()`/`detach()`
|
|
283
|
+
3. ✅ Supporting multiple attribute conventions
|
|
284
|
+
4. ✅ Careful span lifecycle management in async code
|
|
285
|
+
5. ✅ Capturing all available SDK data
|
|
286
|
+
6. ✅ Avoiding arbitrary truncation
|
|
287
|
+
7. ✅ Being aware of timing limitations in streaming contexts
|
|
288
|
+
8. ✅ Centralizing attribute definitions
|
|
289
|
+
9. ✅ Preserving event sequences for debugging
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
# Copyright (c) Metis. All rights reserved.
|
|
2
|
+
|
|
3
|
+
"""Instrumentor registry for MantisDK tracing.
|
|
4
|
+
|
|
5
|
+
This module provides a registry-based system for discovering and managing
|
|
6
|
+
OpenTelemetry instrumentors. It supports lazy loading and optional dependencies.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import importlib.util
|
|
12
|
+
import logging
|
|
13
|
+
from abc import ABC, abstractmethod
|
|
14
|
+
from typing import Callable, Dict, List, Optional, TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from openinference.instrumentation import TraceConfig
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
# Singleton registry instance
|
|
22
|
+
_registry: Optional["InstrumentorRegistry"] = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class BaseInstrumentor(ABC):
|
|
26
|
+
"""Base class for instrumentor adapters.
|
|
27
|
+
|
|
28
|
+
Each instrumentor adapter wraps a specific instrumentation library
|
|
29
|
+
(OpenInference, AgentOps, etc.) and provides a consistent interface.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def name(self) -> str:
|
|
35
|
+
"""Unique name for this instrumentor (e.g., 'openai', 'langchain')."""
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def package_name(self) -> str:
|
|
41
|
+
"""The package name to check for availability."""
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
@abstractmethod
|
|
45
|
+
def is_available(self) -> bool:
|
|
46
|
+
"""Check if the instrumentor and its target library are available."""
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def instrument(self, trace_config: Optional["TraceConfig"] = None) -> None:
|
|
51
|
+
"""Activate the instrumentation."""
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
def uninstrument(self) -> None:
|
|
56
|
+
"""Deactivate the instrumentation."""
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class OpenInferenceInstrumentor(BaseInstrumentor):
|
|
61
|
+
"""Adapter for OpenInference instrumentors.
|
|
62
|
+
|
|
63
|
+
OpenInference provides high-quality instrumentors for major LLM libraries
|
|
64
|
+
that emit semantic conventions compatible with observability platforms.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self,
|
|
69
|
+
name: str,
|
|
70
|
+
instrumentor_package: str,
|
|
71
|
+
instrumentor_class: str,
|
|
72
|
+
target_package: str,
|
|
73
|
+
):
|
|
74
|
+
"""Initialize the adapter.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
name: Unique name for this instrumentor.
|
|
78
|
+
instrumentor_package: Full package path to the instrumentor module.
|
|
79
|
+
instrumentor_class: Class name of the instrumentor.
|
|
80
|
+
target_package: The target library package (e.g., "openai").
|
|
81
|
+
"""
|
|
82
|
+
self._name = name
|
|
83
|
+
self._instrumentor_package = instrumentor_package
|
|
84
|
+
self._instrumentor_class = instrumentor_class
|
|
85
|
+
self._target_package = target_package
|
|
86
|
+
self._instance: Optional[object] = None
|
|
87
|
+
self._instrumented = False
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def name(self) -> str:
|
|
91
|
+
return self._name
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def package_name(self) -> str:
|
|
95
|
+
return self._instrumentor_package
|
|
96
|
+
|
|
97
|
+
def is_available(self) -> bool:
|
|
98
|
+
"""Check if both the instrumentor and target library are installed."""
|
|
99
|
+
# Check target library
|
|
100
|
+
if not _is_package_available(self._target_package):
|
|
101
|
+
return False
|
|
102
|
+
|
|
103
|
+
# Check instrumentor package
|
|
104
|
+
if not _is_package_available(self._instrumentor_package):
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
return True
|
|
108
|
+
|
|
109
|
+
def instrument(self, trace_config: Optional["TraceConfig"] = None) -> None:
|
|
110
|
+
"""Activate the OpenInference instrumentation."""
|
|
111
|
+
if self._instrumented:
|
|
112
|
+
logger.debug("Already instrumented: %s", self._name)
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
if not self.is_available():
|
|
116
|
+
logger.debug("Cannot instrument %s: dependencies not available", self._name)
|
|
117
|
+
return
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
module = importlib.import_module(self._instrumentor_package)
|
|
121
|
+
instrumentor_class = getattr(module, self._instrumentor_class)
|
|
122
|
+
self._instance = instrumentor_class()
|
|
123
|
+
|
|
124
|
+
# OpenInference instrumentors accept trace_config
|
|
125
|
+
if trace_config is not None:
|
|
126
|
+
self._instance.instrument(tracer_provider=None, trace_config=trace_config)
|
|
127
|
+
else:
|
|
128
|
+
self._instance.instrument()
|
|
129
|
+
|
|
130
|
+
self._instrumented = True
|
|
131
|
+
logger.debug("Instrumented: %s", self._name)
|
|
132
|
+
except Exception as e:
|
|
133
|
+
logger.warning("Failed to instrument %s: %s", self._name, e)
|
|
134
|
+
raise
|
|
135
|
+
|
|
136
|
+
def uninstrument(self) -> None:
|
|
137
|
+
"""Deactivate the OpenInference instrumentation."""
|
|
138
|
+
if not self._instrumented or self._instance is None:
|
|
139
|
+
return
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
self._instance.uninstrument()
|
|
143
|
+
self._instrumented = False
|
|
144
|
+
self._instance = None
|
|
145
|
+
logger.debug("Uninstrumented: %s", self._name)
|
|
146
|
+
except Exception as e:
|
|
147
|
+
logger.warning("Failed to uninstrument %s: %s", self._name, e)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class InstrumentorRegistry:
|
|
151
|
+
"""Registry for managing instrumentors.
|
|
152
|
+
|
|
153
|
+
The registry provides a central place to discover, configure, and manage
|
|
154
|
+
instrumentors from various sources.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
def __init__(self):
|
|
158
|
+
self._instrumentors: Dict[str, BaseInstrumentor] = {}
|
|
159
|
+
self._register_defaults()
|
|
160
|
+
|
|
161
|
+
def _register_defaults(self) -> None:
|
|
162
|
+
"""Register the default set of instrumentors."""
|
|
163
|
+
# Claude Agent SDK instrumentor (native mantisdk instrumentor)
|
|
164
|
+
from .claude_agent_sdk import ClaudeAgentSDKInstrumentor
|
|
165
|
+
self.register(ClaudeAgentSDKInstrumentor())
|
|
166
|
+
|
|
167
|
+
# OpenInference instrumentors (core set)
|
|
168
|
+
openinference_instrumentors = [
|
|
169
|
+
("openai", "openinference.instrumentation.openai", "OpenAIInstrumentor", "openai"),
|
|
170
|
+
("anthropic", "openinference.instrumentation.anthropic", "AnthropicInstrumentor", "anthropic"),
|
|
171
|
+
("langchain", "openinference.instrumentation.langchain", "LangChainInstrumentor", "langchain"),
|
|
172
|
+
("llama_index", "openinference.instrumentation.llama_index", "LlamaIndexInstrumentor", "llama_index"),
|
|
173
|
+
("litellm", "openinference.instrumentation.litellm", "LiteLLMInstrumentor", "litellm"),
|
|
174
|
+
# Additional instrumentors
|
|
175
|
+
("google_adk", "openinference.instrumentation.google_adk", "GoogleADKInstrumentor", "google.adk"),
|
|
176
|
+
("mistral", "openinference.instrumentation.mistralai", "MistralAIInstrumentor", "mistralai"),
|
|
177
|
+
("groq", "openinference.instrumentation.groq", "GroqInstrumentor", "groq"),
|
|
178
|
+
("bedrock", "openinference.instrumentation.bedrock", "BedrockInstrumentor", "boto3"),
|
|
179
|
+
("vertexai", "openinference.instrumentation.vertexai", "VertexAIInstrumentor", "vertexai"),
|
|
180
|
+
("dspy", "openinference.instrumentation.dspy", "DSPyInstrumentor", "dspy"),
|
|
181
|
+
("instructor", "openinference.instrumentation.instructor", "InstructorInstrumentor", "instructor"),
|
|
182
|
+
("crewai", "openinference.instrumentation.crewai", "CrewAIInstrumentor", "crewai"),
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
for name, package, class_name, target in openinference_instrumentors:
|
|
186
|
+
instrumentor = OpenInferenceInstrumentor(
|
|
187
|
+
name=name,
|
|
188
|
+
instrumentor_package=package,
|
|
189
|
+
instrumentor_class=class_name,
|
|
190
|
+
target_package=target,
|
|
191
|
+
)
|
|
192
|
+
self.register(instrumentor)
|
|
193
|
+
|
|
194
|
+
def register(self, instrumentor: BaseInstrumentor) -> None:
|
|
195
|
+
"""Register an instrumentor.
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
instrumentor: The instrumentor to register.
|
|
199
|
+
"""
|
|
200
|
+
self._instrumentors[instrumentor.name] = instrumentor
|
|
201
|
+
logger.debug("Registered instrumentor: %s", instrumentor.name)
|
|
202
|
+
|
|
203
|
+
def get(self, name: str) -> Optional[BaseInstrumentor]:
|
|
204
|
+
"""Get an instrumentor by name.
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
name: The instrumentor name.
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
The instrumentor if found and available, None otherwise.
|
|
211
|
+
"""
|
|
212
|
+
instrumentor = self._instrumentors.get(name)
|
|
213
|
+
if instrumentor is None:
|
|
214
|
+
return None
|
|
215
|
+
|
|
216
|
+
if not instrumentor.is_available():
|
|
217
|
+
return None
|
|
218
|
+
|
|
219
|
+
return instrumentor
|
|
220
|
+
|
|
221
|
+
def list_available(self) -> List[str]:
|
|
222
|
+
"""List all available instrumentor names.
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
List of instrumentor names that are currently available.
|
|
226
|
+
"""
|
|
227
|
+
return [
|
|
228
|
+
name for name, instrumentor in self._instrumentors.items()
|
|
229
|
+
if instrumentor.is_available()
|
|
230
|
+
]
|
|
231
|
+
|
|
232
|
+
def list_all(self) -> List[str]:
|
|
233
|
+
"""List all registered instrumentor names.
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
List of all registered instrumentor names (including unavailable).
|
|
237
|
+
"""
|
|
238
|
+
return list(self._instrumentors.keys())
|
|
239
|
+
|
|
240
|
+
def instrument_all(
|
|
241
|
+
self,
|
|
242
|
+
names: Optional[List[str]] = None,
|
|
243
|
+
skip: Optional[List[str]] = None,
|
|
244
|
+
trace_config: Optional["TraceConfig"] = None,
|
|
245
|
+
) -> List[str]:
|
|
246
|
+
"""Instrument multiple instrumentors at once.
|
|
247
|
+
|
|
248
|
+
Args:
|
|
249
|
+
names: List of instrumentor names to enable. If None, uses the core set.
|
|
250
|
+
skip: List of instrumentor names to skip.
|
|
251
|
+
trace_config: Optional TraceConfig for OpenInference instrumentors.
|
|
252
|
+
|
|
253
|
+
Returns:
|
|
254
|
+
List of successfully instrumented names.
|
|
255
|
+
"""
|
|
256
|
+
skip_set = set(skip or [])
|
|
257
|
+
|
|
258
|
+
if names is None:
|
|
259
|
+
# Core set (includes claude_agent_sdk for automatic tracing)
|
|
260
|
+
target_names = ["claude_agent_sdk", "openai", "anthropic", "langchain", "llama_index", "litellm"]
|
|
261
|
+
else:
|
|
262
|
+
target_names = names
|
|
263
|
+
|
|
264
|
+
# Filter out skipped
|
|
265
|
+
target_names = [n for n in target_names if n not in skip_set]
|
|
266
|
+
|
|
267
|
+
instrumented = []
|
|
268
|
+
for name in target_names:
|
|
269
|
+
instrumentor = self.get(name)
|
|
270
|
+
if instrumentor is not None:
|
|
271
|
+
try:
|
|
272
|
+
instrumentor.instrument(trace_config=trace_config)
|
|
273
|
+
instrumented.append(name)
|
|
274
|
+
except Exception as e:
|
|
275
|
+
logger.debug("Failed to instrument %s: %s", name, e)
|
|
276
|
+
|
|
277
|
+
return instrumented
|
|
278
|
+
|
|
279
|
+
def uninstrument_all(self) -> None:
|
|
280
|
+
"""Uninstrument all active instrumentors."""
|
|
281
|
+
for instrumentor in self._instrumentors.values():
|
|
282
|
+
try:
|
|
283
|
+
instrumentor.uninstrument()
|
|
284
|
+
except Exception as e:
|
|
285
|
+
logger.debug("Failed to uninstrument %s: %s", instrumentor.name, e)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def get_registry() -> InstrumentorRegistry:
|
|
289
|
+
"""Get the global instrumentor registry.
|
|
290
|
+
|
|
291
|
+
Returns:
|
|
292
|
+
The singleton InstrumentorRegistry instance.
|
|
293
|
+
"""
|
|
294
|
+
global _registry
|
|
295
|
+
if _registry is None:
|
|
296
|
+
_registry = InstrumentorRegistry()
|
|
297
|
+
return _registry
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _is_package_available(package_name: str) -> bool:
|
|
301
|
+
"""Check if a package is available for import.
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
package_name: The package name (can be dotted path).
|
|
305
|
+
|
|
306
|
+
Returns:
|
|
307
|
+
True if the package is available.
|
|
308
|
+
"""
|
|
309
|
+
try:
|
|
310
|
+
spec = importlib.util.find_spec(package_name)
|
|
311
|
+
return spec is not None
|
|
312
|
+
except (ModuleNotFoundError, ValueError):
|
|
313
|
+
return False
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mantisdk
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Mantisdk - AI Agent Training and Evaluation Platform
|
|
5
5
|
Project-URL: Homepage, https://github.com/withmetis/mantis
|
|
6
6
|
Project-URL: Documentation, https://withmetis.github.io/mantis/mantisdk/
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
mantisdk/__init__.py,sha256=
|
|
1
|
+
mantisdk/__init__.py,sha256=LsOsUrIvnZ8LtDNfw5o2rCqCJn0aDJk4JIAYB1Q5HQo,718
|
|
2
2
|
mantisdk/client.py,sha256=QTH-0LDAKWLuLH7PSaZWj7XUcnbBGrJHH-w2dJiNmKM,17059
|
|
3
3
|
mantisdk/config.py,sha256=9RAdsbTkSG_qnLvTwH0xTG5srLFaj1nk5Ufz1dQHGwc,14355
|
|
4
4
|
mantisdk/env_var.py,sha256=OCb5CvZo5U7-HBnqC0ZmdsrS2giAh7dV85TTjz6wLqA,4372
|
|
@@ -158,6 +158,16 @@ mantisdk/tracer/base.py,sha256=icW8Xb-PFv_CJop71kGN8POujVCCZqn6K-jGja9hHlY,9454
|
|
|
158
158
|
mantisdk/tracer/dummy.py,sha256=p-MO-sfS5t7J1Lae-ep7OW7y4rwklHUm_Jf0swM12RA,3399
|
|
159
159
|
mantisdk/tracer/otel.py,sha256=-0NHS08BLFA3N2lbzecYwjz3SdCnrDS6psJaHxjBq1o,24430
|
|
160
160
|
mantisdk/tracer/weave.py,sha256=UxEgESCI7bGIXhigW8YG6hCtr3ZhB86d9LOZPmFeOdc,26921
|
|
161
|
+
mantisdk/tracing/__init__.py,sha256=8WSvcB62ogBo3oLyu6non4eoDTDh5NQyIYgbyq7ZXaI,1551
|
|
162
|
+
mantisdk/tracing/api.py,sha256=bbBsABnfvp0B1fHbsW3pM-g2VY0AUZ2THx_tVXDLQXA,17273
|
|
163
|
+
mantisdk/tracing/attributes.py,sha256=eCB1dmGol-OnFCOvMvVqdpgHDMNKTLIV7Y_w4ktTXxU,8617
|
|
164
|
+
mantisdk/tracing/init.py,sha256=1AI92hS8HY96pfiApczb4_Ciu1g8YHwe2P8e-FLo3tQ,12472
|
|
165
|
+
mantisdk/tracing/exporters/__init__.py,sha256=Y2cnRuApIc34dJpoJGLakZfrzyzyFrr0dchHuTiDpmw,191
|
|
166
|
+
mantisdk/tracing/exporters/insight.py,sha256=kVq7syVpT06nD6Lu8-tjthnIqALuBGkFb6ICOAajLy8,6753
|
|
167
|
+
mantisdk/tracing/instrumentors/__init__.py,sha256=NXwK6SYJgrEg9zQM8uvthL43a2lyfRgx0gYAzcEnFNg,392
|
|
168
|
+
mantisdk/tracing/instrumentors/claude_agent_sdk.py,sha256=B0gbp8RmDkLHEEEgo-WMM6gJh4sLAr5pARTgs_SI09o,28336
|
|
169
|
+
mantisdk/tracing/instrumentors/instrumentation_principles.md,sha256=9IsNYaUdwo87LrFdJF6yCHW49jb9U0bVzAv1_o1hfQ8,8886
|
|
170
|
+
mantisdk/tracing/instrumentors/registry.py,sha256=jSZDRiK84oUU_lqOYF-rrvCxT52dAKS12J65FAcA3Q0,10872
|
|
161
171
|
mantisdk/trainer/__init__.py,sha256=sxlcew_92J02isSqlSp3gFNNOeGfbTalBUQDE6hYoko,160
|
|
162
172
|
mantisdk/trainer/init_utils.py,sha256=R-HM4izFbQUQKcJooeBZIPnMHCawnV7DeVH1f7N7PHg,10755
|
|
163
173
|
mantisdk/trainer/legacy.py,sha256=-AvYGBMRZSua_0sNGqtAdSSbEV0aEQY9cWj8wb046dQ,16389
|
|
@@ -183,8 +193,8 @@ mantisdk/verl/daemon.py,sha256=cvIGDkotN6MPp-DubCeKBfD9hSoZCgRyIACI9X7D7uw,54493
|
|
|
183
193
|
mantisdk/verl/dataset.py,sha256=Hr0K9oXuj7q8pf08BuvNRTbIgyNo0XY_N_JVA5jHUvI,1175
|
|
184
194
|
mantisdk/verl/entrypoint.py,sha256=dnZ7TwCe9hC1kRXm_ypRTrSR9cPXz3LcxXYjtPVuOHs,8627
|
|
185
195
|
mantisdk/verl/trainer.py,sha256=TCaVXo65iNcEEGujuIbx2yQ_cMfBYyU5Uudx9gOBSNI,25447
|
|
186
|
-
mantisdk-0.1.
|
|
187
|
-
mantisdk-0.1.
|
|
188
|
-
mantisdk-0.1.
|
|
189
|
-
mantisdk-0.1.
|
|
190
|
-
mantisdk-0.1.
|
|
196
|
+
mantisdk-0.1.2.dist-info/METADATA,sha256=P2Skwi3Cf5XLGN0un6XEgc5NHgKi-fvNTZ3V9f99VpE,3418
|
|
197
|
+
mantisdk-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
198
|
+
mantisdk-0.1.2.dist-info/entry_points.txt,sha256=yDYe8wx2pXbBXBaRPtRwt6OL01VPHf6GiFsfauISW3M,42
|
|
199
|
+
mantisdk-0.1.2.dist-info/licenses/LICENSE,sha256=j3Flk3DFJo2aHclipGIyVA6PymNGJYbY76qVqrSSogg,1046
|
|
200
|
+
mantisdk-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|