thinkhive 4.2.0__tar.gz → 4.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {thinkhive-4.2.0 → thinkhive-4.2.2}/PKG-INFO +41 -12
- {thinkhive-4.2.0 → thinkhive-4.2.2}/README.md +40 -11
- {thinkhive-4.2.0 → thinkhive-4.2.2}/setup.py +1 -1
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/__init__.py +11 -7
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/api_keys.py +3 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/business_metrics.py +9 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/calibration.py +9 -2
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/conversation_eval.py +20 -10
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/customer_context.py +5 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/deterministic_graders.py +25 -16
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/documents.py +8 -2
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/linking.py +41 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/llm_costs.py +6 -2
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/nondeterminism.py +23 -6
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/notifications.py +6 -3
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/quality_metrics.py +4 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/roi_analytics.py +7 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/runs.py +8 -2
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/sessions.py +6 -2
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/shadow_tests.py +4 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/signals.py +16 -4
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/client.py +6 -1
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/guardrails.py +8 -1
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive.egg-info/PKG-INFO +41 -12
- {thinkhive-4.2.0 → thinkhive-4.2.2}/setup.cfg +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/tests/__init__.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/tests/test_client.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/tests/test_init.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/__init__.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/agents.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/analyzer.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/claims.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/drift.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/eval_health.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/eval_runs.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/human_review.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/issues.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/traces.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive/api/transcript_patterns.py +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive.egg-info/SOURCES.txt +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive.egg-info/dependency_links.txt +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive.egg-info/requires.txt +0 -0
- {thinkhive-4.2.0 → thinkhive-4.2.2}/thinkhive.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: thinkhive
|
|
3
|
-
Version: 4.2.
|
|
3
|
+
Version: 4.2.2
|
|
4
4
|
Summary: AI agent observability SDK with business metrics, ROI analytics, and 25+ trace format support
|
|
5
5
|
Home-page: https://github.com/Abdul-Omira/ThinkHiveMind
|
|
6
6
|
Author: ThinkHive
|
|
@@ -30,7 +30,7 @@ Dynamic: requires-dist
|
|
|
30
30
|
Dynamic: requires-python
|
|
31
31
|
Dynamic: summary
|
|
32
32
|
|
|
33
|
-
# ThinkHive Python SDK v4.2.
|
|
33
|
+
# ThinkHive Python SDK v4.2.1
|
|
34
34
|
|
|
35
35
|
OpenTelemetry-based observability SDK for AI agents supporting 25 trace formats including LangSmith, Langfuse, Opik, Braintrust, Datadog, MLflow, and more.
|
|
36
36
|
|
|
@@ -53,7 +53,7 @@ thinkhive.init(
|
|
|
53
53
|
)
|
|
54
54
|
|
|
55
55
|
# Trace LLM calls
|
|
56
|
-
@thinkhive.trace_llm(model_name="gpt-4", provider="openai")
|
|
56
|
+
@thinkhive.trace_llm(name="generate-response", model_name="gpt-4", provider="openai")
|
|
57
57
|
def call_llm(prompt):
|
|
58
58
|
response = openai.chat.completions.create(
|
|
59
59
|
model="gpt-4",
|
|
@@ -62,13 +62,13 @@ def call_llm(prompt):
|
|
|
62
62
|
return response
|
|
63
63
|
|
|
64
64
|
# Trace retrieval operations
|
|
65
|
-
@thinkhive.trace_retrieval()
|
|
65
|
+
@thinkhive.trace_retrieval(name="search-kb", query="refund policy")
|
|
66
66
|
def search_documents(query):
|
|
67
67
|
results = vector_db.search(query)
|
|
68
68
|
return results
|
|
69
69
|
|
|
70
70
|
# Trace tool calls
|
|
71
|
-
@thinkhive.trace_tool(tool_name="web_search")
|
|
71
|
+
@thinkhive.trace_tool(name="lookup-order", tool_name="web_search")
|
|
72
72
|
def search_web(query):
|
|
73
73
|
return requests.get(f"https://api.example.com/search?q={query}")
|
|
74
74
|
```
|
|
@@ -317,7 +317,7 @@ result = api_keys.create(
|
|
|
317
317
|
print(f"Key: {result['key']}") # Only shown once!
|
|
318
318
|
|
|
319
319
|
# List all keys
|
|
320
|
-
keys = api_keys.list_keys()
|
|
320
|
+
keys = api_keys.list() # or api_keys.list_keys()
|
|
321
321
|
|
|
322
322
|
# Revoke a key
|
|
323
323
|
api_keys.revoke("key-id")
|
|
@@ -433,7 +433,7 @@ from thinkhive import (
|
|
|
433
433
|
try:
|
|
434
434
|
result = runs.create(agent_id="agent-123", ...)
|
|
435
435
|
except RateLimitError as e:
|
|
436
|
-
print(f"Rate limited. Retry after {e.
|
|
436
|
+
print(f"Rate limited. Retry after {e.retry_after}ms") # or e.retry_after_ms
|
|
437
437
|
except AgentScopeError as e:
|
|
438
438
|
print(f"No access to agent. Allowed: {e.allowed_agents}")
|
|
439
439
|
except NotFoundError as e:
|
|
@@ -511,7 +511,7 @@ print(f"Estimated cost: {cost['estimatedCredits']} credits")
|
|
|
511
511
|
from thinkhive.api import signals
|
|
512
512
|
|
|
513
513
|
# List all signals
|
|
514
|
-
all_signals = signals.list_signals()
|
|
514
|
+
all_signals = signals.list() # or signals.list_signals()
|
|
515
515
|
|
|
516
516
|
# Create a custom signal
|
|
517
517
|
signals.create(
|
|
@@ -540,7 +540,8 @@ notifications.create_rule({
|
|
|
540
540
|
})
|
|
541
541
|
|
|
542
542
|
# List notifications
|
|
543
|
-
alerts = notifications.
|
|
543
|
+
alerts = notifications.list("agent-123", unread_only=True)
|
|
544
|
+
# or: notifications.list_notifications("agent-123", unread_only=True)
|
|
544
545
|
```
|
|
545
546
|
|
|
546
547
|
## Documents (RAG)
|
|
@@ -552,7 +553,7 @@ from thinkhive.api import documents
|
|
|
552
553
|
documents.upload("agent-123", file_name="faq.txt", file_type="text/plain", file_size=1024)
|
|
553
554
|
|
|
554
555
|
# List documents
|
|
555
|
-
docs = documents.list_documents("agent-123")
|
|
556
|
+
docs = documents.list("agent-123") # or documents.list_documents("agent-123")
|
|
556
557
|
```
|
|
557
558
|
|
|
558
559
|
## Shadow Tests
|
|
@@ -576,7 +577,8 @@ shadow_tests.create(
|
|
|
576
577
|
from thinkhive.api import sessions
|
|
577
578
|
|
|
578
579
|
# List conversation sessions
|
|
579
|
-
all_sessions = sessions.
|
|
580
|
+
all_sessions = sessions.list("agent-123", limit=20)
|
|
581
|
+
# or: sessions.list_sessions("agent-123", limit=20)
|
|
580
582
|
|
|
581
583
|
# Get all traces in a session
|
|
582
584
|
traces = sessions.get_session_traces("session-789", "agent-123")
|
|
@@ -601,7 +603,8 @@ from thinkhive.api import llm_costs
|
|
|
601
603
|
from thinkhive import format_cost
|
|
602
604
|
|
|
603
605
|
# Get cost summary
|
|
604
|
-
summary = llm_costs.
|
|
606
|
+
summary = llm_costs.summary(period="30d")
|
|
607
|
+
# or: llm_costs.get_summary(period="30d")
|
|
605
608
|
print(f"Total cost: {format_cost(summary.get('totalCost', 0))}")
|
|
606
609
|
|
|
607
610
|
# Get per-agent breakdown
|
|
@@ -625,6 +628,32 @@ savings = llm_costs.get_savings()
|
|
|
625
628
|
|
|
626
629
|
## Upgrading
|
|
627
630
|
|
|
631
|
+
### v4.2.0 → v4.2.1
|
|
632
|
+
|
|
633
|
+
Improvements in v4.2.1:
|
|
634
|
+
- **Decorators** now accept `name=` parameter for custom span names (`@trace_llm(name="my-llm")`)
|
|
635
|
+
- **`configure()`** now accepts `service_name` parameter
|
|
636
|
+
- **`RateLimitError`** has `retry_after` alias (in addition to `retry_after_ms`)
|
|
637
|
+
- **Rule helpers** (`create_regex_rule`, etc.) now return named rule objects with `{type, name, config}` structure
|
|
638
|
+
- **`calculate_pass_at_k`** now supports both `(pass_rate, k)` and `(n, c, k)` calling conventions
|
|
639
|
+
- **`aggregate_worst`** / **`aggregate_average`** accept simple `[float]` lists in addition to `[dict]`
|
|
640
|
+
- **Method aliases added** for cross-SDK consistency:
|
|
641
|
+
- `runs.list()` / `runs.delete()`
|
|
642
|
+
- `calibration.status()` / `calibration.all_metrics()`
|
|
643
|
+
- `signals.list()` / `signals.delete()`
|
|
644
|
+
- `api_keys.list()`
|
|
645
|
+
- `linking.create()` / `linking.get_for_run()` / `linking.stats()` / `linking.verify()` / `linking.delete()`
|
|
646
|
+
- `roi_analytics.summary()` / `roi_analytics.correlations()`
|
|
647
|
+
- `business_metrics.current()` / `business_metrics.history()` / `business_metrics.record()`
|
|
648
|
+
- `quality_metrics.evaluate()`
|
|
649
|
+
- `llm_costs.summary()`
|
|
650
|
+
- `notifications.list()`
|
|
651
|
+
- `documents.list()` / `documents.delete()`
|
|
652
|
+
- `shadow_tests.list()`
|
|
653
|
+
- `sessions.list()`
|
|
654
|
+
- `customer_context.capture()`
|
|
655
|
+
- `guardrails.evaluate()`
|
|
656
|
+
|
|
628
657
|
### v4.1.0 → v4.2.0
|
|
629
658
|
|
|
630
659
|
New in v4.2.0:
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# ThinkHive Python SDK v4.2.
|
|
1
|
+
# ThinkHive Python SDK v4.2.1
|
|
2
2
|
|
|
3
3
|
OpenTelemetry-based observability SDK for AI agents supporting 25 trace formats including LangSmith, Langfuse, Opik, Braintrust, Datadog, MLflow, and more.
|
|
4
4
|
|
|
@@ -21,7 +21,7 @@ thinkhive.init(
|
|
|
21
21
|
)
|
|
22
22
|
|
|
23
23
|
# Trace LLM calls
|
|
24
|
-
@thinkhive.trace_llm(model_name="gpt-4", provider="openai")
|
|
24
|
+
@thinkhive.trace_llm(name="generate-response", model_name="gpt-4", provider="openai")
|
|
25
25
|
def call_llm(prompt):
|
|
26
26
|
response = openai.chat.completions.create(
|
|
27
27
|
model="gpt-4",
|
|
@@ -30,13 +30,13 @@ def call_llm(prompt):
|
|
|
30
30
|
return response
|
|
31
31
|
|
|
32
32
|
# Trace retrieval operations
|
|
33
|
-
@thinkhive.trace_retrieval()
|
|
33
|
+
@thinkhive.trace_retrieval(name="search-kb", query="refund policy")
|
|
34
34
|
def search_documents(query):
|
|
35
35
|
results = vector_db.search(query)
|
|
36
36
|
return results
|
|
37
37
|
|
|
38
38
|
# Trace tool calls
|
|
39
|
-
@thinkhive.trace_tool(tool_name="web_search")
|
|
39
|
+
@thinkhive.trace_tool(name="lookup-order", tool_name="web_search")
|
|
40
40
|
def search_web(query):
|
|
41
41
|
return requests.get(f"https://api.example.com/search?q={query}")
|
|
42
42
|
```
|
|
@@ -285,7 +285,7 @@ result = api_keys.create(
|
|
|
285
285
|
print(f"Key: {result['key']}") # Only shown once!
|
|
286
286
|
|
|
287
287
|
# List all keys
|
|
288
|
-
keys = api_keys.list_keys()
|
|
288
|
+
keys = api_keys.list() # or api_keys.list_keys()
|
|
289
289
|
|
|
290
290
|
# Revoke a key
|
|
291
291
|
api_keys.revoke("key-id")
|
|
@@ -401,7 +401,7 @@ from thinkhive import (
|
|
|
401
401
|
try:
|
|
402
402
|
result = runs.create(agent_id="agent-123", ...)
|
|
403
403
|
except RateLimitError as e:
|
|
404
|
-
print(f"Rate limited. Retry after {e.
|
|
404
|
+
print(f"Rate limited. Retry after {e.retry_after}ms") # or e.retry_after_ms
|
|
405
405
|
except AgentScopeError as e:
|
|
406
406
|
print(f"No access to agent. Allowed: {e.allowed_agents}")
|
|
407
407
|
except NotFoundError as e:
|
|
@@ -479,7 +479,7 @@ print(f"Estimated cost: {cost['estimatedCredits']} credits")
|
|
|
479
479
|
from thinkhive.api import signals
|
|
480
480
|
|
|
481
481
|
# List all signals
|
|
482
|
-
all_signals = signals.list_signals()
|
|
482
|
+
all_signals = signals.list() # or signals.list_signals()
|
|
483
483
|
|
|
484
484
|
# Create a custom signal
|
|
485
485
|
signals.create(
|
|
@@ -508,7 +508,8 @@ notifications.create_rule({
|
|
|
508
508
|
})
|
|
509
509
|
|
|
510
510
|
# List notifications
|
|
511
|
-
alerts = notifications.
|
|
511
|
+
alerts = notifications.list("agent-123", unread_only=True)
|
|
512
|
+
# or: notifications.list_notifications("agent-123", unread_only=True)
|
|
512
513
|
```
|
|
513
514
|
|
|
514
515
|
## Documents (RAG)
|
|
@@ -520,7 +521,7 @@ from thinkhive.api import documents
|
|
|
520
521
|
documents.upload("agent-123", file_name="faq.txt", file_type="text/plain", file_size=1024)
|
|
521
522
|
|
|
522
523
|
# List documents
|
|
523
|
-
docs = documents.list_documents("agent-123")
|
|
524
|
+
docs = documents.list("agent-123") # or documents.list_documents("agent-123")
|
|
524
525
|
```
|
|
525
526
|
|
|
526
527
|
## Shadow Tests
|
|
@@ -544,7 +545,8 @@ shadow_tests.create(
|
|
|
544
545
|
from thinkhive.api import sessions
|
|
545
546
|
|
|
546
547
|
# List conversation sessions
|
|
547
|
-
all_sessions = sessions.
|
|
548
|
+
all_sessions = sessions.list("agent-123", limit=20)
|
|
549
|
+
# or: sessions.list_sessions("agent-123", limit=20)
|
|
548
550
|
|
|
549
551
|
# Get all traces in a session
|
|
550
552
|
traces = sessions.get_session_traces("session-789", "agent-123")
|
|
@@ -569,7 +571,8 @@ from thinkhive.api import llm_costs
|
|
|
569
571
|
from thinkhive import format_cost
|
|
570
572
|
|
|
571
573
|
# Get cost summary
|
|
572
|
-
summary = llm_costs.
|
|
574
|
+
summary = llm_costs.summary(period="30d")
|
|
575
|
+
# or: llm_costs.get_summary(period="30d")
|
|
573
576
|
print(f"Total cost: {format_cost(summary.get('totalCost', 0))}")
|
|
574
577
|
|
|
575
578
|
# Get per-agent breakdown
|
|
@@ -593,6 +596,32 @@ savings = llm_costs.get_savings()
|
|
|
593
596
|
|
|
594
597
|
## Upgrading
|
|
595
598
|
|
|
599
|
+
### v4.2.0 → v4.2.1
|
|
600
|
+
|
|
601
|
+
Improvements in v4.2.1:
|
|
602
|
+
- **Decorators** now accept `name=` parameter for custom span names (`@trace_llm(name="my-llm")`)
|
|
603
|
+
- **`configure()`** now accepts `service_name` parameter
|
|
604
|
+
- **`RateLimitError`** has `retry_after` alias (in addition to `retry_after_ms`)
|
|
605
|
+
- **Rule helpers** (`create_regex_rule`, etc.) now return named rule objects with `{type, name, config}` structure
|
|
606
|
+
- **`calculate_pass_at_k`** now supports both `(pass_rate, k)` and `(n, c, k)` calling conventions
|
|
607
|
+
- **`aggregate_worst`** / **`aggregate_average`** accept simple `[float]` lists in addition to `[dict]`
|
|
608
|
+
- **Method aliases added** for cross-SDK consistency:
|
|
609
|
+
- `runs.list()` / `runs.delete()`
|
|
610
|
+
- `calibration.status()` / `calibration.all_metrics()`
|
|
611
|
+
- `signals.list()` / `signals.delete()`
|
|
612
|
+
- `api_keys.list()`
|
|
613
|
+
- `linking.create()` / `linking.get_for_run()` / `linking.stats()` / `linking.verify()` / `linking.delete()`
|
|
614
|
+
- `roi_analytics.summary()` / `roi_analytics.correlations()`
|
|
615
|
+
- `business_metrics.current()` / `business_metrics.history()` / `business_metrics.record()`
|
|
616
|
+
- `quality_metrics.evaluate()`
|
|
617
|
+
- `llm_costs.summary()`
|
|
618
|
+
- `notifications.list()`
|
|
619
|
+
- `documents.list()` / `documents.delete()`
|
|
620
|
+
- `shadow_tests.list()`
|
|
621
|
+
- `sessions.list()`
|
|
622
|
+
- `customer_context.capture()`
|
|
623
|
+
- `guardrails.evaluate()`
|
|
624
|
+
|
|
596
625
|
### v4.1.0 → v4.2.0
|
|
597
626
|
|
|
598
627
|
New in v4.2.0:
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name="thinkhive",
|
|
8
|
-
version="4.2.
|
|
8
|
+
version="4.2.2",
|
|
9
9
|
author="ThinkHive",
|
|
10
10
|
author_email="support@thinkhive.ai",
|
|
11
11
|
description="AI agent observability SDK with business metrics, ROI analytics, and 25+ trace format support",
|
|
@@ -20,7 +20,7 @@ from typing import Optional, Dict, Any, Callable
|
|
|
20
20
|
import os
|
|
21
21
|
import logging
|
|
22
22
|
|
|
23
|
-
__version__ = "4.2.
|
|
23
|
+
__version__ = "4.2.2"
|
|
24
24
|
|
|
25
25
|
# Global tracer
|
|
26
26
|
_tracer: Optional[trace.Tracer] = None
|
|
@@ -103,6 +103,7 @@ def get_tracer() -> trace.Tracer:
|
|
|
103
103
|
def trace_llm(
|
|
104
104
|
model_name: Optional[str] = None,
|
|
105
105
|
provider: Optional[str] = None,
|
|
106
|
+
name: Optional[str] = None,
|
|
106
107
|
):
|
|
107
108
|
"""
|
|
108
109
|
Decorator for tracing LLM calls
|
|
@@ -116,8 +117,9 @@ def trace_llm(
|
|
|
116
117
|
@functools.wraps(func)
|
|
117
118
|
def wrapper(*args, **kwargs):
|
|
118
119
|
tracer = get_tracer()
|
|
120
|
+
span_name = name or func.__name__
|
|
119
121
|
with tracer.start_as_current_span(
|
|
120
|
-
|
|
122
|
+
span_name,
|
|
121
123
|
attributes={
|
|
122
124
|
"openinference.span.kind": "LLM",
|
|
123
125
|
"llm.model_name": model_name,
|
|
@@ -148,7 +150,7 @@ def trace_llm(
|
|
|
148
150
|
return decorator
|
|
149
151
|
|
|
150
152
|
|
|
151
|
-
def trace_retrieval(query: Optional[str] = None):
|
|
153
|
+
def trace_retrieval(query: Optional[str] = None, name: Optional[str] = None):
|
|
152
154
|
"""
|
|
153
155
|
Decorator for tracing retrieval/RAG operations
|
|
154
156
|
|
|
@@ -161,8 +163,9 @@ def trace_retrieval(query: Optional[str] = None):
|
|
|
161
163
|
@functools.wraps(func)
|
|
162
164
|
def wrapper(*args, **kwargs):
|
|
163
165
|
tracer = get_tracer()
|
|
166
|
+
span_name = name or func.__name__
|
|
164
167
|
with tracer.start_as_current_span(
|
|
165
|
-
|
|
168
|
+
span_name,
|
|
166
169
|
attributes={
|
|
167
170
|
"openinference.span.kind": "RETRIEVER",
|
|
168
171
|
"retrieval.query": query or (args[0] if args else None),
|
|
@@ -193,7 +196,7 @@ def trace_retrieval(query: Optional[str] = None):
|
|
|
193
196
|
return decorator
|
|
194
197
|
|
|
195
198
|
|
|
196
|
-
def trace_tool(tool_name: Optional[str] = None):
|
|
199
|
+
def trace_tool(tool_name: Optional[str] = None, name: Optional[str] = None):
|
|
197
200
|
"""
|
|
198
201
|
Decorator for tracing tool/function calls
|
|
199
202
|
|
|
@@ -206,11 +209,12 @@ def trace_tool(tool_name: Optional[str] = None):
|
|
|
206
209
|
@functools.wraps(func)
|
|
207
210
|
def wrapper(*args, **kwargs):
|
|
208
211
|
tracer = get_tracer()
|
|
212
|
+
span_name = name or tool_name or func.__name__
|
|
209
213
|
with tracer.start_as_current_span(
|
|
210
|
-
|
|
214
|
+
span_name,
|
|
211
215
|
attributes={
|
|
212
216
|
"openinference.span.kind": "TOOL",
|
|
213
|
-
"tool.name": tool_name or func.__name__,
|
|
217
|
+
"tool.name": tool_name or name or func.__name__,
|
|
214
218
|
}
|
|
215
219
|
) as span:
|
|
216
220
|
try:
|
|
@@ -119,6 +119,9 @@ class ApiKeysClient:
|
|
|
119
119
|
response = api_request("POST", "/api-keys/test", api_version="v2")
|
|
120
120
|
return response.get("data", response) if isinstance(response, dict) else response
|
|
121
121
|
|
|
122
|
+
# Convenience alias for cross-SDK parity
|
|
123
|
+
list = list_keys
|
|
124
|
+
|
|
122
125
|
|
|
123
126
|
def has_permission(key: Dict[str, Any], permission: str) -> bool:
|
|
124
127
|
"""
|
|
@@ -392,6 +392,11 @@ def format_metric_value(value: float, unit: str) -> str:
|
|
|
392
392
|
return str(round(value * 100) / 100)
|
|
393
393
|
|
|
394
394
|
|
|
395
|
+
# Convenience aliases for cross-SDK parity
|
|
396
|
+
current = get_current
|
|
397
|
+
history = get_history
|
|
398
|
+
record = record_value
|
|
399
|
+
|
|
395
400
|
__all__ = [
|
|
396
401
|
# Dataclasses
|
|
397
402
|
"MetricTrend",
|
|
@@ -413,4 +418,8 @@ __all__ = [
|
|
|
413
418
|
"get_status_message",
|
|
414
419
|
"get_trace_progress",
|
|
415
420
|
"format_metric_value",
|
|
421
|
+
# Aliases
|
|
422
|
+
"current",
|
|
423
|
+
"history",
|
|
424
|
+
"record",
|
|
416
425
|
]
|
|
@@ -5,7 +5,7 @@ Prediction accuracy tracking with Brier scores and calibration metrics
|
|
|
5
5
|
|
|
6
6
|
from typing import Optional, Dict, Any, List, Literal
|
|
7
7
|
from dataclasses import dataclass
|
|
8
|
-
from ..client import post, get
|
|
8
|
+
from ..client import post, get, api_request
|
|
9
9
|
|
|
10
10
|
PredictionType = Literal[
|
|
11
11
|
"outcome",
|
|
@@ -115,7 +115,7 @@ def get_status(agent_id: str, prediction_type: PredictionType) -> CalibrationSta
|
|
|
115
115
|
>>> print(f"Is calibrated: {status.is_calibrated}")
|
|
116
116
|
"""
|
|
117
117
|
params = {"predictionType": prediction_type}
|
|
118
|
-
response =
|
|
118
|
+
response = api_request("GET", f"/calibration/{agent_id}/status", params=params, api_version="v3")
|
|
119
119
|
return CalibrationStatus.from_dict(response)
|
|
120
120
|
|
|
121
121
|
|
|
@@ -267,6 +267,10 @@ def format_brier_score(score: float) -> str:
|
|
|
267
267
|
return f"{score:.4f}"
|
|
268
268
|
|
|
269
269
|
|
|
270
|
+
# Convenience aliases for cross-SDK parity
|
|
271
|
+
status = get_status
|
|
272
|
+
all_metrics = get_metrics
|
|
273
|
+
|
|
270
274
|
__all__ = [
|
|
271
275
|
# Types
|
|
272
276
|
"PredictionType",
|
|
@@ -284,4 +288,7 @@ __all__ = [
|
|
|
284
288
|
"is_well_calibrated",
|
|
285
289
|
"get_calibration_quality",
|
|
286
290
|
"format_brier_score",
|
|
291
|
+
# Aliases
|
|
292
|
+
"status",
|
|
293
|
+
"all_metrics",
|
|
287
294
|
]
|
|
@@ -69,18 +69,23 @@ conversation_eval = ConversationEvalApi()
|
|
|
69
69
|
|
|
70
70
|
# Aggregation helper functions
|
|
71
71
|
|
|
72
|
-
def aggregate_worst(turn_results
|
|
72
|
+
def aggregate_worst(turn_results):
|
|
73
73
|
"""
|
|
74
74
|
Calculate worst-turn aggregation
|
|
75
75
|
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
Accepts List[Dict] (turn results) or List[float] (simple scores):
|
|
77
|
+
- aggregate_worst([{"score": 0.9, "passed": True}, ...]) → {"passed": ..., "score": ...}
|
|
78
|
+
- aggregate_worst([0.9, 0.5, 0.7]) → 0.5
|
|
78
79
|
|
|
79
80
|
Returns:
|
|
80
|
-
Dict with passed and score
|
|
81
|
+
Dict with passed and score, or float if given simple numbers
|
|
81
82
|
"""
|
|
82
83
|
if not turn_results:
|
|
83
|
-
return {"passed": False, "score": 0}
|
|
84
|
+
return 0 if (turn_results is not None and isinstance(turn_results, list)) else {"passed": False, "score": 0}
|
|
85
|
+
|
|
86
|
+
# Simple number list
|
|
87
|
+
if isinstance(turn_results[0], (int, float)):
|
|
88
|
+
return min(turn_results)
|
|
84
89
|
|
|
85
90
|
scores = [t.get("score", 0) for t in turn_results]
|
|
86
91
|
worst_score = min(scores)
|
|
@@ -89,18 +94,23 @@ def aggregate_worst(turn_results: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
|
89
94
|
return {"passed": passed, "score": worst_score}
|
|
90
95
|
|
|
91
96
|
|
|
92
|
-
def aggregate_average(turn_results
|
|
97
|
+
def aggregate_average(turn_results):
|
|
93
98
|
"""
|
|
94
99
|
Calculate average aggregation
|
|
95
100
|
|
|
96
|
-
|
|
97
|
-
|
|
101
|
+
Accepts List[Dict] (turn results) or List[float] (simple scores):
|
|
102
|
+
- aggregate_average([{"score": 0.6, "passed": True}, ...]) → {"passed": ..., "score": ...}
|
|
103
|
+
- aggregate_average([0.6, 0.8]) → 0.7
|
|
98
104
|
|
|
99
105
|
Returns:
|
|
100
|
-
Dict with passed (majority vote) and average score
|
|
106
|
+
Dict with passed (majority vote) and average score, or float if given simple numbers
|
|
101
107
|
"""
|
|
102
108
|
if not turn_results:
|
|
103
|
-
return {"passed": False, "score": 0}
|
|
109
|
+
return 0 if (turn_results is not None and isinstance(turn_results, list)) else {"passed": False, "score": 0}
|
|
110
|
+
|
|
111
|
+
# Simple number list
|
|
112
|
+
if isinstance(turn_results[0], (int, float)):
|
|
113
|
+
return sum(turn_results) / len(turn_results)
|
|
104
114
|
|
|
105
115
|
scores = [t.get("score", 0) for t in turn_results]
|
|
106
116
|
avg_score = sum(scores) / len(scores)
|
|
@@ -304,6 +304,9 @@ def format_arr(arr: float) -> str:
|
|
|
304
304
|
return f"${arr:,.0f}"
|
|
305
305
|
|
|
306
306
|
|
|
307
|
+
# Convenience alias for cross-SDK parity
|
|
308
|
+
capture = create_snapshot
|
|
309
|
+
|
|
307
310
|
__all__ = [
|
|
308
311
|
# Types
|
|
309
312
|
"CustomerSegment",
|
|
@@ -320,4 +323,6 @@ __all__ = [
|
|
|
320
323
|
"is_at_risk",
|
|
321
324
|
"get_segment_priority",
|
|
322
325
|
"format_arr",
|
|
326
|
+
# Aliases
|
|
327
|
+
"capture",
|
|
323
328
|
]
|
|
@@ -71,70 +71,79 @@ deterministic_graders = DeterministicGradersApi()
|
|
|
71
71
|
|
|
72
72
|
# Helper functions
|
|
73
73
|
|
|
74
|
-
def create_regex_rule(pattern: str, flags: str = "gi") -> Dict[str,
|
|
74
|
+
def create_regex_rule(name: str, field: str = "output", pattern: str = "", flags: str = "gi") -> Dict[str, Any]:
|
|
75
75
|
"""
|
|
76
|
-
Create a regex rule configuration
|
|
76
|
+
Create a named regex rule configuration
|
|
77
77
|
|
|
78
78
|
Args:
|
|
79
|
+
name: Rule name for identification
|
|
80
|
+
field: Field to check ('output', 'input', etc.)
|
|
79
81
|
pattern: Regular expression pattern
|
|
80
82
|
flags: Regex flags (default: 'gi')
|
|
81
83
|
|
|
82
84
|
Returns:
|
|
83
|
-
|
|
85
|
+
Named rule configuration object with type, name, and config
|
|
84
86
|
"""
|
|
85
|
-
return {"pattern": pattern, "flags": flags}
|
|
87
|
+
return {"type": "regex", "name": name, "config": {"field": field, "pattern": pattern, "flags": flags}}
|
|
86
88
|
|
|
87
89
|
|
|
88
90
|
def create_contains_rule(
|
|
89
|
-
|
|
91
|
+
name: str,
|
|
92
|
+
field: str = "output",
|
|
93
|
+
values: Optional[List[str]] = None,
|
|
90
94
|
case_sensitive: bool = False,
|
|
91
95
|
) -> Dict[str, Any]:
|
|
92
96
|
"""
|
|
93
|
-
Create a contains rule configuration
|
|
97
|
+
Create a named contains rule configuration
|
|
94
98
|
|
|
95
99
|
Args:
|
|
100
|
+
name: Rule name for identification
|
|
101
|
+
field: Field to check ('output', 'input', etc.)
|
|
96
102
|
values: Strings to check for
|
|
97
103
|
case_sensitive: Whether comparison is case-sensitive
|
|
98
104
|
|
|
99
105
|
Returns:
|
|
100
|
-
|
|
106
|
+
Named rule configuration object with type, name, and config
|
|
101
107
|
"""
|
|
102
|
-
return {"values": values, "caseSensitive": case_sensitive}
|
|
108
|
+
return {"type": "contains", "name": name, "config": {"field": field, "values": values or [], "caseSensitive": case_sensitive}}
|
|
103
109
|
|
|
104
110
|
|
|
105
111
|
def create_length_rule(
|
|
112
|
+
name: str,
|
|
106
113
|
min_length: Optional[int] = None,
|
|
107
114
|
max_length: Optional[int] = None,
|
|
108
115
|
) -> Dict[str, Any]:
|
|
109
116
|
"""
|
|
110
|
-
Create a length rule configuration
|
|
117
|
+
Create a named length rule configuration
|
|
111
118
|
|
|
112
119
|
Args:
|
|
120
|
+
name: Rule name for identification
|
|
113
121
|
min_length: Minimum length
|
|
114
122
|
max_length: Maximum length
|
|
115
123
|
|
|
116
124
|
Returns:
|
|
117
|
-
|
|
125
|
+
Named rule configuration object with type, name, and config
|
|
118
126
|
"""
|
|
119
|
-
config = {}
|
|
127
|
+
config: Dict[str, Any] = {}
|
|
120
128
|
if min_length is not None:
|
|
121
129
|
config["min"] = min_length
|
|
122
130
|
if max_length is not None:
|
|
123
131
|
config["max"] = max_length
|
|
124
|
-
return config
|
|
132
|
+
return {"type": "length", "name": name, "config": config}
|
|
125
133
|
|
|
126
134
|
|
|
127
|
-
def create_json_schema_rule(schema: Dict[str, Any]) -> Dict[str, Any]:
|
|
135
|
+
def create_json_schema_rule(name: str, schema: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
128
136
|
"""
|
|
129
|
-
Create a JSON schema rule configuration
|
|
137
|
+
Create a named JSON schema rule configuration
|
|
130
138
|
|
|
131
139
|
Args:
|
|
140
|
+
name: Rule name for identification
|
|
132
141
|
schema: JSON Schema object
|
|
133
142
|
|
|
134
143
|
Returns:
|
|
135
|
-
|
|
144
|
+
Named rule configuration object with type, name, and config
|
|
136
145
|
"""
|
|
137
|
-
return {"schema": schema}
|
|
146
|
+
return {"type": "json_schema", "name": name, "config": {"schema": schema or {}}}
|
|
138
147
|
|
|
139
148
|
|
|
140
149
|
def all_rules_passed(results: List[Dict[str, Any]]) -> bool:
|
|
@@ -5,7 +5,7 @@ Agent document management for RAG (Retrieval-Augmented Generation)
|
|
|
5
5
|
|
|
6
6
|
from typing import Optional, Dict, Any, List
|
|
7
7
|
from dataclasses import dataclass
|
|
8
|
-
from ..client import post, get, delete
|
|
8
|
+
from ..client import post, get, delete as _client_delete
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@dataclass
|
|
@@ -82,12 +82,18 @@ def delete_document(agent_id: str, doc_id: str) -> None:
|
|
|
82
82
|
agent_id: The agent ID
|
|
83
83
|
doc_id: The document ID to delete
|
|
84
84
|
"""
|
|
85
|
-
|
|
85
|
+
_client_delete(f"/agents/{agent_id}/documents/{doc_id}", api_version="")
|
|
86
86
|
|
|
87
87
|
|
|
88
|
+
# Convenience aliases for cross-SDK parity
|
|
89
|
+
list = list_documents
|
|
90
|
+
delete = delete_document
|
|
91
|
+
|
|
88
92
|
__all__ = [
|
|
89
93
|
"Document",
|
|
90
94
|
"list_documents",
|
|
91
95
|
"upload",
|
|
92
96
|
"delete_document",
|
|
97
|
+
"list",
|
|
98
|
+
"delete",
|
|
93
99
|
]
|
|
@@ -255,6 +255,41 @@ def get_supported_platforms() -> List[str]:
|
|
|
255
255
|
return list(_SUPPORTED_PLATFORMS)
|
|
256
256
|
|
|
257
257
|
|
|
258
|
+
# Convenience aliases for cross-SDK parity
|
|
259
|
+
create = link_run_to_ticket
|
|
260
|
+
get_for_run = get_linked_ticket
|
|
261
|
+
delete = unlink_run
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def stats() -> Dict[str, Any]:
|
|
265
|
+
"""
|
|
266
|
+
Get supported platform stats.
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
Dict with platform count and list of supported platforms
|
|
270
|
+
"""
|
|
271
|
+
platforms = get_supported_platforms()
|
|
272
|
+
return {"platform_count": len(platforms), "platforms": platforms}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def verify(
|
|
276
|
+
run_id: str,
|
|
277
|
+
*,
|
|
278
|
+
verified: bool = True,
|
|
279
|
+
) -> Dict[str, Any]:
|
|
280
|
+
"""
|
|
281
|
+
Update link verification status for a run.
|
|
282
|
+
|
|
283
|
+
Args:
|
|
284
|
+
run_id: The run ID whose link to verify
|
|
285
|
+
verified: Whether the link is verified (default True)
|
|
286
|
+
|
|
287
|
+
Returns:
|
|
288
|
+
Updated run data
|
|
289
|
+
"""
|
|
290
|
+
return put(f"/runs/{run_id}", {"ticketLinking": {"verified": verified}}, api_version="v3")
|
|
291
|
+
|
|
292
|
+
|
|
258
293
|
__all__ = [
|
|
259
294
|
# Types
|
|
260
295
|
"LinkMethod",
|
|
@@ -271,4 +306,10 @@ __all__ = [
|
|
|
271
306
|
"get_method_confidence",
|
|
272
307
|
"is_high_confidence_link",
|
|
273
308
|
"get_supported_platforms",
|
|
309
|
+
# Aliases
|
|
310
|
+
"create",
|
|
311
|
+
"get_for_run",
|
|
312
|
+
"delete",
|
|
313
|
+
"stats",
|
|
314
|
+
"verify",
|
|
274
315
|
]
|
|
@@ -5,7 +5,7 @@ LLM cost tracking and optimization insights
|
|
|
5
5
|
|
|
6
6
|
from typing import Optional, Dict, Any
|
|
7
7
|
from dataclasses import dataclass
|
|
8
|
-
from ..client import get
|
|
8
|
+
from ..client import get, api_request
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@dataclass
|
|
@@ -57,7 +57,7 @@ def get_summary(*, period: str = "30d") -> CostSummary:
|
|
|
57
57
|
CostSummary with total costs and breakdown
|
|
58
58
|
"""
|
|
59
59
|
params: Dict[str, Any] = {"period": period}
|
|
60
|
-
response =
|
|
60
|
+
response = api_request("GET", "/llm-costs/summary", params=params, api_version="")
|
|
61
61
|
return CostSummary.from_dict(response)
|
|
62
62
|
|
|
63
63
|
|
|
@@ -110,6 +110,9 @@ def format_cost(amount: float) -> str:
|
|
|
110
110
|
return f"${amount:,.2f}"
|
|
111
111
|
|
|
112
112
|
|
|
113
|
+
# Convenience alias for cross-SDK parity
|
|
114
|
+
summary = get_summary
|
|
115
|
+
|
|
113
116
|
__all__ = [
|
|
114
117
|
"CostSummary",
|
|
115
118
|
"CostBreakdown",
|
|
@@ -118,4 +121,5 @@ __all__ = [
|
|
|
118
121
|
"get_savings",
|
|
119
122
|
"get_optimization_stats",
|
|
120
123
|
"format_cost",
|
|
124
|
+
"summary",
|
|
121
125
|
]
|
|
@@ -178,18 +178,35 @@ nondeterminism = NondeterminismApi()
|
|
|
178
178
|
|
|
179
179
|
# Helper functions
|
|
180
180
|
|
|
181
|
-
def calculate_pass_at_k(
|
|
181
|
+
def calculate_pass_at_k(pass_rate_or_n: float, k_or_c: int, k: Optional[int] = None) -> float:
|
|
182
182
|
"""
|
|
183
|
-
Calculate pass@k probability
|
|
183
|
+
Calculate pass@k probability
|
|
184
184
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
185
|
+
Supports two calling conventions:
|
|
186
|
+
- calculate_pass_at_k(pass_rate, k) — from pass rate directly
|
|
187
|
+
- calculate_pass_at_k(n, c, k) — from n total runs, c correct, sample k
|
|
188
188
|
|
|
189
189
|
Returns:
|
|
190
190
|
Probability that at least 1 of k runs passes
|
|
191
191
|
"""
|
|
192
|
-
|
|
192
|
+
if k is not None:
|
|
193
|
+
# (n, c, k) form: 1 - C(n-c, k) / C(n, k)
|
|
194
|
+
n = int(pass_rate_or_n)
|
|
195
|
+
c = int(k_or_c)
|
|
196
|
+
if c >= n:
|
|
197
|
+
return 1.0
|
|
198
|
+
if c == 0:
|
|
199
|
+
return 0.0
|
|
200
|
+
if k > n:
|
|
201
|
+
return 1.0 if c > 0 else 0.0
|
|
202
|
+
|
|
203
|
+
# Use logarithmic calculation to avoid overflow
|
|
204
|
+
log_num = sum(math.log(n - c - i) for i in range(k) if (n - c - i) > 0)
|
|
205
|
+
log_den = sum(math.log(n - i) for i in range(k))
|
|
206
|
+
return 1 - math.exp(log_num - log_den)
|
|
207
|
+
|
|
208
|
+
# (pass_rate, k) form
|
|
209
|
+
return 1 - math.pow(1 - pass_rate_or_n, k_or_c)
|
|
193
210
|
|
|
194
211
|
|
|
195
212
|
def calculate_pass_to_k(pass_rate: float, k: int) -> float:
|
|
@@ -23,7 +23,7 @@ Usage:
|
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
from typing import Optional, Dict, Any, List
|
|
26
|
-
from ..client import get, post, patch, delete
|
|
26
|
+
from ..client import get, post, patch, delete, api_request
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class NotificationsClient:
|
|
@@ -42,7 +42,7 @@ class NotificationsClient:
|
|
|
42
42
|
List of notification rules
|
|
43
43
|
"""
|
|
44
44
|
params: Dict[str, Any] = {"agentId": agent_id}
|
|
45
|
-
return
|
|
45
|
+
return api_request("GET", "/notification-rules", params=params, api_version="")
|
|
46
46
|
|
|
47
47
|
def get_rule(
|
|
48
48
|
self,
|
|
@@ -119,7 +119,7 @@ class NotificationsClient:
|
|
|
119
119
|
if unread_only:
|
|
120
120
|
params["unreadOnly"] = True
|
|
121
121
|
|
|
122
|
-
return
|
|
122
|
+
return api_request("GET", "/notifications", params=params, api_version="")
|
|
123
123
|
|
|
124
124
|
def mark_as_read(self, notification_id: str) -> Dict[str, Any]:
|
|
125
125
|
"""
|
|
@@ -133,6 +133,9 @@ class NotificationsClient:
|
|
|
133
133
|
"""
|
|
134
134
|
return patch(f"/notifications/{notification_id}/read", api_version="")
|
|
135
135
|
|
|
136
|
+
# Convenience alias for cross-SDK parity
|
|
137
|
+
list = list_notifications
|
|
138
|
+
|
|
136
139
|
|
|
137
140
|
# Singleton instance
|
|
138
141
|
notifications = NotificationsClient()
|
|
@@ -372,6 +372,9 @@ def needs_improvement(evaluation: RAGEvaluation, *, threshold: float = 0.5) -> b
|
|
|
372
372
|
return any(score < threshold for score in dimensions)
|
|
373
373
|
|
|
374
374
|
|
|
375
|
+
# Convenience alias for cross-SDK parity
|
|
376
|
+
evaluate = evaluate_rag
|
|
377
|
+
|
|
375
378
|
__all__ = [
|
|
376
379
|
"RAGEvaluation",
|
|
377
380
|
"HallucinationInstance",
|
|
@@ -387,4 +390,5 @@ __all__ = [
|
|
|
387
390
|
"has_critical_hallucinations",
|
|
388
391
|
"get_grade_color",
|
|
389
392
|
"needs_improvement",
|
|
393
|
+
"evaluate",
|
|
390
394
|
]
|
|
@@ -457,6 +457,10 @@ def get_trend_v3(
|
|
|
457
457
|
return get("/roi/trend", params=params, api_version="v3")
|
|
458
458
|
|
|
459
459
|
|
|
460
|
+
# Convenience aliases for cross-SDK parity
|
|
461
|
+
summary = get_summary
|
|
462
|
+
correlations = get_correlations
|
|
463
|
+
|
|
460
464
|
__all__ = [
|
|
461
465
|
# Dataclasses
|
|
462
466
|
"IndustryConfig",
|
|
@@ -478,4 +482,7 @@ __all__ = [
|
|
|
478
482
|
"get_config_versions",
|
|
479
483
|
"calculate_v3",
|
|
480
484
|
"get_trend_v3",
|
|
485
|
+
# Aliases
|
|
486
|
+
"summary",
|
|
487
|
+
"correlations",
|
|
481
488
|
]
|
|
@@ -6,7 +6,7 @@ Run-centric API for creating and managing runs (v3 atomic unit)
|
|
|
6
6
|
from typing import Optional, Dict, Any, List, Literal
|
|
7
7
|
from dataclasses import dataclass
|
|
8
8
|
from datetime import datetime
|
|
9
|
-
from ..client import post, get, put, delete
|
|
9
|
+
from ..client import post, get, put, delete as _client_delete
|
|
10
10
|
|
|
11
11
|
RunOutcome = Literal[
|
|
12
12
|
"resolved",
|
|
@@ -245,7 +245,7 @@ def delete_run(run_id: str) -> None:
|
|
|
245
245
|
Args:
|
|
246
246
|
run_id: The run ID to delete
|
|
247
247
|
"""
|
|
248
|
-
|
|
248
|
+
_client_delete(f"/runs/{run_id}", api_version="v3")
|
|
249
249
|
|
|
250
250
|
|
|
251
251
|
def batch_create(runs_data: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
@@ -329,6 +329,10 @@ def get_stats(
|
|
|
329
329
|
return RunStats.from_dict(response)
|
|
330
330
|
|
|
331
331
|
|
|
332
|
+
# Convenience aliases for cross-SDK parity
|
|
333
|
+
list = list_runs
|
|
334
|
+
delete = delete_run
|
|
335
|
+
|
|
332
336
|
__all__ = [
|
|
333
337
|
"RunResult",
|
|
334
338
|
"RunStats",
|
|
@@ -340,4 +344,6 @@ __all__ = [
|
|
|
340
344
|
"delete_run",
|
|
341
345
|
"batch_create",
|
|
342
346
|
"get_stats",
|
|
347
|
+
"list",
|
|
348
|
+
"delete",
|
|
343
349
|
]
|
|
@@ -4,7 +4,7 @@ Trace session grouping for managing conversation sessions
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
from typing import Optional, Dict, Any, List
|
|
7
|
-
from ..client import get
|
|
7
|
+
from ..client import get, api_request
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def list_sessions(
|
|
@@ -30,7 +30,7 @@ def list_sessions(
|
|
|
30
30
|
"offset": offset,
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
return
|
|
33
|
+
return api_request("GET", "/sessions", params=params, api_version="")
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
def get_session_traces(session_id: str, agent_id: str) -> Dict[str, Any]:
|
|
@@ -52,7 +52,11 @@ def get_session_traces(session_id: str, agent_id: str) -> Dict[str, Any]:
|
|
|
52
52
|
return get("/traces", params=params, api_version="")
|
|
53
53
|
|
|
54
54
|
|
|
55
|
+
# Convenience alias for cross-SDK parity
|
|
56
|
+
list = list_sessions
|
|
57
|
+
|
|
55
58
|
__all__ = [
|
|
56
59
|
"list_sessions",
|
|
57
60
|
"get_session_traces",
|
|
61
|
+
"list",
|
|
58
62
|
]
|
|
@@ -117,6 +117,9 @@ def update(test_id: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
117
117
|
return patch(f"/shadow-tests/{test_id}", data, api_version="")
|
|
118
118
|
|
|
119
119
|
|
|
120
|
+
# Convenience alias for cross-SDK parity
|
|
121
|
+
list = list_tests
|
|
122
|
+
|
|
120
123
|
__all__ = [
|
|
121
124
|
"ShadowTest",
|
|
122
125
|
"list_tests",
|
|
@@ -124,4 +127,5 @@ __all__ = [
|
|
|
124
127
|
"get_by_fix",
|
|
125
128
|
"create",
|
|
126
129
|
"update",
|
|
130
|
+
"list",
|
|
127
131
|
]
|
|
@@ -5,7 +5,7 @@ Behavioral signal management for detecting patterns in agent interactions
|
|
|
5
5
|
|
|
6
6
|
from typing import Optional, Dict, Any, List
|
|
7
7
|
from dataclasses import dataclass
|
|
8
|
-
from ..client import post, get, put, delete
|
|
8
|
+
from ..client import post, get, put, delete as _client_delete, api_request
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@dataclass
|
|
@@ -58,7 +58,10 @@ def list_signals(
|
|
|
58
58
|
if is_enabled is not None:
|
|
59
59
|
params["isEnabled"] = is_enabled
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
response = api_request("GET", "/signals/", params=params, api_version="v1")
|
|
62
|
+
if type(response) is type([]):
|
|
63
|
+
return response
|
|
64
|
+
return response.get("signals", [])
|
|
62
65
|
|
|
63
66
|
|
|
64
67
|
def create(
|
|
@@ -140,7 +143,7 @@ def delete_signal(signal_id: str) -> None:
|
|
|
140
143
|
Args:
|
|
141
144
|
signal_id: The signal ID to delete
|
|
142
145
|
"""
|
|
143
|
-
|
|
146
|
+
_client_delete(f"/signals/{signal_id}", api_version="v1")
|
|
144
147
|
|
|
145
148
|
|
|
146
149
|
def seed_defaults() -> Dict[str, Any]:
|
|
@@ -179,7 +182,10 @@ def get_stats(
|
|
|
179
182
|
if agent_id is not None:
|
|
180
183
|
params["agentId"] = agent_id
|
|
181
184
|
|
|
182
|
-
|
|
185
|
+
response = api_request("GET", "/signals/stats", params=params, api_version="v1")
|
|
186
|
+
if type(response) is type([]):
|
|
187
|
+
return {"stats": response}
|
|
188
|
+
return response
|
|
183
189
|
|
|
184
190
|
|
|
185
191
|
def get_trends(
|
|
@@ -272,6 +278,10 @@ def get_events(
|
|
|
272
278
|
return get(f"/signals/{signal_id}/events", params=params, api_version="v1")
|
|
273
279
|
|
|
274
280
|
|
|
281
|
+
# Convenience aliases for cross-SDK parity
|
|
282
|
+
list = list_signals
|
|
283
|
+
delete = delete_signal
|
|
284
|
+
|
|
275
285
|
__all__ = [
|
|
276
286
|
"Signal",
|
|
277
287
|
"list_signals",
|
|
@@ -283,4 +293,6 @@ __all__ = [
|
|
|
283
293
|
"get_trends",
|
|
284
294
|
"get_traces",
|
|
285
295
|
"get_events",
|
|
296
|
+
"list",
|
|
297
|
+
"delete",
|
|
286
298
|
]
|
|
@@ -11,7 +11,7 @@ from dataclasses import dataclass
|
|
|
11
11
|
|
|
12
12
|
T = TypeVar('T')
|
|
13
13
|
|
|
14
|
-
SDK_VERSION = "4.0
|
|
14
|
+
SDK_VERSION = "4.2.0"
|
|
15
15
|
|
|
16
16
|
# Retry configuration
|
|
17
17
|
MAX_RETRIES = 3
|
|
@@ -70,6 +70,7 @@ class RateLimitError(ThinkHiveApiError):
|
|
|
70
70
|
def __init__(self, message: str, retry_after_ms: Optional[int] = None):
|
|
71
71
|
super().__init__(message, status_code=429, code="RATE_LIMIT_EXCEEDED")
|
|
72
72
|
self.retry_after_ms = retry_after_ms
|
|
73
|
+
self.retry_after = retry_after_ms # Alias for cross-SDK parity
|
|
73
74
|
|
|
74
75
|
|
|
75
76
|
class IpWhitelistError(ThinkHiveApiError):
|
|
@@ -95,6 +96,7 @@ def configure(
|
|
|
95
96
|
endpoint: Optional[str] = None,
|
|
96
97
|
timeout: Optional[int] = None,
|
|
97
98
|
max_retries: Optional[int] = None,
|
|
99
|
+
service_name: Optional[str] = None,
|
|
98
100
|
) -> None:
|
|
99
101
|
"""
|
|
100
102
|
Configure the HTTP client
|
|
@@ -105,6 +107,7 @@ def configure(
|
|
|
105
107
|
endpoint: API endpoint URL
|
|
106
108
|
timeout: Request timeout in seconds (default: 30)
|
|
107
109
|
max_retries: Maximum number of retries for transient failures (default: 3)
|
|
110
|
+
service_name: Service name for tracing (default: 'my-ai-agent')
|
|
108
111
|
"""
|
|
109
112
|
global _config
|
|
110
113
|
|
|
@@ -118,6 +121,8 @@ def configure(
|
|
|
118
121
|
_config["timeout"] = timeout
|
|
119
122
|
if max_retries is not None:
|
|
120
123
|
_config["max_retries"] = max_retries
|
|
124
|
+
if service_name is not None:
|
|
125
|
+
_config["service_name"] = service_name
|
|
121
126
|
|
|
122
127
|
# Try environment variables as fallback
|
|
123
128
|
if not _config["api_key"]:
|
|
@@ -97,7 +97,10 @@ def _guardrails_request(method: str, path: str, body: Optional[Dict[str, Any]] =
|
|
|
97
97
|
data = response.json()
|
|
98
98
|
if not data.get("success", True):
|
|
99
99
|
raise ThinkHiveApiError(data.get("message", "Request failed"), 500)
|
|
100
|
-
|
|
100
|
+
# Support both wrapped {success, data} and raw responses
|
|
101
|
+
if "data" in data:
|
|
102
|
+
return data["data"]
|
|
103
|
+
return data
|
|
101
104
|
|
|
102
105
|
except requests.exceptions.Timeout:
|
|
103
106
|
if attempt < max_retries:
|
|
@@ -211,3 +214,7 @@ def list_scanners() -> List[Dict[str, Any]]:
|
|
|
211
214
|
"""
|
|
212
215
|
data = _guardrails_request("GET", "/v1/guardrails/scanners")
|
|
213
216
|
return data.get("scanners", [])
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# Convenience alias for cross-SDK parity
|
|
220
|
+
evaluate = scan
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: thinkhive
|
|
3
|
-
Version: 4.2.
|
|
3
|
+
Version: 4.2.2
|
|
4
4
|
Summary: AI agent observability SDK with business metrics, ROI analytics, and 25+ trace format support
|
|
5
5
|
Home-page: https://github.com/Abdul-Omira/ThinkHiveMind
|
|
6
6
|
Author: ThinkHive
|
|
@@ -30,7 +30,7 @@ Dynamic: requires-dist
|
|
|
30
30
|
Dynamic: requires-python
|
|
31
31
|
Dynamic: summary
|
|
32
32
|
|
|
33
|
-
# ThinkHive Python SDK v4.2.
|
|
33
|
+
# ThinkHive Python SDK v4.2.1
|
|
34
34
|
|
|
35
35
|
OpenTelemetry-based observability SDK for AI agents supporting 25 trace formats including LangSmith, Langfuse, Opik, Braintrust, Datadog, MLflow, and more.
|
|
36
36
|
|
|
@@ -53,7 +53,7 @@ thinkhive.init(
|
|
|
53
53
|
)
|
|
54
54
|
|
|
55
55
|
# Trace LLM calls
|
|
56
|
-
@thinkhive.trace_llm(model_name="gpt-4", provider="openai")
|
|
56
|
+
@thinkhive.trace_llm(name="generate-response", model_name="gpt-4", provider="openai")
|
|
57
57
|
def call_llm(prompt):
|
|
58
58
|
response = openai.chat.completions.create(
|
|
59
59
|
model="gpt-4",
|
|
@@ -62,13 +62,13 @@ def call_llm(prompt):
|
|
|
62
62
|
return response
|
|
63
63
|
|
|
64
64
|
# Trace retrieval operations
|
|
65
|
-
@thinkhive.trace_retrieval()
|
|
65
|
+
@thinkhive.trace_retrieval(name="search-kb", query="refund policy")
|
|
66
66
|
def search_documents(query):
|
|
67
67
|
results = vector_db.search(query)
|
|
68
68
|
return results
|
|
69
69
|
|
|
70
70
|
# Trace tool calls
|
|
71
|
-
@thinkhive.trace_tool(tool_name="web_search")
|
|
71
|
+
@thinkhive.trace_tool(name="lookup-order", tool_name="web_search")
|
|
72
72
|
def search_web(query):
|
|
73
73
|
return requests.get(f"https://api.example.com/search?q={query}")
|
|
74
74
|
```
|
|
@@ -317,7 +317,7 @@ result = api_keys.create(
|
|
|
317
317
|
print(f"Key: {result['key']}") # Only shown once!
|
|
318
318
|
|
|
319
319
|
# List all keys
|
|
320
|
-
keys = api_keys.list_keys()
|
|
320
|
+
keys = api_keys.list() # or api_keys.list_keys()
|
|
321
321
|
|
|
322
322
|
# Revoke a key
|
|
323
323
|
api_keys.revoke("key-id")
|
|
@@ -433,7 +433,7 @@ from thinkhive import (
|
|
|
433
433
|
try:
|
|
434
434
|
result = runs.create(agent_id="agent-123", ...)
|
|
435
435
|
except RateLimitError as e:
|
|
436
|
-
print(f"Rate limited. Retry after {e.
|
|
436
|
+
print(f"Rate limited. Retry after {e.retry_after}ms") # or e.retry_after_ms
|
|
437
437
|
except AgentScopeError as e:
|
|
438
438
|
print(f"No access to agent. Allowed: {e.allowed_agents}")
|
|
439
439
|
except NotFoundError as e:
|
|
@@ -511,7 +511,7 @@ print(f"Estimated cost: {cost['estimatedCredits']} credits")
|
|
|
511
511
|
from thinkhive.api import signals
|
|
512
512
|
|
|
513
513
|
# List all signals
|
|
514
|
-
all_signals = signals.list_signals()
|
|
514
|
+
all_signals = signals.list() # or signals.list_signals()
|
|
515
515
|
|
|
516
516
|
# Create a custom signal
|
|
517
517
|
signals.create(
|
|
@@ -540,7 +540,8 @@ notifications.create_rule({
|
|
|
540
540
|
})
|
|
541
541
|
|
|
542
542
|
# List notifications
|
|
543
|
-
alerts = notifications.
|
|
543
|
+
alerts = notifications.list("agent-123", unread_only=True)
|
|
544
|
+
# or: notifications.list_notifications("agent-123", unread_only=True)
|
|
544
545
|
```
|
|
545
546
|
|
|
546
547
|
## Documents (RAG)
|
|
@@ -552,7 +553,7 @@ from thinkhive.api import documents
|
|
|
552
553
|
documents.upload("agent-123", file_name="faq.txt", file_type="text/plain", file_size=1024)
|
|
553
554
|
|
|
554
555
|
# List documents
|
|
555
|
-
docs = documents.list_documents("agent-123")
|
|
556
|
+
docs = documents.list("agent-123") # or documents.list_documents("agent-123")
|
|
556
557
|
```
|
|
557
558
|
|
|
558
559
|
## Shadow Tests
|
|
@@ -576,7 +577,8 @@ shadow_tests.create(
|
|
|
576
577
|
from thinkhive.api import sessions
|
|
577
578
|
|
|
578
579
|
# List conversation sessions
|
|
579
|
-
all_sessions = sessions.
|
|
580
|
+
all_sessions = sessions.list("agent-123", limit=20)
|
|
581
|
+
# or: sessions.list_sessions("agent-123", limit=20)
|
|
580
582
|
|
|
581
583
|
# Get all traces in a session
|
|
582
584
|
traces = sessions.get_session_traces("session-789", "agent-123")
|
|
@@ -601,7 +603,8 @@ from thinkhive.api import llm_costs
|
|
|
601
603
|
from thinkhive import format_cost
|
|
602
604
|
|
|
603
605
|
# Get cost summary
|
|
604
|
-
summary = llm_costs.
|
|
606
|
+
summary = llm_costs.summary(period="30d")
|
|
607
|
+
# or: llm_costs.get_summary(period="30d")
|
|
605
608
|
print(f"Total cost: {format_cost(summary.get('totalCost', 0))}")
|
|
606
609
|
|
|
607
610
|
# Get per-agent breakdown
|
|
@@ -625,6 +628,32 @@ savings = llm_costs.get_savings()
|
|
|
625
628
|
|
|
626
629
|
## Upgrading
|
|
627
630
|
|
|
631
|
+
### v4.2.0 → v4.2.1
|
|
632
|
+
|
|
633
|
+
Improvements in v4.2.1:
|
|
634
|
+
- **Decorators** now accept `name=` parameter for custom span names (`@trace_llm(name="my-llm")`)
|
|
635
|
+
- **`configure()`** now accepts `service_name` parameter
|
|
636
|
+
- **`RateLimitError`** has `retry_after` alias (in addition to `retry_after_ms`)
|
|
637
|
+
- **Rule helpers** (`create_regex_rule`, etc.) now return named rule objects with `{type, name, config}` structure
|
|
638
|
+
- **`calculate_pass_at_k`** now supports both `(pass_rate, k)` and `(n, c, k)` calling conventions
|
|
639
|
+
- **`aggregate_worst`** / **`aggregate_average`** accept simple `[float]` lists in addition to `[dict]`
|
|
640
|
+
- **Method aliases added** for cross-SDK consistency:
|
|
641
|
+
- `runs.list()` / `runs.delete()`
|
|
642
|
+
- `calibration.status()` / `calibration.all_metrics()`
|
|
643
|
+
- `signals.list()` / `signals.delete()`
|
|
644
|
+
- `api_keys.list()`
|
|
645
|
+
- `linking.create()` / `linking.get_for_run()` / `linking.stats()` / `linking.verify()` / `linking.delete()`
|
|
646
|
+
- `roi_analytics.summary()` / `roi_analytics.correlations()`
|
|
647
|
+
- `business_metrics.current()` / `business_metrics.history()` / `business_metrics.record()`
|
|
648
|
+
- `quality_metrics.evaluate()`
|
|
649
|
+
- `llm_costs.summary()`
|
|
650
|
+
- `notifications.list()`
|
|
651
|
+
- `documents.list()` / `documents.delete()`
|
|
652
|
+
- `shadow_tests.list()`
|
|
653
|
+
- `sessions.list()`
|
|
654
|
+
- `customer_context.capture()`
|
|
655
|
+
- `guardrails.evaluate()`
|
|
656
|
+
|
|
628
657
|
### v4.1.0 → v4.2.0
|
|
629
658
|
|
|
630
659
|
New in v4.2.0:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|