aiqa-client 0.4.7__tar.gz → 0.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aiqa_client-0.4.7/aiqa_client.egg-info → aiqa_client-0.6.1}/PKG-INFO +1 -1
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/__init__.py +9 -3
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/client.py +113 -16
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/constants.py +1 -1
- aiqa_client-0.6.1/aiqa/experiment_runner.py +490 -0
- aiqa_client-0.6.1/aiqa/http_utils.py +143 -0
- aiqa_client-0.6.1/aiqa/llm_as_judge.py +281 -0
- aiqa_client-0.6.1/aiqa/span_helpers.py +511 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/tracing.py +202 -566
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/tracing_llm_utils.py +20 -9
- aiqa_client-0.6.1/aiqa/types.py +61 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1/aiqa_client.egg-info}/PKG-INFO +1 -1
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa_client.egg-info/SOURCES.txt +7 -1
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/pyproject.toml +1 -1
- aiqa_client-0.6.1/tests/test_chatbot.py +87 -0
- aiqa_client-0.6.1/tests/test_integration.py +322 -0
- aiqa_client-0.6.1/tests/test_integration_api_key.py +96 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/tests/test_object_serialiser.py +0 -33
- aiqa_client-0.6.1/tests/test_span_helpers.py +345 -0
- aiqa_client-0.6.1/tests/test_startup_reliability.py +108 -0
- aiqa_client-0.6.1/tests/test_tracing.py +798 -0
- aiqa_client-0.4.7/aiqa/aiqa_exporter.py +0 -772
- aiqa_client-0.4.7/aiqa/experiment_runner.py +0 -319
- aiqa_client-0.4.7/aiqa/http_utils.py +0 -69
- aiqa_client-0.4.7/tests/test_startup_reliability.py +0 -249
- aiqa_client-0.4.7/tests/test_tracing.py +0 -230
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/LICENSE.txt +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/MANIFEST.in +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/README.md +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/object_serialiser.py +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa/py.typed +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa_client.egg-info/dependency_links.txt +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa_client.egg-info/requires.txt +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/aiqa_client.egg-info/top_level.txt +0 -0
- {aiqa_client-0.4.7 → aiqa_client-0.6.1}/setup.cfg +0 -0
|
@@ -6,7 +6,7 @@ The client initializes automatically when WithTracing is first used.
|
|
|
6
6
|
|
|
7
7
|
Set environment variables:
|
|
8
8
|
AIQA_SERVER_URL: URL of the AIQA server
|
|
9
|
-
AIQA_API_KEY: API key for authentication
|
|
9
|
+
AIQA_API_KEY: API key for authentication (required)
|
|
10
10
|
AIQA_COMPONENT_TAG: Optional component identifier
|
|
11
11
|
AIQA_STARTUP_DELAY_SECONDS: Optional delay before first flush (default: 10s)
|
|
12
12
|
|
|
@@ -26,8 +26,8 @@ Example:
|
|
|
26
26
|
result = my_function()
|
|
27
27
|
"""
|
|
28
28
|
|
|
29
|
-
from .tracing import
|
|
30
|
-
|
|
29
|
+
from .tracing import WithTracing
|
|
30
|
+
from .span_helpers import (
|
|
31
31
|
flush_tracing,
|
|
32
32
|
set_span_attribute,
|
|
33
33
|
set_span_name,
|
|
@@ -39,7 +39,10 @@ from .tracing import (
|
|
|
39
39
|
extract_trace_context,
|
|
40
40
|
set_conversation_id,
|
|
41
41
|
set_component_tag,
|
|
42
|
+
set_token_usage,
|
|
43
|
+
set_provider_and_model,
|
|
42
44
|
get_span,
|
|
45
|
+
submit_feedback,
|
|
43
46
|
)
|
|
44
47
|
from .client import get_aiqa_client
|
|
45
48
|
from .experiment_runner import ExperimentRunner
|
|
@@ -60,7 +63,10 @@ __all__ = [
|
|
|
60
63
|
"extract_trace_context",
|
|
61
64
|
"set_conversation_id",
|
|
62
65
|
"set_component_tag",
|
|
66
|
+
"set_token_usage",
|
|
67
|
+
"set_provider_and_model",
|
|
63
68
|
"get_span",
|
|
69
|
+
"submit_feedback",
|
|
64
70
|
"VERSION",
|
|
65
71
|
]
|
|
66
72
|
|
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
import os
|
|
3
3
|
import logging
|
|
4
4
|
from functools import lru_cache
|
|
5
|
-
from typing import Optional, TYPE_CHECKING, Any
|
|
5
|
+
from typing import Optional, TYPE_CHECKING, Any, Dict
|
|
6
6
|
from opentelemetry import trace
|
|
7
7
|
from opentelemetry.sdk.trace import TracerProvider
|
|
8
|
-
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
8
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult, SpanExporter as SpanExporterBase
|
|
9
|
+
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
10
|
+
from opentelemetry.sdk.trace import ReadableSpan
|
|
11
|
+
from opentelemetry.trace import SpanContext
|
|
12
|
+
import requests
|
|
9
13
|
|
|
10
14
|
from .constants import AIQA_TRACER_NAME, LOG_TAG
|
|
11
15
|
|
|
@@ -30,7 +34,7 @@ except ImportError:
|
|
|
30
34
|
# Set to None so we can check later
|
|
31
35
|
TraceIdRatioBased = None
|
|
32
36
|
|
|
33
|
-
from .http_utils import get_server_url, get_api_key
|
|
37
|
+
from .http_utils import get_server_url, get_api_key, build_headers, format_http_error
|
|
34
38
|
|
|
35
39
|
class AIQAClient:
|
|
36
40
|
"""
|
|
@@ -100,14 +104,14 @@ class AIQAClient:
|
|
|
100
104
|
self.enabled = False
|
|
101
105
|
if self._provider:
|
|
102
106
|
self._provider.shutdown()
|
|
103
|
-
|
|
104
|
-
self._exporter.shutdown()
|
|
107
|
+
# OTLP exporter doesn't have a separate shutdown method - it's handled by the provider
|
|
105
108
|
except Exception as e:
|
|
106
109
|
logger.error(f"Error shutting down tracing: {e}")
|
|
107
110
|
# Still disable even if shutdown had errors
|
|
108
111
|
self.enabled = False
|
|
109
112
|
|
|
110
113
|
|
|
114
|
+
|
|
111
115
|
# Global singleton instance
|
|
112
116
|
client: AIQAClient = AIQAClient()
|
|
113
117
|
|
|
@@ -176,11 +180,10 @@ def _init_tracing() -> None:
|
|
|
176
180
|
server_url = get_server_url()
|
|
177
181
|
api_key = get_api_key()
|
|
178
182
|
|
|
179
|
-
if not
|
|
183
|
+
if not api_key:
|
|
180
184
|
client.enabled = False
|
|
181
|
-
missing_vars = [var for var, val in [("AIQA_SERVER_URL", server_url), ("AIQA_API_KEY", api_key)] if not val]
|
|
182
185
|
logger.warning(
|
|
183
|
-
f"AIQA tracing is disabled: missing required environment variables:
|
|
186
|
+
f"AIQA tracing is disabled: missing required environment variables: AIQA_API_KEY"
|
|
184
187
|
)
|
|
185
188
|
client._initialized = True
|
|
186
189
|
return
|
|
@@ -227,20 +230,52 @@ def _init_tracing() -> None:
|
|
|
227
230
|
|
|
228
231
|
def _attach_aiqa_processor(provider: TracerProvider) -> None:
|
|
229
232
|
"""Attach AIQA span processor to the provider. Idempotent - safe to call multiple times."""
|
|
230
|
-
from .aiqa_exporter import AIQASpanExporter
|
|
231
|
-
|
|
232
233
|
try:
|
|
233
234
|
# Check if already attached
|
|
234
235
|
for p in provider._active_span_processor._span_processors:
|
|
235
|
-
if isinstance(getattr(p, "exporter", None),
|
|
236
|
+
if isinstance(getattr(p, "exporter", None), OTLPSpanExporter):
|
|
236
237
|
logger.debug(f"AIQA span processor already attached, skipping")
|
|
237
238
|
return
|
|
238
239
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
240
|
+
server_url = get_server_url()
|
|
241
|
+
api_key = get_api_key()
|
|
242
|
+
|
|
243
|
+
# Build headers for authentication
|
|
244
|
+
# OTLP exporter sets its own Content-Type, so we only need Authorization
|
|
245
|
+
auth_headers = {}
|
|
246
|
+
if api_key:
|
|
247
|
+
auth_headers["Authorization"] = f"ApiKey {api_key}"
|
|
248
|
+
elif os.getenv("AIQA_API_KEY"):
|
|
249
|
+
auth_headers["Authorization"] = f"ApiKey {os.getenv('AIQA_API_KEY')}"
|
|
250
|
+
|
|
251
|
+
# OTLP HTTP exporter requires the full endpoint URL including /v1/traces
|
|
252
|
+
# Ensure server_url doesn't have trailing slash or /v1/traces, then append /v1/traces
|
|
253
|
+
base_url = server_url.rstrip('/')
|
|
254
|
+
if base_url.endswith('/v1/traces'):
|
|
255
|
+
endpoint = base_url
|
|
256
|
+
else:
|
|
257
|
+
endpoint = f"{base_url}/v1/traces"
|
|
258
|
+
|
|
259
|
+
# Get timeout from environment variable (in seconds)
|
|
260
|
+
# Supports OTEL_EXPORTER_OTLP_TIMEOUT (standard) or AIQA_EXPORT_TIMEOUT (custom)
|
|
261
|
+
# Default is 30 seconds (more generous than OTLP default of 10s)
|
|
262
|
+
timeout = 30.0
|
|
263
|
+
otlp_timeout = os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT")
|
|
264
|
+
|
|
265
|
+
if otlp_timeout:
|
|
266
|
+
try:
|
|
267
|
+
timeout = float(otlp_timeout)
|
|
268
|
+
except ValueError:
|
|
269
|
+
logger.warning(f"Invalid OTEL_EXPORTER_OTLP_TIMEOUT value '{otlp_timeout}', using default 30.0")
|
|
270
|
+
|
|
271
|
+
# Create OTLP exporter with authentication headers and timeout
|
|
272
|
+
# The exporter will set Content-Type and other headers automatically
|
|
273
|
+
exporter = OTLPSpanExporter(
|
|
274
|
+
endpoint=endpoint,
|
|
275
|
+
headers=auth_headers if auth_headers else None,
|
|
276
|
+
timeout=timeout,
|
|
243
277
|
)
|
|
278
|
+
|
|
244
279
|
provider.add_span_processor(BatchSpanProcessor(exporter))
|
|
245
280
|
global client
|
|
246
281
|
client.exporter = exporter
|
|
@@ -266,4 +301,66 @@ def get_aiqa_tracer() -> trace.Tracer:
|
|
|
266
301
|
except Exception as e:
|
|
267
302
|
# Log issue but still return a tracer
|
|
268
303
|
logger.info(f"Issue getting AIQA tracer with version: {e}, using fallback")
|
|
269
|
-
return trace.get_tracer(AIQA_TRACER_NAME)
|
|
304
|
+
return trace.get_tracer(AIQA_TRACER_NAME)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def get_organisation(
|
|
308
|
+
organisation_id: str,
|
|
309
|
+
server_url: Optional[str] = None,
|
|
310
|
+
api_key: Optional[str] = None
|
|
311
|
+
) -> Dict[str, Any]:
|
|
312
|
+
"""
|
|
313
|
+
Get organisation information based on API key via an API call.
|
|
314
|
+
|
|
315
|
+
Args:
|
|
316
|
+
organisation_id: ID of the organisation to retrieve
|
|
317
|
+
server_url: Optional server URL (defaults to AIQA_SERVER_URL env var)
|
|
318
|
+
api_key: Optional API key (defaults to AIQA_API_KEY env var)
|
|
319
|
+
|
|
320
|
+
Returns:
|
|
321
|
+
Organisation object as a dictionary
|
|
322
|
+
"""
|
|
323
|
+
url = get_server_url(server_url)
|
|
324
|
+
key = get_api_key(api_key)
|
|
325
|
+
headers = build_headers(key)
|
|
326
|
+
|
|
327
|
+
response = requests.get(
|
|
328
|
+
f"{url}/organisation/{organisation_id}",
|
|
329
|
+
headers=headers,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
if not response.ok:
|
|
333
|
+
raise Exception(format_http_error(response, "get organisation"))
|
|
334
|
+
|
|
335
|
+
return response.json()
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def get_api_key_info(
|
|
339
|
+
api_key_id: str,
|
|
340
|
+
server_url: Optional[str] = None,
|
|
341
|
+
api_key: Optional[str] = None
|
|
342
|
+
) -> Dict[str, Any]:
|
|
343
|
+
"""
|
|
344
|
+
Get API key information via an API call.
|
|
345
|
+
|
|
346
|
+
Args:
|
|
347
|
+
api_key_id: ID of the API key to retrieve
|
|
348
|
+
server_url: Optional server URL (defaults to AIQA_SERVER_URL env var)
|
|
349
|
+
api_key: Optional API key (defaults to AIQA_API_KEY env var)
|
|
350
|
+
|
|
351
|
+
Returns:
|
|
352
|
+
ApiKey object as a dictionary
|
|
353
|
+
"""
|
|
354
|
+
url = get_server_url(server_url)
|
|
355
|
+
key = get_api_key(api_key)
|
|
356
|
+
headers = build_headers(key)
|
|
357
|
+
|
|
358
|
+
response = requests.get(
|
|
359
|
+
f"{url}/api-key/{api_key_id}",
|
|
360
|
+
headers=headers,
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
if not response.ok:
|
|
364
|
+
raise Exception(format_http_error(response, "get api key info"))
|
|
365
|
+
|
|
366
|
+
return response.json()
|
|
@@ -3,6 +3,6 @@ Constants used across the AIQA client package.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
AIQA_TRACER_NAME = "aiqa-tracer"
|
|
6
|
-
VERSION = "0.
|
|
6
|
+
VERSION = "0.6.1" # automatically updated by set-version-json.sh
|
|
7
7
|
|
|
8
8
|
LOG_TAG = "AIQA" # Used in all logging output to identify AIQA messages
|