aiqa-client 0.5.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

aiqa/tracing_llm_utils.py

@@ -38,16 +38,18 @@ def _extract_and_set_token_usage(span: trace.Span, result: Any) -> None:
     Extract OpenAI API style token usage from result and add to span attributes
     using OpenTelemetry semantic conventions for gen_ai.
     
-    Looks for usage dict with prompt_tokens, completion_tokens, and total_tokens.
+    Looks for usage dict or object with prompt_tokens, completion_tokens, and total_tokens.
     Sets gen_ai.usage.input_tokens, gen_ai.usage.output_tokens, and gen_ai.usage.total_tokens.
     Only sets attributes that are not already set.
     
     This function detects token usage from OpenAI API response patterns:
-    - OpenAI Chat Completions API: The 'usage' object contains 'prompt_tokens', 'completion_tokens', and 'total_tokens'.
+    - OpenAI Chat Completions API: The 'usage' object (dict or Usage object) contains 'prompt_tokens', 'completion_tokens', and 'total_tokens'.
       See https://platform.openai.com/docs/api-reference/chat/object (usage field)
     - OpenAI Completions API: The 'usage' object contains 'prompt_tokens', 'completion_tokens', and 'total_tokens'.
       See https://platform.openai.com/docs/api-reference/completions/object (usage field)
     
+    Handles both dict and object cases (e.g., OpenAI SDK Usage objects).
+    
     This function is safe against exceptions and will not derail tracing or program execution.
     """
     try:
@@ -74,15 +76,24 @@ def _extract_and_set_token_usage(span: trace.Span, result: Any) -> None:
             # If accessing result properties fails, just return silently
             return
         
-        # Extract token usage if found
-        if isinstance(usage, dict):
+        # Extract token usage if found (handle both dict and object cases)
+        if usage is not None:
             try:
                 # Support both OpenAI format (prompt_tokens/completion_tokens) and Bedrock format (input_tokens/output_tokens)
-                prompt_tokens = usage.get("prompt_tokens") or usage.get("PromptTokens")
-                completion_tokens = usage.get("completion_tokens") or usage.get("CompletionTokens")
-                input_tokens = usage.get("input_tokens") or usage.get("InputTokens")
-                output_tokens = usage.get("output_tokens") or usage.get("OutputTokens")
-                total_tokens = usage.get("total_tokens") or usage.get("TotalTokens")
+                # Handle dict case
+                if isinstance(usage, dict):
+                    prompt_tokens = usage.get("prompt_tokens") or usage.get("PromptTokens")
+                    completion_tokens = usage.get("completion_tokens") or usage.get("CompletionTokens")
+                    input_tokens = usage.get("input_tokens") or usage.get("InputTokens")
+                    output_tokens = usage.get("output_tokens") or usage.get("OutputTokens")
+                    total_tokens = usage.get("total_tokens") or usage.get("TotalTokens")
+                # Handle object case (e.g., OpenAI Usage object)
+                else:
+                    prompt_tokens = getattr(usage, "prompt_tokens", None) or getattr(usage, "PromptTokens", None)
+                    completion_tokens = getattr(usage, "completion_tokens", None) or getattr(usage, "CompletionTokens", None)
+                    input_tokens = getattr(usage, "input_tokens", None) or getattr(usage, "InputTokens", None)
+                    output_tokens = getattr(usage, "output_tokens", None) or getattr(usage, "OutputTokens", None)
+                    total_tokens = getattr(usage, "total_tokens", None) or getattr(usage, "TotalTokens", None)
                 
                 # Use Bedrock format if OpenAI format not available
                 if prompt_tokens is None:

aiqa/types.py

@@ -0,0 +1,61 @@
+
+
+from typing import TypedDict, Dict, Optional, Awaitable, Callable, Any, Union, Annotated, List
+from numbers import Number
+
+class Metric(TypedDict):
+  """Definition of a metric to score. See Metric.ts for more details."""
+  id: str
+  type: str
+  name: Optional[str] = None
+  description: Optional[str] = None
+  provider: Optional[str] = None
+  model: Optional[str] = None
+  prompt: Optional[str] = None
+  code: Optional[str] = None
+
+class Example(TypedDict):
+  """Definition of an example to score. See Example.ts for more details."""
+  id: str
+  input: Optional[str] = None
+  spans: Optional[List[Dict[str, Any]]] = None
+  metrics: Optional[List[Metric]] = None
+
+class MetricResult(TypedDict):
+  """Result of evaluating a metric on an output (i.e. a single metric for a single example)."""
+  score: Annotated[Number, "Numeric score for the metric evaluation, typically a float in the range [0, 1]"]
+  message: Optional[str] = None
+  error: Optional[str] = None
+
+class Result(TypedDict):
+  """Result of evaluating a set of metrics on an output (i.e. the full set of metrics for a single example)."""
+  example: str
+  scores: Dict[str, Number]
+  messages: Optional[Dict[str, str]] = None
+  errors: Optional[Dict[str, str]] = None
+
+
+# Function that processes input and parameters to produce an output (sync or async)
+# Args:
+#   input: The input data for the example (typically a dict with the example's input fields)
+#   parameters: Dictionary of parameters to pass to the function (e.g., model settings, temperature)
+# Returns:
+#   The output result, which can be any type. If the function is async, returns an Awaitable.
+CallMyCodeType = Callable[[Any, Dict[str, Any]], Union[Any, Awaitable[Any]]]
+
+# Function that calls an LLM with a system prompt and user message (async)
+# Args:
+#   system_prompt: The system prompt/instructions for the LLM
+#   user_message: The "user" message containing the content to process (e.g., input and output to score)
+# Returns:
+#   The raw response content string from the LLM (typically JSON for scoring)
+CallLLMType = Callable[[str, str], Awaitable[str]]
+
+# Function that scores a given output, using input, example, and parameters (usually async)
+# Args:
+#   input: The input data for the example (typically a dict with the example's input fields)
+#   output: The output to score
+#   metric: The metric to score
+# Returns:
+#   MetricResult object with score:[0,1], message (optional), and error (optional)
+ScoreThisInputOutputMetricType = Callable[[Any, Any, Metric], Awaitable[MetricResult]]

{aiqa_client-0.5.2.dist-info → aiqa_client-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiqa-client
-Version: 0.5.2
+Version: 0.7.0
 Summary: OpenTelemetry-based Python client for tracing functions and sending traces to the AIQA server
 Author-email: AIQA <info@aiqa.dev>
 License: MIT

aiqa_client-0.7.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+aiqa/__init__.py,sha256=JLQjgQgsyGQ1mRl4kcYygJq9i_91jN4WDem3dF1eMGA,1888
+aiqa/client.py,sha256=zS9OQQhdvVeIoBz0o8qrz-rjXngEbS9Lrli2ZWNIsrM,15993
+aiqa/constants.py,sha256=Z1Z8wzO5_tNGCAvhKtxdCB1zCjQFvL8tctwLkL1og3w,226
+aiqa/experiment_runner.py,sha256=c_XxeM3W3EfsK4WoC6BCh6wr0Ph1M-W6Bhi9wkmFSKo,17313
+aiqa/http_utils.py,sha256=OIB4tRI2TiDl4VKDmtbLWg9Q7TicMBeL7scLYEhVPXI,4944
+aiqa/llm_as_judge.py,sha256=ESmqQfaYpypCNfsODkdn5s85n_nzJ4WKbhUMVTb2djE,10087
+aiqa/object_serialiser.py,sha256=mzd2U_mFcAPalN2m9wxq35-BBeRJOhNK1k0-BmRSfQM,17055
+aiqa/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aiqa/span_helpers.py,sha256=Ht4T_JJXK4HlqBY_Qwe8QDk9XwWCjagx_DkOUVY-PmY,18189
+aiqa/tracing.py,sha256=yOQoeiJi1VeITnrJL2giz1gMN8xKt-ZR35HeY40pj5U,35281
+aiqa/tracing_llm_utils.py,sha256=zQSxzkEhPmgel1P2kFueNWTr846re-qHEFxD-_EHhNQ,10241
+aiqa/types.py,sha256=Rv27oC1R0P1soJz5wsdwkVW-jfHQEVi4vUhwRJid270,2529
+aiqa_client-0.7.0.dist-info/licenses/LICENSE.txt,sha256=kIzkzLuzG0HHaWYm4F4W5FeJ1Yxut3Ec6bhLWyw798A,1062
+aiqa_client-0.7.0.dist-info/METADATA,sha256=diEKUaiP3xtihOXOsFN6dPozS9EjzqY_NoR5XrijaUY,7705
+aiqa_client-0.7.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+aiqa_client-0.7.0.dist-info/top_level.txt,sha256=nwcsuVVSuWu27iLxZd4n1evVzv1W6FVTrSnCXCc-NQs,5
+aiqa_client-0.7.0.dist-info/RECORD,,

{aiqa_client-0.5.2.dist-info → aiqa_client-0.7.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

aiqa_client-0.5.2.dist-info/RECORD

@@ -1,14 +0,0 @@
-aiqa/__init__.py,sha256=V1VLfpxU_BXfkvKyhGckZsPYF43NJhoXeVX6FkeHr9g,1721
-aiqa/client.py,sha256=Ba3v-voBlTSCr-RU88INLXsF_5vqp42QiQWCFciSJbU,12542
-aiqa/constants.py,sha256=tZuh7XvKs6hFvWc-YnQ5Na6uogJMsRrMy-rWOauvcIA,226
-aiqa/experiment_runner.py,sha256=XAZsjVP70UH_QTk5ANSOQYAhmozuGXwKB5qWWHs-zeE,11186
-aiqa/http_utils.py,sha256=OIB4tRI2TiDl4VKDmtbLWg9Q7TicMBeL7scLYEhVPXI,4944
-aiqa/object_serialiser.py,sha256=DBv7EyXIwfwjwXHDsIwdZNFmQffRb5fKAE0r8qhoqgc,16958
-aiqa/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aiqa/tracing.py,sha256=gdmtpvBVbvc_HrJHgtr78_XH9sIWRjUoEkXuCuNmuc0,45662
-aiqa/tracing_llm_utils.py,sha256=rNx6v6Wh_Mhv-_DPU9_aWS7YQcO46oiv0YPdBK1KVL8,9338
-aiqa_client-0.5.2.dist-info/licenses/LICENSE.txt,sha256=kIzkzLuzG0HHaWYm4F4W5FeJ1Yxut3Ec6bhLWyw798A,1062
-aiqa_client-0.5.2.dist-info/METADATA,sha256=xMaQSnI3AiNE6lYs2vM6BV9VxQWMHXyDoIl6JXwdi3I,7705
-aiqa_client-0.5.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aiqa_client-0.5.2.dist-info/top_level.txt,sha256=nwcsuVVSuWu27iLxZd4n1evVzv1W6FVTrSnCXCc-NQs,5
-aiqa_client-0.5.2.dist-info/RECORD,,