aiqa-client 0.6.1__tar.gz → 0.7.0__tar.gz

{aiqa_client-0.6.1/aiqa_client.egg-info → aiqa_client-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiqa-client
-Version: 0.6.1
+Version: 0.7.0
 Summary: OpenTelemetry-based Python client for tracing functions and sending traces to the AIQA server
 Author-email: AIQA <info@aiqa.dev>
 License: MIT

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/client.py

@@ -2,10 +2,10 @@
 import os
 import logging
 from functools import lru_cache
-from typing import Optional, TYPE_CHECKING, Any, Dict
+from typing import Optional, TYPE_CHECKING, Any, Dict, List
 from opentelemetry import trace
 from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult, SpanExporter as SpanExporterBase
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.trace import ReadableSpan
 from opentelemetry.trace import SpanContext
@@ -52,6 +52,8 @@ class AIQAClient:
             cls._instance._exporter = None # reduce circular import issues by not importing for typecheck here
             cls._instance._enabled: bool = True
             cls._instance._initialized: bool = False
+            cls._instance._default_ignore_patterns: List[str] = ["_*"]  # Default: filter properties starting with '_'
+            cls._instance._ignore_recursive: bool = True  # Default: recursive filtering enabled
         return cls._instance
     
     @property
@@ -90,6 +92,76 @@ class AIQAClient:
         logger.info(f"AIQA tracing {'enabled' if value else 'disabled'}")
         self._enabled = value
     
+    @property
+    def default_ignore_patterns(self) -> List[str]:
+        """
+        Get the default ignore patterns applied to all traced inputs and outputs.
+        
+        Default: ["_*"] (filters properties starting with '_')
+        
+        Returns:
+            List of ignore patterns (supports wildcards like "_*")
+        """
+        return self._default_ignore_patterns.copy()
+    
+    @default_ignore_patterns.setter
+    def default_ignore_patterns(self, value: Optional[List[str]]) -> None:
+        """
+        Set the default ignore patterns applied to all traced inputs and outputs.
+        
+        Args:
+            value: List of patterns to ignore (e.g., ["_*", "password"]).
+                   Set to None or [] to disable default ignore patterns.
+                   Supports wildcards (e.g., "_*" matches "_apple", "_fruit").
+        
+        Example:
+            from aiqa import get_aiqa_client
+            
+            client = get_aiqa_client()
+            # Add password to default ignore patterns
+            client.default_ignore_patterns = ["_*", "password", "api_key"]
+            # Disable default ignore patterns
+            client.default_ignore_patterns = []
+        """
+        if value is None:
+            self._default_ignore_patterns = []
+        else:
+            self._default_ignore_patterns = list(value)
+        logger.info(f"Default ignore patterns set to: {self._default_ignore_patterns}")
+    
+    @property
+    def ignore_recursive(self) -> bool:
+        """
+        Get whether ignore patterns are applied recursively to nested objects.
+        
+        Default: True (recursive filtering enabled)
+        
+        Returns:
+            True if recursive filtering is enabled, False otherwise
+        """
+        return self._ignore_recursive
+    
+    @ignore_recursive.setter
+    def ignore_recursive(self, value: bool) -> None:
+        """
+        Set whether ignore patterns are applied recursively to nested objects.
+        
+        When True (default), ignore patterns are applied at all nesting levels.
+        When False, ignore patterns are only applied to top-level keys.
+        
+        Args:
+            value: True to enable recursive filtering, False to disable
+        
+        Example:
+            from aiqa import get_aiqa_client
+            
+            client = get_aiqa_client()
+            # Disable recursive filtering (only filter top-level keys)
+            client.ignore_recursive = False
+        """
+        self._ignore_recursive = bool(value)
+        logger.info(f"Ignore recursive filtering {'enabled' if self._ignore_recursive else 'disabled'}")
+    
     def shutdown(self) -> None:
         """
         Shutdown the tracer provider and exporter.
@@ -245,8 +317,6 @@ def _attach_aiqa_processor(provider: TracerProvider) -> None:
         auth_headers = {}
         if api_key:
             auth_headers["Authorization"] = f"ApiKey {api_key}"
-        elif os.getenv("AIQA_API_KEY"):
-            auth_headers["Authorization"] = f"ApiKey {os.getenv('AIQA_API_KEY')}"
         
         # OTLP HTTP exporter requires the full endpoint URL including /v1/traces
         # Ensure server_url doesn't have trailing slash or /v1/traces, then append /v1/traces

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/constants.py

@@ -3,6 +3,6 @@ Constants used across the AIQA client package.
 """
 
 AIQA_TRACER_NAME = "aiqa-tracer"
-VERSION = "0.6.1" # automatically updated by set-version-json.sh
+VERSION = "0.7.0" # automatically updated by set-version-json.sh
 
 LOG_TAG = "AIQA" # Used in all logging output to identify AIQA messages

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/experiment_runner.py

@@ -123,7 +123,17 @@ class ExperimentRunner:
         
         return dataset
 
-    def get_example_inputs(self, limit: int = 10000) -> List[Dict[str, Any]]:
+    def get_example(self, example_id: str) -> Dict[str, Any]:
+        """
+        Fetch an example by ID.
+        """
+        response = requests.get(
+            f"{self.server_url}/example/{example_id}",
+            headers=self._get_headers(),
+        )
+        return response.json()
+
+    def get_examples_for_dataset(self, limit: int = 10000) -> List[Dict[str, Any]]:
         """
         Fetch example inputs from the dataset.
 
@@ -162,7 +172,6 @@ class ExperimentRunner:
             experiment_setup: Optional setup for the experiment object. You may wish to set:
                 - name (recommended for labelling the experiment)
                 - parameters
-                - comparison_parameters
 
         Returns:
             The created experiment object
@@ -184,7 +193,7 @@ class ExperimentRunner:
             "organisation": self.organisation,
             "dataset": self.dataset_id,
             "results": [],
-            "summary_results": {},
+            "summaries": {},
         }
 
         print(f"Creating experiment")
@@ -226,10 +235,7 @@ class ExperimentRunner:
         if not example_id:
             raise ValueError("Example must have an 'id' field")
         if result is None:
-            example_id = example.get("id")
-            if not example_id:
-                raise ValueError("Example must have an 'id' field")
-            result = Result(exampleId=example_id, scores={}, messages={}, errors={})
+            result = Result(example=example_id, scores={}, messages={}, errors={})
         scores = result.get("scores") or {}
 
         
@@ -243,7 +249,7 @@ class ExperimentRunner:
                 f"{self.server_url}/experiment/{self.experiment_id}/example/{example_id}/scoreAndStore",
                 json={
                     "output": result,
-                    "traceId": example.get("traceId"),
+                    "traceId": example.get("trace"),  # Server returns 'trace' (lowercase), but API expects 'traceId' (camelCase)
                     "scores": scores,
                 },
                 headers=self._get_headers(),
@@ -271,7 +277,7 @@ class ExperimentRunner:
             engine: Function that takes input, returns output (can be async)
             scorer: Optional function that scores the output given the example
         """
-        examples = self.get_example_inputs()
+        examples = self.get_examples_for_dataset()
 
         # Wrap engine to match run_example signature (input, parameters)
         async def wrapped_engine(input_data, parameters):
@@ -304,8 +310,7 @@ class ExperimentRunner:
         scorer_for_metric_id: Optional[Dict[str, ScoreThisInputOutputMetricType]] = None,
     ) -> List[Result]:
         """
-        Run the engine on an example with the given parameters (looping over comparison parameters),
-        and score the result. Also calls scoreAndStore to store the result in the server.
+        Run the engine on an example with the experiment's parameters, score the result, and store it.
 
         Args:
             example: The example to run. See Example.ts type
@@ -313,117 +318,76 @@ class ExperimentRunner:
             scorer_for_metric_id: Optional dictionary of metric IDs to functions that score the output given the example and parameters
 
         Returns:
-            One set of scores for each comparison parameter set. If no comparison parameters,
-            returns an array of one.
+            List of one result (for API compatibility).
         """
-        # Ensure experiment exists
         if not self.experiment:
             self.create_experiment()
         if not self.experiment:
             raise Exception("Failed to create experiment")
 
-        # Make the parameters
-        parameters_fixed = self.experiment.get("parameters") or {}
-        # If comparison_parameters is empty/undefined, default to [{}] so we run at least once
-        parameters_loop = self.experiment.get("comparison_parameters") or [{}]
-
-        # Handle both spans array and input field
+        parameters_here = self.experiment.get("parameters") or {}
         input_data = example.get("input")
         if not input_data and example.get("spans") and len(example["spans"]) > 0:
             input_data = example["spans"][0].get("attributes", {}).get("input")
-
         if not input_data:
-            print(f"Warning: Example has no input field or spans with input attribute: {example}"
-            )
-            # Run engine anyway -- this could make sense if it's all about the parameters
+            print(f"Warning: Example has no input field or spans with input attribute: {example}")
 
-        # Set example.id on the root span (created by @WithTracing decorator)
-        # This ensures the root span from the trace has example=Example.id set
         example_id = example.get("id")
         if not example_id:
             raise ValueError("Example must have an 'id' field")
         set_span_attribute("example", example_id)
-        
-        all_scores: List[Dict[str, Any]] = []
-        dataset_metrics = self.get_dataset().get("metrics", [])
-        specific_metrics = example.get("metrics", [])
-        metrics = [*dataset_metrics, *specific_metrics]
-        # This loop should not be parallelized - it should run sequentially, one after the other
-        # to avoid creating interference between the runs.
-        for parameters in parameters_loop:
-            parameters_here = {**parameters_fixed, **parameters}
-            print(f"Running with parameters: {parameters_here}")
-
-            # Save original env var values for cleanup
-            original_env_vars: Dict[str, Optional[str]] = {}
-            # Set env vars from parameters_here
-            for key, value in parameters_here.items():
-                if value:
-                    original_env_vars[key] = os.environ.get(key)
-                    os.environ[key] = str(value)
 
-            try:
-                start = time.time() * 1000  # milliseconds
-                output = call_my_code(input_data, parameters_here)
-                # Handle async functions
-                if hasattr(output, "__await__"):
-                    output = await output
-                end = time.time() * 1000  # milliseconds
-                duration = int(end - start)
-
-                print(f"Output: {output}")
-                # Score it
-                result = Result(exampleId=example_id, scores={}, messages={}, errors={})
-                for metric in metrics:
-                    metric_id = metric.get("id")
-                    if not metric_id:
-                        print(f"Warning: Metric missing 'id' field, skipping: {metric}")
-                        continue
-                    scorer = scorer_for_metric_id.get(metric_id) if scorer_for_metric_id else None
-                    if scorer:
-                        metric_result = await scorer(input_data, output, metric)
-                    elif metric.get("type") == "llm":
-                        metric_result = await self._score_llm_metric(input_data, output, example, metric)
-                    else:
-                        metric_type = metric.get("type", "unknown")
-                        print(f"Skipping metric: {metric_id} {metric_type} - no scorer")
-                        continue
-                    
-                    # Handle None metric_result (e.g., if scoring failed)
-                    if not metric_result:
-                        print(f"Warning: Metric {metric_id} returned None result, skipping")
-                        result["errors"][metric_id] = "Scoring function returned None"
-                        continue
-                    
-                    result["scores"][metric_id] = metric_result.get("score")
-                    result["messages"][metric_id] = metric_result.get("message")
-                    result["errors"][metric_id] = metric_result.get("error")
-                # Always add duration to scores as a system metric
-                result["scores"]["duration"] = duration
-
-                # Flush spans before scoreAndStore to ensure they're indexed in ES
-                # This prevents race condition where scoreAndStore looks up spans before they're indexed
-                await flush_tracing()
-
-                print(f"Call scoreAndStore ... for example: {example_id} with scores: {result['scores']}")
-                result = await self.score_and_store(example, output, result)
-                print(f"scoreAndStore returned: {result}")
-                all_scores.append(result)
-            finally:
-                # Restore original env var values
-                for key, original_value in original_env_vars.items():
-                    if original_value is None:
-                        # Variable didn't exist before, remove it
-                        os.environ.pop(key, None)
-                    else:
-                        # Restore original value
-                        os.environ[key] = original_value
-
-        return all_scores
-
-    def get_summary_results(self) -> Dict[str, Any]:
+        print(f"Running with parameters: {parameters_here}")
+        original_env_vars: Dict[str, Optional[str]] = {}
+        for key, value in parameters_here.items():
+            if value:
+                original_env_vars[key] = os.environ.get(key)
+                os.environ[key] = str(value)
+        try:
+            start = time.time() * 1000
+            output = call_my_code(input_data, parameters_here)
+            if hasattr(output, "__await__"):
+                output = await output
+            duration = int((time.time() * 1000) - start)
+            print(f"Output: {output}")
+
+            dataset_metrics = self.get_dataset().get("metrics", [])
+            specific_metrics = example.get("metrics", [])
+            metrics = [*dataset_metrics, *specific_metrics]
+            result = Result(example=example_id, scores={}, messages={}, errors={})
+            for metric in metrics:
+                metric_id = metric.get("id")
+                if not metric_id:
+                    continue
+                scorer = scorer_for_metric_id.get(metric_id) if scorer_for_metric_id else None
+                if scorer:
+                    metric_result = await scorer(input_data, output, metric)
+                elif metric.get("type") == "llm":
+                    metric_result = await self._score_llm_metric(input_data, output, example, metric)
+                else:
+                    continue
+                if not metric_result:
+                    result["errors"][metric_id] = "Scoring function returned None"
+                    continue
+                result["scores"][metric_id] = metric_result.get("score")
+                result["messages"][metric_id] = metric_result.get("message")
+                result["errors"][metric_id] = metric_result.get("error")
+            result["scores"]["duration"] = duration
+            await flush_tracing()
+            print(f"Call scoreAndStore ... for example: {example_id} with scores: {result['scores']}")
+            result = await self.score_and_store(example, output, result)
+            print(f"scoreAndStore returned: {result}")
+            return [result]
+        finally:
+            for key, original_value in original_env_vars.items():
+                if original_value is None:
+                    os.environ.pop(key, None)
+                else:
+                    os.environ[key] = original_value
+
+    def get_summaries(self) -> Dict[str, Any]:
         """
-        Get summary results from the experiment.
+        Get summaries from the experiment.
 
         Returns:
             Dictionary of metric names to summary statistics
@@ -435,12 +399,12 @@ class ExperimentRunner:
             f"{self.server_url}/experiment/{self.experiment_id}",
             headers=self._get_headers(),
         )
-
+    
         if not response.ok:
             raise Exception(format_http_error(response, "fetch summary results"))
 
         experiment2 = response.json()
-        return experiment2.get("summary_results", {})
+        return experiment2.get("summaries", {})
 
     async def _score_llm_metric(
         self,
@@ -471,7 +435,8 @@ class ExperimentRunner:
                 model_id, self.server_url, self._get_headers()
             )
             if model_data:
-                api_key = model_data.get("api_key")
+                # Server returns 'apiKey' (camelCase)
+                api_key = model_data.get("apiKey")
                 # If provider not set in metric, try to get it from model
                 if not provider and model_data.get("provider"):
                     provider = model_data.get("provider")

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/llm_as_judge.py

@@ -52,14 +52,15 @@ async def get_model_from_server(
     try:
         def _do_request():
             return requests.get(
-                f"{server_url}/model/{model_id}?fields=api_key",
+                f"{server_url}/model/{model_id}?fields=apiKey",  # Server uses camelCase 'apiKey' (also accepts 'api_key')
                 headers=headers,
             )
         
         response = await asyncio.to_thread(_do_request)
         if response.ok:
             model = response.json()
-            if model.get("api_key"):
+            # Server returns 'apiKey' (camelCase)
+            if model.get("apiKey"):
                 return model
         return None
     except Exception as e:

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/object_serialiser.py

@@ -25,7 +25,7 @@ def sanitize_string_for_utf8(text: str) -> str:
     Returns:
         A string with surrogate characters replaced by the Unicode replacement character (U+FFFD)
     """
-    if text == None:
+    if text is None:
         return None
     if not isinstance(text, str): # paranoia
         text = str(text)
@@ -43,7 +43,10 @@ def toNumber(value: str|int|None) -> int:
     if value is None:
         return 0
     if isinstance(value, int):
-        return value    
+        return value
+    # Convert to string if not already
+    if not isinstance(value, str):
+        value = str(value)
     if value.endswith("b"): # drop the b
         value = value[:-1]
     if value.endswith("g"):

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/tracing.py

@@ -47,16 +47,16 @@ class TracingOptions:
             
             ignore_input: Iterable of keys (e.g., list, set) to exclude from
                 input data when recording span attributes. Applies after filter_input if both are set. 
-                Only applies when
-                input is a dictionary. Supports simple wildcards (e.g., `"_*"` 
-                matches `"_apple"`, `"_fruit"`). For example, use `["password", "api_key"]`
-                or `["_*", "password"]` to exclude sensitive fields from being traced.
+                Supports "self" and simple wildcards (e.g., `"_*"` 
+                matches `"_apple"`, `"_fruit"`). The pattern `"_*"` is applied by default
+                to filter properties starting with '_' in nested objects.
             
             ignore_output: Iterable of keys (e.g., list, set) to exclude from
                 output data when recording span attributes. Only applies when
                 output is a dictionary. Supports simple wildcards (e.g., `"_*"` 
-                matches `"_apple"`, `"_fruit"`). Useful for excluding large or sensitive
-                fields from traces.
+                matches `"_apple"`, `"_fruit"`). The pattern `"_*"` is applied by default
+                to filter properties starting with '_' in nested objects. Useful for excluding 
+                large or sensitive fields from traces.
             
             filter_input: Callable function that receives the same arguments as the
                 decorated function (*args, **kwargs) and returns a filtered/transformed
@@ -96,7 +96,7 @@ class TracingOptions:
                     filter_input=lambda self, example: {
                         "dataset": self.dataset_id,
                         "experiment": self.experiment_id,
-                        "example_id": example.id if hasattr(example, 'id') else None
+                        "example": example.id if hasattr(example, 'id') else None
                     }
                 )
                 def run_example(self, example):
@@ -168,33 +168,89 @@ def _prepare_input(args: tuple, kwargs: dict, sig: Optional[inspect.Signature] =
     return result
 
 
-def _apply_ignore_patterns(data_dict: dict, ignore_patterns: Optional[List[str]]) -> dict:
+def _apply_ignore_patterns(
+    data_dict: dict, 
+    ignore_patterns: Optional[List[str]], 
+    recursive: bool = True,
+    max_depth: int = 100,
+    current_depth: int = 0
+) -> dict:
     """
-    Apply ignore patterns to a dict.
+    Apply ignore patterns to a dict, optionally recursively.
     Supports string keys, wildcard patterns (*), and list of patterns.
     Used for both ignore_input and ignore_output.
     
     Args:
-        data_dict: Dictionary to filter
+        data_dict: Dictionary to filter (may contain nested dictionaries)
         ignore_patterns: List of patterns to exclude (e.g., ["self", "_*", "password"])
+        recursive: Whether to apply patterns recursively to nested dictionaries
+        max_depth: Maximum recursion depth to prevent infinite loops (default: 100)
+        current_depth: Current recursion depth (internal use)
     
     Returns:
         Filtered dictionary with matching keys removed
     """
-    if not ignore_patterns or not isinstance(data_dict, dict):
+    if not isinstance(data_dict, dict):
         return data_dict
     
-    result = data_dict.copy()
-    keys_to_delete = [
-        key for key in result.keys()
-        if _matches_ignore_pattern(key, ignore_patterns)
-    ]
-    for key in keys_to_delete:
-        del result[key]
+    # Safety check: prevent infinite loops from extremely deep nesting
+    if current_depth >= max_depth:
+        logger.warning(
+            f"_apply_ignore_patterns: max depth {max_depth} reached, "
+            f"stopping recursion to prevent infinite loop"
+        )
+        return data_dict
+    
+    # If no patterns, return copy (no filtering needed, even if recursive=True)
+    if not ignore_patterns:
+        return data_dict.copy()
+    
+    result = {}
+    for key, value in data_dict.items():
+        # Skip keys that match ignore patterns
+        if _matches_ignore_pattern(key, ignore_patterns):
+            continue
+        
+        # Recursively process nested dictionaries if recursive=True
+        if recursive and isinstance(value, dict):
+            result[key] = _apply_ignore_patterns(
+                value, ignore_patterns, recursive, max_depth, current_depth + 1
+            )
+        else:
+            result[key] = value
     
     return result
 
 
+def _merge_with_default_ignore_patterns(
+    ignore_patterns: Optional[List[str]], 
+    client: Optional[Any] = None
+) -> List[str]:
+    """
+    Merge user-provided ignore patterns with client's default ignore patterns.
+    
+    Args:
+        ignore_patterns: Optional list of user-provided patterns
+        client: Optional client instance (to avoid repeated get_aiqa_client() calls)
+    
+    Returns:
+        List of patterns including client's default ignore patterns
+    """
+    if client is None:
+        client = get_aiqa_client()
+    default_patterns = client.default_ignore_patterns
+    
+    if ignore_patterns is None:
+        return default_patterns.copy() if default_patterns else []
+    
+    # Merge patterns, avoiding duplicates
+    merged = list(default_patterns)
+    for pattern in ignore_patterns:
+        if pattern not in merged:
+            merged.append(pattern)
+    return merged
+
+
 def _prepare_and_filter_input(
     args: tuple,
     kwargs: dict,
@@ -209,6 +265,7 @@ def _prepare_and_filter_input(
     1. Apply filter_input to args, kwargs (receives same inputs as decorated function, including self)
     2. Convert into dict ready for span.attributes.input
     3. Apply ignore_input to the dict (supports string, wildcard, and list patterns)
+       Client's default ignore patterns are automatically merged with ignore_input.
     
     Args:
         args: Positional arguments (including self for bound methods)
@@ -218,7 +275,7 @@ def _prepare_and_filter_input(
             including `self` for bound methods. This allows extracting properties from any object.
         ignore_input: Optional list of keys/patterns to exclude from the final dict.
             If "self" is in ignore_input, it will be removed from the final dict but filter_input
-            still receives it.
+            still receives it. Client's default ignore patterns are automatically merged.
         sig: Optional function signature for proper arg name resolution
     
     Returns:
@@ -251,15 +308,23 @@ def _prepare_and_filter_input(
         input_data = _prepare_input(args, kwargs, sig)
     
     # Step 3: Apply ignore_input to the dict (removes "self" from final dict if specified)
-    should_ignore_self = ignore_input and "self" in ignore_input
+    # Merge with client's default ignore patterns
+    client = get_aiqa_client()
+    merged_ignore_input = _merge_with_default_ignore_patterns(ignore_input, client)
+    should_ignore_self = "self" in merged_ignore_input
+    
     if isinstance(input_data, dict):
-        input_data = _apply_ignore_patterns(input_data, ignore_input)
+        input_data = _apply_ignore_patterns(
+            input_data, 
+            merged_ignore_input, 
+            recursive=client.ignore_recursive
+        )
         # Handle case where we removed self and there are no remaining args/kwargs
         if should_ignore_self and not input_data:
             return None
-    elif ignore_input:
-        # Warn if ignore_input is set but input_data is not a dict
-        logger.warning(f"_prepare_and_filter_input: skip: ignore_input is set but input_data is not a dict: {type(input_data)}")
+    elif merged_ignore_input:
+        # Warn if ignore patterns are set but input_data is not a dict
+        logger.warning(f"_prepare_and_filter_input: skip: ignore patterns are set but input_data is not a dict: {type(input_data)}")
     
     return input_data
 
@@ -269,7 +334,10 @@ def _filter_and_serialize_output(
     filter_output: Optional[Callable[[Any], Any]],
     ignore_output: Optional[List[str]],
 ) -> Any:
-    """Filter and serialize output for span attributes."""
+    """
+    Filter and serialize output for span attributes.
+    Client's default ignore patterns are automatically merged with ignore_output.
+    """
     output_data = result
     if filter_output:
         if isinstance(output_data, dict):
@@ -277,11 +345,19 @@ def _filter_and_serialize_output(
         output_data = filter_output(output_data)
     
     # Apply ignore_output patterns (supports key, wildcard, and list patterns)
+    # Merge with client's default ignore patterns
+    client = get_aiqa_client()
+    merged_ignore_output = _merge_with_default_ignore_patterns(ignore_output, client)
+    
     if isinstance(output_data, dict):
-        output_data = _apply_ignore_patterns(output_data, ignore_output)
-    elif ignore_output:
-        # Warn if ignore_output is set but output_data is not a dict
-        logger.warning(f"_filter_and_serialize_output: skip: ignore_output is set but output_data is not a dict: {type(output_data)}")
+        output_data = _apply_ignore_patterns(
+            output_data, 
+            merged_ignore_output,
+            recursive=client.ignore_recursive
+        )
+    elif merged_ignore_output:
+        # Warn if ignore patterns are set but output_data is not a dict
+        logger.warning(f"_filter_and_serialize_output: skip: ignore patterns are set but output_data is not a dict: {type(output_data)}")
     
     # Serialize immediately to create immutable result (removes mutable structures)
     return serialize_for_span(output_data)
@@ -500,12 +576,14 @@ def WithTracing(
         ignore_input: List of keys to exclude from input data when recording span attributes.
             self is handled as "self"
             Supports simple wildcards (e.g., "_*" 
-            matches "_apple", "_fruit"). For example, use ["password", "api_key"] or 
-            ["_*", "password"] to exclude sensitive fields from being traced.
+            matches "_apple", "_fruit"). The pattern "_*" is applied by default
+            to filter properties starting with '_' in nested objects. For example, use 
+            ["password", "api_key"] to exclude additional sensitive fields from being traced.
         ignore_output: List of keys to exclude from output data when recording span attributes.
             Only applies when output is a dictionary. Supports simple wildcards (e.g., "_*" 
-            matches "_apple", "_fruit"). Useful for excluding large or sensitive
-            fields from traces.
+            matches "_apple", "_fruit"). The pattern "_*" is applied by default
+            to filter properties starting with '_' in nested objects. Useful for excluding 
+            large or sensitive fields from traces.
         filter_input: Function to filter/transform input before recording.
             Receives the same arguments as the decorated function (*args, **kwargs),
             including `self` for bound methods. This allows you to extract specific
@@ -678,7 +756,8 @@ def WithTracing(
             # This is called lazily when the function runs, not at decorator definition time
             client = get_aiqa_client()
             if not client.enabled:
-                return await executor()
+                # executor() returns an async generator object, not a coroutine, so don't await it
+                return executor()
             
             # Get tracer after initialization (lazy)
             tracer = get_aiqa_tracer()

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/aiqa/types.py

@@ -29,7 +29,7 @@ class MetricResult(TypedDict):
 
 class Result(TypedDict):
   """Result of evaluating a set of metrics on an output (i.e. the full set of metrics for a single example)."""
-  exampleId: str
+  example: str
   scores: Dict[str, Number]
   messages: Optional[Dict[str, str]] = None
   errors: Optional[Dict[str, str]] = None

{aiqa_client-0.6.1 → aiqa_client-0.7.0/aiqa_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiqa-client
-Version: 0.6.1
+Version: 0.7.0
 Summary: OpenTelemetry-based Python client for tracing functions and sending traces to the AIQA server
 Author-email: AIQA <info@aiqa.dev>
 License: MIT

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aiqa-client"
-version = "0.6.1"
+version = "0.7.0"
 description = "OpenTelemetry-based Python client for tracing functions and sending traces to the AIQA server"
 readme = "README.md"
 requires-python = ">=3.8"

{aiqa_client-0.6.1 → aiqa_client-0.7.0}/tests/test_tracing.py

@@ -8,7 +8,7 @@ import pytest
 from unittest.mock import patch, MagicMock
 from aiqa.span_helpers import get_span
 from aiqa.tracing import _prepare_input, _prepare_and_filter_input, _filter_and_serialize_output
-from aiqa.tracing import _matches_ignore_pattern
+from aiqa.tracing import _matches_ignore_pattern, _apply_ignore_patterns, _merge_with_default_ignore_patterns
 
 
 class TestGetSpan:
@@ -796,3 +796,367 @@ class TestFilterInputWithSelf:
         assert "_trace_me_not" not in result  # filtered by wildcard
         assert "arg2" in result
         assert "trace_me_yes" in result
+
+
+class TestDefaultUnderscoreIgnore:
+    """Tests for default '_*' pattern that filters properties starting with '_'."""
+
+    def test_default_underscore_ignore_input(self):
+        """Test that '_*' pattern is applied by default to ignore_input."""
+        def test_func(user: dict, x: int):
+            pass
+        
+        sig = inspect.signature(test_func)
+        user_data = {"name": "Alice", "_sa_instance_state": "object"}
+        result = _prepare_and_filter_input((user_data, 5), {}, None, None, sig)
+        
+        assert isinstance(result, dict)
+        assert "user" in result
+        user_result = result["user"]
+        assert isinstance(user_result, dict)
+        assert "name" in user_result
+        assert user_result["name"] == "Alice"
+        assert "_sa_instance_state" not in user_result  # filtered by default '_*' pattern
+        assert "x" in result
+
+    def test_default_underscore_ignore_output(self):
+        """Test that '_*' pattern is applied by default to ignore_output."""
+        output_data = {
+            "user": {
+                "name": "Alice",
+                "_sa_instance_state": "object"
+            },
+            "result": "success"
+        }
+        
+        result = _filter_and_serialize_output(output_data, None, None)
+        
+        # Result is serialized to JSON string for dicts
+        assert isinstance(result, str)
+        
+        # Parse it back to verify filtering worked
+        import json
+        parsed = json.loads(result)
+        assert "user" in parsed
+        assert "name" in parsed["user"]
+        assert parsed["user"]["name"] == "Alice"
+        assert "_sa_instance_state" not in parsed["user"]  # filtered by default '_*' pattern
+        assert "result" in parsed
+
+    def test_nested_underscore_filtering(self):
+        """Test that nested objects are filtered recursively."""
+        data = {
+            "level1": {
+                "public": "value1",
+                "_private": "value2",
+                "nested": {
+                    "public": "value3",
+                    "_private": "value4"
+                }
+            },
+            "_top_level": "value5"
+        }
+        
+        result = _apply_ignore_patterns(data, None)
+        
+        assert "level1" in result
+        assert "public" in result["level1"]
+        assert "_private" not in result["level1"]
+        assert "nested" in result["level1"]
+        assert "public" in result["level1"]["nested"]
+        assert "_private" not in result["level1"]["nested"]
+        assert "_top_level" not in result
+
+    def test_merge_with_default_patterns(self):
+        """Test that user-provided patterns are merged with default '_*' pattern."""
+        user_patterns = ["password", "api_key"]
+        merged = _merge_with_default_ignore_patterns(user_patterns)
+        
+        assert "_*" in merged
+        assert "password" in merged
+        assert "api_key" in merged
+        assert len(merged) == 3
+
+    def test_merge_with_default_patterns_none(self):
+        """Test that None patterns result in just default '_*' pattern."""
+        merged = _merge_with_default_ignore_patterns(None)
+        
+        assert merged == ["_*"]
+
+    def test_merge_with_default_patterns_duplicate(self):
+        """Test that duplicate patterns are not added."""
+        user_patterns = ["_*", "password"]
+        merged = _merge_with_default_ignore_patterns(user_patterns)
+        
+        assert merged == ["_*", "password"]  # _* is first, password is second
+        assert merged.count("_*") == 1
+
+    def test_default_underscore_with_custom_patterns(self):
+        """Test that default '_*' works alongside custom patterns."""
+        def test_func(user: dict, password: str, x: int):
+            pass
+        
+        sig = inspect.signature(test_func)
+        user_data = {"name": "Alice", "_sa_instance_state": "object"}
+        result = _prepare_and_filter_input(
+            (user_data, "secret", 5), 
+            {}, 
+            None, 
+            ["password"],  # Only specify password, '_*' should be added automatically
+            sig
+        )
+        
+        assert isinstance(result, dict)
+        assert "user" in result
+        user_result = result["user"]
+        assert "name" in user_result
+        assert "_sa_instance_state" not in user_result  # filtered by default '_*'
+        assert "password" not in result  # filtered by custom pattern
+        assert "x" in result
+
+    def test_nested_underscore_in_output(self):
+        """Test nested underscore filtering in output."""
+        output_data = {
+            "result": "success",
+            "user": {
+                "name": "Alice",
+                "_sa_instance_state": "object",
+                "profile": {
+                    "email": "alice@example.com",
+                    "_internal_id": "12345"
+                }
+            }
+        }
+        
+        result = _filter_and_serialize_output(output_data, None, None)
+        
+        import json
+        parsed = json.loads(result)
+        assert "user" in parsed
+        assert "name" in parsed["user"]
+        assert "_sa_instance_state" not in parsed["user"]
+        assert "profile" in parsed["user"]
+        assert "email" in parsed["user"]["profile"]
+        assert "_internal_id" not in parsed["user"]["profile"]
+
+    def test_apply_ignore_patterns_with_none(self):
+        """Test that _apply_ignore_patterns handles None patterns by recursively processing nested dicts."""
+        data = {
+            "public": "value",
+            "_private": "hidden",
+            "nested": {
+                "public": "value2",
+                "_private": "hidden2"
+            }
+        }
+        
+        result = _apply_ignore_patterns(data, None)
+        
+        # When patterns is None, it should still recursively process nested dicts
+        # but won't filter top-level keys (since no patterns provided)
+        assert "public" in result
+        assert "_private" in result  # Not filtered when patterns is None
+        assert "nested" in result
+        # Nested dicts are still processed recursively (structure preserved)
+        assert isinstance(result["nested"], dict)
+        assert "public" in result["nested"]
+        assert "_private" in result["nested"]  # Also not filtered when patterns is None
+
+
+class TestClientIgnoreProperties:
+    """Tests for client default_ignore_patterns and ignore_recursive properties."""
+
+    def test_default_ignore_patterns_property(self):
+        """Test setting and getting default_ignore_patterns on client."""
+        from aiqa import get_aiqa_client
+        
+        client = get_aiqa_client()
+        original = client.default_ignore_patterns
+        
+        # Test setting new patterns
+        client.default_ignore_patterns = ["_*", "password"]
+        assert client.default_ignore_patterns == ["_*", "password"]
+        
+        # Test that it's a copy (modifying returned list doesn't affect client)
+        patterns = client.default_ignore_patterns
+        patterns.append("new")
+        assert client.default_ignore_patterns == ["_*", "password"]
+        
+        # Test setting to None (disables defaults)
+        client.default_ignore_patterns = None
+        assert client.default_ignore_patterns == []
+        
+        # Test setting to empty list
+        client.default_ignore_patterns = []
+        assert client.default_ignore_patterns == []
+        
+        # Restore original
+        client.default_ignore_patterns = original
+
+    def test_ignore_recursive_property(self):
+        """Test setting and getting ignore_recursive on client."""
+        from aiqa import get_aiqa_client
+        
+        client = get_aiqa_client()
+        original = client.ignore_recursive
+        
+        # Test setting to False
+        client.ignore_recursive = False
+        assert client.ignore_recursive is False
+        
+        # Test setting to True
+        client.ignore_recursive = True
+        assert client.ignore_recursive is True
+        
+        # Restore original
+        client.ignore_recursive = original
+
+    def test_custom_default_ignore_patterns_used(self):
+        """Test that custom default ignore patterns are used in tracing."""
+        from aiqa import get_aiqa_client
+        from aiqa.tracing import _prepare_and_filter_input
+        import inspect
+        
+        client = get_aiqa_client()
+        original_patterns = client.default_ignore_patterns
+        
+        try:
+            # Set custom default patterns
+            client.default_ignore_patterns = ["password", "secret"]
+            
+            def test_func(password: str, secret: str, public: str):
+                pass
+            
+            sig = inspect.signature(test_func)
+            result = _prepare_and_filter_input(
+                ("pwd", "sec", "pub"), 
+                {}, 
+                None, 
+                None,  # No explicit ignore_input
+                sig
+            )
+            
+            # Custom patterns should be applied
+            assert "password" not in result
+            assert "secret" not in result
+            assert "public" in result
+        finally:
+            # Restore original
+            client.default_ignore_patterns = original_patterns
+
+    def test_ignore_recursive_false(self):
+        """Test that ignore_recursive=False only filters top-level keys."""
+        from aiqa import get_aiqa_client
+        
+        client = get_aiqa_client()
+        original_recursive = client.ignore_recursive
+        
+        try:
+            client.ignore_recursive = False
+            
+            data = {
+                "top_level": "value",
+                "_top_private": "hidden",
+                "nested": {
+                    "public": "value2",
+                    "_nested_private": "hidden2"
+                }
+            }
+            
+            result = _apply_ignore_patterns(data, ["_*"], recursive=False)
+            
+            # Top-level _* should be filtered
+            assert "_top_private" not in result
+            assert "top_level" in result
+            # Nested dict should be preserved as-is (not filtered recursively)
+            assert "nested" in result
+            assert "_nested_private" in result["nested"]  # Not filtered when recursive=False
+            assert "public" in result["nested"]
+        finally:
+            client.ignore_recursive = original_recursive
+
+    def test_ignore_recursive_true(self):
+        """Test that ignore_recursive=True filters nested keys."""
+        from aiqa import get_aiqa_client
+        
+        client = get_aiqa_client()
+        original_recursive = client.ignore_recursive
+        
+        try:
+            client.ignore_recursive = True
+            
+            data = {
+                "top_level": "value",
+                "_top_private": "hidden",
+                "nested": {
+                    "public": "value2",
+                    "_nested_private": "hidden2"
+                }
+            }
+            
+            result = _apply_ignore_patterns(data, ["_*"], recursive=True)
+            
+            # Top-level _* should be filtered
+            assert "_top_private" not in result
+            assert "top_level" in result
+            # Nested dict should also be filtered
+            assert "nested" in result
+            assert "_nested_private" not in result["nested"]  # Filtered when recursive=True
+            assert "public" in result["nested"]
+        finally:
+            client.ignore_recursive = original_recursive
+
+    def test_max_depth_prevents_infinite_loop(self):
+        """Test that max_depth prevents infinite loops from deep nesting."""
+        # Create a deeply nested structure
+        data = {"level": 0}
+        current = data
+        for i in range(150):  # Exceeds default max_depth of 100
+            current["nested"] = {"level": i + 1}
+            current = current["nested"]
+        
+        # Should not raise exception or hang
+        result = _apply_ignore_patterns(data, ["_*"], recursive=True, max_depth=100)
+        
+        # Should return something (may be truncated at max_depth)
+        assert isinstance(result, dict)
+        assert "level" in result
+
+    def test_client_ignore_recursive_affects_tracing(self):
+        """Test that client.ignore_recursive setting affects actual tracing."""
+        from aiqa import get_aiqa_client
+        from aiqa.tracing import _filter_and_serialize_output
+        
+        client = get_aiqa_client()
+        original_recursive = client.ignore_recursive
+        original_patterns = client.default_ignore_patterns
+        
+        try:
+            client.default_ignore_patterns = ["_*"]
+            
+            output_data = {
+                "top": "value",
+                "_top_private": "hidden",
+                "nested": {
+                    "public": "value2",
+                    "_nested_private": "hidden2"
+                }
+            }
+            
+            # Test with recursive=True (default)
+            client.ignore_recursive = True
+            result_recursive = _filter_and_serialize_output(output_data, None, None)
+            import json
+            parsed_recursive = json.loads(result_recursive)
+            assert "_top_private" not in parsed_recursive
+            assert "_nested_private" not in parsed_recursive["nested"]  # Filtered recursively
+            
+            # Test with recursive=False
+            client.ignore_recursive = False
+            result_non_recursive = _filter_and_serialize_output(output_data, None, None)
+            parsed_non_recursive = json.loads(result_non_recursive)
+            assert "_top_private" not in parsed_non_recursive
+            assert "_nested_private" in parsed_non_recursive["nested"]  # Not filtered when recursive=False
+        finally:
+            client.ignore_recursive = original_recursive
+            client.default_ignore_patterns = original_patterns