aiqa-client 0.2.1__py3-none-any.whl → 0.3.4__py3-none-any.whl

aiqa/object_serialiser.py

@@ -0,0 +1,396 @@
+"""
+Object serialization utilities for converting Python objects to JSON-safe formats.
+Handles objects, dataclasses, circular references, and size limits.
+"""
+
+import json
+import os
+import dataclasses
+import logging
+from datetime import datetime, date, time
+from typing import Any, Callable, Set
+
+logger = logging.getLogger("aiqa")
+
+def toNumber(value: str|int|None) -> int:
+    """Convert string to number. handling units like g, m, k, (also mb kb gb though these should be avoided)"""
+    if value is None:
+        return 0
+    if isinstance(value, int):
+        return value    
+    if value.endswith("b"): # drop the b
+        value = value[:-1]
+    if value.endswith("g"):
+        return int(value[:-1]) * 1024 * 1024 * 1024
+    elif value.endswith("m"):
+        return int(value[:-1]) * 1024 * 1024
+    elif value.endswith("k"):
+        return int(value[:-1]) * 1024
+    return int(value)
+
+
+# Configurable limit for object string representation (in characters)
+AIQA_MAX_OBJECT_STR_CHARS = toNumber(os.getenv("AIQA_MAX_OBJECT_STR_CHARS", "1m"))
+
+# Data filters configuration
+def _get_enabled_filters() -> Set[str]:
+    """Get set of enabled filter names from AIQA_DATA_FILTERS env var."""
+    filters_env = os.getenv("AIQA_DATA_FILTERS", "RemovePasswords, RemoveJWT, RemoveAuthHeaders, RemoveAPIKeys")
+    if not filters_env or filters_env.lower() == "false":
+        return set()
+    return {f.strip() for f in filters_env.split(",") if f.strip()}
+
+_ENABLED_FILTERS = _get_enabled_filters()
+
+def _is_jwt_token(value: Any) -> bool:
+    """Check if a value looks like a JWT token (starts with 'eyJ' and has 3 parts separated by dots)."""
+    if not isinstance(value, str):
+        return False
+    # JWT tokens have format: header.payload.signature (3 parts separated by dots)
+    # They typically start with 'eyJ' (base64 encoded '{"')
+    parts = value.split('.')
+    return len(parts) == 3 and value.startswith('eyJ') and all(len(p) > 0 for p in parts)
+
+def _is_api_key(value: Any) -> bool:
+    """Check if a value looks like an API key based on common patterns."""
+    if not isinstance(value, str):
+        return False
+    value = value.strip()
+    # Common API key prefixes:
+    api_key_prefixes = [
+        'sk-',    # OpenAI secret key
+        'pk-',    # possibly public key
+        'AKIA',   # AWS access key
+        'ghp_',   # GitHub personal access token
+        'gho_',   # GitHub OAuth token
+        'ghu_',   # GitHub unidentified token
+        'ghs_',   # GitHub SAML token
+        'ghr_'    # GitHub refresh token
+    ]
+    return any(value.startswith(prefix) for prefix in api_key_prefixes)
+
+def _apply_data_filters(key: str, value: Any) -> Any:
+    """Apply data filters to a key-value pair based on enabled filters."""
+    if not value:  # Don't filter falsy values
+        return value
+    
+    key_lower = str(key).lower()
+    
+    # RemovePasswords filter: if key contains "password", replace value with "****"
+    if "RemovePasswords" in _ENABLED_FILTERS and "password" in key_lower:
+        return "****"
+    
+    # RemoveJWT filter: if value looks like a JWT token, replace with "****"
+    if "RemoveJWT" in _ENABLED_FILTERS and _is_jwt_token(value):
+        return "****"
+    
+    # RemoveAuthHeaders filter: if key is "authorization" (case-insensitive), replace value with "****"
+    if "RemoveAuthHeaders" in _ENABLED_FILTERS and key_lower == "authorization":
+        return "****"
+    
+    # RemoveAPIKeys filter: if key contains API key patterns or value looks like an API key
+    if "RemoveAPIKeys" in _ENABLED_FILTERS:
+        # Check key patterns
+        api_key_key_patterns = ['api_key', 'apikey', 'api-key', 'apikey']
+        if any(pattern in key_lower for pattern in api_key_key_patterns):
+            return "****"
+        # Check value patterns
+        if _is_api_key(value):
+            return "****"
+    
+    return value
+
+
+def serialize_for_span(value: Any) -> Any:
+    """
+    Serialize a value for span attributes.
+    OpenTelemetry only accepts primitives (bool, str, bytes, int, float) or sequences of those.
+    Complex types (dicts, lists, objects) are converted to JSON strings.
+    
+    Handles objects by attempting to convert them to dicts, with safeguards against:
+    - Circular references
+    - Unconvertible parts
+    - Large objects (size limits)
+    """
+    # Keep primitives as is (including None)
+    if value is None or isinstance(value, (str, int, float, bool, bytes)):
+        return value
+    
+    # For sequences, check if all elements are primitives
+    if isinstance(value, (list, tuple)):
+        # If all elements are primitives, return as list
+        if all(isinstance(item, (str, int, float, bool, bytes, type(None))) for item in value):
+            return list(value)
+        # Otherwise serialize to JSON string
+        try:
+            return safe_json_dumps(value)
+        except Exception:
+            return str(value)
+    
+    # For dicts and other complex types, serialize to JSON string
+    try:
+        return safe_json_dumps(value)
+    except Exception:
+        # If JSON serialization fails, convert to string
+        return safe_str_repr(value)
+
+
+def safe_str_repr(value: Any) -> str:
+    """
+    Safely convert a value to string representation.
+    Handles objects with __repr__ that might raise exceptions.
+    Uses AIQA_MAX_OBJECT_STR_CHARS environment variable (default: 100000) to limit length.
+    """
+    try:
+        # Try __repr__ first (usually more informative)
+        repr_str = repr(value)
+        # Limit length to avoid huge strings
+        if len(repr_str) > AIQA_MAX_OBJECT_STR_CHARS:
+            return repr_str[:AIQA_MAX_OBJECT_STR_CHARS] + "... (truncated)"
+        return repr_str
+    except Exception:
+        # Fallback to type name
+        try:
+            return f"<{type(value).__name__} object>"
+        except Exception:
+            return "<unknown object>"
+
+
+def object_to_dict(obj: Any, visited: Set[int], max_depth: int = 10, current_depth: int = 0) -> Any:
+    """
+    Convert an object to a dictionary representation.
+    
+    Args:
+        obj: The object to convert
+        visited: Set of object IDs to detect circular references
+        max_depth: Maximum recursion depth
+        current_depth: Current recursion depth
+    
+    Returns:
+        Dictionary representation of the object, or a string if conversion fails
+    """
+    if current_depth > max_depth:
+        return "<max depth exceeded>"
+    
+    obj_id = id(obj)
+    if obj_id in visited:
+        return "<circular reference>"
+    
+    # Handle None
+    if obj is None:
+        return None
+    
+    # Handle primitives
+    if isinstance(obj, (str, int, float, bool, bytes)):
+        return obj
+    
+    # Handle datetime objects
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    if isinstance(obj, date):
+        return obj.isoformat()
+    if isinstance(obj, time):
+        return obj.isoformat()
+    
+    # Handle dict
+    if isinstance(obj, dict):
+        visited.add(obj_id)
+        try:
+            result = {}
+            for k, v in obj.items():
+                try:
+                    key_str = str(k) if not isinstance(k, (str, int, float, bool)) else k
+                    filtered_value = _apply_data_filters(key_str, v)
+                    result[key_str] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
+                except Exception as e:
+                    # If one key-value pair fails, log and use string representation for the value
+                    key_str = str(k) if not isinstance(k, (str, int, float, bool)) else k
+                    logger.debug(f"Failed to convert dict value for key '{key_str}': {e}")
+                    result[key_str] = safe_str_repr(v)
+            visited.remove(obj_id)
+            return result
+        except Exception as e:
+            visited.discard(obj_id)
+            logger.debug(f"Failed to convert dict to dict: {e}")
+            return safe_str_repr(obj)
+    
+    # Handle list/tuple
+    if isinstance(obj, (list, tuple)):
+        visited.add(obj_id)
+        try:
+            result = []
+            for item in obj:
+                try:
+                    result.append(object_to_dict(item, visited, max_depth, current_depth + 1))
+                except Exception as e:
+                    # If one item fails, log and use its string representation
+                    logger.debug(f"Failed to convert list item {type(item).__name__} to dict: {e}")
+                    result.append(safe_str_repr(item))
+            visited.remove(obj_id)
+            return result
+        except Exception as e:
+            visited.discard(obj_id)
+            logger.debug(f"Failed to convert list/tuple to dict: {e}")
+            return safe_str_repr(obj)
+    
+    # Handle dataclasses
+    if dataclasses.is_dataclass(obj):
+        visited.add(obj_id)
+        try:
+            result = {}
+            for field in dataclasses.fields(obj):
+                try:
+                    value = getattr(obj, field.name, None)
+                    filtered_value = _apply_data_filters(field.name, value)
+                    result[field.name] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
+                except Exception as e:
+                    # If accessing a field fails, log and skip it
+                    logger.debug(f"Failed to access field {field.name} on {type(obj).__name__}: {e}")
+                    result[field.name] = "<error accessing field>"
+            visited.remove(obj_id)
+            return result
+        except Exception as e:
+            visited.discard(obj_id)
+            logger.debug(f"Failed to convert dataclass {type(obj).__name__} to dict: {e}")
+            return safe_str_repr(obj)
+    
+    # Handle objects with __dict__
+    if hasattr(obj, "__dict__"):
+        visited.add(obj_id)
+        try:
+            result = {}
+            for key, value in obj.__dict__.items():
+                # Skip private attributes that start with __
+                if not (isinstance(key, str) and key.startswith("__")):
+                    filtered_value = _apply_data_filters(key, value)
+                    result[key] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
+            visited.remove(obj_id)
+            return result
+        except Exception as e:
+            visited.discard(obj_id)
+            # Log the error for debugging, but still return string representation
+            logger.debug(f"Failed to convert object {type(obj).__name__} to dict: {e}")
+            return safe_str_repr(obj)
+    
+    # Handle objects with __slots__
+    if hasattr(obj, "__slots__"):
+        visited.add(obj_id)
+        try:
+            result = {}
+            for slot in obj.__slots__:
+                try:
+                    if hasattr(obj, slot):
+                        value = getattr(obj, slot, None)
+                        filtered_value = _apply_data_filters(slot, value)
+                        result[slot] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
+                except Exception as e:
+                    # If accessing a slot fails, log and skip it
+                    logger.debug(f"Failed to access slot {slot} on {type(obj).__name__}: {e}")
+                    result[slot] = "<error accessing slot>"
+            visited.remove(obj_id)
+            return result
+        except Exception as e:
+            visited.discard(obj_id)
+            logger.debug(f"Failed to convert slotted object {type(obj).__name__} to dict: {e}")
+            return safe_str_repr(obj)
+    
+    # Fallback: try to get a few common attributes
+    try:
+        result = {}
+        for attr in ["name", "id", "value", "type", "status"]:
+            if hasattr(obj, attr):
+                value = getattr(obj, attr, None)
+                filtered_value = _apply_data_filters(attr, value)
+                result[attr] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
+        if result:
+            return result
+    except Exception:
+        pass
+    
+    # Final fallback: string representation
+    return safe_str_repr(obj)
+
+
+def safe_json_dumps(value: Any) -> str:
+    """
+    Safely serialize a value to JSON string with safeguards against:
+    - Circular references
+    - Large objects (size limits)
+    - Unconvertible parts
+    
+    Args:
+        value: The value to serialize
+
+    Uses AIQA_MAX_OBJECT_STR_CHARS environment variable (default: 1000000) to limit length.
+    
+    Returns:
+        JSON string representation
+    """    
+    max_size_chars = AIQA_MAX_OBJECT_STR_CHARS
+    visited: Set[int] = set()
+    
+    # Convert the entire structure to ensure circular references are detected
+    # across the whole object graph
+    try:
+        converted = object_to_dict(value, visited)
+    except Exception as e:
+        # If conversion fails, try with a fresh visited set and json default handler
+        logger.debug(f"object_to_dict failed for {type(value).__name__}, trying json.dumps with default handler: {e}")
+        try:
+            json_str = json.dumps(value, default=json_default_handler_factory(set()))
+            if len(json_str) > max_size_chars:
+                return f"<object {type(value)} too large: {len(json_str)} chars (limit: {max_size_chars} chars) begins: {json_str[:100]}... conversion error: {e}>"
+            return json_str
+        except Exception as e2:
+            logger.debug(f"json.dumps with default handler also failed for {type(value).__name__}: {e2}")
+            return safe_str_repr(value)
+    
+    # Try JSON serialization of the converted structure
+    try:
+        json_str = json.dumps(converted, default=json_default_handler_factory(set()))
+        # Check size
+        if len(json_str) > max_size_chars:
+            return f"<object {type(value)} too large: {len(json_str)} chars (limit: {max_size_chars} chars) begins: {json_str[:100]}...>"
+        return json_str
+    except Exception as e:
+        logger.debug(f"json.dumps total fail for {type(value).__name__}: {e2}")
+        # Final fallback
+        return safe_str_repr(value)
+
+
+def json_default_handler_factory(visited: Set[int]) -> Callable[[Any], Any]:
+    """
+    Create a JSON default handler with a shared visited set for circular reference detection.
+    """
+    def handler(obj: Any) -> Any:
+        # Handle datetime objects
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        if isinstance(obj, date):
+            return obj.isoformat()
+        if isinstance(obj, time):
+            return obj.isoformat()
+        
+        # Handle bytes
+        if isinstance(obj, bytes):
+            try:
+                return obj.decode('utf-8')
+            except UnicodeDecodeError:
+                return f"<bytes: {len(obj)} bytes>"
+        
+        # Try object conversion with the shared visited set
+        try:
+            return object_to_dict(obj, visited)
+        except Exception:
+            return safe_str_repr(obj)
+    
+    return handler
+
+
+def json_default_handler(obj: Any) -> Any:
+    """
+    Default handler for JSON serialization of non-serializable objects.
+    This is a fallback that creates its own visited set.
+    """
+    return json_default_handler_factory(set())(obj)
+

aiqa/test_experiment_runner.py

@@ -0,0 +1,176 @@
+"""
+Example usage of the ExperimentRunner class.
+"""
+
+import asyncio
+import os
+from dotenv import load_dotenv
+from aiqa import ExperimentRunner
+
+# Load environment variables
+load_dotenv()
+
+
+# A dummy test engine that returns a dummy response
+async def my_engine(input_data):
+    """
+    Example engine function that simulates an API call.
+    Note: For run(), the engine only takes input_data.
+    For run_example(), you can use an engine that takes (input_data, parameters).
+    """
+    # Imitate an OpenAI API response
+    # Sleep for random about 0.5 - 1 seconds
+    import random
+
+    sleep_time = random.random() * 0.5 + 0.5
+    await asyncio.sleep(sleep_time)
+    return {
+        "choices": [
+            {
+                "message": {
+                    "content": f"hello {input_data}",
+                },
+            },
+        ],
+    }
+
+
+async def scorer(output, example):
+    """
+    Example scorer function that scores the output.
+    In a real scenario, you would use the metrics from the dataset.
+    Note: For run(), the scorer only takes (output, example).
+    For run_example(), you can use a scorer that takes (output, example, parameters).
+    """
+    # This is a simple example - in practice, you'd use the metrics from the dataset
+    # and call the scoring functions accordingly
+    scores = {}
+    # Add your scoring logic here
+    return scores
+
+
+async def example_basic_usage():
+    """
+    Basic example of using ExperimentRunner.
+    """
+    if not os.getenv("AIQA_API_KEY"):
+        print("Warning: AIQA_API_KEY environment variable is not set. Example may fail.")
+
+    dataset_id = "your-dataset-id-here"
+    organisation_id = "your-organisation-id-here"
+
+    experiment_runner = ExperimentRunner(
+        dataset_id=dataset_id,
+        organisation_id=organisation_id,
+    )
+
+    # Get metrics from the dataset
+    dataset = experiment_runner.get_dataset()
+    metrics = dataset.get("metrics", [])
+    print(f"Found {len(metrics)} metrics in dataset: {[m['name'] for m in metrics]}")
+
+    # Create scorer that scores all metrics from the dataset
+    # (In practice, you'd implement this based on your metrics)
+    async def dataset_scorer(output, example):
+        # Use the metrics from the dataset to score
+        # This is a placeholder - implement based on your actual metrics
+        return await scorer(output, example)
+
+    # Get example inputs
+    example_inputs = experiment_runner.get_example_inputs()
+    print(f"Processing {len(example_inputs)} examples")
+
+    # Run experiments on each example
+    for example in example_inputs:
+        result = await experiment_runner.run_example(example, my_engine, dataset_scorer)
+        if result and len(result) > 0:
+            print(f"Scored example {example['id']}: {result}")
+        else:
+            print(f"No results for example {example['id']}")
+
+    # Get summary results
+    summary_results = experiment_runner.get_summary_results()
+    print(f"Summary results: {summary_results}")
+
+
+async def example_with_experiment_setup():
+    """
+    Example of creating an experiment with custom setup.
+    """
+    dataset_id = "your-dataset-id-here"
+    organisation_id = "your-organisation-id-here"
+
+    experiment_runner = ExperimentRunner(
+        dataset_id=dataset_id,
+        organisation_id=organisation_id,
+    )
+
+    # Create experiment with custom parameters
+    experiment = experiment_runner.create_experiment(
+        {
+            "name": "My Custom Experiment",
+            "parameters": {
+                "model": "gpt-4",
+                "temperature": 0.7,
+            },
+            "comparison_parameters": [
+                {"temperature": 0.5},
+                {"temperature": 0.9},
+            ],
+        }
+    )
+
+    print(f"Created experiment: {experiment['id']}")
+
+    # Now run the experiment
+    await experiment_runner.run(my_engine, scorer)
+
+
+async def example_stepwise():
+    """
+    Example of running experiments step by step (more control).
+    """
+    dataset_id = "your-dataset-id-here"
+    organisation_id = "your-organisation-id-here"
+
+    experiment_runner = ExperimentRunner(
+        dataset_id=dataset_id,
+        organisation_id=organisation_id,
+    )
+
+    # Get the dataset
+    dataset = experiment_runner.get_dataset()
+    metrics = dataset.get("metrics", [])
+    print(f"Found {len(metrics)} metrics in dataset")
+
+    # Create scorer for run_example (takes parameters)
+    async def my_scorer(output, example, parameters):
+        # Implement your scoring logic here
+        # Note: run_example() passes parameters, so this scorer can use them
+        return {"score": 0.8}  # Placeholder
+
+    # Get examples
+    examples = experiment_runner.get_example_inputs(limit=100)
+    print(f"Processing {len(examples)} examples")
+
+    # Process each example individually
+    for example in examples:
+        try:
+            result = await experiment_runner.run_example(example, my_engine, my_scorer)
+            print(f"Example {example['id']} completed: {result}")
+        except Exception as e:
+            print(f"Example {example['id']} failed: {e}")
+
+    # Get final summary
+    summary = experiment_runner.get_summary_results()
+    print(f"Final summary: {summary}")
+
+
+if __name__ == "__main__":
+    # Uncomment the example you want to run:
+    # asyncio.run(example_basic_usage())
+    # asyncio.run(example_with_experiment_setup())
+    # asyncio.run(example_stepwise())
+    print("Please uncomment one of the examples above to run it.")
+    print("Make sure to set your dataset_id and organisation_id in the example functions.")
+