aiqa-client 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

aiqa/object_serialiser.py

@@ -1,361 +0,0 @@
-"""
-Object serialization utilities for converting Python objects to JSON-safe formats.
-Handles objects, dataclasses, circular references, and size limits.
-"""
-
-import json
-import os
-import dataclasses
-from datetime import datetime, date, time
-from typing import Any, Callable, Set
-
-def toNumber(value: str|int|None) -> int:
-    """Convert string to number. handling units like g, m, k, (also mb kb gb though these should be avoided)"""
-    if value is None:
-        return 0
-    if isinstance(value, int):
-        return value    
-    if value.endswith("b"): # drop the b
-        value = value[:-1]
-    if value.endswith("g"):
-        return int(value[:-1]) * 1024 * 1024 * 1024
-    elif value.endswith("m"):
-        return int(value[:-1]) * 1024 * 1024
-    elif value.endswith("k"):
-        return int(value[:-1]) * 1024
-    return int(value)
-
-
-# Configurable limit for object string representation (in characters)
-AIQA_MAX_OBJECT_STR_CHARS = toNumber(os.getenv("AIQA_MAX_OBJECT_STR_CHARS", "1m"))
-
-# Data filters configuration
-def _get_enabled_filters() -> Set[str]:
-    """Get set of enabled filter names from AIQA_DATA_FILTERS env var."""
-    filters_env = os.getenv("AIQA_DATA_FILTERS", "RemovePasswords, RemoveJWT, RemoveAuthHeaders, RemoveAPIKeys")
-    if not filters_env or filters_env.lower() == "false":
-        return set()
-    return {f.strip() for f in filters_env.split(",") if f.strip()}
-
-_ENABLED_FILTERS = _get_enabled_filters()
-
-def _is_jwt_token(value: Any) -> bool:
-    """Check if a value looks like a JWT token (starts with 'eyJ' and has 3 parts separated by dots)."""
-    if not isinstance(value, str):
-        return False
-    # JWT tokens have format: header.payload.signature (3 parts separated by dots)
-    # They typically start with 'eyJ' (base64 encoded '{"')
-    parts = value.split('.')
-    return len(parts) == 3 and value.startswith('eyJ') and all(len(p) > 0 for p in parts)
-
-def _is_api_key(value: Any) -> bool:
-    """Check if a value looks like an API key based on common patterns."""
-    if not isinstance(value, str):
-        return False
-    value = value.strip()
-    # Common API key prefixes:
-    api_key_prefixes = [
-        'sk-',    # OpenAI secret key
-        'pk-',    # possibly public key
-        'AKIA',   # AWS access key
-        'ghp_',   # GitHub personal access token
-        'gho_',   # GitHub OAuth token
-        'ghu_',   # GitHub unidentified token
-        'ghs_',   # GitHub SAML token
-        'ghr_'    # GitHub refresh token
-    ]
-    return any(value.startswith(prefix) for prefix in api_key_prefixes)
-
-def _apply_data_filters(key: str, value: Any) -> Any:
-    """Apply data filters to a key-value pair based on enabled filters."""
-    if not value:  # Don't filter falsy values
-        return value
-    
-    key_lower = str(key).lower()
-    
-    # RemovePasswords filter: if key contains "password", replace value with "****"
-    if "RemovePasswords" in _ENABLED_FILTERS and "password" in key_lower:
-        return "****"
-    
-    # RemoveJWT filter: if value looks like a JWT token, replace with "****"
-    if "RemoveJWT" in _ENABLED_FILTERS and _is_jwt_token(value):
-        return "****"
-    
-    # RemoveAuthHeaders filter: if key is "authorization" (case-insensitive), replace value with "****"
-    if "RemoveAuthHeaders" in _ENABLED_FILTERS and key_lower == "authorization":
-        return "****"
-    
-    # RemoveAPIKeys filter: if key contains API key patterns or value looks like an API key
-    if "RemoveAPIKeys" in _ENABLED_FILTERS:
-        # Check key patterns
-        api_key_key_patterns = ['api_key', 'apikey', 'api-key', 'apikey']
-        if any(pattern in key_lower for pattern in api_key_key_patterns):
-            return "****"
-        # Check value patterns
-        if _is_api_key(value):
-            return "****"
-    
-    return value
-
-
-def serialize_for_span(value: Any) -> Any:
-    """
-    Serialize a value for span attributes.
-    OpenTelemetry only accepts primitives (bool, str, bytes, int, float) or sequences of those.
-    Complex types (dicts, lists, objects) are converted to JSON strings.
-    
-    Handles objects by attempting to convert them to dicts, with safeguards against:
-    - Circular references
-    - Unconvertible parts
-    - Large objects (size limits)
-    """
-    # Keep primitives as is (including None)
-    if value is None or isinstance(value, (str, int, float, bool, bytes)):
-        return value
-    
-    # For sequences, check if all elements are primitives
-    if isinstance(value, (list, tuple)):
-        # If all elements are primitives, return as list
-        if all(isinstance(item, (str, int, float, bool, bytes, type(None))) for item in value):
-            return list(value)
-        # Otherwise serialize to JSON string
-        try:
-            return safe_json_dumps(value)
-        except Exception:
-            return str(value)
-    
-    # For dicts and other complex types, serialize to JSON string
-    try:
-        return safe_json_dumps(value)
-    except Exception:
-        # If JSON serialization fails, convert to string
-        return safe_str_repr(value)
-
-
-def safe_str_repr(value: Any) -> str:
-    """
-    Safely convert a value to string representation.
-    Handles objects with __repr__ that might raise exceptions.
-    Uses AIQA_MAX_OBJECT_STR_CHARS environment variable (default: 100000) to limit length.
-    """
-    try:
-        # Try __repr__ first (usually more informative)
-        repr_str = repr(value)
-        # Limit length to avoid huge strings
-        if len(repr_str) > AIQA_MAX_OBJECT_STR_CHARS:
-            return repr_str[:AIQA_MAX_OBJECT_STR_CHARS] + "... (truncated)"
-        return repr_str
-    except Exception:
-        # Fallback to type name
-        try:
-            return f"<{type(value).__name__} object>"
-        except Exception:
-            return "<unknown object>"
-
-
-def object_to_dict(obj: Any, visited: Set[int], max_depth: int = 10, current_depth: int = 0) -> Any:
-    """
-    Convert an object to a dictionary representation.
-    
-    Args:
-        obj: The object to convert
-        visited: Set of object IDs to detect circular references
-        max_depth: Maximum recursion depth
-        current_depth: Current recursion depth
-    
-    Returns:
-        Dictionary representation of the object, or a string if conversion fails
-    """
-    if current_depth > max_depth:
-        return "<max depth exceeded>"
-    
-    obj_id = id(obj)
-    if obj_id in visited:
-        return "<circular reference>"
-    
-    # Handle None
-    if obj is None:
-        return None
-    
-    # Handle primitives
-    if isinstance(obj, (str, int, float, bool, bytes)):
-        return obj
-    
-    # Handle datetime objects
-    if isinstance(obj, datetime):
-        return obj.isoformat()
-    if isinstance(obj, date):
-        return obj.isoformat()
-    if isinstance(obj, time):
-        return obj.isoformat()
-    
-    # Handle dict
-    if isinstance(obj, dict):
-        visited.add(obj_id)
-        try:
-            result = {}
-            for k, v in obj.items():
-                key_str = str(k) if not isinstance(k, (str, int, float, bool)) else k
-                filtered_value = _apply_data_filters(key_str, v)
-                result[key_str] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
-            visited.remove(obj_id)
-            return result
-        except Exception:
-            visited.discard(obj_id)
-            return safe_str_repr(obj)
-    
-    # Handle list/tuple
-    if isinstance(obj, (list, tuple)):
-        visited.add(obj_id)
-        try:
-            result = [object_to_dict(item, visited, max_depth, current_depth + 1) for item in obj]
-            visited.remove(obj_id)
-            return result
-        except Exception:
-            visited.discard(obj_id)
-            return safe_str_repr(obj)
-    
-    # Handle dataclasses
-    if dataclasses.is_dataclass(obj):
-        visited.add(obj_id)
-        try:
-            result = {}
-            for field in dataclasses.fields(obj):
-                value = getattr(obj, field.name, None)
-                filtered_value = _apply_data_filters(field.name, value)
-                result[field.name] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
-            visited.remove(obj_id)
-            return result
-        except Exception:
-            visited.discard(obj_id)
-            return safe_str_repr(obj)
-    
-    # Handle objects with __dict__
-    if hasattr(obj, "__dict__"):
-        visited.add(obj_id)
-        try:
-            result = {}
-            for key, value in obj.__dict__.items():
-                # Skip private attributes that start with __
-                if not (isinstance(key, str) and key.startswith("__")):
-                    filtered_value = _apply_data_filters(key, value)
-                    result[key] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
-            visited.remove(obj_id)
-            return result
-        except Exception:
-            visited.discard(obj_id)
-            return safe_str_repr(obj)
-    
-    # Handle objects with __slots__
-    if hasattr(obj, "__slots__"):
-        visited.add(obj_id)
-        try:
-            result = {}
-            for slot in obj.__slots__:
-                if hasattr(obj, slot):
-                    value = getattr(obj, slot, None)
-                    filtered_value = _apply_data_filters(slot, value)
-                    result[slot] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
-            visited.remove(obj_id)
-            return result
-        except Exception:
-            visited.discard(obj_id)
-            return safe_str_repr(obj)
-    
-    # Fallback: try to get a few common attributes
-    try:
-        result = {}
-        for attr in ["name", "id", "value", "type", "status"]:
-            if hasattr(obj, attr):
-                value = getattr(obj, attr, None)
-                filtered_value = _apply_data_filters(attr, value)
-                result[attr] = object_to_dict(filtered_value, visited, max_depth, current_depth + 1)
-        if result:
-            return result
-    except Exception:
-        pass
-    
-    # Final fallback: string representation
-    return safe_str_repr(obj)
-
-
-def safe_json_dumps(value: Any) -> str:
-    """
-    Safely serialize a value to JSON string with safeguards against:
-    - Circular references
-    - Large objects (size limits)
-    - Unconvertible parts
-    
-    Args:
-        value: The value to serialize
-
-    Uses AIQA_MAX_OBJECT_STR_CHARS environment variable (default: 1000000) to limit length.
-    
-    Returns:
-        JSON string representation
-    """    
-    max_size_chars = AIQA_MAX_OBJECT_STR_CHARS
-    visited: Set[int] = set()
-    
-    # Convert the entire structure to ensure circular references are detected
-    # across the whole object graph
-    try:
-        converted = object_to_dict(value, visited)
-    except Exception:
-        # If conversion fails, try with a fresh visited set and json default handler
-        try:
-            json_str = json.dumps(value, default=json_default_handler_factory(set()))
-            if len(json_str) > max_size_chars:
-                return f"<object {type(value)} too large: {len(json_str)} chars (limit: {max_size_chars} chars) begins: {json_str[:100]}... conversion error: {e}>"
-            return json_str
-        except Exception:
-            return safe_str_repr(value)
-    
-    # Try JSON serialization of the converted structure
-    try:
-        json_str = json.dumps(converted, default=json_default_handler_factory(set()))
-        # Check size
-        if len(json_str) > max_size_chars:
-            return f"<object {type(value)} too large: {len(json_str)} chars (limit: {max_size_chars} chars) begins: {json_str[:100]}...>"
-        return json_str
-    except Exception:
-        # Final fallback
-        return safe_str_repr(value)
-
-
-def json_default_handler_factory(visited: Set[int]) -> Callable[[Any], Any]:
-    """
-    Create a JSON default handler with a shared visited set for circular reference detection.
-    """
-    def handler(obj: Any) -> Any:
-        # Handle datetime objects
-        if isinstance(obj, datetime):
-            return obj.isoformat()
-        if isinstance(obj, date):
-            return obj.isoformat()
-        if isinstance(obj, time):
-            return obj.isoformat()
-        
-        # Handle bytes
-        if isinstance(obj, bytes):
-            try:
-                return obj.decode('utf-8')
-            except UnicodeDecodeError:
-                return f"<bytes: {len(obj)} bytes>"
-        
-        # Try object conversion with the shared visited set
-        try:
-            return object_to_dict(obj, visited)
-        except Exception:
-            return safe_str_repr(obj)
-    
-    return handler
-
-
-def json_default_handler(obj: Any) -> Any:
-    """
-    Default handler for JSON serialization of non-serializable objects.
-    This is a fallback that creates its own visited set.
-    """
-    return json_default_handler_factory(set())(obj)
-

aiqa/test_experiment_runner.py

@@ -1,176 +0,0 @@
-"""
-Example usage of the ExperimentRunner class.
-"""
-
-import asyncio
-import os
-from dotenv import load_dotenv
-from aiqa import ExperimentRunner
-
-# Load environment variables
-load_dotenv()
-
-
-# A dummy test engine that returns a dummy response
-async def my_engine(input_data):
-    """
-    Example engine function that simulates an API call.
-    Note: For run(), the engine only takes input_data.
-    For run_example(), you can use an engine that takes (input_data, parameters).
-    """
-    # Imitate an OpenAI API response
-    # Sleep for random about 0.5 - 1 seconds
-    import random
-
-    sleep_time = random.random() * 0.5 + 0.5
-    await asyncio.sleep(sleep_time)
-    return {
-        "choices": [
-            {
-                "message": {
-                    "content": f"hello {input_data}",
-                },
-            },
-        ],
-    }
-
-
-async def scorer(output, example):
-    """
-    Example scorer function that scores the output.
-    In a real scenario, you would use the metrics from the dataset.
-    Note: For run(), the scorer only takes (output, example).
-    For run_example(), you can use a scorer that takes (output, example, parameters).
-    """
-    # This is a simple example - in practice, you'd use the metrics from the dataset
-    # and call the scoring functions accordingly
-    scores = {}
-    # Add your scoring logic here
-    return scores
-
-
-async def example_basic_usage():
-    """
-    Basic example of using ExperimentRunner.
-    """
-    if not os.getenv("AIQA_API_KEY"):
-        print("Warning: AIQA_API_KEY environment variable is not set. Example may fail.")
-
-    dataset_id = "your-dataset-id-here"
-    organisation_id = "your-organisation-id-here"
-
-    experiment_runner = ExperimentRunner(
-        dataset_id=dataset_id,
-        organisation_id=organisation_id,
-    )
-
-    # Get metrics from the dataset
-    dataset = experiment_runner.get_dataset()
-    metrics = dataset.get("metrics", [])
-    print(f"Found {len(metrics)} metrics in dataset: {[m['name'] for m in metrics]}")
-
-    # Create scorer that scores all metrics from the dataset
-    # (In practice, you'd implement this based on your metrics)
-    async def dataset_scorer(output, example):
-        # Use the metrics from the dataset to score
-        # This is a placeholder - implement based on your actual metrics
-        return await scorer(output, example)
-
-    # Get example inputs
-    example_inputs = experiment_runner.get_example_inputs()
-    print(f"Processing {len(example_inputs)} examples")
-
-    # Run experiments on each example
-    for example in example_inputs:
-        result = await experiment_runner.run_example(example, my_engine, dataset_scorer)
-        if result and len(result) > 0:
-            print(f"Scored example {example['id']}: {result}")
-        else:
-            print(f"No results for example {example['id']}")
-
-    # Get summary results
-    summary_results = experiment_runner.get_summary_results()
-    print(f"Summary results: {summary_results}")
-
-
-async def example_with_experiment_setup():
-    """
-    Example of creating an experiment with custom setup.
-    """
-    dataset_id = "your-dataset-id-here"
-    organisation_id = "your-organisation-id-here"
-
-    experiment_runner = ExperimentRunner(
-        dataset_id=dataset_id,
-        organisation_id=organisation_id,
-    )
-
-    # Create experiment with custom parameters
-    experiment = experiment_runner.create_experiment(
-        {
-            "name": "My Custom Experiment",
-            "parameters": {
-                "model": "gpt-4",
-                "temperature": 0.7,
-            },
-            "comparison_parameters": [
-                {"temperature": 0.5},
-                {"temperature": 0.9},
-            ],
-        }
-    )
-
-    print(f"Created experiment: {experiment['id']}")
-
-    # Now run the experiment
-    await experiment_runner.run(my_engine, scorer)
-
-
-async def example_stepwise():
-    """
-    Example of running experiments step by step (more control).
-    """
-    dataset_id = "your-dataset-id-here"
-    organisation_id = "your-organisation-id-here"
-
-    experiment_runner = ExperimentRunner(
-        dataset_id=dataset_id,
-        organisation_id=organisation_id,
-    )
-
-    # Get the dataset
-    dataset = experiment_runner.get_dataset()
-    metrics = dataset.get("metrics", [])
-    print(f"Found {len(metrics)} metrics in dataset")
-
-    # Create scorer for run_example (takes parameters)
-    async def my_scorer(output, example, parameters):
-        # Implement your scoring logic here
-        # Note: run_example() passes parameters, so this scorer can use them
-        return {"score": 0.8}  # Placeholder
-
-    # Get examples
-    examples = experiment_runner.get_example_inputs(limit=100)
-    print(f"Processing {len(examples)} examples")
-
-    # Process each example individually
-    for example in examples:
-        try:
-            result = await experiment_runner.run_example(example, my_engine, my_scorer)
-            print(f"Example {example['id']} completed: {result}")
-        except Exception as e:
-            print(f"Example {example['id']} failed: {e}")
-
-    # Get final summary
-    summary = experiment_runner.get_summary_results()
-    print(f"Final summary: {summary}")
-
-
-if __name__ == "__main__":
-    # Uncomment the example you want to run:
-    # asyncio.run(example_basic_usage())
-    # asyncio.run(example_with_experiment_setup())
-    # asyncio.run(example_stepwise())
-    print("Please uncomment one of the examples above to run it.")
-    print("Make sure to set your dataset_id and organisation_id in the example functions.")
-