fluxloop 0.1.0__py3-none-any.whl

fluxloop/models.py

@@ -0,0 +1,92 @@
+"""
+Internal SDK models for observations and traces.
+These are lightweight versions optimized for SDK use.
+"""
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Dict, List, Optional
+from uuid import UUID, uuid4
+
+from pydantic import BaseModel, Field
+
+
+class ObservationType(str, Enum):
+    """Type of observation."""
+    
+    SPAN = "span"
+    EVENT = "event"
+    GENERATION = "generation"
+    TOOL = "tool"
+    AGENT = "agent"
+    CHAIN = "chain"
+    EVALUATION = "evaluation"
+
+
+class ObservationLevel(str, Enum):
+    """Log level for observations."""
+    
+    DEBUG = "debug"
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+class ObservationData(BaseModel):
+    """Lightweight observation data for SDK use."""
+    
+    id: UUID = Field(default_factory=uuid4)
+    type: ObservationType
+    name: str
+    
+    # Timing
+    start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    end_time: Optional[datetime] = None
+    
+    # Content
+    input: Optional[Any] = None
+    output: Optional[Any] = None
+    error: Optional[str] = None
+    
+    # Metadata
+    level: ObservationLevel = ObservationLevel.INFO
+    status_message: Optional[str] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+    
+    # LLM specific
+    model: Optional[str] = None
+    llm_parameters: Optional[Dict[str, Any]] = None
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
+    total_tokens: Optional[int] = None
+    
+    # Parent reference (set by context)
+    parent_observation_id: Optional[UUID] = None
+    trace_id: Optional[UUID] = None
+
+
+class TraceData(BaseModel):
+    """Lightweight trace data for SDK use."""
+    
+    id: UUID = Field(default_factory=uuid4)
+    name: str
+    
+    # Timing
+    start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    end_time: Optional[datetime] = None
+    
+    # Context
+    session_id: Optional[UUID] = None
+    user_id: Optional[str] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+    tags: List[str] = Field(default_factory=list)
+    
+    # Experiment specific
+    experiment_id: Optional[str] = None
+    iteration: Optional[int] = None
+    persona: Optional[str] = None
+    variation_seed: Optional[str] = None
+    
+    # Input/Output
+    input: Optional[Any] = None
+    output: Optional[Any] = None

fluxloop/recording.py

@@ -0,0 +1,205 @@
+"""Argument recording utilities for the FluxLoop SDK (MVP implementation)."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+_SENSITIVE_KEY_PATTERNS = [
+    "token",
+    "password",
+    "secret",
+    "key",
+    "auth",
+    "credential",
+]
+
+try:
+    from pydantic import BaseModel
+except ImportError:  # pragma: no cover - fallback if optional dependency missing
+    class BaseModel:  # type: ignore
+        def __init__(self, **kwargs):
+            for key, value in kwargs.items():
+                setattr(self, key, value)
+
+
+class ArgsRecorder:
+    """Simple argument recorder that writes call metadata to a JSONL file."""
+
+    def __init__(self, output_file: Path) -> None:
+        self.output_file = output_file
+        self.output_file.parent.mkdir(parents=True, exist_ok=True)
+        self._iteration_counters: Dict[str, int] = {}
+
+    def record(self, target: str, *, iteration: Optional[int], **kwargs: Any) -> None:
+        """Record call arguments for the given target."""
+
+        serializable_kwargs: Dict[str, Any] = {}
+
+        for key, value in kwargs.items():
+            if callable(value):
+                serializable_kwargs[key] = self._serialize_callable(value)
+                continue
+
+            if self._is_sensitive_key(key):
+                serializable_kwargs[key] = "***"
+                continue
+
+            safe_value = self._coerce_to_json_safe(value)
+
+            try:
+                json.dumps(safe_value)
+            except (TypeError, ValueError):
+                serializable_kwargs[key] = self._serialize_non_json_value(key, value)
+            else:
+                serializable_kwargs[key] = safe_value
+
+        resolved_iteration = self._resolve_iteration(target, iteration)
+
+        record = {
+            "_version": "1",
+            "iteration": resolved_iteration,
+            "target": target,
+            "kwargs": serializable_kwargs,
+            "timestamp": datetime.now().isoformat(),
+        }
+
+        with self.output_file.open("a", encoding="utf-8") as fp:
+            fp.write(json.dumps(record, default=str) + "\n")
+
+    def _resolve_iteration(self, target: str, iteration: Optional[int]) -> int:
+        if iteration is not None:
+            self._iteration_counters[target] = iteration
+            return iteration
+
+        next_value = self._iteration_counters.get(target, -1) + 1
+        self._iteration_counters[target] = next_value
+        return next_value
+
+    def _serialize_callable(self, value: Any) -> str:
+        marker = getattr(value, "__fluxloop_builtin__", None)
+        if marker:
+            return f"<builtin:{marker}>"
+
+        if hasattr(value, "messages"):
+            return "<builtin:collector.send>"
+
+        if hasattr(value, "errors"):
+            return "<builtin:collector.error>"
+
+        name = getattr(value, "__name__", "unknown")
+        return f"<callable:{name}>"
+
+    def _serialize_non_json_value(self, key: str, value: Any) -> Any:
+        if self._is_sensitive_key(key):
+            return "***"
+
+        coerced = self._coerce_to_json_safe(value)
+        try:
+            json.dumps(coerced)
+            return coerced
+        except (TypeError, ValueError):
+            representation = repr(value)
+            if len(representation) > 100:
+                representation = representation[:100]
+            return f"<repr:{representation}>"
+
+    def _coerce_to_json_safe(self, value: Any, *, depth: int = 0) -> Any:
+        if depth > 3:
+            return f"<repr:{type(value).__name__}>"
+
+        if value is None or isinstance(value, (str, int, float, bool)):
+            return value
+
+        if isinstance(value, datetime):
+            return value.isoformat()
+
+        if isinstance(value, Mapping):
+            result: Dict[str, Any] = {}
+            for key, item in value.items():
+                mask_key = str(key)
+                if self._is_sensitive_key(mask_key):
+                    result[mask_key] = "***"
+                else:
+                    result[mask_key] = self._coerce_to_json_safe(item, depth=depth + 1)
+            return result
+
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+            return [
+                "***"
+                if self._is_collection_with_sensitive_keys(item)
+                else self._coerce_to_json_safe(item, depth=depth + 1)
+                for item in value
+            ]
+
+        if hasattr(value, "__dict__"):
+            return {
+                str(attr): self._coerce_to_json_safe(attr_value, depth=depth + 1)
+                for attr, attr_value in vars(value).items()
+                if not attr.startswith("__")
+            }
+
+        return value
+
+    def _is_collection_with_sensitive_keys(self, value: Any) -> bool:
+        if isinstance(value, Mapping):
+            return any(self._is_sensitive_key(str(key)) for key in value.keys())
+
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+            return any(self._is_collection_with_sensitive_keys(item) for item in value)
+
+        return False
+
+    def _is_sensitive_key(self, key: str) -> bool:
+        key_lower = key.lower()
+        return any(pattern in key_lower for pattern in _SENSITIVE_KEY_PATTERNS)
+
+
+_global_recorder: Optional[ArgsRecorder] = None
+
+class RecordingConfig(BaseModel):
+    iteration_auto_increment: bool = True
+
+
+_recording_config = RecordingConfig()
+
+
+def enable_recording(output_file: str) -> None:
+    """Enable argument recording by configuring the global recorder."""
+
+    global _global_recorder
+    resolved_path = Path(output_file).expanduser().resolve()
+    _global_recorder = ArgsRecorder(resolved_path)
+
+
+def record_call_args(target: str, *, iteration: Optional[int] = None, **kwargs: Any) -> None:
+    """Record call arguments if recording is enabled."""
+
+    if _global_recorder is None:
+        return
+
+    recorded_iteration = iteration
+    if iteration is None and not _recording_config.iteration_auto_increment:
+        recorded_iteration = 0
+
+    _global_recorder.record(target, iteration=recorded_iteration, **kwargs)
+
+
+def disable_recording() -> None:
+    """Disable argument recording."""
+
+    global _global_recorder
+    _global_recorder = None
+
+
+def set_recording_options(*, iteration_auto_increment: Optional[bool] = None) -> None:
+    """Update global recording behaviour."""
+
+    if iteration_auto_increment is not None:
+        _recording_config.iteration_auto_increment = iteration_auto_increment
+
+

fluxloop/schemas/__init__.py

@@ -0,0 +1,48 @@
+"""
+FluxLoop Shared Schemas
+
+Common data models shared across all FluxLoop components.
+Based on Langfuse data model for compatibility.
+"""
+
+from .trace import (
+    Trace,
+    Observation,
+    ObservationType,
+    Score,
+    ScoreDataType,
+    TraceStatus,
+    ObservationLevel,
+)
+from .config import (
+    ExperimentConfig,
+    InputGenerationConfig,
+    InputGenerationMode,
+    LLMGeneratorConfig,
+    PersonaConfig,
+    ReplayArgsConfig,
+    VariationStrategy,
+    EvaluatorConfig,
+    RunnerConfig,
+)
+
+__all__ = [
+    # Trace models
+    "Trace",
+    "Observation",
+    "ObservationType",
+    "Score",
+    "ScoreDataType",
+    "TraceStatus",
+    "ObservationLevel",
+    # Config models
+    "ExperimentConfig",
+    "InputGenerationConfig",
+    "InputGenerationMode",
+    "LLMGeneratorConfig",
+    "PersonaConfig",
+    "ReplayArgsConfig",
+    "VariationStrategy",
+    "EvaluatorConfig",
+    "RunnerConfig",
+]

fluxloop/schemas/config.py

@@ -0,0 +1,312 @@
+"""
+Configuration schemas for experiments and simulations.
+"""
+
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, PrivateAttr, field_validator, model_validator
+
+
+class VariationStrategy(str, Enum):
+    """Strategy for generating prompt variations."""
+    
+    REPHRASE = "rephrase"  # Rephrase the same intent
+    TYPO = "typo"  # Add typos/errors
+    VERBOSE = "verbose"  # Make more verbose
+    CONCISE = "concise"  # Make more concise
+    PERSONA_BASED = "persona_based"  # Based on persona characteristics
+    ADVERSARIAL = "adversarial"  # Edge cases and attacks
+    MULTILINGUAL = "multilingual"  # Different languages
+    CUSTOM = "custom"  # Custom variation prompt
+
+
+class InputGenerationMode(str, Enum):
+    """Supported input generation approaches."""
+
+    DETERMINISTIC = "deterministic"
+    LLM = "llm"
+
+
+class LLMGeneratorConfig(BaseModel):
+    """Configuration for LLM-backed input generation."""
+
+    enabled: bool = False
+    provider: str = "openai"
+    model: str = "gpt-4o-mini"
+    api_key: Optional[str] = None
+    system_prompt: Optional[str] = None
+    user_prompt_template: Optional[str] = None
+    strategy_prompts: Dict[str, str] = Field(default_factory=dict)
+    max_outputs: int = Field(default=3, ge=1, le=20)
+    temperature: float = Field(default=0.7, ge=0.0, le=2.0)
+    top_p: float = Field(default=1.0, ge=0.0, le=1.0)
+    frequency_penalty: float = Field(default=0.0, ge=-2.0, le=2.0)
+    presence_penalty: float = Field(default=0.0, ge=-2.0, le=2.0)
+    max_tokens: int = Field(default=1024, ge=16, le=4096)
+    request_timeout: int = Field(default=60, ge=1, le=600)
+    batch_size: int = Field(default=1, ge=1, le=10)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    # GPT-5 specific controls
+    reasoning_effort: Optional[str] = Field(
+        default=None,
+        description="Reasoning effort for GPT-5 models: minimal, low, medium, high",
+    )
+    text_verbosity: Optional[str] = Field(
+        default=None,
+        description="Output verbosity for GPT-5 models: low, medium, high",
+    )
+
+
+class InputGenerationConfig(BaseModel):
+    """Wrapper around input generation options."""
+
+    mode: InputGenerationMode = InputGenerationMode.LLM
+    llm: LLMGeneratorConfig = Field(default_factory=LLMGeneratorConfig)
+
+
+class PersonaConfig(BaseModel):
+    """User persona for simulation."""
+    
+    name: str
+    description: str
+    characteristics: List[str] = Field(default_factory=list)
+    language: str = "en"
+    expertise_level: str = "intermediate"  # novice, intermediate, expert
+    goals: List[str] = Field(default_factory=list)
+    constraints: List[str] = Field(default_factory=list)
+    custom_attributes: Dict[str, Any] = Field(default_factory=dict)
+    
+    def to_prompt(self) -> str:
+        """Convert persona to a prompt description."""
+        prompt_parts = [
+            f"User Persona: {self.name}",
+            f"Description: {self.description}",
+        ]
+        
+        if self.characteristics:
+            prompt_parts.append(f"Characteristics: {', '.join(self.characteristics)}")
+        
+        if self.goals:
+            prompt_parts.append(f"Goals: {', '.join(self.goals)}")
+        
+        if self.constraints:
+            prompt_parts.append(f"Constraints: {', '.join(self.constraints)}")
+        
+        prompt_parts.append(f"Language: {self.language}")
+        prompt_parts.append(f"Expertise Level: {self.expertise_level}")
+        
+        return "\n".join(prompt_parts)
+
+
+class EvaluatorConfig(BaseModel):
+    """Configuration for evaluation methods."""
+    
+    name: str
+    type: str  # "llm_judge", "rule_based", "metric", "custom"
+    enabled: bool = True
+    
+    # For LLM judge
+    model: Optional[str] = None
+    prompt_template: Optional[str] = None
+    
+    # For rule-based
+    rules: List[Dict[str, Any]] = Field(default_factory=list)
+    
+    # For metrics
+    metric_name: Optional[str] = None
+    threshold: Optional[float] = None
+    
+    # Custom evaluator
+    module_path: Optional[str] = None
+    class_name: Optional[str] = None
+    
+    # Common
+    weight: float = 1.0
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+
+class RunnerConfig(BaseModel):
+    """Configuration for agent execution."""
+    
+    # Entry point
+    module_path: str  # e.g., "my_agent.main"
+    function_name: str = "run"  # Function to call
+    target: Optional[str] = Field(
+        default=None,
+        description=(
+            "Optional combined target specification. Use 'module:function' or "
+            "'module:Class.method'. When provided, takes precedence over "
+            "module_path + function_name."
+        ),
+    )
+    
+    # Execution environment
+    working_directory: Optional[str] = None
+    python_path: Optional[str] = None
+    environment_vars: Dict[str, str] = Field(default_factory=dict)
+    
+    # Dependencies
+    requirements_file: Optional[str] = None
+    setup_commands: List[str] = Field(default_factory=list)
+    
+    # Execution settings
+    timeout_seconds: int = 300
+    max_retries: int = 3
+    retry_delay: int = 5
+    
+    # Docker settings (optional)
+    use_docker: bool = False
+    docker_image: Optional[str] = None
+    docker_build_context: Optional[str] = None
+    docker_volumes: List[str] = Field(default_factory=list)
+
+
+class ExperimentConfig(BaseModel):
+    """Complete experiment configuration."""
+
+    _source_dir: Optional[Path] = PrivateAttr(default=None)
+    _resolved_input_count: Optional[int] = PrivateAttr(default=None)
+    _resolved_persona_count: Optional[int] = PrivateAttr(default=None)
+
+    # Basic info
+    name: str
+    description: Optional[str] = None
+    version: str = "1.0.0"
+
+    # Simulation settings
+    iterations: int = Field(default=10, ge=1, le=1000)
+    parallel_runs: int = Field(default=1, ge=1, le=10)
+    seed: Optional[int] = None
+    run_delay_seconds: float = Field(default=0.0, ge=0.0)
+
+    # Personas
+    personas: List[PersonaConfig] = Field(default_factory=list)
+
+    # Variation settings
+    variation_strategies: List[VariationStrategy] = Field(default_factory=list)
+    variation_count: int = Field(default=1, ge=1, le=10)
+    variation_temperature: float = Field(default=0.7, ge=0, le=2)
+    variation_model: str = "gpt-3.5-turbo"
+    custom_variation_prompt: Optional[str] = None
+
+    # Base prompts/inputs
+    base_inputs: List[Dict[str, Any]] = Field(default_factory=list)
+    inputs_file: Optional[str] = None
+    input_template: Optional[str] = None
+    input_generation: InputGenerationConfig = Field(default_factory=InputGenerationConfig)
+
+    # Runner configuration
+    runner: RunnerConfig
+
+    # Argument replay (optional)
+    replay_args: Optional["ReplayArgsConfig"] = None
+
+    # Evaluators
+    evaluators: List[EvaluatorConfig] = Field(default_factory=list)
+
+    # Output settings
+    output_directory: str = "./experiments"
+    save_traces: bool = True
+    save_aggregated_metrics: bool = True
+
+    # Collector settings
+    collector_url: Optional[str] = None
+    collector_api_key: Optional[str] = None
+
+    # Metadata
+    tags: List[str] = Field(default_factory=list)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+    
+    @field_validator("iterations")
+    def validate_iterations(cls, v):
+        """Ensure reasonable iteration count."""
+        if v > 1000:
+            raise ValueError("iterations must be <= 1000 for safety")
+        return v
+    
+    @model_validator(mode="after")
+    def validate_input_sources(cls, values: "ExperimentConfig") -> "ExperimentConfig":
+        """Ensure at least one input source is configured."""
+        if not values.base_inputs and not values.inputs_file:
+            # Allow both sources to be disabled if base_inputs is provided
+            if not values.base_inputs:
+                raise ValueError("Either base_inputs or inputs_file must be provided")
+        return values
+
+    def has_external_inputs(self) -> bool:
+        """Return True when inputs should be loaded from an external file."""
+        return bool(self.inputs_file)
+
+    def set_source_dir(self, source_dir: Path) -> None:
+        """Remember the directory where this config file was loaded from."""
+        self._source_dir = source_dir
+
+    def get_source_dir(self) -> Optional[Path]:
+        """Return the directory where the config file was loaded from."""
+        return self._source_dir
+
+    def set_resolved_input_count(self, count: int) -> None:
+        """Record the effective input count after resolution."""
+        if count < 0:
+            raise ValueError("resolved input count must be non-negative")
+        self._resolved_input_count = count
+
+    def get_resolved_input_count(self) -> Optional[int]:
+        """Return the resolved input count if it has been set."""
+        return self._resolved_input_count
+
+    def set_resolved_persona_count(self, count: int) -> None:
+        """Record the effective persona multiplier after resolution."""
+        if count < 1:
+            raise ValueError("resolved persona count must be >= 1")
+        self._resolved_persona_count = count
+
+    def get_resolved_persona_count(self) -> Optional[int]:
+        """Return the resolved persona multiplier if available."""
+        return self._resolved_persona_count
+
+    def _default_input_count(self) -> int:
+        """Fallback calculation when no resolved count is available."""
+        base_count = len(self.base_inputs)
+        if self.has_external_inputs():
+            return base_count if base_count else 1
+        variation_multiplier = max(1, self.variation_count)
+        return base_count * variation_multiplier if base_count else variation_multiplier
+
+    def get_input_count(self) -> int:
+        """Return the effective number of inputs that will be executed."""
+        return self._resolved_input_count if self._resolved_input_count is not None else self._default_input_count()
+
+    def estimate_total_runs(self) -> int:
+        """Calculate total number of runs."""
+        if self._resolved_persona_count is not None:
+            persona_count = self._resolved_persona_count
+        else:
+            persona_count = len(self.personas) if self.personas else 1
+        input_count = self.get_input_count()
+        return self.iterations * persona_count * input_count
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return self.model_dump(exclude_none=True)
+
+
+class ReplayArgsConfig(BaseModel):
+    """Configuration for argument replay (MVP scope)."""
+
+    enabled: bool = False
+    recording_file: Optional[str] = None
+    callable_providers: Dict[str, str] = Field(
+        default_factory=lambda: {
+            "send_message_callback": "builtin:collector.send",
+            "send_error_callback": "builtin:collector.error",
+        },
+        description="Mapping of callable parameter names to builtin providers",
+    )
+    override_param_path: Optional[str] = Field(
+        default="data.content",
+        description="Single dot-notation path whose value should be overridden with runtime input",
+    )