flashlite 0.1.0__tar.gz → 0.1.1__tar.gz

{flashlite-0.1.0 → flashlite-0.1.1}/.gitignore

@@ -8,3 +8,6 @@ wheels/
 
 # Virtual environments
 .venv
+.env
+
+logs/

{flashlite-0.1.0 → flashlite-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashlite
-Version: 0.1.0
+Version: 0.1.1
 Summary: Batteries-included wrapper for litellm with rate limiting, retries, templating, and more
 Author-email: ndalton12 <niall.dalton12@gmail.com>
 License-File: LICENSE.md
@@ -16,6 +16,8 @@ Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
 Requires-Dist: pytest>=8.0.0; extra == 'dev'
 Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: evals
+Requires-Dist: inspect-ai>=0.3.0; extra == 'evals'
 Description-Content-Type: text/markdown
 
 # Flashlite

{flashlite-0.1.0 → flashlite-0.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flashlite"
-version = "0.1.0"
+version = "0.1.1"
 description = "Batteries-included wrapper for litellm with rate limiting, retries, templating, and more"
 readme = "README.md"
 authors = [
@@ -23,6 +23,9 @@ dev = [
     "mypy>=1.0.0",
     "ruff>=0.1.0",
 ]
+evals = [
+    "inspect-ai>=0.3.0",
+]
 
 [build-system]
 requires = ["hatchling"]

{flashlite-0.1.0 → flashlite-0.1.1}/src/flashlite/observability/__init__.py

@@ -7,7 +7,12 @@ from .callbacks import (
     OnResponseCallback,
     create_logging_callbacks,
 )
-from .inspect_compat import FlashliteModelAPI, InspectLogEntry, InspectLogger
+from .inspect_compat import (
+    FlashliteModelAPI,
+    InspectLogEntry,
+    InspectLogger,
+    convert_flashlite_logs_to_inspect,
+)
 from .logging import RequestContext, RequestLogEntry, ResponseLogEntry, StructuredLogger
 from .metrics import BudgetExceededError, CostMetrics, CostTracker
 
@@ -31,4 +36,5 @@ __all__ = [
     "InspectLogger",
     "InspectLogEntry",
     "FlashliteModelAPI",
+    "convert_flashlite_logs_to_inspect",
 ]

flashlite-0.1.1/src/flashlite/observability/inspect_compat.py

@@ -0,0 +1,527 @@
+"""Inspect framework compatibility layer for flashlite.
+
+This module provides interoperability with the UK AISI's Inspect framework
+(https://inspect.ai-safety-institute.org.uk/).
+
+It includes:
+- Log format conversion to Inspect's native eval log format
+- ModelAPI protocol implementation for use as an Inspect solver backend
+- Functions to convert flashlite JSONL logs to Inspect-viewable format
+"""
+
+import json
+import logging
+import uuid
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from ..types import CompletionRequest, CompletionResponse
+
+if TYPE_CHECKING:
+    from ..client import Flashlite
+
+logger = logging.getLogger(__name__)
+
+
+def convert_flashlite_logs_to_inspect(
+    input_path: str | Path,
+    output_path: str | Path | None = None,
+    task_name: str | None = None,
+) -> Path:
+    """
+    Convert flashlite JSONL logs to Inspect-compatible format.
+
+    This allows logs generated by flashlite's InspectLogger to be viewed
+    in Inspect's log viewer (`inspect view`).
+
+    Args:
+        input_path: Path to flashlite JSONL log file
+        output_path: Output path for Inspect log file (defaults to same dir with proper naming)
+        task_name: Task name for the evaluation (defaults to eval_id from logs)
+
+    Returns:
+        Path to the generated Inspect log file
+
+    Raises:
+        FileNotFoundError: If input file doesn't exist
+
+    Example:
+        >>> from flashlite.observability import convert_flashlite_logs_to_inspect
+        >>> convert_flashlite_logs_to_inspect("logs/my_eval.jsonl")
+        PosixPath('logs/2026-02-05T12-00-00_my_eval_abc123.json')
+    """
+    input_path = Path(input_path)
+    if not input_path.exists():
+        raise FileNotFoundError(f"Log file not found: {input_path}")
+
+    # Read all entries from JSONL
+    entries: list[dict[str, Any]] = []
+    with open(input_path) as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                entries.append(json.loads(line))
+
+    if not entries:
+        raise ValueError(f"No log entries found in {input_path}")
+
+    # Extract metadata from first entry
+    first_entry = entries[0]
+    eval_id = first_entry.get("eval_id", "flashlite_eval")
+    model_name = first_entry.get("model", "unknown")
+    task = task_name or eval_id
+
+    # Get timestamp from entries or generate one
+    timestamps = [e.get("timestamp", "") for e in entries if e.get("timestamp")]
+    if timestamps:
+        # Parse and format for filename (Inspect uses format like 2024-05-29T12-38-43)
+        started_at = min(timestamps)
+        # Convert ISO format to Inspect's filename format
+        ts_for_filename = started_at.replace(":", "-").split(".")[0]
+    else:
+        ts_for_filename = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S")
+
+    # Generate a short unique ID
+    short_id = uuid.uuid4().hex[:8]
+
+    # Determine output path with Inspect's naming convention: {timestamp}_{task}_{id}.json
+    if output_path is None:
+        output_dir = input_path.parent
+        output_filename = f"{ts_for_filename}_{task}_{short_id}.json"
+        output_path = output_dir / output_filename
+    else:
+        output_path = Path(output_path)
+
+    # Build EvalLog structure as dict (Inspect's JSON format)
+    eval_log = _build_eval_log_dict(
+        entries=entries,
+        eval_id=eval_id,
+        task_name=task,
+        model_name=model_name,
+    )
+
+    # Write JSON directly
+    with open(output_path, "w") as f:
+        json.dump(eval_log, f, indent=2)
+
+    logger.info(f"Converted {len(entries)} entries to Inspect format: {output_path}")
+    return output_path
+
+
+def _build_eval_log_dict(
+    entries: list[dict[str, Any]],
+    eval_id: str,
+    task_name: str,
+    model_name: str,
+) -> dict[str, Any]:
+    """Build an Inspect-compatible EvalLog dict from flashlite log entries."""
+    # Calculate timestamps
+    timestamps = [e.get("timestamp", "") for e in entries if e.get("timestamp")]
+    started_at = min(timestamps) if timestamps else datetime.now(UTC).isoformat()
+    completed_at = max(timestamps) if timestamps else datetime.now(UTC).isoformat()
+
+    # Calculate total token usage
+    total_input_tokens = sum(e.get("tokens", {}).get("input", 0) for e in entries)
+    total_output_tokens = sum(e.get("tokens", {}).get("output", 0) for e in entries)
+
+    # Build samples
+    samples = [_build_eval_sample_dict(entry) for entry in entries]
+
+    # Get unique epochs
+    epochs = len(set(e.get("epoch", 0) for e in entries))
+
+    return {
+        "version": 2,
+        "status": "success",
+        "eval": {
+            "eval_id": eval_id,
+            "run_id": str(uuid.uuid4()),
+            "created": started_at,
+            "task": task_name,
+            "task_id": f"{task_name}_{eval_id}",
+            "task_version": 1,
+            "task_file": None,
+            "task_attribs": {},
+            "task_args": {},
+            "task_args_passed": {},
+            "solver": None,
+            "solver_args": None,
+            "dataset": {
+                "name": task_name,
+                "location": None,
+                "samples": len(entries),
+                "shuffled": False,
+            },
+            "sandbox": None,
+            "model": model_name,
+            "model_generate_config": {},
+            "model_base_url": None,
+            "model_args": {},
+            "config": {
+                "epochs": epochs,
+                "log_samples": True,
+            },
+            "revision": None,
+            "packages": {"flashlite": "0.1.0"},
+            "metadata": {"source": "flashlite"},
+        },
+        "plan": {
+            "name": "flashlite",
+            "steps": [],
+            "finish": None,
+            "config": {},
+        },
+        "results": {
+            "total_samples": len(samples),
+            "completed_samples": len(samples),
+            "scores": [],
+        },
+        "stats": {
+            "started_at": started_at,
+            "completed_at": completed_at,
+            "model_usage": {
+                model_name: {
+                    "input_tokens": total_input_tokens,
+                    "output_tokens": total_output_tokens,
+                    "total_tokens": total_input_tokens + total_output_tokens,
+                }
+            },
+        },
+        "error": None,
+        "samples": samples,
+        "reductions": None,
+    }
+
+
+def _build_eval_sample_dict(entry: dict[str, Any]) -> dict[str, Any]:
+    """Build an Inspect-compatible EvalSample dict from a flashlite log entry."""
+    # Convert input messages to ChatMessage format
+    input_messages = entry.get("input", [])
+
+    # Get tokens
+    tokens = entry.get("tokens", {})
+    model_name = entry.get("model", "unknown")
+
+    # Build messages list (input + assistant response)
+    messages = list(input_messages) + [
+        {"role": "assistant", "content": entry.get("output", "")}
+    ]
+
+    return {
+        "id": entry.get("sample_id", 0),
+        "epoch": entry.get("epoch", 0) + 1,  # Inspect uses 1-based epochs
+        "input": input_messages,
+        "choices": None,
+        "target": "",  # flashlite logs don't have targets
+        "sandbox": None,
+        "files": None,
+        "setup": None,
+        "messages": messages,
+        "output": {
+            "model": model_name,
+            "choices": [
+                {
+                    "message": {
+                        "role": "assistant",
+                        "content": entry.get("output", ""),
+                    },
+                    "stop_reason": "stop",
+                }
+            ],
+            "usage": {
+                "input_tokens": tokens.get("input", 0),
+                "output_tokens": tokens.get("output", 0),
+                "total_tokens": tokens.get("total", 0),
+            },
+        },
+        "scores": None,
+        "metadata": entry.get("metadata", {}),
+        "store": {},
+        "events": [],
+        "model_usage": {
+            model_name: {
+                "input_tokens": tokens.get("input", 0),
+                "output_tokens": tokens.get("output", 0),
+                "total_tokens": tokens.get("total", 0),
+            }
+        },
+    }
+
+
+@dataclass
+class InspectLogEntry:
+    """A log entry in Inspect-compatible format."""
+
+    eval_id: str
+    sample_id: str | int
+    epoch: int
+    model: str
+    input: list[dict[str, Any]]
+    output: str
+    tokens: dict[str, int]
+    timestamp: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to Inspect log format."""
+        return {
+            "eval_id": self.eval_id,
+            "sample_id": self.sample_id,
+            "epoch": self.epoch,
+            "model": self.model,
+            "input": self.input,
+            "output": self.output,
+            "tokens": self.tokens,
+            "timestamp": self.timestamp,
+            "metadata": self.metadata,
+        }
+
+
+class InspectLogger:
+    """
+    A logger that outputs in Inspect-compatible format.
+
+    This allows Flashlite logs to be analyzed alongside Inspect eval logs,
+    enabling unified observability across evaluation runs.
+
+    Example:
+        inspect_logger = InspectLogger(
+            log_dir="./logs",
+            eval_id="my-eval-001",
+        )
+
+        # Log a completion
+        inspect_logger.log(
+            request=request,
+            response=response,
+            sample_id="sample_123",
+            epoch=0,
+        )
+
+        # Close when done
+        inspect_logger.close()
+    """
+
+    def __init__(
+        self,
+        log_dir: str | Path,
+        eval_id: str | None = None,
+        append: bool = True,
+    ):
+        """
+        Initialize the Inspect logger.
+
+        Args:
+            log_dir: Directory to write log files
+            eval_id: Evaluation ID (auto-generated if not provided)
+            append: Whether to append to existing log file
+        """
+        self._log_dir = Path(log_dir)
+        self._log_dir.mkdir(parents=True, exist_ok=True)
+
+        self._eval_id = eval_id or self._generate_eval_id()
+        self._log_file = self._log_dir / f"{self._eval_id}.jsonl"
+        self._mode = "a" if append else "w"
+        self._file_handle = open(self._log_file, self._mode)
+        self._sample_count = 0
+
+    def _generate_eval_id(self) -> str:
+        """Generate a unique evaluation ID."""
+        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+        return f"flashlite_eval_{timestamp}"
+
+    def log(
+        self,
+        request: CompletionRequest,
+        response: CompletionResponse,
+        sample_id: str | int | None = None,
+        epoch: int = 0,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Log a request/response pair in Inspect format.
+
+        Args:
+            request: The completion request
+            response: The completion response
+            sample_id: Sample identifier (auto-incremented if not provided)
+            epoch: Epoch number for multi-epoch evals
+            metadata: Additional metadata to include
+        """
+        if sample_id is None:
+            sample_id = self._sample_count
+            self._sample_count += 1
+
+        # Convert messages to Inspect format
+        input_messages = [
+            {"role": msg.get("role", "user"), "content": msg.get("content", "")}
+            for msg in request.messages
+        ]
+
+        entry = InspectLogEntry(
+            eval_id=self._eval_id,
+            sample_id=sample_id,
+            epoch=epoch,
+            model=response.model,
+            input=input_messages,
+            output=response.content,
+            tokens={
+                "input": response.usage.input_tokens if response.usage else 0,
+                "output": response.usage.output_tokens if response.usage else 0,
+                "total": response.usage.total_tokens if response.usage else 0,
+            },
+            timestamp=datetime.now(UTC).isoformat(),
+            metadata=metadata or {},
+        )
+
+        json_str = json.dumps(entry.to_dict())
+        self._file_handle.write(json_str + "\n")
+        self._file_handle.flush()
+
+    def close(self) -> None:
+        """Close the log file."""
+        if self._file_handle:
+            self._file_handle.close()
+
+    @property
+    def eval_id(self) -> str:
+        """Get the evaluation ID."""
+        return self._eval_id
+
+    @property
+    def log_file(self) -> Path:
+        """Get the log file path."""
+        return self._log_file
+
+
+class FlashliteModelAPI:
+    """
+    An adapter that implements a ModelAPI-like interface for Inspect integration.
+
+    This allows Flashlite to be used as a model backend in Inspect evaluations.
+
+    Example:
+        from flashlite import Flashlite
+        from flashlite.observability import FlashliteModelAPI
+
+        # Create Flashlite client
+        client = Flashlite(rate_limit=RateLimitConfig(requests_per_minute=60))
+
+        # Wrap for Inspect
+        model_api = FlashliteModelAPI(client, model="gpt-4o")
+
+        # Use in Inspect eval (pseudocode)
+        # @task
+        # def my_eval():
+        #     return Task(
+        #         dataset=my_dataset,
+        #         solver=my_solver,
+        #         model=model_api,
+        #     )
+    """
+
+    def __init__(
+        self,
+        client: "Flashlite",
+        model: str | None = None,
+        **default_kwargs: Any,
+    ):
+        """
+        Initialize the Inspect model adapter.
+
+        Args:
+            client: The Flashlite client to use
+            model: Default model to use (can be overridden per-request)
+            **default_kwargs: Default parameters for completions
+        """
+        self._client = client
+        self._model = model
+        self._default_kwargs = default_kwargs
+
+    async def generate(
+        self,
+        messages: list[dict[str, Any]],
+        model: str | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Generate a completion (Inspect-compatible interface).
+
+        Args:
+            messages: List of messages
+            model: Model to use (overrides default)
+            **kwargs: Additional parameters
+
+        Returns:
+            Inspect-compatible response dict
+        """
+        # Merge kwargs
+        call_kwargs = {**self._default_kwargs, **kwargs}
+
+        # Call Flashlite
+        response = await self._client.complete(
+            model=model or self._model,
+            messages=messages,
+            **call_kwargs,
+        )
+
+        # Return in Inspect-compatible format
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "role": "assistant",
+                        "content": response.content,
+                    },
+                    "finish_reason": response.finish_reason,
+                }
+            ],
+            "usage": {
+                "prompt_tokens": response.usage.input_tokens if response.usage else 0,
+                "completion_tokens": response.usage.output_tokens if response.usage else 0,
+                "total_tokens": response.usage.total_tokens if response.usage else 0,
+            },
+            "model": response.model,
+        }
+
+    @property
+    def model_name(self) -> str | None:
+        """Get the default model name."""
+        return self._model
+
+
+def convert_logs_cli() -> None:
+    """CLI entry point for converting flashlite logs to Inspect format.
+
+    Usage:
+        python -m flashlite.observability.inspect_compat input.jsonl [output.json]
+    """
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python -m flashlite.observability.inspect_compat <input.jsonl> [output.json]")
+        print("\nConverts flashlite JSONL logs to Inspect-viewable format.")
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2] if len(sys.argv) > 2 else None
+
+    try:
+        result = convert_flashlite_logs_to_inspect(input_path, output_path)
+        print(f"Successfully converted to: {result}")
+        print(f"\nView with: inspect view --log-dir {result.parent}")
+    except ImportError as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    except FileNotFoundError as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error converting logs: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    convert_logs_cli()

{flashlite-0.1.0 → flashlite-0.1.1}/src/flashlite/types.py

@@ -58,7 +58,9 @@ class CompletionRequest:
     """A request to complete a chat conversation."""
 
     model: str
-    messages: Messages
+    messages: Messages = {}
+    template: str | None = None
+    variables: dict[str, Any] | None = None
     temperature: float | None = None
     max_tokens: int | None = None
     max_completion_tokens: int | None = None