flashlite 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

flashlite/observability/__init__.py

@@ -7,7 +7,12 @@ from .callbacks import (
     OnResponseCallback,
     create_logging_callbacks,
 )
-from .inspect_compat import FlashliteModelAPI, InspectLogEntry, InspectLogger
+from .inspect_compat import (
+    FlashliteModelAPI,
+    InspectLogEntry,
+    InspectLogger,
+    convert_flashlite_logs_to_inspect,
+)
 from .logging import RequestContext, RequestLogEntry, ResponseLogEntry, StructuredLogger
 from .metrics import BudgetExceededError, CostMetrics, CostTracker
 
@@ -31,4 +36,5 @@ __all__ = [
     "InspectLogger",
     "InspectLogEntry",
     "FlashliteModelAPI",
+    "convert_flashlite_logs_to_inspect",
 ]

flashlite/observability/inspect_compat.py

@@ -4,13 +4,14 @@ This module provides interoperability with the UK AISI's Inspect framework
 (https://inspect.ai-safety-institute.org.uk/).
 
 It includes:
-- Log format compatible with Inspect's eval logging
+- Log format conversion to Inspect's native eval log format
 - ModelAPI protocol implementation for use as an Inspect solver backend
-- Hooks for Inspect's TaskState integration
+- Functions to convert flashlite JSONL logs to Inspect-viewable format
 """
 
 import json
 import logging
+import uuid
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
@@ -24,6 +25,231 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+def convert_flashlite_logs_to_inspect(
+    input_path: str | Path,
+    output_path: str | Path | None = None,
+    task_name: str | None = None,
+) -> Path:
+    """
+    Convert flashlite JSONL logs to Inspect-compatible format.
+
+    This allows logs generated by flashlite's InspectLogger to be viewed
+    in Inspect's log viewer (`inspect view`).
+
+    Args:
+        input_path: Path to flashlite JSONL log file
+        output_path: Output path for Inspect log file (defaults to same dir with proper naming)
+        task_name: Task name for the evaluation (defaults to eval_id from logs)
+
+    Returns:
+        Path to the generated Inspect log file
+
+    Raises:
+        FileNotFoundError: If input file doesn't exist
+
+    Example:
+        >>> from flashlite.observability import convert_flashlite_logs_to_inspect
+        >>> convert_flashlite_logs_to_inspect("logs/my_eval.jsonl")
+        PosixPath('logs/2026-02-05T12-00-00_my_eval_abc123.json')
+    """
+    input_path = Path(input_path)
+    if not input_path.exists():
+        raise FileNotFoundError(f"Log file not found: {input_path}")
+
+    # Read all entries from JSONL
+    entries: list[dict[str, Any]] = []
+    with open(input_path) as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                entries.append(json.loads(line))
+
+    if not entries:
+        raise ValueError(f"No log entries found in {input_path}")
+
+    # Extract metadata from first entry
+    first_entry = entries[0]
+    eval_id = first_entry.get("eval_id", "flashlite_eval")
+    model_name = first_entry.get("model", "unknown")
+    task = task_name or eval_id
+
+    # Get timestamp from entries or generate one
+    timestamps = [e.get("timestamp", "") for e in entries if e.get("timestamp")]
+    if timestamps:
+        # Parse and format for filename (Inspect uses format like 2024-05-29T12-38-43)
+        started_at = min(timestamps)
+        # Convert ISO format to Inspect's filename format
+        ts_for_filename = started_at.replace(":", "-").split(".")[0]
+    else:
+        ts_for_filename = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S")
+
+    # Generate a short unique ID
+    short_id = uuid.uuid4().hex[:8]
+
+    # Determine output path with Inspect's naming convention: {timestamp}_{task}_{id}.json
+    if output_path is None:
+        output_dir = input_path.parent
+        output_filename = f"{ts_for_filename}_{task}_{short_id}.json"
+        output_path = output_dir / output_filename
+    else:
+        output_path = Path(output_path)
+
+    # Build EvalLog structure as dict (Inspect's JSON format)
+    eval_log = _build_eval_log_dict(
+        entries=entries,
+        eval_id=eval_id,
+        task_name=task,
+        model_name=model_name,
+    )
+
+    # Write JSON directly
+    with open(output_path, "w") as f:
+        json.dump(eval_log, f, indent=2)
+
+    logger.info(f"Converted {len(entries)} entries to Inspect format: {output_path}")
+    return output_path
+
+
+def _build_eval_log_dict(
+    entries: list[dict[str, Any]],
+    eval_id: str,
+    task_name: str,
+    model_name: str,
+) -> dict[str, Any]:
+    """Build an Inspect-compatible EvalLog dict from flashlite log entries."""
+    # Calculate timestamps
+    timestamps = [e.get("timestamp", "") for e in entries if e.get("timestamp")]
+    started_at = min(timestamps) if timestamps else datetime.now(UTC).isoformat()
+    completed_at = max(timestamps) if timestamps else datetime.now(UTC).isoformat()
+
+    # Calculate total token usage
+    total_input_tokens = sum(e.get("tokens", {}).get("input", 0) for e in entries)
+    total_output_tokens = sum(e.get("tokens", {}).get("output", 0) for e in entries)
+
+    # Build samples
+    samples = [_build_eval_sample_dict(entry) for entry in entries]
+
+    # Get unique epochs
+    epochs = len(set(e.get("epoch", 0) for e in entries))
+
+    return {
+        "version": 2,
+        "status": "success",
+        "eval": {
+            "eval_id": eval_id,
+            "run_id": str(uuid.uuid4()),
+            "created": started_at,
+            "task": task_name,
+            "task_id": f"{task_name}_{eval_id}",
+            "task_version": 1,
+            "task_file": None,
+            "task_attribs": {},
+            "task_args": {},
+            "task_args_passed": {},
+            "solver": None,
+            "solver_args": None,
+            "dataset": {
+                "name": task_name,
+                "location": None,
+                "samples": len(entries),
+                "shuffled": False,
+            },
+            "sandbox": None,
+            "model": model_name,
+            "model_generate_config": {},
+            "model_base_url": None,
+            "model_args": {},
+            "config": {
+                "epochs": epochs,
+                "log_samples": True,
+            },
+            "revision": None,
+            "packages": {"flashlite": "0.1.0"},
+            "metadata": {"source": "flashlite"},
+        },
+        "plan": {
+            "name": "flashlite",
+            "steps": [],
+            "finish": None,
+            "config": {},
+        },
+        "results": {
+            "total_samples": len(samples),
+            "completed_samples": len(samples),
+            "scores": [],
+        },
+        "stats": {
+            "started_at": started_at,
+            "completed_at": completed_at,
+            "model_usage": {
+                model_name: {
+                    "input_tokens": total_input_tokens,
+                    "output_tokens": total_output_tokens,
+                    "total_tokens": total_input_tokens + total_output_tokens,
+                }
+            },
+        },
+        "error": None,
+        "samples": samples,
+        "reductions": None,
+    }
+
+
+def _build_eval_sample_dict(entry: dict[str, Any]) -> dict[str, Any]:
+    """Build an Inspect-compatible EvalSample dict from a flashlite log entry."""
+    # Convert input messages to ChatMessage format
+    input_messages = entry.get("input", [])
+
+    # Get tokens
+    tokens = entry.get("tokens", {})
+    model_name = entry.get("model", "unknown")
+
+    # Build messages list (input + assistant response)
+    messages = list(input_messages) + [
+        {"role": "assistant", "content": entry.get("output", "")}
+    ]
+
+    return {
+        "id": entry.get("sample_id", 0),
+        "epoch": entry.get("epoch", 0) + 1,  # Inspect uses 1-based epochs
+        "input": input_messages,
+        "choices": None,
+        "target": "",  # flashlite logs don't have targets
+        "sandbox": None,
+        "files": None,
+        "setup": None,
+        "messages": messages,
+        "output": {
+            "model": model_name,
+            "choices": [
+                {
+                    "message": {
+                        "role": "assistant",
+                        "content": entry.get("output", ""),
+                    },
+                    "stop_reason": "stop",
+                }
+            ],
+            "usage": {
+                "input_tokens": tokens.get("input", 0),
+                "output_tokens": tokens.get("output", 0),
+                "total_tokens": tokens.get("total", 0),
+            },
+        },
+        "scores": None,
+        "metadata": entry.get("metadata", {}),
+        "store": {},
+        "events": [],
+        "model_usage": {
+            model_name: {
+                "input_tokens": tokens.get("input", 0),
+                "output_tokens": tokens.get("output", 0),
+                "total_tokens": tokens.get("total", 0),
+            }
+        },
+    }
+
+
 @dataclass
 class InspectLogEntry:
     """A log entry in Inspect-compatible format."""
@@ -264,3 +490,38 @@ class FlashliteModelAPI:
     def model_name(self) -> str | None:
         """Get the default model name."""
         return self._model
+
+
+def convert_logs_cli() -> None:
+    """CLI entry point for converting flashlite logs to Inspect format.
+
+    Usage:
+        python -m flashlite.observability.inspect_compat input.jsonl [output.json]
+    """
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python -m flashlite.observability.inspect_compat <input.jsonl> [output.json]")
+        print("\nConverts flashlite JSONL logs to Inspect-viewable format.")
+        sys.exit(1)
+
+    input_path = sys.argv[1]
+    output_path = sys.argv[2] if len(sys.argv) > 2 else None
+
+    try:
+        result = convert_flashlite_logs_to_inspect(input_path, output_path)
+        print(f"Successfully converted to: {result}")
+        print(f"\nView with: inspect view --log-dir {result.parent}")
+    except ImportError as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    except FileNotFoundError as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error converting logs: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    convert_logs_cli()

flashlite/types.py

@@ -58,7 +58,9 @@ class CompletionRequest:
     """A request to complete a chat conversation."""
 
     model: str
-    messages: Messages
+    messages: Messages = field(default_factory=list)
+    template: str | None = None
+    variables: dict[str, Any] | None = None
     temperature: float | None = None
     max_tokens: int | None = None
     max_completion_tokens: int | None = None

{flashlite-0.1.0.dist-info → flashlite-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flashlite
-Version: 0.1.0
+Version: 0.1.2
 Summary: Batteries-included wrapper for litellm with rate limiting, retries, templating, and more
 Author-email: ndalton12 <niall.dalton12@gmail.com>
 License-File: LICENSE.md
@@ -16,6 +16,8 @@ Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
 Requires-Dist: pytest>=8.0.0; extra == 'dev'
 Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: evals
+Requires-Dist: inspect-ai>=0.3.0; extra == 'evals'
 Description-Content-Type: text/markdown
 
 # Flashlite

{flashlite-0.1.0.dist-info → flashlite-0.1.2.dist-info}/RECORD

@@ -2,7 +2,7 @@ flashlite/__init__.py,sha256=RlXjsK7zvZXStMvfz4FGqBxTWHev9VkyHYy-35TuTuM,3585
 flashlite/client.py,sha256=zQH_eLWZxnkX9acwI-y9c3uxeGybA-C0I9UPU6HrzvI,25081
 flashlite/config.py,sha256=3RMEIAejBPlBG_VOgD8mpZKEDNZvK0k0cVv3vMM9kW8,4818
 flashlite/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-flashlite/types.py,sha256=Odh18srphi8KhE8DSg9TW4nDCy_mNhFZowEjarirz3o,6392
+flashlite/types.py,sha256=OqtgtnZaEWRYfaI6esqzv4-HKwu76Y-RDu3-Mhsae9w,6498
 flashlite/cache/__init__.py,sha256=T8O7oiZ0U181_tacJzfK6IGEAt1m3NdaIlBjq9wmB44,325
 flashlite/cache/base.py,sha256=IaDAI4EzewhJe0quh2JQK9-BxQxGxUDwrsd9BCaHFFc,5663
 flashlite/cache/disk.py,sha256=pGPI7eJW6RqVCQC4laTYhQr0iU-AkjA4aFFYt-wg8ls,8777
@@ -20,9 +20,9 @@ flashlite/middleware/cache.py,sha256=R1YwAZBg5YJGTiqgNWdkl7VSN1xpmqmupTSBQnpyH-s
 flashlite/middleware/logging.py,sha256=D3x8X1l1LN1Um_qOWuELyO8Fgo9WulFJTIx6s94Ure4,4919
 flashlite/middleware/rate_limit.py,sha256=nf0-Ul0CGnX0VRKtxB2dfoplkBin3P2cMLrbks76lcg,7059
 flashlite/middleware/retry.py,sha256=_3Lz9Gmes2sNk6rO10WamH6yrwJy8TQi-esIl8NIMag,4832
-flashlite/observability/__init__.py,sha256=5896gbmuAY3qHZzcu9UwsrGd9Umq2AkaUrufX-72Epo,852
+flashlite/observability/__init__.py,sha256=VHdYteU9KmVkgSHrkA-Ssz6_qoi9uL-2JFDhSH5sgwI,949
 flashlite/observability/callbacks.py,sha256=yz1oZh7f7WVxvKmt7XyHbj4WDC2xnvM3SJiTSxfAkoQ,4897
-flashlite/observability/inspect_compat.py,sha256=1bEowfjBdkvbo7nmWXYJMw9mRIfHSKaT2Cj9zBynRRA,7812
+flashlite/observability/inspect_compat.py,sha256=IrsdEiV-qn_wOlgAvWLcIJ_7WxU0Bpq7DcHaS_KWXPw,16366
 flashlite/observability/logging.py,sha256=UxBH2RN8rNcGZHYgC_QYiuEpaIRXEQFs1OjiKjxbuf0,9273
 flashlite/observability/metrics.py,sha256=blRx5N3uN4ilnPpxBe7k_uDhYV3GmQWXoKPLVxnk8_s,7466
 flashlite/structured/__init__.py,sha256=9k5bwkzFo_JD3WZ1Tm4iyZqoZ1A51EIINI8N1H2_2ew,750
@@ -35,7 +35,7 @@ flashlite/templating/registry.py,sha256=wp8RaibHKNyu5q4tCdOXJ0B4tey7bv-c0qb9h1a7
 flashlite/tools/__init__.py,sha256=zpQ5KyvZwZaVvaulnpMmL_JjCnMfD08nD_foI95TjVg,1791
 flashlite/tools/definitions.py,sha256=cqyk6GR1qeMkTPFqsadnJc-YkCG15QVafiaf-OjGYNU,11519
 flashlite/tools/execution.py,sha256=iQC7V3R5Tx19suISnnuaDpjpgl8wURwOHmKZbsHL16s,10814
-flashlite-0.1.0.dist-info/METADATA,sha256=Wmpytj9YfLQpYccBCJ9po6Jx_bCD51z4-7VRC6LFcgk,4220
-flashlite-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-flashlite-0.1.0.dist-info/licenses/LICENSE.md,sha256=z2KZcyoH16ayjxlbeBM01uD-bXn1WTcKFab5ZKBhfJE,1068
-flashlite-0.1.0.dist-info/RECORD,,
+flashlite-0.1.2.dist-info/METADATA,sha256=vWQl0DuuE16hbq9n1lLRL8ASCgxwBrHZsuibi8YD-u4,4293
+flashlite-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+flashlite-0.1.2.dist-info/licenses/LICENSE.md,sha256=z2KZcyoH16ayjxlbeBM01uD-bXn1WTcKFab5ZKBhfJE,1068
+flashlite-0.1.2.dist-info/RECORD,,