timbal 2.2.8__tar.gz → 2.2.9__tar.gz

{timbal-2.2.8 → timbal-2.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: timbal
-Version: 2.2.8
+Version: 2.2.9
 Summary: Simple, performant, battle-tested framework for building reliable AI applications
 Project-URL: Source, https://github.com/timbal-ai/timbal
 Author-email: Timbal <team@timbal.ai>

{timbal-2.2.8 → timbal-2.2.9}/python/timbal/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.2.8'
-__version_tuple__ = version_tuple = (2, 2, 8)
+__version__ = version = '2.2.9'
+__version_tuple__ = version_tuple = (2, 2, 9)
 
 __commit_id__ = commit_id = None

{timbal-2.2.8 → timbal-2.2.9}/python/timbal/codegen/__main__.py

@@ -83,10 +83,42 @@ def main() -> None:
         "--stream", "-s", action="store_true", help="Print every event instead of only the final output event."
     )
 
+    # Evals operation
+    evals_parser = subparsers.add_parser(
+        "evals",
+        help="Run evals against the workspace entry point.",
+    )
+    evals_parser.add_argument(
+        "evals_path",
+        nargs="?",
+        default=None,
+        help="Path to an eval file or directory (defaults to the workspace path). "
+        "Use ::eval_name to run a single eval.",
+    )
+    evals_parser.add_argument(
+        "--tags",
+        default=None,
+        help="Comma-separated tag filter; evals matching ANY tag will run.",
+    )
+    evals_parser.add_argument(
+        "--jobs",
+        "-j",
+        type=int,
+        default=1,
+        help="Run up to N evals concurrently (default 1). Results are emitted as they complete.",
+    )
+    evals_parser.add_argument(
+        "--format",
+        choices=["json", "pretty"],
+        default="json",
+        help="Output format: 'json' (default) streams one JSON event per line to stdout as each eval "
+        "completes; 'pretty' is the rich terminal report.",
+    )
+
     # Defer transformer module loading (pulls in libcst + timbal.codegen which
     # are expensive) — only needed for transformer operations, not for
     # list-tools, get-flow, or test.
-    _lightweight_ops = {"get-models", "get-tools", "get-flow", "test"}
+    _lightweight_ops = {"get-models", "get-tools", "get-flow", "test", "evals"}
     if not (_lightweight_ops & set(sys.argv[1:])):
         from timbal.codegen.transformers import load_modules
 
@@ -230,6 +262,73 @@ def main() -> None:
         asyncio.run(run_test(import_spec, params, run_context=run_context, stream=args.stream))
         return
 
+    if operation == "evals":
+        import asyncio
+        import os
+
+        os.environ.setdefault("TIMBAL_SUPPRESS_EVENTS", "tracing_setup")
+        os.environ["TIMBAL_LOG_LEVEL"] = "CRITICAL"
+
+        from ..logs import setup_logging
+
+        setup_logging()
+
+        from dotenv import load_dotenv
+
+        load_dotenv(override=True)
+
+        from . import parse_fqn
+
+        # The workspace entry point is the default runnable for all evals;
+        # individual evals can still override it with their own 'runnable' key.
+        try:
+            runnable = parse_fqn(workspace_path).load()
+        except (FileNotFoundError, ValueError) as e:
+            print(f"error: {e}", file=sys.stderr)
+            sys.exit(1)
+        except Exception as e:
+            print(f"error: failed to load entry point: {e}", file=sys.stderr)
+            sys.exit(1)
+
+        from timbal.evals.utils import collect_evals
+
+        evals_path_arg = args.evals_path if args.evals_path is not None else str(workspace_path)
+        eval_name = None
+        if "::" in evals_path_arg:
+            evals_path_arg, eval_name = evals_path_arg.rsplit("::", 1)
+        tags = {t.strip() for t in args.tags.split(",")} if args.tags else None
+
+        # Resolve relative eval paths against the workspace (--path), not the
+        # process cwd, so running codegen from outside the workspace works.
+        evals_path = Path(evals_path_arg)
+        if not evals_path.is_absolute():
+            evals_path = workspace_path / evals_path
+
+        try:
+            evals = collect_evals(evals_path, runnable, eval_name, tags)
+        except ValueError as e:
+            print(f"error: {e}", file=sys.stderr)
+            sys.exit(1)
+
+        if not evals:
+            # Warn but keep going, matching the evals CLI: an empty run must
+            # still emit the start/summary events consumers expect.
+            print(f"warning: no evals found in {evals_path}", file=sys.stderr)
+
+        from timbal.evals.runner import run_evals
+
+        if args.format == "pretty":
+            from timbal.evals.reporters import PrettyReporter
+
+            reporter = PrettyReporter()
+        else:
+            from timbal.evals.reporters import JsonReporter
+
+            reporter = JsonReporter()
+
+        summary = asyncio.run(run_evals(evals, reporter=reporter, max_concurrency=args.jobs))
+        sys.exit(0 if summary.failed == 0 else 1)
+
     # Transformer operations — heavy imports already loaded above.
     from timbal.codegen import parse_fqn
     from timbal.codegen.transformers import apply_operation

{timbal-2.2.8 → timbal-2.2.9}/python/timbal/core/llm_router.py

@@ -416,8 +416,9 @@ async def _llm_router(
     # Per-request headers: change every call, so passed via extra_headers on each .create().
     request_headers: dict[str, str] = {
         "x-timbal-run-id": run_context.id,
-        "x-timbal-call-id": call_id,
     }
+    if call_id:
+        request_headers["x-timbal-call-id"] = call_id
     if run_context.platform_config and run_context.platform_config.subject:
         if run_context.platform_config.subject.app_id:
             request_headers["x-timbal-app-id"] = run_context.platform_config.subject.app_id

{timbal-2.2.8 → timbal-2.2.9}/python/timbal/core/models.py

@@ -54,6 +54,9 @@ Model = Literal[
     "anthropic/claude-sonnet-4-5",
     "anthropic/claude-haiku-4-5",
     "openai/gpt-5.5",
+    "openai/gpt-5.6-sol",
+    "openai/gpt-5.6-terra",
+    "openai/gpt-5.6-luna",
     "openai/gpt-5.4",
     "openai/gpt-5.4-mini",
     "openai/gpt-5.4-nano",
@@ -108,6 +111,7 @@ Model = Literal[
     "google/gemini-2.5-flash-lite",
     "google/gemini-2.5-flash-image",
     "google/gemini-2.5-flash-preview-tts",
+    "xai/grok-4.5",
     "xai/grok-4",
     "xai/grok-4-fast-reasoning",
     "xai/grok-4-fast-non-reasoning",
@@ -120,13 +124,17 @@ Model = Literal[
     "groq/openai/gpt-oss-120b",
     "groq/openai/gpt-oss-20b",
     "fireworks/accounts/fireworks/models/deepseek-v4-pro",
+    "fireworks/accounts/fireworks/models/deepseek-v4-flash",
     "fireworks/accounts/fireworks/models/qwen3p6-plus",
+    "fireworks/accounts/fireworks/models/qwen3p7-plus",
     "fireworks/accounts/fireworks/models/kimi-k2p6",
     "fireworks/accounts/fireworks/models/kimi-k2p5",
     "fireworks/accounts/fireworks/models/minimax-m2p5",
+    "fireworks/accounts/fireworks/models/minimax-m2p7",
     "fireworks/accounts/fireworks/models/gpt-oss-120b",
     "fireworks/accounts/fireworks/models/gpt-oss-20b",
     "fireworks/accounts/fireworks/models/glm-5p1",
+    "fireworks/accounts/fireworks/models/glm-5p2",
     "xiaomi/mimo-v2-pro",
     "xiaomi/mimo-v2-omni",
     "xiaomi/mimo-v2-flash",

{timbal-2.2.8 → timbal-2.2.9}/python/timbal/evals/cli.py

@@ -20,7 +20,7 @@ from .. import __version__
 from ..logs import setup_logging
 from ..utils import ImportSpec
 from .runner import run_evals
-from .utils import discover_config, discover_eval_files, parse_eval_file
+from .utils import collect_evals, discover_config, dump_summary
 
 console = Console()
 
@@ -79,6 +79,29 @@ if __name__ == "__main__":
         help="Filter evals by tags. Comma-separated list (e.g., 'smoke,fast'). "
         "Evals matching ANY of the tags will run.",
     )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default=None,
+        help="Write results as JSON to the given file. Use '-' to print JSON to stdout.",
+    )
+    parser.add_argument(
+        "-f",
+        "--format",
+        type=str,
+        choices=["pretty", "json"],
+        default="pretty",
+        help="Output format: 'pretty' (default) is the rich terminal report; "
+        "'json' streams one JSON event per line (JSONL) to stdout as each eval completes.",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=1,
+        help="Run up to N evals concurrently (default 1). Results are reported as they complete.",
+    )
     parser.add_argument(
         "-V",
         "--version",
@@ -87,6 +110,19 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
 
+    if args.format == "json" and args.output == "-":
+        console.print("[red]Error:[/red] --format json already streams JSON to stdout; use -o <file> instead of '-'")
+        sys.exit(1)
+
+    if args.output == "-" or args.format == "json":
+        # Reserve stdout for JSON; route all human-readable output (headers,
+        # eval trees, failures, summary, warnings) to stderr so pipelines can
+        # parse stdout reliably.
+        from . import display
+
+        console.file = sys.stderr
+        display.console.file = sys.stderr
+
     if args.version:
         console.print(f"timbal.eval {__version__}")
         sys.exit(0)
@@ -125,47 +161,50 @@ if __name__ == "__main__":
             console.print(f"[red]Error:[/red] Failed to load runnable: {e}")
             sys.exit(1)
 
-    eval_files = discover_eval_files(path)
-    if not eval_files:
-        console.print(f"[yellow]Warning:[/yellow] No eval files found in {path}")
-        sys.exit(0)
+    tag_filters = {t.strip() for t in args.tags.split(",")} if args.tags else None
+
+    try:
+        evals = collect_evals(path, runnable, eval_name_filter, tag_filters)
+    except ValueError as e:
+        console.print(f"[red]Error:[/red] {e}")
+        sys.exit(1)
 
-    evals = [eval for eval_file in eval_files for eval in parse_eval_file(eval_file, runnable)]
     if not evals:
-        console.print(f"[yellow]Warning:[/yellow] No evals found in {len(eval_files)} file(s)")
-        sys.exit(0)
+        # Warn but keep going: an empty run must still produce the JSON
+        # events/document that -o and --format json consumers expect.
+        suffix = f" matching tags: {', '.join(sorted(tag_filters))}" if tag_filters else ""
+        console.print(f"[yellow]Warning:[/yellow] No evals found in {path}{suffix}")
 
-    # Check for duplicate eval names
-    seen_names: dict[str, Path] = {}
-    for e in evals:
-        if e.name in seen_names:
-            console.print(
-                f"[red]Error:[/red] Duplicate eval name '{e.name}' found in:\n  - {seen_names[e.name]}\n  - {e.path}"
-            )
-            sys.exit(1)
-        seen_names[e.name] = e.path
+    if args.format == "json":
+        from .reporters import JsonReporter
 
-    # Filter by eval name if specified
-    if eval_name_filter:
-        evals = [e for e in evals if e.name == eval_name_filter]
-        if not evals:
-            console.print(f"[red]Error:[/red] No eval found with name '{eval_name_filter}'")
-            sys.exit(1)
+        reporter = JsonReporter()
+    else:
+        from .reporters import PrettyReporter
 
-    # Filter by tags if specified
-    if args.tags:
-        tag_filters = {t.strip() for t in args.tags.split(",")}
-        evals = [e for e in evals if tag_filters & set(e.tags)]
-        if not evals:
-            console.print(f"[yellow]Warning:[/yellow] No evals found matching tags: {', '.join(sorted(tag_filters))}")
-            sys.exit(0)
+        reporter = PrettyReporter()
 
     summary = asyncio.run(
         run_evals(
             evals,
             capture=not args.no_capture,
+            reporter=reporter,
+            max_concurrency=args.jobs,
         )
     )
 
+    if args.output:
+        import json
+
+        summary_dict = asyncio.run(dump_summary(summary))
+        summary_json = json.dumps(summary_dict, indent=2, default=str)
+        if args.output == "-":
+            sys.stdout.write(summary_json + "\n")
+        else:
+            output_path = Path(args.output).expanduser()
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            output_path.write_text(summary_json)
+            console.print(f"[dim]Results written to {output_path}[/dim]")
+
     # Exit with error code if any eval failed
     sys.exit(0 if summary.failed == 0 else 1)

timbal-2.2.9/python/timbal/evals/reporters.py

@@ -0,0 +1,80 @@
+import json
+import sys
+from typing import Any, TextIO
+
+from .models import Eval, EvalResult, EvalSummary
+from .utils import dump_result
+
+
+class Reporter:
+    """Receives eval lifecycle callbacks.
+
+    `result()` is called as each eval completes — in file order for sequential
+    runs, in completion order for parallel runs.
+    """
+
+    def start(self, evals: list[Eval]) -> None:
+        pass
+
+    async def result(self, result: EvalResult) -> None:
+        pass
+
+    def finish(self, summary: EvalSummary) -> None:
+        pass
+
+
+class PrettyReporter(Reporter):
+    """Human-readable Rich output (the default CLI experience)."""
+
+    def __init__(self) -> None:
+        from . import display  # deferred: pulls in rich
+
+        self._display = display
+
+    def start(self, evals: list[Eval]) -> None:
+        self._display.print_header(evals)
+
+    async def result(self, result: EvalResult) -> None:
+        self._display.print_eval_result(result)
+
+    def finish(self, summary: EvalSummary) -> None:
+        self._display.print_failures(summary.results)
+        self._display.print_summary(summary)
+
+
+class JsonReporter(Reporter):
+    """Streams one JSON object per line (JSONL) as evals complete.
+
+    Events:
+        {"event": "start", "total": N, "evals": [{"name", "path"}, ...]}
+        {"event": "result", ...}   # one per eval, in completion order
+        {"event": "summary", "total", "passed", "failed", "total_duration"}
+    """
+
+    def __init__(self, stream: TextIO | None = None) -> None:
+        # Grab the stream reference eagerly: output capture swaps sys.stdout
+        # globally while evals run, and events must reach the real stream.
+        self._stream = stream if stream is not None else sys.stdout
+
+    def _emit(self, obj: dict[str, Any]) -> None:
+        self._stream.write(json.dumps(obj, default=str) + "\n")
+        self._stream.flush()
+
+    def start(self, evals: list[Eval]) -> None:
+        self._emit({
+            "event": "start",
+            "total": len(evals),
+            "evals": [{"name": e.name, "path": str(e.path)} for e in evals],
+        })
+
+    async def result(self, result: EvalResult) -> None:
+        self._emit({"event": "result", **await dump_result(result)})
+
+    def finish(self, summary: EvalSummary) -> None:
+        self._emit({
+            "event": "summary",
+            "total": summary.total,
+            "passed": summary.passed,
+            "failed": summary.failed,
+            "total_duration": summary.total_duration,
+        })