10xscale-agentflow-cli 0.3.2.5__tar.gz → 0.3.2.7__tar.gz

{10xscale_agentflow_cli-0.3.2.5 → 10xscale_agentflow_cli-0.3.2.7}/10xscale_agentflow_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: 10xscale-agentflow-cli
-Version: 0.3.2.5
+Version: 0.3.2.7
 Summary: CLI and API for 10xscale AgentFlow
 Author-email: 10xscale <contact@10xscale.ai>
 Maintainer-email: Shudipto Trafder <shudiptotrafder@gmail.com>

{10xscale_agentflow_cli-0.3.2.5 → 10xscale_agentflow_cli-0.3.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: 10xscale-agentflow-cli
-Version: 0.3.2.5
+Version: 0.3.2.7
 Summary: CLI and API for 10xscale AgentFlow
 Author-email: 10xscale <contact@10xscale.ai>
 Maintainer-email: Shudipto Trafder <shudiptotrafder@gmail.com>

{10xscale_agentflow_cli-0.3.2.5 → 10xscale_agentflow_cli-0.3.2.7}/agentflow_cli/cli/commands/eval.py

@@ -15,11 +15,17 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from agentflow.qa.evaluation import CriterionConfig, EvalConfig, MatchType
-from agentflow.qa.evaluation.collectors.trajectory_collector import TrajectoryCollector
+from agentflow.qa.evaluation.collectors.trajectory_collector import (
+    TrajectoryCollector,
+    make_trajectory_callback,
+)
 from agentflow.qa.evaluation.config.eval_config import ReporterConfig
-from agentflow.qa.evaluation.eval_result import EvalReport as ER, EvalCaseResult
+from agentflow.qa.evaluation.eval_result import EvalCaseResult
+from agentflow.qa.evaluation.eval_result import EvalReport as ER
 from agentflow.qa.evaluation.evaluator import AgentEvaluator
 from agentflow.qa.evaluation.reporters.manager import ReporterManager
+from agentflow.utils.callbacks import CallbackManager
+from injectq.testing import override_dependency
 
 from agentflow_cli.cli.commands import BaseCommand
 from agentflow_cli.cli.core.config import ConfigManager
@@ -48,6 +54,32 @@ class _PendingSimulation:
     eval_set_name: str
 
 
+def _reset_inject_proxy(service_type: type) -> None:
+    """Reset the lazy Inject[T] proxy cached on Node.execute's default parameters.
+
+    Inject[T] proxies cache their resolved value in _injected_value on first use.
+    container.override() clears the singleton scope, but the proxy's own cache is
+    independent — it must be cleared so the proxy re-resolves from the container
+    on next invocation instead of returning the stale value.
+    """
+    try:
+        from agentflow.core.graph.node import Node as _GraphNode
+
+        defaults = getattr(_GraphNode.execute, "__defaults__", None) or ()
+        for d in defaults:
+            if (
+                hasattr(d, "_injected_value")
+                and hasattr(d, "service_type")
+                and d.service_type is service_type
+            ):
+                d._injected_value = None
+    except Exception:  # nosec B110  # noqa: S110
+        pass
+
+
+DEFAULT_CONCURRENCY = 4
+
+
 class EvalCommand(BaseCommand):
     """Discover and run agent evaluations; always write HTML + JSON reports."""
 
@@ -112,8 +144,44 @@ class EvalCommand(BaseCommand):
             },
         )
 
+    # ------------------------------------------------------------------
+    # confeval.py loading
+    # ------------------------------------------------------------------
+
+    def _load_confeval(self, evals_dir: Path) -> EvalConfig | None:
+        """Find and load confeval.py from evals_dir, returning its EvalConfig.
+
+        Tries get_eval_config() first, then falls back to the EVAL_CONFIG
+        constant.  Returns None if no confeval.py is found so callers can
+        distinguish "no confeval" from "confeval with empty criteria".
+        """
+        confeval_path = evals_dir / "confeval.py"
+        if not confeval_path.exists():
+            return None
+
+        try:
+            mod = self._load_module(confeval_path)
+        except Exception as exc:
+            self.logger.warning("Failed to load confeval.py: %s", exc)
+            return None
+
+        if hasattr(mod, "get_eval_config"):
+            try:
+                return mod.get_eval_config()
+            except Exception as exc:
+                self.logger.warning("get_eval_config() in confeval.py failed: %s", exc)
+
+        if hasattr(mod, "EVAL_CONFIG"):
+            return mod.EVAL_CONFIG
+
+        self.logger.warning(
+            "confeval.py found but has no get_eval_config() or EVAL_CONFIG — ignoring"
+        )
+        return None
+
     def _collect_eval_functions(self, mod: Any) -> tuple[list[tuple[str, Any]], Any]:
-        """Pytest-style discovery: functions annotated -> EvalSet are evals, -> EvalConfig is config."""
+        """Pytest-style discovery: functions annotated -> EvalSet are evals, ->
+        EvalConfig is config."""
         from agentflow.qa.evaluation import EvalConfig, EvalSet
 
         eval_pairs: list[tuple[str, Any]] = []
@@ -150,10 +218,15 @@ class EvalCommand(BaseCommand):
     # ------------------------------------------------------------------
 
     def _collect_from_file(
-        self, path: Path, global_config: EvalConfig
+        self, path: Path, global_config: EvalConfig | None
     ) -> list[_PendingCase | _PendingSimulation]:
         """Load a module and return pending work for every eval case or simulation scenario.
 
+        Config priority chain (highest → lowest):
+          1. confeval.py   — global_config (not None when confeval was found)
+          2. Per-file      — EVAL_CONFIG / get_eval_config() inside the file
+          3. Built-in defaults — _default_config()
+
         Returns _PendingSimulation items when the file exposes get_scenarios() or SCENARIOS.
         Returns _PendingCase items for the standard get_eval_set() / pytest-style protocols.
         """
@@ -175,13 +248,13 @@ class EvalCommand(BaseCommand):
 
         # get_eval_set() protocol
         if hasattr(mod, "get_eval_set"):
+            file_config: EvalConfig | None = None
             if hasattr(mod, "get_eval_config"):
                 file_config = mod.get_eval_config()
             elif hasattr(mod, "EVAL_CONFIG"):
                 file_config = mod.EVAL_CONFIG
-            else:
-                file_config = global_config
-            config = global_config if global_config.criteria else file_config
+            # Priority: confeval > per-file > defaults
+            config = global_config or file_config or self._default_config()
             return self._make_pending(mod, mod.get_eval_set(), config, file_name)
 
         # pytest-style discovery
@@ -192,9 +265,10 @@ class EvalCommand(BaseCommand):
                 if hasattr(mod, "get_eval_config")
                 else mod.EVAL_CONFIG
                 if hasattr(mod, "EVAL_CONFIG")
-                else global_config
+                else None
             )
-            config = global_config if global_config.criteria else file_config
+            # Priority: confeval > per-file > defaults
+            config = global_config or file_config or self._default_config()
             pending: list[_PendingCase] = []
             for _, es in eval_pairs:
                 pending.extend(self._make_pending(mod, es, config, file_name))
@@ -262,6 +336,38 @@ class EvalCommand(BaseCommand):
             for c in eval_set.eval_cases
         ]
 
+    # ------------------------------------------------------------------
+    # Criteria display
+    # ------------------------------------------------------------------
+
+    def _print_criteria_block(
+        self,
+        confeval_config: EvalConfig | None,
+        target_path: Path,
+    ) -> None:
+        """Print the active criteria and their source before any case runs."""
+        if confeval_config is not None:
+            source = str(target_path / "confeval.py")
+            criteria = confeval_config.criteria
+        else:
+            source = "built-in defaults"
+            criteria = self._default_config().criteria
+
+        print(f"Criteria  source: {source}", flush=True)  # noqa: T201
+        for name in criteria.model_fields:
+            cfg = getattr(criteria, name)
+            if cfg is None:
+                continue
+
+            parts = [f"threshold={cfg.threshold}"]
+            parts.append(cfg.match_type.value)
+            if cfg.judge_model:
+                parts.append(f"judge={cfg.judge_model}")
+            if cfg.num_samples and cfg.num_samples != 1:
+                parts.append(f"samples={cfg.num_samples}")
+            print(f"  {name:<40} {'  '.join(parts)}", flush=True)  # noqa: T201
+        print("", flush=True)  # noqa: T201
+
     # ------------------------------------------------------------------
     # Progress printing
     # ------------------------------------------------------------------
@@ -277,10 +383,12 @@ class EvalCommand(BaseCommand):
         status = "PASSED" if result.passed else ("ERROR" if result.is_error else "FAILED")
         duration = f"{result.duration_seconds:.2f}s"
         label = f"{file_name}::{case_name}"
-        status_colored = (
-            f"\033[32m{status}\033[0m" if result.passed else f"\033[31m{status}\033[0m"
-        )
-        print(f"[{index:3d}/{total}] {label}  {status_colored}  {duration}", flush=True)
+        status_colored = f"\033[32m{status}\033[0m" if result.passed else f"\033[31m{status}\033[0m"
+        tok = getattr(result, "token_usage", None)
+        tok_str = ""
+        if tok and (tok.input_tokens or tok.output_tokens):
+            tok_str = f"  in={tok.input_tokens} out={tok.output_tokens}"
+        print(f"[{index:3d}/{total}] {label}  {status_colored}  {duration}{tok_str}", flush=True)  # noqa: T201
 
     # ------------------------------------------------------------------
     # Flat pool execution — single asyncio event loop for all cases
@@ -303,24 +411,34 @@ class EvalCommand(BaseCommand):
             local_collector = TrajectoryCollector(
                 capture_all_events=pc.evaluator.collector.capture_all_events,
             )
-            try:
-                result = await pc.evaluator._evaluate_case(
-                    pc.case, collector_override=local_collector
-                )
-            except Exception as exc:
-                result = EvalCaseResult.failure(
-                    eval_id=pc.case.eval_id,
-                    error=str(exc),
-                    name=pc.case.name,
-                )
+            _, local_mgr = make_trajectory_callback(local_collector)
+            container = pc.evaluator.graph._state_graph._container
+            # The Inject[CallbackManager] proxy on Node.execute caches its resolved
+            # value independently of the container — reset it so it re-resolves
+            # inside the override context instead of returning the stale instance.
+            _reset_inject_proxy(CallbackManager)
+            with override_dependency(CallbackManager, local_mgr, container=container):
+                try:
+                    result = await pc.evaluator._evaluate_case(
+                        pc.case, collector_override=local_collector
+                    )
+                except Exception as exc:
+                    result = EvalCaseResult.failure(
+                        eval_id=pc.case.eval_id,
+                        error=str(exc),
+                        name=pc.case.name,
+                    )
             return (pc.file_name, pc.eval_set_id, pc.eval_set_name, result)
 
         async def _run_simulation(ps: _PendingSimulation) -> tuple[str, str, str, EvalCaseResult]:
             from agentflow.qa.evaluation.eval_result import CriterionResult
+            from agentflow.qa.evaluation.token_usage import TokenUsage
 
             try:
                 sim_result = await ps.simulator.run(ps.graph, ps.scenario)
-                criterion_results = [
+                # Use the full CriterionResult objects (with per-criterion token_usage)
+                # if available; fall back to rebuilding from scores dict.
+                criterion_results = sim_result.criterion_results or [
                     CriterionResult.success(
                         criterion=name,
                         score=score,
@@ -342,14 +460,19 @@ class EvalCommand(BaseCommand):
                         )
                     ]
                 conversation_text = "\n".join(
-                    f"{m['role'].upper()}: {m['content']}"
-                    for m in sim_result.conversation
+                    f"{m['role'].upper()}: {m['content']}" for m in sim_result.conversation
+                )
+                criteria_token_usage = sum(
+                    (cr.token_usage for cr in criterion_results), TokenUsage()
                 )
+                total_token_usage = sim_result.simulator_token_usage + criteria_token_usage
                 result = EvalCaseResult.success(
                     eval_id=ps.scenario.scenario_id,
                     name=ps.scenario.description or ps.scenario.scenario_id,
                     criterion_results=criterion_results,
                     actual_response=conversation_text,
+                    token_usage=total_token_usage,
+                    agent_token_usage=sim_result.simulator_token_usage,
                     metadata={
                         "turns": sim_result.turns,
                         "goals_achieved": sim_result.goals_achieved,
@@ -457,10 +580,13 @@ class EvalCommand(BaseCommand):
         quiet: bool = False,
         **kwargs: Any,
     ) -> int:
-        # 1. Load global config from agentflow.json
-        global_eval_cfg: dict[str, Any] = {}
+        # 1. Load infra settings from agentflow.json — no criteria here.
+        # Criteria come exclusively from confeval.py (step 4) or per-file config.
         config_manager = ConfigManager()
         discovered = config_manager.auto_discover_config()
+        json_directory = "evals"
+        effective_parallel = parallel
+        effective_concurrency = max_concurrency
         if discovered:
             try:
                 config_manager.load_config(str(discovered))
@@ -469,33 +595,19 @@ class EvalCommand(BaseCommand):
                     output_dir = global_eval_cfg.get("output_dir", output_dir)
                 if threshold is None:
                     threshold = global_eval_cfg.get("threshold")
+                if not parallel:
+                    effective_parallel = global_eval_cfg.get("parallel", False)
+                if max_concurrency == DEFAULT_CONCURRENCY:
+                    effective_concurrency = global_eval_cfg.get(
+                        "max_concurrency", DEFAULT_CONCURRENCY
+                    )
+                json_directory = global_eval_cfg.get("directory", json_directory)
             except Exception:
                 self.logger.warning(
                     "Failed to load eval config from agentflow.json; using defaults"
                 )
 
-        # 2. Build typed EvalConfig; CLI flags override everything.
-        # If agentflow.json has an evaluation section but no criteria key, inject
-        # the built-in defaults so at least one criterion always runs.
-        try:
-            global_config = (
-                EvalConfig.model_validate(global_eval_cfg)
-                if global_eval_cfg
-                else self._default_config()
-            )
-        except Exception:
-            global_config = self._default_config()
-
-        if not global_config.criteria:
-            global_config.criteria = self._default_config().criteria
-
-        if parallel:
-            global_config.parallel = True
-        if max_concurrency != 4:
-            global_config.max_concurrency = max_concurrency
-
-        effective_parallel = global_config.parallel
-        effective_concurrency = global_config.max_concurrency
+        # 2. CLI flags already captured as function parameters (highest priority).
 
         # 3. Resolve target path
         if target:
@@ -504,7 +616,7 @@ class EvalCommand(BaseCommand):
                 self.output.error(f"Path not found: {target}")
                 return 1
         else:
-            target_path = self._resolve_eval_dir()
+            target_path = Path.cwd() / json_directory
             if not target_path.exists():
                 self.output.error(
                     f"Eval directory '{target_path}' not found. "
@@ -512,17 +624,20 @@ class EvalCommand(BaseCommand):
                 )
                 return 1
 
-        # 4. Discover files
+        # 4. Load confeval.py → global criteria config (None if not found)
+        confeval_config = self._load_confeval(target_path)
+
+        # 5. Discover files
         files = self._discover(target_path)
         if not files:
             self.output.error(f"No eval files found in {target_path}")
             return 1
 
-        # 5. Collect all pending cases across every file
+        # 6. Collect all pending cases across every file
         pending: list[_PendingCase] = []
         for f in files:
             try:
-                cases = self._collect_from_file(f, global_config)
+                cases = self._collect_from_file(f, confeval_config)
                 pending.extend(cases)
             except Exception as exc:
                 self.output.error(f"Error loading {f.name}: {exc}")
@@ -548,22 +663,21 @@ class EvalCommand(BaseCommand):
             f"Found {', '.join(parts)} across {n_files} file(s) in {target_path}",
         )
 
-        # 6. Run all cases under a single asyncio event loop
-        quads = asyncio.run(
-            self._run_flat_pool(pending, effective_concurrency, effective_parallel)
-        )
+        # 7. Print criteria block so users know what is being evaluated
+        self._print_criteria_block(confeval_config, target_path)
+
+        # 8. Run all cases under a single asyncio event loop
+        quads = asyncio.run(self._run_flat_pool(pending, effective_concurrency, effective_parallel))
 
         if not quads:
             self.output.error("No results produced.")
             return 1
 
         # 7. Group by eval_set_id → one EvalReport per set
-        groups: dict[str, tuple[str, list[EvalCaseResult]]] = defaultdict(
-            lambda: ("", [])
-        )
-        for file_name, eval_set_id, eval_set_name, result in quads:
+        groups: dict[str, tuple[str, list[EvalCaseResult]]] = defaultdict(lambda: ("", []))
+        for _, eval_set_id, eval_set_name, result in quads:
             name, results_list = groups[eval_set_id]
-            groups[eval_set_id] = (eval_set_name or name, results_list + [result])
+            groups[eval_set_id] = (eval_set_name or name, [*results_list, result])
 
         reports: list[EvalReport] = []
         for eval_set_id, (eval_set_name, results) in groups.items():
@@ -572,7 +686,7 @@ class EvalCommand(BaseCommand):
                     eval_set_id=eval_set_id,
                     eval_set_name=eval_set_name,
                     results=results,
-                    config_used=global_config.model_dump(),
+                    config_used=(confeval_config or self._default_config()).model_dump(),
                 )
             )
 
@@ -618,5 +732,13 @@ class EvalCommand(BaseCommand):
             f"({summary.pass_rate:.1%})",
             emoji=False,
         )
+        tok = summary.total_token_usage
+        if tok.total_tokens:
+            cache_part = f"  cache_read: {tok.cache_read_tokens:,}" if tok.cache_read_tokens else ""
+            self.output.info(
+                f"Tokens  — input: {tok.input_tokens:,}  output: {tok.output_tokens:,}"
+                f"{cache_part}  total: {tok.total_tokens:,}",
+                emoji=False,
+            )
 
         return return_code

{10xscale_agentflow_cli-0.3.2.5 → 10xscale_agentflow_cli-0.3.2.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "10xscale-agentflow-cli"
-version = "0.3.2.5"
+version = "0.3.2.7"
 description = "CLI and API for 10xscale AgentFlow"
 readme = "README.md"
 license = {text = "MIT"}