10xscale-agentflow-cli 0.3.2.2__tar.gz → 0.3.2.4__tar.gz

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/10xscale_agentflow_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: 10xscale-agentflow-cli
-Version: 0.3.2.2
+Version: 0.3.2.4
 Summary: CLI and API for 10xscale AgentFlow
 Author-email: 10xscale <contact@10xscale.ai>
 Maintainer-email: Shudipto Trafder <shudiptotrafder@gmail.com>

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/10xscale_agentflow_cli.egg-info/SOURCES.txt

@@ -19,8 +19,10 @@ agentflow_cli/cli/main.py
 agentflow_cli/cli/commands/__init__.py
 agentflow_cli/cli/commands/api.py
 agentflow_cli/cli/commands/build.py
+agentflow_cli/cli/commands/eval.py
 agentflow_cli/cli/commands/init.py
 agentflow_cli/cli/commands/skills.py
+agentflow_cli/cli/commands/test.py
 agentflow_cli/cli/commands/version.py
 agentflow_cli/cli/core/__init__.py
 agentflow_cli/cli/core/config.py
@@ -36,7 +38,7 @@ agentflow_cli/cli/templates/prod/agentflow.json
 agentflow_cli/cli/templates/prod/auth/__init__.py
 agentflow_cli/cli/templates/prod/auth/agent_auth.py
 agentflow_cli/cli/templates/prod/evals/__init__.py
-agentflow_cli/cli/templates/prod/evals/weather_agents.py
+agentflow_cli/cli/templates/prod/evals/weather_agents_eval.py
 agentflow_cli/cli/templates/prod/graph/__init__.py
 agentflow_cli/cli/templates/prod/graph/agent.py
 agentflow_cli/cli/templates/prod/graph/state.py
@@ -72,6 +74,7 @@ agentflow_cli/cli/templates/skills/agent-skills/references/client-messages-invok
 agentflow_cli/cli/templates/skills/agent-skills/references/client-threads-memory-files.md
 agentflow_cli/cli/templates/skills/agent-skills/references/context-id-background.md
 agentflow_cli/cli/templates/skills/agent-skills/references/dependency-injection.md
+agentflow_cli/cli/templates/skills/agent-skills/references/evaluation.md
 agentflow_cli/cli/templates/skills/agent-skills/references/id-and-thread-name-generators.md
 agentflow_cli/cli/templates/skills/agent-skills/references/media-and-files.md
 agentflow_cli/cli/templates/skills/agent-skills/references/memory-and-store.md
@@ -88,6 +91,7 @@ agentflow_cli/cli/templates/skills/agent-skills/references/state-graph.md
 agentflow_cli/cli/templates/skills/agent-skills/references/stream-emitter.md
 agentflow_cli/cli/templates/skills/agent-skills/references/streaming.md
 agentflow_cli/cli/templates/skills/agent-skills/references/testing-and-evaluation.md
+agentflow_cli/cli/templates/skills/agent-skills/references/unit-testing.md
 agentflow_cli/cli/templates/skills/copilot/agentflow.instructions.md
 agentflow_cli/src/__init__.py
 agentflow_cli/src/app/__init__.py

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: 10xscale-agentflow-cli
-Version: 0.3.2.2
+Version: 0.3.2.4
 Summary: CLI and API for 10xscale AgentFlow
 Author-email: 10xscale <contact@10xscale.ai>
 Maintainer-email: Shudipto Trafder <shudiptotrafder@gmail.com>

10xscale_agentflow_cli-0.3.2.4/agentflow_cli/cli/commands/eval.py

@@ -0,0 +1,273 @@
+"""Eval command — discover and run agentflow evaluations, always generating reports."""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import importlib.util
+import inspect
+import sys
+import webbrowser
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from agentflow.qa.evaluation import CriterionConfig, EvalConfig, MatchType
+from agentflow.qa.evaluation.collectors.trajectory_collector import TrajectoryCollector
+from agentflow.qa.evaluation.config.eval_config import ReporterConfig
+from agentflow.qa.evaluation.eval_result import EvalReport as ER
+from agentflow.qa.evaluation.evaluator import AgentEvaluator
+from agentflow.qa.evaluation.reporters.manager import ReporterManager
+
+from agentflow_cli.cli.commands import BaseCommand
+from agentflow_cli.cli.core.config import ConfigManager
+
+
+if TYPE_CHECKING:
+    from agentflow.qa.evaluation.eval_result import EvalReport
+
+
+class EvalCommand(BaseCommand):
+    """Discover and run agent evaluations; always write HTML + JSON reports."""
+
+    # ------------------------------------------------------------------
+    # Discovery
+    # ------------------------------------------------------------------
+
+    def _discover(self, target: Path) -> list[Path]:
+        """Return eval files under target. If target is a file, return it directly."""
+        if target.is_file():
+            return [target]
+
+        seen: dict[Path, None] = {}
+        for pattern in ("*_eval.py", "eval_*.py"):
+            for p in sorted(target.rglob(pattern)):
+                seen[p] = None
+        return list(seen)
+
+    # ------------------------------------------------------------------
+    # Module loading
+    # ------------------------------------------------------------------
+
+    def _load_module(self, path: Path) -> Any:
+        project_root = str(Path.cwd())
+        if project_root not in sys.path:
+            sys.path.insert(0, project_root)
+
+        spec = importlib.util.spec_from_file_location("_agentflow_eval", path)
+        mod = importlib.util.module_from_spec(spec)  # type: ignore[arg-type]
+        spec.loader.exec_module(mod)  # type: ignore[union-attr]
+        return mod
+
+    # ------------------------------------------------------------------
+    # Agent loading from agentflow.json
+    # ------------------------------------------------------------------
+
+    def _load_agent_from_config(self) -> Any:
+        config_manager = ConfigManager()
+        discovered = config_manager.auto_discover_config()
+        if not discovered:
+            raise RuntimeError("No agentflow.json found — cannot auto-load agent.")
+        config_manager.load_config(str(discovered))
+        agent_spec: str = config_manager.get_config_value("agent", default="")
+        if not agent_spec or ":" not in agent_spec:
+            raise RuntimeError(f"Invalid 'agent' field in agentflow.json: {agent_spec!r}")
+        module_path, attr = agent_spec.rsplit(":", 1)
+        mod = importlib.import_module(module_path)
+        return getattr(mod, attr)
+
+    # ------------------------------------------------------------------
+    # Per-file runner (sync wrapper — avoids nested asyncio.run() issues)
+    # ------------------------------------------------------------------
+
+    def _default_config(self) -> Any:
+        return EvalConfig(
+            criteria={
+                "response_match": CriterionConfig(threshold=0.6, match_type=MatchType.ANY_ORDER),
+                "tool_name_match_score": CriterionConfig(
+                    threshold=1.0, match_type=MatchType.EXACT, check_args=False
+                ),
+                "node_order": CriterionConfig(threshold=0.8, match_type=MatchType.IN_ORDER),
+            },
+        )
+
+    def _run_file_sync(self, path: Path) -> EvalReport | None:
+        """Load and run a single eval file. Returns EvalReport or None if skipped."""
+        self.output.info(f"Running: {path.name}", emoji=False)
+        mod = self._load_module(path)
+
+        # Primary protocol: run() — full control when the standard pipeline isn't enough.
+        # Must return an EvalReport.
+        if hasattr(mod, "run"):
+            result = mod.run()
+            if inspect.isawaitable(result):
+                return asyncio.run(result)  # type: ignore[arg-type]
+            return result  # type: ignore[return-value]
+
+        # Fallback protocol: get_eval_set() — CLI handles graph loading, evaluation, and reports.
+        # Config is optional; omit it and the CLI default is used.
+        if hasattr(mod, "get_eval_set"):
+            if hasattr(mod, "get_eval_config"):
+                config = mod.get_eval_config()
+            elif hasattr(mod, "EVAL_CONFIG"):
+                config = mod.EVAL_CONFIG
+            else:
+                config = self._default_config()
+            return self._run_with_evaluator(mod, mod.get_eval_set(), config)
+
+        self.output.warning(f"Skipping {path.name} — no get_eval_set() or run() found.")
+        return None
+
+    def _run_with_evaluator(self, mod: Any, eval_set: Any, config: Any) -> EvalReport:
+        # Prefer the graph already imported in the module (most common pattern)
+        graph = getattr(mod, "app", None) or self._load_agent_from_config()
+        collector = TrajectoryCollector(capture_all_events=True)
+        evaluator = AgentEvaluator(graph, collector, config=config)
+        return asyncio.run(evaluator.evaluate(eval_set))
+
+    # ------------------------------------------------------------------
+    # Report merging
+    # ------------------------------------------------------------------
+
+    def _merge_reports(self, reports: list[EvalReport]) -> EvalReport:
+        if len(reports) == 1:
+            return reports[0]
+
+        all_results = []
+        for r in reports:
+            all_results.extend(r.results)
+        return ER.create(
+            eval_set_id="combined_eval",
+            eval_set_name="Combined Evaluation",
+            results=all_results,
+        )
+
+    # ------------------------------------------------------------------
+    # Eval directory from agentflow.json
+    # ------------------------------------------------------------------
+
+    def _resolve_eval_dir(self) -> Path:
+        config_manager = ConfigManager()
+        discovered = config_manager.auto_discover_config()
+        directory = "evals"
+        if discovered:
+            try:
+                config_manager.load_config(str(discovered))
+                eval_cfg = config_manager.get_evaluation_config()
+                directory = eval_cfg.get("directory", "evals")
+            except Exception:
+                self.logger.warning(
+                    "Failed to load eval directory from config; using default 'evals/'"
+                )
+        return Path.cwd() / directory
+
+    # ------------------------------------------------------------------
+    # Main entry point
+    # ------------------------------------------------------------------
+
+    def execute(  # noqa: PLR0912, PLR0915
+        self,
+        target: str | None = None,
+        output_dir: str = "eval_reports",
+        no_report: bool = False,
+        threshold: float | None = None,
+        open_report: bool = False,
+        verbose: bool = False,
+        quiet: bool = False,
+        **kwargs: Any,
+    ) -> int:
+        # Load optional overrides from agentflow.json
+        config_manager = ConfigManager()
+        discovered = config_manager.auto_discover_config()
+        if discovered:
+            try:
+                config_manager.load_config(str(discovered))
+                eval_cfg = config_manager.get_evaluation_config()
+                if output_dir == "eval_reports":
+                    output_dir = eval_cfg.get("output_dir", output_dir)
+                if threshold is None:
+                    threshold = eval_cfg.get("threshold")
+            except Exception:
+                self.logger.warning(
+                    "Failed to load eval config from agentflow.json; using defaults"
+                )
+
+        # Resolve target path
+        if target:
+            target_path = Path(target)
+            if not target_path.exists():
+                self.output.error(f"Path not found: {target}")
+                return 1
+        else:
+            target_path = self._resolve_eval_dir()
+            if not target_path.exists():
+                self.output.error(
+                    f"Eval directory '{target_path}' not found. "
+                    "Create an evals/ directory or pass a file/folder path."
+                )
+                return 1
+
+        files = self._discover(target_path)
+        if not files:
+            self.output.error(f"No eval files found in {target_path}")
+            return 1
+
+        self.output.print_banner("Eval", f"Found {len(files)} eval file(s) in {target_path}")
+
+        # Run each file
+        reports: list[EvalReport] = []
+        for f in files:
+            try:
+                report = self._run_file_sync(f)
+                if report is not None:
+                    reports.append(report)
+            except Exception as exc:
+                self.output.error(f"Error in {f.name}: {exc}")
+                self.logger.exception("Eval file failed: %s", f)
+
+        if not reports:
+            self.output.error("No reports produced. Ensure eval files expose run().")
+            return 1
+
+        merged = self._merge_reports(reports)
+
+        # Determine exit code
+        if threshold is not None and merged.summary.pass_rate < threshold:
+            self.output.error(
+                f"Pass rate {merged.summary.pass_rate:.1%} is below threshold {threshold:.1%}"
+            )
+            return_code = 1
+        else:
+            return_code = 0 if merged.summary.pass_rate == 1.0 else 1
+
+        # Always generate file reports unless --no-report
+        if not no_report:
+            manager = ReporterManager(
+                ReporterConfig(
+                    output_dir=output_dir,
+                    html=True,
+                    json_report=True,
+                    console=False,  # console output already handled by run() modules
+                    timestamp_files=True,
+                )
+            )
+            result = manager.run_all(merged)
+
+            if result.html_path:
+                self.output.success(f"HTML report: {result.html_path}")
+            if result.json_path:
+                self.output.info(f"JSON report: {result.json_path}", emoji=False)
+            if result.has_errors:
+                for name, err in result.errors:
+                    self.output.warning(f"Reporter error [{name}]: {err}")
+
+            if open_report and result.html_path:
+                webbrowser.open(Path(result.html_path).as_uri())
+
+        summary = merged.summary
+        self.output.info(
+            f"Results: {summary.passed_cases}/{summary.total_cases} passed "
+            f"({summary.pass_rate:.1%})",
+            emoji=False,
+        )
+
+        return return_code

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/agentflow_cli/cli/commands/init.py

@@ -16,7 +16,7 @@ from agentflow_cli.cli.exceptions import FileOperationError
 
 
 _TEMPLATES_DIR = Path(__file__).parent.parent / "templates"
-_SKIP_DIRS = {"__pycache__"}
+_SKIP_DIRS = {"__pycache__", ".ruff_cache"}
 _DIVIDER = Colors.colorize("  " + "─" * 46, "cyan")
 
 # Directories inside prod/ that are only included based on user choices

10xscale_agentflow_cli-0.3.2.4/agentflow_cli/cli/commands/test.py

@@ -0,0 +1,87 @@
+"""Test command implementation — thin pytest wrapper."""
+
+import subprocess  # nosec: B404
+import sys
+import webbrowser
+from pathlib import Path
+from typing import Any
+
+from agentflow_cli.cli.commands import BaseCommand
+from agentflow_cli.cli.core.config import ConfigManager
+
+
+class TestCommand(BaseCommand):
+    """Run the project's test suite via pytest."""
+
+    def execute(
+        self,
+        path: str | None = None,
+        coverage: bool = False,
+        html: bool = False,
+        keyword: str | None = None,
+        verbose: bool = False,
+        quiet: bool = False,
+        extra_args: tuple[str, ...] = (),
+        **kwargs: Any,
+    ) -> int:
+        project_root = Path.cwd()
+
+        # Load optional overrides from agentflow.json
+        cfg: dict[str, Any] = {}
+        config_manager = ConfigManager()
+        discovered = config_manager.auto_discover_config()
+        if discovered:
+            try:
+                config_manager.load_config(str(discovered))
+                cfg = config_manager.get_test_config()
+            except Exception:  # nosec: B110
+                self.logger.warning("Failed to load test configuration from %s", discovered)
+
+        # Explicit CLI path wins; fall back to agentflow.json; None = pytest auto-discovery
+        resolved_path: str | None = path or cfg.get("path") or None
+        resolved_coverage = coverage or cfg.get("coverage", False)
+        coverage_threshold: int | None = cfg.get("coverage_threshold")
+
+        location = str(project_root / resolved_path) if resolved_path else str(project_root)
+        self.output.print_banner("Test", f"Running tests in {location}")
+
+        cmd = [sys.executable, "-m", "pytest"]
+        if resolved_path:
+            cmd.append(resolved_path)
+
+        if verbose:
+            cmd.append("-v")
+        elif quiet:
+            cmd.append("-q")
+        else:
+            cmd.append("-v")
+
+        if resolved_coverage:
+            cmd += [
+                "--cov=.",
+                "--cov-report=term-missing",
+                "--cov-report=html:htmlcov",
+            ]
+            if coverage_threshold is not None:
+                cmd.append(f"--cov-fail-under={coverage_threshold}")
+
+        if keyword:
+            cmd += ["-k", keyword]
+
+        cmd += list(extra_args)
+
+        self.logger.info("Running: %s", " ".join(cmd))
+
+        result = subprocess.run(cmd, cwd=project_root, check=False)  # nosec: B603  # noqa: S603
+
+        if result.returncode == 0:
+            self.output.success("All tests passed.")
+        else:
+            self.output.error(f"Tests finished with exit code {result.returncode}.")
+
+        if html and resolved_coverage:
+            report_path = (project_root / "htmlcov" / "index.html").as_uri()
+            self.output.info(f"Opening coverage report: {report_path}", emoji=False)
+            webbrowser.open(report_path)
+
+        return result.returncode

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/agentflow_cli/cli/core/config.py

@@ -214,6 +214,28 @@ class ConfigManager:
 
         return value
 
+    def get_test_config(self) -> dict[str, Any]:
+        """Return the optional 'test' section from agentflow.json.
+
+        Returns a dict with keys: path, coverage, coverage_threshold.
+        All fields are optional; callers should use .get() with their own defaults.
+        """
+        raw = self.get_config_value("test", default={})
+        if not isinstance(raw, dict):
+            return {}
+        return raw
+
+    def get_evaluation_config(self) -> dict[str, Any]:
+        """Return the optional 'evaluation' section from agentflow.json.
+
+        Returns a dict with keys: directory, output_dir, threshold, timestamp_files.
+        All fields are optional; callers should use .get() with their own defaults.
+        """
+        raw = self.get_config_value("evaluation", default={})
+        if not isinstance(raw, dict):
+            return {}
+        return raw
+
     def resolve_env_file(self) -> Path | None:
         """Resolve environment file path from configuration.
 

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/agentflow_cli/cli/main.py

@@ -7,8 +7,10 @@ from dotenv import load_dotenv
 
 from agentflow_cli.cli.commands.api import APICommand
 from agentflow_cli.cli.commands.build import BuildCommand
+from agentflow_cli.cli.commands.eval import EvalCommand
 from agentflow_cli.cli.commands.init import InitCommand
 from agentflow_cli.cli.commands.skills import SkillsCommand
+from agentflow_cli.cli.commands.test import TestCommand
 from agentflow_cli.cli.commands.version import VersionCommand
 from agentflow_cli.cli.constants import (
     DEFAULT_CONFIG_FILE,
@@ -362,6 +364,103 @@ def skills(
         sys.exit(handle_exception(e))
 
 
+@app.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+)
+def test(
+    ctx: typer.Context,
+    path: str | None = typer.Argument(
+        None, help="Path to tests directory or file (omit to let pytest auto-discover)"
+    ),
+    coverage: bool = typer.Option(False, "--coverage", "-C", help="Run with coverage"),
+    html: bool = typer.Option(
+        False, "--html", help="Open HTML coverage report after run (requires --coverage)"
+    ),
+    keyword: str | None = typer.Option(None, "-k", help="Only run tests matching this expression"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"),
+    quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress all output except errors"),
+) -> None:
+    """Run project tests with pytest.
+
+    Any arguments after -- are forwarded verbatim to pytest.
+    """
+    setup_cli_logging(verbose=verbose, quiet=quiet)
+
+    try:
+        command = TestCommand(output)
+        exit_code = command.execute(
+            path=path,
+            coverage=coverage,
+            html=html,
+            keyword=keyword,
+            verbose=verbose,
+            quiet=quiet,
+            extra_args=tuple(ctx.args),
+        )
+        sys.exit(exit_code)
+    except Exception as e:
+        sys.exit(handle_exception(e))
+
+
+@app.command(name="eval")
+def eval_cmd(
+    target: str | None = typer.Argument(
+        None,
+        help="File or directory to evaluate (default: evals/ from agentflow.json or cwd)",
+    ),
+    output_dir: str = typer.Option(
+        "eval_reports",
+        "--output",
+        "-o",
+        help="Directory for generated report files",
+    ),
+    no_report: bool = typer.Option(
+        False,
+        "--no-report",
+        help="Skip file report generation (console summary only)",
+    ),
+    threshold: float | None = typer.Option(
+        None,
+        "--threshold",
+        "-t",
+        help="Fail if overall pass rate is below this value (0.0-1.0)",
+    ),
+    open_report: bool = typer.Option(
+        False,
+        "--open",
+        help="Open the HTML report in the default browser after the run",
+    ),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"),
+    quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress all output except errors"),
+) -> None:
+    """Run agent evaluations.
+
+    Discovers *_eval.py / eval_*.py files in the target directory (default: evals/).
+    Always generates HTML + JSON reports in eval_reports/ unless --no-report is set.
+
+    Each eval file must expose one of:
+      run()                           # full control, returns EvalReport
+      get_eval_set() + get_eval_config()  # CLI loads agent from agentflow.json
+      EVAL_CONFIG + get_eval_set()        # same, config as a constant
+    """
+    setup_cli_logging(verbose=verbose, quiet=quiet)
+
+    try:
+        command = EvalCommand(output)
+        exit_code = command.execute(
+            target=target,
+            output_dir=output_dir,
+            no_report=no_report,
+            threshold=threshold,
+            open_report=open_report,
+            verbose=verbose,
+            quiet=quiet,
+        )
+        sys.exit(exit_code)
+    except Exception as e:
+        sys.exit(handle_exception(e))
+
+
 def main() -> None:
     """Main CLI entry point."""
     try:

10xscale_agentflow_cli-0.3.2.4/agentflow_cli/cli/templates/dev/agentflow.json

@@ -0,0 +1,12 @@
+{
+  "agent": "graph.agent:app",
+  "env": ".env",
+  "auth": null,
+  "thread_name_generator": null,
+  "evaluation": {
+    "directory": "evals",
+    "output_dir": "eval_reports",
+    "threshold": 0.75,
+    "timestamp_files": true
+  }
+}

{10xscale_agentflow_cli-0.3.2.2 → 10xscale_agentflow_cli-0.3.2.4}/agentflow_cli/cli/templates/prod/agentflow.json

@@ -15,5 +15,16 @@
     "by": "ip",
     "trusted_proxy_headers": false,
     "exclude_paths": ["/health", "/docs", "/redoc", "/openapi.json"]
+  },
+  "test": {
+    "path": "tests",
+    "coverage": true,
+    "coverage_threshold": 70
+  },
+  "evaluation": {
+    "directory": "evals",
+    "output_dir": "eval_reports",
+    "threshold": 0.75,
+    "timestamp_files": true
   }
 }

10xscale_agentflow_cli-0.3.2.4/agentflow_cli/cli/templates/prod/evals/weather_agents_eval.py

@@ -0,0 +1,41 @@
+from agentflow.qa.evaluation import EvalConfig, EvalSet, EvalSetBuilder
+
+
+def get_eval_config() -> EvalConfig:
+    """Return the default eval configuration for CLI auto-discovery."""
+    return EvalConfig()
+
+
+def get_eval_set() -> EvalSet:
+    return (
+        EvalSetBuilder(name="weather-agent-regression")
+        .add_case(
+            query="Hi",
+            expected="assistant",
+            expected_node_order=["MAIN"],
+            name="greeting_response",
+            description="The agent responds to a greeting.",
+        )
+        .add_tool_test(
+            query="What is the weather in London?",
+            tool_name="get_weather",
+            tool_args={"location": "London"},
+            expected_response="London",
+            case_id="weather_london",
+        )
+        .add_tool_test(
+            query="Tell me the current weather in New York",
+            tool_name="get_weather",
+            tool_args={"location": "New York"},
+            expected_response="New York",
+            case_id="weather_new_york",
+        )
+        .add_tool_test(
+            query="How is the weather in Tokyo today?",
+            tool_name="get_weather",
+            tool_args={"location": "Tokyo"},
+            expected_response="Tokyo",
+            case_id="weather_tokyo",
+        )
+        .build()
+    )