evalgate-sdk 3.5.1__tar.gz → 3.6.0__tar.gz

{evalgate_sdk-3.5.1 → evalgate_sdk-3.6.0}/CHANGELOG.md

@@ -9,6 +9,11 @@ Version numbering is aligned with the TypeScript SDK (`@evalgate/sdk`) and the p
 
 **Version history note:** The Python SDK jumped from 1.0.0 → 1.9.x → 2.0.0 to stay in sync with the TypeScript SDK. The TypeScript SDK had many releases (1.1–1.9) before the Python SDK existed. We now align both SDKs on the same major.minor version.
 
+## Unreleased
+
+### Added
+- **Production-loop scaffold parity** — `evalgate init --template production-loop`, `evalgate verify --local`, configured `discover`, JSON `validate`, and JSON/dry-run `gate` now work together from a clean Python SDK install.
+
 ## [3.5.1] - 2026-06-26
 
 ### Fixed

{evalgate_sdk-3.5.1 → evalgate_sdk-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalgate-sdk
-Version: 3.5.1
+Version: 3.6.0
 Summary: EvalGate Python SDK — CI for AI behavior. Traces, evaluations, assertions, and regression gates for LLM apps.
 Project-URL: Homepage, https://evalgate.com
 Project-URL: Documentation, https://github.com/evalgate/ai-evaluation-platform#readme

{evalgate_sdk-3.5.1 → evalgate_sdk-3.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "evalgate-sdk"
-version = "3.5.1"
+version = "3.6.0"
 description = "EvalGate Python SDK — CI for AI behavior. Traces, evaluations, assertions, and regression gates for LLM apps."
 readme = "README.md"
 license = "MIT"

{evalgate_sdk-3.5.1 → evalgate_sdk-3.6.0}/src/evalgate_sdk/__init__.py

@@ -392,12 +392,17 @@ from evalgate_sdk.types import (
     AutoPlanPreviewResponse,
     AutoSessionWorkflowProfile,
     CamelModel,
+    CreateProviderKeyParams,
     EvaluationIntegritySnapshot,
     EvaluationRunDetail,
     KnowledgeHintSummary,
     KnowledgeResponse,
+    ListProviderKeysParams,
+    ProviderKey,
     QualityBreakdown,
     QualityScore,
+    RotateProviderKeyResult,
+    UpdateProviderKeyParams,
 )
 from evalgate_sdk.utils.input_hash import normalize_input, sha256_input
 from evalgate_sdk.workflows import WorkflowTracer, create_workflow_tracer, trace_workflow_step
@@ -700,6 +705,11 @@ __all__ = [
     "CamelModel",
     "QualityScore",
     "QualityBreakdown",
+    "CreateProviderKeyParams",
+    "ListProviderKeysParams",
+    "ProviderKey",
+    "RotateProviderKeyResult",
+    "UpdateProviderKeyParams",
     # Knowledge API
     "KnowledgeAPI",
     # Collector (T2)

evalgate_sdk-3.6.0/src/evalgate_sdk/_version.py

@@ -0,0 +1,3 @@
+__version__ = "3.6.0"
+SDK_VERSION = __version__
+SPEC_VERSION = "3.6.0"

{evalgate_sdk-3.5.1 → evalgate_sdk-3.6.0}/src/evalgate_sdk/cli/__init__.py

@@ -76,6 +76,7 @@ def get_app():
         share,
         sync,
         upgrade,
+        verify,
         version,
     )
     from evalgate_sdk.cli.judge_commands import judge
@@ -103,6 +104,7 @@ def get_app():
     app.command("sync")(sync)
     app.command("configure")(configure)
     app.command("upgrade")(upgrade)
+    app.command("verify")(verify)
     app.command("version")(version)
     app.command("impact-analysis")(impact_analysis)
     app.command("migrate")(migrate)
@@ -132,6 +134,21 @@ def get_app():
     app.command("replay-decision")(replay_decision)
     app.add_typer(auto_app, name="auto")
 
+    from evalgate_sdk.cli.sprint4_commands import failure_modes_command, mcp_command, run_cli_command
+
+    app.command(
+        "run-cli",
+        context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    )(run_cli_command)
+    app.command(
+        "mcp",
+        context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    )(mcp_command)
+    app.command(
+        "failure-modes",
+        context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    )(failure_modes_command)
+
     return app
 
 

evalgate_sdk-3.6.0/src/evalgate_sdk/cli/agent_evidence_utils.py

@@ -0,0 +1,87 @@
+"""Utilities for agent evidence capture (hashing, redaction, JSON I/O).
+
+Port of ``cli/agent-evidence-utils.ts``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+SECRET_PATTERNS: list[re.Pattern[str]] = [
+    re.compile(
+        r"\b((?:api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password|client[_-]?secret)\s*[:=]\s*)([\"']?)([^\s\"',;]+)",
+        re.I,
+    ),
+    re.compile(r"(Bearer\s+)[A-Za-z0-9._~+/=-]+", re.I),
+    re.compile(r"\b(sk-[A-Za-z0-9_-]{20,})\b"),
+    re.compile(r"\b(gh[pousr]_[A-Za-z0-9_]{20,})\b"),
+    re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]{20,})\b"),
+    re.compile(r"\b(npm_[A-Za-z0-9]{20,})\b"),
+    re.compile(r"\b(AKIA[0-9A-Z]{16})\b"),
+    re.compile(r"\b(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b"),
+    re.compile(r"(postgres(?:ql)?:\/\/[^:\s/@]+:)([^@\s]+)(@[^\s]+)", re.I),
+]
+
+
+def sha256_hex(value: str) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def _sort_value(value: Any) -> Any:
+    if isinstance(value, list):
+        return [_sort_value(item) for item in value]
+    if isinstance(value, dict):
+        return {key: _sort_value(value[key]) for key in sorted(value)}
+    return value
+
+
+def canonicalize(value: Any) -> str:
+    return json.dumps(_sort_value(value), separators=(",", ":"), default=str)
+
+
+def stable_hash(value: Any) -> str:
+    return sha256_hex(canonicalize(value))
+
+
+def redact_text(value: str) -> dict[str, Any]:
+    text = value
+    secrets_found = 0
+    for pattern in SECRET_PATTERNS:
+
+        def _replace(match: re.Match[str]) -> str:
+            nonlocal secrets_found
+            secrets_found += 1
+            groups = match.groups()
+            first = groups[0] if groups else ""
+            if first.lower().startswith("bearer"):
+                return f"{first}[REDACTED]"
+            if first.lower().startswith("postgres"):
+                suffix = groups[2] if len(groups) > 2 else ""
+                return f"{first}[REDACTED]{suffix}"
+            if len(groups) >= 3 and first:
+                quote = groups[1] if len(groups) > 1 else ""
+                return f"{first}{quote}[REDACTED]"
+            return "[REDACTED]"
+
+        text = pattern.sub(_replace, text)
+    return {"text": text, "secretsFound": secrets_found}
+
+
+def bounded(value: str, max_bytes: int = 64 * 1024) -> str:
+    encoded = value.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return value
+    truncated = encoded[:max_bytes].decode("utf-8", errors="ignore")
+    return f"{truncated}\n[TRUNCATED]"
+
+
+def read_json_file(file_path: str) -> dict[str, Any]:
+    raw = Path(file_path).read_text(encoding="utf-8-sig")
+    parsed = json.loads(raw)
+    if not isinstance(parsed, dict):
+        raise ValueError(f"{file_path} must contain a JSON object")
+    return parsed

evalgate_sdk-3.6.0/src/evalgate_sdk/cli/check_runner.py

@@ -0,0 +1,291 @@
+"""evalgate check runner — CI/CD evaluation gate.
+
+Port of ``cli/check.ts`` (core path: fetch quality, evaluate gate, build report).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import asdict, dataclass, is_dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from evalgate_sdk.cli.api import fetch_quality_latest, fetch_run_details
+from evalgate_sdk.cli.cli_constants import EXIT
+from evalgate_sdk.cli.config import load_config, merge_config_with_args
+from evalgate_sdk.cli.gate_evaluator import CheckGateArgs, evaluate_gate, gate_result_to_dict
+from evalgate_sdk.cli.report.build_check_report import build_check_report
+from evalgate_sdk.formatters.human import format_human
+from evalgate_sdk.formatters.json_fmt import format_json
+
+FormatType = Literal["human", "json", "github"]
+
+
+@dataclass
+class CheckArgs:
+    base_url: str
+    api_key: str
+    min_score: float
+    max_drop: float | None = None
+    warn_drop: float | None = None
+    min_n: int | None = None
+    allow_weak_evidence: bool = False
+    evaluation_id: str = ""
+    policy: str | None = None
+    baseline: str = "published"
+    format: FormatType = "human"
+    explain: bool = False
+    dry_run: bool = False
+    allow_empty: bool = False
+    max_cost_usd: float | None = None
+    max_latency_ms: float | None = None
+    max_cost_delta_usd: float | None = None
+
+
+def _to_camel(key: str) -> str:
+    parts = key.split("_")
+    return parts[0] + "".join(part[:1].upper() + part[1:] for part in parts[1:])
+
+
+def _serialize(value: Any) -> Any:
+    if is_dataclass(value) and not isinstance(value, type):
+        return {_to_camel(k): _serialize(v) for k, v in asdict(value).items() if v is not None}
+    if isinstance(value, list):
+        return [_serialize(item) for item in value]
+    if isinstance(value, dict):
+        return {_to_camel(k): _serialize(v) for k, v in value.items() if v is not None}
+    return value
+
+
+def parse_check_args(argv: list[str]) -> tuple[CheckArgs | None, int, str | None]:
+    args: dict[str, str] = {}
+    index = 0
+    while index < len(argv):
+        arg = argv[index]
+        if arg.startswith("--"):
+            key = arg[2:]
+            next_value = argv[index + 1] if index + 1 < len(argv) else None
+            if next_value is not None and not next_value.startswith("--"):
+                args[key] = next_value
+                index += 2
+            else:
+                args[key] = "true"
+                index += 1
+        else:
+            index += 1
+
+    base_url = args.get("baseUrl") or args.get("base-url") or ""
+    api_key = args.get("apiKey") or args.get("api-key") or ""
+    min_score = int(args.get("minScore") or args.get("min-score") or "0")
+    max_drop = (
+        int(args["maxDrop"]) if args.get("maxDrop") else (int(args["max-drop"]) if args.get("max-drop") else None)
+    )
+    warn_drop = (
+        int(args["warnDrop"]) if args.get("warnDrop") else (int(args["warn-drop"]) if args.get("warn-drop") else None)
+    )
+    min_n = int(args["minN"]) if args.get("minN") else (int(args["min-n"]) if args.get("min-n") else None)
+    allow_weak_evidence = args.get("allowWeakEvidence") in {"true", "1"} or args.get("allow-weak-evidence") in {
+        "true",
+        "1",
+    }
+    evaluation_id = args.get("evaluationId") or args.get("evaluation-id") or ""
+    policy = args.get("policy")
+    format_raw = args.get("format") or "human"
+    fmt: FormatType = "json" if format_raw == "json" else ("github" if format_raw == "github" else "human")
+    explain = args.get("explain") in {"true", "1"}
+    dry_run = args.get("dry-run") == "true" or args.get("dryRun") == "true"
+    allow_empty = args.get("allow-empty") == "true" or args.get("allowEmpty") == "true"
+    profile = args.get("profile")
+    baseline = args.get("baseline") or "published"
+    if baseline not in {"published", "previous", "production", "auto"}:
+        baseline = "published"
+    max_cost_usd = (
+        float(args.get("max-cost-usd") or args.get("maxCostUsd") or "nan")
+        if (args.get("max-cost-usd") or args.get("maxCostUsd"))
+        else None
+    )
+    max_latency_ms = (
+        int(args.get("max-latency-ms") or args.get("maxLatencyMs") or "0")
+        if (args.get("max-latency-ms") or args.get("maxLatencyMs"))
+        else None
+    )
+    max_cost_delta_usd = (
+        float(args.get("max-cost-delta-usd") or args.get("maxCostDeltaUsd") or "nan")
+        if (args.get("max-cost-delta-usd") or args.get("maxCostDeltaUsd"))
+        else None
+    )
+
+    config = load_config()
+    merged = merge_config_with_args(
+        config,
+        {
+            "evaluation_id": evaluation_id or None,
+            "base_url": base_url or None,
+            "min_score": str(min_score) if args.get("minScore") or args.get("min-score") else None,
+            "max_drop": str(max_drop) if max_drop is not None else None,
+            "warn_drop": str(warn_drop) if warn_drop is not None else None,
+            "min_n": str(min_n) if min_n is not None else None,
+            "allow_weak_evidence": "true" if allow_weak_evidence else None,
+            "baseline": baseline,
+            "profile": profile,
+        },
+    )
+    if not evaluation_id and merged.get("evaluation_id"):
+        evaluation_id = str(merged["evaluation_id"])
+    if merged.get("base_url"):
+        base_url = str(merged["base_url"])
+    if merged.get("min_score") is not None and not (args.get("minScore") or args.get("min-score")):
+        min_score = int(merged["min_score"])
+    if merged.get("max_drop") is not None and max_drop is None:
+        max_drop = int(merged["max_drop"])
+    if merged.get("warn_drop") is not None and warn_drop is None:
+        warn_drop = int(merged["warn_drop"])
+    if merged.get("min_n") is not None and min_n is None:
+        min_n = int(merged["min_n"])
+    if merged.get("allow_weak_evidence") is not None and not (
+        args.get("allowWeakEvidence") or args.get("allow-weak-evidence")
+    ):
+        allow_weak_evidence = bool(merged["allow_weak_evidence"])
+    if merged.get("baseline") and not args.get("baseline"):
+        baseline = str(merged["baseline"])
+
+    import os
+
+    if not base_url:
+        from evalgate_sdk.constants import DEFAULT_BASE_URL
+
+        base_url = os.environ.get("EVALGATE_BASE_URL") or DEFAULT_BASE_URL
+    if not api_key:
+        api_key = os.environ.get("EVALGATE_API_KEY") or ""
+
+    if not api_key:
+        return None, EXIT.BAD_ARGS, "Error: --apiKey or EVALGATE_API_KEY is required"
+    if not evaluation_id:
+        return None, EXIT.BAD_ARGS, "Run evalgate init and paste your evaluationId, or pass --evaluationId."
+    if min_score < 0 or min_score > 100:
+        return None, EXIT.BAD_ARGS, "Error: --minScore must be 0-100"
+    if min_n is not None and min_n < 1:
+        return None, EXIT.BAD_ARGS, "Error: --minN must be a positive number"
+
+    return (
+        CheckArgs(
+            base_url=base_url,
+            api_key=api_key,
+            min_score=min_score,
+            max_drop=max_drop,
+            warn_drop=warn_drop,
+            min_n=min_n,
+            allow_weak_evidence=allow_weak_evidence,
+            evaluation_id=evaluation_id,
+            policy=policy,
+            baseline=baseline,
+            format=fmt,
+            explain=explain,
+            dry_run=dry_run,
+            allow_empty=allow_empty,
+            max_cost_usd=max_cost_usd,
+            max_latency_ms=max_latency_ms,
+            max_cost_delta_usd=max_cost_delta_usd,
+        ),
+        EXIT.PASS,
+        None,
+    )
+
+
+async def run_check(args: CheckArgs) -> int:
+    quality_result = await fetch_quality_latest(args.base_url, args.api_key, args.evaluation_id, args.baseline)
+    if not quality_result.get("ok"):
+        status = quality_result.get("status", 0)
+        body = quality_result.get("body", "")
+        if status == 0:
+            print(f"EvalGate gate ERROR: Network failure — {body}")
+        else:
+            print(f"EvalGate gate ERROR: API returned {status} — {body}")
+        return EXIT.API_ERROR
+
+    quality = quality_result["data"]
+    request_id = quality_result.get("request_id")
+    evaluation_run_id = quality.get("evaluationRunId") or quality.get("evaluation_run_id")
+
+    run_details = None
+    if evaluation_run_id is not None:
+        run_res = await fetch_run_details(args.base_url, args.api_key, args.evaluation_id, int(evaluation_run_id))
+        if run_res.get("ok"):
+            run_details = run_res.get("data")
+
+    gate_args = CheckGateArgs(
+        min_score=args.min_score,
+        max_drop=args.max_drop,
+        warn_drop=args.warn_drop,
+        min_n=args.min_n,
+        allow_weak_evidence=args.allow_weak_evidence,
+        policy=args.policy,
+        baseline=args.baseline,
+        allow_empty=args.allow_empty,
+        max_cost_usd=args.max_cost_usd,
+        max_latency_ms=args.max_latency_ms,
+        max_cost_delta_usd=args.max_cost_delta_usd,
+    )
+    gate_result = evaluate_gate(gate_args, quality)
+    gate_dict = gate_result_to_dict(gate_result)
+
+    report = build_check_report(
+        evaluation_id=args.evaluation_id,
+        quality=quality,
+        gate_result=gate_dict,
+        base_url=args.base_url,
+        run_details=run_details,
+        request_id=request_id,
+        policy=args.policy,
+        min_score=args.min_score,
+        max_drop=args.max_drop,
+        warn_drop=args.warn_drop,
+        min_n=args.min_n,
+        allow_weak_evidence=args.allow_weak_evidence,
+        baseline=args.baseline,
+        max_cost_usd=args.max_cost_usd,
+        max_latency_ms=args.max_latency_ms,
+        max_cost_delta_usd=args.max_cost_delta_usd,
+        explain=args.explain,
+    )
+    report_dict = _serialize(report)
+
+    report_dir = Path.cwd() / ".evalgate"
+    try:
+        report_dir.mkdir(parents=True, exist_ok=True)
+        (report_dir / "last-report.json").write_text(json.dumps(report_dict, indent=2) + "\n", encoding="utf-8")
+    except OSError as exc:
+        print(f"EvalGate: failed to write .evalgate/last-report.json: {exc}")
+        return EXIT.API_ERROR
+
+    if args.format == "json":
+        print(format_json(report_dict))
+    elif args.format == "github":
+        from evalgate_sdk.formatters.github import format_github
+
+        print(format_github(report_dict))
+    else:
+        print(format_human(report_dict))
+
+    if not gate_result.passed:
+        print("\nNext: evalgate explain")
+
+    if args.dry_run:
+        print(f"\n[dry-run] Gate exits with code {gate_result.exit_code} (severity={gate_result.severity})")
+        return gate_result.exit_code
+
+    return gate_result.exit_code
+
+
+def run_check_sync(argv: list[str]) -> int:
+    parsed, exit_code, message = parse_check_args(argv)
+    if parsed is None:
+        if message:
+            print(message)
+        return exit_code
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(run_check(parsed))
+    return loop.run_until_complete(run_check(parsed))

{evalgate_sdk-3.5.1 → evalgate_sdk-3.6.0}/src/evalgate_sdk/cli/cli_constants.py

@@ -18,3 +18,8 @@ class EXIT:
     LOW_N = 6
     WEAK_EVIDENCE = 7
     WARN_REGRESSION = 8
+    FAILURE_MODE_THRESHOLD = 9
+    JUDGE_CREDIBILITY_UNTRUSTWORTHY = 10
+    IO_ERROR = 11
+    BASELINE_TAMPER = 12
+    INTERNAL_ERROR = 99