evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/runtime/adapters/config_to_dsl.py

@@ -0,0 +1,270 @@
+"""Config → DSL Adapter — LAYER 2 Compatibility Bridge.
+
+Migrates existing evalgate.config.json and TestSuite configurations
+to the new define_eval() DSL without breaking user workflows.
+
+Port of ``runtime/adapters/config-to-dsl.ts``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class MigrationResult:
+    """Migration result information."""
+
+    success: bool = True
+    specs_generated: int = 0
+    errors: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    output_path: str = ""
+
+
+@dataclass
+class EvalAIConfig:
+    """Configuration file structure (existing evalgate.config.json)."""
+
+    evaluation_id: str | None = None
+    gate: dict[str, str] | None = None
+    packages: dict[str, Any] | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+def migrate_config_to_dsl(
+    config_path: str,
+    output_path: str,
+) -> MigrationResult:
+    """Convert evalgate.config.json to DSL specifications."""
+    result = MigrationResult(output_path=output_path)
+
+    if not os.path.exists(config_path):
+        result.success = False
+        result.errors.append(f"Configuration file not found: {config_path}")
+        return result
+
+    try:
+        with open(config_path, encoding="utf-8") as f:
+            config_data = json.load(f)
+
+        config = EvalAIConfig(
+            evaluation_id=config_data.get("evaluationId"),
+            gate=config_data.get("gate"),
+            packages=config_data.get("packages"),
+        )
+
+        dsl_content = _generate_dsl_from_config(config)
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        Path(output_path).write_text(dsl_content, encoding="utf-8")
+
+        result.specs_generated = 1
+        result.warnings.append("Generated basic DSL structure from evalgate.config.json. Manual completion required.")
+    except Exception as exc:
+        result.success = False
+        result.errors.append(f"Config migration failed: {exc}")
+
+    return result
+
+
+def migrate_testsuite_to_dsl(
+    suite_data: dict[str, Any],
+    output_path: str,
+) -> MigrationResult:
+    """Convert TestSuite data to defineEval() DSL specifications."""
+    result = MigrationResult(output_path=output_path)
+
+    try:
+        name = suite_data.get("name", "test-suite")
+        cases = suite_data.get("cases", [])
+
+        dsl_content = _generate_dsl_from_suite(name, cases)
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        Path(output_path).write_text(dsl_content, encoding="utf-8")
+
+        result.specs_generated = len(cases)
+        result.warnings.append(f"Migrated {len(cases)} test cases from TestSuite to define_eval() DSL")
+    except Exception as exc:
+        result.success = False
+        result.errors.append(f"Migration failed: {exc}")
+
+    return result
+
+
+def migrate_project_to_dsl(
+    project_root: str,
+    output_dir: str | None = None,
+    dry_run: bool = False,
+) -> MigrationResult:
+    """Discover and migrate all configurations in a project."""
+    out_dir = output_dir or os.path.join(project_root, ".evalgate", "migrated")
+    result = MigrationResult(output_path=out_dir)
+
+    try:
+        # Find config files
+        config_names = [
+            "evalgate.config.json",
+            "evalai.config.json",
+        ]
+        config_path: str | None = None
+        for name in config_names:
+            candidate = os.path.join(project_root, name)
+            if os.path.exists(candidate):
+                config_path = candidate
+                break
+
+        if config_path:
+            output_path = os.path.join(out_dir, "evalgate.config.migrated.py")
+            if not dry_run:
+                sub = migrate_config_to_dsl(config_path, output_path)
+                result.specs_generated += sub.specs_generated
+                result.errors.extend(sub.errors)
+                result.warnings.extend(sub.warnings)
+            else:
+                result.warnings.append(f"Would migrate {config_path} to {output_path}")
+
+        # Find TestSuite usage in Python files
+        test_files = _find_testsuite_files(project_root)
+        for test_file in test_files:
+            output_path = os.path.join(
+                out_dir,
+                Path(test_file).stem + ".migrated.py",
+            )
+            if not dry_run:
+                Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+                content = _generate_placeholder_dsl(test_file)
+                Path(output_path).write_text(content, encoding="utf-8")
+                result.specs_generated += 1
+                result.warnings.append(f"Created migration placeholder for {test_file}")
+            else:
+                result.warnings.append(f"Would migrate {test_file} to {output_path}")
+
+        if result.specs_generated == 0:
+            result.warnings.append("No TestSuite configurations found to migrate")
+    except Exception as exc:
+        result.success = False
+        result.errors.append(f"Project migration failed: {exc}")
+
+    return result
+
+
+def _find_testsuite_files(project_root: str) -> list[str]:
+    """Find files that might contain TestSuite usage."""
+    found: list[str] = []
+    for root, dirs, files in os.walk(project_root):
+        # Skip hidden dirs and common non-source dirs
+        dirs[:] = [
+            d for d in dirs if not d.startswith(".") and d not in ("node_modules", "__pycache__", ".git", "venv")
+        ]
+        for fname in files:
+            if not fname.endswith(".py"):
+                continue
+            full = os.path.join(root, fname)
+            try:
+                text = Path(full).read_text(encoding="utf-8", errors="ignore")
+                if "create_test_suite" in text or "TestSuite" in text:
+                    found.append(full)
+            except OSError:
+                pass
+    return found
+
+
+def _generate_dsl_from_config(config: EvalAIConfig) -> str:
+    """Generate DSL code from configuration."""
+    now = datetime.now(timezone.utc).isoformat()
+    eval_id = json.dumps(config.evaluation_id) if config.evaluation_id else "None"
+    return f"""# Auto-generated from evalgate.config.json
+# Generated at: {now}
+# This is a basic DSL structure — complete with your actual evaluations
+
+from evalgate_sdk import define_eval, create_result
+
+
+define_eval("basic-evaluation", _basic_eval)
+
+
+async def _basic_eval(ctx):
+    input_text = ctx.input
+
+    # TODO: Replace with your actual agent/LLM call
+    output = f"Agent response to: {{input_text}}"
+
+    passed = len(output) > 0
+    score = 100 if passed else 0
+
+    return create_result(
+        passed=passed,
+        score=score,
+        output=output,
+        metadata={{
+            "evaluation_id": {eval_id},
+            "input": input_text,
+        }},
+    )
+"""
+
+
+def _generate_dsl_from_suite(name: str, cases: list[dict[str, Any]]) -> str:
+    """Generate DSL code from TestSuite data."""
+    now = datetime.now(timezone.utc).isoformat()
+    lines = [
+        f"# Auto-generated from TestSuite: {name}",
+        f"# Generated at: {now}",
+        "# This file replaces the old TestSuite configuration",
+        "",
+        "from evalgate_sdk import define_eval, create_result",
+        "",
+    ]
+
+    for i, case in enumerate(cases):
+        case_id = case.get("id", f"{name}-case-{i + 1}")
+        case_input = json.dumps(case.get("input", ""))
+        case_expected = json.dumps(case.get("expected"))
+        lines.extend(
+            [
+                f'define_eval("{case_id}", lambda ctx: _eval_{i}(ctx))',
+                "",
+                f"async def _eval_{i}(ctx):",
+                f"    # Original input: {case_input}",
+                f"    # Original expected: {case_expected}",
+                "    input_text = ctx.input",
+                "    # TODO: Replace with your actual agent/LLM call",
+                '    output = f"Agent response to: {input_text}"',
+                "    passed = len(output) > 0",
+                "    return create_result(passed=passed, score=100 if passed else 0, output=output)",
+                "",
+            ]
+        )
+
+    return "\n".join(lines)
+
+
+def _generate_placeholder_dsl(original_file: str) -> str:
+    """Generate placeholder DSL for files that need manual migration."""
+    now = datetime.now(timezone.utc).isoformat()
+    basename = Path(original_file).stem
+    return f"""# Migration placeholder for: {original_file}
+# Generated at: {now}
+# This file contains TestSuite usage that needs manual migration
+
+from evalgate_sdk import define_eval, create_result
+
+
+define_eval("placeholder-from-{basename}", _placeholder_eval)
+
+
+async def _placeholder_eval(ctx):
+    # TODO: Manually migrate TestSuite from {original_file}
+    input_text = ctx.input
+    output = f"Response to: {{input_text}}"
+    return create_result(
+        passed=len(output) > 0,
+        score=100 if len(output) > 0 else 0,
+        metadata={{"migrated_from": {json.dumps(original_file)}}},
+    )
+"""

evalgate_sdk/runtime/adapters/testsuite_to_dsl.py

@@ -0,0 +1,213 @@
+"""COMPAT-202: Legacy TestSuite → define_eval adapter.
+
+Converts legacy TestSuite instances to define_eval specifications
+without forcing migration. Enables lossless where possible.
+
+Port of ``runtime/adapters/testsuite-to-dsl.ts``.
+"""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+from evalgate_sdk.runtime.eval import create_result
+
+
+@dataclass
+class TestSuiteAdapterOptions:
+    """Adapter configuration options."""
+
+    include_provenance: bool = True
+    preserve_ids: bool = True
+    generate_helpers: bool = True
+
+
+@dataclass
+class TestDefinition:
+    """A single test definition from a legacy TestSuite."""
+
+    id: str = ""
+    input: str = ""
+    expected: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    assertions: list[Any] = field(default_factory=list)
+
+    @property
+    def has_assertions(self) -> bool:
+        return len(self.assertions) > 0
+
+    @property
+    def assertion_count(self) -> int:
+        return len(self.assertions)
+
+
+@dataclass
+class AdaptedSpec:
+    """An adapted EvalSpec from a legacy TestSuite."""
+
+    id: str = ""
+    name: str = ""
+    file_path: str = "legacy://testsuite"
+    position: dict[str, int] = field(default_factory=lambda: {"line": 1, "column": 1})
+    description: str = ""
+    tags: list[str] = field(default_factory=lambda: ["legacy", "migrated"])
+    metadata: dict[str, Any] = field(default_factory=dict)
+    config: dict[str, Any] = field(default_factory=dict)
+    executor: Callable[..., Any] | None = None
+
+
+def adapt_test_suite(
+    tests: list[TestDefinition],
+    suite_name: str = "legacy-suite",
+    options: TestSuiteAdapterOptions | None = None,
+) -> list[AdaptedSpec]:
+    """Convert TestSuite test definitions to adapted EvalSpec list.
+
+    Parameters
+    ----------
+    tests:
+        List of test definitions extracted from a TestSuite.
+    suite_name:
+        Name of the original suite.
+    options:
+        Adapter configuration.
+    """
+    opts = options or TestSuiteAdapterOptions()
+    specs: list[AdaptedSpec] = []
+
+    for test in tests:
+        spec_id = _generate_spec_id(test, suite_name, opts.preserve_ids)
+
+        metadata: dict[str, Any] = dict(test.metadata)
+        if opts.include_provenance:
+            metadata.update(
+                {
+                    "source": "legacy",
+                    "legacy_suite_name": suite_name,
+                    "legacy_test_id": test.id,
+                    "original_input": test.input,
+                    "original_expected": test.expected,
+                }
+            )
+
+        executor = _create_executor_from_test(test, opts.generate_helpers)
+
+        spec = AdaptedSpec(
+            id=spec_id,
+            name=test.id,
+            description=f"Legacy test: {test.id}",
+            tags=["legacy", "migrated"],
+            metadata=metadata,
+            executor=executor,
+        )
+        specs.append(spec)
+
+    return specs
+
+
+def generate_define_eval_code(
+    tests: list[TestDefinition],
+    suite_name: str = "legacy-suite",
+    options: TestSuiteAdapterOptions | None = None,
+) -> str:
+    """Generate define_eval() Python code from TestSuite data.
+
+    Returns generated Python source code as a string.
+    """
+    specs = adapt_test_suite(tests, suite_name, options)
+    now = datetime.now(timezone.utc).isoformat()
+
+    lines = [
+        f"# Auto-generated from TestSuite: {suite_name}",
+        f"# Generated at: {now}",
+        "# This file replaces the legacy TestSuite with define_eval() specifications",
+        "",
+        "from evalgate_sdk import define_eval, create_result",
+        "",
+    ]
+
+    for i, spec in enumerate(specs):
+        original_input = repr(spec.metadata.get("original_input", ""))
+        original_expected = repr(spec.metadata.get("original_expected"))
+
+        lines.extend(
+            [
+                f'define_eval("{spec.name}", _eval_{i}, options={{'
+                f'"description": "{spec.description}", "tags": {spec.tags!r}}})',
+                "",
+                f"async def _eval_{i}(ctx):",
+                f"    # Original input: {original_input}",
+                f"    # Original expected: {original_expected}",
+                "    input_text = ctx.input",
+                "",
+                "    # TODO: Replace with your actual agent/LLM call",
+                '    output = f"Response to: {input_text}"',
+                "",
+                "    # Legacy evaluation logic",
+                f"    expected = {original_expected}",
+                "    if expected is not None:",
+                "        passed = output == expected",
+                "    else:",
+                "        passed = len(output) > 0",
+                "    return create_result(passed=passed, score=100 if passed else 0, output=output)",
+                "",
+            ]
+        )
+
+    return "\n".join(lines)
+
+
+def _generate_spec_id(
+    test: TestDefinition,
+    suite_name: str,
+    preserve_ids: bool,
+) -> str:
+    """Generate specification ID for legacy test."""
+    import re
+
+    if preserve_ids and test.id and not test.id.startswith("case-"):
+        cleaned = re.sub(r"[^a-zA-Z0-9_-]", "_", test.id)[:20]
+        return cleaned
+
+    content = f"{suite_name}|{test.id}|{test.input}|{test.expected or ''}"
+    raw = base64.b64encode(content.encode("utf-8")).decode("ascii")
+    cleaned = raw.replace("+", "").replace("/", "").replace("=", "")[:20].lower()
+    return cleaned
+
+
+def _create_executor_from_test(
+    test: TestDefinition,
+    generate_helpers: bool,
+) -> Callable[..., Any]:
+    """Create executor function from test case."""
+
+    async def executor(ctx: Any) -> Any:
+        input_text = getattr(ctx, "input", str(ctx))
+
+        if test.expected is not None:
+            # In a real migration the caller replaces this with their agent call.
+            # The default stub echoes input so the comparison is meaningful.
+            output = input_text
+            exact_match = output == test.expected
+            return create_result(
+                passed=exact_match,
+                score=100 if exact_match else 0,
+                output=output,
+                metadata={
+                    "test_case_id": test.id,
+                    "original_input": test.input,
+                    "original_expected": test.expected,
+                },
+            )
+
+        return create_result(
+            passed=False,
+            score=0,
+            error="No executor or expected output available for legacy test",
+        )
+
+    return executor

evalgate_sdk/runtime/context.py

@@ -0,0 +1,68 @@
+"""EvalGate Runtime Context — Layer 1 Foundation.
+
+Execution context management for specifications.
+Port of ``runtime/context.ts``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def create_runtime_context(
+    input: str,
+    metadata: dict[str, Any] | None = None,
+    options: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Create a new execution context."""
+    return {
+        "input": input,
+        "metadata": metadata or {},
+        "options": options,
+    }
+
+
+def merge_runtime_contexts(
+    base: dict[str, Any],
+    *overrides: dict[str, Any],
+) -> dict[str, Any]:
+    """Merge contexts with proper precedence. Later contexts override earlier ones."""
+    if not base.get("input"):
+        raise ValueError("Base context must have a valid input")
+
+    merged = dict(base)
+    for override in overrides:
+        merged = {
+            "input": override.get("input") or merged.get("input"),
+            "metadata": {**(merged.get("metadata") or {}), **(override.get("metadata") or {})},
+            "options": (
+                {**(merged.get("options") or {}), **(override.get("options") or {})}
+                if override.get("options")
+                else merged.get("options")
+            ),
+        }
+    return merged
+
+
+def clone_runtime_context(context: dict[str, Any]) -> dict[str, Any]:
+    """Clone a context for safe modification."""
+    return {
+        "input": context.get("input"),
+        "metadata": dict(context.get("metadata") or {}),
+        "options": dict(context.get("options") or {}) if context.get("options") else None,
+    }
+
+
+def validate_runtime_context(context: Any) -> None:
+    """Validate context structure."""
+    if not isinstance(context, dict):
+        raise TypeError("Context must be a dict")
+
+    if not isinstance(context.get("input"), str):
+        raise TypeError("Context input must be a string")
+
+    if context.get("metadata") is not None and not isinstance(context.get("metadata"), dict):
+        raise TypeError("Context metadata must be a dict")
+
+    if context.get("options") is not None and not isinstance(context.get("options"), dict):
+        raise TypeError("Context options must be a dict")