metamorphic-guard 1.1.0__py3-none-any.whl

metamorphic_guard/__init__.py

@@ -0,0 +1,47 @@
+"""
+Metamorphic Guard v1 - A Python library for comparing program versions using metamorphic testing.
+"""
+
+from .specs import task, Spec, Property, MetamorphicRelation
+from .generators import gen_top_k_inputs
+from .relations import permute_input, add_noise_below_min
+from .stability import multiset_equal
+
+__version__ = "1.1.0"
+
+
+@task("top_k")
+def top_k_spec() -> Spec:
+    """Specification for the top_k task."""
+    return Spec(
+        gen_inputs=gen_top_k_inputs,
+        properties=[
+            Property(
+                check=lambda out, L, k: len(out) == min(k, len(L)),
+                description="Output length equals min(k, len(L))"
+            ),
+            Property(
+                check=lambda out, L, k: sorted(out, reverse=True) == out,
+                description="Output is sorted in descending order"
+            ),
+            Property(
+                check=lambda out, L, k: all(x in L for x in out),
+                description="All output elements are from input list"
+            )
+        ],
+        relations=[
+            MetamorphicRelation(
+                name="permute_input",
+                transform=permute_input,
+                expect="equal"
+            ),
+            MetamorphicRelation(
+                name="add_noise_below_min", 
+                transform=add_noise_below_min,
+                expect="equal"
+            )
+        ],
+        equivalence=multiset_equal,
+        fmt_in=lambda args: f"L={args[0]}, k={args[1]}",
+        fmt_out=lambda result: f"top_k={result}"
+    )

metamorphic_guard/cli.py

@@ -0,0 +1,163 @@
+"""
+Command-line interface for Metamorphic Guard.
+"""
+
+import sys
+
+import click
+
+from .gate import decide_adopt
+from .harness import run_eval
+from .specs import list_tasks
+from .util import write_report
+
+
+@click.command()
+@click.option("--task", required=True, help="Task name to evaluate")
+@click.option("--baseline", required=True, help="Path to baseline implementation")
+@click.option("--candidate", required=True, help="Path to candidate implementation")
+@click.option("--n", default=400, show_default=True, help="Number of test cases to generate")
+@click.option("--seed", default=42, show_default=True, help="Random seed for generators")
+@click.option("--timeout-s", default=2.0, show_default=True, help="Timeout per test (seconds)")
+@click.option("--mem-mb", default=512, show_default=True, help="Memory limit per test (MB)")
+@click.option("--alpha", default=0.05, show_default=True, help="Significance level for bootstrap CI")
+@click.option(
+    "--improve-delta",
+    default=0.02,
+    show_default=True,
+    help="Minimum improvement threshold for adoption",
+)
+@click.option("--violation-cap", default=25, show_default=True, help="Maximum violations to record")
+@click.option(
+    "--parallel",
+    type=int,
+    default=1,
+    show_default=True,
+    help="Number of concurrent workers for sandbox execution",
+)
+@click.option(
+    "--bootstrap-samples",
+    type=int,
+    default=1000,
+    show_default=True,
+    help="Bootstrap resamples for confidence interval estimation",
+)
+@click.option(
+    "--ci-method",
+    type=click.Choice(["bootstrap", "newcombe", "wilson"], case_sensitive=False),
+    default="bootstrap",
+    show_default=True,
+    help="Method for the pass-rate delta confidence interval",
+)
+@click.option(
+    "--rr-ci-method",
+    type=click.Choice(["log"], case_sensitive=False),
+    default="log",
+    show_default=True,
+    help="Method for relative risk confidence interval",
+)
+def main(
+    task: str,
+    baseline: str,
+    candidate: str,
+    n: int,
+    seed: int,
+    timeout_s: float,
+    mem_mb: int,
+    alpha: float,
+    improve_delta: float,
+    violation_cap: int,
+    parallel: int,
+    bootstrap_samples: int,
+    ci_method: str,
+    rr_ci_method: str,
+) -> None:
+    """Compare baseline and candidate implementations using metamorphic testing."""
+
+    available_tasks = list_tasks()
+    if task not in available_tasks:
+        click.echo(
+            f"Error: Task '{task}' not found. Available tasks: {available_tasks}",
+            err=True,
+        )
+        sys.exit(1)
+
+    try:
+        click.echo(f"Running evaluation: {task}")
+        click.echo(f"Baseline: {baseline}")
+        click.echo(f"Candidate: {candidate}")
+        click.echo(f"Test cases: {n}, Seed: {seed}")
+        click.echo(f"Parallel workers: {parallel}")
+        click.echo(f"CI method: {ci_method}")
+        click.echo(f"RR CI method: {rr_ci_method}")
+
+        result = run_eval(
+            task_name=task,
+            baseline_path=baseline,
+            candidate_path=candidate,
+            n=n,
+            seed=seed,
+            timeout_s=timeout_s,
+            mem_mb=mem_mb,
+            alpha=alpha,
+            violation_cap=violation_cap,
+            parallel=parallel,
+            improve_delta=improve_delta,
+            bootstrap_samples=bootstrap_samples,
+            ci_method=ci_method,
+            rr_ci_method=rr_ci_method,
+        )
+
+        decision = decide_adopt(result, improve_delta)
+        result["decision"] = decision
+
+        report_path = write_report(result)
+
+        click.echo("\n" + "=" * 60)
+        click.echo("EVALUATION SUMMARY")
+        click.echo("=" * 60)
+        click.echo(f"Task: {result['task']}")
+        click.echo(f"Test cases: {result['n']}")
+        click.echo(f"Seed: {result['seed']}")
+        click.echo()
+        click.echo("BASELINE:")
+        click.echo(
+            f"  Pass rate: {result['baseline']['pass_rate']:.3f} "
+            f"({result['baseline']['passes']}/{result['baseline']['total']})"
+        )
+        click.echo()
+        click.echo("CANDIDATE:")
+        click.echo(
+            f"  Pass rate: {result['candidate']['pass_rate']:.3f} "
+            f"({result['candidate']['passes']}/{result['candidate']['total']})"
+        )
+        click.echo(f"  Property violations: {len(result['candidate']['prop_violations'])}")
+        click.echo(f"  MR violations: {len(result['candidate']['mr_violations'])}")
+        click.echo()
+        click.echo("IMPROVEMENT:")
+        click.echo(f"  Delta: {result['delta_pass_rate']:.3f}")
+        click.echo(f"  95% CI: [{result['delta_ci'][0]:.3f}, {result['delta_ci'][1]:.3f}]")
+        click.echo(f"  Relative risk: {result['relative_risk']:.3f}")
+        rr_ci = result["relative_risk_ci"]
+        click.echo(f"  RR 95% CI: [{rr_ci[0]:.3f}, {rr_ci[1]:.3f}]")
+        click.echo()
+        click.echo("DECISION:")
+        click.echo(f"  Adopt: {decision['adopt']}")
+        click.echo(f"  Reason: {decision['reason']}")
+        click.echo()
+        click.echo(f"Report saved to: {report_path}")
+
+        if decision["adopt"]:
+            click.echo("✅ Candidate accepted!")
+            sys.exit(0)
+
+        click.echo("❌ Candidate rejected!")
+        sys.exit(1)
+
+    except Exception as exc:  # pragma: no cover - defensive surface
+        click.echo(f"Error during evaluation: {exc}", err=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

metamorphic_guard/gate.py

@@ -0,0 +1,59 @@
+"""
+Adoption gate logic for deciding whether to accept a candidate implementation.
+"""
+
+from typing import Dict, Any
+
+
+def decide_adopt(
+    result: Dict[str, Any], 
+    improve_delta: float = 0.02, 
+    min_pass_rate: float = 0.80
+) -> Dict[str, Any]:
+    """
+    Decide whether to adopt the candidate based on evaluation results.
+    
+    Args:
+        result: Full evaluation result from harness
+        improve_delta: Minimum improvement threshold for CI lower bound
+        min_pass_rate: Minimum pass rate required for candidate
+        
+    Returns:
+        Dict with 'adopt' boolean and 'reason' string
+    """
+    candidate = result["candidate"]
+    delta_ci = result["delta_ci"]
+    
+    # Check for property violations
+    if candidate["prop_violations"]:
+        return {
+            "adopt": False,
+            "reason": f"Property violations: {len(candidate['prop_violations'])} violations found"
+        }
+    
+    # Check for metamorphic relation violations
+    if candidate["mr_violations"]:
+        return {
+            "adopt": False,
+            "reason": f"Metamorphic relation violations: {len(candidate['mr_violations'])} violations found"
+        }
+    
+    # Check minimum pass rate
+    if candidate["pass_rate"] < min_pass_rate:
+        return {
+            "adopt": False,
+            "reason": f"Pass rate too low: {candidate['pass_rate']:.3f} < {min_pass_rate}"
+        }
+    
+    # Check improvement threshold
+    if delta_ci[0] < improve_delta:
+        return {
+            "adopt": False,
+            "reason": f"Improvement insufficient: CI lower bound {delta_ci[0]:.3f} < {improve_delta}"
+        }
+    
+    # All conditions met
+    return {
+        "adopt": True,
+        "reason": "meets_gate"
+    }

metamorphic_guard/generators.py

@@ -0,0 +1,126 @@
+"""
+Input generators for test cases.
+"""
+
+import random
+from typing import Callable, List, Sequence, Tuple
+
+
+def gen_top_k_inputs(n: int, seed: int) -> List[Tuple[List[int], int]]:
+    """
+    Generate diverse test cases for the top_k task.
+
+    The generator aims to cover the following scenarios:
+    * Empty lists and single-element lists.
+    * Lists with heavy duplication.
+    * Already sorted ascending/descending inputs.
+    * Very long lists (up to ~200 elements).
+    * k larger than the list length and k == 0.
+    * Negative-only, mixed-sign, and extreme magnitude values.
+    """
+    rng = random.Random(seed)
+    scenarios: Sequence[Callable[[random.Random], Tuple[List[int], int]]] = (
+        _case_empty,
+        _case_single,
+        _case_duplicates,
+        _case_sorted_ascending,
+        _case_sorted_descending,
+        _case_long,
+        _case_random_dense,
+        _case_k_zero,
+        _case_k_bigger_than_len,
+        _case_negatives_only,
+        _case_extreme_values,
+        _case_small_range_many_duplicates,
+    )
+
+    test_cases: List[Tuple[List[int], int]] = []
+    for i in range(n):
+        case_fn = rng.choice(scenarios)
+        test_cases.append(case_fn(rng))
+    return test_cases
+
+
+def _case_empty(rng: random.Random) -> Tuple[List[int], int]:
+    return [], 0
+
+
+def _case_single(rng: random.Random) -> Tuple[List[int], int]:
+    value = rng.randint(-1000, 1000)
+    return [value], rng.choice([0, 1, 2])
+
+
+def _case_duplicates(rng: random.Random) -> Tuple[List[int], int]:
+    base = rng.randint(-100, 100)
+    dup_count = rng.randint(3, 15)
+    noise = [rng.randint(-150, 150) for _ in range(rng.randint(1, 6))]
+    values = [base] * dup_count + noise
+    rng.shuffle(values)
+    k = rng.randint(1, max(1, len(values)))
+    return values, k
+
+
+def _case_sorted_ascending(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(2, 40)
+    values = sorted(rng.randint(-500, 500) for _ in range(length))
+    k = rng.randint(1, length)
+    return values, k
+
+
+def _case_sorted_descending(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(2, 40)
+    values = sorted((rng.randint(-500, 500) for _ in range(length)), reverse=True)
+    k = rng.randint(1, length)
+    return values, k
+
+
+def _case_long(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(100, 200)
+    values = [rng.randint(-10**4, 10**4) for _ in range(length)]
+    k = rng.randint(1, length)
+    return values, k
+
+
+def _case_random_dense(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(5, 80)
+    values = [rng.randint(-500, 500) for _ in range(length)]
+    k = rng.randint(0, length + 5)
+    return values, k
+
+
+def _case_k_zero(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(1, 25)
+    values = [rng.randint(-200, 200) for _ in range(length)]
+    return values, 0
+
+
+def _case_k_bigger_than_len(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(1, 25)
+    values = [rng.randint(-100, 100) for _ in range(length)]
+    k = length + rng.randint(1, 20)
+    return values, k
+
+
+def _case_negatives_only(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(3, 30)
+    values = [rng.randint(-10**5, -1) for _ in range(length)]
+    k = rng.randint(1, length)
+    return values, k
+
+
+def _case_extreme_values(rng: random.Random) -> Tuple[List[int], int]:
+    palette = [-10**9, -10**6, -10**3, 0, 10**3, 10**6, 10**9]
+    length = rng.randint(5, 30)
+    values = [rng.choice(palette) for _ in range(length)]
+    k = rng.randint(1, length)
+    return values, k
+
+
+def _case_small_range_many_duplicates(rng: random.Random) -> Tuple[List[int], int]:
+    length = rng.randint(10, 60)
+    base_values = [rng.randint(-3, 3) for _ in range(length)]
+    values = base_values + [rng.randint(-3, 3) for _ in range(length // 2)]
+    rng.shuffle(values)
+    k = rng.randint(1, max(1, len(values)))
+    return values, k
+