stackfix 0.1.0__py3-none-any.whl

cloudgym/__init__.py

@@ -0,0 +1,3 @@
+"""Cloud-Gym: IaC Repair Benchmark via Environment Inversion."""
+
+__version__ = "0.1.0"

cloudgym/benchmark/dataset.py

@@ -0,0 +1,188 @@
+"""Benchmark dataset management.
+
+Curates a balanced subset from the test split for evaluation.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BenchmarkEntry:
+    """A single benchmark entry."""
+
+    id: str
+    format: str
+    broken_config: str
+    errors: list[str]
+    warnings: list[str]
+    fault_types: list[str]
+    difficulty: str
+    gold_config: str
+    gold_hash: str
+
+
+class BenchmarkDataset:
+    """Manages the curated benchmark dataset."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.entries: list[BenchmarkEntry] = []
+        if self.path.exists():
+            self._load()
+
+    def _load(self):
+        """Load benchmark entries from JSONL."""
+        with open(self.path) as f:
+            for line in f:
+                data = json.loads(line)
+                self.entries.append(BenchmarkEntry(**{
+                    k: data[k] for k in BenchmarkEntry.__dataclass_fields__
+                    if k in data
+                }))
+        logger.info("Loaded %d benchmark entries from %s", len(self.entries), self.path)
+
+    def __len__(self) -> int:
+        return len(self.entries)
+
+    def __iter__(self):
+        return iter(self.entries)
+
+    @staticmethod
+    def build(
+        test_jsonl: str | Path,
+        output_path: str | Path,
+        target_size: int = 200,
+    ) -> BenchmarkDataset:
+        """Curate a benchmark dataset from the test split.
+
+        Curation rules:
+        - Single-fault only (one fault type per record)
+        - Balanced across categories and difficulties
+        - Min 10-line configs (non-trivial)
+        - Deduplicated per gold config (max 1 entry per gold hash per fault category)
+
+        Args:
+            test_jsonl: Path to test.jsonl from format_and_split.
+            output_path: Path to write benchmark.jsonl.
+            target_size: Target number of benchmark entries.
+
+        Returns:
+            BenchmarkDataset with curated entries.
+        """
+        test_path = Path(test_jsonl)
+        out_path = Path(output_path)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Load test records
+        records = []
+        with open(test_path) as f:
+            for line in f:
+                records.append(json.loads(line))
+
+        logger.info("Loaded %d test records for curation", len(records))
+
+        # Filter: single-fault, min 10 lines
+        candidates = [
+            r for r in records
+            if len(r.get("fault_types", [])) == 1
+            and len(r.get("broken_config", "").splitlines()) >= 10
+            and r.get("errors")  # Must have validation errors
+        ]
+        logger.info("%d candidates after filtering", len(candidates))
+
+        # Deduplicate: max 1 entry per (gold_hash, fault_category)
+        seen: set[tuple[str, str]] = set()
+        deduped = []
+        for r in candidates:
+            fault_id = r["fault_types"][0]
+            category = fault_id.split(".")[0] if "." in fault_id else fault_id
+            key = (r.get("gold_hash", ""), category)
+            if key not in seen:
+                seen.add(key)
+                deduped.append(r)
+        logger.info("%d after deduplication", len(deduped))
+
+        # Balance across categories and difficulties
+        selected = _balance_select(deduped, target_size)
+        logger.info("Selected %d entries for benchmark", len(selected))
+
+        # Write benchmark JSONL
+        with open(out_path, "w") as f:
+            for r in selected:
+                entry = {
+                    "id": r["id"],
+                    "format": r["format"],
+                    "broken_config": r["broken_config"],
+                    "errors": r["errors"],
+                    "warnings": r.get("warnings", []),
+                    "fault_types": r["fault_types"],
+                    "difficulty": r["difficulty"],
+                    "gold_config": r["gold_config"],
+                    "gold_hash": r.get("gold_hash", ""),
+                }
+                f.write(json.dumps(entry) + "\n")
+
+        # Write metadata
+        meta = {
+            "total_entries": len(selected),
+            "source": str(test_path),
+            "category_distribution": _count_categories(selected),
+            "difficulty_distribution": _count_field(selected, "difficulty"),
+            "format_distribution": _count_field(selected, "format"),
+        }
+        meta_path = out_path.parent / "benchmark_meta.json"
+        with open(meta_path, "w") as f:
+            json.dump(meta, f, indent=2)
+
+        return BenchmarkDataset(out_path)
+
+
+def _balance_select(records: list[dict], target: int) -> list[dict]:
+    """Select records with balanced category/difficulty distribution."""
+    by_category: dict[str, list[dict]] = {}
+    for r in records:
+        fault_id = r["fault_types"][0]
+        cat = fault_id.split(".")[0] if "." in fault_id else fault_id
+        by_category.setdefault(cat, []).append(r)
+
+    if not by_category:
+        return []
+
+    per_category = max(1, target // len(by_category))
+    selected = []
+
+    for cat, cat_records in by_category.items():
+        # Within category, balance by difficulty
+        by_diff: dict[str, list[dict]] = {}
+        for r in cat_records:
+            by_diff.setdefault(r["difficulty"], []).append(r)
+
+        per_diff = max(1, per_category // max(len(by_diff), 1))
+        for diff, diff_records in by_diff.items():
+            selected.extend(diff_records[:per_diff])
+
+    return selected[:target]
+
+
+def _count_categories(records: list[dict]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for r in records:
+        fault_id = r["fault_types"][0]
+        cat = fault_id.split(".")[0] if "." in fault_id else fault_id
+        counts[cat] = counts.get(cat, 0) + 1
+    return counts
+
+
+def _count_field(records: list[dict], field: str) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for r in records:
+        val = r.get(field, "unknown")
+        counts[val] = counts.get(val, 0) + 1
+    return counts

cloudgym/benchmark/evaluator.py

@@ -0,0 +1,275 @@
+"""Evaluation harness for IaC repair benchmark.
+
+Validates model-generated fixes via terraform validate / cfn-lint
+and computes pass@k metrics using the unbiased Codex estimator.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import math
+import shutil
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Awaitable
+
+from cloudgym.benchmark.dataset import BenchmarkDataset, BenchmarkEntry
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class EvalReport:
+    """Evaluation report for a model on the benchmark."""
+
+    model_name: str
+    n_attempts: int
+    total_entries: int
+    pass_at_k: dict[int, float] = field(default_factory=dict)
+    per_category: dict[str, dict[int, float]] = field(default_factory=dict)
+    per_difficulty: dict[str, dict[int, float]] = field(default_factory=dict)
+    per_format: dict[str, dict[int, float]] = field(default_factory=dict)
+    raw_results: list[dict] = field(default_factory=list)
+
+
+# Type alias for model repair function
+ModelFn = Callable[[str, list[str]], Awaitable[str]]
+
+
+class Evaluator:
+    """Evaluates model repair attempts against the benchmark."""
+
+    # Max concurrent validations (terraform/cfn-lint)
+    DEFAULT_CONCURRENCY = 8
+
+    def __init__(self, benchmark_path: str | Path, concurrency: int | None = None):
+        self.dataset = BenchmarkDataset(benchmark_path)
+        self._tf_cache_dir: Path | None = None
+        self._concurrency = concurrency or self.DEFAULT_CONCURRENCY
+
+    async def evaluate_model(
+        self,
+        model_fn: ModelFn,
+        model_name: str = "unknown",
+        n_attempts: int = 5,
+        k_values: list[int] | None = None,
+    ) -> EvalReport:
+        """Evaluate a model's repair ability on the benchmark.
+
+        Args:
+            model_fn: Async function (broken_config, errors) -> repaired_config.
+            model_name: Name of the model being evaluated.
+            n_attempts: Number of repair attempts per benchmark entry.
+            k_values: k values for pass@k computation. Default [1, 3].
+
+        Returns:
+            EvalReport with pass@k metrics and breakdowns.
+        """
+        if k_values is None:
+            k_values = [1, 3]
+
+        # Phase 1: Generate all repairs (serial — model inference isn't parallel-safe)
+        # Store as list of (entry, [repaired_configs])
+        all_repairs: list[tuple[BenchmarkEntry, list[str | None]]] = []
+        for entry in self.dataset:
+            repairs: list[str | None] = []
+            for attempt in range(n_attempts):
+                try:
+                    repaired = await model_fn(entry.broken_config, entry.errors)
+                    repairs.append(repaired)
+                except Exception:
+                    logger.exception(
+                        "Model failed on entry %s attempt %d", entry.id, attempt
+                    )
+                    repairs.append(None)
+            all_repairs.append((entry, repairs))
+
+        # Phase 2: Validate all repairs concurrently
+        sem = asyncio.Semaphore(self._concurrency)
+
+        async def _validate(repaired: str | None, fmt: str) -> bool:
+            if repaired is None:
+                return False
+            async with sem:
+                return await self._check_repair(repaired, fmt)
+
+        raw_results = []
+        for entry, repairs in all_repairs:
+            tasks = [_validate(r, entry.format) for r in repairs]
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            passes = sum(
+                1 for r in results if r is True
+            )
+            raw_results.append({
+                "id": entry.id,
+                "format": entry.format,
+                "fault_types": entry.fault_types,
+                "difficulty": entry.difficulty,
+                "n": n_attempts,
+                "c": passes,
+            })
+
+        # Compute metrics
+        report = EvalReport(
+            model_name=model_name,
+            n_attempts=n_attempts,
+            total_entries=len(self.dataset),
+            raw_results=raw_results,
+        )
+
+        # Overall pass@k
+        for k in k_values:
+            report.pass_at_k[k] = _compute_pass_at_k(raw_results, k)
+
+        # Per-category breakdown
+        categories = set()
+        for r in raw_results:
+            for ft in r["fault_types"]:
+                cat = ft.split(".")[0] if "." in ft else ft
+                categories.add(cat)
+
+        for cat in categories:
+            cat_results = [
+                r for r in raw_results
+                if any(ft.startswith(cat) for ft in r["fault_types"])
+            ]
+            report.per_category[cat] = {
+                k: _compute_pass_at_k(cat_results, k) for k in k_values
+            }
+
+        # Per-difficulty breakdown
+        difficulties = {r["difficulty"] for r in raw_results}
+        for diff in difficulties:
+            diff_results = [r for r in raw_results if r["difficulty"] == diff]
+            report.per_difficulty[diff] = {
+                k: _compute_pass_at_k(diff_results, k) for k in k_values
+            }
+
+        # Per-format breakdown
+        formats = {r["format"] for r in raw_results}
+        for fmt in formats:
+            fmt_results = [r for r in raw_results if r["format"] == fmt]
+            report.per_format[fmt] = {
+                k: _compute_pass_at_k(fmt_results, k) for k in k_values
+            }
+
+        return report
+
+    async def _ensure_tf_cache(self) -> Path:
+        """Create a cached terraform init directory for fast validation."""
+        if self._tf_cache_dir and self._tf_cache_dir.exists():
+            return self._tf_cache_dir
+
+        cache = Path(tempfile.mkdtemp(prefix="cloudgym_tf_cache_"))
+        # Write a minimal .tf that requires the AWS provider
+        (cache / "providers.tf").write_text(
+            'terraform {\n  required_providers {\n'
+            '    aws = {\n      source  = "hashicorp/aws"\n'
+            '      version = "~> 5.0"\n    }\n  }\n}\n'
+        )
+        import asyncio
+        proc = await asyncio.create_subprocess_exec(
+            "terraform", "init", "-backend=false", "-no-color",
+            cwd=cache,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await proc.communicate()
+        # Remove the providers.tf so it doesn't interfere with validation
+        (cache / "providers.tf").unlink(missing_ok=True)
+        self._tf_cache_dir = cache
+        logger.info("Cached terraform init at %s", cache)
+        return cache
+
+    async def _check_repair(self, repaired: str, iac_format: str) -> bool:
+        """Check if a repaired config passes validation."""
+        if not repaired or not repaired.strip():
+            return False
+
+        if iac_format in ("terraform", "opentofu"):
+            return await self._check_terraform_repair(repaired)
+        else:
+            return await self._check_cf_repair(repaired)
+
+    async def _check_terraform_repair(self, repaired: str) -> bool:
+        """Validate terraform repair using cached init directory."""
+        try:
+            cache = await self._ensure_tf_cache()
+            tmpdir = Path(tempfile.mkdtemp(prefix="cloudgym_eval_"))
+            # Symlink .terraform and lock file from cache (much faster than copy)
+            tf_dir = cache / ".terraform"
+            lock_file = cache / ".terraform.lock.hcl"
+            if tf_dir.exists():
+                (tmpdir / ".terraform").symlink_to(tf_dir)
+            if lock_file.exists():
+                (tmpdir / ".terraform.lock.hcl").symlink_to(lock_file)
+
+            (tmpdir / "repaired.tf").write_text(repaired)
+
+            # Skip init, go straight to validate
+            proc = await asyncio.create_subprocess_exec(
+                "terraform", "validate", "-json", "-no-color",
+                cwd=tmpdir,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            stdout, _ = await proc.communicate()
+            result = json.loads(stdout.decode(errors="replace"))
+            return result.get("valid", False)
+        except Exception:
+            logger.exception("Terraform validation error during eval")
+            return False
+        finally:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+    async def _check_cf_repair(self, repaired: str) -> bool:
+        """Validate CloudFormation repair via cfn-lint."""
+        tmpdir = Path(tempfile.mkdtemp(prefix="cloudgym_eval_"))
+        tmp_file = tmpdir / "repaired.yaml"
+        tmp_file.write_text(repaired)
+        try:
+            from cloudgym.validator.cloudformation import validate
+            result = await validate(tmp_file)
+            return result.valid and len(result.errors) == 0
+        except Exception:
+            logger.exception("CF validation error during eval")
+            return False
+        finally:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _compute_pass_at_k(results: list[dict], k: int) -> float:
+    """Compute pass@k using the unbiased Codex estimator.
+
+    pass@k = 1 - C(n-c, k) / C(n, k)
+
+    Where n = total attempts, c = number of correct attempts.
+    """
+    if not results:
+        return 0.0
+
+    total = 0.0
+    for r in results:
+        n = r["n"]
+        c = r["c"]
+        if n < k:
+            # Not enough attempts
+            total += 1.0 if c > 0 else 0.0
+        elif c == 0:
+            total += 0.0
+        elif n - c < k:
+            total += 1.0
+        else:
+            total += 1.0 - _comb(n - c, k) / _comb(n, k)
+
+    return total / len(results)
+
+
+def _comb(n: int, k: int) -> float:
+    """Compute combination C(n, k) using math.comb."""
+    if k < 0 or k > n:
+        return 0.0
+    return float(math.comb(n, k))

cloudgym/cli.py

@@ -0,0 +1,61 @@
+"""Cloud-Gym CLI entry point."""
+
+from __future__ import annotations
+
+import click
+
+
+@click.group()
+@click.version_option(package_name="cloud-gym")
+def main():
+    """Cloud-Gym: IaC Repair Benchmark via Environment Inversion."""
+
+
+@main.command()
+@click.option("--skip-github", is_flag=True)
+@click.option("--skip-registry", is_flag=True)
+@click.option("--skip-aws", is_flag=True)
+@click.option("--skip-validate", is_flag=True)
+def scrape(skip_github: bool, skip_registry: bool, skip_aws: bool, skip_validate: bool):
+    """Collect gold IaC configurations from various sources."""
+    import asyncio
+    from scripts.scrape import run_scrape
+
+    asyncio.run(run_scrape(skip_github, skip_registry, skip_aws, skip_validate))
+
+
+@main.command()
+def taxonomy():
+    """Display the fault taxonomy."""
+    from rich.console import Console
+    from rich.table import Table
+
+    from cloudgym.taxonomy.base import REGISTRY, FaultCategory, IaCFormat
+    import cloudgym.taxonomy.terraform  # noqa: F401 — triggers registration
+    import cloudgym.taxonomy.cloudformation  # noqa: F401
+
+    console = Console()
+    table = Table(title="Cloud-Gym Fault Taxonomy")
+    table.add_column("ID", style="cyan")
+    table.add_column("Category", style="magenta")
+    table.add_column("Severity")
+    table.add_column("Formats")
+    table.add_column("Description")
+
+    for fault in sorted(REGISTRY.all(), key=lambda f: (f.category.name, f.severity.value)):
+        formats = ", ".join(f.value for f in fault.applicable_formats)
+        sev_style = {"low": "green", "medium": "yellow", "high": "red"}[fault.severity.value]
+        table.add_row(
+            fault.id,
+            fault.category.name,
+            f"[{sev_style}]{fault.severity.value}[/{sev_style}]",
+            formats,
+            fault.description,
+        )
+
+    console.print(table)
+    console.print(f"\nTotal fault types: {len(REGISTRY)}")
+    for cat in FaultCategory:
+        count = len(REGISTRY.list_by_category(cat))
+        if count > 0:
+            console.print(f"  {cat.name}: {count}")

cloudgym/fixer/__init__.py

@@ -0,0 +1 @@
+"""stackfix: AI-powered Infrastructure-as-Code repair tool."""