ethibench 1.0.0__py3-none-any.whl

ethibench/__init__.py

@@ -0,0 +1,7 @@
+"""ethibench — An evaluation framework for AI pentesting agents."""
+
+__version__ = "0.1.0"
+
+from dotenv import load_dotenv
+
+load_dotenv()

ethibench/analysis/duplicates.py

@@ -0,0 +1,89 @@
+"""Duplicate detection — findings that matched GT in raw matching but were removed by bipartite."""
+
+import json
+from pathlib import Path
+
+from loguru import logger
+
+
+def calculate_duplicates_for_run(raw_matching: dict, matching: dict) -> tuple[int, list[str]]:
+    """Count duplicates and return their UUIDs.
+
+    A duplicate is a finding with non-empty selected_gt_ids in raw_matching
+    but empty selected_gt_ids in the final matching.
+    """
+    dup_count = 0
+    dup_uuids: list[str] = []
+
+    for uuid, raw_data in raw_matching.items():
+        raw_gt_ids = raw_data.get("selected_gt_ids", [])
+        if not raw_gt_ids:
+            continue
+        final_gt_ids = matching.get(uuid, {}).get("selected_gt_ids", [])
+        if not final_gt_ids:
+            dup_count += 1
+            dup_uuids.append(uuid)
+
+    return dup_count, dup_uuids
+
+
+def extract_duplicates(output_dir: Path, findings_file: Path | None = None) -> dict:
+    """Analyze duplicates across all subsets in an evaluation output directory.
+
+    Args:
+        output_dir: The evaluation_outputs directory containing raw_matchings/ and matchings/.
+        findings_file: Optional findings_parsed.jsonl for extracting full finding objects.
+
+    Returns:
+        Dict mapping subset name → {"count": int, "uuids": list[str], "findings": list[dict]}.
+    """
+    raw_dir = output_dir / "raw_matchings"
+    match_dir = output_dir / "matchings"
+
+    if not raw_dir.is_dir() or not match_dir.is_dir():
+        logger.warning("No raw_matchings/ or matchings/ found in output_dir.")
+        return {}
+
+    # Load findings if provided
+    findings_by_uuid: dict[str, dict] = {}
+    if findings_file and findings_file.exists():
+        with open(findings_file) as f:
+            for line in f:
+                if line.strip():
+                    item = json.loads(line)
+                    if "uuid" in item:
+                        findings_by_uuid[item["uuid"]] = item
+
+    results: dict[str, dict] = {}
+
+    for raw_file in sorted(raw_dir.glob("matchings_*.json")):
+        subset = raw_file.stem.replace("matchings_", "")
+        match_file = match_dir / raw_file.name
+
+        if not match_file.exists():
+            continue
+
+        with open(raw_file) as f:
+            raw_list = json.load(f)
+        with open(match_file) as f:
+            match_list = json.load(f)
+
+        all_uuids: list[str] = []
+        total_dups = 0
+
+        for raw_m, final_m in zip(raw_list, match_list):
+            count, uuids = calculate_duplicates_for_run(raw_m, final_m)
+            total_dups += count
+            all_uuids.extend(uuids)
+
+        unique_uuids = list(set(all_uuids))
+        dup_findings = [findings_by_uuid[u] for u in unique_uuids if u in findings_by_uuid]
+
+        results[subset] = {
+            "count": total_dups,
+            "uuids": unique_uuids,
+            "findings": dup_findings,
+        }
+        logger.info(f"  {subset}: {total_dups} duplicates ({len(unique_uuids)} unique)")
+
+    return results

ethibench/analysis/statistics.py

@@ -0,0 +1,95 @@
+"""Matching statistics — GT coverage, findings-per-GT distribution."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+from loguru import logger
+
+
+def load_ground_truths(gt_dir: Path) -> tuple[dict[str, dict], dict[str, list[str]]]:
+    """Load all GT files and return (gt_by_id, gt_ids_by_subset)."""
+    gt_by_id: dict[str, dict] = {}
+    gt_ids_by_subset: dict[str, list[str]] = defaultdict(list)
+
+    for gt_file in gt_dir.glob("*_gt.jsonl"):
+        with open(gt_file) as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                gt = json.loads(line)
+                gt_id = gt.get("id")
+                subset = gt.get("subset_name")
+                if gt_id and subset:
+                    gt_by_id[gt_id] = gt
+                    gt_ids_by_subset[subset].append(gt_id)
+
+    return gt_by_id, dict(gt_ids_by_subset)
+
+
+def compute_statistics(
+    output_dir: Path,
+    gt_dir: Path,
+) -> dict:
+    """Compute matching statistics from raw_matchings and ground truth.
+
+    Returns a dict with:
+      - total_gt: total ground truth entries
+      - total_subsets: number of subsets
+      - per_subset: dict of subset → {total_gt, matched_gt, unmatched_gt, findings_per_gt_avg, ...}
+      - unmatched_gts: list of GT entries with no findings across any run
+    """
+    gt_by_id, gt_ids_by_subset = load_ground_truths(gt_dir)
+    raw_dir = output_dir / "raw_matchings"
+
+    if not raw_dir.is_dir():
+        logger.warning("No raw_matchings/ found.")
+        return {}
+
+    # Build gt_id → list of finding UUIDs from raw matchings
+    gt_to_findings: dict[str, set[str]] = defaultdict(set)
+
+    for raw_file in raw_dir.glob("matchings_*.json"):
+        with open(raw_file) as f:
+            raw_list = json.load(f)
+        for raw_m in raw_list:
+            for uuid, data in raw_m.items():
+                for gt_id in data.get("selected_gt_ids", []):
+                    gt_to_findings[gt_id].add(uuid)
+
+    # Per-subset stats
+    per_subset: dict[str, dict] = {}
+    all_unmatched: list[dict] = []
+
+    for subset, gt_ids in sorted(gt_ids_by_subset.items()):
+        matched = 0
+        findings_counts: list[int] = []
+        for gt_id in gt_ids:
+            n = len(gt_to_findings.get(gt_id, set()))
+            findings_counts.append(n)
+            if n > 0:
+                matched += 1
+            else:
+                all_unmatched.append(gt_by_id[gt_id])
+
+        total = len(gt_ids)
+        avg = sum(findings_counts) / total if total else 0
+        max_f = max(findings_counts) if findings_counts else 0
+        min_f = min(findings_counts) if findings_counts else 0
+
+        per_subset[subset] = {
+            "total_gt": total,
+            "matched_gt": matched,
+            "unmatched_gt": total - matched,
+            "findings_per_gt_avg": round(avg, 2),
+            "findings_per_gt_max": max_f,
+            "findings_per_gt_min": min_f,
+        }
+        logger.info(f"  {subset}: {matched}/{total} GTs matched, avg {avg:.1f} findings/GT")
+
+    return {
+        "total_gt": len(gt_by_id),
+        "total_subsets": len(gt_ids_by_subset),
+        "per_subset": per_subset,
+        "unmatched_gts": all_unmatched,
+    }

ethibench/analysis/unmatched.py

@@ -0,0 +1,53 @@
+"""Extract findings with no ground truth match (false positives)."""
+
+import json
+from pathlib import Path
+
+from loguru import logger
+
+
+def extract_unmatched(output_dir: Path, findings_file: Path | None = None) -> dict:
+    """Find findings that had no match in raw_matchings (true false positives).
+
+    Args:
+        output_dir: The evaluation_outputs directory containing raw_matchings/.
+        findings_file: Optional findings_parsed.jsonl for full finding objects.
+
+    Returns:
+        Dict mapping subset → {"uuids": list[str], "findings": list[dict]}.
+    """
+    raw_dir = output_dir / "raw_matchings"
+    if not raw_dir.is_dir():
+        logger.warning("No raw_matchings/ found in output_dir.")
+        return {}
+
+    findings_by_uuid: dict[str, dict] = {}
+    if findings_file and findings_file.exists():
+        with open(findings_file) as f:
+            for line in f:
+                if line.strip():
+                    item = json.loads(line)
+                    if "uuid" in item:
+                        findings_by_uuid[item["uuid"]] = item
+
+    results: dict[str, dict] = {}
+
+    for raw_file in sorted(raw_dir.glob("matchings_*.json")):
+        subset = raw_file.stem.replace("matchings_", "")
+
+        with open(raw_file) as f:
+            raw_list = json.load(f)
+
+        unmatched_uuids: set[str] = set()
+        for raw_m in raw_list:
+            for uuid, data in raw_m.items():
+                if not data.get("selected_gt_ids", []):
+                    unmatched_uuids.add(uuid)
+
+        uuids = sorted(unmatched_uuids)
+        findings = [findings_by_uuid[u] for u in uuids if u in findings_by_uuid]
+
+        results[subset] = {"uuids": uuids, "findings": findings}
+        logger.info(f"  {subset}: {len(uuids)} unmatched findings")
+
+    return results