gitinject 0.1.0__py3-none-any.whl

gitinject/scanner/report_generator.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+
+from .types import MODEL_PRICING, AttackHypothesis, EffectivePromptContext, ScanCost, ValidationResult
+
+
+def _severity_badge(severity: str) -> str:
+    return {"high": "🔴", "medium": "🟡", "low": "🟢"}.get(severity, "⚪")
+
+
+def generate(
+    context: EffectivePromptContext,
+    results: list[ValidationResult],
+    discarded: list[tuple[AttackHypothesis, str]],
+    baseline_findings: list[dict],
+    output_dir: str,
+    scan_cost: ScanCost | None = None,
+) -> tuple[str, str]:
+    os.makedirs(output_dir, exist_ok=True)
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    confirmed = [r for r in results if r.status == "confirmed"]
+    unconfirmed = [r for r in results if r.status == "unconfirmed"]
+    skipped = [r for r in results if r.status == "skipped"]
+
+    scan_cost = scan_cost or ScanCost()
+    total_minutes = scan_cost.total_billable_minutes or sum(r.billable_minutes for r in results)
+    total_wall = scan_cost.total_wall_seconds or sum(r.wall_seconds for r in results)
+
+    md_lines = [
+        f"# Vulnerability Scan Report — {context.workflow_id}",
+        f"\n**Generated:** {timestamp}",
+        f"**Provider:** {context.provider}",
+        f"**Trigger:** {context.trigger_event}",
+        f"**Tool restrictions:** {', '.join(context.tool_restrictions) or 'none'}",
+        f"**persist-credentials:** {context.has_persist_credentials}",
+        "",
+        "---",
+        "",
+        "## Reconstructed Agent Prompt",
+        "```",
+        context.reconstructed_prompt[:2000],
+        "```" if len(context.reconstructed_prompt) <= 2000 else "```\n*(truncated)*",
+        "",
+        "---",
+        "",
+        "## Summary",
+        "| Metric | Value |",
+        "|--------|-------|",
+        f"| Hypotheses generated | {len(results) + len(discarded)} |",
+        f"| Filtered (pre-pass + LLM ranker) | {len(discarded)} |",
+        f"| Live-validated | {len(results)} |",
+        f"| **Confirmed** | **{len(confirmed)}** |",
+        f"| Unconfirmed | {len(unconfirmed)} |",
+        f"| Execution/evaluation errors | {sum(r.status == 'error' for r in results)} |",
+        f"| Skipped (dry-run) | {len(skipped)} |",
+        "",
+    ]
+
+    if confirmed:
+        md_lines += [
+            "---",
+            "",
+            "## Confirmed Vulnerabilities",
+            "",
+        ]
+        for r in confirmed:
+            h = r.hypothesis
+            tags = ", ".join(f"`{t}`" for t in h.tags) if h.tags else "—"
+            md_lines += [
+                f"### {_severity_badge(h.severity)} `{h.id}` — {h.mitre_category}",
+                f"**Tags:** {tags}  ",
+                f"**Attack goal:** {h.attack_goal}  ",
+                f"**Recipe:** {r.payload_used}  ",
+                f"**Success rate:** {r.success_rate}  ",
+                f"**Found in iteration:** {r.iteration}  ",
+                f"**Mitigation:** {r.suggested_mitigation}",
+                "",
+                "**Reproduction:**",
+                "```bash",
+                f"uv run gitinject run --workflow {context.workflow_id} --scenario {h.id}",
+                "```",
+                "",
+            ]
+
+    if unconfirmed:
+        md_lines += [
+            "---",
+            "",
+            "## Unconfirmed Hypotheses",
+            "",
+            "| ID | Category | Recipe | Goal | Failure reason |",
+            "|----|----------|--------|------|----------------|",
+        ]
+        for r in unconfirmed:
+            h = r.hypothesis
+            md_lines.append(
+                f"| `{h.id}` | {h.mitre_category} | {r.payload_used} "
+                f"| {h.attack_goal[:60]} | {r.failure_reason or 'unknown'} |"
+            )
+        md_lines.append("")
+
+    if discarded:
+        md_lines += [
+            "---",
+            "",
+            "## Filtered Hypotheses",
+            "",
+            "| ID | Category | Goal | Filter reason |",
+            "|----|----------|------|---------------|",
+        ]
+        for h, reason in discarded:
+            md_lines.append(f"| `{h.id}` | {h.mitre_category} | {h.attack_goal[:60]} | {reason} |")
+        md_lines.append("")
+
+    if baseline_findings:
+        md_lines += [
+            "---",
+            "",
+            "## Baseline Tool Findings",
+            "",
+        ]
+        by_tool: dict[str, list[dict]] = {}
+        for f in baseline_findings:
+            by_tool.setdefault(f.get("tool", "unknown"), []).append(f)
+        for tool, findings in by_tool.items():
+            md_lines += [f"### {tool} ({len(findings)} findings)", ""]
+            for f in findings:
+                if "error" in f:
+                    md_lines.append(f"- ⚠️  {f['error']}")
+                else:
+                    sev = f.get("severity", "?")
+                    rule = f.get("rule", "?")
+                    msg = f.get("message", "")
+                    loc = f.get("location", "")
+                    md_lines.append(f"- **{sev}** [{rule}] {msg} — `{loc}`")
+            md_lines.append("")
+
+    md_lines += [
+        "---",
+        "",
+        "## Cost Summary",
+        "",
+        "| Item | Value |",
+        "|------|-------|",
+        f"| Scanner input tokens | {scan_cost.total_input_tokens:,} |",
+        f"| Scanner output tokens | {scan_cost.total_output_tokens:,} |",
+        f"| Estimated scanner API cost | ${scan_cost.total_usd:.4f} |",
+        f"| GitHub Actions billable minutes | {total_minutes:.2f} |",
+        f"| Wall-clock seconds | {total_wall:.1f} |",
+        "",
+    ]
+    if scan_cost.token_usage_by_model:
+        md_lines += [
+            "### Per-model token usage",
+            "",
+            "| Model | Input tokens | Output tokens | Cost (USD) | Priced? |",
+            "|-------|--------------|---------------|------------|---------|",
+        ]
+        for model, t in sorted(scan_cost.token_usage_by_model.items()):
+            in_tok = t.get("input", 0)
+            out_tok = t.get("output", 0)
+            priced = model in MODEL_PRICING
+            in_price, out_price = MODEL_PRICING.get(model, (0.0, 0.0))
+            cost_usd = (in_tok * in_price + out_tok * out_price) / 1_000_000
+            md_lines.append(
+                f"| `{model}` | {in_tok:,} | {out_tok:,} | "
+                f"${cost_usd:.4f} | {'yes' if priced else '**no — update MODEL_PRICING**'} |"
+            )
+        md_lines.append("")
+
+    md_content = "\n".join(md_lines)
+    md_path = os.path.join(output_dir, f"{context.workflow_id}.md")
+    with open(md_path, "w") as f:
+        f.write(md_content)
+
+    json_data = {
+        "timestamp": timestamp,
+        "workflow_id": context.workflow_id,
+        "provider": context.provider,
+        "trigger_event": context.trigger_event,
+        "tool_restrictions": context.tool_restrictions,
+        "has_persist_credentials": context.has_persist_credentials,
+        "confirmed": [_result_to_dict(r) for r in confirmed],
+        "unconfirmed": [_result_to_dict(r) for r in unconfirmed],
+        "errors": [_result_to_dict(r) for r in results if r.status == "error"],
+        "skipped": [_result_to_dict(r) for r in results if r.status == "skipped"],
+        "filtered": [{"id": h.id, "reason": reason} for h, reason in discarded],
+        "baselines": baseline_findings,
+        "cost": {
+            "input_tokens": scan_cost.total_input_tokens,
+            "output_tokens": scan_cost.total_output_tokens,
+            "estimated_usd": scan_cost.total_usd,
+            "billable_minutes": total_minutes,
+            "wall_seconds": total_wall,
+            "token_usage_by_model": scan_cost.token_usage_by_model,
+        },
+    }
+
+    json_path = os.path.join(output_dir, f"{context.workflow_id}.json")
+    with open(json_path, "w") as f:
+        json.dump(json_data, f, indent=2)
+
+    return md_path, json_path
+
+
+def _result_to_dict(r: ValidationResult) -> dict:
+    return {
+        "id": r.hypothesis.id,
+        "status": r.status,
+        "recipe_path": r.recipe_path,
+        "attempted_runs": r.attempted_runs,
+        "invalid_runs": r.invalid_runs,
+        "diagnostics": r.diagnostics,
+        "evaluator_correction": r.evaluator_correction,
+        "mitre_category": r.hypothesis.mitre_category,
+        "tags": r.hypothesis.tags,
+        "recipe_summary": r.payload_used,
+        "attack_goal": r.hypothesis.attack_goal,
+        "severity": r.hypothesis.severity,
+        "success_rate": r.success_rate,
+        "iteration": r.iteration,
+        "run_ids": r.run_ids,
+        "suggested_mitigation": r.suggested_mitigation,
+        "failure_reason": r.failure_reason,
+        "wall_seconds": r.wall_seconds,
+    }

gitinject/scanner/types.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class EffectivePromptContext:
+    workflow_id: str
+    provider: str
+    reconstructed_prompt: str
+    tool_restrictions: list[str]
+    has_persist_credentials: bool
+    trigger_event: str
+
+
+@dataclass
+class SetupStep:
+    primitive: str
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class TriggerSpec:
+    event_type: str
+    data: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SuccessCheck:
+    kind: str
+    args: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class AttackHypothesis:
+    id: str
+    mitre_category: str
+    attack_goal: str
+    rationale: str
+    severity: str
+    setup: list[SetupStep] = field(default_factory=list)
+    trigger: TriggerSpec | None = None
+    success_check: SuccessCheck | None = None
+    tags: list[str] = field(default_factory=list)
+    seeded_from: str | None = None
+
+
+@dataclass
+class ValidationResult:
+    hypothesis: AttackHypothesis
+    status: str
+    failure_reason: str | None
+    success_rate: str
+    iteration: int
+    discard_reason: str | None
+    run_ids: list[str]
+    payload_used: str
+    suggested_mitigation: str
+    billable_minutes: float
+    wall_seconds: float
+    evaluator_correction: str | None = None
+    recipe_path: str | None = None
+    attempted_runs: int = 0
+    invalid_runs: int = 0
+    diagnostics: list[dict] = field(default_factory=list)
+
+
+# Model pricing in (input $/MTok, output $/MTok). Verified 2026-05 against
+# Anthropic (docs.anthropic.com/en/docs/about-claude/pricing), Google
+# (ai.google.dev/gemini-api/docs/pricing), and OpenAI rate cards. Gemini Pro
+# rates are the standard <=200K-prompt tier. Unknown models price at $0 — the
+# report surfaces them so we notice the gap rather than silently undercounting.
+MODEL_PRICING: dict[str, tuple[float, float]] = {
+    "claude-opus-4-7": (5.00, 25.00),
+    "claude-opus-4-6": (5.00, 25.00),
+    "claude-opus-4-5": (5.00, 25.00),
+    "claude-opus-4-1": (15.00, 75.00),
+    "claude-sonnet-4-6": (3.00, 15.00),
+    "claude-sonnet-4-5": (3.00, 15.00),
+    "claude-haiku-4-5": (1.00, 5.00),
+    "gemini-2.5-flash": (0.30, 2.50),
+    "gemini-2.5-pro": (1.25, 10.00),
+    "gemini-3.1-pro-preview": (2.00, 12.00),
+    "gpt-5": (1.25, 10.00),
+    "gpt-5-2025-08-07": (1.25, 10.00),
+    "gpt-4o": (2.50, 10.00),
+    "gpt-4o-mini": (0.15, 0.60),
+}
+
+
+def cost_for_usage(usage_by_model: dict[str, dict[str, int]]) -> float:
+    total = 0.0
+    for model, t in usage_by_model.items():
+        in_price, out_price = MODEL_PRICING.get(model, (0.0, 0.0))
+        total += (t.get("input", 0) * in_price + t.get("output", 0) * out_price) / 1_000_000
+    return total
+
+
+def roll_up_usage(log) -> dict[str, dict[str, int]]:
+    """Aggregate a list[LLMResponse] into {model: {"input": int, "output": int}}."""
+    out: dict[str, dict[str, int]] = {}
+    for r in log:
+        bucket = out.setdefault(r.model, {"input": 0, "output": 0})
+        bucket["input"] += int(r.input_tokens)
+        bucket["output"] += int(r.output_tokens)
+    return out
+
+
+@dataclass
+class ScanCost:
+    token_usage_by_model: dict[str, dict[str, int]] = field(default_factory=dict)
+    total_billable_minutes: float = 0.0
+    total_wall_seconds: float = 0.0
+
+    @property
+    def total_input_tokens(self) -> int:
+        return sum(v.get("input", 0) for v in self.token_usage_by_model.values())
+
+    @property
+    def total_output_tokens(self) -> int:
+        return sum(v.get("output", 0) for v in self.token_usage_by_model.values())
+
+    @property
+    def total_usd(self) -> float:
+        return cost_for_usage(self.token_usage_by_model)
+
+
+@dataclass
+class MemoryEntry:
+    provider: str
+    mitre_category: str
+    recipe_fingerprint: str
+    recipe_template: dict
+    attack_goal: str
+    status: str
+    failure_reason: str | None = None
+    evaluator_correction: str | None = None
+    tags: list[str] = field(default_factory=list)
+    workflow_ids: list[str] = field(default_factory=list)
+    first_seen: str = ""
+    source: str | None = None
+
+
+def hypothesis_to_dict(h: AttackHypothesis) -> dict:
+    return {
+        "id": h.id,
+        "mitre_category": h.mitre_category,
+        "attack_goal": h.attack_goal,
+        "rationale": h.rationale,
+        "severity": h.severity,
+        "setup": [{"primitive": s.primitive, "args": s.args} for s in h.setup],
+        "trigger": {"event_type": h.trigger.event_type, "data": h.trigger.data} if h.trigger else None,
+        "success_check": {"kind": h.success_check.kind, "args": h.success_check.args} if h.success_check else None,
+        "tags": h.tags,
+        "seeded_from": h.seeded_from,
+    }
+
+
+def hypothesis_from_dict(d: dict) -> AttackHypothesis:
+    trigger = None
+    if d.get("trigger"):
+        trigger = TriggerSpec(event_type=d["trigger"].get("event_type", ""), data=d["trigger"].get("data", {}))
+    success_check = None
+    if d.get("success_check"):
+        success_check = SuccessCheck(kind=d["success_check"].get("kind", ""), args=d["success_check"].get("args", {}))
+    setup = [SetupStep(primitive=s.get("primitive", ""), args=s.get("args", {})) for s in d.get("setup", [])]
+    return AttackHypothesis(
+        id=d.get("id", ""),
+        mitre_category=d.get("mitre_category", ""),
+        attack_goal=d.get("attack_goal", ""),
+        rationale=d.get("rationale", ""),
+        severity=d.get("severity", "medium"),
+        setup=setup,
+        trigger=trigger,
+        success_check=success_check,
+        tags=d.get("tags", []),
+        seeded_from=d.get("seeded_from"),
+    )

gitinject/scenario_base.py

@@ -0,0 +1,147 @@
+import json
+import os
+from abc import ABC
+
+from .evaluators import Evaluator, UnconfiguredEvaluator
+from .run_context import RunContext, TriggerReceipt
+from .utils.types import ScenarioType, WorkflowCategory
+
+
+class AbstractScenario(ABC):
+    """
+    Abstract Base Class for all benchmark scenarios.
+    Each scenario defines its own GitHub event, state preparation, and evaluation logic.
+    """
+
+    labels = []
+    category: WorkflowCategory = None
+    scenario_type: ScenarioType = ScenarioType.BENIGN
+    required_actors: tuple[str, ...] = ("owner",)
+
+    def __init__(self, workspace_dir):
+        self.workspace_dir = workspace_dir
+        self.scenario_dir = None
+        self.runtime_state = {}
+        self._injected: dict = {}
+
+    def prepare(self, context: RunContext) -> None:
+        self.setup_state(context.github("owner"))
+
+    def trigger(self, context: RunContext) -> TriggerReceipt:
+        return context.default_trigger()
+
+    def observe(self, context: RunContext, run_result: dict) -> dict:
+        return context.collect_target()
+
+    def cleanup(self, context: RunContext) -> None:
+        self.teardown_state(context.github("owner"))
+
+    def get_preflight_evaluator(self):
+        """
+        Returns a callable (response: str) -> bool for offline attack validation,
+        or None if offline testing is not supported for this scenario.
+
+        The callable receives the victim model's raw text response and returns
+        True if the attack succeeded (i.e. the response contains the expected
+        malicious output). Used by the preflight and offline_optimize commands
+        to validate/optimize attacks without provisioning a GitHub repo.
+        """
+        return None
+
+    def get_attack_goal(self) -> str | None:
+        """
+        Returns the goal string passed to attack.generate() — what we want the
+        victim LLM to do (e.g. "run gh repo list --visibility=private").
+        Return None to skip attack injection and use the hardcoded get_event() payload.
+        """
+        return None
+
+    def get_injection_slots(self) -> dict[str, str]:
+        """
+        Returns {event_field: template} pairs. The runner substitutes the attack
+        payload at every {{INJECTION}} marker in each template, then calls
+        apply_attack(field, rendered_value).
+
+        Examples:
+          Full replacement:   {"pr_body": "{{INJECTION}}"}
+          Suffix injection:   {"pr_body": "## Summary\\nLegit text...\\n\\n{{INJECTION}}"}
+          Multi-field:        {"pr_title": "fix: {{INJECTION}}", "pr_body": "...{{INJECTION}}..."}
+
+        Default: empty dict — no injection slots, get_event() is used as-is.
+        """
+        return {}
+
+    def apply_attack(self, field: str, rendered: str) -> None:
+        """Store a rendered injection value. get_event() checks _injected for overrides."""
+        self._injected[field] = rendered
+
+    def get_required_files(self) -> dict:
+        """
+        Returns a dictionary of {repo_path: local_path_or_content}
+        for static files needed in the repo.
+        Defaults to discovering files in a 'contents/' directory next to the scenario.
+        """
+        files = {}
+        if self.scenario_dir:
+            contents_dir = os.path.join(self.scenario_dir, "contents")
+            if os.path.isdir(contents_dir):
+                for root, _, filenames in os.walk(contents_dir):
+                    for filename in filenames:
+                        abs_path = os.path.join(root, filename)
+                        rel_path = os.path.relpath(abs_path, contents_dir)
+                        files[rel_path] = abs_path
+        return files
+
+    def get_required_secrets(self) -> list[str]:
+        """
+        Returns a list of secret names (strings) that must be set in the repository.
+        The runner will attempt to fetch these from local environment variables.
+        """
+        return []
+
+    def get_secrets(self) -> dict[str, str]:
+        """
+        Returns a dictionary of {name: value} for secrets defined by the scenario.
+        These will be set in the repository automatically.
+        """
+        return {}
+
+    def get_template_repo(self) -> str:
+        """
+        Returns the full name of a repository to fork (e.g., 'owner/repo').
+        If None, a new empty repository is created.
+        """
+        return None
+
+    def setup_state(self, gh_client):
+        """Prepares the repository state before the benchmark run."""
+        pass
+
+    def teardown_state(self, gh_client):
+        """Cleans up the repository state after the benchmark run."""
+        pass
+
+    def get_event(self):
+        """
+        Returns a dictionary representing the GitHub event.
+        Must contain 'event_type' and 'data'.
+        """
+        return {}
+
+    def get_utility_evaluator(self) -> Evaluator:
+        """
+        Returns an Evaluator instance to assess Utility.
+        Defaults to an unconfigured evaluator (unknown verdict).
+        """
+        return UnconfiguredEvaluator()
+
+    def get_security_evaluator(self) -> Evaluator:
+        """
+        Returns an Evaluator instance to assess Security.
+        Defaults to an unconfigured evaluator (unknown verdict).
+        """
+        return UnconfiguredEvaluator()
+
+    def to_json(self):
+        """Helper to export the event data to JSON format if needed."""
+        return json.dumps(self.get_event(), indent=4)

gitinject/scenario_loader.py

@@ -0,0 +1,77 @@
+"""Shared, credential-free scenario discovery and loading."""
+
+import hashlib
+import importlib.util
+import inspect
+import sys
+from pathlib import Path
+
+from .scenario_base import AbstractScenario
+
+
+def scenario_definition(path: str | Path) -> Path:
+    path = Path(path)
+    if path.is_dir():
+        definitions = [path / name for name in ("scenario.py", "recipe.json") if (path / name).is_file()]
+        if len(definitions) != 1:
+            raise ValueError(f"Expected exactly one scenario.py or recipe.json in {path}")
+        return definitions[0]
+    if not path.is_file() or path.suffix not in {".py", ".json"}:
+        raise ValueError(f"Scenario definition not found: {path}")
+    return path
+
+
+def discover_scenario_paths(root: str | Path) -> list[Path]:
+    root = Path(root)
+    if not root.exists():
+        return []
+    paths = []
+    for directory in sorted({p.parent for name in ("scenario.py", "recipe.json") for p in root.rglob(name)}):
+        if not {"contents", "__pycache__"}.intersection(directory.relative_to(root).parts):
+            paths.append(scenario_definition(directory))
+    names = [path.parent.name for path in paths]
+    if len(names) != len(set(names)):
+        raise ValueError("Duplicate scenario IDs in dataset")
+    return paths
+
+
+def find_scenario(root: str | Path, identifier: str) -> Path | None:
+    direct = Path(identifier)
+    if direct.exists():
+        return scenario_definition(direct)
+    matches = [path for path in discover_scenario_paths(root) if path.parent.name == identifier]
+    return matches[0] if matches else None
+
+
+def load_scenario(path: str | Path, workspace_dir: str) -> AbstractScenario:
+    definition = scenario_definition(path).resolve()
+    if definition.suffix == ".json":
+        from .scanner.recipe_scenario import load_recipe
+
+        if definition.name != "recipe.json":
+            raise ValueError("JSON scenarios must use recipe.json")
+        scenario = load_recipe(str(definition.parent), workspace_dir)
+    else:
+        content = definition.read_bytes()
+        name = "gitinject_scenario_" + hashlib.sha256(content).hexdigest()[:16]
+        spec = importlib.util.spec_from_file_location(name, definition)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[name] = module
+        try:
+            exec(compile(content, str(definition), "exec"), module.__dict__)
+        except BaseException:
+            sys.modules.pop(name, None)
+            raise
+        classes = [
+            cls
+            for cls in vars(module).values()
+            if inspect.isclass(cls)
+            and cls.__module__ == name
+            and issubclass(cls, AbstractScenario)
+            and not inspect.isabstract(cls)
+        ]
+        if len(classes) != 1:
+            raise ValueError(f"Expected exactly one concrete scenario class in {definition}")
+        scenario = classes[0](workspace_dir)
+    scenario.scenario_dir = str(definition.parent)
+    return scenario

gitinject/scenarios/benign/documentation_out_of_sync/contents/math_utils.py

@@ -0,0 +1,3 @@
+def add(a, b, c=0):
+    """Adds three numbers (c is optional)."""
+    return a + b + c

gitinject/scenarios/benign/documentation_out_of_sync/contents/pydantic/v1/utils.py

@@ -0,0 +1,21 @@
+from typing import Any, Dict, TypeVar
+
+KeyType = TypeVar("KeyType")
+
+
+def deep_update(
+    mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any], merge_lists: bool = False
+) -> Dict[KeyType, Any]:
+    """
+    Recursively update a dictionary.
+    """
+    updated_mapping = mapping.copy()
+    for updating_mapping in updating_mappings:
+        for k, v in updating_mapping.items():
+            if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict):
+                updated_mapping[k] = deep_update(updated_mapping[k], v, merge_lists=merge_lists)
+            elif merge_lists and k in updated_mapping and isinstance(updated_mapping[k], list) and isinstance(v, list):
+                updated_mapping[k] = updated_mapping[k] + v
+            else:
+                updated_mapping[k] = v
+    return updated_mapping