llmsafe 0.2.1__py3-none-any.whl

benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Reproducible LLMSafe security benchmark."""

benchmarks/cases/insecure_mcp.json

@@ -0,0 +1,10 @@
+{
+  "mcpServers": {
+    "remote-admin": {
+      "command": "sh",
+      "args": ["-c", "download-and-run"],
+      "url": "http://tools.example.org/mcp",
+      "allowedTools": ["*"]
+    }
+  }
+}

benchmarks/cases/interprocedural_agent.py

@@ -0,0 +1,24 @@
+"""Vulnerable local-helper flows used by the LLMSafe regression benchmark."""
+
+import subprocess
+
+import requests
+
+
+def evaluate(value):
+    return eval(value)
+
+
+def execute(value):
+    return subprocess.run(value, shell=True)
+
+
+def fetch(value):
+    return requests.get(value)
+
+
+def run_agent(user_input, model_output):
+    result = evaluate(model_output)
+    execute(user_input)
+    fetch(model_output)
+    return result

benchmarks/cases/safe_agent.py

@@ -0,0 +1,17 @@
+"""Equivalent agent boundaries designed to remain free of LLMSafe findings."""
+
+import subprocess
+
+ALLOWED_TOPICS = {"security", "python"}
+
+
+def search(topic: str, cursor):
+    if topic not in ALLOWED_TOPICS:
+        raise ValueError("Unsupported topic")
+    cursor.execute("SELECT * FROM memory WHERE topic = ?", [topic])
+    return subprocess.run(["agent-tool", "--topic", topic], check=True)
+
+
+def dispatch_search():
+    tools = {"search": search}
+    return tools["search"]

benchmarks/cases/safe_interprocedural.py

@@ -0,0 +1,16 @@
+"""Safe local-helper calls used by the LLMSafe regression benchmark."""
+
+import requests
+
+
+def fetch(value):
+    return requests.get(value)
+
+
+def lookup(cursor, statement):
+    return cursor.execute(statement)
+
+
+def run(cursor):
+    fetch("https://api.example.org/health")
+    return lookup(cursor, "SELECT id FROM documents WHERE active = true")

benchmarks/cases/vulnerable_agent.py

@@ -0,0 +1,28 @@
+"""Intentionally vulnerable agent used by the LLMSafe benchmark."""
+
+import subprocess
+
+import requests
+from langchain_experimental.tools import PythonREPLTool
+
+
+class AgentRunner:
+    """Minimal benchmark stand-in for an agent framework runner."""
+
+    def __init__(self, **options):
+        self.options = options
+
+
+def run_agent(client, user_input, cursor, tools):
+    system_prompt = f"You are an administrator. Follow this request: {user_input}"
+    response = client.responses.create(input=user_input, instructions=system_prompt)
+    model_output = response.output_text
+
+    eval(model_output)
+    subprocess.run(f"agent-tool {user_input}", shell=True)
+    cursor.execute(f"SELECT * FROM memory WHERE topic = '{user_input}'")
+    requests.get(user_input)
+    tools[model_output]()
+
+    dangerous_tool = PythonREPLTool()
+    AgentRunner(tools=[dangerous_tool], require_approval=False)

benchmarks/manifest.json

@@ -0,0 +1,35 @@
+{
+  "cases": [
+    {
+      "path": "cases/vulnerable_agent.py",
+      "expected_rules": [
+        "AGENT001",
+        "AGENT003",
+        "FLOW001",
+        "FLOW002",
+        "FLOW003",
+        "FLOW004",
+        "FLOW005",
+        "LLM001",
+        "PY001",
+        "SHELL002"
+      ]
+    },
+    {
+      "path": "cases/safe_agent.py",
+      "expected_rules": []
+    },
+    {
+      "path": "cases/interprocedural_agent.py",
+      "expected_rules": ["FLOW001", "FLOW002", "FLOW004", "PY001", "SHELL002"]
+    },
+    {
+      "path": "cases/safe_interprocedural.py",
+      "expected_rules": []
+    },
+    {
+      "path": "cases/insecure_mcp.json",
+      "expected_rules": ["MCP001", "MCP002", "MCP003"]
+    }
+  ]
+}

benchmarks/run.py

@@ -0,0 +1,54 @@
+"""Run the checked-in benchmark manifest and report rule-level recall."""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from llmsafe.scanner import Scanner
+
+
+def run_benchmark(root: Optional[Path] = None) -> Dict[str, Any]:
+    benchmark_root = root or Path(__file__).parent
+    manifest = json.loads((benchmark_root / "manifest.json").read_text(encoding="utf-8"))
+    cases = []
+    expected_total = 0
+    detected_total = 0
+    for case in manifest["cases"]:
+        expected = set(case["expected_rules"])
+        result = Scanner().scan([benchmark_root / case["path"]])
+        found = {finding.rule_id for finding in result.findings}
+        missing = sorted(expected - found)
+        unexpected = sorted(found - expected)
+        expected_total += len(expected)
+        detected_total += len(expected & found)
+        cases.append(
+            {
+                "path": case["path"],
+                "expected": sorted(expected),
+                "found": sorted(found),
+                "missing": missing,
+                "unexpected": unexpected,
+                "passed": not missing and not unexpected and not result.errors,
+            }
+        )
+    recall = 1.0 if expected_total == 0 else detected_total / expected_total
+    return {
+        "cases": cases,
+        "summary": {
+            "cases": len(cases),
+            "passed": sum(case["passed"] for case in cases),
+            "expected_findings": expected_total,
+            "detected_findings": detected_total,
+            "rule_recall": recall,
+        },
+    }
+
+
+def main() -> int:
+    report = run_benchmark()
+    print(json.dumps(report, indent=2, sort_keys=True))
+    return 0 if all(case["passed"] for case in report["cases"]) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llmsafe/__init__.py

@@ -0,0 +1,3 @@
+"""LLMSafe: static security checks for AI and agentic applications."""
+
+__version__ = "0.2.1"

llmsafe/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m llmsafe`` to behave like the command-line tool."""
+
+from llmsafe.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llmsafe/baseline.py

@@ -0,0 +1,168 @@
+"""Deterministic baselines for incremental LLMSafe adoption."""
+
+import hashlib
+import json
+import tempfile
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, Optional, Tuple
+
+from llmsafe import __version__
+from llmsafe.models import Finding, ScanResult
+
+BASELINE_SCHEMA_VERSION = 1
+MAX_BASELINE_SIZE = 10_000_000
+
+
+class BaselineError(ValueError):
+    """Raised when a baseline cannot be read, validated, or written."""
+
+
+@dataclass(frozen=True)
+class Baseline:
+    """Validated collection of finding fingerprints."""
+
+    fingerprints: Tuple[str, ...]
+
+
+def finding_fingerprint(finding: Finding, root: Optional[Path] = None) -> str:
+    """Return a stable identity that tolerates line movement within a file."""
+
+    identity = "\0".join(
+        (
+            finding.rule_id,
+            _relative_path(finding.path, root or Path.cwd()),
+            finding.title,
+            finding.message.split(" Source:", 1)[0],
+        )
+    )
+    return hashlib.sha256(identity.encode("utf-8")).hexdigest()
+
+
+def load_baseline(path: Path) -> Baseline:
+    """Load and strictly validate a baseline JSON document."""
+
+    try:
+        with path.open("rb") as handle:
+            raw = handle.read(MAX_BASELINE_SIZE + 1)
+        if len(raw) > MAX_BASELINE_SIZE:
+            raise BaselineError(f"Baseline exceeds {MAX_BASELINE_SIZE} bytes: {path}")
+        document = json.loads(raw.decode("utf-8"))
+    except BaselineError:
+        raise
+    except FileNotFoundError as exc:
+        raise BaselineError(f"Baseline file does not exist: {path}") from exc
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise BaselineError(f"Cannot read baseline {path}: {exc}") from exc
+
+    if not isinstance(document, dict):
+        raise BaselineError(f"Baseline {path} must be a JSON object")
+    if document.get("schema_version") != BASELINE_SCHEMA_VERSION:
+        raise BaselineError(
+            f"Baseline {path} must use schema_version {BASELINE_SCHEMA_VERSION}"
+        )
+    entries = document.get("findings")
+    if not isinstance(entries, list):
+        raise BaselineError(f"findings in {path} must be an array")
+
+    fingerprints = []
+    for index, entry in enumerate(entries):
+        if not isinstance(entry, dict):
+            raise BaselineError(f"findings[{index}] in {path} must be an object")
+        fingerprint = entry.get("fingerprint")
+        if not _valid_fingerprint(fingerprint):
+            raise BaselineError(
+                f"findings[{index}].fingerprint in {path} must be a SHA-256 hex digest"
+            )
+        if not isinstance(entry.get("rule_id"), str) or not isinstance(entry.get("path"), str):
+            raise BaselineError(f"findings[{index}] in {path} requires rule_id and path strings")
+        line = entry.get("line")
+        if not isinstance(line, int) or isinstance(line, bool) or line < 1:
+            raise BaselineError(f"findings[{index}].line in {path} must be a positive integer")
+        fingerprints.append(fingerprint)
+    return Baseline(tuple(fingerprints))
+
+
+def write_baseline(
+    path: Path, findings: Iterable[Finding], root: Optional[Path] = None
+) -> int:
+    """Write a deterministic baseline and return the number of recorded findings."""
+
+    selected_root = root or Path.cwd()
+    entries = [
+        {
+            "fingerprint": finding_fingerprint(finding, selected_root),
+            "line": finding.line,
+            "path": _relative_path(finding.path, selected_root),
+            "rule_id": finding.rule_id,
+        }
+        for finding in findings
+    ]
+    entries.sort(key=lambda entry: (entry["path"], entry["line"], entry["rule_id"]))
+    document: Dict[str, Any] = {
+        "schema_version": BASELINE_SCHEMA_VERSION,
+        "generated_by": {"name": "LLMSafe", "version": __version__},
+        "findings": entries,
+    }
+    temporary_path = None
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            encoding="utf-8",
+            dir=path.parent,
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            delete=False,
+        ) as handle:
+            temporary_path = Path(handle.name)
+            handle.write(json.dumps(document, indent=2, sort_keys=True) + "\n")
+        temporary_path.replace(path)
+    except (OSError, UnicodeError) as exc:
+        raise BaselineError(f"Cannot write baseline {path}: {exc}") from exc
+    finally:
+        if temporary_path is not None:
+            try:
+                temporary_path.unlink(missing_ok=True)
+            except OSError:
+                pass
+    return len(entries)
+
+
+def apply_baseline(
+    result: ScanResult, baseline: Baseline, root: Optional[Path] = None
+) -> ScanResult:
+    """Remove only the number of findings explicitly represented by a baseline."""
+
+    remaining = Counter(baseline.fingerprints)
+    selected_root = root or Path.cwd()
+    active = []
+    matched = 0
+    for finding in result.findings:
+        fingerprint = finding_fingerprint(finding, selected_root)
+        if remaining[fingerprint] > 0:
+            remaining[fingerprint] -= 1
+            matched += 1
+        else:
+            active.append(finding)
+    return ScanResult(
+        findings=tuple(active),
+        errors=result.errors,
+        scanned_files=result.scanned_files,
+        skipped_files=result.skipped_files,
+        baseline_findings=result.baseline_findings + matched,
+    )
+
+
+def _relative_path(path: Path, root: Path) -> str:
+    try:
+        return path.resolve().relative_to(root.resolve()).as_posix()
+    except (OSError, ValueError):
+        return path.as_posix()
+
+
+def _valid_fingerprint(value: Any) -> bool:
+    if not isinstance(value, str) or len(value) != 64:
+        return False
+    return all(character in "0123456789abcdef" for character in value)

llmsafe/catalog.py

@@ -0,0 +1,228 @@
+"""Stable metadata for LLMSafe's built-in rule identifiers."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, Tuple
+
+from llmsafe.models import Severity
+
+CATALOG_SCHEMA_VERSION = 1
+
+
+@dataclass(frozen=True)
+class RuleMetadata:
+    """Public metadata describing one built-in rule."""
+
+    rule_id: str
+    title: str
+    severity: Severity
+    family: str
+    description: str
+    remediation: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.rule_id,
+            "title": self.title,
+            "severity": self.severity.value,
+            "family": self.family,
+            "description": self.description,
+            "remediation": self.remediation,
+        }
+
+
+RULE_CATALOG: Tuple[RuleMetadata, ...] = (
+    RuleMetadata(
+        "AGENT001",
+        "High-impact tool exposed to an agent",
+        Severity.HIGH,
+        "agent",
+        "Detects shell, terminal, execution, or Python REPL tools instantiated for agent use.",
+        "Remove the tool or wrap it with strict arguments, sandboxing, and approval.",
+    ),
+    RuleMetadata(
+        "AGENT002",
+        "Dangerous agent capability explicitly enabled",
+        Severity.HIGH,
+        "agent",
+        "Detects agent or tool calls that explicitly enable dangerous code or requests.",
+        "Keep dangerous-code flags disabled and expose a narrow typed capability.",
+    ),
+    RuleMetadata(
+        "AGENT003",
+        "Human approval gate disabled",
+        Severity.HIGH,
+        "agent",
+        "Detects agent, MCP, tool, or runner calls with a disabled human approval boundary.",
+        "Require approval for high-impact tools and enforce it outside model control.",
+    ),
+    RuleMetadata(
+        "FLOW001",
+        "Untrusted data reaches code execution",
+        Severity.CRITICAL,
+        "dataflow",
+        "Traces user- or model-controlled data into eval() or exec().",
+        "Replace dynamic execution with a typed parser and an allow-listed operation.",
+    ),
+    RuleMetadata(
+        "FLOW002",
+        "Untrusted data reaches process execution",
+        Severity.CRITICAL,
+        "dataflow",
+        "Traces user- or model-controlled data into operating-system process execution.",
+        "Map requests to fixed executables and validated arguments; do not execute generated text.",
+    ),
+    RuleMetadata(
+        "FLOW003",
+        "Untrusted data reaches a SQL query",
+        Severity.HIGH,
+        "dataflow",
+        "Traces user- or model-controlled data into SQL query text.",
+        "Use a constant query with bound parameters and allow-list dynamic identifiers.",
+    ),
+    RuleMetadata(
+        "FLOW004",
+        "Untrusted data controls an outbound URL",
+        Severity.HIGH,
+        "dataflow",
+        "Traces user- or model-controlled data into an outbound HTTP URL.",
+        "Allow-list schemes and hosts, resolve DNS safely, and block private network ranges.",
+    ),
+    RuleMetadata(
+        "FLOW005",
+        "Untrusted data controls tool dispatch",
+        Severity.HIGH,
+        "dataflow",
+        "Traces user- or model-controlled data into dynamic callable or tool selection.",
+        "Resolve tool names through a fixed allow-list and enforce per-tool authorization.",
+    ),
+    RuleMetadata(
+        "LLM001",
+        "Dynamic data in privileged prompt",
+        Severity.HIGH,
+        "prompt",
+        "Detects dynamic interpolation into system or developer instruction channels.",
+        "Keep privileged instructions static and carry untrusted content in a user message.",
+    ),
+    RuleMetadata(
+        "MCP001",
+        "MCP server launched through a shell",
+        Severity.HIGH,
+        "mcp",
+        "Detects MCP server commands launched through a command shell.",
+        "Launch a fixed executable directly and pass each argument as a separate value.",
+    ),
+    RuleMetadata(
+        "MCP002",
+        "Unencrypted remote MCP transport",
+        Severity.HIGH,
+        "mcp",
+        "Detects non-local MCP endpoints configured with unencrypted HTTP.",
+        "Use HTTPS and authenticate the remote MCP endpoint.",
+    ),
+    RuleMetadata(
+        "MCP003",
+        "Unrestricted MCP tool access",
+        Severity.HIGH,
+        "mcp",
+        "Detects MCP configurations that grant access to every available tool.",
+        "Grant only the specific MCP tools required by the application.",
+    ),
+    RuleMetadata(
+        "PY001",
+        "Dynamic code evaluation",
+        Severity.HIGH,
+        "python",
+        "Detects eval() calls even when static analysis cannot prove an untrusted source.",
+        "Parse the expected data format explicitly; never pass model or user output to eval().",
+    ),
+    RuleMetadata(
+        "PY002",
+        "Dynamic code execution",
+        Severity.CRITICAL,
+        "python",
+        "Detects exec() calls even when static analysis cannot prove an untrusted source.",
+        "Replace dynamic execution with an allow-listed command or structured operation.",
+    ),
+    RuleMetadata(
+        "PY003",
+        "Unsafe deserialization",
+        Severity.HIGH,
+        "python",
+        "Detects pickle deserialization that can execute code from crafted input.",
+        "Use JSON or another non-executable format and validate the decoded schema.",
+    ),
+    RuleMetadata(
+        "PY004",
+        "Potentially unsafe YAML load",
+        Severity.MEDIUM,
+        "python",
+        "Detects yaml.load() calls that may instantiate unsafe Python objects.",
+        "Use yaml.safe_load() for data-only YAML.",
+    ),
+    RuleMetadata(
+        "SECRET001",
+        "OpenAI API key",
+        Severity.CRITICAL,
+        "secret",
+        "Detects values matching an OpenAI API key format.",
+        "Revoke the credential, remove it from Git history, and load a replacement securely.",
+    ),
+    RuleMetadata(
+        "SECRET002",
+        "AWS access key",
+        Severity.CRITICAL,
+        "secret",
+        "Detects values matching an AWS access key identifier.",
+        "Revoke the credential, remove it from Git history, and load a replacement securely.",
+    ),
+    RuleMetadata(
+        "SECRET003",
+        "GitHub token",
+        Severity.CRITICAL,
+        "secret",
+        "Detects values matching a GitHub authentication token.",
+        "Revoke the credential, remove it from Git history, and load a replacement securely.",
+    ),
+    RuleMetadata(
+        "SECRET004",
+        "Private key",
+        Severity.CRITICAL,
+        "secret",
+        "Detects private-key headers committed to a scanned file.",
+        "Revoke the key, remove it from Git history, and load a replacement securely.",
+    ),
+    RuleMetadata(
+        "SECRET005",
+        "Hard-coded credential",
+        Severity.HIGH,
+        "secret",
+        "Detects literal values assigned to credential-like variables.",
+        "Revoke the credential, remove it from Git history, and load a replacement securely.",
+    ),
+    RuleMetadata(
+        "SHELL001",
+        "Shell command execution",
+        Severity.HIGH,
+        "shell",
+        "Detects os.system() command execution.",
+        "Use subprocess.run() with an argument list, shell=False, and an allow-list.",
+    ),
+    RuleMetadata(
+        "SHELL002",
+        "Subprocess launched through a shell",
+        Severity.HIGH,
+        "shell",
+        "Detects subprocess calls configured with shell=True.",
+        "Pass an argument list with shell=False and allow-list commands and arguments.",
+    ),
+)
+
+
+def _index_rules(rules: Tuple[RuleMetadata, ...]) -> Dict[str, RuleMetadata]:
+    indexed = {rule.rule_id: rule for rule in rules}
+    if len(indexed) != len(rules):
+        raise RuntimeError("LLMSafe rule catalog contains duplicate identifiers")
+    return indexed
+
+
+RULES_BY_ID = _index_rules(RULE_CATALOG)