universal-dev-security 1.0.0__py3-none-any.whl

universal_dev_security/recovery.py

@@ -0,0 +1,168 @@
+"""Recovery points and rollback for UDS-made changes.
+
+Every UDS modification (fixes, integration installs) follows:
+
+    checkpoint = rm.create(label, paths)   # snapshot "before"
+    ... modify files (safe_write) ...
+    rm.seal(checkpoint)                    # record "after" hashes
+
+Rollback restores the "before" state only for files whose current content still
+matches the recorded "after" hash, so later user edits are never clobbered
+unless ``force`` is given.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import shutil
+import tempfile
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+CHECKPOINT_ID_RE = re.compile(r"^\d{8}T\d{6}(?:\d{6})?Z-[0-9a-f]{6}$")
+
+
+class RecoveryError(ValueError):
+    pass
+
+
+def sha256_file(path: Path) -> str | None:
+    if not path.is_file():
+        return None
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def resolve_inside(root: Path, relative: str) -> Path:
+    """Resolve ``relative`` under ``root``; reject traversal and symlink escapes."""
+    if not relative or "\x00" in relative:
+        raise RecoveryError("Empty or invalid path")
+    # Treat backslashes as separators and reject drive prefixes on every OS, so the
+    # same input is accepted or rejected identically on Windows, Linux and macOS.
+    normalized = relative.replace("\\", "/")
+    if normalized.startswith("/") or re.match(r"^[A-Za-z]:", normalized):
+        raise RecoveryError(f"Absolute paths are not allowed: {relative}")
+    candidate = Path(normalized)
+    if candidate.is_absolute() or candidate.drive or ".." in candidate.parts:
+        raise RecoveryError(f"Path traversal is not allowed: {relative}")
+    resolved = (root / candidate).resolve()
+    try:
+        resolved.relative_to(root.resolve())
+    except ValueError as exc:
+        raise RecoveryError(f"Path escapes the project directory: {relative}") from exc
+    return resolved
+
+
+def safe_write(root: Path, relative: str, content: str) -> Path:
+    """Atomically write a text file inside ``root`` (temp file + replace)."""
+    target = resolve_inside(root, relative)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    fd, temp = tempfile.mkstemp(prefix=".uds-tmp-", dir=str(target.parent))
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write(content)
+        if target.exists():
+            shutil.copymode(target, temp)
+        os.replace(temp, target)
+    except BaseException:
+        Path(temp).unlink(missing_ok=True)
+        raise
+    return target
+
+
+class RecoveryManager:
+    def __init__(self, root: Path) -> None:
+        self.root = root.expanduser().resolve()
+        self.base = self.root / ".uds" / "backups"
+
+    def _dir(self, checkpoint_id: str) -> Path:
+        if not CHECKPOINT_ID_RE.match(checkpoint_id):
+            raise RecoveryError(f"Invalid checkpoint id: {checkpoint_id!r}")
+        return self.base / checkpoint_id
+
+    def create(self, label: str, paths: list[str]) -> str:
+        from universal_dev_security.history import ensure_uds_dir
+
+        ensure_uds_dir(self.root)
+        checkpoint_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + "-" + uuid.uuid4().hex[:6]
+        directory = self._dir(checkpoint_id)
+        files_dir = directory / "files"
+        files_dir.mkdir(parents=True)
+        entries = []
+        for relative in dict.fromkeys(paths):
+            source = resolve_inside(self.root, relative)
+            if source.is_symlink():
+                raise RecoveryError(f"Refusing to manage a symbolic link: {relative}")
+            existed = source.is_file()
+            if existed:
+                backup = files_dir / Path(relative)
+                backup.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(source, backup)
+            entries.append({"path": Path(relative).as_posix(), "existed": existed,
+                            "sha256_before": sha256_file(source), "sha256_after": None})
+        manifest = {"id": checkpoint_id, "label": label[:200], "created_at": datetime.now(timezone.utc).isoformat(),
+                    "sealed": False, "restored": False, "entries": entries}
+        self._save(checkpoint_id, manifest)
+        return checkpoint_id
+
+    def seal(self, checkpoint_id: str) -> dict[str, Any]:
+        manifest = self.load(checkpoint_id)
+        for entry in manifest["entries"]:
+            entry["sha256_after"] = sha256_file(resolve_inside(self.root, entry["path"]))
+        manifest["sealed"] = True
+        self._save(checkpoint_id, manifest)
+        return manifest
+
+    def _save(self, checkpoint_id: str, manifest: dict[str, Any]) -> None:
+        path = self._dir(checkpoint_id) / "manifest.json"
+        path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8", newline="\n")
+
+    def load(self, checkpoint_id: str) -> dict[str, Any]:
+        path = self._dir(checkpoint_id) / "manifest.json"
+        if not path.is_file():
+            raise RecoveryError(f"Checkpoint not found: {checkpoint_id}")
+        return json.loads(path.read_text(encoding="utf-8"))
+
+    def list(self) -> list[dict[str, Any]]:
+        if not self.base.is_dir():
+            return []
+        result = []
+        for directory in sorted(self.base.iterdir()):
+            if CHECKPOINT_ID_RE.match(directory.name) and (directory / "manifest.json").is_file():
+                manifest = self.load(directory.name)
+                result.append({"id": manifest["id"], "label": manifest["label"], "created_at": manifest["created_at"],
+                               "files": [e["path"] for e in manifest["entries"]], "restored": manifest.get("restored", False)})
+        return result
+
+    def restore(self, checkpoint_id: str, *, force: bool = False) -> dict[str, Any]:
+        manifest = self.load(checkpoint_id)
+        restored, conflicts, unchanged = [], [], []
+        for entry in manifest["entries"]:
+            target = resolve_inside(self.root, entry["path"])
+            current = sha256_file(target)
+            if current == entry["sha256_before"]:
+                unchanged.append(entry["path"])
+                continue
+            expected_after = entry.get("sha256_after")
+            if not force and manifest.get("sealed") and current != expected_after:
+                conflicts.append(entry["path"])  # modified after UDS changed it: keep the user's work
+                continue
+            if entry["existed"]:
+                backup = self._dir(checkpoint_id) / "files" / Path(entry["path"])
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(backup, target)
+            elif target.is_file():
+                target.unlink()
+            restored.append(entry["path"])
+        if not conflicts:
+            manifest["restored"] = True
+            self._save(checkpoint_id, manifest)
+        return {"checkpoint": checkpoint_id, "restored": restored, "conflicts": conflicts, "unchanged": unchanged,
+                "status": "conflicts" if conflicts else "restored"}

universal_dev_security/redaction.py

@@ -0,0 +1,132 @@
+"""Secret detection heuristics and redaction for text UDS handles itself.
+
+This is *not* a replacement for Gitleaks. It is a small, dependency-free
+safety net used where UDS must make a decision without an external scanner,
+for example when an AI coding tool is about to write a file or run a command,
+and to make sure UDS never echoes likely secrets into logs or reports.
+"""
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass
+
+REDACTED = "[REDACTED]"
+
+
+@dataclass(frozen=True)
+class SecretPattern:
+    rule_id: str
+    description: str
+    regex: re.Pattern[str]
+    group: int = 0
+
+
+def _p(rule_id: str, description: str, pattern: str, group: int = 0, flags: int = 0) -> SecretPattern:
+    return SecretPattern(rule_id, description, re.compile(pattern, flags), group)
+
+
+# High-confidence, well-documented token formats. Kept conservative to avoid
+# blocking legitimate work with false positives.
+PATTERNS: tuple[SecretPattern, ...] = (
+    _p("private-key", "Private key block", r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----"),
+    _p("aws-access-key-id", "AWS access key ID", r"\b(?:AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{16}\b"),
+    _p("github-token", "GitHub token", r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,255}\b"),
+    _p("github-fine-grained-pat", "GitHub fine-grained token", r"\bgithub_pat_[A-Za-z0-9_]{60,255}\b"),
+    # Every quantifier is bounded: these patterns run on untrusted AI-generated
+    # content inside a hook, so catastrophic backtracking (ReDoS) must be impossible.
+    _p("gitlab-pat", "GitLab personal access token", r"\bglpat-[A-Za-z0-9_\-]{20,100}\b"),
+    _p("slack-token", "Slack token", r"\bxox[abposr]-[A-Za-z0-9-]{10,200}\b"),
+    _p("stripe-secret-key", "Stripe secret key", r"\b(?:sk|rk)_live_[A-Za-z0-9]{20,200}\b"),
+    _p("google-api-key", "Google API key", r"\bAIza[0-9A-Za-z_\-]{35}\b"),
+    _p("anthropic-api-key", "Anthropic API key", r"\bsk-ant-[A-Za-z0-9_\-]{20,300}\b"),
+    _p("openai-api-key", "OpenAI API key", r"\bsk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_\-]{32,300}\b"),
+    _p("npm-token", "npm access token", r"\bnpm_[A-Za-z0-9]{36}\b"),
+    _p("pypi-token", "PyPI upload token", r"\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_\-]{50,400}\b"),
+    _p("jwt", "JSON Web Token",
+       r"\beyJ[A-Za-z0-9_\-]{10,4000}\.eyJ[A-Za-z0-9_\-]{10,8000}\.[A-Za-z0-9_\-]{10,2000}\b"),
+    _p(
+        "connection-string-password",
+        "Credentials embedded in a connection URL",
+        r"\b[a-z][a-z0-9+.\-]{1,20}://[^\s:/@'\"]{1,64}:([^\s@/'\"]{6,128})@[^\s'\"]{1,500}",
+        group=1,
+        flags=re.IGNORECASE,
+    ),
+    _p(
+        "generic-assigned-secret",
+        "High-entropy value assigned to a secret-like name",
+        r"(?i)\b[\w.\-]{0,40}?(?:secret|passwd|password|api[_-]?key|access[_-]?token|auth[_-]?token|private[_-]?key|client[_-]?secret)[\w.\-]{0,40}\s{0,5}[:=]\s{0,5}[\"']?([A-Za-z0-9+/_\-.~]{16,200})[\"']?",
+        group=1,
+    ),
+)
+
+_PLACEHOLDER_HINTS = (
+    "example", "placeholder", "changeme", "change_me", "your_", "your-", "xxxx", "dummy",
+    "sample", "fake", "test", "redacted", "<", "${", "{{", "process.env", "os.environ", "getenv",
+)
+
+
+@dataclass(frozen=True)
+class SecretMatch:
+    rule_id: str
+    description: str
+    line: int
+
+
+def shannon_entropy(value: str) -> float:
+    if not value:
+        return 0.0
+    frequencies = {char: value.count(char) for char in set(value)}
+    length = len(value)
+    return -sum((count / length) * math.log2(count / length) for count in frequencies.values())
+
+
+def _looks_like_placeholder(value: str) -> bool:
+    lowered = value.lower()
+    return any(hint in lowered for hint in _PLACEHOLDER_HINTS) or len(set(value)) <= 3
+
+
+def find_secrets(text: str, *, max_chars: int = 2_000_000) -> list[SecretMatch]:
+    """Return locations of likely secrets. Never returns the secret value."""
+    if not text:
+        return []
+    text = text[:max_chars]
+    matches: list[SecretMatch] = []
+    seen: set[tuple[str, int]] = set()
+    for pattern in PATTERNS:
+        for match in pattern.regex.finditer(text):
+            value = match.group(pattern.group) or ""
+            if pattern.rule_id == "generic-assigned-secret":
+                if _looks_like_placeholder(value) or shannon_entropy(value) < 3.5:
+                    continue
+            elif pattern.rule_id == "connection-string-password" and _looks_like_placeholder(value):
+                continue
+            line = text.count("\n", 0, match.start()) + 1
+            key = (pattern.rule_id, line)
+            if key not in seen:
+                seen.add(key)
+                matches.append(SecretMatch(pattern.rule_id, pattern.description, line))
+    return matches
+
+
+def redact(text: str) -> str:
+    """Replace likely secret values in ``text`` with a fixed marker."""
+    if not text:
+        return text
+    result = text
+    for pattern in PATTERNS:
+        def _replace(match: re.Match[str], pattern: SecretPattern = pattern) -> str:
+            value = match.group(pattern.group) or ""
+            if pattern.rule_id == "generic-assigned-secret" and (
+                _looks_like_placeholder(value) or shannon_entropy(value) < 3.5
+            ):
+                return match.group(0)
+            if pattern.group == 0:
+                return REDACTED
+            whole = match.group(0)
+            start = match.start(pattern.group) - match.start(0)
+            end = match.end(pattern.group) - match.start(0)
+            return whole[:start] + REDACTED + whole[end:]
+
+        result = pattern.regex.sub(_replace, result)
+    return result