patchi 0.7.5__py3-none-any.whl

patchi/core/hosted/ip_reputation.py

@@ -0,0 +1,169 @@
+"""
+IP Reputation database for hosted mode.
+
+Maintains a local database of known-bad IPs from public blocklists.
+Supports:
+  - Loading public blocklists (abuse.ch, firehol, etc.)
+  - Fast O(1) lookup via set
+  - Periodic refresh (daily)
+  - Auto-blocking for high-threat IPs
+
+All data stored locally in .patchi/hosted/ip_reputation.json.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+from pathlib import Path
+
+_REPUTATION_FILE = ".patchi/hosted/ip_reputation.json"
+_BLOCKLIST_URLS = [
+    "https://raw.githubusercontent.com/stamparm/ipsum/master/ipsum.txt",
+    "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset",
+]
+
+# IPs from these lists are automatically blocked
+_AUTO_BLOCK_THRESHOLD = 3  # IPs seen on 3+ lists
+
+
+_log = logging.getLogger("patchi.core.ip_reputation")
+
+
+def _load(root: Path) -> dict:
+    path = root / _REPUTATION_FILE
+    if not path.exists():
+        return {"bad_ips": {}, "last_refresh": 0, "blocked": []}
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception as e:
+        _log.warning("_load failed: %s", e)
+        return {"bad_ips": {}, "last_refresh": 0, "blocked": []}
+
+
+def _save(root: Path, data: dict) -> None:
+    path = root / _REPUTATION_FILE
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    tmp.replace(path)
+
+
+def lookup(ip: str, root: Path) -> dict:
+    """
+    Look up an IP's reputation.
+    Returns {"known_bad": bool, "threat_level": str, "lists_hit": int, "blocked": bool}
+    """
+    data = _load(root)
+    bad_ips = data.get("bad_ips", {})
+    blocked = set(data.get("blocked", []))
+
+    if ip in bad_ips:
+        info = bad_ips[ip]
+        return {
+            "known_bad": True,
+            "threat_level": _threat_level(info.get("lists_hit", 0)),
+            "lists_hit": info.get("lists_hit", 0),
+            "sources": info.get("sources", []),
+            "blocked": ip in blocked,
+        }
+
+    return {
+        "known_bad": False,
+        "threat_level": "clean",
+        "lists_hit": 0,
+        "sources": [],
+        "blocked": ip in blocked,
+    }
+
+
+def record_hit(ip: str, source: str, root: Path) -> None:
+    """Record that an IP was flagged by a detector."""
+    data = _load(root)
+    bad_ips = data.setdefault("bad_ips", {})
+
+    if ip not in bad_ips:
+        bad_ips[ip] = {"lists_hit": 0, "sources": [], "first_seen": time.time()}
+
+    entry = bad_ips[ip]
+    if source not in entry["sources"]:
+        entry["sources"].append(source)
+        entry["lists_hit"] = len(entry["sources"])
+    entry["last_seen"] = time.time()
+
+    # Auto-block if threshold crossed
+    if entry["lists_hit"] >= _AUTO_BLOCK_THRESHOLD:
+        blocked = data.setdefault("blocked", [])
+        if ip not in blocked:
+            blocked.append(ip)
+
+    _save(root, data)
+
+
+def is_blocked(ip: str, root: Path) -> bool:
+    """Check if an IP is auto-blocked."""
+    data = _load(root)
+    return ip in data.get("blocked", [])
+
+
+def block(ip: str, root: Path) -> None:
+    """Manually block an IP."""
+    data = _load(root)
+    blocked = data.setdefault("blocked", [])
+    if ip not in blocked:
+        blocked.append(ip)
+    _save(root, data)
+
+
+def unblock(ip: str, root: Path) -> bool:
+    """Unblock an IP. Returns True if it was blocked."""
+    data = _load(root)
+    blocked = data.get("blocked", [])
+    if ip in blocked:
+        blocked.remove(ip)
+        data["blocked"] = blocked
+        _save(root, data)
+        return True
+    return False
+
+
+def needs_refresh(root: Path, max_age_hours: int = 24) -> bool:
+    """Check if the reputation database needs refreshing."""
+    data = _load(root)
+    last_refresh = data.get("last_refresh", 0)
+    age_hours = (time.time() - last_refresh) / 3600
+    return age_hours > max_age_hours
+
+
+def get_top_threats(root: Path, limit: int = 20) -> list[dict]:
+    """Get the top threatening IPs sorted by lists_hit."""
+    data = _load(root)
+    bad_ips = data.get("bad_ips", {})
+
+    sorted_ips = sorted(
+        bad_ips.items(),
+        key=lambda x: x[1].get("lists_hit", 0),
+        reverse=True,
+    )[:limit]
+
+    return [
+        {
+            "ip": ip,
+            "lists_hit": info.get("lists_hit", 0),
+            "sources": info.get("sources", []),
+            "threat_level": _threat_level(info.get("lists_hit", 0)),
+            "first_seen": info.get("first_seen", 0),
+            "last_seen": info.get("last_seen", 0),
+        }
+        for ip, info in sorted_ips
+    ]
+
+
+def _threat_level(lists_hit: int) -> str:
+    if lists_hit >= 5:
+        return "critical"
+    if lists_hit >= 3:
+        return "high"
+    if lists_hit >= 1:
+        return "medium"
+    return "clean"

patchi/core/hosted/log_parsers.py

@@ -0,0 +1,251 @@
+"""
+Log parsers for Patchi hosted mode.
+
+Parses 7 log formats into a unified LogEntry structure.
+Uses loguru for internal structured logging of parse errors.
+
+Supported formats:
+  nginx, apache, caddy, uvicorn, gunicorn, cloudflare (JSON), generic JSON
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from dataclasses import dataclass, field
+
+from loguru import logger
+
+
+@dataclass
+class LogEntry:
+    """Unified log entry across all formats."""
+
+    timestamp: float
+    level: str  # INFO / WARNING / ERROR / CRITICAL
+    method: str  # HTTP method or ""
+    path: str  # URL path or ""
+    status: int  # HTTP status or 0
+    ip: str  # client IP or ""
+    message: str  # raw log line or parsed message
+    source: str  # parser name
+    extra: dict = field(default_factory=dict)
+
+
+# ── Shared patterns ────────────────────────────────────────────────────────────
+
+_NGINX_RE = re.compile(
+    r"(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] "
+    r'"(?P<method>\w+) (?P<path>\S+) [^"]*" (?P<status>\d+) (?P<bytes>\d+)'
+)
+
+_APACHE_RE = re.compile(
+    r"(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] "
+    r'"(?P<method>\w+) (?P<path>\S+) [^"]*" (?P<status>\d+)'
+)
+
+_CADDY_RE = re.compile(
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\S+)\s+(?P<level>\w+)\s+"
+    r'http\.log\.access.*?"method":"(?P<method>\w+)".*?"uri":"(?P<path>[^"]+)"'
+    r'.*?"status":(?P<status>\d+).*?"remote_ip":"(?P<ip>[^"]+)"'
+)
+
+_UVICORN_RE = re.compile(
+    r'(?P<ip>\S+):(?:\d+) - "(?P<method>\w+) (?P<path>\S+) [^"]*" (?P<status>\d+)'
+)
+
+_GUNICORN_RE = re.compile(
+    r"(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] "
+    r'"(?P<method>\w+) (?P<path>\S+) [^"]*" (?P<status>\d+)'
+)
+
+
+def _status_to_level(status: int) -> str:
+    if status >= 500:
+        return "ERROR"
+    if status >= 400:
+        return "WARNING"
+    return "INFO"
+
+
+def _now() -> float:
+    return time.time()
+
+
+# ── Individual parsers ─────────────────────────────────────────────────────────
+
+
+def parse_nginx(line: str) -> LogEntry | None:
+    m = _NGINX_RE.match(line)
+    if not m:
+        return None
+    status = int(m.group("status"))
+    return LogEntry(
+        timestamp=_now(),
+        level=_status_to_level(status),
+        method=m.group("method"),
+        path=m.group("path"),
+        status=status,
+        ip=m.group("ip"),
+        message=line,
+        source="nginx",
+        extra={"bytes": m.group("bytes"), "raw_time": m.group("time")},
+    )
+
+
+def parse_apache(line: str) -> LogEntry | None:
+    m = _APACHE_RE.match(line)
+    if not m:
+        return None
+    status = int(m.group("status"))
+    return LogEntry(
+        timestamp=_now(),
+        level=_status_to_level(status),
+        method=m.group("method"),
+        path=m.group("path"),
+        status=status,
+        ip=m.group("ip"),
+        message=line,
+        source="apache",
+    )
+
+
+def parse_caddy(line: str) -> LogEntry | None:
+    m = _CADDY_RE.search(line)
+    if not m:
+        return None
+    status = int(m.group("status"))
+    return LogEntry(
+        timestamp=_now(),
+        level=_status_to_level(status),
+        method=m.group("method"),
+        path=m.group("path"),
+        status=status,
+        ip=m.group("ip"),
+        message=line,
+        source="caddy",
+    )
+
+
+def parse_uvicorn(line: str) -> LogEntry | None:
+    m = _UVICORN_RE.search(line)
+    if not m:
+        return None
+    status = int(m.group("status"))
+    return LogEntry(
+        timestamp=_now(),
+        level=_status_to_level(status),
+        method=m.group("method"),
+        path=m.group("path"),
+        status=status,
+        ip=m.group("ip"),
+        message=line,
+        source="uvicorn",
+    )
+
+
+def parse_gunicorn(line: str) -> LogEntry | None:
+    m = _GUNICORN_RE.match(line)
+    if not m:
+        return None
+    status = int(m.group("status"))
+    return LogEntry(
+        timestamp=_now(),
+        level=_status_to_level(status),
+        method=m.group("method"),
+        path=m.group("path"),
+        status=status,
+        ip=m.group("ip"),
+        message=line,
+        source="gunicorn",
+    )
+
+
+def parse_cloudflare(line: str) -> LogEntry | None:
+    """Cloudflare logs ship as JSON objects."""
+    try:
+        d = json.loads(line)
+        status = int(d.get("EdgeResponseStatus", d.get("CacheResponseStatus", 0)))
+        return LogEntry(
+            timestamp=_now(),
+            level=_status_to_level(status),
+            method=d.get("ClientRequestMethod", ""),
+            path=d.get("ClientRequestURI", ""),
+            status=status,
+            ip=d.get("ClientIP", ""),
+            message=line,
+            source="cloudflare",
+            extra={
+                k: v
+                for k, v in d.items()
+                if k
+                not in (
+                    "ClientRequestMethod",
+                    "ClientRequestURI",
+                    "ClientIP",
+                    "EdgeResponseStatus",
+                    "CacheResponseStatus",
+                )
+            },
+        )
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+def parse_json(line: str) -> LogEntry | None:
+    """Generic JSON log line — best-effort field extraction."""
+    try:
+        d = json.loads(line)
+        status = int(d.get("status", d.get("status_code", 0)))
+        return LogEntry(
+            timestamp=float(d.get("timestamp", d.get("ts", _now()))),
+            level=d.get("level", d.get("severity", _status_to_level(status))).upper(),
+            method=d.get("method", ""),
+            path=d.get("path", d.get("url", d.get("uri", ""))),
+            status=status,
+            ip=d.get("ip", d.get("remote_addr", d.get("client_ip", ""))),
+            message=d.get("message", d.get("msg", line)),
+            source="json",
+            extra=d,
+        )
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+# ── Auto-detect and parse ──────────────────────────────────────────────────────
+
+_PARSERS = [
+    parse_cloudflare,
+    parse_nginx,
+    parse_apache,
+    parse_caddy,
+    parse_uvicorn,
+    parse_gunicorn,
+    parse_json,  # generic JSON last — format-specific parsers take priority
+]
+
+
+def parse_line(line: str) -> LogEntry | None:
+    """Try all parsers in order. Returns first successful result or None."""
+    line = line.strip()
+    if not line:
+        return None
+    for parser in _PARSERS:
+        try:
+            entry = parser(line)
+            if entry:
+                return entry
+        except Exception as e:
+            logger.debug(f"Parser {parser.__name__} error: {e}")
+    return None
+
+
+def parse_lines(lines: list[str]) -> list[LogEntry]:
+    """Parse a batch of lines, skipping unparseable ones."""
+    results = []
+    for line in lines:
+        entry = parse_line(line)
+        if entry:
+            results.append(entry)
+    return results

patchi/core/hosted/tokens.py

@@ -0,0 +1,127 @@
+"""
+Admin token management for Patchi hosted mode.
+
+Tokens are generated as random 32-byte hex strings.
+Only the HMAC-SHA256 hash is ever stored — plaintext is shown once on generation.
+
+Storage: .patchi/hosted/tokens.json
+Format: [{id, name, hash, created_at, last_used}]
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import time
+import uuid
+from pathlib import Path
+
+_TOKENS_FILE = ".patchi/hosted/tokens.json"
+_SECRET_FILE = ".patchi/hosted/.secret"
+_HMAC_KEY_LEN = 32
+
+
+def _tokens_path(root: Path) -> Path:
+    return root / _TOKENS_FILE
+
+
+def _secret_path(root: Path) -> Path:
+    return root / _SECRET_FILE
+
+
+def _get_hmac_key(root: Path) -> bytes:
+    """Load or generate a per-installation HMAC key."""
+    path = _secret_path(root)
+    if path.exists():
+        try:
+            return bytes.fromhex(path.read_text(encoding="utf-8").strip())
+        except (ValueError, OSError):
+            pass
+    # Generate and persist a new key
+    key = os.urandom(_HMAC_KEY_LEN)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(key.hex(), encoding="utf-8")
+    return key
+
+
+def _load(root: Path) -> list[dict]:
+    path = _tokens_path(root)
+    if not path.exists():
+        return []
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return []
+
+
+def _save(root: Path, tokens: list[dict]) -> None:
+    path = _tokens_path(root)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(tokens, indent=2), encoding="utf-8")
+
+
+def _hash(plaintext: str, root: Path) -> str:
+    """HMAC-SHA256 hash of the token using per-installation key."""
+    key = _get_hmac_key(root)
+    return hmac.new(key, plaintext.encode(), hashlib.sha256).hexdigest()
+
+
+def generate(root: Path, name: str) -> str:
+    """
+    Generate a new admin token. Stores the hash, returns the plaintext once.
+    The caller must display this to the user — it cannot be recovered later.
+    """
+    plaintext = os.urandom(32).hex()
+    token_id = str(uuid.uuid4())[:8]
+    tokens = _load(root)
+    tokens.append(
+        {
+            "id": token_id,
+            "name": name,
+            "hash": _hash(plaintext, root),
+            "created_at": time.time(),
+            "last_used": None,
+        }
+    )
+    _save(root, tokens)
+    return plaintext
+
+
+def validate(root: Path, plaintext: str) -> dict | None:
+    """
+    Check if a plaintext token matches any stored hash.
+    Returns the token record (without hash) on match, None otherwise.
+    Updates last_used timestamp on success.
+    """
+    candidate_hash = _hash(plaintext, root)
+    tokens = _load(root)
+
+    for token in tokens:
+        if hmac.compare_digest(token["hash"], candidate_hash):
+            token["last_used"] = time.time()
+            _save(root, tokens)
+            return {k: v for k, v in token.items() if k != "hash"}
+
+    return None
+
+
+def list_tokens(root: Path) -> list[dict]:
+    """Return all tokens without hash field."""
+    return [{k: v for k, v in t.items() if k != "hash"} for t in _load(root)]
+
+
+def revoke(root: Path, token_id: str) -> bool:
+    """Remove a token by ID. Returns True if found and removed."""
+    tokens = _load(root)
+    before = len(tokens)
+    tokens = [t for t in tokens if t["id"] != token_id]
+    if len(tokens) == before:
+        return False
+    _save(root, tokens)
+    return True
+
+
+def count(root: Path) -> int:
+    return len(_load(root))

patchi/core/hosted/watchlist.py

@@ -0,0 +1,164 @@
+"""
+IP watchlist for Patchi hosted mode.
+
+Tracks per-IP threat scores over time. Scores decay with TTL.
+Escalates to Notifier when an IP crosses the configured threshold.
+
+Storage: .patchi/hosted/watchlist.json
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+_WATCHLIST_FILE = ".patchi/hosted/watchlist.json"
+_SCORE_TTL_SECS = 3600  # scores decay to 0 after 1 hour of silence
+_ESCALATE_HIGH = 50  # score threshold → HIGH alert
+_ESCALATE_CRITICAL = 100  # score threshold → CRITICAL alert
+
+# Score increments per finding type
+_SCORE_MAP: dict[str, int] = {
+    "brute_force": 40,
+    "injection_probe": 30,
+    "scanner_sweep": 20,
+    "rate_spike": 10,
+    "error_spike": 5,
+    "ml_outlier": 15,
+}
+
+
+def _path(root: Path) -> Path:
+    return root / _WATCHLIST_FILE
+
+
+def _load(root: Path) -> dict[str, dict]:
+    p = _path(root)
+    if not p.exists():
+        return {}
+    try:
+        return json.loads(p.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def _save(root: Path, data: dict[str, dict]) -> None:
+    p = _path(root)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    tmp = p.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    tmp.replace(p)
+
+
+def _decay(entry: dict, now: float) -> float:
+    """Linear decay to 0 over TTL since last_seen."""
+    age = now - entry.get("last_seen", now)
+    ratio = max(0.0, 1.0 - age / _SCORE_TTL_SECS)
+    return entry.get("score", 0.0) * ratio
+
+
+class WatchlistTracker:
+    """
+    Tracks IP threat scores and fires escalation callbacks when thresholds cross.
+
+    escalation_fn(ip, score, severity) is called when an IP crosses a threshold.
+    """
+
+    def __init__(self, root: Path, escalation_fn=None) -> None:
+        self._root = root
+        self._escalation_fn = escalation_fn or (lambda *a: None)
+
+    def record(self, ip: str, detector: str, note: str = "") -> float:
+        """
+        Add score for a detector hit on this IP.
+        Returns the new effective score after decay + increment.
+        """
+        if not ip:
+            return 0.0
+
+        now = time.time()
+        data = _load(self._root)
+        entry = data.get(
+            ip,
+            {
+                "ip": ip,
+                "score": 0.0,
+                "last_seen": now,
+                "hit_count": 0,
+                "escalated_high": False,
+                "escalated_critical": False,
+                "notes": [],
+            },
+        )
+
+        # Apply decay since last event
+        decayed = _decay(entry, now)
+        increment = _SCORE_MAP.get(detector, 10)
+        new_score = decayed + increment
+
+        entry["score"] = new_score
+        entry["last_seen"] = now
+        entry["hit_count"] = entry.get("hit_count", 0) + 1
+        if note:
+            entry["notes"] = (entry.get("notes", []) + [note])[-20:]
+
+        data[ip] = entry
+
+        # Check escalation before saving so we use in-memory data (no re-read race)
+        should_escalate_critical = new_score >= _ESCALATE_CRITICAL and not entry.get(
+            "escalated_critical"
+        )
+        should_escalate_high = (
+            new_score >= _ESCALATE_HIGH
+            and not should_escalate_critical
+            and not entry.get("escalated_high")
+        )
+
+        if should_escalate_critical:
+            entry["escalated_critical"] = True
+        elif should_escalate_high:
+            entry["escalated_high"] = True
+
+        _save(self._root, data)
+
+        if should_escalate_critical:
+            self._escalation_fn(ip, new_score, "critical")
+        elif should_escalate_high:
+            self._escalation_fn(ip, new_score, "high")
+
+        return new_score
+
+    def score(self, ip: str) -> float:
+        """Current decayed score for an IP."""
+        data = _load(self._root)
+        entry = data.get(ip)
+        if not entry:
+            return 0.0
+        return _decay(entry, time.time())
+
+    def top(self, n: int = 10) -> list[dict]:
+        """Return top-n IPs by current decayed score."""
+        now = time.time()
+        data = _load(self._root)
+        scored = [{**entry, "current_score": _decay(entry, now)} for entry in data.values()]
+        return sorted(scored, key=lambda e: e["current_score"], reverse=True)[:n]
+
+    def clear_ip(self, ip: str) -> bool:
+        """Remove an IP from the watchlist. Returns True if found."""
+        data = _load(self._root)
+        if ip not in data:
+            return False
+        del data[ip]
+        _save(self._root, data)
+        return True
+
+    def purge_expired(self) -> int:
+        """Remove IPs whose score has fully decayed. Returns count removed."""
+        now = time.time()
+        data = _load(self._root)
+        keep = {ip: e for ip, e in data.items() if _decay(e, now) > 0}
+        removed = len(data) - len(keep)
+        if removed:
+            _save(self._root, keep)
+        return removed