umbermark-cli 1.0.0__py3-none-any.whl

umbermark/core/labels.py

@@ -0,0 +1,29 @@
+"""Product status label, vendored from starhold-ops `docs/shared/labels.json`.
+
+The JSON next to the templates is a byte-for-byte copy of the ops file; the
+label for this product is looked up by key, never typed here.
+"""
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from pathlib import Path
+
+LABELS_PATH = Path(__file__).parent.parent / "web" / "labels.json"
+PRODUCT_KEY = "umbermark"
+
+
+@lru_cache(maxsize=1)
+def _load() -> dict:
+    return json.loads(LABELS_PATH.read_text(encoding="utf-8"))
+
+
+LAUNCHED_LABEL = "Open · paid plans"
+
+
+def product_label(launched: bool = False) -> str:
+    data = _load()
+    label = LAUNCHED_LABEL if launched else data["products"][PRODUCT_KEY]
+    if label not in data["labels"]:
+        raise ValueError(f"label {label!r} is not in the allowed label set")
+    return label

umbermark/core/lifecycle.py

@@ -0,0 +1,100 @@
+"""Report lifecycle rules shared by the web app and the purge job.
+
+  - Free anonymous / upload / outside-in reports: 30 days (expires_at set at
+    creation from ANONYMOUS_REPORT_TTL_SECONDS).
+  - Paid ($5) report: 3 months from payment, or until deleted, whichever is
+    first. A report claimed into an account has no expiry (it lives with the
+    account).
+"""
+from __future__ import annotations
+
+import calendar
+import datetime as dt
+import logging
+import sqlite3
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+PAID_RETENTION_MONTHS = 3
+# After this many failed PDF removals the purge job logs an ALERT every run.
+PURGE_ALERT_AFTER = 3
+
+
+def add_months(moment: dt.datetime, months: int) -> dt.datetime:
+    """Calendar-month addition; the day is clamped to the target month's length."""
+    index = moment.month - 1 + months
+    year = moment.year + index // 12
+    month = index % 12 + 1
+    day = min(moment.day, calendar.monthrange(year, month)[1])
+    return moment.replace(year=year, month=month, day=day)
+
+
+def paid_expiry(paid_at: dt.datetime) -> str:
+    return add_months(paid_at, PAID_RETENTION_MONTHS).isoformat()
+
+
+def is_expired(
+    expires_at: str | None, now: dt.datetime | None = None, *, row_id: object = None
+) -> bool:
+    """True once `expires_at` has passed. None/empty means no expiry.
+
+    Fails CLOSED: a value that cannot be parsed (or has no timezone) counts as
+    expired, so a corrupted row is purged rather than kept forever. Only the row
+    id is logged, never the value or any content.
+    """
+    if not expires_at:
+        return False
+    try:
+        moment = dt.datetime.fromisoformat(expires_at)
+        return (now or dt.datetime.now(dt.UTC)) >= moment
+    except (ValueError, TypeError):
+        log.warning("unparseable expires_at on row %s: treated as expired", row_id)
+        return True
+
+
+def remove_pdf(pdf_dir: str | Path | None, short_id: str) -> bool:
+    """Delete the cached PDF for a report if one exists. Never raises.
+
+    Returns True when no PDF remains (deleted, or none existed) and False when
+    an unlink failed, so the caller can keep the row and retry later.
+    """
+    if not pdf_dir:
+        return True
+    try:
+        path = Path(pdf_dir) / f"{short_id}.pdf"
+        # short_id comes from our own rows, but never trust a path component.
+        if path.parent == Path(pdf_dir) and path.is_file():
+            path.unlink()
+        return True
+    except OSError:
+        return False
+
+
+def erase_report(conn: sqlite3.Connection, report, pdf_dir: str | Path | None) -> bool:
+    """Erase a report: PDF FIRST, then the row (keys, intents, tokens go with it).
+
+    Returns True when fully erased. If the PDF cannot be removed the row is kept,
+    soft-deleted (invisible at once, all its keys and intents revoked) and marked
+    `purge_failed_at`; the purge job retries it on every run and logs an ALERT
+    after PURGE_ALERT_AFTER failures. Call inside a transaction.
+    """
+    from umbermark.db.repos import reports as reports_repo
+
+    if not remove_pdf(pdf_dir, report.short_id):
+        now = dt.datetime.now(dt.UTC).isoformat()
+        conn.execute("DELETE FROM report_keys WHERE report_id = ?", (report.id,))
+        conn.execute("DELETE FROM purchase_intents WHERE report_id = ?", (report.id,))
+        conn.execute(
+            "UPDATE reports SET deleted_at = COALESCE(deleted_at, ?), purge_failed_at = ?, "
+            "purge_failures = purge_failures + 1 WHERE id = ?",
+            (now, now, report.id),
+        )
+        failures = conn.execute("SELECT purge_failures FROM reports WHERE id = ?", (report.id,)).fetchone()[0]
+        if failures >= PURGE_ALERT_AFTER:
+            log.error("ALERT: PDF removal failed %s times for report row %s", failures, report.id)
+        else:
+            log.warning("PDF removal failed for report row %s (attempt %s); will retry", report.id, failures)
+        return False
+    reports_repo.hard_delete(conn, report.id)
+    return True

umbermark/core/logredact.py

@@ -0,0 +1,55 @@
+"""Log redaction: no sign-in token, claim token or key may reach a log line.
+
+uvicorn's access log prints the request line, which includes `?token=...` on
+magic-link verification and `/claim/<token>` on claim links. This filter
+rewrites those before the record is emitted. It is installed on the access
+logger at app creation and is cheap enough to attach to the root logger too.
+"""
+from __future__ import annotations
+
+import logging
+import re
+
+_PATTERNS = (
+    (re.compile(r"([?&](?:token|key|ownership_key|delete_key)=)[^&\s\"']+", re.I), r"\1[redacted]"),
+    (re.compile(r"(/claim/)[^\s/?\"']+"), r"\1[redacted]"),
+    (re.compile(r"(/auth/verify\?token=)[^\s\"']+"), r"\1[redacted]"),
+    (re.compile(r"\bumk_[A-Za-z0-9_\-]{8,}"), "umk_[redacted]"),
+)
+
+
+def redact(text: str) -> str:
+    for pattern, repl in _PATTERNS:
+        text = pattern.sub(repl, text)
+    return text
+
+
+class RedactFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        try:
+            message = record.getMessage()
+        except Exception:
+            return True
+        cleaned = redact(message)
+        if cleaned == message:
+            return True
+        # Redact inside the args when they are a plain tuple, and keep the tuple. uvicorn's access formatter unpacks
+        # exactly five args (client, method, path, version, status); emptying them made it raise inside logging and
+        # the whole access line was lost. Anything else (dict args, no args) falls back to a flattened message.
+        if isinstance(record.args, tuple) and record.args:
+            record.args = tuple(redact(a) if isinstance(a, str) else a for a in record.args)
+            try:
+                if redact(record.getMessage()) == record.getMessage():
+                    return True
+            except Exception:
+                pass
+        record.msg = cleaned
+        record.args = ()
+        return True
+
+
+def install(logger_names: tuple[str, ...] = ("uvicorn.access", "uvicorn.error", "umbermark")) -> None:
+    for name in logger_names:
+        logger = logging.getLogger(name)
+        if not any(isinstance(f, RedactFilter) for f in logger.filters):
+            logger.addFilter(RedactFilter())

umbermark/core/partial.py

@@ -0,0 +1,57 @@
+"""Outside-in partial score (UMBR-67).
+
+An external scan of a host whose ownership is NOT verified runs the passive
+checks only. Its number is computed from those checks alone, so it is shown as
+a labelled "Outside-in score" with the count of checks that ran, never as the
+full hardening score and never with a band. The stored band stays whatever the
+scoring engine said (normally "incomplete"), so history and deltas keep
+treating it as not comparable with a full score.
+
+Pure functions over the stored payload dict: no I/O.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+LABEL = "Outside-in score"
+KIND_PARTIAL = "outside_in_partial"
+KIND_FULL = "hardening"
+
+
+def outside_in(payload: Any) -> dict[str, int] | None:
+    """{'score', 'ran', 'total', 'gated'} for an unverified outside-in scan, else None.
+
+    None means "show this report the usual way": a verified external scan, an
+    on-box scan, a blocked target, or a scan where no check produced a verdict
+    (there is nothing to score, so the "?" stays).
+    """
+    if not isinstance(payload, dict):
+        return None
+    gate = payload.get("external_gate")
+    if not isinstance(gate, dict) or gate.get("verified") or gate.get("blocked"):
+        return None
+    try:
+        gated = int(gate.get("gated_count") or 0)
+        result = payload.get("result") or {}
+        ran = sum(int(result.get(k) or 0) for k in ("pass_count", "warn_count", "fail_count"))
+        total = ran + int(result.get("skip_count") or 0)
+        score = int(result["score"])
+    except (KeyError, TypeError, ValueError, AttributeError):
+        return None
+    if gated <= 0 or ran <= 0:
+        return None
+    return {"score": max(0, min(100, score)), "ran": ran, "total": total, "gated": gated}
+
+
+def outside_in_json(payload_json: str | None) -> dict[str, int] | None:
+    """`outside_in` over a stored payload_json string (bad JSON -> None)."""
+    try:
+        return outside_in(json.loads(payload_json or ""))
+    except ValueError:
+        return None
+
+
+def short_label(p: dict[str, int]) -> str:
+    """'Outside-in score 97 (partial, 2 of 7 checks)' for one-line places."""
+    return f"{LABEL} {p['score']} (partial, {p['ran']} of {p['total']} checks)"

umbermark/core/pdf/renderer.py

@@ -0,0 +1,26 @@
+"""PdfRenderer Protocol + FakePdfRenderer.
+
+The Fake returns a bytestring that starts with the PDF magic header so
+downstream code that sniffs for valid PDFs is happy, and embeds the
+original HTML in the body so integration tests can assert on rendered
+content without parsing real PDFs.
+
+Real impl lives in weasyprint_renderer.py and is only loaded in
+production / Linux CI.
+"""
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class PdfRenderer(Protocol):
+    def render(self, *, html: str, base_url: str) -> bytes: ...
+
+
+class FakePdfRenderer:
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str]] = []
+
+    def render(self, *, html: str, base_url: str) -> bytes:
+        self.calls.append((html, base_url))
+        return b"%PDF-1.4\n" + html.encode("utf-8") + b"\n%%EOF\n"

umbermark/core/pdf/weasyprint_renderer.py

@@ -0,0 +1,18 @@
+"""Production PDF renderer using WeasyPrint.
+
+Imported lazily — on Windows + macOS WeasyPrint needs GTK + cairo
+system libs that are non-trivial to install. On a Linux container
+(the prod target) WeasyPrint is a clean pip install. Tests that
+need a "real" PDF renderer are marked skip-if-not-installed.
+"""
+from __future__ import annotations
+
+
+class WeasyPrintRenderer:
+    def __init__(self) -> None:
+        from weasyprint import HTML  # noqa: F401
+
+    def render(self, *, html: str, base_url: str) -> bytes:
+        from weasyprint import HTML
+
+        return HTML(string=html, base_url=base_url).write_pdf()

umbermark/core/platform.py

@@ -0,0 +1,73 @@
+"""Distro / platform detection — parses /etc/os-release into a normalized family."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from umbermark.core.ssh import CommandRunner
+
+# Map of distro ID (or any token in ID_LIKE) -> family the checks use.
+_FAMILY_BY_ID = {
+    "debian": "debian",
+    "ubuntu": "debian",
+    "linuxmint": "debian",
+    "pop": "debian",
+    "rhel": "rhel",
+    "centos": "rhel",
+    "rocky": "rhel",
+    "almalinux": "rhel",
+    "fedora": "rhel",
+    "ol": "rhel",
+}
+
+
+@dataclass
+class PlatformFacts:
+    distro: str = "unknown"
+    distro_version: str = "unknown"
+    distro_family: str = "unknown"
+    arch: str = "unknown"
+    is_container: bool = False
+    raw: dict[str, str] = field(default_factory=dict)
+
+
+def _parse_os_release(text: str) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for line in text.splitlines():
+        m = re.match(r"^([A-Z_]+)=(.*)$", line.strip())
+        if m:
+            key = m.group(1)
+            val = m.group(2).strip().strip('"').strip("'")
+            out[key] = val
+    return out
+
+
+def _family_for(distro_id: str, id_like: str) -> str:
+    if distro_id in _FAMILY_BY_ID:
+        return _FAMILY_BY_ID[distro_id]
+    for token in id_like.split():
+        if token in _FAMILY_BY_ID:
+            return _FAMILY_BY_ID[token]
+    return "unknown"
+
+
+def detect_platform(runner: CommandRunner) -> PlatformFacts:
+    facts = PlatformFacts()
+    r = runner.run("cat /etc/os-release 2>/dev/null", timeout=5)
+    if r.ok and r.stdout:
+        parsed = _parse_os_release(r.stdout)
+        facts.raw = parsed
+        facts.distro = parsed.get("ID", "unknown")
+        facts.distro_version = parsed.get("VERSION_ID", "unknown")
+        facts.distro_family = _family_for(facts.distro, parsed.get("ID_LIKE", ""))
+    r2 = runner.run("uname -m 2>/dev/null", timeout=5)
+    if r2.ok and r2.stdout:
+        facts.arch = r2.stdout.strip()
+    # Build the access context now: this issues every detector's collector
+    # commands, so capture enumeration (which drives off this function) sees them.
+    from umbermark.core.access import maybe_context
+
+    # Cached on the runner; checks fetch it with get_access_context(runner). It is
+    # deliberately NOT stored on PlatformFacts, which is serialised into reports.
+    maybe_context(runner)
+    return facts

umbermark/core/redact.py

@@ -0,0 +1,75 @@
+"""Public-view scrubbing for reports.
+
+Spec §12.3: public `/r/<short_id>` shows score + band + counts + deduction
+titles (severity-only) + each finding's title/severity/verdict/generic
+remediation. It MUST NOT include specific evidence, skip_reason details,
+platform_facts, or the unscrubbed target identifier.
+
+`redact_report(report, target)` is the canonical entry point. It returns a
+NEW dict (does not mutate the input).
+"""
+from __future__ import annotations
+
+import re
+from copy import deepcopy
+from typing import Any
+
+from umbermark.core.wording import skip_note
+
+_IPV4 = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b")
+# IPv6 — accepts forms with `::` shorthand (e.g., 2001:db8::1).
+# Boundary lookarounds prevent partial mid-word matches.
+_IPV6 = re.compile(
+    r"(?<![\w:])[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{0,4}){2,7}(?![\w:])"
+)
+_ABS_PATH = re.compile(r"(?<![A-Za-z0-9])/[A-Za-z0-9_.][A-Za-z0-9_./-]*")
+# Version: dotted numeric core + optional alphanumeric tail (e.g., 9.2p1-2+deb12u3)
+_VERSION = re.compile(r"\b\d+(?:\.\d+){1,3}[A-Za-z0-9_.+~-]*\b")
+
+REDACTED = "<redacted>"
+
+# UMBR-20: checks whose evidence is built only from port numbers, bind-scope words and process names (never an
+# address or a path), so the public view can keep it. A finding without its evidence is not actionable. The kept
+# text still goes through redact_evidence as a second line of defence.
+PUBLIC_SAFE_EVIDENCE: frozenset[str] = frozenset({"public_listen_anomalies"})
+
+
+def redact_evidence(text: str, target: str = "") -> str:
+    """Strip identifying tokens from a free-form evidence string.
+
+    Replaces: the literal target string, IPv4/IPv6 addresses, absolute paths,
+    and dotted-version strings. Order matters: target first (so we don't
+    accidentally treat the target as a version), then IPs, then paths, then
+    versions.
+    """
+    if not text:
+        return text
+    result = text
+    if target:
+        result = result.replace(target, REDACTED)
+    result = _IPV4.sub(REDACTED, result)
+    result = _IPV6.sub(REDACTED, result)
+    result = _ABS_PATH.sub("<path>", result)
+    result = _VERSION.sub("<version>", result)
+    return result
+
+
+def redact_report(report: dict[str, Any], target: str) -> dict[str, Any]:
+    """Return a scrubbed copy of `report` for public consumption."""
+    scrubbed = deepcopy(report)
+    scrubbed["target"] = REDACTED
+    scrubbed.pop("platform_facts", None)
+    findings = scrubbed.get("findings", [])
+    for f in findings:
+        # Public view drops specific evidence and skip-reason details entirely, except the allow-listed
+        # IP-free evidence above (scrubbed again regardless).
+        if f.get("check_id") in PUBLIC_SAFE_EVIDENCE and f.get("verdict") in ("warn", "fail"):
+            f["evidence"] = redact_evidence(f.get("evidence", ""), target)
+        else:
+            f["evidence"] = ""
+        # Keep a generic, human reason ("needs root", "not applicable") before
+        # the raw detail is dropped, so skip rows can still explain themselves.
+        if f.get("verdict") == "skip":
+            f["skip_note"] = skip_note(f.get("skip_reason", ""))
+        f["skip_reason"] = ""
+    return scrubbed

umbermark/core/remediation.py

@@ -0,0 +1,121 @@
+"""Remediation bundle: one reviewable shell file per server, built from its latest scan (UMBR-66; Pro and Agency).
+
+Every failing or warning finding that costs points (and is not an accepted risk) becomes one function holding the
+Fix Pack steps for the server's distribution family. Nothing runs by default:
+
+    bash umbermark-fix-<id>.sh list            # what is in the file
+    sudo bash umbermark-fix-<id>.sh run ID ... # run the named fixes, each in its own subshell
+
+Steps the catalogue marks as needing judgement are included commented out, with a note, so they are never run
+blindly. The snippets are the same reviewed Fix Pack text shown on the report page.
+"""
+from __future__ import annotations
+
+import re
+
+from umbermark.catalogue import get_entry
+
+_RHEL = ("rhel", "rocky", "alma", "almalinux", "fedora", "centos", "ol", "oracle")
+_DEBIAN = ("debian", "ubuntu", "raspbian", "linuxmint", "pop")
+_ID_RE = re.compile(r"^[a-z0-9_]{1,64}$")
+
+
+def family(payload: dict) -> str | None:
+    """'debian', 'rhel' or None (unknown: both variants are included, each labelled)."""
+    distro = str((payload.get("platform_facts") or {}).get("distro", "")).lower()
+    if any(distro.startswith(d) for d in _DEBIAN):
+        return "debian"
+    if any(distro.startswith(d) for d in _RHEL):
+        return "rhel"
+    return None
+
+
+def _comment(text: str) -> str:
+    return "\n".join("  # " + line if line else "  #" for line in text.splitlines())
+
+
+def _step_lines(step, fam: str | None) -> list[str]:
+    out = [f"  # --- {step.title}"]
+    picks: list[tuple[str, str]] = []
+    if step.any is not None:
+        picks.append(("Any distro", step.any))
+    if fam in (None, "debian") and step.debian is not None:
+        picks.append(("Debian / Ubuntu", step.debian))
+    if fam in (None, "rhel") and step.rhel is not None:
+        picks.append(("RHEL / Rocky / Alma / Fedora", step.rhel))
+    for label, snippet in picks:
+        if fam is None and label != "Any distro":
+            out.append(f"  # {label} only: commented out because the scan could not tell the distribution.")
+            out.append(_comment(snippet))
+        elif step.judgement:
+            out.append("  # Needs your judgement: read it, adapt it, then uncomment.")
+            out.append(_comment(snippet))
+        else:
+            # Not indented: a heredoc terminator inside a snippet must stay at column 0.
+            out.extend(snippet.rstrip("\n").splitlines())
+    return out
+
+
+def build_script(*, payload: dict, server_label: str, scanned_at: str, score: int, accepted_ids: set[str]) -> str:
+    fam = family(payload)
+    lost = {d.get("check_id"): d.get("points_lost", 0) for d in (payload.get("result") or {}).get("deductions", [])}
+    blocks: list[tuple[str, str, list[str]]] = []
+    for f in payload.get("findings", []) or []:
+        cid = f.get("check_id", "")
+        if not _ID_RE.match(cid) or str(f.get("verdict")) not in ("fail", "warn") or not lost.get(cid):
+            continue
+        if cid in accepted_ids:
+            continue
+        entry = get_entry(cid)
+        if entry is None or not entry.fix_steps:
+            continue
+        lines: list[str] = []
+        for step in entry.fix_steps:
+            lines.extend(_step_lines(step, fam))
+        blocks.append((cid, entry.title.replace("\n", " "), lines))
+    label = re.sub(r"[^\w .:@-]", "_", server_label)[:80]
+    head = [
+        "#!/usr/bin/env bash",
+        "# =============================================================================",
+        f"# Umbermark remediation bundle for {label}",
+        f"# From the scan of {scanned_at[:16].replace('T', ' ')} UTC (score {score}).",
+        f"# Distribution family: {fam or 'unknown (distro-specific steps are commented out)'}",
+        "#",
+        "# REVIEW EVERY BLOCK BEFORE RUNNING. Nothing runs unless you name it:",
+        "#   bash this-file.sh list",
+        "#   sudo bash this-file.sh run CHECK_ID [CHECK_ID ...]",
+        "# Each fix runs in its own subshell. Accepted risks are left out. Re-scan afterwards to verify.",
+        "# =============================================================================",
+        "set -uo pipefail",
+        "",
+    ]
+    body: list[str] = []
+    for cid, title, lines in blocks:
+        body.append(f"fix_{cid}() {{  # {title}")
+        body.extend(lines)
+        body.append("  :")
+        body.append("}")
+        body.append("")
+    ids = " ".join(cid for cid, _, _ in blocks)
+    listing = "\n".join(f"    echo '  {cid}  {title.replace(chr(39), '')}'" for cid, title, _ in blocks) or "    echo '  (none)'"
+    tail = [
+        'case "${1:-list}" in',
+        "  list)",
+        f"    echo 'Fixes in this file ({len(blocks)}):'",
+        listing,
+        "    ;;",
+        "  run)",
+        "    shift",
+        '    [ "$#" -gt 0 ] || { echo "name at least one fix: run CHECK_ID ..." >&2; exit 2; }',
+        '    for id in "$@"; do',
+        '      case " ' + ids + ' " in',
+        '        *" $id "*) echo "== $id"; ( "fix_$id" ) || echo "!! $id exited with $?" >&2 ;;',
+        '        *) echo "unknown fix: $id (see: list)" >&2; exit 2 ;;',
+        "      esac",
+        "    done",
+        "    ;;",
+        '  *) echo "usage: $0 list | run CHECK_ID ..." >&2; exit 2 ;;',
+        "esac",
+        "",
+    ]
+    return "\n".join(head + body + tail)

umbermark/core/report.py

@@ -0,0 +1,25 @@
+"""Report serialization — JSON for phase 0; HTML/PDF arrive in phase 2."""
+from __future__ import annotations
+
+import json
+from dataclasses import fields, is_dataclass
+from enum import Enum
+from typing import Any
+
+from umbermark.core.engine import ScanReport
+
+
+def _to_dict(obj: Any) -> Any:
+    if isinstance(obj, Enum):
+        return obj.value
+    if is_dataclass(obj):
+        return {f.name: _to_dict(getattr(obj, f.name)) for f in fields(obj)}
+    if isinstance(obj, list):
+        return [_to_dict(x) for x in obj]
+    if isinstance(obj, dict):
+        return {k: _to_dict(v) for k, v in obj.items()}
+    return obj
+
+
+def to_json(report: ScanReport, indent: int | None = 2) -> str:
+    return json.dumps(_to_dict(report), indent=indent)