memdebug 0.2.0__py3-none-any.whl

memdebug/stores.py

@@ -0,0 +1,276 @@
+"""The stores memdebug watches: a small settings file, finding likely stores, and opening them.
+
+The settings file is read back into the program, so it is validated strictly (names, kinds, option types, size) and a
+damaged or hostile file produces a clear message, never a crash or a surprise. Finding likely stores only looks at
+folder names and whether they hold markdown files; it never reads their contents.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import secrets
+import sqlite3
+import stat
+from contextlib import closing
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+from .adapters.base import MemoryAdapter
+from .adapters.folder import FolderAdapter
+from .adapters.markdown_git import MarkdownGitAdapter, _is_reparse_point, _valid_relpath
+from .adapters.mem0 import Mem0Adapter, build_mem0_memory, validate_scope
+from .adapters.openwebui import OpenWebUIAdapter
+from .docker_source import Docker, copy_database, list_open_webui, valid_container
+from .errors import MemdebugError
+from .textsafe import has_unsafe_chars, safe_text
+
+KINDS = ("markdown", "folder", "mem0", "openwebui")
+KIND_NAMES = {"markdown": "markdown notes in a git repository", "folder": "a folder of markdown notes",
+              "mem0": "self-hosted Mem0", "openwebui": "Open WebUI's memory"}
+MAX_SETTINGS_BYTES = 1_000_000
+MAX_STORES = 200
+_NAME = re.compile(r"^[a-z0-9][a-z0-9._-]{0,39}\Z")
+_OPTIONS = ("subdir", "user_id", "agent_id", "run_id", "docker", "files")
+
+
+class SettingsError(MemdebugError):
+    """The settings file is unusable or a store's settings are wrong."""
+
+
+@dataclass(frozen=True)
+class StoreConfig:
+    name: str
+    kind: str
+    path: str
+    subdir: str | None = None
+    user_id: str | None = None
+    agent_id: str | None = None
+    run_id: str | None = None
+    docker: str | None = None  # Open WebUI only: the container its database is copied from, before every look
+    files: str | None = None  # folder only: watch just these files in it (comma-separated names), nothing else in the folder
+
+    def describe(self) -> str:
+        return f"{self.name} ({KIND_NAMES[self.kind]})"
+
+
+@dataclass
+class Registry:
+    stores: list[StoreConfig] = field(default_factory=list)
+    witness: str | None = None
+
+    def get(self, name: str) -> StoreConfig | None:
+        return next((s for s in self.stores if s.name == name), None)
+
+    def add(self, store: StoreConfig) -> None:
+        if self.get(store.name) is not None:
+            raise SettingsError(f"there is already a store called {safe_text(store.name, 40)}; choose another name or remove it first")
+        if len(self.stores) >= MAX_STORES:
+            raise SettingsError("too many stores")
+        self.stores.append(store)
+
+    def remove(self, name: str) -> StoreConfig:
+        store = self.get(name)
+        if store is None:
+            raise SettingsError(f"no store is called {safe_text(name, 40)}")
+        self.stores.remove(store)
+        return store
+
+
+def valid_name(name: str) -> bool:
+    return bool(_NAME.match(name))
+
+
+def _text(value: object, what: str, *, limit: int = 1000) -> str:
+    if not isinstance(value, str) or not value or len(value) > limit or has_unsafe_chars(value) or "\0" in value:
+        raise SettingsError(f"the settings file has an invalid {what}")
+    return value
+
+
+def validate_store(data: object) -> StoreConfig:
+    if not isinstance(data, dict) or not {"name", "kind", "path"} <= set(data) or set(data) - {"name", "kind", "path", *_OPTIONS}:
+        raise SettingsError("the settings file has a store entry with unexpected fields")
+    name, kind = data["name"], data["kind"]
+    if not isinstance(name, str) or not valid_name(name):
+        raise SettingsError("a store name must be 1 to 40 lowercase letters, digits, dots, dashes or underscores")
+    if kind not in KINDS:
+        raise SettingsError(f"unknown store type for {safe_text(name, 40)}")
+    options = {key: (None if data.get(key) is None else _text(data[key], key, limit=256)) for key in _OPTIONS}
+    if kind == "mem0" and options["user_id"] is None:
+        raise SettingsError(f"the Mem0 store {name} needs a user id")
+    if options["files"] is not None:
+        names = options["files"].split(",")
+        if kind != "folder" or len(names) > 10 or any("/" in n or _valid_relpath(n, (".md",)) is None for n in names):
+            raise SettingsError(f"{safe_text(name, 40)}: 'files' must be 1 to 10 plain markdown file names, for a folder store")
+    if options["docker"] is not None and (kind != "openwebui" or not valid_container(options["docker"])):
+        raise SettingsError(f"{safe_text(name, 40)}: a container can only be named for an Open WebUI store, with a plain container name")
+    return StoreConfig(name=name, kind=kind, path=_text(data["path"], "path"), **options)
+
+
+def load_registry(path: Path) -> Registry:
+    """The registry from its settings file; an empty one if there is no file yet."""
+    try:
+        info = os.lstat(path)
+    except FileNotFoundError:
+        return Registry()
+    except OSError as exc:
+        raise SettingsError(f"cannot read the settings file ({exc.strerror})") from exc
+    if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode) or info.st_size > MAX_SETTINGS_BYTES:
+        raise SettingsError("the settings file is not a plain file of reasonable size")
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise SettingsError(f"the settings file is damaged ({safe_text(exc, 80)}); fix or delete {safe_text(path, 120)}") from exc
+    if not isinstance(data, dict) or data.get("version") != 1 or set(data) - {"version", "stores", "witness"}:
+        raise SettingsError("the settings file is not in a format this version understands")
+    raw_stores = data.get("stores", [])
+    if not isinstance(raw_stores, list) or len(raw_stores) > MAX_STORES:
+        raise SettingsError("the settings file lists too many stores")
+    registry = Registry()
+    for item in raw_stores:
+        registry.add(validate_store(item))
+    witness = data.get("witness")
+    registry.witness = None if witness is None else _text(witness, "witness path")
+    return registry
+
+
+def save_registry(registry: Registry, path: Path) -> None:
+    """Write the settings atomically and privately."""
+    document = {"version": 1, "stores": [{k: v for k, v in asdict(s).items() if v is not None} for s in registry.stores]}
+    if registry.witness:
+        document["witness"] = registry.witness
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        temporary = path.with_name(f".{path.name}.{secrets.token_hex(6)}.tmp")
+        fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600)
+        try:
+            with os.fdopen(fd, "wb") as handle:
+                handle.write((json.dumps(document, indent=2, ensure_ascii=True) + "\n").encode("utf-8"))
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(temporary, path)
+        except BaseException:
+            try:
+                os.unlink(temporary)
+            except OSError:
+                pass
+            raise
+    except OSError as exc:
+        raise SettingsError(f"cannot save the settings file ({exc.strerror})") from exc
+
+
+# -- opening a store ----------------------------------------------------------------------------------------------------
+
+@dataclass
+class OpenedStore:
+    adapter: MemoryAdapter
+    scope: dict[str, str]
+    notes: list[str] = field(default_factory=list)
+
+
+def open_store(cfg: StoreConfig, *, refresh: bool = True) -> OpenedStore:
+    """Open a store for reading. An Open WebUI store that lives in Docker gets a fresh copy of its database first
+    (`refresh=False` reads the copy that is already there, for commands that only look)."""
+    if cfg.docker and refresh:
+        copy_database(Docker(), cfg.docker, Path(cfg.path))
+    if cfg.kind == "markdown":
+        adapter = MarkdownGitAdapter(cfg.path, store=cfg.name, subdir=cfg.subdir)
+        return OpenedStore(adapter, {"store": adapter.store})
+    if cfg.kind == "folder":
+        folder = FolderAdapter(cfg.path, store=cfg.name, subdir=cfg.subdir, only=tuple(cfg.files.split(",")) if cfg.files else None)
+        return OpenedStore(folder, {"store": folder.store})
+    if cfg.kind == "openwebui":
+        webui = OpenWebUIAdapter(cfg.path, user_id=cfg.user_id)
+        return OpenedStore(webui, webui.scope)
+    scope = validate_scope({k: v for k, v in (("user_id", cfg.user_id), ("agent_id", cfg.agent_id), ("run_id", cfg.run_id)) if v is not None})
+    memory, notes = build_mem0_memory(None)
+    return OpenedStore(Mem0Adapter(memory, Path(cfg.path)), scope, notes)
+
+
+# -- telling what a path is -------------------------------------------------------------------------------------------
+
+def detect_kind(path: Path) -> str:
+    """markdown, folder, openwebui or mem0, from what the path is. Reads structure only, never contents."""
+    try:
+        info = os.stat(path)
+    except OSError as exc:
+        raise SettingsError(f"cannot find that path ({exc.strerror})") from exc
+    if stat.S_ISDIR(info.st_mode):
+        return "markdown" if (path / ".git").exists() else "folder"
+    if not stat.S_ISREG(info.st_mode):
+        raise SettingsError("that is neither a folder nor a file")
+    try:
+        with closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True, timeout=3)) as db:
+            tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
+            if "memory" in tables:
+                columns = {row[1] for row in db.execute('PRAGMA table_info("memory")')}
+                if {"id", "user_id", "content"} <= columns:
+                    return "openwebui"
+            if "history" in tables:
+                return "mem0"
+    except sqlite3.Error:
+        pass
+    raise SettingsError("that file is not a database memdebug recognises (Open WebUI's webui.db or Mem0's history.db)")
+
+
+# -- finding likely stores ---------------------------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class Candidate:
+    kind: str
+    name: str
+    path: Path
+    why: str
+    docker: str | None = None
+    files: tuple[str, ...] | None = None
+
+
+def _slug(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
+
+
+def discover(home: Path | None = None) -> list[Candidate]:
+    """Stores that are probably on this computer. Looks only at folder names and for markdown files, never inside them."""
+    base = (home or Path.home()) / ".claude" / "projects"
+    found: list[Candidate] = []
+    taken: set[str] = set()
+    try:
+        entries = sorted(os.scandir(base), key=lambda e: e.name)[:300]
+    except OSError:
+        return []
+    for entry in entries:
+        try:
+            if not entry.is_dir(follow_symlinks=False) or has_unsafe_chars(entry.name):
+                continue
+            memory = Path(entry.path) / "memory"
+            info = os.lstat(memory)
+            if stat.S_ISLNK(info.st_mode) or _is_reparse_point(info) or not stat.S_ISDIR(info.st_mode):
+                continue
+            names = os.listdir(memory)[:2000]
+        except OSError:
+            continue
+        if not any(n.lower().endswith(".md") for n in names):
+            continue
+        slug = "claude-" + (_slug(entry.name)[-24:].strip("-") or "project")
+        name, n = slug[:40], 2
+        while name in taken:
+            name, n = f"{slug[:36]}-{n}", n + 1
+        taken.add(name)
+        found.append(Candidate("folder", name, memory, "Claude Code's memory notes for one project"))
+        if len(found) >= 50:
+            break
+    return found
+
+
+def discover_docker(copies_dir: Path, docker: Docker | None = None) -> list[Candidate]:
+    """Open WebUI containers that are running right now. Quietly nothing if Docker is missing or not running: it is only a convenience."""
+    try:
+        client = docker or Docker()
+        names = list_open_webui(client)
+    except MemdebugError:
+        return []
+    found = []
+    for container in names:
+        slug = _slug(container)[:40] or "open-webui"
+        found.append(Candidate("openwebui", slug, copies_dir / slug / "webui.db", f"Open WebUI's memory, running in Docker as '{container}'", container))
+    return found

memdebug/sync.py

@@ -0,0 +1,177 @@
+"""One sync: copy new backend history into the ledger, then look for changes that bypassed it.
+
+Safety rules baked in:
+  * history rows are deduplicated by the backend's own row id, so repeating a sync is harmless;
+  * a change outside the API is only recorded if it is seen in two passes, so a write that is
+    halfway through (store updated, history row not yet written) is not a false alarm;
+  * no deletions are claimed from a listing that may be cut short, and no changes at all are
+    claimed from a history that was only partly read;
+  * history rows that were recorded earlier but have since disappeared are reported;
+  * everything is appended in one transaction.
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Callable
+
+from .adapters.base import HistoryRead, LiveMemories, MemoryAdapter
+from .errors import LedgerConflictError
+from .ledger import Ledger
+from .models import MemoryEvent, Op, Source, derive_trust
+from .reconcile import find_external_changes, replay_events
+from .textsafe import safe_text
+
+
+@dataclass
+class SyncReport:
+    history_events: int = 0
+    external_events: int = 0
+    observed_events: int = 0  # changes seen in a store that keeps no history
+    unconfirmed: int = 0
+    skipped_rows: int = 0
+    reconcile_skipped: bool = False
+    warnings: list[str] = field(default_factory=list)
+    live: LiveMemories | None = None  # the listing the final pass looked at
+
+
+def _key(event: MemoryEvent) -> tuple[str, str | None, str | None]:
+    return (event.memory_id, event.before, event.after)
+
+
+ROLLBACK_ACTOR = "memdebug rollback"
+
+
+def _with_source(event: MemoryEvent, memory) -> MemoryEvent:
+    """A change that adds or alters a memory carries what the backend says about where that memory came from."""
+    if memory is None or memory.source is None or event.after is None or event.source is not None:
+        return event
+    return event.model_copy(update={"source": memory.source, "trust": derive_trust(memory.source)})
+
+
+def _observed(event: MemoryEvent) -> MemoryEvent:
+    """A change seen in a store that keeps no history: an ordinary ADD, UPDATE or DELETE, time-stamped when it was noticed."""
+    op = Op.DELETE if event.after is None else Op.ADD if event.before is None else Op.UPDATE
+    return event.model_copy(update={"op": op})
+
+
+def _acknowledged(event: MemoryEvent, written: dict[str, str | None]) -> MemoryEvent:
+    """An outside change that is exactly what a rollback just wrote becomes an ordinary change with a named actor."""
+    if event.memory_id not in written or written[event.memory_id] != event.after:
+        return event
+    op = Op.DELETE if event.after is None else Op.ADD if event.before is None else Op.UPDATE
+    return event.model_copy(update={"op": op, "source": Source(actor_id=ROLLBACK_ACTOR)})
+
+
+def sync(
+    adapter: MemoryAdapter,
+    ledger: Ledger,
+    scope: dict[str, str],
+    *,
+    now: datetime | None = None,
+    settle_seconds: float = 1.0,
+    sleep: Callable[[float], None] = time.sleep,
+    adopt_existing: bool = False,
+    max_history_rows: int = 100_000,
+    acknowledge: dict[str, str | None] | None = None,
+) -> SyncReport:
+    """adopt_existing: on the very first sync of a store, record memories that have no history
+    as observed ADDs instead of as changes made outside the API.
+
+    acknowledge: memory id -> the exact text (or None for removed) that this tool itself just wrote. A change that
+    matches is recorded as an ordinary ADD, UPDATE or DELETE by "memdebug rollback", not as a change made outside
+    the history, so a rollback does not raise a false alarm. Anything that differs is still reported."""
+    last_error: LedgerConflictError | None = None
+    for _ in range(3):  # another process may have written to the ledger meanwhile
+        try:
+            return _sync_once(
+                adapter, ledger, scope, now or datetime.now(timezone.utc),
+                settle_seconds, sleep, adopt_existing, max_history_rows, acknowledge or {},
+            )
+        except LedgerConflictError as exc:
+            last_error = exc
+    assert last_error is not None
+    raise last_error
+
+
+def _sync_once(adapter, ledger, scope, now, settle_seconds, sleep, adopt_existing, max_rows, acknowledge) -> SyncReport:
+    known_refs = ledger.known_refs(adapter.name)
+    had_events = ledger.has_events(adapter.name)
+
+    def observe() -> tuple[list[MemoryEvent], list[MemoryEvent], HistoryRead, LiveMemories]:
+        history = adapter.read_history(max_rows)  # history first, then the live listing
+        live = adapter.list_memories(scope)
+        scopes = {m.id: m.scope for m in live.memories}
+        new: list[MemoryEvent] = []
+        seen: set[str] = set()
+        for event in history.events:
+            ref = event.backend_ref
+            if ref is None or ref in known_refs or ref in seen:
+                continue
+            seen.add(ref)
+            if not event.scope and event.memory_id in scopes:
+                event = event.model_copy(update={"scope": scopes[event.memory_id]})
+            new.append(event)
+        if history.truncated:
+            candidates: list[MemoryEvent] = []  # a partial history cannot support any claim
+        else:
+            prior = [e.event for e in ledger.entries() if e.event.backend == adapter.name]
+            state = replay_events(prior + new)
+            candidates = find_external_changes(
+                state, live.memories, adapter.name, now, scope=scope, complete=live.complete
+            )
+        return new, candidates, history, live
+
+    report = SyncReport()
+    new, candidates, history, live = observe()
+    confirmed = candidates
+    if candidates:
+        sleep(settle_seconds)
+        new, second, history, live = observe()
+        wanted = {_key(c) for c in candidates}
+        confirmed = [c for c in second if _key(c) in wanted]
+        report.unconfirmed = len(candidates) - len(confirmed)
+        if report.unconfirmed:
+            report.warnings.append(
+                f"{report.unconfirmed} change(s) were not seen twice and were not recorded; "
+                "they will be checked again on the next sync"
+            )
+
+    if adopt_existing and not had_events:
+        confirmed = [
+            c.model_copy(update={"op": Op.ADD}) if c.before is None and c.after is not None else c
+            for c in confirmed
+        ]
+
+    if acknowledge:
+        confirmed = [_acknowledged(c, acknowledge) for c in confirmed]
+
+    observed_only = "history" not in adapter.capabilities
+    if observed_only:  # no history to bypass: a difference is simply a change seen between two looks
+        confirmed = [_observed(c) for c in confirmed]
+    by_id = {m.id: m for m in live.memories}
+    confirmed = [_with_source(c, by_id.get(c.memory_id)) for c in confirmed]
+
+    if history.truncated:
+        report.reconcile_skipped = True
+        report.warnings.append("history was only partly read, so changes outside the API were not checked")
+    else:
+        missing = known_refs - history.refs
+        if missing:
+            sample = ", ".join(safe_text(ref, 40) for ref in sorted(missing)[:3])
+            report.warnings.append(
+                f"{len(missing)} history row(s) recorded earlier are no longer in the backend "
+                f"history (removed or scrubbed); for example: {sample}"
+            )
+    report.warnings += history.warnings + live.warnings
+    report.skipped_rows = history.skipped
+    report.live = live
+
+    ledger.append_many(new + confirmed)
+    report.history_events = len(new)
+    if observed_only:
+        report.observed_events = len(confirmed)
+    else:
+        report.external_events = len(confirmed)
+    return report

memdebug/textsafe.py

@@ -0,0 +1,73 @@
+"""Helpers for handling text that came from a memory store.
+
+Memory text is untrusted. It may be planted by an attacker, for example through an email an
+agent read. Two rules follow:
+  * bound its size before storing it (bound_text), and
+  * never print it raw (safe_text), so it cannot move the cursor, recolour the terminal,
+    forge extra timeline lines or reorder what the reader sees.
+"""
+from __future__ import annotations
+
+import hashlib
+import unicodedata
+
+MAX_TEXT_CHARS = 100_000
+
+_NAMED = {"\n": "\\n", "\r": "\\r", "\t": "\\t"}
+_BIDI = set("\u200e\u200f\u061c\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
+_UNSAFE_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
+
+
+def bound_text(text: str, max_chars: int = MAX_TEXT_CHARS) -> str:
+    """Cap length. A cut text carries a hash of the full text, so two different long texts
+    that share a prefix still compare as different."""
+    if len(text) <= max_chars:
+        return text
+    digest = hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()[:16]
+    return f"{text[:max_chars]}...[cut: {len(text)} chars, sha256 {digest}]"
+
+
+def _escape(ch: str) -> str:
+    code = ord(ch)
+    if code < 0x100:
+        return f"\\x{code:02x}"
+    if code < 0x10000:
+        return f"\\u{code:04x}"
+    return f"\\U{code:08x}"
+
+
+def safe_text(value: object, limit: int | None = 200) -> str:
+    """Single-line, printable rendering of untrusted text. Control, format, bidi and
+    line-separator characters are shown as visible escapes."""
+    pieces: list[str] = []
+    length = 0
+    for ch in str(value):
+        if ch in _NAMED:
+            piece = _NAMED[ch]
+        elif ch in _BIDI or unicodedata.category(ch) in _UNSAFE_CATEGORIES:
+            piece = _escape(ch)
+        else:
+            piece = ch
+        if limit is not None and length + len(piece) > limit:
+            pieces.append("...")
+            break
+        pieces.append(piece)
+        length += len(piece)
+    return "".join(pieces)
+
+
+def has_unsafe_chars(text: str) -> bool:
+    """True if the text contains anything safe_text would have to escape."""
+    return any(ch in _BIDI or unicodedata.category(ch) in _UNSAFE_CATEGORIES for ch in text)
+
+
+def console_safe(text: str, encoding: str | None = None) -> str:
+    """Make text printable on a console whose encoding cannot show every character (a Windows
+    code page, for example). Unencodable characters become visible escapes instead of raising."""
+    import sys
+
+    target = encoding or getattr(sys.stdout, "encoding", None) or "utf-8"
+    try:
+        return text.encode(target, "backslashreplace").decode(target)
+    except LookupError:
+        return text.encode("ascii", "backslashreplace").decode("ascii")

memdebug/viewer/__init__.py

@@ -0,0 +1 @@
+"""A read-only local web viewer for the ledger. See server.py for the security model."""

memdebug/viewer/html.py

@@ -0,0 +1,136 @@
+"""Safe-by-construction HTML.
+
+Memory text is untrusted: an attacker can plant `<script>` or markup in a memory. Instead of
+remembering to escape at every use, pages are built only through `el()`:
+
+  * a plain string child is ALWAYS escaped (and control, bidi and line-separator characters are
+    made visible), so forgetting to escape is not possible;
+  * only `Markup` (created by this module) is inserted as-is;
+  * tags and attributes come from fixed allow-lists: no script, style, iframe, image, object or
+    event-handler attributes can be produced at all;
+  * `href` and `action` accept only internal paths that start with a single `/`.
+
+The pages carry no JavaScript and the server adds a Content-Security-Policy that forbids it, so even
+a bug here could not run code.
+"""
+from __future__ import annotations
+
+import html as _html
+import re
+import unicodedata
+from typing import Iterable
+
+from ..textsafe import _BIDI, _UNSAFE_CATEGORIES, safe_text
+
+__all__ = ["Markup", "el", "inline", "block", "raw_doctype"]
+
+
+class Markup(str):
+    """HTML that is already safe. Only functions in this module create it."""
+
+    __slots__ = ()
+
+
+_TAGS = frozenset({
+    "html", "head", "meta", "title", "link", "body", "header", "nav", "main", "aside", "section", "footer",
+    "h1", "h2", "h3", "p", "div", "span", "a", "table", "thead", "tbody", "tr", "th", "td", "caption",
+    "ul", "ol", "li", "pre", "code", "details", "summary", "form", "label", "select", "option", "button",
+    "input", "strong", "em", "br", "small", "time", "dl", "dt", "dd",
+})
+_VOID = frozenset({"meta", "link", "br", "input"})
+_ATTRS = frozenset({
+    "class", "href", "id", "lang", "charset", "name", "content", "rel", "type", "for", "value", "selected",
+    "checked", "scope", "colspan", "aria-label", "aria-current", "method", "action", "datetime", "title",
+    "role", "open", "disabled", "http-equiv", "media",
+})
+_URL_ATTRS = frozenset({"href", "action"})
+_INTERNAL_URL = re.compile(r"^/(?!/)[A-Za-z0-9_\-./?=&%:~+,]*\Z")
+
+
+def raw_doctype() -> Markup:
+    return Markup("<!doctype html>")
+
+
+def _attr_name(name: str) -> str:
+    return name.rstrip("_").replace("_", "-")
+
+
+def _render_attrs(attrs: dict) -> str:
+    parts: list[str] = []
+    for key, value in attrs.items():
+        name = _attr_name(key)
+        if name not in _ATTRS:
+            raise ValueError(f"attribute not allowed: {name!r}")
+        if value is None or value is False:
+            continue
+        if value is True:
+            parts.append(name)
+            continue
+        text = str(value)
+        if name in _URL_ATTRS and not _INTERNAL_URL.match(text):
+            raise ValueError(f"only internal paths are allowed in {name}: {text!r}")
+        parts.append(f'{name}="{_html.escape(safe_text(text, None), quote=True)}"')
+    return (" " + " ".join(parts)) if parts else ""
+
+
+def _render_children(children: Iterable) -> str:
+    out: list[str] = []
+    for child in children:
+        if child is None or child is False:
+            continue
+        if isinstance(child, Markup):
+            out.append(str(child))
+        elif isinstance(child, (list, tuple)):
+            out.append(_render_children(child))
+        elif isinstance(child, (int, float)) and not isinstance(child, bool):
+            out.append(_html.escape(str(child)))
+        else:  # any ordinary string is untrusted by default
+            out.append(_html.escape(safe_text(child, None), quote=True))
+    return "".join(out)
+
+
+def el(tag: str, *children, **attrs) -> Markup:
+    if tag not in _TAGS:
+        raise ValueError(f"tag not allowed: {tag!r}")
+    opening = f"<{tag}{_render_attrs(attrs)}>"
+    if tag in _VOID:
+        if children:
+            raise ValueError(f"<{tag}> cannot have children")
+        return Markup(opening)
+    return Markup(f"{opening}{_render_children(children)}</{tag}>")
+
+
+def inline(text: object, limit: int | None = 200) -> Markup:
+    """Untrusted text on one line, escaped and shortened."""
+    return Markup(_html.escape(safe_text(text, limit), quote=True))
+
+
+def block(text: object, limit: int = 20_000) -> Markup:
+    """Untrusted multi-line text for a <pre>: line breaks and tabs are kept, every other control,
+    format, bidi or line-separator character is shown as a visible escape."""
+    value = str(text if text is not None else "")
+    total = len(value)
+    shown = value[:limit]
+    pieces: list[str] = []
+    skip_next = False
+    for index, ch in enumerate(shown):
+        if skip_next:
+            skip_next = False
+            continue
+        if ch == "\r":
+            if shown[index + 1:index + 2] == "\n":
+                skip_next = True
+                pieces.append("\n")
+                continue
+            pieces.append("\\r")
+        elif ch in ("\n", "\t"):
+            pieces.append(ch)
+        elif ch in _BIDI or unicodedata.category(ch) in _UNSAFE_CATEGORIES:
+            code = ord(ch)
+            pieces.append(f"\\x{code:02x}" if code < 0x100 else f"\\u{code:04x}" if code < 0x10000 else f"\\U{code:08x}")
+        else:
+            pieces.append(ch)
+    out = _html.escape("".join(pieces), quote=True)
+    if total > limit:
+        out += _html.escape(f"\n... (shortened: showing the first {limit} of {total} characters)")
+    return Markup(out)