notmuch-browser 0.2.0rc1__py3-none-any.whl

notmuch_browser/__init__.py

@@ -0,0 +1,3 @@
+"""A small local web UI for notmuch."""
+
+__version__ = "0.1.0"

notmuch_browser/__main__.py

@@ -0,0 +1,67 @@
+"""Command line entry point: `notmuch-browser`."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import shutil
+import sys
+import webbrowser
+
+from aiohttp import web
+
+from . import __version__
+from .app import create_app, default_static_dir
+from .notmuch import Notmuch, NotmuchError
+
+HOST = "127.0.0.1"  # the server never binds anything else
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="notmuch-browser",
+        description="Browse and tag a notmuch database in a local web UI.",
+    )
+    parser.add_argument("--port", type=int, default=8765, help="port on 127.0.0.1 (default 8765)")
+    parser.add_argument(
+        "--notmuch-config",
+        metavar="FILE",
+        help="notmuch configuration file (default: the usual notmuch lookup)",
+    )
+    parser.add_argument("--no-browser", action="store_true", help="do not open a browser window")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    if shutil.which("notmuch") is None:
+        print("error: the notmuch command was not found on PATH", file=sys.stderr)
+        return 1
+    nm = Notmuch(config=args.notmuch_config)
+    try:
+        version = asyncio.run(nm.check_version())
+    except NotmuchError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    if not (default_static_dir() / "index.html").is_file():
+        print("warning: the UI bundle is missing, run `npm ci && npm run build`", file=sys.stderr)
+    url = f"http://{HOST}:{args.port}/"
+    app = create_app(nm)
+
+    async def announce(_app: web.Application) -> None:
+        print(f"notmuch {'.'.join(map(str, version))}, serving on {url} (Ctrl+C to stop)", flush=True)
+        if not args.no_browser:
+            webbrowser.open(url)
+
+    app.on_startup.append(announce)
+    try:
+        web.run_app(app, host=HOST, port=args.port, print=None)
+    except OSError as exc:
+        print(f"error: cannot listen on {HOST}:{args.port}: {exc.strerror}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

notmuch_browser/app.py

@@ -0,0 +1,320 @@
+"""The aiohttp application: a small REST API over notmuch plus the bundled UI."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from importlib import resources
+from pathlib import Path
+from urllib.parse import quote
+
+from aiohttp import web
+
+from . import render
+from .notmuch import (
+    BadQuery,
+    DatabaseBusy,
+    NotFound,
+    Notmuch,
+    NotmuchError,
+    message_query,
+    thread_query,
+)
+
+PAGE_SIZE = 50
+COUNTS_TTL = 30.0
+MAX_BATCH = 1000
+MAX_TAGS_PER_REQUEST = 20
+NM = web.AppKey("nm", Notmuch)
+STATIC = web.AppKey("static", Path)
+COUNTS = web.AppKey("counts", dict)
+
+APP_CSP = (
+    "default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' data:; "
+    "connect-src 'self'; frame-src 'self'; base-uri 'none'; form-action 'none'; "
+    "frame-ancestors 'self'"
+)
+ATTACHMENT_CSP = "default-src 'none'; sandbox"
+# Types a browser could execute if it ever rendered them. They are downloaded as opaque bytes.
+RISKY_TYPES = ("text/html", "application/xhtml+xml", "image/svg+xml", "text/xml", "application/xml")
+
+
+def default_static_dir() -> Path:
+    return Path(str(resources.files("notmuch_browser") / "static"))
+
+
+# ------------------------------------------------------------------ middleware
+
+
+def listening_port(request: web.Request) -> int | None:
+    transport = request.transport
+    sockname = transport.get_extra_info("sockname") if transport else None
+    return sockname[1] if sockname else None
+
+
+def forbidden(reason: str) -> web.Response:
+    return web.json_response({"error": reason, "kind": "forbidden"}, status=403)
+
+
+def is_inline_image(request: web.Request) -> bool:
+    """The one cross-site request the app serves itself.
+
+    HTML mail is shown in a sandboxed frame without allow-same-origin, so the browser
+    treats that frame as a foreign origin and labels its inline `cid:` image requests
+    cross-site. They are plain image loads of a message part, which a foreign page could
+    only use to learn that a part exists and how large the image is, and only if it
+    already knew the message id. Everything else cross-site stays refused.
+    """
+    return (
+        request.method == "GET"
+        and request.headers.get("Sec-Fetch-Dest") == "image"
+        and request.path.startswith("/api/message/")
+        and "/part/" in request.path
+    )
+
+
+@web.middleware
+async def security(request: web.Request, handler):
+    port = listening_port(request)
+    host = request.headers.get("Host", "")
+    if port is None or host not in (f"127.0.0.1:{port}", f"localhost:{port}"):
+        return forbidden("unexpected Host header")
+    if request.headers.get("Sec-Fetch-Site") == "cross-site" and not is_inline_image(request):
+        return forbidden("cross-site request")
+    if request.method not in ("GET", "HEAD"):
+        if request.headers.get("Origin") != f"http://{host}":
+            return forbidden("unexpected Origin header")
+        if request.content_type != "application/json":
+            return web.json_response(
+                {"error": "content type must be application/json", "kind": "bad_request"},
+                status=415,
+            )
+    try:
+        response = await handler(request)
+    except web.HTTPException:
+        raise
+    except BadQuery as exc:
+        return web.json_response({"error": str(exc), "kind": "bad_request"}, status=400)
+    except NotFound as exc:
+        return web.json_response({"error": str(exc), "kind": "not_found"}, status=404)
+    except DatabaseBusy as exc:
+        return web.json_response(
+            {"error": str(exc), "kind": "busy"}, status=503, headers={"Retry-After": "5"}
+        )
+    except NotmuchError as exc:
+        return web.json_response({"error": str(exc), "kind": "error"}, status=500)
+    response.headers.setdefault("X-Content-Type-Options", "nosniff")
+    response.headers.setdefault("Referrer-Policy", "no-referrer")
+    response.headers.setdefault("Content-Security-Policy", APP_CSP)
+    if request.path.startswith("/api/"):
+        response.headers.setdefault("Cache-Control", "no-store")
+    return response
+
+
+# --------------------------------------------------------------------- helpers
+
+
+def int_param(request: web.Request, name: str, default: int, low: int, high: int) -> int:
+    raw = request.query.get(name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        raise BadQuery(f"{name} must be an integer") from None
+    if not low <= value <= high:
+        raise BadQuery(f"{name} must be between {low} and {high}")
+    return value
+
+
+async def read_json(request: web.Request) -> dict:
+    try:
+        body = await request.json()
+    except ValueError:
+        raise BadQuery("request body is not valid JSON") from None
+    if not isinstance(body, dict):
+        raise BadQuery("request body must be a JSON object")
+    return body
+
+
+def tag_list(body: dict, key: str) -> list[str]:
+    value = body.get(key, [])
+    if not isinstance(value, list) or len(value) > MAX_TAGS_PER_REQUEST:
+        raise BadQuery(f"{key} must be a short list of tag names")
+    if not all(isinstance(t, str) for t in value):
+        raise BadQuery(f"{key} must contain strings")
+    return value
+
+
+def invalidate_counts(app: web.Application) -> None:
+    app[COUNTS].clear()
+
+
+# ------------------------------------------------------------------- handlers
+
+
+async def api_search(request: web.Request) -> web.Response:
+    nm = request.app[NM]
+    query = request.query.get("q", "").strip() or "*"
+    offset = int_param(request, "offset", 0, 0, 10_000_000)
+    limit = int_param(request, "limit", PAGE_SIZE, 1, PAGE_SIZE)
+    rows = await nm.search(query, offset, limit + 1)
+    keep = ("thread", "timestamp", "date_relative", "matched", "total", "authors", "subject", "tags")
+    threads = [{k: row.get(k) for k in keep} for row in rows[:limit]]
+    return web.json_response(
+        {"threads": threads, "offset": offset, "limit": limit, "has_more": len(rows) > limit}
+    )
+
+
+async def api_count(request: web.Request) -> web.Response:
+    query = request.query.get("q", "").strip() or "*"
+    unit = request.query.get("unit", "messages")
+    count = await request.app[NM].count(query, unit)
+    return web.json_response({"count": count, "unit": unit})
+
+
+async def api_tags(request: web.Request) -> web.Response:
+    return web.json_response({"tags": await request.app[NM].tags()})
+
+
+async def api_tag_counts(request: web.Request) -> web.Response:
+    cache = request.app[COUNTS]
+    async with cache.setdefault("lock", asyncio.Lock()):
+        fresh = cache.get("at") is not None and time.monotonic() - cache["at"] < COUNTS_TTL
+        if not fresh:
+            nm = request.app[NM]
+            counts = await nm.unread_counts(await nm.tags())
+            cache["counts"], cache["at"] = counts, time.monotonic()
+    return web.json_response({"counts": cache["counts"], "unit": "unread threads"})
+
+
+async def api_thread(request: web.Request) -> web.Response:
+    thread_id = request.match_info["id"]
+    nested = await request.app[NM].show_thread(thread_id)
+    return web.json_response(render.thread_view(thread_id, nested))
+
+
+def changed_payload(changed: dict) -> web.Response:
+    return web.json_response({"changed": changed})
+
+
+async def api_thread_tags(request: web.Request) -> web.Response:
+    body = await read_json(request)
+    changed = await request.app[NM].apply_tag_changes(
+        thread_query(request.match_info["id"]), tag_list(body, "add"), tag_list(body, "remove")
+    )
+    invalidate_counts(request.app)
+    return changed_payload(changed)
+
+
+async def api_message_tags(request: web.Request) -> web.Response:
+    body = await read_json(request)
+    changed = await request.app[NM].apply_tag_changes(
+        message_query(request.match_info["id"]), tag_list(body, "add"), tag_list(body, "remove")
+    )
+    invalidate_counts(request.app)
+    return changed_payload(changed)
+
+
+async def api_messages_tags(request: web.Request) -> web.Response:
+    body = await read_json(request)
+    changes = body.get("changes")
+    if not isinstance(changes, list) or not changes or len(changes) > MAX_BATCH:
+        raise BadQuery(f"changes must be a list of 1 to {MAX_BATCH} entries")
+    entries = []
+    for change in changes:
+        if not isinstance(change, dict) or not isinstance(change.get("id"), str):
+            raise BadQuery("each change needs a message id")
+        entries.append(
+            {"id": change["id"], "add": tag_list(change, "add"), "remove": tag_list(change, "remove")}
+        )
+    await request.app[NM].tag_batch(entries)
+    invalidate_counts(request.app)
+    return web.json_response({"applied": len(entries)})
+
+
+def find_part(message: dict, part: int) -> dict:
+    for candidate in render.iter_parts(message.get("body", [])):
+        if candidate.get("id") == part:
+            return candidate
+    raise NotFound("no such part")
+
+
+def attachment_filename(part: dict, number: int) -> str:
+    name = part.get("filename") or f"part-{number}"
+    return name.replace("\\", "_").replace("/", "_").replace("\r", " ").replace("\n", " ")
+
+
+async def api_part(request: web.Request) -> web.Response:
+    nm = request.app[NM]
+    message_id = request.match_info["id"]
+    try:
+        number = int(request.match_info["part"])
+    except ValueError:
+        raise BadQuery("invalid part number") from None
+    message = await nm.show_message(message_id)
+    part = find_part(message, number)
+    data = await nm.get_part(message_id, number)
+    ctype = part.get("content-type", "application/octet-stream").split(";")[0].strip().lower()
+    if ctype.startswith(RISKY_TYPES) or "/" not in ctype:
+        ctype = "application/octet-stream"
+    name = attachment_filename(part, number)
+    return web.Response(
+        body=data,
+        content_type=ctype,
+        headers={
+            "Content-Disposition": f"attachment; filename*=UTF-8''{quote(name, safe='')}",
+            "Content-Security-Policy": ATTACHMENT_CSP,
+        },
+    )
+
+
+async def api_html(request: web.Request) -> web.Response:
+    message = await request.app[NM].show_message(request.match_info["id"])
+    document = render.html_document_for(message)
+    if document is None:
+        raise NotFound("message has no HTML part")
+    origin = f"http://{request.headers['Host']}"
+    csp = (
+        f"default-src 'none'; img-src {origin}; style-src 'unsafe-inline'; "
+        "base-uri 'none'; form-action 'none'; frame-ancestors 'self'"
+    )
+    return web.Response(
+        text=document, content_type="text/html", headers={"Content-Security-Policy": csp}
+    )
+
+
+async def index(request: web.Request) -> web.StreamResponse:
+    page = request.app[STATIC] / "index.html"
+    if not page.is_file():
+        return web.Response(
+            status=503,
+            text="The UI bundle is missing. Run `npm ci && npm run build` and restart.",
+        )
+    return web.FileResponse(page, headers={"Cache-Control": "no-cache"})
+
+
+# ------------------------------------------------------------------------ app
+
+
+def create_app(nm: Notmuch, static_dir: Path | None = None) -> web.Application:
+    app = web.Application(middlewares=[security])
+    app[NM] = nm
+    app[STATIC] = static_dir or default_static_dir()
+    app[COUNTS] = {}
+    app.router.add_get("/api/search", api_search)
+    app.router.add_get("/api/count", api_count)
+    app.router.add_get("/api/tags", api_tags)
+    app.router.add_get("/api/tags/counts", api_tag_counts)
+    app.router.add_get("/api/thread/{id}", api_thread)
+    app.router.add_post("/api/thread/{id}/tags", api_thread_tags)
+    app.router.add_post("/api/message/{id}/tags", api_message_tags)
+    app.router.add_post("/api/messages/tags", api_messages_tags)
+    app.router.add_get(r"/api/message/{id}/part/{part}", api_part)
+    app.router.add_get("/api/message/{id}/html", api_html)
+    app.router.add_get("/", index)
+    assets = app[STATIC] / "assets"
+    if assets.is_dir():
+        app.router.add_static("/assets", assets, follow_symlinks=False)
+    return app
+

notmuch_browser/notmuch.py

@@ -0,0 +1,269 @@
+"""Async wrapper over the notmuch command line.
+
+Every call spawns `notmuch` with an argument list, never through a shell. User
+supplied queries always come after a `--` separator. Tag names and identifiers
+are validated before they reach an argument list or a batch line.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import re
+from dataclasses import dataclass, field
+
+MIN_VERSION = (0, 36)  # developed and tested against 0.40
+READ_TIMEOUT = 60.0
+# A running `notmuch new` or another writer holds the Xapian lock and notmuch
+# then waits for it instead of failing. A short write timeout turns that wait
+# into a clear "database busy" error.
+WRITE_TIMEOUT = 15.0
+
+TAG_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.@:/=-]{0,63}$")
+THREAD_RE = re.compile(r"^[0-9a-f]{1,32}$")
+_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]")
+
+
+class NotmuchError(Exception):
+    """A notmuch command failed."""
+
+
+class DatabaseBusy(NotmuchError):
+    """The notmuch database is locked by another process."""
+
+
+class BadQuery(NotmuchError):
+    """The query or an argument is not valid."""
+
+
+class NotFound(NotmuchError):
+    """The thread, message or part does not exist."""
+
+
+def validate_tag(tag: str) -> str:
+    if not isinstance(tag, str) or not TAG_RE.match(tag):
+        raise BadQuery(f"invalid tag name: {tag!r}")
+    return tag
+
+
+def thread_query(thread_id: str) -> str:
+    if not isinstance(thread_id, str) or not THREAD_RE.match(thread_id):
+        raise BadQuery("invalid thread id")
+    return f"thread:{thread_id}"
+
+
+def quote_term(value: str) -> str:
+    """Quote a value with the Xapian boolean term rules (double quotes doubled)."""
+    if not isinstance(value, str) or not value or _CONTROL_RE.search(value):
+        raise BadQuery("invalid identifier")
+    return '"' + value.replace('"', '""') + '"'
+
+
+def message_query(message_id: str) -> str:
+    return "id:" + quote_term(message_id)
+
+
+def tag_query(tag: str) -> str:
+    return "tag:" + quote_term(tag)
+
+
+@dataclass
+class Notmuch:
+    config: str | None = None
+    binary: str = "notmuch"
+    read_timeout: float = READ_TIMEOUT
+    write_timeout: float = WRITE_TIMEOUT
+    max_processes: int = 4
+    _sem: asyncio.Semaphore = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self._sem = asyncio.Semaphore(self.max_processes)
+
+    # ------------------------------------------------------------------ core
+
+    def _env(self) -> dict[str, str]:
+        env = dict(os.environ)
+        if self.config:
+            env["NOTMUCH_CONFIG"] = self.config
+        return env
+
+    async def _run(
+        self, *args: str, stdin: bytes | None = None, timeout: float | None = None
+    ) -> bytes:
+        timeout = self.read_timeout if timeout is None else timeout
+        async with self._sem:
+            proc = await asyncio.create_subprocess_exec(
+                self.binary,
+                *args,
+                stdin=asyncio.subprocess.PIPE if stdin is not None else asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                env=self._env(),
+            )
+            try:
+                out, err = await asyncio.wait_for(proc.communicate(stdin), timeout)
+            except asyncio.TimeoutError:
+                proc.kill()
+                await proc.wait()
+                raise DatabaseBusy(
+                    "notmuch did not answer in time; the database may be locked by "
+                    "another process such as `notmuch new`"
+                ) from None
+        if proc.returncode != 0:
+            message = err.decode("utf-8", "replace").strip() or f"exit status {proc.returncode}"
+            lowered = message.lower()
+            if "lock" in lowered:
+                raise DatabaseBusy(message)
+            if "parsing query" in lowered:
+                raise BadQuery(message)
+            raise NotmuchError(message)
+        return out
+
+    async def _json(self, *args: str, timeout: float | None = None):
+        out = await self._run(*args, timeout=timeout)
+        try:
+            return json.loads(out) if out.strip() else []
+        except json.JSONDecodeError as exc:
+            raise NotmuchError(f"unexpected notmuch output: {exc}") from exc
+
+    # ---------------------------------------------------------------- version
+
+    async def version(self) -> tuple[int, ...]:
+        out = (await self._run("--version")).decode().strip()
+        match = re.search(r"(\d+)\.(\d+)", out)
+        if not match:
+            raise NotmuchError(f"cannot read the notmuch version from {out!r}")
+        return int(match.group(1)), int(match.group(2))
+
+    async def check_version(self) -> tuple[int, ...]:
+        found = await self.version()
+        if found < MIN_VERSION:
+            need = ".".join(map(str, MIN_VERSION))
+            have = ".".join(map(str, found))
+            raise NotmuchError(f"notmuch {have} is too old, {need} or newer is required")
+        return found
+
+    # ------------------------------------------------------------------ reads
+
+    async def search(self, query: str, offset: int = 0, limit: int = 50) -> list[dict]:
+        return await self._json(
+            "search",
+            "--format=json",
+            "--output=summary",
+            "--sort=newest-first",
+            f"--offset={int(offset)}",
+            f"--limit={int(limit)}",
+            "--",
+            query,
+        )
+
+    async def count(self, query: str, unit: str = "messages") -> int:
+        if unit not in ("messages", "threads"):
+            raise BadQuery("unit must be messages or threads")
+        args = ["count"]
+        if unit == "threads":
+            args.append("--output=threads")
+        out = await self._run(*args, "--", query)
+        return int(out.strip() or 0)
+
+    async def tags(self) -> list[str]:
+        return await self._json("search", "--format=json", "--output=tags", "--", "*")
+
+    async def unread_counts(self, tags: list[str]) -> dict[str, int]:
+        async def one(tag: str) -> tuple[str, int]:
+            n = await self.count(f"{tag_query(tag)} and tag:unread", "threads")
+            return tag, n
+
+        return dict(await asyncio.gather(*(one(t) for t in tags)))
+
+    async def show_thread(self, thread_id: str) -> list:
+        """Return the nested notmuch JSON of a whole thread, HTML parts included."""
+        data = await self._json(
+            "show",
+            "--format=json",
+            "--entire-thread=true",
+            "--include-html",
+            "--",
+            thread_query(thread_id),
+        )
+        if not data:
+            raise NotFound("no such thread")
+        return data
+
+    async def show_message(self, message_id: str) -> dict:
+        data = await self._json(
+            "show",
+            "--format=json",
+            "--entire-thread=false",
+            "--include-html",
+            "--",
+            message_query(message_id),
+        )
+        for msg in iter_messages(data):
+            return msg
+        raise NotFound("no such message")
+
+    async def get_part(self, message_id: str, part: int) -> bytes:
+        if not isinstance(part, int) or part < 1:
+            raise BadQuery("invalid part number")
+        return await self._run(
+            "show", "--format=raw", f"--part={part}", "--", message_query(message_id)
+        )
+
+    # ----------------------------------------------------------------- writes
+
+    async def _message_ids(self, query: str) -> list[str]:
+        return await self._json("search", "--format=json", "--output=messages", "--", query)
+
+    async def apply_tag_changes(
+        self, scope: str, add: list[str], remove: list[str]
+    ) -> dict[str, dict[str, list[str]]]:
+        """Add and remove tags on every message matching `scope`.
+
+        Returns the ids of the messages that actually changed, per tag, so a
+        caller can undo exactly this change and nothing more.
+        """
+        add = [validate_tag(t) for t in dict.fromkeys(add)]
+        remove = [validate_tag(t) for t in dict.fromkeys(remove)]
+        if set(add) & set(remove):
+            raise BadQuery("a tag cannot be added and removed in the same request")
+        if not add and not remove:
+            raise BadQuery("nothing to do")
+        changed: dict[str, dict[str, list[str]]] = {"add": {}, "remove": {}}
+        for tag in add:
+            changed["add"][tag] = await self._message_ids(f"({scope}) and not {tag_query(tag)}")
+        for tag in remove:
+            changed["remove"][tag] = await self._message_ids(f"({scope}) and {tag_query(tag)}")
+        entries = []
+        for tag, ids in changed["add"].items():
+            entries += [{"id": i, "add": [tag], "remove": []} for i in ids]
+        for tag, ids in changed["remove"].items():
+            entries += [{"id": i, "add": [], "remove": [tag]} for i in ids]
+        await self.tag_batch(entries)
+        return changed
+
+    async def tag_batch(self, entries: list[dict]) -> None:
+        """Apply `{id, add, remove}` entries with a single `notmuch tag --batch`."""
+        lines = []
+        for entry in entries:
+            ops = [f"+{validate_tag(t)}" for t in entry.get("add", [])]
+            ops += [f"-{validate_tag(t)}" for t in entry.get("remove", [])]
+            if not ops:
+                continue
+            lines.append(" ".join(ops) + " -- " + message_query(entry["id"]))
+        if not lines:
+            return
+        await self._run(
+            "tag", "--batch", stdin=("\n".join(lines) + "\n").encode(), timeout=self.write_timeout
+        )
+
+
+def iter_messages(node):
+    """Yield every message dict in the nested `notmuch show` structure."""
+    if isinstance(node, dict):
+        if "id" in node and "headers" in node:
+            yield node
+    elif isinstance(node, list):
+        for child in node:
+            yield from iter_messages(child)