primarysource 0.1.0__py3-none-any.whl

primarysource/draft/catalogue.py

@@ -0,0 +1,117 @@
+"""The drafts the portfolio tracks: why each is tracked, the revision last read, and the pins. Metadata only.
+
+`tracked.json` is package data. For each draft it holds what the importer was last run against: the latest
+revision read, the datatracker state then, what replaced it, and the sha256 of the files fetched for each
+revision read. Those digests are the pins: an import of a revision that has one refuses a different file. A
+revision with no pin is trusted on first use and its digest recorded in the library's `meta.json`; add it here
+after checking it (`primarysource draft import --print-pin`).
+
+The snapshot says "as of `checked`". It is the catalogue's memory, not the datatracker's current word: `info`
+prefers the library's own `status.json` and falls back to this when the draft was never fetched.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+
+from primarysource.draft import names
+from primarysource.errors import LibError
+
+PLACEMENTS = ("layer", "volume", "none")
+
+
+@dataclass(frozen=True, slots=True)
+class Terms:
+    """What the IETF Trust's terms allow, and so where drafts may be placed (docs/draft.md)."""
+
+    licence: str
+    placement: str
+    terms_url: str
+    terms_read: str
+    summary: str
+
+
+@dataclass(frozen=True, slots=True)
+class Pin:
+    """sha256 of the files fetched for one revision; None for a file that was not fetched (no XML)."""
+
+    txt: str | None
+    xml: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class Tracked:
+    name: str
+    why: str
+    consumers: tuple[str, ...]
+    rev_read: str
+    state: str
+    replaced_by: tuple[str, ...]
+    became_rfc: str | None
+    expires: str
+    checked: str
+    pins: dict[str, Pin]
+
+
+def _digest(value: Any, where: str) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str) or len(value) != 64 or value.strip("0123456789abcdef"):
+        raise ValueError(f"{where}: a sha256 must be 64 lowercase hex digits or null")
+    return value
+
+
+def parse(raw: str) -> tuple[Terms, dict[str, Tracked]]:
+    """Validate the catalogue text. A malformed catalogue is a packaging bug, so it raises ValueError."""
+    data = json.loads(raw)
+    if data.get("schema") != 1:
+        raise ValueError("catalogue schema must be 1")
+    t = data["terms"]
+    if t["placement"] not in PLACEMENTS:
+        raise ValueError(f"placement {t['placement']!r} is not one of {PLACEMENTS}")
+    terms = Terms(t["licence"], t["placement"], t["terms_url"], t["terms_read"], t["summary"])
+    drafts: dict[str, Tracked] = {}
+    for entry in data["drafts"]:
+        name = names.parse_name(entry["name"])
+        if name in drafts:
+            raise ValueError(f"duplicate draft {name}")
+        pins: dict[str, Pin] = {}
+        for rev, files in entry["pins"].items():
+            names.valid_revision(rev)
+            pins[rev] = Pin(_digest(files.get("txt"), f"{name}-{rev}"), _digest(files.get("xml"), f"{name}-{rev}"))
+        if entry["rev_read"]:
+            names.valid_revision(entry["rev_read"])
+        drafts[name] = Tracked(
+            name,
+            entry["why"],
+            tuple(entry["consumers"]),
+            entry["rev_read"],
+            entry["state"],
+            tuple(entry["replaced_by"]),
+            entry["became_rfc"],
+            entry["expires"],
+            entry["checked"],
+            pins,
+        )
+    return terms, drafts
+
+
+TERMS, TRACKED = parse(resources.files("primarysource.draft").joinpath("tracked.json").read_text(encoding="utf-8"))
+
+
+def pin_for(name: str, rev: str) -> Pin | None:
+    """The pinned digests for one revision of a tracked draft, or None."""
+    tracked = TRACKED.get(name)
+    return tracked.pins.get(rev) if tracked else None
+
+
+def tracked(name: str) -> Tracked:
+    """The catalogue entry, or a LibError that lists what is tracked."""
+    found = TRACKED.get(names.parse_name(name))
+    if found is None:
+        known = ", ".join(sorted(TRACKED)) or "none"
+        raise LibError(f"{name} is not in the tracked catalogue (tracked: {known})", "")
+    return found

primarysource/draft/cli.py

@@ -0,0 +1,149 @@
+"""`primarysource draft ...`: the collection's own commands.
+
+primarysource draft import <name>[-rev] ...   fetch drafts from the IETF (one request a second); --tracked for all
+primarysource draft refresh [<name> ...]      fetch only the datatracker's current status; --tracked for all
+primarysource draft list                      tracked and stored drafts, with state and replacement
+primarysource draft info <name>               standing, revisions, expiry, replaced-by, licence legend
+primarysource draft outline <name>[-rev]      a stored draft's sections
+primarysource draft read <ref> [--max-tokens N]
+primarysource draft search <words> [--name N] [--limit N]
+primarysource draft cite <ref>
+
+`import` and `refresh` are the only commands that use the network. They refuse when `PRIMARYSOURCE_READONLY` is
+set. The others read the library and are what the MCP tools call.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Callable, Sequence
+
+from primarysource.draft import catalogue, importer, names, views
+from primarysource.draft.net import Fetcher
+from primarysource.draft.store import Store
+from primarysource.errors import LibError
+
+fetcher_factory: Callable[[], Fetcher] = Fetcher
+"""How the importer gets its fetcher; tests replace it with one over a fake transport."""
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="primarysource draft", description="IETF Internet-Drafts")
+    commands = parser.add_subparsers(dest="command", required=True)
+    imp = commands.add_parser("import", help="fetch drafts from the IETF into the library")
+    imp.add_argument("drafts", nargs="*", help="draft-<words>[-NN]; without -NN the latest revision")
+    imp.add_argument("--tracked", action="store_true", help="every draft in the tracked catalogue (latest revision)")
+    imp.add_argument("--no-xml", action="store_true", help="do not fetch the author's .xml")
+    imp.add_argument("--sha256", help="expected sha256 of the .txt (one draft with an explicit revision only)")
+    imp.add_argument("--print-pin", action="store_true", help="print each revision's digests as tracked.json pins")
+    ref = commands.add_parser("refresh", help="fetch the datatracker's current status of drafts")
+    ref.add_argument("drafts", nargs="*")
+    ref.add_argument("--tracked", action="store_true", help="every draft in the tracked catalogue")
+    commands.add_parser("list", help="tracked and stored drafts")
+    commands.add_parser("info", help="standing of a draft").add_argument("name")
+    commands.add_parser("outline", help="a stored draft's sections").add_argument("name")
+    read = commands.add_parser("read", help="read by ref")
+    read.add_argument("ref")
+    read.add_argument("--max-tokens", type=int, default=2000)
+    search = commands.add_parser("search", help="search the stored drafts")
+    search.add_argument("query")
+    search.add_argument("--name")
+    search.add_argument("--limit", type=int, default=10)
+    commands.add_parser("cite", help="the citation for a ref").add_argument("ref")
+    return parser
+
+
+def _emit(text: str) -> int:
+    out = sys.stdout
+    if hasattr(out, "reconfigure"):
+        out.reconfigure(encoding="utf-8")  # refs carry §, and a Windows console pipe is not UTF-8 by default
+    out.write(text if text.endswith("\n") else text + "\n")
+    return 0
+
+
+def _targets(args: argparse.Namespace) -> list[tuple[str, str | None]]:
+    found = [names.split_doc(d) for d in args.drafts]
+    if args.tracked:
+        found += [(n, None) for n in catalogue.TRACKED if n not in {f[0] for f in found}]
+    if not found:
+        raise LibError(
+            "name a draft, or pass --tracked", "e.g. primarysource draft import draft-ietf-sidrops-8210bis"
+        )
+    return found
+
+
+def _import(args: argparse.Namespace, store: Store) -> int:
+    targets = _targets(args)
+    if args.sha256 and (len(targets) != 1 or targets[0][1] is None):
+        raise LibError(
+            "--sha256 needs exactly one draft with an explicit revision", "e.g. draft-x-y-03 --sha256 <hex>"
+        )
+    fetcher = fetcher_factory()
+    failed = 0
+    pins: dict[str, dict[str, dict[str, str]]] = {}
+    for name, rev in targets:
+        try:
+            report = importer.import_draft(
+                name, rev, fetcher=fetcher, store=store, want_xml=not args.no_xml, expected_sha256=args.sha256
+            )
+        except LibError as exc:
+            failed += 1
+            print(f"{name}: {exc.message}" + (f" ({exc.hint})" if exc.hint else ""), file=sys.stderr)
+            continue
+        verb = "imported" if report.new else "already stored"
+        pin = "pinned" if report.pinned else "not pinned (trusted on first use)"
+        print(
+            f"{report.name}-{report.rev}: {verb}, text {report.txt_bytes:,} bytes sha256 {report.txt_sha256}, "
+            f"xml {report.xml_state}, state {report.status.state}, {pin}, {report.requests} requests -> {report.path}"
+        )
+        if report.xml_sha256 or report.txt_sha256:
+            pins.setdefault(report.name, {})[report.rev] = {
+                "txt": report.txt_sha256,
+                **({"xml": report.xml_sha256} if report.xml_sha256 else {}),
+            }
+    if args.print_pin and pins:
+        print(json.dumps(pins, indent=1))
+    return 1 if failed else 0
+
+
+def _refresh(args: argparse.Namespace, store: Store) -> int:
+    fetcher = fetcher_factory()
+    failed = 0
+    for name, _rev in _targets(args):
+        try:
+            status = importer.refresh(name, fetcher=fetcher, store=store)
+        except LibError as exc:
+            failed += 1
+            print(f"{name}: {exc.message}" + (f" ({exc.hint})" if exc.hint else ""), file=sys.stderr)
+            continue
+        extra = (f", replaced by {', '.join(status.replaced_by)}" if status.replaced_by else "") + (
+            f", became {status.became_rfc}" if status.became_rfc else ""
+        )
+        print(f"{name}: latest -{status.rev}, state {status.state}, expires {status.expires[:10] or '?'}{extra}")
+    return 1 if failed else 0
+
+
+def main(argv: Sequence[str]) -> int:
+    args = _parser().parse_args(list(argv))
+    store = Store()
+    try:
+        if args.command == "import":
+            return _import(args, store)
+        if args.command == "refresh":
+            return _refresh(args, store)
+        if args.command == "list":
+            return _emit(views.list_tracked(store))
+        if args.command == "info":
+            return _emit(views.info(store, args.name))
+        if args.command == "outline":
+            return _emit(views.outline(store, args.name))
+        if args.command == "read":
+            return _emit(views.read(store, args.ref, max_tokens=args.max_tokens))
+        if args.command == "search":
+            return _emit(views.search(store, args.query, limit=args.limit, name=args.name))
+        return _emit(views.cite(store, args.ref))
+    except LibError as exc:
+        print(exc.as_text(), file=sys.stderr)
+        return 1

primarysource/draft/collection.py

@@ -0,0 +1,21 @@
+"""IETF Internet-Drafts as a primarysource collection. Importing this needs nothing beyond the core."""
+
+from __future__ import annotations
+
+from primarysource.draft.names import GRAMMAR
+from primarysource.model import Unit
+from primarysource.registry import Collection
+
+COLLECTION = Collection(
+    name="draft",
+    title="IETF Internet-Drafts (from the datatracker and the IETF archive)",
+    prefix="draft_",
+    unit=Unit.PARAGRAPH,
+    grammar=GRAMMAR,
+    server="primarysource.draft.server:server",
+    cli="primarysource.draft.cli:main",
+)
+"""Refs: `draft-ietf-sidrops-8210bis-17§5.1¶2`. The `-17` is the revision; without it a ref resolves to the latest
+stored revision and the result says so. `§` names a section as the draft prints its number (`5.1`, `A.2` for
+Appendix A.2) or, for an unnumbered section, a title slug (`abstract`, `security-considerations`). `¶n` counts the
+paragraph blocks of that section (paragraphs, lists, figures, tables), as the rfc collection does."""

primarysource/draft/datatracker.py

@@ -0,0 +1,158 @@
+"""Draft metadata from the IETF datatracker: three small JSON documents per draft, validated before use.
+
+- `/doc/<name>/doc.json`, the datatracker's simplified document API (no key, no parameters): title, state,
+  latest revision, expiry, authors and the whole revision history with posting dates;
+- `/api/v1/doc/relateddocument/` filtered by relationship, twice: which draft *replaces* this one, and which RFC
+  this draft *became* (the `replaces` and `became_rfc` relationship names).
+
+Everything the server sends is data, never trusted: every field is type-checked and length-capped, a name
+extracted from a resource URI must match the draft or RFC name grammar, and a response for a different draft
+than the one asked about is refused.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import replace
+from datetime import UTC, datetime
+from typing import Any
+from urllib.parse import quote
+
+from primarysource.draft import names
+from primarysource.draft.net import ARCHIVE, DATATRACKER, Fetcher, Response
+from primarysource.draft.status import Status
+from primarysource.errors import LibError
+
+_DRAFT_URI = re.compile(r"/api/v1/doc/document/(draft-[a-z0-9]+(?:-[a-z0-9]+)*)/")
+_RFC_URI = re.compile(r"/api/v1/doc/document/rfc([0-9]{1,5})/")
+_MAX_TEXT = 500
+_MAX_REVISIONS = 100
+_MAX_AUTHORS = 50
+_MAX_LINKS = 20
+
+
+def doc_url(name: str) -> str:
+    return f"https://{DATATRACKER}/doc/{quote(names.parse_name(name), safe='')}/doc.json"
+
+
+def replaced_by_url(name: str) -> str:
+    return (
+        f"https://{DATATRACKER}/api/v1/doc/relateddocument/?target__name={quote(names.parse_name(name), safe='')}"
+        "&relationship__slug=replaces&format=json"
+    )
+
+
+def became_rfc_url(name: str) -> str:
+    return (
+        f"https://{DATATRACKER}/api/v1/doc/relateddocument/?source__name={quote(names.parse_name(name), safe='')}"
+        "&relationship__slug=became_rfc&format=json"
+    )
+
+
+def archive_url(name: str, rev: str, suffix: str) -> str:
+    """The IETF archive's copy of one revision: `suffix` is `txt` or `xml`."""
+    if suffix not in ("txt", "xml"):
+        raise ValueError(suffix)
+    return f"https://{ARCHIVE}/archive/id/{names.with_revision(name, rev)}.{suffix}"
+
+
+def html_url(name: str, rev: str) -> str:
+    """The datatracker's rendering of one revision, which the cite string points at."""
+    return f"https://{DATATRACKER}/doc/html/{names.with_revision(name, rev)}"
+
+
+def _json(reply: Response, what: str) -> dict[str, Any]:
+    try:
+        data = json.loads(reply.body.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError) as exc:
+        raise LibError(f"the datatracker's answer for {what} is not JSON", "try again later") from exc
+    if not isinstance(data, dict):
+        raise LibError(f"the datatracker's answer for {what} is not a JSON object", "")
+    return data
+
+
+def _text(data: dict[str, Any], key: str, required: bool = False) -> str:
+    value = data.get(key)
+    if value is None and not required:
+        return ""
+    if not isinstance(value, str):
+        raise LibError(f"the datatracker's {key!r} is not text", "")
+    return " ".join(value.split())[:_MAX_TEXT]
+
+
+def parse_doc(reply: Response, name: str, checked: datetime) -> Status:
+    """`doc.json` -> a `Status` without the relations (`replaced_by`, `became_rfc`), which come separately."""
+    data = _json(reply, name)
+    if data.get("name") != name:
+        raise LibError(f"asked the datatracker for {name} and it answered for {str(data.get('name'))[:60]!r}", "")
+    rev = _text(data, "rev", required=True)
+    names.valid_revision(rev)
+    history: list[tuple[str, str]] = []
+    raw_history = data.get("rev_history")
+    if not isinstance(raw_history, list):
+        raise LibError("the datatracker's revision history is not a list", "")
+    for item in raw_history[:_MAX_REVISIONS]:
+        if isinstance(item, dict) and isinstance(item.get("rev"), str) and re.fullmatch(r"[0-9]{2}", item["rev"]):
+            history.append((item["rev"], _text(item, "published")))
+    history.sort()
+    authors: list[str] = []
+    for person in data.get("authors") or []:
+        if isinstance(person, dict) and isinstance(person.get("name"), str) and person["name"].strip():
+            authors.append(" ".join(person["name"].split())[:100])
+    group = data.get("group")
+    pages = data.get("pages")
+    return Status(
+        name=name,
+        title=_text(data, "title", required=True),
+        state=_text(data, "state", required=True),
+        rev=rev,
+        expires=_text(data, "expires"),
+        replaced_by=(),
+        became_rfc=None,
+        revisions=tuple(history),
+        authors=tuple(authors[:_MAX_AUTHORS]),
+        group=_text(group, "acronym") if isinstance(group, dict) else "",
+        stream=_text(data, "stream"),
+        pages=pages if isinstance(pages, int) and not isinstance(pages, bool) else None,
+        checked=checked.replace(microsecond=0).isoformat(),
+    )
+
+
+def _related(reply: Response, field: str, pattern: re.Pattern[str]) -> list[str]:
+    data = _json(reply, "a relationship query")
+    objects = data.get("objects")
+    if not isinstance(objects, list):
+        raise LibError("the datatracker's relationship answer has no object list", "")
+    found: list[str] = []
+    for item in objects[:_MAX_LINKS]:
+        match = (
+            pattern.fullmatch(item.get(field, ""))
+            if isinstance(item, dict) and isinstance(item.get(field), str)
+            else None
+        )
+        if match:
+            found.append(match.group(1))
+    return found
+
+
+def fetch_status(fetcher: Fetcher, name: str, now: datetime | None = None) -> Status:
+    """Three requests: the document, who replaces it, and whether it became an RFC. A 404 on the document means
+    the datatracker has never heard of the name."""
+    name = names.parse_name(name)
+    checked = now or datetime.now(UTC)
+    reply = fetcher.get(doc_url(name))
+    if reply.status == 404:
+        raise LibError(
+            f"the datatracker has no draft named {name}", "check the spelling, or search datatracker.ietf.org"
+        )
+    if reply.status != 200:
+        raise LibError(f"the datatracker answered {reply.status} for {name}", "try again later")
+    base = parse_doc(reply, name, checked)
+    replaced = fetcher.get(replaced_by_url(name))
+    became = fetcher.get(became_rfc_url(name))
+    if replaced.status != 200 or became.status != 200:
+        raise LibError(f"the datatracker's relationship queries for {name} failed", "try again later")
+    by = tuple(sorted(_related(replaced, "source", _DRAFT_URI)))
+    rfcs = _related(became, "target", _RFC_URI)
+    return replace(base, replaced_by=by, became_rfc=f"RFC{int(rfcs[0])}" if rfcs else None)

primarysource/draft/importer.py

@@ -0,0 +1,163 @@
+"""Import named drafts from the IETF into the library: `primarysource draft import <name>[-rev]`.
+
+For one draft the importer makes at most five requests, one second apart (net.Fetcher), in this order:
+
+1. `doc.json`, the replaced-by query and the became-RFC query (datatracker.fetch_status): the draft's state,
+   latest revision, expiry and revision history. The snapshot is written first, so a draft found to be dead is
+   recorded as dead even if the text fetch then fails;
+2. the revision's `.txt` from the IETF archive: required, because it is what is segmented and cited;
+3. the revision's `.xml`, the author's source: optional (many drafts and every pre-xml2rfc draft have none), and
+   skipped with `want_xml=False`.
+
+Each file is verified before anything is written. A digest pinned in the catalogue (or given as
+`expected_sha256`) must match; a file with no pin is trusted on first use and its digest recorded in the
+revision's `meta.json`. A text that is not text (NUL bytes, absurd lines, an HTML error page behind a 200) is
+refused. A revision already stored is not fetched again: the archive's revisions do not change.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from primarysource.draft import catalogue, datatracker, licence, names, segment, xmlmeta
+from primarysource.draft.net import Fetcher, Response
+from primarysource.draft.status import Status
+from primarysource.draft.store import RevMeta, Store, readonly
+from primarysource.errors import LibError
+
+
+@dataclass(frozen=True, slots=True)
+class Report:
+    """What one import did. `new` is False when the revision was already stored (only the status was refreshed)."""
+
+    name: str
+    rev: str
+    path: Path
+    new: bool
+    txt_bytes: int
+    txt_sha256: str
+    xml_state: str
+    xml_sha256: str
+    pinned: bool
+    status: Status
+    requests: int
+
+
+def sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _body(reply: Response, url: str, kind: str) -> bytes | None:
+    """The body of a 200 for the right kind of file. None for a 404, and for an optional file (`xml`) that came
+    back as something else; a LibError for a `txt` that did, because the archive answers a missing file with an
+    HTML page and that is not the draft."""
+    if reply.status == 404:
+        return None
+    if reply.status != 200:
+        raise LibError(f"the IETF archive answered {reply.status} for {url}", "try again later")
+    kind_ok = "text/plain" if kind == "txt" else "xml"
+    if kind_ok not in reply.content_type.lower() or "html" in reply.content_type.lower() or not reply.body.strip():
+        if kind == "xml":
+            return None
+        raise LibError(
+            f"{url} came back as {reply.content_type or 'no content type'!r} or empty, not a text draft", ""
+        )
+    return reply.body
+
+
+def _check(got: str, pinned: str | None, expected: str | None, what: str) -> bool:
+    """Compare a digest with the explicit one (it wins) or the catalogue's; True if something pinned it."""
+    want = expected or pinned
+    if want is not None and got != want:
+        raise LibError(
+            f"{what} has sha256 {got}, not the {want} pinned for it: the archive's copy changed or the download "
+            "is damaged, and nothing was stored",
+            "try again; if it persists, the pin in tracked.json or --sha256 is what to re-check",
+        )
+    return want is not None
+
+
+def refresh(name: str, *, fetcher: Fetcher, store: Store, now: datetime | None = None) -> Status:
+    """Fetch and store the datatracker's current word on `name` (three requests). Needs no stored revision."""
+    if readonly():
+        raise LibError("the library is read-only (PRIMARYSOURCE_READONLY): nothing is fetched or written", "")
+    status = datatracker.fetch_status(fetcher, names.parse_name(name), now)
+    store.write_status(status)
+    return status
+
+
+def import_draft(
+    name: str,
+    rev: str | None = None,
+    *,
+    fetcher: Fetcher,
+    store: Store,
+    want_xml: bool = True,
+    expected_sha256: str | None = None,
+    now: datetime | None = None,
+) -> Report:
+    """Import one revision of `name`: the latest the datatracker knows when `rev` is None."""
+    name = names.parse_name(name)
+    if rev is not None:
+        names.valid_revision(rev)
+    started = fetcher.requests
+    status = refresh(name, fetcher=fetcher, store=store, now=now)
+    known = [r for r, _when in status.revisions]
+    rev = rev or status.rev
+    if known and rev not in known:
+        raise LibError(f"{name} has no revision {rev} (the datatracker lists: {', '.join(known)})", "")
+    if store.has(name, rev):
+        meta = store.meta(name, rev)
+        return Report(
+            name, rev, store.path(name, rev), False, meta.txt_bytes, meta.txt_sha256, meta.xml_state, meta.xml_sha256,
+            catalogue.pin_for(name, rev) is not None, status, fetcher.requests - started,
+        )  # fmt: skip
+    pin = catalogue.pin_for(name, rev)
+    txt_url = datatracker.archive_url(name, rev, "txt")
+    txt = _body(fetcher.get(txt_url), txt_url, "txt")
+    if txt is None:
+        raise LibError(
+            f"the IETF archive has no text for {name}-{rev}",
+            "a draft submitted as XML only is not supported yet (docs/draft.md)",
+        )
+    pinned = _check(sha256(txt), pin.txt if pin else None, expected_sha256, f"{name}-{rev}.txt")
+    text, encoding = segment.decode(txt)
+    found = segment.segment(text)
+    xml: bytes | None = None
+    info = xmlmeta.XmlInfo("skipped")
+    if want_xml:
+        xml_url = datatracker.archive_url(name, rev, "xml")
+        xml = _body(fetcher.get(xml_url), xml_url, "xml")
+        if xml is None:
+            info = xmlmeta.XmlInfo("absent")
+        else:
+            _check(sha256(xml), pin.xml if pin else None, None, f"{name}-{rev}.xml")
+            info = xmlmeta.inspect(xml)
+            if info.state == xmlmeta.REFUSED:
+                xml = None  # hostile or unusual source: recorded as refused, not stored
+    legend = licence.detect(text, info.ipr)
+    meta = RevMeta(
+        name=name,
+        rev=rev,
+        title=status.title,
+        published=status.published(rev),
+        authors=status.authors,
+        txt_sha256=sha256(txt),
+        txt_bytes=len(txt),
+        encoding=encoding,
+        xml_state=info.state,
+        xml_sha256=sha256(xml) if xml is not None else "",
+        xml_bytes=len(xml) if xml is not None else 0,
+        restriction=legend.restriction,
+        copyright_year=legend.year,
+        ipr=legend.ipr,
+        imported=(now or datetime.now(UTC)).replace(microsecond=0).isoformat(),
+    )
+    path = store.install(meta, txt, xml, found)
+    return Report(
+        name, rev, path, True, len(txt), meta.txt_sha256, info.state, meta.xml_sha256, pinned, status,
+        fetcher.requests - started,
+    )  # fmt: skip