curu-memory 0.2.0__py3-none-any.whl

curu_memory/okf.py

@@ -0,0 +1,271 @@
+"""Parse and validate one OKF v0.2 concept file, with the rules of the curu index.
+
+The rules are the rules of curu (research R-3), so that each file that curu accepts
+also indexes here. Only the `description` field is required. A link is a Markdown
+link to a sibling `.md` file, with a title that is one of the eight link kinds.
+"""
+
+from __future__ import annotations
+
+import datetime
+import hashlib
+import re
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+from typing import Any
+
+import yaml
+
+from curu_memory.ports import GLOBAL, Concept
+
+#: The closed vocabulary of link kinds, the same as in curu. Each kind says something
+#: specific, so there is no general "related" kind.
+LINK_KINDS: tuple[str, ...] = (
+    "USES_COMPONENT",
+    "HAS_TURBO_VARIANT",
+    "SAME_LINEAGE_AS",
+    "SHARES_CLIP_TYPE_LABEL_WITH",
+    "EVIDENCED_BY",
+    "SHARES_GUARD_CONVENTION_WITH",
+    "DISTINCT_FAILURE_CLASS_FROM",
+    "RETRACTS_CLAIM_IN",
+)
+
+UNTYPED = "Untyped"
+
+_MARKDOWN_LINK = re.compile(r'\[[^\]]+\]\(\s*(?P<target>[^\s)]+)(?:\s+"(?P<kind>[^"]*)")?\s*\)')
+_WIKI_LINK = re.compile(r"\[\[(?P<target>[^\[\]\n]+)\]\]")
+_FENCE = re.compile(r"^ {0,3}(```+|~~~+)")
+_INLINE_CODE = re.compile(r"(`+)(?:(?!\1).)+?\1")
+_HEADING = re.compile(r"^## +(?P<title>.+?)\s*#*\s*$")
+
+
+class ConceptFault(ValueError):
+    """A concept file that is not valid OKF. The index command skips it and reports it."""
+
+    def __init__(self, path: str, reason: str, line: int | None = None) -> None:
+        where = f"{path}:{line}" if line is not None else path
+        super().__init__(f"{where}: {reason}")
+        self.path = path
+        self.reason = reason
+        self.line = line
+
+
+@dataclass(frozen=True)
+class ParsedConcept:
+    concept: Concept
+    #: Each (target identifier, kind), in the order of the body. The first link wins.
+    links: tuple[tuple[str, str], ...]
+    #: Problems that do not stop the index, for example a missing type.
+    warnings: tuple[str, ...]
+    #: The `media:` paths, relative to the concept file (ADR-007).
+    media: tuple[str, ...] = ()
+
+
+def concept_id(relative_path: str) -> str:
+    return PurePosixPath(relative_path).stem
+
+
+def parse_concept(relative_path: str, text: str, tier_space: str = GLOBAL) -> ParsedConcept:
+    """Parse the text of one concept file. Raise `ConceptFault` when it is not valid."""
+    path = PurePosixPath(relative_path).as_posix()
+    front, body, body_line = _split_front_matter(path, text)
+    identifier = concept_id(path)
+
+    description = front.get("description")
+    if not description or not str(description).strip():
+        raise ConceptFault(path, "the front matter has no 'description', which is required")
+
+    warnings: list[str] = []
+    kind = front.get("type")
+    if not kind or not str(kind).strip():
+        warnings.append(f"the front matter has no 'type', so the type is {UNTYPED}")
+        kind = UNTYPED
+    title = front.get("title")
+    status = front.get("status")
+
+    concept = Concept(
+        id=identifier,
+        tier_space=tier_space,
+        path=path,
+        type=str(kind).strip(),
+        title=str(title).strip() if title else identifier,
+        description=str(description).strip(),
+        tags=_tags(front.get("tags")),
+        status=str(status) if status is not None else None,
+        sources=_sources(front.get("sources")),
+        body=body,
+        content_sha256=hashlib.sha256(text.encode()).hexdigest(),
+    )
+    media = front.get("media")
+    if media is None:
+        media = []
+    elif not isinstance(media, list) or not all(isinstance(item, str) for item in media):
+        warnings.append("the front-matter key 'media' is not a list of paths, so it is ignored")
+        media = []
+    return ParsedConcept(
+        concept=concept,
+        links=_links(path, body, body_line),
+        warnings=tuple(warnings),
+        media=tuple(media),
+    )
+
+
+#: The slug of the unit of the title and the description.
+SUMMARY = "summary"
+
+
+def split_units(body: str) -> list[tuple[str, str]]:
+    """Split a body at its `##` headings. Give each (slug, text) in order (research R-4).
+
+    The text before the first heading is the unit `overview`. A heading in a fenced
+    code block does not split. A repeated slug gets a number: `notes`, `notes-2`.
+    A slug is never the slug of another unit of the concept. The index uses `summary`
+    for the title and the description, so a `## Summary` heading is `summary-2`.
+    """
+    sections: list[tuple[str | None, list[str]]] = [(None, [])]
+    fence: str | None = None
+    for line in body.splitlines():
+        opened = _FENCE.match(line)
+        if fence is None and opened:
+            fence = opened.group(1)
+        elif fence is not None and line.strip().startswith(fence[0] * len(fence)):
+            fence = None
+        elif fence is None and (heading := _HEADING.match(line)):
+            sections.append((heading.group("title"), []))
+        sections[-1][1].append(line)
+
+    units: list[tuple[str, str]] = []
+    used = {SUMMARY}
+    for title, lines in sections:
+        text = "\n".join(lines).strip()
+        if not text:
+            continue
+        base = "overview" if title is None else _slug(title)
+        slug, number = base, 1
+        while slug in used:
+            number += 1
+            slug = f"{base}-{number}"
+        used.add(slug)
+        units.append((slug, text))
+    return units
+
+
+def _split_front_matter(path: str, text: str) -> tuple[dict[str, Any], str, int]:
+    """Give the front matter, the body and the file line where the body starts."""
+    lines = text.splitlines(keepends=True)
+    if not lines or lines[0].rstrip("\r\n") != "---":
+        raise ConceptFault(path, "the file has no front matter between two '---' lines")
+    for index in range(1, len(lines)):
+        if lines[index].rstrip("\r\n") == "---":
+            break
+    else:
+        raise ConceptFault(path, "the front matter has no closing '---' line")
+    source = "".join(lines[1:index])
+    try:
+        front = yaml.safe_load(source)
+    except yaml.YAMLError as error:
+        mark = getattr(error, "problem_mark", None)
+        line = mark.line + 2 if mark is not None else None
+        problem = getattr(error, "problem", None) or "it cannot be read"
+        raise ConceptFault(path, f"the front matter is not valid YAML: {problem}", line) from None
+    if front is None:
+        front = {}
+    if not isinstance(front, dict):
+        raise ConceptFault(path, "the front matter must be a YAML mapping of fields")
+    return front, "".join(lines[index + 1 :]), index + 2
+
+
+def _links(path: str, body: str, body_line: int) -> tuple[tuple[str, str], ...]:
+    unread = _WIKI_LINK.search(_mask_code(body))
+    if unread is not None:
+        target = unread.group("target").strip()
+        raise ConceptFault(
+            path,
+            f"the body has the wiki-link [[{target}]], which is not a Markdown link. "
+            f'Write [text](./{target}.md "KIND") with one of: {", ".join(LINK_KINDS)}',
+            _line_of(body, unread.start(), body_line),
+        )
+    links: list[tuple[str, str]] = []
+    seen: set[str] = set()
+    for match in _MARKDOWN_LINK.finditer(body):
+        target = match.group("target").strip()
+        if target.startswith(("http://", "https://")) or not target.endswith(".md"):
+            continue
+        target_id = PurePosixPath(target).stem
+        kind = match.group("kind")
+        line = _line_of(body, match.start(), body_line)
+        if kind is None:
+            raise ConceptFault(
+                path,
+                f'the link to {target!r} has no kind. Write [text]({target} "KIND") '
+                f"with one of: {', '.join(LINK_KINDS)}",
+                line,
+            )
+        if kind not in LINK_KINDS:
+            raise ConceptFault(
+                path,
+                f"the link to {target!r} has the unknown kind {kind!r}. "
+                f"Use one of: {', '.join(LINK_KINDS)}",
+                line,
+            )
+        if target_id in seen:
+            continue
+        seen.add(target_id)
+        links.append((target_id, kind))
+    return tuple(links)
+
+
+def _mask_code(body: str) -> str:
+    """Blank fenced code blocks and inline code spans, but keep the line breaks."""
+    masked: list[str] = []
+    fence: str | None = None
+    for line in body.splitlines(keepends=True):
+        opened = _FENCE.match(line)
+        if fence is None and opened:
+            fence = opened.group(1)
+            masked.append(_blank(line))
+        elif fence is not None:
+            if line.strip().startswith(fence[0] * len(fence)):
+                fence = None
+            masked.append(_blank(line))
+        else:
+            masked.append(_INLINE_CODE.sub(lambda m: _blank(m.group(0)), line))
+    return "".join(masked)
+
+
+def _blank(text: str) -> str:
+    return "".join(c if c in "\r\n" else " " for c in text)
+
+
+def _line_of(body: str, offset: int, body_line: int) -> int:
+    return body_line + body.count("\n", 0, offset)
+
+
+def _slug(title: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") or "section"
+
+
+def _tags(value: Any) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if isinstance(value, list | tuple):
+        return tuple(str(tag) for tag in value)
+    return (str(value),)
+
+
+def _sources(value: Any) -> tuple[dict[str, Any], ...]:
+    if not isinstance(value, list):
+        return ()
+    return tuple(_plain(item) for item in value if isinstance(item, dict))
+
+
+def _plain(value: Any) -> Any:
+    """Make YAML values plain JSON values: a date becomes ISO text."""
+    if isinstance(value, dict):
+        return {str(k): _plain(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_plain(v) for v in value]
+    if isinstance(value, datetime.date | datetime.datetime):
+        return value.isoformat()
+    return value

curu_memory/ports.py

@@ -0,0 +1,344 @@
+"""The records of the index and the two ports: `IndexStore` and `Embedder`.
+
+The application depends on these protocols, not on SQLite or HTTP (constitution,
+article VII). Each read method of `IndexStore` takes the tier spaces that the caller
+can read, so the filter is part of each query (ADR-001).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from contextlib import AbstractContextManager
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+from curu_memory.media import MediaFile
+
+GLOBAL = "global"
+#: A space name: `global`, `team/<slug>` or `personal/<handle>` (ADR-016).
+_SPACE = re.compile(r"(?:team|personal)/[a-z][a-z0-9-]{0,39}")
+#: Version 2 adds the kind of a unit, and the path and SHA-256 of an image (ADR-007).
+#: Version 3 adds the concept media and the media bytes (ADR-009).
+SCHEMA_VERSION = 3
+TEXT = "text"
+IMAGE = "image"
+AUDIO = "audio"
+VIDEO = "video"
+
+
+def parse_space(text: str) -> str:
+    """The space name in `text`, or `ValueError` with the reason (ADR-016)."""
+    if text == GLOBAL or _SPACE.fullmatch(text):
+        return text
+    raise ValueError(
+        f"the space {text!r} is not valid. Give global, team/<slug> or personal/<handle>. "
+        "A slug or a handle has 1 to 40 characters from a to z, 0 to 9 and -, first a letter."
+    )
+
+
+def space_file_name(space: str) -> str:
+    """The index file name of a space. `/` becomes `--`, so `team/look` is `team--look.sqlite`."""
+    return parse_space(space).replace("/", "--") + ".sqlite"
+
+
+def space_of_file_name(name: str) -> str | None:
+    """The space of an index file name, or None for each other file."""
+    if not name.endswith(".sqlite"):
+        return None
+    try:
+        return parse_space(name.removesuffix(".sqlite").replace("--", "/", 1))
+    except ValueError:
+        return None
+
+
+class IndexLocked(Exception):
+    """Another index run holds the lock of the data directory."""
+
+
+@dataclass(frozen=True)
+class Concept:
+    id: str
+    tier_space: str
+    path: str
+    type: str
+    title: str
+    description: str
+    tags: tuple[str, ...]
+    status: str | None
+    sources: tuple[dict[str, Any], ...]
+    body: str
+    content_sha256: str
+
+
+@dataclass(frozen=True)
+class Unit:
+    concept_id: str
+    slug: str
+    ordinal: int
+    #: The text of a text unit, or the media path of an image unit.
+    text: str
+    vector: tuple[float, ...] = field(default=(), repr=False)
+    #: `text` or `image`.
+    kind: str = TEXT
+    #: The path of the image, relative to the concept directory. None for text.
+    media_path: str | None = None
+    #: The SHA-256 of the image bytes. None for text.
+    media_sha256: str | None = None
+
+
+@dataclass(frozen=True)
+class ConceptMedia:
+    """One accepted image of a concept, with or without a vector (ADR-009)."""
+
+    concept_id: str
+    #: The place of the image in the `media:` list, from 1.
+    number: int
+    #: The path relative to the concept directory.
+    path: str
+    sha256: str
+    mime: str
+
+
+@dataclass(frozen=True)
+class Link:
+    source_id: str
+    target_id: str
+    kind: str
+    resolved: bool
+
+
+@dataclass(frozen=True)
+class IndexMeta:
+    tier_space: str
+    concept_count: int
+    embedder_model: str | None
+    embedder_dimension: int | None
+    #: The fingerprint of the prompts of the embeddings, or None without embeddings.
+    embedder_prompts: str | None = None
+    schema_version: int = SCHEMA_VERSION
+
+
+@dataclass(frozen=True)
+class Prompts:
+    """The task prompts that put a query or a document into the form that a model needs.
+
+    `{text}` is the text. `{title}` is the title of the concept, in a document prompt only.
+    """
+
+    query: str = "{text}"
+    document: str = "{text}"
+
+    def for_query(self, text: str) -> str:
+        return self.query.replace("{text}", text)
+
+    def for_document(self, title: str, text: str) -> str:
+        # The text goes in last, so that braces in the text stay as they are.
+        return self.document.replace("{title}", title).replace("{text}", text)
+
+    @property
+    def fingerprint(self) -> str:
+        return f"query={self.query}|document={self.document}"
+
+
+#: The retrieval prompts of the EmbeddingGemma model card (ADR-003).
+EMBEDDINGGEMMA = Prompts(
+    query="task: search result | query: {text}",
+    document="title: {title} | text: {text}",
+)
+
+
+def prompts_for(model: str) -> Prompts:
+    """The default prompts for a model name: EmbeddingGemma prompts, or no prompts."""
+    return EMBEDDINGGEMMA if _is_embeddinggemma(model) else Prompts()
+
+
+#: The lowest cosine similarity that counts as a match of meaning, for a model with no
+#: measured limit.
+DEFAULT_MIN_SIMILARITY = 0.35
+#: The measured limit of EmbeddingGemma 2 (ADR-005). Unrelated questions reach about
+#: 0.67, and known answers start at about 0.70.
+EMBEDDINGGEMMA_MIN_SIMILARITY = 0.68
+#: The measured limit of EmbeddingGemma 2 for a match between words and an image
+#: (ADR-008). Known images scored from 0.741. Wrong images and unrelated questions
+#: reached 0.655.
+EMBEDDINGGEMMA_MIN_IMAGE_SIMILARITY = 0.68
+#: The measured limits of EmbeddingGemma 2 when the question is an image (ADR-010).
+EMBEDDINGGEMMA_MIN_IMAGE_TO_IMAGE_SIMILARITY = 0.89
+EMBEDDINGGEMMA_MIN_IMAGE_TO_TEXT_SIMILARITY = 0.70
+#: The measured limit of EmbeddingGemma 2 for words and an audio clip (ADR-011).
+EMBEDDINGGEMMA_MIN_AUDIO_SIMILARITY = 0.68
+#: The measured limit of EmbeddingGemma 2 for words and a video (ADR-013).
+EMBEDDINGGEMMA_MIN_VIDEO_SIMILARITY = 0.71
+#: The measured limits of EmbeddingGemma 2 when the question is a video (ADR-014).
+EMBEDDINGGEMMA_MIN_VIDEO_TO_VIDEO_SIMILARITY = 0.92
+EMBEDDINGGEMMA_MIN_VIDEO_TO_TEXT_SIMILARITY = 0.73
+#: The measured limits of EmbeddingGemma 2 when the question is a clip (ADR-012).
+EMBEDDINGGEMMA_MIN_AUDIO_TO_AUDIO_SIMILARITY = 0.87
+EMBEDDINGGEMMA_MIN_AUDIO_TO_TEXT_SIMILARITY = 0.70
+
+
+def min_similarity_for(model: str) -> float:
+    """The default similarity limit for a model name."""
+    return EMBEDDINGGEMMA_MIN_SIMILARITY if _is_embeddinggemma(model) else DEFAULT_MIN_SIMILARITY
+
+
+def min_image_similarity_for(model: str) -> float:
+    """The default similarity limit of an image unit for a model name."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_IMAGE_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_audio_similarity_for(model: str) -> float:
+    """The default limit when a question in words meets an audio unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_AUDIO_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_video_similarity_for(model: str) -> float:
+    """The default limit when a question in words meets a video unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_VIDEO_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_video_to_video_similarity_for(model: str) -> float:
+    """The default limit when a video question meets a video unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_VIDEO_TO_VIDEO_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_video_to_text_similarity_for(model: str) -> float:
+    """The default limit when a video question meets a text unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_VIDEO_TO_TEXT_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_audio_to_audio_similarity_for(model: str) -> float:
+    """The default limit when a clip question meets an audio unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_AUDIO_TO_AUDIO_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_audio_to_text_similarity_for(model: str) -> float:
+    """The default limit when a clip question meets a text unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_AUDIO_TO_TEXT_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_image_to_image_similarity_for(model: str) -> float:
+    """The default limit when an image question meets an image unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_IMAGE_TO_IMAGE_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def min_image_to_text_similarity_for(model: str) -> float:
+    """The default limit when an image question meets a text unit."""
+    if _is_embeddinggemma(model):
+        return EMBEDDINGGEMMA_MIN_IMAGE_TO_TEXT_SIMILARITY
+    return DEFAULT_MIN_SIMILARITY
+
+
+def _is_embeddinggemma(model: str) -> bool:
+    return "embeddinggemma" in model.lower()
+
+
+@dataclass(frozen=True)
+class Filters:
+    """Optional limits on a search: any of `types`, and all of `tags`."""
+
+    types: tuple[str, ...] = ()
+    tags: tuple[str, ...] = ()
+
+
+class Embedder(Protocol):
+    """Turns texts into vectors. An adapter never loads model weights in this process."""
+
+    @property
+    def model(self) -> str: ...
+
+    @property
+    def dimension(self) -> int | None: ...
+
+    def embed(self, texts: Sequence[str]) -> list[list[float]]: ...
+
+    def embed_media(self, files: Sequence[MediaFile]) -> list[list[float]]:
+        """One vector for each image, audio or video file, in the space of the text, with no prompt.
+
+        Raise an error when the embedder cannot embed a file (ADR-007, ADR-011).
+        """
+        ...
+
+
+class IndexStore(Protocol):
+    """The derived index. Only `build` writes, and it replaces a tier space atomically."""
+
+    def build(
+        self,
+        meta: IndexMeta,
+        concepts: Sequence[Concept],
+        units: Sequence[Unit],
+        links: Sequence[Link],
+        media: Sequence[ConceptMedia] = (),
+        blobs: Sequence[MediaFile] = (),
+    ) -> None:
+        """Replace the tier space. `blobs` holds the bytes; each SHA-256 is kept one time."""
+        ...
+
+    def meta(self, tier_space: str) -> IndexMeta | None: ...
+
+    def spaces(self) -> tuple[str, ...]:
+        """Each space that has an index, in name order."""
+        ...
+
+    def concept_ids(self, tier_space: str) -> set[str]:
+        """The concept identifiers of one space. A space with no index has none."""
+        ...
+
+    def index_lock(self) -> AbstractContextManager[None]:
+        """Hold the lock of an index run, or raise `IndexLocked`."""
+        ...
+
+    def get(self, spaces: Sequence[str], concept_id: str) -> Concept | None: ...
+
+    def word_search(
+        self, spaces: Sequence[str], words: Sequence[str], filters: Filters, limit: int
+    ) -> list[str]:
+        """The identifiers that match any of the words, best first."""
+        ...
+
+    def unit_vectors(
+        self, spaces: Sequence[str], filters: Filters
+    ) -> list[tuple[str, str, str, tuple[float, ...]]]:
+        """Each (concept id, unit slug, unit kind, vector) that has a vector.
+
+        Each unit of an index of schema version 1 has the kind `text`.
+        """
+        ...
+
+    def links_from(self, spaces: Sequence[str], concept_id: str) -> list[Link]:
+        """The outgoing links. A target resolves only when a readable space holds it."""
+        ...
+
+    def media_of(self, spaces: Sequence[str], concept_id: str) -> list[ConceptMedia]:
+        """The images of a concept, in list order. An index before schema 3 has none."""
+        ...
+
+    def media_bytes(self, spaces: Sequence[str], sha256: str) -> tuple[str, bytes] | None:
+        """The MIME type and the bytes of an image, or None."""
+        ...
+
+    def links_to(self, spaces: Sequence[str], concept_id: str) -> list[Link]: ...
+
+    def titles(self, spaces: Sequence[str], concept_ids: Sequence[str]) -> dict[str, str]: ...
+
+    def canonical_dump(self, tier_space: str) -> str:
+        """A text of all the rows in key order. Equal dumps mean identical indexes."""
+        ...

curu_memory/render.py

@@ -0,0 +1,61 @@
+"""Markdown to safe HTML for the read-only browser (ADR-004, research R-2).
+
+A concept body comes from persons and agents, so its HTML must run nothing:
+
+- Raw HTML in a body shows as text (`html: False`).
+- The link check of markdown-it-py refuses `javascript:`, `vbscript:`, `file:` and `data:`.
+- A link to the `.md` file of a known concept goes to the page of that concept.
+- A link to a missing concept keeps its text and loses its address.
+- An external link gets `rel="noopener noreferrer"`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Set
+from pathlib import PurePosixPath
+from urllib.parse import quote, urlsplit
+
+from markdown_it import MarkdownIt
+from markdown_it.rules_core import StateCore
+from markdown_it.token import Token
+
+
+def render_markdown(body: str, known_ids: Set[str]) -> str:
+    """The safe HTML of one concept body."""
+    parser = MarkdownIt("commonmark", {"html": False})
+    parser.core.ruler.push("concept_links", lambda state: _rewrite_links(state, known_ids))
+    return parser.render(body)
+
+
+def concept_href(concept_id: str) -> str:
+    return "/concepts/" + quote(concept_id, safe="._-")
+
+
+def _rewrite_links(state: StateCore, known_ids: Set[str]) -> None:
+    for block in state.tokens:
+        if block.type != "inline" or not block.children:
+            continue
+        opened: list[Token] = []
+        for token in block.children:
+            if token.type == "link_open":
+                _rewrite_open(token, known_ids)
+                opened.append(token)
+            elif token.type == "link_close" and opened:
+                token.tag = opened.pop().tag
+
+
+def _rewrite_open(token: Token, known_ids: Set[str]) -> None:
+    href = str(token.attrGet("href") or "")
+    parts = urlsplit(href)
+    if parts.scheme or parts.netloc:
+        token.attrSet("rel", "noopener noreferrer")
+        return
+    if not parts.path.endswith(".md"):
+        return
+    concept_id = PurePosixPath(parts.path).stem
+    if concept_id in known_ids:
+        token.attrSet("href", concept_href(concept_id))
+        return
+    # A link to a missing concept: keep the text, and give no address.
+    token.tag = "span"
+    token.attrs = {"class": "broken", "title": f"No concept {concept_id!r} is in the index."}