reqctl 0.1.38__py3-none-any.whl

reqctl/corpus.py

@@ -0,0 +1,1283 @@
+import base64
+import contextlib
+import functools
+import hashlib
+import json
+import os
+import re
+import secrets
+import shutil
+from pathlib import Path
+from urllib.parse import urldefrag, urljoin, urlsplit
+
+import yaml
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError
+from jsonschema.validators import validator_for
+from referencing import Registry
+from referencing.jsonschema import DRAFT202012, specification_with
+
+Loader = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader
+NEST_LIMIT = 64
+
+CONFLICTED = re.compile(r"^<{7}", re.M)
+
+PREFIXES = {"REQ-": "requirement", "GUARD-": "guard",
+            "PARAM-": "parameter", "TERM-": "term", "DATA-": "data"}
+FOLDERS = {"requirement": "reqs", "guard": "guards", "parameter": "params",
+           "term": "terms", "data": "data"}
+KINDS = "|".join(prefix.rstrip("-") for prefix in PREFIXES)
+NAMED = ", ".join(prefix.rstrip("-") for prefix in PREFIXES)
+KINDS_NAMED = ", ".join(FOLDERS)
+
+NAME = r"[a-z][a-z0-9_]*"
+UID = re.compile(rf"^({KINDS})-\d{{8}}$")
+UID_IN_PROSE = re.compile(rf"\b(?:{KINDS})-\d{{8}}\b")
+PARAM_REF = re.compile(rf"\$\{{({NAME})((?:\.[^}}\s]+)?)\}}")
+PARAM_NEAR = re.compile(r"\$\{?(?:param|data)[-_]\d+(?:\.[\w.-]+)?\}?", re.IGNORECASE)
+NAME_FORM = "${name}"
+# @req> REQ-11930677@dH7Pu7k4K4YI lzcna6
+CONCEPT_TARGET = re.compile(rf"{NAME}(?:\.{NAME}){{0,2}}")
+CONCEPT_LINK = re.compile(rf"\[([^\]\n]+)\]\(({CONCEPT_TARGET.pattern})\)")
+TERM_NEAR = re.compile(r"\bterm[-_]\d+\b", re.IGNORECASE)
+LINK_FORM = "[the words](handle)"
+CONCEPT_NEAR = re.compile(r"\]\(((?:TERM|PARAM|DATA)-\d+[^)]*)\)")
+TAG_STAMP = 12
+TEST_PATH = re.compile(r"(^|/)tests?/|(^|/)test_[^/]+$|[._](test|spec)\.[A-Za-z]+$")
+
+STATEMENT_STAMPED = ("text", "acceptance_criteria")
+ITEM_STAMPED = ("text", "value_type", "unit", "default")
+DATA_KEY = re.compile(r"^[a-z][a-z0-9_]*$")
+STATED_SKIP = ("aliases",)
+# @req> REQ-76787560@yx4uV-ApjJfe sfuabl
+POLICED_SKIP = ("aliases", "word", "name", "carried")
+# @req> REQ-68407569@9WdKt66SbMuQ bzk766
+POLICED_ITEM_SKIP = POLICED_SKIP + ("default", "unit")
+# @req> REQ-83702299@kBGzIwAyKDHL zgqncd
+ENUMERATED = ("kind", "type", "verification", "priority", "value_type")
+CITATION_LIST = "citations"
+WHOLE_FILES = "whole_file_citations"
+# @req> REQ-88203622@wlZ9orsr9c7q otmh6w
+CITABLE = ("requirement", "guard", "parameter", "data")
+# @req> REQ-22682482@F_OIMxlkifKy utfemd
+ITEM_SKIP = ("entries", "acceptance_criteria", "text", "fields",
+             "assessed", "relations", "status", CITATION_LIST, WHOLE_FILES)
+MEMBERS = "*"
+ADDRESS = re.compile(rf"^((?:{KINDS})-\d{{8}}|{NAME})(?:\.(.+))?$")
+
+# @req> REQ-10966072@RC1_0V_R-34U zikuq2
+# @req+ skipped_folders@EwHTRPcwGsLx p2lhya
+# @req> REQ-87066486@AtzPEAcAbJ1v m5r2gd
+SCAN_SKIP = {
+    ".git", "node_modules", ".venv", "venv", "dist", "build", "__pycache__",
+    ".mypy_cache", ".ruff_cache", ".pytest_cache", ".elucidate", ".reqctl",
+}
+
+
+def skipped(root, folder):
+    return (folder.name in SCAN_SKIP or folder.name.endswith(".egg-info")
+            or folder == Path(root) / "requirements")
+# @req- p2lhya
+
+
+class ReqctlError(Exception):
+    pass
+
+
+class Item:
+    def __init__(self, uid, path, data):
+        self.uid = uid
+        self.path = path
+        self.data = data
+
+
+class Store:
+    def __init__(self, root):
+        self.root = root
+        self._items = None
+        self._index = None
+
+
+def find_root(start=None):
+    here = Path(start or Path.cwd()).resolve()
+    for candidate in [here, *here.parents]:
+        if (candidate / "requirements").is_dir() and (candidate / ".git").exists():
+            return candidate
+    # @req> REQ-61212158@EISsRx_ntdvz pdpnj6
+    for candidate in [here, *here.parents]:
+        if (candidate / ".git").exists():
+            return candidate
+    # @req> REQ-30898926@Vu28eWaG_1mE nq47sk
+    raise ReqctlError("not inside a git repository (no .git here or above) -- "
+                      "run reqctl inside the repository")
+
+
+def load(root=None):
+    root = Path(root) if root else find_root()
+    return Store(root), root
+
+
+def _for_prefix(uid, table):
+    for prefix, value in table.items():
+        if uid.startswith(prefix):
+            return value
+    # @req> REQ-30898926@Vu28eWaG_1mE vtwjpf
+    raise ReqctlError(f"{uid}: not a {NAMED} uid -- use a uid of one of those "
+                      "kinds; `reqctl list` names them")
+
+
+# @req> REQ-14917237@a9P7pldigUrg 7vs2op
+def kind_of(uid, data):
+    return kind_for_uid(uid) or (data or {}).get("kind")
+
+
+def folder_for(root, kind):
+    if kind not in FOLDERS:
+        # @req> REQ-30898926@Vu28eWaG_1mE 5vz2qe
+        raise ReqctlError(f"{kind}: not a {KINDS_NAMED} kind -- use one of those kinds")
+    return Path(root) / "requirements" / FOLDERS[kind]
+
+
+def item_files(root):
+    found = []
+    for folder in FOLDERS.values():
+        directory = Path(root) / "requirements" / folder
+        found += sorted(
+            path
+            for pattern in ("*.yml", "*.yaml")
+            for path in directory.glob(pattern)
+            if not path.name.startswith(".")
+        )
+    seen = {}
+    # @req> REQ-17446443@86KE0VfkVuKG 2mb54k
+    for path in found:
+        first = seen.setdefault(path.stem, path)
+        if first is not path:
+            # @req> REQ-30898926@Vu28eWaG_1mE k6zhqw
+            raise ReqctlError(
+                f"{path.stem}: two files claim this uid: "
+                f"{first.relative_to(root)} and {path.relative_to(root)} -- "
+                "remove one of the two files, or rename it to a different uid"
+            )
+    return found
+
+
+def _repeated(node):
+    if not isinstance(node, yaml.MappingNode):
+        return []
+    stated = [key.value for key, _ in node.value
+              if isinstance(key, yaml.ScalarNode)]
+    return sorted(set(key for key in stated if stated.count(key) > 1))
+
+
+def _folded(loader, node):
+    walked, stack = set(), [node]
+    while stack:
+        held = stack.pop()
+        if held in walked:
+            continue
+        walked.add(held)
+        if isinstance(held, yaml.SequenceNode):
+            stack.extend(held.value)
+        if isinstance(held, yaml.MappingNode):
+            keys = {}
+            for key, value in held.value:
+                stack.append(value)
+                if isinstance(key, yaml.ScalarNode):
+                    read = loader.construct_object(key)
+                    if read in keys:
+                        return [keys[read]]
+                    keys[read] = read
+    return []
+
+
+def loads(text, where, aliases=True, reading=Loader):
+    loader = reading(text)
+    try:
+        # @req+ REQ-49870454@KKNV-vdY6oM1 ts6cfc
+        named = [] if aliases else [
+            f"*{event.anchor} on line {event.start_mark.line + 1}"
+            for event in yaml.parse(text, Loader=Loader)
+            if isinstance(event, yaml.AliasEvent)]
+        if named:
+            # @req> REQ-30898926@Vu28eWaG_1mE danz6r
+            raise ReqctlError(f"{where} holds the YAML alias {', '.join(named)} "
+                              "-- write each value out in full in place of its alias")
+        # @req- ts6cfc
+        node = loader.get_single_node()
+        twice = _repeated(node)
+        # @req> REQ-84465433@iXBRKIfkf18a pkvikc
+        if twice:
+            # @req> REQ-30898926@Vu28eWaG_1mE e2tsff
+            raise ReqctlError(
+                f"{where} states {', '.join(twice)} more than once -- YAML "
+                "keeps the last, so drop the one that does not govern"
+            )
+        # @req+ REQ-19913588@hBNJrpn4vtZe 3222m4
+        try:
+            document = loader.construct_document(node) if node is not None else None
+        except (yaml.YAMLError, RecursionError):
+            raise
+        except Exception as error:
+            # @req> REQ-30898926@Vu28eWaG_1mE obrq2j
+            raise ReqctlError(f"{where} holds a value YAML cannot build, "
+                              f"{type(error).__name__}: {error} -- replace the "
+                              "value with plain YAML") from error
+        # @req- 3222m4
+        # @req+ REQ-48650108@-jXFakmxSYbB i4wv7k
+        # @req+ REQ-83085784@mB9K6Zw3dGnC lxmmuv
+        folded = _folded(loader, node)
+        if folded:
+            # @req> REQ-30898926@Vu28eWaG_1mE q232n2
+            raise ReqctlError(
+                f"{where} states the key {folded[0]} twice in one mapping -- YAML "
+                "keeps the last, so drop the one that does not govern"
+            )
+        # @req- lxmmuv
+        # @req- i4wv7k
+        # @req> REQ-39368456@LvbuCW5-SvUW bpdfcl
+        for field, value in (document.items() if isinstance(document, dict) else ()):
+            fault = _unreadable(value)
+            looped = _looped(text) if fault else None
+            if looped:
+                # @req> REQ-30898926@Vu28eWaG_1mE 5l5zyz
+                raise ReqctlError(
+                    f"{where}: the YAML alias {looped} forms a cycle, so it "
+                    "cannot be read as text -- state it without an alias to itself")
+            if fault:
+                # @req> REQ-30898926@Vu28eWaG_1mE xnluqf
+                raise ReqctlError(f"{where}: {field} {fault}")
+        return document
+    except yaml.YAMLError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE 5ek57f
+        raise ReqctlError(f"{where} is not YAML: {error} -- correct the YAML where "
+                          "the parser points; a settings file is corrected on "
+                          "the default branch") from error
+    except RecursionError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE aykasy
+        raise ReqctlError(f"{where} nests too deeply to read -- flatten the "
+                          "nesting") from error
+    finally:
+        loader.dispose()
+
+
+def _looped(text):
+    open_ = []
+    for event in yaml.parse(text, Loader=Loader):
+        if isinstance(event, yaml.CollectionStartEvent):
+            open_.append(event.anchor)
+        elif isinstance(event, yaml.CollectionEndEvent):
+            open_.pop()
+        elif isinstance(event, yaml.AliasEvent) and event.anchor in open_:
+            return f"*{event.anchor} on line {event.start_mark.line + 1}"
+    return None
+
+
+def _unreadable(value):
+    height, open_ = {}, set()
+    stack = [(value, False)]
+    while stack:
+        node, closing = stack.pop()
+        if not isinstance(node, (dict, list, tuple)):
+            continue
+        children = list(node.values() if isinstance(node, dict) else node)
+        if closing:
+            open_.discard(id(node))
+            height[id(node)] = 1 + max((height.get(id(child), 0) for child in children),
+                                       default=0)
+            if height[id(node)] > NEST_LIMIT:
+                return (f"nests deeper than {NEST_LIMIT} levels, so it cannot be read "
+                        "as text -- state it flatter")
+        elif id(node) in open_:
+            return ("forms a cycle, so it cannot be read as text -- state it "
+                    "without an alias to itself")
+        elif id(node) not in height:
+            open_.add(id(node))
+            stack.append((node, True))
+            stack.extend((child, False) for child in children)
+    return None
+
+
+def read_text(path):
+    try:
+        return Path(path).read_text()
+    except UnicodeDecodeError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE 6yjen4
+        raise ReqctlError(f"cannot read {path}: not valid UTF-8 -- save the file "
+                          "as UTF-8") from error
+    except OSError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE acn4ae
+        raise ReqctlError(f"cannot read {path}: {error.strerror} -- make the file "
+                          "readable, then retry") from error
+    except ValueError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE epjjtu
+        raise ReqctlError(f"cannot read {path}: {error} -- name a path that can "
+                          "be read, then retry") from error
+
+
+def read(path, reading=Loader):
+    return loads(read_text(path), path, reading=reading)
+
+
+def sides(text):
+    ours, theirs, taking = [], [], "both"
+    for line in text.splitlines(keepends=True):
+        if line.startswith("<<<<<<<"):
+            taking = "ours"
+        elif line.startswith("|||||||"):
+            taking = "base"
+        elif line.startswith("======="):
+            taking = "theirs"
+        elif line.startswith(">>>>>>>"):
+            taking = "both"
+        else:
+            if taking in ("both", "ours"):
+                ours.append(line)
+            if taking in ("both", "theirs"):
+                theirs.append(line)
+    return "".join(ours), "".join(theirs)
+
+
+def same(value, prior):
+    if type(value) is not type(prior):
+        return False
+    if isinstance(value, list):
+        return len(value) == len(prior) and all(
+            same(v, p) for v, p in zip(value, prior)
+        )
+    if isinstance(value, dict):
+        return value.keys() == prior.keys() and all(
+            same(v, prior[k]) for k, v in value.items()
+        )
+    return value == prior
+
+
+def path_for(root, uid):
+    for path in item_files(root):
+        if path.stem == uid:
+            return path
+    # @req> REQ-30898926@Vu28eWaG_1mE vdl325
+    raise ReqctlError(f"no such item: {uid} -- `reqctl list` names the uids that "
+                      "exist")
+
+
+def items(store):
+    if store._items is None:
+        loaded = []
+        for path in item_files(store.root):
+            text = read_text(path)
+            # @req> REQ-38498011@eJXtv0pwMf-A q7wu74
+            if CONFLICTED.search(text):
+                from .resolve import merged
+                # @req> REQ-30898926@Vu28eWaG_1mE tts4hh
+                raise ReqctlError(merged(path.relative_to(store.root), text))
+            # @req> REQ-49870454@KKNV-vdY6oM1 4sqskc
+            data = loads(text, path, aliases=False)
+            if not isinstance(data, dict):
+                # @req> REQ-30898926@Vu28eWaG_1mE kytafn
+                raise ReqctlError(f"{path.stem}: item file is not a mapping -- edit the "
+                                  "file by hand so it holds a YAML mapping of fields")
+            refuse_misshapen(path.relative_to(store.root), data)
+            # @req> REQ-56223366@zS6FN-Id9V3H wvbhnl
+            loaded.append(Item(path.stem, path, data))
+        store._items = loaded
+    return store._items
+
+
+# @req> REQ-19913588@hBNJrpn4vtZe 5es3j4
+def refuse_misshapen(where, data):
+    for field in ("kind", "status"):
+        if not isinstance(data.get(field, ""), str):
+            # @req> REQ-30898926@Vu28eWaG_1mE 3grind
+            raise ReqctlError(f"{where}: {field} holds {data[field]!r}, not one name -- "
+                              f"state one {field}")
+    held = data.get("entries")
+    for key, fields in (held.items() if isinstance(held, dict) else ()):
+        if not isinstance(fields, dict):
+            # @req> REQ-30898926@Vu28eWaG_1mE mmffva
+            raise ReqctlError(f"{where}: entry {key} holds {fields!r}, not fields -- "
+                              "state its fields under it")
+    if found := _untexted(data):
+        # @req> REQ-30898926@Vu28eWaG_1mE d5vilj
+        raise ReqctlError(f"{where}: the key {found[-1]!r} under "
+                          f"{'.'.join(map(str, found[:-1])) or 'the item'} is not text -- "
+                          "quote it or rename it")
+
+
+def _untexted(value, at=()):
+    pairs = (value.items() if isinstance(value, dict)
+             else enumerate(value) if isinstance(value, list) else ())
+    for key, held in pairs:
+        if isinstance(value, dict) and not isinstance(key, str):
+            return (*at, key)
+        if found := _untexted(held, (*at, key)):
+            return found
+    return None
+
+
+def invalidate(store):
+    store._items = None
+    store._index = None
+
+
+def find(store, uid):
+    if store._index is None:
+        index = {item.uid: item for item in items(store)}
+        for item in items(store):
+            name = name_of(item.uid, item.data)
+            if name is not None:
+                index.setdefault(name, item)
+        store._index = index
+    try:
+        return store._index[uid]
+    except KeyError:
+        # @req> REQ-30898926@Vu28eWaG_1mE j6yens
+        raise ReqctlError(f"no such item: {uid} -- `reqctl list` names the uids that "
+                          "exist") from None
+
+
+def raw(item):
+    return item.data
+
+
+# @req> REQ-79557189@9se-A9aoBjZx np3ttn
+def atomic_write(path, content):
+    target = Path(path).resolve()
+    temp = target.with_name(f"{target.name}.{secrets.token_hex(8)}.tmp")
+    staged = False
+    try:
+        # @req> REQ-24406170@ffuKefeAdU7p ig234p
+        _hold(target)
+        # @req> REQ-81667762@82D0B66GuW3u lkbxat
+        make_folder(target.parent)
+        with open(temp, "xb" if isinstance(content, bytes) else "x") as sink:
+            staged = True
+            sink.write(content)
+            sink.flush()
+            os.fsync(sink.fileno())
+        if target.exists():
+            shutil.copymode(target, temp)
+        os.replace(temp, target)
+        staged = False
+    except OSError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE azwvi6
+        raise ReqctlError(
+            f"cannot write {path}: {error.strerror or error} -- make the path "
+            "writable, then retry"
+        ) from error
+    finally:
+        if staged:
+            with contextlib.suppress(OSError):
+                temp.unlink()
+
+
+# @req> REQ-65668011@WuPZe9UcmiaP l5i6id
+def write_all(changes):
+    with all_or_nothing():
+        for path, content in changes:
+            atomic_write(path, content)
+
+
+# @req+ REQ-24406170@ffuKefeAdU7p u5qcba
+_held = None
+_made = None
+
+
+@contextlib.contextmanager
+def all_or_nothing():
+    global _held, _made
+    if _held is not None:
+        yield
+        return
+    _held, _made = {}, []
+    try:
+        yield
+    except BaseException as error:
+        if isinstance(error, SystemExit) and error.code in (None, 0):
+            raise
+        held, _held = _held, None
+        kept = [str(path) for path, before in reversed(held.items())
+                if not _put_back(path, before)]
+        # @req> REQ-81667762@82D0B66GuW3u cdcyzh
+        for folder in reversed(_made):
+            with contextlib.suppress(OSError):
+                folder.rmdir()
+        # @req+ REQ-83236530@f_SrFVNH8QSC v2zljr
+        kept += [f"{folder}{os.sep}" for folder in reversed(_made)
+                 if _stands(folder)]
+        if kept:
+            # @req> REQ-30898926@Vu28eWaG_1mE vtahuy
+            left = (f"not put back: {', '.join(kept)} -- restore each file by hand "
+                    "(`git restore PATH`) and "
+                    "remove each folder")
+            if isinstance(error, SystemExit):
+                # @req> REQ-30898926@Vu28eWaG_1mE qdhgis
+                raise SystemExit(f"{error}; {left}") from error
+            if isinstance(error, (ReqctlError, OSError, UnicodeError)):
+                # @req> REQ-30898926@Vu28eWaG_1mE exlmdp
+                raise ReqctlError(f"{error}; {left}") from error
+            error.add_note(left)
+        # @req- v2zljr
+        raise
+    finally:
+        _held = None
+
+
+def atomically(command):
+    with all_or_nothing():
+        return command()
+
+
+def _hold(path):
+    if _held is not None and path not in _held:
+        _held[path] = path.read_bytes() if path.exists() else None
+
+
+def _put_back(path, before):
+    try:
+        if before is None:
+            if path.exists():
+                path.unlink()
+        elif not path.is_file() or path.read_bytes() != before:
+            atomic_write(path, before)
+    except (OSError, ReqctlError):
+        return False
+    return True
+
+
+def remove(path, missing_ok=False):
+    held = Path(path).absolute()
+    _hold(held)
+    held.unlink(missing_ok=missing_ok)
+# @req- u5qcba
+
+
+# @req> REQ-81667762@82D0B66GuW3u 3rsgq3
+def make_folder(path):
+    folder = Path(path).absolute()
+    if _held is not None:
+        _made.extend(one for one in (*reversed(folder.parents), folder)
+                     if not one.exists())
+    folder.mkdir(parents=True, exist_ok=True)
+
+
+# @req> REQ-83236530@f_SrFVNH8QSC 3j7lib
+def _stands(folder):
+    try:
+        return folder.is_dir()
+    except OSError:
+        return True
+
+
+# @req+ REQ-41148688@SFtnFhcEeNvS xm3u4i
+class _Unaliased(yaml.SafeDumper):
+    def ignore_aliases(self, data):
+        return True
+
+
+def dump(data):
+    return yaml.dump(data, Dumper=_Unaliased, default_flow_style=False,
+                     sort_keys=True, allow_unicode=True)
+# @req- xm3u4i
+
+
+def save(store, item):
+    atomic_write(item.path, dump(item.data))
+    invalidate(store)
+
+
+def _lexical(value):
+    return f"!{type(value).__name__}:{value}"
+
+
+def digest(payload):
+    content = json.dumps(payload, sort_keys=True, separators=(",", ":"),
+                         default=_lexical)
+    hashed = hashlib.sha256(content.encode()).digest()
+    return base64.urlsafe_b64encode(hashed).decode().rstrip("=")
+
+
+def entries(data):
+    held = data.get("entries")
+    return held if isinstance(held, dict) else None
+
+
+def split_address(address):
+    found = ADDRESS.match(str(address))
+    if not found:
+        # @req> REQ-30898926@Vu28eWaG_1mE ibipo5
+        raise ReqctlError(f"{address}: not a UID or UID.entry address -- write UID, "
+                          "or UID.entry")
+    return found.group(1), found.group(2)
+
+
+def kind_for_uid(uid):
+    try:
+        return _for_prefix(uid, PREFIXES)
+    except ReqctlError:
+        return None
+
+
+def name_of(uid, data):
+    kind = kind_of(uid, data)
+    if kind == "term":
+        return next(iter(entries(data) or {}), None)
+    if kind in ("parameter", "data"):
+        name = data.get("name")
+        return name if isinstance(name, str) and name else None
+    return None
+
+
+def carried_of(uid, data):
+    held = data.get("carried")
+    if held is None:
+        return []
+    if not isinstance(held, list) or not all(
+            isinstance(one, str) for one in held):
+        # @req> REQ-30898926@Vu28eWaG_1mE pgm2dk
+        raise ReqctlError(
+            f"{uid}: carried is not a list of names, so what this item has "
+            "answered to cannot be read -- `reqctl validate` names the fault; edit "
+            "the item by hand so carried is a list of names, or remove carried"
+        )
+    return held
+
+
+def carriers(tree):
+    held = {}
+    for item in items(tree):
+        for name in set(carried_of(item.uid, raw(item))):
+            held.setdefault(name, []).append(item.uid)
+    return held
+
+
+def reachable(records):
+    held = {uid: uid for uid in records}
+    for uid, data in records.items():
+        name = name_of(uid, data)
+        if name is not None:
+            held.setdefault(name, uid)
+    return held
+
+
+def addressable(records):
+    return set(reachable(records))
+
+
+def stamp_of(uid, data):
+    return digest([uid, covered(uid, data)])
+
+
+def _term_digest(uid, fields):
+    return digest([uid, {"definition": fields.get("definition")}])
+
+
+def flagged(data, flag):
+    held = entries(data) or {}
+    if not isinstance(flag, str) or flag in held:
+        return None
+    carried = [fields.get(flag) for fields in held.values()
+               if isinstance(fields, dict) and flag in fields]
+    if not carried:
+        return None
+    # @req> REQ-79222815@smBL1zf2tP96 3wyszr
+    return {key: fields for key, fields in held.items()
+            if isinstance(fields, dict) and fields.get(flag) is True}
+
+
+def held_block(data, name):
+    if name in (entries(data) or {}) or name in ITEM_SKIP:
+        return None
+    held = data.get(name)
+    return held if isinstance(held, dict) else None
+
+
+def entry_at(data, key):
+    held = entries(data) or {}
+    if key in held:
+        fields = held[key]
+        return fields if isinstance(fields, dict) else None
+    if not isinstance(key, str):
+        return None
+    steps = key.split(".")
+    # @req+ REQ-11930677@dH7Pu7k4K4YI 2nario
+    block = held_block(data, steps[0])
+    if block is not None and len(steps) > 1:
+        held, steps = block, steps[1:]
+    # @req- 2nario
+    while steps:
+        name = steps.pop(0)
+        if not isinstance(held, dict) or name not in held:
+            return None
+        fields = held[name]
+        if not isinstance(fields, dict):
+            return None
+        if not steps:
+            return fields
+        field = steps.pop(0)
+        if not isinstance(fields.get(field), dict):
+            return None
+        held = fields[field]
+    return None
+
+
+def entry_stamp_of(uid, data, key):
+    # @req+ REQ-23898341@oXCyxMhvY0ka nvrect
+    fields = entry_at(data, key)
+    if fields is None:
+        return None
+    if kind_of(uid, data) == "term":
+        # @req> REQ-23281207@-EoSDW-v6e5y 5spkdj
+        return _term_digest(uid, fields)
+    # @req> REQ-80106209@1V8R51ejNtrH hydltf
+    return digest([uid, str(key), dict(fields)])
+    # @req- nvrect
+
+
+def address_stamp_of(uid, data, entry_key):
+    if entry_key is None:
+        return stamp_of(uid, data)
+    if entry_key == MEMBERS:
+        return digest([uid, MEMBERS, entries(data) or {}])
+    # @req+ REQ-63712597@Lvnsc2CjyqjN okjacr
+    held = flagged(data, entry_key)
+    if held is not None:
+        return digest([uid, entry_key, held])
+    # @req- okjacr
+    return entry_stamp_of(uid, data, entry_key)
+
+
+def stamp_at(item, entry_key=None):
+    return address_stamp_of(item.uid, item.data, entry_key)
+
+
+# @req> REQ-24608716@rrTjNN2Ay0D0 kzig7g
+def covered(uid, data, entry_key=None):
+    if entry_key is None:
+        # @req> REQ-54851939@n6YOz94UUTn7 njypn2
+        if uid.startswith(("REQ-", "GUARD-")):
+            # @req> REQ-38833983@N77mQ5lbSN4A ekeubm
+            return {key: data.get(key) for key in STATEMENT_STAMPED}
+        # @req+ REQ-56581795@ZHEasHzzTZLC ixanwu
+        if kind_of(uid, data) == "term":
+            # @req> REQ-23281207@-EoSDW-v6e5y vnctjd
+            return {"definition": term_fields(data).get("definition")}
+        # @req> REQ-15984677@POiZGBTxrqb6 ltlklp
+        return {key: data.get(key) for key in ITEM_STAMPED}
+        # @req- ixanwu
+    if entry_key == MEMBERS:
+        return {"entries": entries(data) or {}}
+    held = flagged(data, entry_key)
+    if held is not None:
+        return {"entries": held}
+    fields = entry_at(data, entry_key)
+    if kind_of(uid, data) == "term":
+        return {"definition": (fields or {}).get("definition")}
+    # @req+ REQ-11930677@dH7Pu7k4K4YI a7a4eb
+    block, _, member = str(entry_key).partition(".")
+    if member and held_block(data, block) is not None:
+        return {block: {member: fields}}
+    # @req- a7a4eb
+    return {"entries": {entry_key: fields}}
+
+
+def stamp(item):
+    return stamp_of(item.uid, item.data)
+
+
+# @req> REQ-88203622@wlZ9orsr9c7q zi6xxa
+def citable(item):
+    return kind_of(item.uid, item.data) in CITABLE
+
+
+# @req> REQ-18176935@A6AhLhUZ7MdW r63o2r
+# @req> REQ-34330878@E2ViDxqgA25l ymaadg
+def cited_stamp(item, entry=None):
+    held = stamp(item)
+    if entry or kind_of(item.uid, item.data) in ("parameter", "data"):
+        held = digest([held, stamp_at(item, entry or MEMBERS)])
+    return tag_stamp(held)
+
+
+# @req> REQ-24608716@rrTjNN2Ay0D0 ayt44q
+def cited_fields(uid, data, entry):
+    held = covered(uid, data)
+    if entry or kind_of(uid, data) in ("parameter", "data"):
+        held |= covered(uid, data, entry or MEMBERS)
+    return held
+
+
+def cited_entries(item):
+    if kind_of(item.uid, item.data) != "data":
+        return {}
+    return entries(item.data) or {}
+
+
+# @req> REQ-22755763@lFaaseccSg81 ez4hkv
+def cited_item(store, address):
+    named, entry = split_address(address)
+    item = find(store, named)
+    # @req> REQ-11930677@dH7Pu7k4K4YI bwqnk4
+    if entry is not None and entry not in cited_entries(item):
+        # @req> REQ-30898926@Vu28eWaG_1mE lxj56m
+        raise ReqctlError(f"{address}: {named} is not a data item holding "
+                          f"{entry} -- only an entry of a data item is cited "
+                          "by its name; name a data item that holds it "
+                          "(`reqctl list data`), or cite the item without an entry")
+    return item, entry
+
+
+def tag_stamp(held):
+    return held[:TAG_STAMP]
+
+
+def mapping(data, key):
+    value = data.get(key)
+    if not isinstance(value, dict):
+        return {}
+    return {name: held for name, held in value.items()
+            if isinstance(name, str)}
+
+
+# @req> REQ-73719888@SQkvdemRTgC1 ww7kzv
+def plain(value):
+    if isinstance(value, dict):
+        return ", ".join(f"{name} ({plain(held)})" if isinstance(held, dict)
+                         else f"{name}: {plain(held)}"
+                         for name, held in value.items())
+    if isinstance(value, list):
+        return ", ".join(f"({plain(member)})" if isinstance(member, dict)
+                         else plain(member) for member in value)
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return "null" if value is None else str(value)
+
+
+def _stated(fields, skip, top=()):
+    held = []
+    if not isinstance(fields, dict):
+        return held
+    for key, value in fields.items():
+        if key in skip or key in top:
+            continue
+        if isinstance(value, dict):
+            held += _stated(value, skip)
+        elif isinstance(value, list):
+            held += [str(one) for one in value]
+        else:
+            held.append(str(value))
+    return held
+
+
+def prose(data, skip=STATED_SKIP, item=True):
+    pieces = [str(data.get("text") or data.get("definition") or "")]
+    if item:
+        pieces += _stated(data, skip, ITEM_SKIP)
+    for fields in (entries(data) or {}).values():
+        pieces += _stated(fields, skip)
+    criteria = data.get("acceptance_criteria")
+    for criterion in criteria if isinstance(criteria, list) else []:
+        if isinstance(criterion, dict):
+            pieces += [str(value) for value in criterion.values()]
+        else:
+            pieces.append(str(criterion))
+    return "\n".join(pieces)
+
+
+def prosed(data, paths, resolve, at=(), under=False):
+    under = under or at in paths
+    if isinstance(data, dict):
+        return {key: prosed(value, paths, resolve, at + (str(key),), under)
+                for key, value in data.items()}
+    if isinstance(data, list):
+        return [prosed(value, paths, resolve, at + (str(index),), under)
+                for index, value in enumerate(data)]
+    return resolve(data) if under and isinstance(data, str) else data
+
+
+def references(data):
+    return sorted({uid + rest for uid, rest in PARAM_REF.findall(prose(data))})
+
+
+def concept_references(data):
+    return sorted({address for _, address in CONCEPT_LINK.findall(prose(data))})
+
+
+def unlinked_prose(data):
+    # @req> REQ-96763217@3jJWb99oL6GQ vintgg
+    if data.get("kind") == "data":
+        data = {**data, "entries": {
+            key: {name: value for name, value in fields.items()
+                  if name != "unit" or isinstance(value, dict)}
+            if isinstance(fields, dict) else fields
+            for key, fields in (entries(data) or {}).items()}}
+    # @req> REQ-83702299@kBGzIwAyKDHL et7k4t
+    return PARAM_REF.sub(
+        " ", CONCEPT_LINK.sub(" ", "\n".join(
+            [prose(data, POLICED_SKIP + ("value_type",), item=False)]
+            + _stated(data, POLICED_SKIP + ("value_type", "unit"),
+                      ITEM_SKIP + POLICED_ITEM_SKIP + ENUMERATED))))
+
+
+def dependencies(data):
+    return sorted(
+        set(mapping(data, "relations"))
+        | set(references(data))
+        | set(concept_references(data))
+    )
+
+
+def expand_address(lookup, address):
+    uid, rest = split_address(address)
+    wanted = {uid}
+    target = lookup(uid)
+    held = entries(target) if target is not None else None
+    if held is None or kind_of(uid, target) == "term":
+        return wanted
+    if rest is None:
+        wanted.add(f"{uid}.{MEMBERS}")
+    elif rest == "default":
+        member = target.get("default")
+        if member is not None:
+            wanted.add(f"{uid}.{member}")
+    else:
+        # @req+ REQ-11930677@dH7Pu7k4K4YI ujoctz
+        head, _, tail = rest.partition(".")
+        member = tail.partition(".")[0]
+        wanted.add(f"{uid}.{head}.{member}" if member and held_block(target, head) is not None
+                   else f"{uid}.{head}")
+        # @req- ujoctz
+    return wanted
+
+
+def pin_addresses(lookup, data):
+    wanted = set()
+    for address in dependencies(data):
+        wanted |= expand_address(lookup, address)
+    return sorted(wanted)
+
+
+def data_lookup(store):
+    def lookup(uid):
+        try:
+            return find(store, uid).data
+        except ReqctlError:
+            return None
+    return lookup
+
+
+def is_suspect(store, item, address):
+    pinned = mapping(item.data, "assessed").get(address)
+    if not isinstance(pinned, str):
+        return True
+    uid, rest = split_address(address)
+    return pinned != stamp_at(find(store, uid), rest)
+
+
+def term_word(data):
+    stated = term_fields(data).get("word")
+    return stated.strip() if isinstance(stated, str) and stated.strip() else None
+
+
+def handle_for(word):
+    return re.sub(r"[^a-z0-9]+", "_", str(word).lower()).strip("_")
+
+
+def term_fields(data):
+    sole = next(iter((entries(data) or {}).values()), {})
+    return sole if isinstance(sole, dict) else {}
+
+
+def unclaimed(data):
+    held = term_fields(data).get("unclaimed")
+    return ({str(phrase): str(reason) for phrase, reason in held.items()}
+            if isinstance(held, dict) else {})
+
+
+def concept_at(root, uid, data, rest):
+    if data is None:
+        return None
+    if rest is not None:
+        # @req+ REQ-11930677@dH7Pu7k4K4YI krnbz5
+        head, dot, _ = rest.partition(".")
+        held = None if dot and head not in held_blocks(root, uid, data) else entry_at(data, rest)
+        # @req- krnbz5
+        return None if held is None else (
+            held.get("name"), str(held.get("definition") or "").strip())
+    if kind_of(uid, data) == "term":
+        return (term_word(data),
+                str(term_fields(data).get("definition") or "").strip())
+    return data.get("name"), str(data.get("text") or "").strip()
+
+
+def schema_path(root, name):
+    return Path(root) / "requirements" / "schemas" / f"{name}.schema.yaml"
+
+
+_VALIDATORS: dict[tuple[str, int], Draft202012Validator] = {}
+_HELD: dict[tuple[int, int], tuple[object, Draft202012Validator]] = {}
+
+
+# @req> REQ-21229437@Hp3-AwgxgHMK bzxs2e
+def validator(schema, format_checker=None):
+    seen = (id(schema), id(format_checker))
+    held = _HELD.get(seen)
+    if held is not None and held[0] is schema:
+        return held[1]
+    key = (digest(schema), id(format_checker))
+    if key not in _VALIDATORS:
+        _VALIDATORS[key] = Draft202012Validator(schema, format_checker=format_checker)
+    if len(_HELD) >= 256:
+        _HELD.clear()
+    _HELD[seen] = (schema, _VALIDATORS[key])
+    return _VALIDATORS[key]
+
+
+@functools.lru_cache(maxsize=None)
+def _parsed_schema(path, stamped):
+    # @req+ REQ-19913588@hBNJrpn4vtZe rnou7b
+    try:
+        loader = Loader(path.read_text())
+        try:
+            node = loader.get_single_node()
+            declared = loader.construct_document(node) if node is not None else None
+            # @req> REQ-81761490@4QFne8WaXttt 45ks43
+            folded = _folded(loader, node)
+        finally:
+            loader.dispose()
+    except UnicodeDecodeError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE hcnzso
+        raise ReqctlError(f"unreadable schema {path}: not valid UTF-8 -- save the "
+                          "schema as UTF-8") from error
+    except (OSError, yaml.YAMLError) as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE phwexs
+        raise ReqctlError(f"unreadable schema {path}: {error} -- correct the schema "
+                          "where the parser points") from error
+    except Exception as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE ytojsb
+        raise ReqctlError(f"unreadable schema {path}: a value YAML cannot build, "
+                          f"{type(error).__name__}: {error} -- replace the value "
+                          "with plain YAML") from error
+    # @req- rnou7b
+    # @req> REQ-81761490@4QFne8WaXttt 35mtk4
+    if folded:
+        # @req> REQ-30898926@Vu28eWaG_1mE 5jzbmw
+        raise ReqctlError(f"{path.name} states the key {folded[0]} twice in one mapping "
+                          "-- YAML keeps the last, so drop the one that does not govern")
+    # @req+ REQ-19913588@hBNJrpn4vtZe mszvfu
+    try:
+        for fault in validator(Draft202012Validator.META_SCHEMA,
+                               Draft202012Validator.FORMAT_CHECKER).iter_errors(declared):
+            raise SchemaError.create_from(fault)
+        _refuse_unfollowable(path, DRAFT202012.create_resource(declared))
+    except SchemaError as error:
+        where = ".".join(str(p) for p in error.absolute_path) or "(schema)"
+        # @req> REQ-30898926@Vu28eWaG_1mE 63jzmw
+        raise ReqctlError(f"{path.name}: {where}: {error.message} -- correct the "
+                          "schema at that place") from error
+    except RecursionError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE aml3cg
+        raise ReqctlError(f"{path.name}: nests too deeply to check -- flatten the "
+                          "schema's nesting") from error
+    # @req- mszvfu
+    return declared
+
+
+# @req> REQ-19913588@hBNJrpn4vtZe 4sbgsj
+def _refuse_unfollowable(path, resource):
+    try:
+        walked = list(_walked(resource, ""))
+        if any(specification_with(schema.get("$schema", ""), default=DRAFT202012)
+               is not DRAFT202012
+               or validator_for(schema, Draft202012Validator) is not Draft202012Validator
+               for _, schema in walked[1:]):
+            return
+    except ValueError as error:
+        # @req> REQ-30898926@Vu28eWaG_1mE hme6mh
+        raise ReqctlError(f"{path.name}: an $id or $schema is not an address, "
+                          f"{error} -- make it an address") from error
+    root = resource.id() or ""
+    registry = Registry().with_resource(root, resource)
+    with contextlib.suppress(ValueError):
+        registry = registry.crawl()
+        if registry[root] is not resource:
+            registry = registry.with_resource(root, resource)
+    for base, keyword, ref in sorted({(base, keyword, schema[keyword])
+                                      for base, schema in walked
+                                      for keyword in ("$ref", "$dynamicRef")
+                                      if keyword in schema}):
+        try:
+            address = base if ref.startswith("#") else urldefrag(urljoin(base, ref)).url
+            if address not in registry and urlsplit(address).scheme:
+                continue
+            target = registry.resolver(base).lookup(ref).contents
+        except Exception as error:
+            # @req> REQ-30898926@Vu28eWaG_1mE hwsp72
+            raise ReqctlError(f"{path.name}: {keyword} {ref!r} cannot be followed -- "
+                              "point it at a schema the file holds") from error
+        if not isinstance(target, (dict, bool)):
+            # @req> REQ-30898926@Vu28eWaG_1mE gphyzw
+            raise ReqctlError(f"{path.name}: {keyword} {ref!r} names a value, not a "
+                              "schema -- point it at a schema")
+
+
+def _walked(resource, base):
+    base = urljoin(base, resource.id() or "")
+    if isinstance(resource.contents, dict):
+        yield base, resource.contents
+    for sub in DRAFT202012.subresources_of(resource.contents):
+        yield from _walked(DRAFT202012.create_resource(sub), base)
+
+
+PACKAGED_SCHEMAS = Path(__file__).resolve().parent / "schemas"
+
+
+def packaged_schema_path(name):
+    return PACKAGED_SCHEMAS / f"{name}{SCHEMA_SUFFIX}"
+
+
+def schema(root, name):
+    # @req+ REQ-13148397@Oh5WbBj3LiW0 4fsesd
+    stated = schema_path(root, name)
+    path = stated
+    # @req- 4fsesd
+    # @req> REQ-44451070@AlFXBWPH1jnw idgq2r
+    if not path.is_file():
+        path = packaged_schema_path(name)
+        if not path.is_file():
+            # @req> REQ-30898926@Vu28eWaG_1mE 7oohd6
+            raise ReqctlError(f"missing schema: {stated} -- add the schema there, or "
+                              "remove what refers to it")
+    stat = path.stat()
+    stamped = (stat.st_mtime_ns, stat.st_size)
+    # @req> REQ-14895892@JjrTwHoJqTRe yz3x2d
+    if path == stated and name in SCHEMA_NAMES.values():
+        lacking = _unmarked(path, stamped, name, _data_held(root))
+        if lacking:
+            # @req> REQ-30898926@Vu28eWaG_1mE sa2ijn
+            raise RefusedCopy(f"{stated}: the shipped schema carries {lacking}, "
+                              "which this copy lacks -- add them to the copy "
+                              "by hand from the schema in the installed reqctl "
+                              "package, or delete the copy to use the shipped one")
+    return _parsed_schema(path, stamped)
+
+
+class RefusedCopy(ReqctlError):
+    pass
+
+
+# @req+ REQ-14895892@JjrTwHoJqTRe a252bv
+@functools.lru_cache(maxsize=None)
+def _unmarked(path, stamped, name, data_held):
+    from . import fields
+    shipped = packaged_schema_path(name)
+    stat = shipped.stat()
+    # @req> REQ-77886246@8iAQsnTLrT4C i2zwhw
+    held = _unheld_dropped(_parsed_schema(shipped, (stat.st_mtime_ns, stat.st_size)),
+                           data_held)
+    stated = _parsed_schema(path, stamped)
+    try:
+        return fields.unmarked(
+            held, stated, [kind for kind, named in SCHEMA_NAMES.items() if named == name])
+    except ReqctlError as fault:
+        raise RefusedCopy(f"{path}: {fault}") from fault
+# @req- a252bv
+
+
+ENTRY_OF = "x-entry-of"
+
+
+# @req> REQ-77886246@8iAQsnTLrT4C hghw3m
+def _data_held(root):
+    held = set()
+    for pattern in ("*.yml", "*.yaml"):
+        for path in folder_for(root, "data").glob(pattern):
+            held.add(path.stem)
+            if path.stem.startswith("DATA-"):
+                stat = path.stat()
+                held.add(_data_name(path, (stat.st_mtime_ns, stat.st_size)))
+    held.discard(None)
+    return frozenset(held)
+
+
+@functools.lru_cache(maxsize=None)
+def _data_name(path, stamped):
+    with contextlib.suppress(ReqctlError):
+        data = read(path)
+        return name_of(path.stem, data if isinstance(data, dict) else {})
+    return None
+
+
+def _unheld_dropped(node, data_held):
+    if isinstance(node, dict):
+        return {key: _unheld_dropped(value, data_held) for key, value in node.items()
+                if not (key == ENTRY_OF and value not in data_held)}
+    if isinstance(node, list):
+        return [_unheld_dropped(one, data_held) for one in node]
+    return node
+
+
+SCHEMA_NAMES = {"requirement": "requirement", "guard": "guard",
+                "parameter": "dictionary", "term": "dictionary",
+                "data": "dictionary"}
+SCHEMA_SUFFIX = ".schema.yaml"
+RESERVED_SCHEMAS = frozenset(SCHEMA_NAMES.values()) | {"baseline"}
+
+
+def schema_names(root):
+    folder = Path(root) / "requirements" / "schemas"
+    return sorted(path.name[: -len(SCHEMA_SUFFIX)]
+                  for path in folder.glob(f"*{SCHEMA_SUFFIX}"))
+
+
+def schema_name_for(kind):
+    if kind not in SCHEMA_NAMES:
+        # @req> REQ-30898926@Vu28eWaG_1mE okae4a
+        raise ReqctlError(f"{kind}: not a {KINDS_NAMED} kind -- use one of those kinds")
+    return SCHEMA_NAMES[kind]
+
+
+def schema_name_at(root, kind, name=None):
+    if (name is not None and name not in RESERVED_SCHEMAS
+            and schema_path(root, name).is_file()):
+        return name
+    return schema_name_for(kind)
+
+
+def schema_for(root, kind, name=None):
+    return schema(root, schema_name_at(root, kind, name))
+
+
+BINDING = "x-binding"
+ADDRESSABLE = "x-addressable"
+
+
+def addressable_blocks(root, uid, data):
+    if kind_of(uid, data) != "data":
+        return frozenset()
+    from . import fields
+    held = schema_for(root, "data", name_of(uid, data))
+    return frozenset(name for name, stated in fields._merged(
+        held if isinstance(held, dict) else {}, "data")[0].items()
+        if name not in ITEM_SKIP and stated.get(ADDRESSABLE) is True)
+
+
+def held_blocks(root, uid, data):
+    held = {name: block for name in data if (block := held_block(data, name)) is not None}
+    marked = addressable_blocks(root, uid, data) if held else ()
+    return {name: held[name] for name in sorted(marked) if name in held}
+
+
+def named_schemas(root):
+    return [name for name in schema_names(root)
+            if name not in RESERVED_SCHEMAS]
+
+
+def binding_dimensions(root):
+    # @req> REQ-68873210@e7MRMGqsYrnx up7le5
+    return [name for name in named_schemas(root)
+            if schema(root, name).get(BINDING) is True]
+
+
+def is_test(relative_path):
+    return bool(TEST_PATH.search(relative_path))