reqctl 0.1.38__py3-none-any.whl

reqctl/fields.py

@@ -0,0 +1,361 @@
+import argparse
+import copy
+import re
+from dataclasses import dataclass
+
+from . import corpus
+from .corpus import ReqctlError
+
+KIND = "kind"
+ENTRIES = "entries"
+
+
+def boolean(text):
+    if text not in ("true", "false"):
+        # @req> REQ-30898926@Vu28eWaG_1mE kyzn7q
+        raise argparse.ArgumentTypeError(
+            f"{text!r} is not a boolean -- pass true or false")
+    return text == "true"
+
+
+CONVERTED = {"integer": int, "number": float, "boolean": boolean}
+PLAIN = ("string", *CONVERTED)
+
+
+def _dest(flag):
+    return flag[2:].replace("-", "_")
+
+
+def _flag(word):
+    return "--" + word.replace("_", "-")
+
+
+@dataclass(frozen=True)
+class Flag:
+    flag: str
+    choices: tuple
+    repeated: bool
+    switch: bool
+    convert: object
+    metavar: str | None
+    entries: bool
+
+    @property
+    def dest(self):
+        return _dest(self.flag)
+
+
+@dataclass(frozen=True)
+class Field:
+    name: str
+    flag: str
+    choices: tuple
+    repeated: bool
+    required: bool
+    clears: str | None
+    drops: str | None
+    empty: object
+    address: bool
+    entry: bool
+    textual: bool
+    prose: bool
+    convert: object
+    parts: tuple
+
+    @property
+    def dest(self):
+        return _dest(self.flag)
+
+    @property
+    def cleared(self):
+        return _dest(self.clears) if self.clears else None
+
+    @property
+    def dropped(self):
+        return _dest(self.drops) if self.drops else None
+
+
+def root():
+    try:
+        return corpus.find_root()
+    except ReqctlError:
+        return "."
+
+
+def _schema(root, kind):
+    held = corpus.schema_for(root, kind)
+    return held if isinstance(held, dict) else {}
+
+
+def _pointed(schema, ref):
+    if not ref.startswith("#/"):
+        return None
+    target = schema
+    for step in ref[2:].split("/"):
+        step = step.replace("~1", "/").replace("~0", "~")
+        if isinstance(target, list) and step.isdecimal() and int(step) < len(target):
+            target = target[int(step)]
+        else:
+            target = target.get(step) if isinstance(target, dict) else None
+    return target
+
+
+def _resolved(schema, node):
+    if not isinstance(node, dict):
+        return {}
+    named = schema.get("$id", "a kind schema")
+    seen = set()
+    while "$ref" in node:
+        ref = node["$ref"]
+        if ref in seen:
+            # @req> REQ-30898926@Vu28eWaG_1mE njzgy6
+            raise ReqctlError(f"{named}: {ref} refers back to itself -- break the "
+                              "self-reference in the schema")
+        target = _pointed(schema, ref)
+        if isinstance(target, bool):
+            target = {} if target else {"not": {}}
+        if not isinstance(target, dict):
+            # @req> REQ-30898926@Vu28eWaG_1mE uzmekp
+            raise ReqctlError(f"{named}: {ref} names nothing within the schema -- "
+                              "point it at a property the schema holds")
+        seen.add(ref)
+        node = {**target, **{key: value for key, value in node.items()
+                             if key != "$ref"}}
+    return node
+
+
+def _forbidden(schema, stated):
+    return stated is False or _resolved(schema, stated).get("not") in ({}, True)
+
+
+def _selects(schema, test, kind):
+    if set(test) - {"properties", "required"}:
+        return False
+    if set(test.get("required", ())) - {KIND}:
+        return False
+    tested = test.get("properties")
+    if not isinstance(tested, dict) or set(tested) != {KIND}:
+        return False
+    said = _resolved(schema, tested[KIND])
+    return said.get("const") == kind or kind in said.get("enum", ())
+
+
+def _merged(schema, kind):
+    properties, required, barred = {}, set(), set()
+    branches = [_resolved(schema, item) for item in schema.get("allOf", ())]
+    chosen = [_resolved(schema, branch.get("then")) for branch in branches
+              if _selects(schema, _resolved(schema, branch.get("if")), kind)]
+    for block in [schema] + branches + chosen:
+        required.update(block.get("required", ()))
+        for name, stated in (block.get("properties") or {}).items():
+            if _forbidden(schema, stated):
+                barred.add(name)
+            else:
+                properties[name] = {**properties.get(name, {}),
+                                    **_resolved(schema, stated)}
+    return {name: stated for name, stated in properties.items()
+            if name not in barred}, required
+
+
+def _prose(node):
+    if isinstance(node, dict):
+        return node.get("x-prose") is True or any(map(_prose, node.values()))
+    if isinstance(node, list):
+        return any(map(_prose, node))
+    return False
+
+
+def _field(schema, name, stated, required, entry):
+    said = stated.get("x-flag", name)
+    if not isinstance(said, str) or not corpus.DATA_KEY.fullmatch(said):
+        # @req> REQ-30898926@Vu28eWaG_1mE 4syjdu
+        raise ReqctlError(f"{schema.get('$id', 'a kind schema')}: {name} would "
+                          f"take the flag {said!r}, which is not a snake_case word -- "
+                          "give the field a snake_case x-flag")
+    shape = stated.get("type")
+    repeated = shape in ("array", "object")
+    items = _resolved(schema, stated.get("items", {})) if shape == "array" else {}
+    typed = items.get("type") if shape == "array" else shape
+    convert = CONVERTED.get(typed) if isinstance(typed, str) else None
+    parts = tuple(items.get("required", ())) if items.get("type") == "object" else ()
+    values = (_resolved(schema, stated.get("additionalProperties", {}))
+              if shape == "object" else {})
+    drops = _flag("drop_" + said) if values.get("type") in PLAIN else None
+    enum = stated.get("enum", ())
+    choices = tuple(enum) if all(isinstance(one, str) for one in enum) else ()
+    clears, empty = None, None
+    if not required:
+        clears = _flag("no_" + said)
+    elif repeated and not stated.get("minItems") and not stated.get("minProperties"):
+        clears, empty = _flag("no_" + said), list if shape == "array" else dict
+    return Field(name=name, flag=_flag(said), choices=choices,
+                 repeated=repeated,
+                 required=required and (entry or "default" not in stated),
+                 clears=clears, drops=drops, empty=empty,
+                 address=stated.get("x-address") is True, entry=entry,
+                 textual=(convert is None and "enum" not in stated
+                          and shape != "object" and not parts),
+                 prose=_prose(stated), convert=convert, parts=parts)
+
+
+def of(root, kind):
+    schema = _schema(root, kind)
+    properties, required = _merged(schema, kind)
+    held = []
+    for name, stated in properties.items():
+        # @req> REQ-54521322@g8kfVhS7XXWx 3acfqk
+        if stated.get("readOnly") is True:
+            continue
+        if name == ENTRIES and stated.get("maxProperties") == 1:
+            entry = _resolved(schema, stated.get("additionalProperties", {}))
+            wanted = set(entry.get("required", ()))
+            # @req> REQ-54521322@g8kfVhS7XXWx rd4w24
+            held += [_field(schema, key, _resolved(schema, value), key in wanted, True)
+                     for key, value in (entry.get("properties") or {}).items()
+                     if not _forbidden(schema, value)
+                     and _resolved(schema, value).get("readOnly") is not True]
+            continue
+        held.append(_field(schema, name, stated, name in required, False))
+    taken = {}
+    for field in held:
+        for flag in filter(None, (field.flag, field.clears, field.drops)):
+            if flag in taken:
+                # @req> REQ-30898926@Vu28eWaG_1mE 3xifyq
+                raise ReqctlError(f"{kind}: {taken[flag]} and {field.name} would "
+                                  f"both take {flag}; give one an x-flag")
+            taken[flag] = field.name
+    return held
+
+
+def names(root, kind):
+    return set(_merged(_schema(root, kind), kind)[0])
+
+
+# @req> REQ-41779958@i5ZjNQdrVe4B 4aox7m
+def defaults(root, kind):
+    properties, _ = _merged(_schema(root, kind), kind)
+    held = {name: copy.deepcopy(stated["default"])
+            for name, stated in properties.items() if "default" in stated}
+    if KIND in properties:
+        held[KIND] = kind
+    return held
+
+
+def flags(root):
+    held = {}
+    for kind in corpus.SCHEMA_NAMES:
+        for field in of(root, kind):
+            stated = held.get(field.flag, field)
+            if (stated.repeated, stated.convert) != (field.repeated, field.convert):
+                # @req> REQ-30898926@Vu28eWaG_1mE me6yhq
+                raise ReqctlError(f"{field.flag} takes a different kind of value "
+                                  f"for a {kind}; give one field an x-flag")
+            choices = (tuple(dict.fromkeys(stated.choices + field.choices))
+                       if stated.choices and field.choices else ())
+            metavar = f"'{' | '.join(field.parts).upper()}'" if field.parts else None
+            held[field.flag] = Flag(field.flag, choices, field.repeated, False,
+                                    field.convert, metavar, field.name == ENTRIES)
+    return list(held.values())
+
+
+def clears(root):
+    held = {}
+    for kind in corpus.SCHEMA_NAMES:
+        for field in of(root, kind):
+            if field.clears:
+                held[field.clears] = Flag(field.clears, (), False, True, None, None, False)
+            if field.drops:
+                held[field.drops] = Flag(field.drops, (), True, False, None, "KEY", False)
+    return list(held.values())
+
+
+# @req+ REQ-14895892@JjrTwHoJqTRe aepkss
+def _expanded(schema, nodes, kind):
+    seen = set()
+
+    def facets(node, read, sure):
+        if (id(node), read, sure) in seen or not isinstance(node, dict):
+            return
+        seen.add((id(node), read, sure))
+        if "$ref" in node:
+            yield from facets(_resolved(schema, {"$ref": node["$ref"]}), read, sure)
+        yield node, read, sure
+        for key in ("allOf", "anyOf", "oneOf"):
+            for member in node.get(key, ()):
+                yield from facets(member, read and key == "allOf", sure and key == "allOf")
+        if "if" in node:
+            kinds = {one for one in corpus.SCHEMA_NAMES
+                     if _selects(schema, _resolved(schema, node["if"]), one)}
+            for branch, applies in (("then", not kinds or kind in kinds),
+                                    ("else", kind not in kinds)):
+                if applies:
+                    yield from facets(node.get(branch), read, sure and bool(kinds))
+
+    return [one for node, read, sure in nodes for one in facets(node, read, sure)]
+
+
+def _children(schema, facets, name):
+    found = []
+    for node, read, sure in facets:
+        named = node.get("properties") or {}
+        if name in named:
+            found.append((named[name], read, sure))
+        found += [(value, read, sure) for pattern, value in
+                  (node.get("patternProperties") or {}).items()
+                  if isinstance(pattern, str) and re.search(pattern, name)]
+    return [] if any(sure and _forbidden(schema, value) for value, _, sure in found) else found
+
+
+def _lacking(shipped, copy, held, stated, at, kind, seen=frozenset()):
+    pair = tuple(frozenset((id(node), read, sure) for node, read, sure in nodes)
+                 for nodes in (held, stated))
+    if pair in seen or not (held and stated):
+        return
+    seen |= {pair}
+    carried = {marker for node, read, _ in stated if read for marker in _resolved(copy, node)}
+    yield from (f"{marker} on {at}" for node, read, _ in held if read
+                for marker in _resolved(shipped, node).keys() - carried
+                if marker.startswith("x-") or marker in ("default", "readOnly"))
+    held, stated = _expanded(shipped, held, kind), _expanded(copy, stated, kind)
+    for name in {name for node, _, _ in held for name in node.get("properties") or {}}:
+        yield from _lacking(shipped, copy, _children(shipped, held, name),
+                            _children(copy, stated, name),
+                            f"{at}.{name}" if at else name, kind, seen)
+    for step in ("items", "additionalProperties"):
+        yield from _lacking(shipped, copy, *([(node[step], read, sure) for node, read, sure in facets
+                                              if isinstance(node.get(step), dict)]
+                                             + [(value, read, False) for node, read, _ in facets
+                                                if step == "additionalProperties"
+                                                for value in
+                                                (node.get("patternProperties") or {}).values()]
+                                             for facets in (held, stated)), at, kind, seen)
+
+
+def _lacking_plain(shipped, copy, held, stated, at, seen=frozenset()):
+    pair = (id(held), id(stated))
+    if pair in seen:
+        return
+    seen |= {pair}
+    held, stated = _resolved(shipped, held), _resolved(copy, stated)
+    yield from (f"{marker} on {at}" for marker in held.keys() - stated.keys()
+                if marker.startswith("x-") or marker in ("default", "readOnly"))
+    inner = stated.get("properties") or {}
+    for name, value in (held.get("properties") or {}).items():
+        if name in inner and not _forbidden(copy, inner[name]):
+            yield from _lacking_plain(shipped, copy, value, inner[name], f"{at}.{name}", seen)
+    for step in ("items", "additionalProperties"):
+        if isinstance(held.get(step), dict) and isinstance(stated.get(step), dict):
+            yield from _lacking_plain(shipped, copy, held[step], stated[step], at, seen)
+
+
+def unmarked(shipped, copy, kinds):
+    found = set()
+    for kind in kinds:
+        found.update(_lacking(shipped, copy, [(shipped, True, True)], [(copy, True, True)],
+                              "", kind))
+        held = _merged(shipped, kind)[0]
+        stated = _merged(copy if isinstance(copy, dict) else {}, kind)[0]
+        for name in held.keys() & stated.keys():
+            found.update(_lacking_plain(shipped, copy, held[name], stated[name], name))
+    return ", ".join(sorted(found))
+# @req- aepkss

reqctl/graph.py

@@ -0,0 +1,207 @@
+from pathlib import Path
+
+from . import cite, corpus, history, uncited
+
+
+def unstamped(uid, path):
+    return (f"{uid}: {path} cites it without the stamp it was written "
+            "against -- cite it again with reqctl tag")
+
+
+def listing(tree, citations, uid=None):
+    named = {(citation["uid"], citation["id"]) for citation in citations}
+    # @req> REQ-77594104@x3D--4236Dno 7nucmz
+    listed = {str(item.uid): corpus.mapping(item.data, corpus.CITATION_LIST)
+              for item in corpus.items(tree)
+              if corpus.citable(item)}
+    problems = []
+    # @req> REQ-13384695@P9NXsZKZp683 3lmvqm
+    for owner, held in listed.items():
+        if uid and owner != uid:
+            continue
+        for identity in sorted(i for i in held if (owner, i) not in named):
+            problems.append(
+                f"{owner}: its citation list holds {identity}, which no "
+                f"statement citation names together with {owner} -- "
+                f"reqctl unlist {owner} {identity}")
+    for citation in citations:
+        owner, identity = citation["uid"], citation["id"]
+        if (uid and owner != uid) or owner not in listed:
+            continue
+        # @req+ REQ-67655319@ezp6TxUzJ72E wqruf3
+        # @req+ REQ-32191310@aZkvQ6hvIwPv pmll7c
+        if identity not in listed[owner]:
+            problems.append(
+                f"{citation['path']}: citation {identity} of {owner} is not in "
+                f"its citation list -- read it against {owner}, then reqctl "
+                f"repin {identity}")
+        elif listed[owner][identity] != citation["digest"]:
+            problems.append(
+                f"{citation['path']}: citation {identity} of {owner} covers "
+                "lines that changed since its citation list took their digest "
+                f"-- read them against {owner}, then reqctl repin {identity}")
+        # @req- pmll7c
+        # @req- wqruf3
+    return problems
+
+
+# @req> REQ-40478019@_dwqsOpSOQXA bkbrwo
+def whole(tree, tags, root=None, uid=None):
+    problems = []
+    for item in corpus.items(tree):
+        owner = str(item.uid)
+        if not corpus.citable(item):
+            continue
+        for path, held in corpus.mapping(item.data, corpus.WHOLE_FILES).items():
+            held = held if isinstance(held, dict) else {}
+            # @req> REQ-19896380@lvCPXiVYZ6ud lbb72g
+            if not corpus.is_test(path):
+                tags.setdefault(owner, []).append((path, held.get("pinned") or "", None))
+            if root is None or (uid and owner != uid):
+                continue
+            # @req> REQ-80254000@aV5K4B7jbtHD sb54ru
+            if corpus.is_test(path):
+                problems.append(f"{path}: the whole-file citation {owner} holds is of "
+                                f"a test -- reqctl untag {path} --req {owner}")
+                continue
+            target = (Path(root) / path).resolve()
+            if not target.is_relative_to(Path(root).resolve()):
+                # @req> REQ-30898926@Vu28eWaG_1mE plek4e
+                problems.append(f"{owner}: it holds a whole-file citation of {path}, "
+                                "which is not a file of the code base -- edit "
+                                f"{owner}'s whole_file_citations by hand to "
+                                f"drop {path}")
+                continue
+            try:
+                text = target.read_bytes().decode()
+            except (OSError, UnicodeDecodeError):
+                # @req> REQ-30898926@Vu28eWaG_1mE kf3fxw
+                problems.append(f"{owner}: it holds a whole-file citation of {path}, "
+                                "which does not read -- make the file readable "
+                                f"(fix its permissions), or reqctl untag {path} "
+                                f"--req {owner}")
+                continue
+            if held.get("digest") != cite.whole(path, text):
+                problems.append(
+                    f"{path}: the whole-file citation {owner} holds covers a file "
+                    f"that changed since it took its digest -- read it against "
+                    f"{owner}, then reqctl tag {path} --req {owner}")
+    return problems
+
+
+def trace(tree, root, uid=None, base=None, record=False):
+    citations, cited_problems = cite.read(root, digested=True, uid=uid)
+    # @req> REQ-21635723@CvxpeHCRTgGz qwvqiw
+    if record:
+        uncited.record(tree, root, citations)
+    # @req+ REQ-19896380@lvCPXiVYZ6ud 6rtw6m
+    shown = [c for c in citations if not corpus.is_test(c["path"])]
+    tags = cite.cited(shown)
+    # @req- 6rtw6m
+    # @req> REQ-40478019@_dwqsOpSOQXA rvd6f2
+    cited_problems += whole(tree, tags, root, uid)
+    # @req> REQ-27534660@OcqGUyoB-NOR bc223j
+    # @req> REQ-64570886@Zckz3LzHUzXn fv2hvw
+    if not uid:
+        cited_problems += uncited.problems(tree, root, citations, base)
+    # @req> REQ-76962559@NU0P43-PPSGC dyday2
+    if uid:
+        corpus.find(tree, uid)
+    known = {str(item.uid) for item in corpus.items(tree)}
+    past = history.History(root)
+
+    rows, problems, unimplemented, stale, deprecated = [], list(cited_problems), [], [], []
+    # @req> REQ-19896380@lvCPXiVYZ6ud aes6o5
+    # @req> REQ-52825760@f9HMlzmiwhyn 5aendb
+    for citation in citations:
+        if corpus.is_test(citation["path"]) and (not uid or citation["uid"] == uid):
+            problems.append(
+                f"{citation['path']}: citation {citation['id']} of "
+                f"{citation['uid']} is in a test -- reqctl untag {citation['id']}")
+    for item in corpus.items(tree):
+        current = str(item.uid)
+        # @req> REQ-64570886@Zckz3LzHUzXn jzenja
+        if uid and current != uid:
+            continue
+        data = corpus.raw(item)
+        cited = tags.get(current, [])
+        files = list(dict.fromkeys(path for path, _, _ in cited))
+        rows.append(
+            {
+                "uid": current,
+                "status": data.get("status"),
+                "implementation": files,
+            }
+        )
+        if files and not corpus.citable(item):
+            # @req> REQ-30898926@Vu28eWaG_1mE 4urczb
+            problems.append(
+                f"{current}: referenced by {', '.join(files)} -- only a "
+                "requirement, a guard, a parameter or a data item is cited "
+                f"-- reqctl untag each citation of {current}, then reqctl tag "
+                "the requirement stated against it"
+            )
+        # @req> REQ-35979865@pd7CVJ7isMys iz2v2g
+        # @req> REQ-77594104@x3D--4236Dno tizjds
+        if (corpus.citable(item)
+                and data.get("status") == "deprecated" and cited):
+            deprecated += [{"uid": current, "path": path} for path
+                           in dict.fromkeys(p for p, pinned, _ in cited if pinned)]
+            problems += [unstamped(current, path)
+                         for path, pinned, _ in cited if not pinned]
+            continue
+        if data.get("status") != "approved":
+            if files:
+                # @req> REQ-30898926@Vu28eWaG_1mE 6ddckc
+                problems.append(
+                    f"{current}: referenced by {', '.join(files)} but is "
+                    f"{data.get('status')}, not approved -- approve it through "
+                    f"a corpus pull request, or reqctl untag each citation of "
+                    f"{current}"
+                )
+            continue
+        # @req> REQ-77594104@x3D--4236Dno lgfhiu
+        if corpus.citable(item):
+            for path, pinned, entry in cited:
+                # @req+ REQ-74122607@f-RdDZCNkgqC pbsx4y
+                address = f"{current}.{entry}" if entry else current
+                if entry and entry not in corpus.cited_entries(item):
+                    # @req> REQ-30898926@Vu28eWaG_1mE fv4ej6
+                    problems.append(
+                        f"{address}: referenced by {path} but no such entry -- "
+                        f"reqctl untag the citation, or add the entry with "
+                        f"reqctl revise {current} --new-entry {entry}")
+                    continue
+                held = corpus.cited_stamp(item, entry)
+                # @req- pbsx4y
+                # @req> REQ-75161909@GUQmRsLIBj6Z 7df3my
+                if not pinned:
+                    problems.append(unstamped(address, path))
+                elif pinned != held:
+                    # @req> REQ-24608716@rrTjNN2Ay0D0 ofe7e3
+                    stale.append({"uid": address, "path": path,
+                                  "pinned": pinned, "held": held,
+                                  **past.cited(current, data, entry, pinned)})
+        # @req> REQ-73027720@2Z15jgYQYfsS 3aqkc5
+        if current.startswith(("REQ-", "GUARD-")) and not files:
+            unimplemented.append(current)
+
+    if not uid:
+        for tagged in sorted(tags):
+            if tagged not in known:
+                # @req> REQ-30898926@Vu28eWaG_1mE li6jfv
+                problems.append(
+                    f"{tagged}: referenced by "
+                    f"{', '.join(dict.fromkeys(p for p, _, _ in tags[tagged]))}"
+                    " but no such requirement -- reqctl untag each citation "
+                    f"of {tagged}, or mint it with reqctl new KIND --uid "
+                    f"{tagged}"
+                )
+    # @req> REQ-67655319@ezp6TxUzJ72E eznv3q
+    # @req> REQ-13384695@P9NXsZKZp683 qtu6ly
+    # @req> REQ-32191310@aZkvQ6hvIwPv 65dur7
+    problems += listing(tree, citations, uid)
+
+    return {"rows": rows, "problems": problems, "stale": stale,
+            "deprecated": deprecated, "unimplemented": sorted(unimplemented),
+            "citations": [c for c in shown if not uid or c["uid"] == uid]}