codemeta-drift 0.1.0__py3-none-any.whl

codemeta_drift/__init__.py

@@ -0,0 +1,4 @@
+"""Report where CodeMeta's context, definitions, property table, examples and crosswalks disagree.
+
+Also export the crosswalks as a JSON mapping.
+"""

codemeta_drift/checks.py

@@ -0,0 +1,243 @@
+"""Find where CodeMeta's context, definitions, property table, examples and crosswalks disagree.
+
+It uses only the standard library and imports nothing from this package, so the
+codemeta repository can copy it as is for a CI check.
+
+Every check reads files from a mapping of repository-relative path to text. When
+a check's input file is missing, it reports ``SKIPPED`` instead of failing, so the
+same code runs on every CodeMeta release.
+"""
+
+import csv
+import difflib
+import io
+import json
+from collections import Counter
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+
+CONTEXT = "codemeta.jsonld"
+DEFINITIONS = "codemeta_all.jsonld"
+TABLE = "properties_description.csv"
+AGGREGATE = "crosswalk.csv"
+
+VALID_EXAMPLES = ("codemeta.json", "examples/codemeta.json", "examples/codemeta-software.json")
+"""Files that should only use terms of the CodeMeta context. The other files in
+examples/ use the schema.org context or are invalid on purpose."""
+
+NOT_PROPERTIES = frozenset({"type", "id", "schema", "codemeta"})
+"""Context keys that are keyword aliases or prefixes, not properties."""
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One disagreement between two CodeMeta files.
+
+    Attributes:
+        code: Stable identifier of the rule, such as ``TABLE_NO_TERM``.
+        file: The file (or group of crosswalk files) where it shows.
+        term: The property or format it is about.
+        detail: What disagrees, for a human reader.
+    """
+
+    code: str
+    file: str
+    term: str
+    detail: str
+
+    def __str__(self) -> str:
+        return f"{self.code} {self.file}: {self.term}: {self.detail}"
+
+
+type Row = tuple[int, list[str]]
+"""A CSV record and the file line it starts on."""
+
+
+def csv_rows(text: str, key: int = 0) -> list[Row]:
+    """Records of a CSV file with a value in column ``key``, each with its first line number."""
+    reader = csv.reader(io.StringIO(text))
+    rows, line = [], 1
+    for row in reader:
+        if len(row) > key and row[key]:
+            rows.append((line, row))
+        line = reader.line_num + 1
+    return rows
+
+
+def context_terms(files: Mapping[str, str]) -> dict[str, str]:
+    """Maps each property term of the context to its prefixed IRI."""
+    context = json.loads(files[CONTEXT])["@context"]
+    terms = {}
+    for key, value in context.items():
+        iri = value.get("@id") if isinstance(value, dict) else value
+        is_class = not key[:1].islower()  # Person, Organization...
+        is_prefix = isinstance(iri, str) and iri.endswith(("/", "#"))  # schema, spdx...
+        if key in NOT_PROPERTIES or is_class or is_prefix or not isinstance(iri, str):
+            continue
+        terms[key] = iri
+    return terms
+
+
+def table_rows(files: Mapping[str, str]) -> list[Row]:
+    """Property table records with a property name, header dropped."""
+    return csv_rows(files[TABLE], key=1)[1:]
+
+
+def crosswalk_paths(files: Mapping[str, str]) -> list[str]:
+    """Paths of the per-format crosswalk files, sorted."""
+    return sorted(p for p in files if p.startswith("crosswalks/") and p.endswith(".csv"))
+
+
+def format_name(crosswalk: str) -> str:
+    """The format a crosswalk file maps to: the second cell of its header, as in "Property,Zenodo"."""
+    (_, header), *_ = csv_rows(crosswalk)
+    return header[1]
+
+
+def used_terms(node: object, found: set[str]) -> set[str]:
+    """Collects every non-keyword key of a JSON document, outside @context."""
+    if isinstance(node, dict):
+        for key, value in node.items():
+            if key == "@context":
+                continue
+            if not key.startswith("@"):
+                found.add(key)
+            used_terms(value, found)
+    elif isinstance(node, list):
+        for value in node:
+            used_terms(value, found)
+    return found
+
+
+def check_table(files: Mapping[str, str]) -> Iterator[Finding]:
+    """Property table against itself and against the context."""
+    rows = [(row[0], row[1]) for _, row in table_rows(files)]
+    for (parent, name), count in sorted(Counter(rows).items()):
+        if count > 1:
+            yield Finding("TABLE_DUPLICATE", TABLE, name, f"{parent},{name} appears {count} times")
+    terms = context_terms(files)
+    names = {name for _, name in rows}
+    for name in sorted(names - terms.keys()):
+        yield Finding("TABLE_NO_TERM", TABLE, name, f"row has no term in {CONTEXT}")
+    for name in sorted(terms.keys() - names):
+        yield Finding("TERM_NO_ROW", CONTEXT, name, f"term has no row in {TABLE}")
+    for parent, name in rows:
+        prefix = parent.split(":")[0]
+        iri = terms.get(name)
+        if iri is not None and not iri.startswith(prefix + ":"):
+            yield Finding("TABLE_NAMESPACE", TABLE, name, f"row {parent},{name} but {CONTEXT} maps {name} to {iri}")
+
+
+def check_definitions(files: Mapping[str, str]) -> Iterator[Finding]:
+    """Property definitions against the context."""
+    terms = context_terms(files)
+    graph = json.loads(files[DEFINITIONS])["@graph"]
+
+    def is_property(node: dict) -> bool:
+        kind = node.get("@type")
+        return "rdf:Property" in (kind if isinstance(kind, list) else [kind])
+
+    for node in sorted(filter(is_property, graph), key=lambda n: n["@id"]):
+        label, iri = node.get("rdfs:label", ""), node["@id"]
+        if label not in terms:
+            alias = [t for t, i in terms.items() if i == iri]
+            hint = f"; {CONTEXT} has {iri} as {alias[0]}" if alias else ""
+            yield Finding("DEFINITION_NO_TERM", DEFINITIONS, label, f"defines {iri} but {CONTEXT} has no {label}{hint}")
+        elif terms[label] != iri:
+            yield Finding(
+                "DEFINITION_IRI", DEFINITIONS, label, f"defines {iri} but {CONTEXT} maps it to {terms[label]}"
+            )
+
+
+def check_examples(files: Mapping[str, str]) -> Iterator[Finding]:
+    """Valid examples use only context terms."""
+    terms = context_terms(files)
+    for path in VALID_EXAMPLES:
+        if path not in files:
+            continue
+        keys = used_terms(json.loads(files[path]), set())
+        for key in sorted(k for k in keys if ":" not in k and k not in terms and k not in NOT_PROPERTIES):
+            yield Finding("EXAMPLE_NO_TERM", path, key, f"not a term of {CONTEXT}")
+
+
+def column_diff(code: str, label: str, column: list[tuple[int, str]], table: list[Row]) -> Iterator[Finding]:
+    """Where a property column departs from the table's, matched by name, not by position.
+
+    ``column`` holds (file line, property) pairs of the checked file.
+    """
+    names, wanted = [name for _, name in column], [row[1] for _, row in table]
+    matcher = difflib.SequenceMatcher(a=names, b=wanted, autojunk=False)
+    for op, a1, a2, b1, b2 in matcher.get_opcodes():
+        if op in {"delete", "replace"}:
+            # Name the table row only for a one-for-one swap, where it is the likely rename.
+            instead = f"; the table has {wanted[b1]} there" if op == "replace" and a2 - a1 == b2 - b1 == 1 else ""
+            for line, name in column[a1:a2]:
+                yield Finding(code, label, name, f"line {line} is not in {TABLE}{instead}")
+        if op in {"insert", "replace"}:
+            for line, row in table[b1:b2]:
+                yield Finding(code, label, row[1], f"no row for {TABLE} line {line}")
+
+
+def check_crosswalks(files: Mapping[str, str]) -> Iterator[Finding]:
+    """Each crosswalk file lists the table's properties in the table's order."""
+    table = table_rows(files)
+    paths = crosswalk_paths(files)
+    # Crosswalks with the same property column share one set of findings.
+    groups: dict[tuple[tuple[int, str], ...], list[str]] = {}
+    for path in paths:
+        column = tuple((line, row[0]) for line, row in csv_rows(files[path])[1:])
+        groups.setdefault(column, []).append(path)
+    for column, members in groups.items():
+        label = f"crosswalks/*.csv (all {len(paths)})" if len(members) == len(paths) > 1 else ", ".join(members)
+        yield from column_diff("CROSSWALK_ROWS", label, list(column), table)
+
+
+AGGREGATE_FIXED_COLUMNS = 4
+"""crosswalk.csv starts with Parent Type, Property, Type and Description; formats follow."""
+
+
+def check_aggregate(files: Mapping[str, str]) -> Iterator[Finding]:
+    """The committed crosswalk.csv has every format and the table's properties."""
+    (_, header), *rows = csv_rows(files[AGGREGATE], key=1)
+    yield from column_diff("AGGREGATE_ROWS", AGGREGATE, [(line, row[1]) for line, row in rows], table_rows(files))
+    paths = crosswalk_paths(files)
+    if not paths:
+        return
+    formats: dict[str, list[str]] = {}
+    for path in paths:
+        formats.setdefault(format_name(files[path]), []).append(path)
+    columns = set(header[AGGREGATE_FIXED_COLUMNS:])
+    for name, members in sorted(formats.items()):
+        if len(members) > 1:
+            yield Finding("CROSSWALK_FORMAT", ", ".join(members), name, "files share one format name")
+        if name not in columns:
+            yield Finding("AGGREGATE_FORMAT", AGGREGATE, name, f"no column for {', '.join(members)}")
+    for name in sorted(columns - formats.keys()):
+        yield Finding("AGGREGATE_FORMAT", AGGREGATE, name, "column has no file in crosswalks/")
+
+
+CHECKS = (
+    (check_table, (CONTEXT, TABLE)),
+    (check_definitions, (CONTEXT, DEFINITIONS)),
+    (check_examples, (CONTEXT,)),
+    (check_crosswalks, (TABLE,)),
+    (check_aggregate, (TABLE, AGGREGATE)),
+)
+
+
+def check(files: Mapping[str, str]) -> list[Finding]:
+    """Every finding for one snapshot of the codemeta repository, SKIPPED ones first.
+
+    Raises ValueError, KeyError, IndexError, TypeError or AttributeError when an input file is
+    malformed (not JSON, not UTF-8, an unexpected shape); a caller must not read that as drift.
+    """
+    findings: list[Finding] = []
+    skipped: set[str] = set()
+    for run, needs in CHECKS:
+        gaps = {p for p in needs if p not in files}
+        if gaps:
+            skipped |= gaps
+            continue
+        findings.extend(run(files))
+    reason = "file not present, checks that need it did not run"
+    return [*(Finding("SKIPPED", path, "", reason) for path in sorted(skipped)), *findings]

codemeta_drift/cli.py

@@ -0,0 +1,166 @@
+"""Command line: ``codemeta-drift <checkout> [--ref REF] [--json]``.
+
+Exit status 0 means no drift, 1 means at least one finding (SKIPPED ones do
+not count), 2 means bad arguments, an unreadable or malformed checkout, or one
+without the context and the property table.
+
+``codemeta-drift export [checkout] --tag TAG [--out FILE] [--schemas DIR]`` writes the
+mapping export from the commit of tag TAG, never the working tree, since every row says
+it was read from that tag. It exits 0 when written, 1 when a row can't be made (such as a
+crosswalk cell with no path and no curated row, or a path not in its format's schema), with
+the findings on standard error so standard output is never a partial file, and 2 as above,
+or when the checkout has no tag TAG or a file can't be read or written.
+"""
+
+import argparse
+import fnmatch
+import json
+import subprocess
+import sys
+from dataclasses import asdict
+from importlib import metadata
+from pathlib import Path
+
+from codemeta_drift.checks import CONTEXT, TABLE, check
+from codemeta_drift.conformance import SCHEMA_FILES, conform
+from codemeta_drift.export import curated_tags, export
+
+JSON_VERSION = 1
+"""The ``--json`` document's version: a minor release may add keys; removing or retyping one bumps it."""
+
+INPUTS = (
+    "codemeta.jsonld",
+    "codemeta_all.jsonld",
+    "properties_description.csv",
+    "crosswalk.csv",
+    "codemeta.json",
+    "examples/*.json",
+    "crosswalks/*.csv",
+)
+"""Glob patterns, relative to the repository root, of every file a check reads."""
+
+
+def is_input(path: str) -> bool:
+    """Whether a repository-relative path matches INPUTS, with ``*`` stopping at ``/`` as in Path.glob."""
+    return any(fnmatch.fnmatchcase(path, g) and path.count("/") == g.count("/") for g in INPUTS)
+
+
+def read_dir(root: Path) -> dict[str, str]:
+    """The input files of a working tree, as path -> text."""
+    return {p.relative_to(root).as_posix(): p.read_text(encoding="utf-8") for g in INPUTS for p in root.glob(g)}
+
+
+def read_ref(root: Path, ref: str) -> dict[str, str]:
+    """The input files at a commit, read with git without touching the working tree."""
+
+    def git(*args: str, stdin: bytes = b"") -> bytes:
+        return subprocess.run(["git", "-C", str(root), *args], input=stdin, check=True, capture_output=True).stdout
+
+    commit = git("rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}").strip().decode()
+
+    # Each entry is "<mode> <type> <object>\t<path>". -z keeps git from quoting
+    # non-ASCII names, which would then not match INPUTS.
+    blobs = {}
+    for entry in git("ls-tree", "-r", "-z", commit).decode("utf-8").split("\0"):
+        if entry:
+            meta, path = entry.split("\t", 1)
+            _, kind, oid = meta.split()
+            if kind == "blob" and is_input(path):
+                blobs[path] = oid
+
+    # One cat-file process reads every file; one git process per file doubles the run time.
+    # Its output is "<object> blob <size>\n<content>\n" per file, in the order asked.
+    out = git("cat-file", "--batch", stdin="".join(f"{oid}\n" for oid in blobs.values()).encode())
+    files, pos = {}, 0
+    for path in blobs:
+        header_end = out.index(b"\n", pos)
+        start = header_end + 1
+        end = start + int(out[pos:header_end].split()[2])
+        files[path] = out[start:end].decode("utf-8")
+        pos = end + 1
+    return files
+
+
+def export_main(argv: list[str]) -> int:
+    """Write the mapping export for one CodeMeta tag, or list the cells that block it."""
+    parser = argparse.ArgumentParser(prog="codemeta-drift export", description="Write a mapping export.")
+    parser.add_argument("checkout", type=Path, nargs="?", default=Path(), help="root of a codemeta clone (default: .)")
+    parser.add_argument("--tag", required=True, help="the CodeMeta release tag to read; picks the curated rows")
+    parser.add_argument("--out", type=Path, help="write here instead of standard output")
+    parser.add_argument("--schemas", type=Path, help="check every path against the format schemas in this directory")
+    args = parser.parse_args(argv)
+
+    if args.tag not in curated_tags():
+        print(f"codemeta-drift: no curated rows for {args.tag}; have {', '.join(curated_tags())}", file=sys.stderr)
+        return 2
+    # Each step names what it failed on: the checkout, a schema file or the output file.
+    target: object = f"{args.checkout} at tag {args.tag}"
+    try:
+        files = read_ref(args.checkout, f"refs/tags/{args.tag}")
+        if TABLE not in files:
+            print(f"codemeta-drift: {target} has no {TABLE}", file=sys.stderr)
+            return 2
+        mapping, findings = export(files, args.tag)
+        if args.schemas:
+            schemas = {}
+            for fmt, name in SCHEMA_FILES.items():
+                target = args.schemas / name
+                schemas[fmt] = json.loads(target.read_text("utf-8"))
+            findings += conform(mapping["rows"], schemas)
+        if findings:
+            print("\n".join(map(str, findings)), file=sys.stderr)
+            return 1
+        text = json.dumps(mapping, indent=2, ensure_ascii=False) + "\n"
+        if args.out:
+            target = args.out
+            args.out.write_text(text, encoding="utf-8")
+        else:
+            print(text, end="")
+    except subprocess.CalledProcessError as err:
+        print(f"codemeta-drift: cannot read {target}: {err.stderr.decode().strip()}", file=sys.stderr)
+        return 2
+    except (OSError, ValueError, KeyError, IndexError, TypeError, AttributeError) as err:
+        print(f"codemeta-drift: cannot read or write {target}: {type(err).__name__}: {err}", file=sys.stderr)
+        return 2
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run every check on one snapshot and print the findings, or run a subcommand."""
+    argv = sys.argv[1:] if argv is None else argv
+    if argv[:1] == ["export"]:
+        return export_main(argv[1:])
+    parser = argparse.ArgumentParser(
+        prog="codemeta-drift",
+        description="Report where CodeMeta's files disagree.",
+        epilog="codemeta-drift export --help describes the mapping export.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {metadata.version('codemeta-drift')}")
+    parser.add_argument("checkout", type=Path, help="root of a codemeta clone")
+    parser.add_argument("--ref", help="read this commit, tag or branch instead of the working tree")
+    parser.add_argument("--json", action="store_true", help="print the findings as one JSON document")
+    args = parser.parse_args(argv)
+
+    try:
+        files = read_ref(args.checkout, args.ref) if args.ref else read_dir(args.checkout)
+        if missing := [p for p in (CONTEXT, TABLE) if p not in files]:
+            print(f"codemeta-drift: {args.checkout} has no {' or '.join(missing)}", file=sys.stderr)
+            return 2
+        findings = check(files)
+    except subprocess.CalledProcessError as err:
+        print(f"codemeta-drift: cannot read {args.checkout}: {err.stderr.decode().strip()}", file=sys.stderr)
+        return 2
+    # A malformed file must not exit 1, which means drift.
+    except (OSError, ValueError, KeyError, IndexError, TypeError, AttributeError) as err:
+        print(f"codemeta-drift: cannot read {args.checkout}: {type(err).__name__}: {err}", file=sys.stderr)
+        return 2
+    if args.json:
+        tool = {"name": "codemeta-drift", "version": metadata.version("codemeta-drift")}
+        print(json.dumps({"version": JSON_VERSION, "tool": tool, "findings": [asdict(f) for f in findings]}, indent=2))
+    else:
+        print("\n".join(map(str, findings)))
+    return 1 if any(f.code != "SKIPPED" for f in findings) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

codemeta_drift/conformance.py

@@ -0,0 +1,89 @@
+"""Check that every path in a mapping export names a property in its format's own JSON Schema.
+
+A step resolves when the schema in reach has the key under ``properties``, a matching
+``patternProperties`` entry, or an ``additionalProperties`` schema. Local ``$ref``s are followed,
+and a key found in any branch of ``allOf``, ``anyOf`` or ``oneOf`` counts. ``key[]`` and
+``key[k=v]`` need the value to be a list (``items``); a filter needs ``k`` in the item schema.
+An external ``$ref`` is not followed, so a key only reachable through one does not resolve.
+"""
+
+import re
+from collections.abc import Iterator, Mapping
+from typing import Any
+
+from codemeta_drift.checks import Finding
+from codemeta_drift.export import PATH
+
+SCHEMA_FILES = {
+    "cff-1.2.0": "cff-1.2.0.json",
+    "pep621": "pyproject.json",
+    "npm": "package.json",
+    "cargo": "cargo.json",
+}
+"""The schema file for each format that has one. Zenodo and codemeta are not checked."""
+
+STEP = re.compile(r"(?P<key>@?[\w-]+)(?:\[(?:(?P<filter>@?[\w-]+)=[^\]]+)?\])?")
+
+type Schema = dict[str, Any]
+
+
+def branches(root: Schema, node: object, seen: frozenset[int] = frozenset()) -> Iterator[Schema]:
+    """The node and every schema it stands for through local $ref, allOf, anyOf and oneOf."""
+    if not isinstance(node, dict) or id(node) in seen:
+        return
+    seen |= {id(node)}
+    yield node
+    ref = node.get("$ref")
+    if isinstance(ref, str) and ref.startswith("#"):
+        target: Any = root
+        for part in filter(None, ref[1:].split("/")):
+            target = target.get(part.replace("~1", "/").replace("~0", "~"), {}) if isinstance(target, dict) else {}
+        yield from branches(root, target, seen)
+    for combinator in ("allOf", "anyOf", "oneOf"):
+        for sub in node.get(combinator, ()):
+            yield from branches(root, sub, seen)
+
+
+def child(node: Schema, key: str) -> object:
+    """The schema of ``key`` inside one object schema, or None when the object has no such key."""
+    if key in node.get("properties", {}):
+        return node["properties"][key]
+    for pattern, sub in node.get("patternProperties", {}).items():
+        if re.search(pattern, key):
+            return sub
+    extra = node.get("additionalProperties")
+    return extra if isinstance(extra, dict) else None
+
+
+def resolve(root: Schema, path: str) -> str | None:
+    """Why one path alternative (no `` | ``) is not in the schema, or None when every step resolves."""
+    if " | " in path or not PATH.fullmatch(path):
+        return "not a path"
+    nodes: list[object] = [root]
+    for match in STEP.finditer(path):
+        key, selector = match["key"], match[0][len(match["key"]) :]
+        found = [c for n in nodes for b in branches(root, n) if (c := child(b, key)) is not None]
+        if not found:
+            return f"no {key!r}"
+        if selector:
+            found = [b["items"] for n in found for b in branches(root, n) if isinstance(b.get("items"), dict)]
+            if not found:
+                return f"{key!r} is not a list"
+            if (k := match["filter"]) and not any(child(b, k) is not None for n in found for b in branches(root, n)):
+                return f"items of {key!r} have no {k!r}"
+        nodes = found
+    return None
+
+
+def conform(rows: list[dict[str, Any]], schemas: Mapping[str, Schema]) -> list[Finding]:
+    """A ``PATH_NOT_IN_SCHEMA`` finding for each path alternative its format's schema lacks.
+
+    Rows of a format with no entry in ``schemas``, and rows whose path is null, are not checked.
+    """
+    return [
+        Finding("PATH_NOT_IN_SCHEMA", row["format"], row["term"], f"{alternative}: {why}")
+        for row in rows
+        if (schema := schemas.get(row["format"])) is not None
+        for alternative in (row["path"] or "").split(" | ")
+        if alternative and (why := resolve(schema, alternative))
+    ]