iirds-validate 0.1.0__py3-none-any.whl

iirds_validate/__init__.py

@@ -0,0 +1,7 @@
+"""Offline, graph-based validator and interoperability linter for iiRDS packages."""
+
+__version__ = "0.1.0"
+
+from .model import Finding, Rule, Severity, Violation  # noqa: F401
+from .package import Package, PackageError  # noqa: F401
+from .runner import check, lint, load  # noqa: F401

iirds_validate/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

iirds_validate/banner.py

@@ -0,0 +1,45 @@
+"""The thing you see when you type `iirdsv` and nothing else.
+
+Deliberately not printed by `check`, `lint` or `all`. Those write to a build
+log or into a pipe, and `--format json` writes a document another program
+parses; a banner in front of either is somewhere between noise and corruption.
+
+Plain ASCII on purpose. Block-drawing characters look better in a modern
+terminal and turn into rubbish in a Windows console or over a serial link, and
+the machines this tool is built for are exactly the ones with the old fonts.
+"""
+from __future__ import annotations
+
+from . import __version__
+
+#: figlet, "slant". 25 columns, which leaves room on an 80-column terminal for
+#: the version to sit on the baseline.
+LOGO = r"""    _ _ ____  ____  _____
+   (_|_) __ \/ __ \/ ___/
+  / / / /_/ / / / /\__ \
+ / / / _, _/ /_/ /___/ /
+/_/_/_/ |_/_____//____/"""
+
+TAGLINE = "conformance and interoperability checking for iiRDS packages, offline"
+
+COMMANDS = (
+    ("<path>", "check and lint it — a package, a directory, either"),
+    ("check <path>", "does it conform to the specification?"),
+    ("lint  <path>", "will anyone else be able to read it?"),
+    ("pack  <directory>", "write it as a conformant .iirds, then check that"),
+    ("rules", "every rule this tool checks, and its source"),
+)
+
+
+def banner(coverage=None) -> str:
+    """The logo, the version on the baseline, and how to start."""
+    lines = LOGO.splitlines()
+    lines[-1] = "%-26s validate %s" % (lines[-1], __version__)
+    out = ["", "\n".join(lines), "", "  " + TAGLINE, ""]
+    for command, purpose in COMMANDS:
+        out.append("  iirdsv %-18s %s" % (command, purpose))
+    out.append("")
+    if coverage:
+        out.append("  " + coverage)
+        out.append("")
+    return "\n".join(out)

iirds_validate/cli.py

@@ -0,0 +1,255 @@
+"""Command line interface.
+
+    iirdsv check pkg.iirds      conformance  (container + graph rules)
+    iirdsv lint  pkg.iirds      interoperability  (can a consumer use it?)
+    iirdsv all   pkg.iirds      both
+    iirdsv rules --kind lint    what this tool knows how to check
+
+Exit codes: 0 clean, 1 errors found, 2 could not run.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from . import __version__, runner
+from .banner import banner
+from .model import VERSIONS, Severity
+from .package import discover
+from .packer import PackError, pack
+from .registry import CATALOG, all_rules, coverage
+from .report import render
+
+EXIT_OK, EXIT_FINDINGS, EXIT_ERROR = 0, 1, 2
+
+
+def _add_target(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("package", nargs="+",
+                        help="packages: a .iirds file, an unpacked container directory, "
+                             "or a directory to search")
+    parser.add_argument("-f", "--format", choices=("text", "json"), default="text")
+    parser.add_argument("--iirds-version", dest="version", default=None, choices=VERSIONS,
+                        metavar="{%s}" % ",".join(VERSIONS),
+                        help="validate against this version instead of the declared one")
+    parser.add_argument("-v", "--verbose", action="store_true", help="include spec links")
+    parser.add_argument("-q", "--quiet", action="store_true", help="exit code only")
+    parser.add_argument("-W", "--warnings-as-errors", action="store_true",
+                        help="fail the run on warnings too")
+
+
+def _targets(paths):
+    """Expand what the user pointed at into packages.
+
+    A path can be a package, a directory that is one unpacked, or a directory
+    with packages somewhere underneath. Pointing at a build output directory
+    should do the obvious thing rather than require a shell glob.
+    """
+    found, missing, empty = [], [], []
+    for path in paths:
+        if not os.path.exists(path):
+            missing.append(path)
+            continue
+        expanded = discover(path)
+        if expanded:
+            found.extend(expanded)
+        else:
+            empty.append(path)
+    return found, missing, empty
+
+
+def _run(args, kinds) -> int:
+    # A path that is not there is an operator error, not a validation result:
+    # exit 2. A file that opens but is not a valid container is a finding about
+    # the package, so it goes through the rules and exits 1.
+    targets, missing, empty = _targets(args.package)
+    for path in missing:
+        print("iirds-validate: no such file or directory: %s" % path, file=sys.stderr)
+    for path in empty:
+        print("iirds-validate: no iiRDS package found under %s" % path, file=sys.stderr)
+    if missing or empty:
+        return EXIT_ERROR
+
+    reports = [runner.run(path, kinds, version=args.version) for path in targets]
+
+    if args.format == "json":
+        payload = [r.as_dict() for r in reports]
+        json.dump(payload[0] if len(payload) == 1 else payload,
+                  sys.stdout, ensure_ascii=False, indent=2)
+        sys.stdout.write("\n")
+    elif not args.quiet:
+        for i, report in enumerate(reports):
+            if i:
+                print()
+            render(report, "text", verbose=args.verbose)
+
+    failed = any(not r.ok for r in reports)
+    if args.warnings_as_errors:
+        failed = failed or any(r.count(Severity.WARNING) for r in reports)
+
+    if len(reports) > 1 and args.format == "text" and not args.quiet:
+        bad = sum(1 for r in reports if not r.ok)
+        print("\n%d packages: %d passed, %d failed"
+              % (len(reports), len(reports) - bad, bad))
+    return EXIT_FINDINGS if failed else EXIT_OK
+
+
+def _cmd_pack(args) -> int:
+    """Write the archive, then validate the archive.
+
+    Validating what was just written rather than the directory is the point:
+    the five requirements about the ZIP that a directory cannot answer are now
+    answerable, and answered against the file that will actually be delivered.
+    """
+    try:
+        output = pack(args.directory, args.output, overwrite=args.overwrite)
+    except PackError as exc:
+        print("iirds-validate: %s" % exc, file=sys.stderr)
+        return EXIT_ERROR
+
+    if not args.quiet and args.format == "text":
+        print("wrote %s (%.0f KB)\n" % (output, output.stat().st_size / 1024))
+
+    args.package = [str(output)]
+    return _run(args, runner.ALL_KINDS)
+
+
+def _cmd_rules(args) -> int:
+    rules = all_rules()
+    if args.kind:
+        rules = [r for r in rules if r.kind == args.kind]
+    if args.ids:
+        wanted = {rid.upper() for rid in args.ids}
+        rules = [r for r in rules if r.id.upper() in wanted]
+        missing = wanted - {r.id.upper() for r in rules}
+        if missing:
+            print("no such rule: %s" % ", ".join(sorted(missing)), file=sys.stderr)
+            return EXIT_ERROR
+        args.verbose = True          # asking for one rule means asking about it
+
+    if args.format == "json":
+        json.dump([{"id": r.id, "kind": r.kind, "priority": r.prio, "severity": str(r.severity),
+                    "versions": list(r.versions), "variants": list(r.variants),
+                    "title": r.title, "spec": r.spec,
+                    "source": "catalogue" if r.id in CATALOG else "iirds-validate",
+                    "covers": list(r.covers), "fix": r.fix} for r in rules],
+                  sys.stdout, ensure_ascii=False, indent=2)
+        sys.stdout.write("\n")
+        return EXIT_OK
+
+    for r in rules:
+        variants = ("/" + ",".join(r.variants)) if r.variants else ""
+        print("%-9s %-9s %-9s %s" % (r.id, r.kind, r.prio + variants, r.title[:96]))
+        if args.verbose:
+            source = "catalogue" if r.id in CATALOG else "this project"
+            print("          versions: %s   source: %s"
+                  % (", ".join(r.versions) if r.versions else "all", source))
+            if r.spec:
+                print("          spec: %s" % r.spec)
+            if r.covers:
+                print("          covers: %s" % ", ".join(r.covers))
+            if r.fix:
+                print("          fix: %s" % r.fix)
+            print()
+
+    if args.ids:
+        return EXIT_OK              # asked about specific rules; no summary tail
+
+    print()
+    cov = coverage()
+    labels = {"container": "the ZIP and its layout",
+              "schema": "the metadata graph",
+              "system": "the run itself",
+              "content": "iiRDS XHTML5 (Appendix B)",
+              "lint": "will a consumer be able to use it"}
+    for kind in ("container", "schema", "system", "content", "lint"):
+        c = cov.get(kind)
+        if not c or not (c["total"] or c["ours"]):
+            continue
+        catalogued = "%d/%d" % (c["implemented"], c["total"]) if c["total"] else "-"
+        ours = ("  +%d of its own" % c["ours"]) if c["ours"] else ""
+        print("%-10s %-8s %s%s" % (kind, catalogued, labels[kind], ours))
+    print()
+    print("%d of %d catalogued rules, plus %d of this project's own" % (
+        sum(c["implemented"] for c in cov.values()), len(CATALOG),
+        sum(c["ours"] for c in cov.values())))
+    return EXIT_OK
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="iirds-validate",
+        description="Offline validator and interoperability linter for iiRDS packages.")
+    parser.add_argument("--version", action="version", version="iirds-validate %s" % __version__)
+    sub = parser.add_subparsers(dest="command")
+
+    p_check = sub.add_parser("check", help="conformance: container structure and metadata graph")
+    _add_target(p_check)
+
+    p_lint = sub.add_parser("lint", help="interoperability: can a consumer actually use this?")
+    _add_target(p_lint)
+
+    p_all = sub.add_parser("all", help="check and lint together")
+    _add_target(p_all)
+
+    p_pack = sub.add_parser(
+        "pack", help="write a directory as a conformant .iirds and check it")
+    p_pack.add_argument("directory")
+    p_pack.add_argument("-o", "--output", default=None,
+                        help="where to write it (default: alongside the directory)")
+    p_pack.add_argument("--overwrite", action="store_true")
+    p_pack.add_argument("-f", "--format", choices=("text", "json"), default="text")
+    p_pack.add_argument("--iirds-version", dest="version", default=None, choices=VERSIONS)
+    p_pack.add_argument("-v", "--verbose", action="store_true")
+    p_pack.add_argument("-q", "--quiet", action="store_true")
+    p_pack.add_argument("-W", "--warnings-as-errors", action="store_true")
+
+    p_rules = sub.add_parser("rules", help="list the rules this tool implements")
+    p_rules.add_argument("ids", nargs="*", metavar="RULE",
+                         help="show only these rules, in full (e.g. M11 B8 R3)")
+    p_rules.add_argument("--kind",
+                         choices=("container", "schema", "system", "content", "lint"))
+    p_rules.add_argument("-v", "--verbose", action="store_true",
+                         help="also print versions, spec link, source and remedy")
+    p_rules.add_argument("-f", "--format", choices=("text", "json"), default="text")
+
+    # `iirdsv some/path` with no subcommand means `all`. Typing the verb is
+    # friction, and "check it" is what anybody pointing at a package wants.
+    argv = list(sys.argv[1:] if argv is None else argv)
+    known = {"check", "lint", "all", "pack", "rules", "-h", "--help", "--version"}
+    if argv and argv[0] not in known and not argv[0].startswith("-"):
+        argv.insert(0, "all")
+
+    args = parser.parse_args(argv)
+
+    if args.command is None:
+        # Bare `iirdsv`. argparse would exit 2 with a usage error, which is a
+        # poor answer to someone who has just installed the thing.
+        cov = coverage()
+        print(banner("%d of %d catalogued rules, plus %d of its own. no network access."
+                     % (sum(v["implemented"] for v in cov.values()), len(CATALOG),
+                        sum(v["ours"] for v in cov.values()))))
+        return EXIT_OK
+
+    try:
+        if args.command == "check":
+            return _run(args, runner.CONFORMANCE_KINDS)
+        if args.command == "lint":
+            return _run(args, runner.LINT_KINDS)
+        if args.command == "all":
+            return _run(args, runner.ALL_KINDS)
+        if args.command == "pack":
+            return _cmd_pack(args)
+        if args.command == "rules":
+            return _cmd_rules(args)
+    except KeyboardInterrupt:
+        return EXIT_ERROR
+    except OSError as exc:
+        print("iirds-validate: %s" % exc, file=sys.stderr)
+        return EXIT_ERROR
+    return EXIT_ERROR
+
+
+if __name__ == "__main__":
+    sys.exit(main())

iirds_validate/context.py

@@ -0,0 +1,269 @@
+"""Everything a rule is handed: the container, the graph, and the ontology.
+
+The whole point of this project lives in `build_graph`. iiRDS metadata is RDF,
+and RDF says nothing about how it is written down. These are the same fact:
+
+    <iirds:Document rdf:about="urn:d1"/>
+
+    <rdf:Description rdf:about="urn:d1">
+      <rdf:type rdf:resource="http://iirds.tekom.de/iirds#Document"/>
+    </rdf:Description>
+
+and so are these:
+
+    <iirds:relates-to-event><iirds:Event rdf:about="urn:e1"/></iirds:relates-to-event>
+    <iirds:relates-to-event rdf:resource="urn:e1"/>
+
+A validator that walks the XML tree sees one form and misses the other, which is
+how a package can be perfectly conformant and still unreadable to the tool that
+is supposed to bless it. Parsing into a graph makes the distinction disappear
+before any rule runs — and lets the same rules apply to metadata.jsonld.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass, field
+from typing import List, Optional, Set
+
+from rdflib import BNode, Graph, URIRef
+from rdflib.namespace import RDF, RDFS
+
+from . import ontology as ontology_mod
+from . import terms as T
+from .model import LATEST_VERSION, METADATA_JSONLD, METADATA_RDF, PACKAGE_BASE, VERSIONS
+from .package import Package
+
+
+@dataclass
+class Context:
+    package: Package
+    graph: Graph
+    ontology: ontology_mod.Ontology
+    version: str
+    variant: str
+    declared_version: Optional[str] = None      # None when the package omits it
+    requested_version: Optional[str] = None     # set when the caller overrode it
+    parse_errors: List[str] = field(default_factory=list)
+    sources: List[str] = field(default_factory=list)
+    #: One graph per metadata file, kept alongside the merged `graph` so a rule
+    #: can ask whether the serialisations agree. Merging is right for every
+    #: other rule and is exactly what hides a disagreement.
+    per_source: dict = field(default_factory=dict)
+
+    # -- graph helpers ------------------------------------------------------
+    def instances_of(self, cls: URIRef, include_subclasses: bool = True) -> List:
+        """Subjects typed `cls`, or any class beneath it."""
+        classes = self.ontology.subclasses_of(cls) if include_subclasses else {cls}
+        out, seen = [], set()
+        for c in classes:
+            for s in self.graph.subjects(RDF.type, c):
+                if s not in seen:
+                    seen.add(s)
+                    out.append(s)
+        return out
+
+    def typed_exactly(self, cls: URIRef) -> List:
+        """Subjects carrying `cls` itself as an rdf:type (no subclasses)."""
+        return list(self.graph.subjects(RDF.type, cls))
+
+    def values(self, subject, prop: URIRef) -> List:
+        return list(self.graph.objects(subject, prop))
+
+    def one(self, subject, prop: URIRef):
+        for o in self.graph.objects(subject, prop):
+            return o
+        return None
+
+    def has(self, subject, prop: URIRef) -> bool:
+        return (subject, prop, None) in self.graph
+
+    def information_units(self) -> List:
+        return self.instances_of(T.InformationUnit)
+
+    def iirds_subjects(self) -> Set:
+        """Every subject that carries at least one iiRDS type."""
+        out = set()
+        for s, o in self.graph.subject_objects(RDF.type):
+            if self.ontology.is_iirds_term(o):
+                out.add(s)
+        return out
+
+    def ref(self, node) -> str:
+        """A name for a node that is the same on every run.
+
+        rdflib mints a fresh identifier for every blank node on every parse, so
+        a finding that reported `str(node)` gave `N8892b8d9…` one run and
+        `N39e7e968…` the next. Two effects, both bad: a JSON report could not be
+        diffed between runs, and the same package written as RDF/XML and as
+        JSON-LD produced different findings — which is the one property this
+        project claims above all others.
+
+        A blank node is named by how you reach it instead: the nearest named
+        subject and the property that points at it, which is also far more use
+        to somebody reading the report than an opaque identifier. Where that
+        fails, a hash of the statements about the node, which is stable because
+        the statements are.
+        """
+        if not isinstance(node, BNode):
+            return str(node)
+        for subject, predicate in sorted(self.graph.subject_predicates(node), key=str):
+            if not isinstance(subject, BNode):
+                return "%s %s" % (subject, str(predicate).split("#")[-1].split("/")[-1])
+        digest = hashlib.sha256()
+        for predicate, obj in sorted(self.graph.predicate_objects(node), key=str):
+            digest.update(("%s %s\n" % (predicate, obj)).encode("utf-8"))
+        return "_:%s" % digest.hexdigest()[:12]
+
+    def label_of(self, node) -> str:
+        for p in (RDFS.label, T.title):
+            v = self.one(node, p)
+            if v is not None:
+                return str(v)
+        return str(node)
+
+
+def _detect(graph: Graph):
+    """Read iirds:iiRDSVersion / iirds:formatRestriction off the package node."""
+    declared, variant = None, None
+    for pkg in graph.subjects(RDF.type, T.Package):
+        v = graph.value(pkg, T.iiRDSVersion)
+        if v is not None and declared is None:
+            declared = str(v).strip()
+        r = graph.value(pkg, T.formatRestriction)
+        if r is not None and variant is None:
+            variant = str(r).strip()
+    return declared, (variant or "unrestricted")
+
+
+#: An .iirds package arrives from a supplier, so its metadata is untrusted
+#: input. Two cheap guards, applied before the parser sees anything.
+MAX_METADATA_BYTES = 64 * 1024 * 1024
+_ENTITY_DECL = re.compile(rb"<!ENTITY", re.IGNORECASE)
+_HAS_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
+
+
+#: rdflib decodes a bytes payload as UTF-8 unconditionally, so a document that
+#: declares — and marks with a byte order mark — any other encoding fails to
+#: parse at all. XML says the BOM decides, so it is honoured here and the
+#: payload handed on as UTF-8.
+_BOMS = ((b"\xff\xfe\x00\x00", "utf-32-le"), (b"\x00\x00\xfe\xff", "utf-32-be"),
+         (b"\xff\xfe", "utf-16-le"), (b"\xfe\xff", "utf-16-be"),
+         (b"\xef\xbb\xbf", "utf-8-sig"))
+
+
+def _decode_by_bom(raw: bytes) -> bytes:
+    for bom, encoding in _BOMS:
+        if raw.startswith(bom):
+            text = raw.decode(encoding)
+            # The declaration would now contradict the bytes.
+            text = re.sub(r'(<\?xml[^>]*?)\s+encoding\s*=\s*(["\'])[^"\']*\2',
+                          r"\1", text, count=1)
+            return text.encode("utf-8")
+    return raw
+
+
+def _remote_contexts(node, found=None):
+    """Every `@context` in the document that names a location to go and fetch.
+
+    JSON-LD lets a context be a URL, and the parser will dereference it. In a
+    package that arrived from a supplier that is two separate problems: it
+    breaks the promise that validation touches no network, and it lets the
+    sender choose a host for a machine inside the plant to connect to.
+
+    Contexts nest, and a context can be an array mixing inline objects with
+    URLs, so the whole document is walked rather than just the top level.
+    """
+    found = [] if found is None else found
+    if isinstance(node, dict):
+        for key, value in node.items():
+            if key == "@context":
+                for candidate in (value if isinstance(value, list) else [value]):
+                    if isinstance(candidate, str) and _HAS_SCHEME.match(candidate):
+                        found.append(candidate)
+                    else:
+                        _remote_contexts(candidate, found)
+            else:
+                _remote_contexts(value, found)
+    elif isinstance(node, list):
+        for item in node:
+            _remote_contexts(item, found)
+    return found
+
+
+def build_graph(package: Package):
+    """Parse every metadata serialisation in the container into one graph."""
+    graph = Graph()
+    errors: List[str] = []
+    sources: List[str] = []
+    per_source = {}
+
+    for name, fmt in ((METADATA_RDF, "xml"), (METADATA_JSONLD, "json-ld")):
+        if not package.has(name):
+            continue
+
+        info = package.info(name)
+        if info is not None and info.file_size > MAX_METADATA_BYTES:
+            errors.append("%s: refused: %d bytes uncompressed, above the %d byte limit"
+                          % (name, info.file_size, MAX_METADATA_BYTES))
+            continue
+
+        raw = _decode_by_bom(package.read(name))
+
+        # Nested internal entities expand geometrically: a few hundred bytes of
+        # declarations can occupy the parser indefinitely. iiRDS metadata has no
+        # legitimate use for them, so refuse rather than try to bound the damage.
+        if fmt == "xml" and _ENTITY_DECL.search(raw):
+            errors.append("%s: refused: the document declares XML entities" % name)
+            continue
+
+        if fmt == "json-ld":
+            try:
+                document = json.loads(raw.decode("utf-8"))
+            except Exception as exc:
+                errors.append("%s: %s: %s" % (name, type(exc).__name__, exc))
+                continue
+            remote = _remote_contexts(document)
+            if remote:
+                errors.append("%s: refused: @context must be inline, not fetched from %s"
+                              % (name, ", ".join(sorted(set(remote))[:3])))
+                continue
+
+        try:
+            single = Graph()
+            single.parse(data=raw, format=fmt, publicID=PACKAGE_BASE)
+        except Exception as exc:
+            errors.append("%s: %s: %s" % (name, type(exc).__name__, exc))
+            continue
+        per_source[name] = single
+        graph += single
+        sources.append(name)
+
+    return graph, errors, sources, per_source
+
+
+def load_context(package: Package, version: Optional[str] = None) -> Context:
+    graph, errors, sources, per_source = build_graph(package)
+    declared, variant = _detect(graph)
+
+    # plusmeta's tool filters its rules by the declared version string, so a
+    # package that omits iirds:iiRDSVersion runs zero schema rules and reports
+    # "no violations". Here a missing or unknown version falls back to the
+    # newest one and is recorded as a note, so nothing passes by silence.
+    effective = version or declared or LATEST_VERSION
+    if effective not in VERSIONS:
+        effective = LATEST_VERSION
+
+    return Context(
+        package=package,
+        graph=graph,
+        ontology=ontology_mod.load(effective if effective in VERSIONS else LATEST_VERSION),
+        version=effective,
+        variant=variant,
+        declared_version=declared,
+        requested_version=version,
+        parse_errors=errors,
+        sources=sources,
+        per_source=per_source,
+    )