sdc-mapper 4.1.0__py3-none-any.whl

sdc_mapper/__init__.py

@@ -0,0 +1,29 @@
+"""sdc-mapper: the shared core of the Semantic Data Charter's Governed Data Record mappers.
+
+A record is checked by the validation gate (valid, valid with Exceptional Values, or refused), read and joined to its
+model by sdcreader, resolved into one shape every mapper receives, mapped by a target mapper, and reported value by
+value: how each value reached the target, or that the target cannot hold it, and for every Exceptional Value whether
+the target could say it too. Every call is appended to a hash-chained log with no values in it.
+
+Outbound only. Each target format is its own package (`sdc-gdr2<target>`) registered under the entry-point group
+`sdc_mapper.targets`.
+"""
+from .contract import (CARRIED, DECLARED, DEGRADED, ENTRY_POINT_GROUP, EV_CARRIED, EV_CLASSES, LOOKUP, LOST,
+                       NOT_REPRESENTABLE, PRESERVED, VALUE_CLASSES, EVRule, MapResult, Mapper, ValueOutcome,
+                       installed_mappers)
+from .engine import ERROR, CallResult, Engine
+from .gate import REFUSED, VALID, VALID_WITH_EV, Gate, GateResult, Recovery
+from .log import Log, entry_hash
+from .mapping import Mapping, MappingError, Rule, load_mapping
+from .placement import ClusterPlacement, Extra, Placement, place, text_of
+from .report import CoverageReport, IncompleteMapping, build_report
+from .resolve import Exceptional, ResolvedRecord, ResolvedValue, Units, resolve
+
+__version__ = "4.1.0"
+__all__ = [
+    "Engine", "CallResult", "Gate", "GateResult", "Recovery", "Log", "entry_hash", "Mapping", "MappingError", "Rule",
+    "load_mapping", "place", "Placement", "ClusterPlacement", "Extra", "text_of", "CoverageReport", "IncompleteMapping", "build_report", "ResolvedRecord", "ResolvedValue",
+    "Exceptional", "Units", "resolve", "Mapper", "MapResult", "ValueOutcome", "EVRule", "installed_mappers",
+    "VALID", "VALID_WITH_EV", "REFUSED", "ERROR", "LOOKUP", "DECLARED", "CARRIED", "NOT_REPRESENTABLE", "VALUE_CLASSES",
+    "PRESERVED", "DEGRADED", "EV_CARRIED", "LOST", "EV_CLASSES", "ENTRY_POINT_GROUP", "__version__",
+]

sdc_mapper/cli.py

@@ -0,0 +1,101 @@
+"""`sdc-map`: the command line for the Governed Data Record mappers.
+
+    sdc-map targets
+    sdc-map map TARGET RECORD [--package DIR | --ct-id CT] [--mapping FILE] [-o DIR] [--log-dir DIR]
+    sdc-map check-mapping FILE [--package DIR | --ct-id CT]
+    sdc-map fidelity RECORD [--package DIR | --ct-id CT] [--mapping NAME=FILE ...]
+    sdc-map log summary [--days N] | sdc-map log verify   [--log-dir DIR]
+
+Exit status: 0 mapped, 1 refused, 2 error.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from sdcreader import DEFAULT_HOST
+
+from .engine import ERROR, Engine
+from .gate import REFUSED
+from .log import Log
+from .mapping import MappingError, load_mapping
+from .service import ServiceError, describe, fidelity, fidelity_text, map_one, targets
+
+
+def _source(p):
+    g = p.add_mutually_exclusive_group()
+    g.add_argument("--package", help="A model package directory (dm-<ct>.jsonld and dm-<ct>.xsd).")
+    g.add_argument("--ct-id", help="A published model's identifier, fetched from --host.")
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(prog="sdc-map", description="Map Governed Data Records to other formats, every value reported.")
+    p.add_argument("--host", default=DEFAULT_HOST)
+    p.add_argument("--log-dir", default=os.environ.get("SDC_MAPPER_LOG_DIR", "sdc-mapper-log"))
+    sub = p.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("targets")
+    m = sub.add_parser("map")
+    m.add_argument("target")
+    m.add_argument("record")
+    _source(m)
+    m.add_argument("--mapping")
+    m.add_argument("-o", "--output-dir")
+    c = sub.add_parser("check-mapping")
+    c.add_argument("mapping")
+    _source(c)
+    f = sub.add_parser("fidelity")
+    f.add_argument("record")
+    _source(f)
+    f.add_argument("--mapping", action="append", default=[], metavar="NAME=FILE")
+    f.add_argument("--json", action="store_true")
+    lg = sub.add_parser("log")
+    lg.add_argument("action", choices=["summary", "verify"])
+    lg.add_argument("--days", type=float, default=7)
+    a = p.parse_args(argv)
+
+    try:
+        if a.cmd == "targets":
+            print(json.dumps(targets(), indent=2))
+            return 0
+        if a.cmd == "log":
+            log = Log(a.log_dir)
+            if a.action == "verify":
+                ok, msg = log.verify()
+                print(msg)
+                return 0 if ok else 2
+            print(json.dumps(log.summary(days=a.days), indent=2))
+            return 0
+        engine = Engine(log_dir=a.log_dir, host=a.host)
+        if a.cmd == "map":
+            r = map_one(engine, a.target, a.record, mapping=a.mapping, ct_id=a.ct_id, directory=a.package, caller="cli")
+            out = describe(r, output_dir=a.output_dir or "sdc-map-out")
+            if r.report:
+                print(r.report.summary())
+                print("written:", ", ".join(out["written"]))
+            else:
+                print(f"{r.outcome}:", *(f"\n  {e.get('kind')}: {e.get('reason', '')}" for e in r.errors))
+            return {REFUSED: 1, ERROR: 2}.get(r.outcome, 0)
+        if a.cmd == "check-mapping":
+            mp = load_mapping(a.mapping)
+            print(f"ok: {mp.id} {mp.version}, {len(mp.rules)} rules, sha256 {mp.sha256}")
+            if a.package or a.ct_id:
+                pkg, model = engine.packages.one({"directory": a.package} if a.package else {"ct_id": a.ct_id})
+                if (mp.model_ct_id, mp.model_schema_sha256) != (model.ct_id, pkg.sha256):
+                    print(f"not bound to this model's current schema ({model.ct_id} {pkg.sha256[:12]})")
+                    return 2
+            return 0
+        if a.cmd == "fidelity":
+            mappings = dict(x.split("=", 1) for x in a.mapping)
+            table = fidelity(engine, a.record, mappings=mappings, ct_id=a.ct_id, directory=a.package, caller="cli")
+            print(json.dumps(table, indent=2) if a.json else fidelity_text(table))
+            return 0
+    except (ServiceError, MappingError, OSError) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sdc_mapper/contract.py

@@ -0,0 +1,79 @@
+"""The contract between the core and a target mapper (PRD sections 10, 11 and the mapper contract).
+
+A mapper receives a `ResolvedRecord` and an optional `Mapping`, and returns a `MapResult`: the output files and one
+`ValueOutcome` for every value it was given. It never writes a rejected value as a value, and it accounts for every
+value: the core checks that, and fails the call if any value is missing from the outcomes.
+
+Mappers register under the entry-point group `sdc_mapper.targets`.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from importlib.metadata import entry_points
+from typing import Protocol, runtime_checkable
+
+from .mapping import Mapping
+from .resolve import ResolvedRecord
+
+#: How a value that holds a value reached the target, in the order the classification tries them (section 11).
+LOOKUP = "lookup"
+DECLARED = "declared"
+CARRIED = "carried"
+NOT_REPRESENTABLE = "not_representable"
+VALUE_CLASSES = (LOOKUP, DECLARED, CARRIED, NOT_REPRESENTABLE)
+
+#: What the target did with an Exceptional Value (section 10).
+PRESERVED = "preserved"
+DEGRADED = "degraded"
+EV_CARRIED = "carried"
+LOST = "lost"
+EV_CLASSES = (PRESERVED, DEGRADED, EV_CARRIED, LOST)
+
+ENTRY_POINT_GROUP = "sdc_mapper.targets"
+
+
+@dataclass
+class ValueOutcome:
+    """What happened to one value. `cls` is a value class, or an Exceptional Value class when the value carries one."""
+    position: int
+    cls: str
+    location: str = ""            # where in the target it went: "MEASUREMENT.value_as_number", "OBX-5", ...
+    target_code: str = ""         # for an Exceptional Value: the code the target wrote ("asked-declined"), if any
+    merged_with: list[str] = field(default_factory=list)   # for degraded: the ISO 21090 codes the target code also covers
+    note: str = ""
+
+
+@dataclass
+class EVRule:
+    """One row of a mapper's Exceptional Value table: what the target writes for an ISO 21090 code."""
+    cls: str                      # preserved, degraded, carried or lost
+    target_code: str = ""
+    merged_with: tuple[str, ...] = ()
+
+
+@dataclass
+class MapResult:
+    files: dict[str, bytes]
+    outcomes: list[ValueOutcome]
+    cannot_hold: list[str] = field(default_factory=list)   # what the target structurally cannot hold for any record
+
+
+@runtime_checkable
+class Mapper(Protocol):
+    name: str                     # "edifact", "omop", ...
+    target: str                   # "UN/EDIFACT"
+    target_version: str           # "D.24A"
+    package: str                  # the distribution name, "sdc-gdr2edifact"
+    version: str                  # the mapper's own version
+    ev_table: dict[str, EVRule]   # one rule per ISO 21090 code
+
+    def map(self, record: ResolvedRecord, mapping: Mapping | None) -> MapResult: ...
+
+
+def installed_mappers() -> dict[str, type]:
+    """The mappers installed in this environment, by name."""
+    found = {}
+    for ep in entry_points(group=ENTRY_POINT_GROUP):
+        cls = ep.load()
+        found[getattr(cls, "name", ep.name)] = cls
+    return found

sdc_mapper/engine.py

@@ -0,0 +1,108 @@
+"""One call, end to end: find the model, check the mapping, gate, read, resolve, map, report, log.
+
+Every call ends in exactly one of four outcomes, and every one is logged:
+- `valid` or `valid_with_ev`: mapped; the files and the report are returned;
+- `refused`: the gate refused the record; the errors are returned, no output is written;
+- `error`: the call could not be completed (a mapping for another model, a mapper that did not account for every
+  value); the reason is returned.
+"""
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from sdcreader import DEFAULT_HOST
+from sdcreader.mcp import PackageCache
+from sdcreader.record import read_record
+
+from .gate import REFUSED, Gate
+from .log import Log
+from .mapping import Mapping, MappingError
+from .report import CoverageReport, IncompleteMapping, build_report
+from .resolve import resolve
+
+ERROR = "error"
+_ROOT = re.compile(rb"<[A-Za-z0-9_]+:dm-([a-z0-9]{24,32})[\s>]")
+
+
+def _sha(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+@dataclass
+class CallResult:
+    outcome: str                                  # valid, valid_with_ev, refused or error
+    files: dict[str, bytes] = field(default_factory=dict)
+    report: CoverageReport | None = None
+    errors: list[dict] = field(default_factory=list)
+    log_hash: str | None = None
+
+    @property
+    def mapped(self) -> bool:
+        return self.report is not None
+
+
+class Engine:
+    def __init__(self, log_dir: str | Path | None = None, host: str = DEFAULT_HOST, schema_dir: str | Path | None = None):
+        self.gate = Gate(schema_dir)
+        self.log = Log(log_dir) if log_dir else None
+        self.packages = PackageCache(host=host, storage={})
+
+    def model_for(self, xml: bytes, ct_id: str | None = None, directory: str | None = None):
+        source = {k: v for k, v in (("ct_id", ct_id), ("directory", directory)) if v}
+        if not source:
+            m = _ROOT.search(xml[:4000])
+            if not m:
+                raise MappingError("the record does not name a model (no sdc4:dm-<ct_id> root); give ct_id or directory")
+            source = {"ct_id": m.group(1).decode()}
+        return self.packages.one(source)
+
+    def map_record(self, xml: bytes, mapper, mapping: Mapping | None = None, ct_id: str | None = None,
+                   directory: str | None = None, caller: str | None = None) -> CallResult:
+        base = {"caller": caller or "", "record_sha256": _sha(xml),
+                "mapper": {"package": mapper.package, "version": mapper.version, "name": mapper.name},
+                "target": {"name": mapper.target, "version": mapper.target_version},
+                "mapping": {"id": mapping.id, "version": mapping.version, "sha256": mapping.sha256} if mapping else None}
+        try:
+            pkg, model = self.model_for(xml, ct_id, directory)
+        except Exception as e:  # noqa: BLE001  an unreachable model is an error result, logged, never a crash
+            return self._finish(CallResult(ERROR, errors=[{"kind": "model", "reason": str(e)}]), base)
+        base["model"] = {"ct_id": model.ct_id, "schema_sha256": pkg.sha256}
+        try:
+            if mapping:
+                mapping.check(model.ct_id, pkg.sha256, mapper.target, mapper.target_version)
+        except MappingError as e:
+            return self._finish(CallResult(ERROR, errors=[{"kind": "mapping", "reason": str(e)}]), base)
+
+        gate = self.gate.check(xml, pkg, model)
+        if gate.outcome == REFUSED:
+            return self._finish(CallResult(REFUSED, errors=gate.errors), base)
+
+        try:
+            resolved = resolve(read_record(gate.xml, model), model, pkg, gate)
+            result = mapper.map(resolved, mapping)
+            report = build_report(resolved, result, mapper, mapping)
+        except IncompleteMapping as e:
+            return self._finish(CallResult(ERROR, errors=[{"kind": "incomplete mapping", "reason": str(e)}]), base)
+        except MappingError as e:                 # a code-table miss, a path the target refuses: never a guess
+            return self._finish(CallResult(ERROR, errors=[{"kind": "mapping", "reason": str(e)}]), base)
+        except Exception as e:  # noqa: BLE001  a mapper's failure is reported and logged, not raised into the stream
+            return self._finish(CallResult(ERROR, errors=[{"kind": type(e).__name__, "reason": str(e)}]), base)
+
+        base["record"] = {"instance_id": resolved.instance_id,
+                          "instance_version": resolved.attributes.get("instance_version", "")}
+        base["counts"] = report.counts
+        base["exceptional"] = [{"position": e["position"], "ct_id": e["ct_id"], "code": e["code"], "origin": e["origin"],
+                                "class": e["class"]} for e in report.exceptional]
+        base["outputs"] = {name: _sha(data) for name, data in sorted(result.files.items())}
+        return self._finish(CallResult(gate.outcome, files=result.files, report=report), base)
+
+    def _finish(self, result: CallResult, base: dict) -> CallResult:
+        if self.log is not None:
+            entry = {**base, "outcome": result.outcome,
+                     # kinds and paths only: a failure reason can quote the value that failed
+                     "errors": [{"kind": e.get("kind", ""), "path": e.get("path", "")} for e in result.errors]}
+            result.log_hash = self.log.append(entry)["hash"]
+        return result

sdc_mapper/gate.py

@@ -0,0 +1,180 @@
+"""The validation gate (PRD section 5): every record is valid, valid with Exceptional Values, or refused.
+
+The gate uses sdcvalidator's public recovery API (`SDC4Validator.recover()`, 4.6.0), which keeps the Exceptional
+Values a record already carries (a refusal, a masked field: what this line exists to report), inserts one beside each
+value that fails its constraint and reports it as data with the constraint that failed, and returns every error it
+cannot cover rather than skipping it. The gate refuses a record with any such error: nothing is projected that the
+gate has not accounted for.
+
+The model's schema includes the SDC4 reference model from https://semanticdatacharter.com/ns/sdc4/sdc4.xsd. The gate
+maps that include to the copy shipped in this package, so validation never touches the network, and a schema is built
+once per SHA-256 and kept for the life of the process.
+"""
+from __future__ import annotations
+
+import hashlib
+import io
+import re
+import tempfile
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from sdcreader import LEAF_TYPES, Model, ModelPackage
+from sdcvalidator import (SDC4StructuralValidationError, SDC4Validator, build_xsd11_schema, etree_tostring,
+                          validate_sdc4_schema_compliance)
+from xmlschema import XMLResource
+
+SDC4_NS = "https://semanticdatacharter.com/ns/sdc4/"
+SDC4_XSD = Path(__file__).with_name("schemas") / "sdc4.xsd"
+
+VALID = "valid"
+VALID_WITH_EV = "valid_with_ev"
+REFUSED = "refused"
+
+
+@dataclass
+class Recovery:
+    """One value the gate covered with an Exceptional Value."""
+    position: int          # the value's position in the record, as sdcreader numbers it
+    ct_id: str
+    code: str              # the ISO 21090 code inserted
+    constraint: str        # what failed, from the validator
+
+
+@dataclass
+class GateResult:
+    outcome: str                                   # VALID, VALID_WITH_EV or REFUSED
+    xml: bytes | None = None                       # the record to read: the input, or the input with recoveries
+    recoveries: list[Recovery] = field(default_factory=list)
+    errors: list[dict] = field(default_factory=list)   # for REFUSED: each error that could not be covered
+
+    @property
+    def refused(self) -> bool:
+        return self.outcome == REFUSED
+
+
+def _map_sdc4(url: str) -> str:
+    return str(SDC4_XSD) if url.rstrip("/").endswith("/ns/sdc4/sdc4.xsd") else url
+
+
+class Gate:
+    """Builds and caches one validator per model schema, keyed by the schema's SHA-256."""
+
+    def __init__(self, workdir: str | Path | None = None):
+        self._dir = Path(workdir) if workdir else Path(tempfile.mkdtemp(prefix="sdc-mapper-schemas-"))
+        self._dir.mkdir(parents=True, exist_ok=True)
+        self._validators: dict[str, SDC4Validator] = {}
+        self._lock = threading.Lock()
+
+    def validator(self, pkg: ModelPackage) -> SDC4Validator:
+        sha = pkg.sha256
+        with self._lock:
+            v = self._validators.get(sha)
+            if v is None:
+                path = self._dir / f"{sha}.xsd"
+                if not path.exists():
+                    path.write_bytes(pkg.xsd_bytes)
+                ok, problems = validate_sdc4_schema_compliance(str(path))
+                if not ok:
+                    raise ValueError("the model's schema violates SDC4 compliance: " + "; ".join(problems))
+                schema = build_xsd11_schema(str(path), validation="lax", uri_mapper=_map_sdc4, allow="local")
+                v = SDC4Validator(schema, check_sdc4_compliance=False, validation="lax")
+                self._validators[sha] = v
+            return v
+
+    def check(self, xml: bytes, pkg: ModelPackage, model: Model) -> GateResult:
+        try:
+            resource = XMLResource(io.BytesIO(xml))
+        except Exception as e:  # noqa: BLE001  xmlschema raises its own and the parser's errors
+            return GateResult(REFUSED, errors=[{"kind": "not well-formed", "reason": str(e)}])
+        if resource.root.tag != f"{{{SDC4_NS}}}dm-{model.ct_id}":
+            return GateResult(REFUSED, errors=[{"kind": "wrong model",
+                                                "reason": f"the record's root is {resource.root.tag}, not sdc4:dm-{model.ct_id}"}])
+        try:
+            result = self.validator(pkg).recover(resource, remove_existing_ev=False)
+        except SDC4StructuralValidationError as e:
+            return GateResult(REFUSED, errors=[_summary(err, "structural") for err in e.errors])
+        if not result.ok:
+            return GateResult(REFUSED, errors=[_summary(err, "not recoverable") for err in result.unrecoverable])
+        if not result.recoveries:
+            return GateResult(VALID, xml=xml)
+
+        root = result.tree.getroot()
+        positions = _value_positions(root, model)
+        recoveries = []
+        for r in result.inserted:     # an Exceptional Value the record already carried is the source's, not the gate's
+            comp = _component_element(root, _find(root, r.path), model)
+            if comp is None:
+                return GateResult(REFUSED, errors=[{"kind": "not recoverable", "path": r.path,
+                                                    "reason": "the recovered value is not inside a value component"}])
+            recoveries.append(Recovery(position=positions[id(comp)], ct_id=comp.tag.split("ms-", 1)[1], code=r.code,
+                                       constraint=r.reason))
+        recovered = etree_tostring(root, namespaces=resource.get_namespaces(root_only=False) or {"sdc4": SDC4_NS},
+                                   xml_declaration=True, encoding="utf-8")
+        return GateResult(VALID_WITH_EV, xml=recovered if isinstance(recovered, bytes) else recovered.encode("utf-8"),
+                          recoveries=recoveries)
+
+
+def _reason(e) -> str:
+    return " ".join((e.reason or e.message or "").split())
+
+
+def _summary(e, kind: str) -> dict:
+    return {"kind": kind, "path": e.path or "", "reason": _reason(e)}
+
+
+_STEP = re.compile(r"/((?:\{[^}]*\})?[^/{\[]+)(?:\[(\d+)\])?")
+
+
+def _find(root, path: str):
+    """The element a Clark-form error path names (`/{ns}dm-x/{ns}ms-y/xdcount-value`), or None. The namespace URI
+    holds slashes, so the path is tokenized by step, not split."""
+    steps = _STEP.findall(path)
+    if not steps or steps[0][0] != root.tag:
+        return None
+    el = root
+    for name, idx in steps[1:]:
+        matches = [c for c in el if c.tag == name]
+        i = int(idx) - 1 if idx else 0
+        if i >= len(matches):
+            return None
+        el = matches[i]
+    return el
+
+
+def _component_element(root, el, model: Model):
+    """The leaf component element (`sdc4:ms-<ct_id>` of a value type) that holds `el`, or None."""
+    parents = {c: p for p in root.iter() for c in p}
+    while el is not None:
+        if el.tag.startswith(f"{{{SDC4_NS}}}ms-"):
+            comp = model.components.get(el.tag.split("ms-", 1)[1])
+            if comp is not None and comp.sdc_type in LEAF_TYPES:
+                return el
+        el = parents.get(el)
+    return None
+
+
+def _value_positions(root, model: Model) -> dict[int, int]:
+    """Each leaf component element's position among the record's values, in document order: the numbering
+    sdcreader's `read_record` gives the same record."""
+    out, n = {}, 0
+
+    def walk(el):
+        nonlocal n
+        for c in el:
+            if not (isinstance(c.tag, str) and c.tag.startswith(f"{{{SDC4_NS}}}ms-")):
+                continue
+            comp = model.components.get(c.tag.split("ms-", 1)[1])
+            if comp is not None and comp.sdc_type in LEAF_TYPES:
+                out[id(c)] = n
+                n += 1
+            else:
+                walk(c)
+
+    walk(root)
+    return out
+
+
+def sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()

sdc_mapper/log.py

@@ -0,0 +1,108 @@
+"""The log (PRD section 12): one JSON Lines entry per call, whatever the outcome, anchored and chained.
+
+Each entry carries the SHA-256 of the input record, the model schema, the mapping file and each output, so it can be
+checked against the artifacts it names, and the SHA-256 of the previous entry, so a deleted or edited entry shows.
+No values: identifiers, classes, codes and hashes only. A failure reason is not logged either, because a validator's
+reason can quote the rejected value.
+
+Files are `mapper-YYYY-MM-DD.jsonl` (UTC) in the log directory; the chain runs across files. Writes are serialized
+within a process by a lock and across processes by an exclusive file lock on `mapper.lock`.
+"""
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import json
+import threading
+from collections import Counter
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+GENESIS = "0" * 64
+LOG_VERSION = 1
+
+
+def entry_hash(entry: dict) -> str:
+    """The SHA-256 of an entry's canonical JSON, without its own `hash` field."""
+    body = {k: v for k, v in entry.items() if k != "hash"}
+    return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest()
+
+
+class Log:
+    def __init__(self, directory: str | Path):
+        self.dir = Path(directory)
+        self.dir.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+
+    def _files(self) -> list[Path]:
+        return sorted(self.dir.glob("mapper-*.jsonl"))
+
+    def _last_hash(self) -> str:
+        for f in reversed(self._files()):
+            lines = [l for l in f.read_text(encoding="utf-8").splitlines() if l.strip()]
+            if lines:
+                return json.loads(lines[-1])["hash"]
+        return GENESIS
+
+    def append(self, entry: dict, now: datetime | None = None) -> dict:
+        now = now or datetime.now(timezone.utc)
+        with self._lock, open(self.dir / "mapper.lock", "a") as lockf:
+            fcntl.flock(lockf, fcntl.LOCK_EX)
+            try:
+                full = {"log_version": LOG_VERSION, "time": now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), **entry,
+                        "previous": self._last_hash()}
+                full["hash"] = entry_hash(full)
+                path = self.dir / f"mapper-{now.strftime('%Y-%m-%d')}.jsonl"
+                with open(path, "a", encoding="utf-8") as f:
+                    f.write(json.dumps(full, sort_keys=True, ensure_ascii=False) + "\n")
+                return full
+            finally:
+                fcntl.flock(lockf, fcntl.LOCK_UN)
+
+    def entries(self, since: datetime | None = None) -> list[dict]:
+        out = []
+        for f in self._files():
+            for line in f.read_text(encoding="utf-8").splitlines():
+                if line.strip():
+                    e = json.loads(line)
+                    if since is None or e["time"] >= since.strftime("%Y-%m-%dT%H:%M:%S.%fZ"):
+                        out.append(e)
+        return out
+
+    def verify(self) -> tuple[bool, str]:
+        """Walk the chain from the first entry: every hash recomputes and every entry names its predecessor."""
+        prev = GENESIS
+        n = 0
+        for e in self.entries():
+            n += 1
+            if e.get("previous") != prev:
+                return False, f"entry {n} ({e.get('time')}) does not follow the entry before it"
+            if entry_hash(e) != e.get("hash"):
+                return False, f"entry {n} ({e.get('time')}) has been altered"
+            prev = e["hash"]
+        return True, f"{n} entries, chain intact"
+
+    def summary(self, days: float = 7, now: datetime | None = None) -> dict:
+        now = now or datetime.now(timezone.utc)
+        rows = self.entries(since=now - timedelta(days=days))
+        by_target: dict[str, dict] = {}
+        for e in rows:
+            t = by_target.setdefault(e.get("target", {}).get("name", "?"), {
+                "calls": 0, "outcomes": Counter(), "values": Counter(), "exceptional": Counter(), "models": set(),
+                "mappings": set()})
+            t["calls"] += 1
+            t["outcomes"][e.get("outcome", "?")] += 1
+            for k, n in (e.get("counts") or {}).items():
+                if k == "exceptional":
+                    t["exceptional"].update(n)
+                else:
+                    t["values"][k] += n
+            if e.get("model", {}).get("ct_id"):
+                t["models"].add(e["model"]["ct_id"])
+            if (e.get("mapping") or {}).get("sha256"):
+                t["mappings"].add(e["mapping"]["sha256"])
+        return {"days": days, "calls": len(rows),
+                "targets": {k: {"calls": v["calls"], "outcomes": dict(v["outcomes"]), "values": {k: n for k, n in v["values"].items() if n},
+                                "exceptional": {k: n for k, n in v["exceptional"].items() if n},
+                                "models": sorted(v["models"]),
+                                "mappings": sorted(v["mappings"])} for k, v in by_target.items()}}