mk2vsc 0.1.2__py3-none-any.whl

mk2vsc/__init__.py

@@ -0,0 +1,35 @@
+"""
+rvms -- read, verify, edit and diff Victron VEConfigure ``.rvms`` configuration files without VEConfigure.
+
+Zero dependencies. Python 3.9+.
+
+Quick tour::
+
+    from mk2vsc import RvmsFile, units_by_serial, decode_file, set_settings, diff_files
+
+    f = RvmsFile.load("system.rvms")
+    f.all_checksums_ok                      # every section's integrity trailer validates
+    units_by_serial(f)["HQ2414U6FVN"].setting(2) / 100   # absorption voltage
+
+Safety model in one paragraph: this library produces *files*.  It never talks to an inverter.  A valid
+file is necessary, not sufficient: editing the right offset is on you (see ``fields.py`` confidence
+levels), and the only proven-safe edits are length-preserving value changes to the settings array.
+Adding, removing or transplanting an assistant (ESS etc.) by file has never worked for us and has
+disrupted live systems.  Read docs/SAFETY.md before uploading anything.
+"""
+from .sections import RvmsFile, Section, RvmsParseError, sum32_le, scan_unit_blocks
+from .units import UnitBlock, unit_blocks, units_by_serial
+from .fields import FIELDS, BY_ID, BY_NAME, lookup, Field
+from .decode import decode_file, decode_bytes
+from .writer import set_settings, WriteRefused
+from .diff import diff_files, diff_bytes
+from .qualify import qualify_file, Intent
+
+__version__ = "0.1.2"
+__all__ = [
+    "RvmsFile", "Section", "RvmsParseError", "sum32_le", "scan_unit_blocks",
+    "UnitBlock", "unit_blocks", "units_by_serial",
+    "FIELDS", "BY_ID", "BY_NAME", "lookup", "Field",
+    "decode_file", "decode_bytes", "set_settings", "WriteRefused", "diff_files", "diff_bytes",
+    "qualify_file", "Intent",
+]

mk2vsc/assistants.py

@@ -0,0 +1,105 @@
+"""
+READ-ONLY parsing of the assistant area that follows the settings array in a unit block.
+
+What we can say with evidence (docs/ASSISTANTS.md has the full story):
+
+* Bare block (no assistant): the area is the 9 bytes ``ff ff ff ff 00 00 ff 00 0b`` (device form) --
+  i.e. an empty record header (``ff ff`` marker, ``ff ff`` subtype, length ``00 00``) plus 3 trailer bytes.
+* Block with the ESS assistant installed by the GUI: one or more records framed
+  ``f5 ff <subtype u16> <len u16> <body>``; body lengths 704 and 1152 in every working install we hold
+  (one per inverter of the pair; the two bodies differ from each other, and are byte-identical across
+  systems except for a single primary/secondary flag byte).  The device pads records with ``0xff`` runs
+  and appends trailer bytes; the GUI's upload form writes the same records compact.
+* Stub: after VEConfigure accepted one of our transplanted files it wrote a 64-byte empty container
+  ``40 00 a7 fe 00 00 57 01`` + ``0xff`` filler + ``c0 0a`` on both inverters and discarded our payload.
+  Its presence in a download is the signature of a failed by-file assistant install.
+
+We do NOT understand the record body.  It looks like a compiled program (entropy ~6.2 bits/byte,
+recurring 2-3 byte opcodes, embedded parameter values such as 48.00 V and 10 %).  This module reports
+structure; it does not author it.
+"""
+from __future__ import annotations
+
+import struct
+from typing import Dict, List
+
+from .units import UnitBlock
+
+RECORD_MARK = b"\xf5\xff"
+EMPTY_MARK = b"\xff\xff"
+CONTAINER_SIG = b"\xa7\xfe\x00\x00\x57\x01"
+STUB_MAGIC = b"\x40\x00" + CONTAINER_SIG     # len 64 + signature, as it appears after the ff ff ff ff header
+
+
+def parse_records(area: bytes):
+    """Walk ``marker(2) subtype(2) len(2) body`` records from the start of the area.
+
+    Returns (records, tail_offset).  Walking stops at the first byte pair that is not a known marker.
+    """
+    records: List[Dict] = []
+    pos = 0
+    while pos + 6 <= len(area) and area[pos: pos + 2] in (EMPTY_MARK, RECORD_MARK):
+        marker = area[pos: pos + 2]
+        subtype, length = struct.unpack_from("<HH", area, pos + 2)
+        body = area[pos + 6: pos + 6 + length]
+        records.append({"offset": pos, "marker": marker.hex(), "subtype": f"{subtype:04x}", "length": length,
+                        "body_sha8": _sha8(body) if length else "", "nonpad_bytes": sum(1 for b in body if b != 0xFF),
+                        "container_signature": body.startswith(CONTAINER_SIG),
+                        "truncated": len(body) < length})
+        pos += 6 + length
+    return records, pos
+
+
+def parse_assistant_area(u: UnitBlock) -> Dict:
+    """Describe the assistant area of a unit block.
+
+    Uniform model (every block in the corpus fits it)::
+
+        area := record* tail
+        record := marker(2) subtype(2) len(2) body[len]
+        marker  ff ff  -> empty slot / container.  Bare blocks: len 0.  Two June-2026 files from an older
+                          tool build: len 6, body a7 fe 00 00 57 01.  Stub written by VEConfigure after it
+                          discarded a transplanted assistant: len 64, same signature + 0xff filler.
+                f5 ff  -> assistant record.  GUI-installed ESS: one 704-byte and one 1152-byte record per
+                          system (one on each inverter), subtype 0101 / 0001.
+        tail   := padding(0xff)* | ff | u16 free
+                  On bare, legacy and stub blocks free == 2816 - bytes used; see docs/FORMAT.md for ESS.
+
+    A ``f5 ff`` header with len 0 where ``ff ff`` is expected is residue seen on downloads taken after a
+    rejected or rolled-back assistant upload; functionally bare.
+    """
+    area = u.assistant_area
+    records, tail_off = parse_records(area)
+    tail = area[tail_off:]
+    out: Dict = {"bytes": len(area), "records": records, "tail_bytes": len(tail), "tail_hex": tail[-8:].hex(" "),
+                 "stub": False, "kind": "unknown", "summary": ""}
+    if len(tail) >= 3 and tail[-3] == 0xFF:
+        out["free"] = struct.unpack_from("<H", tail, len(tail) - 2)[0]
+        out["used"] = tail_off
+        out["free_plus_used"] = out["free"] + tail_off
+    if any(r["truncated"] for r in records):
+        out["kind"] = "malformed"
+        out["summary"] = "record length exceeds the area (malformed)"
+        return out
+    real = [r for r in records if r["marker"] == "f5ff" and r["length"] > 0]
+    containers = [r for r in records if r["marker"] == "ffff" and r["length"] > 0]
+    if any(r["length"] >= 64 and r["container_signature"] for r in containers):
+        out["kind"], out["stub"] = "stub", True
+        out["summary"] = "EMPTY STUB container (signature of a failed by-file install)"
+    elif real:
+        out["kind"] = "records"
+        out["summary"] = "assistant records: " + ", ".join(f"{r['length']}B/{r['subtype']}" for r in real)
+    elif containers:
+        out["kind"] = "container"
+        out["summary"] = f"empty {containers[0]['length']}-byte container (no program)"
+    elif records:
+        out["kind"] = "none"
+        out["summary"] = "no assistant" + (" (empty record residue)" if records[0]["marker"] == "f5ff" else "")
+    else:
+        out["summary"] = f"{len(area)} unrecognised bytes"
+    return out
+
+
+def _sha8(b: bytes) -> str:
+    import hashlib
+    return hashlib.sha256(b).hexdigest()[:8]

mk2vsc/cli.py

@@ -0,0 +1,249 @@
+"""
+Command-line interface.
+
+    mk2vsc info      FILE...                 one-screen summary (structure, inverters, confirmed settings)
+    mk2vsc validate  FILE...                 checksum + structure check; exit 1 on any failure
+    mk2vsc decode    FILE [--json] [--all]   every setting with label/confidence
+    mk2vsc diff      A B [--json]            by-serial comparison; says whether only bookkeeping changed
+    mk2vsc set       IN OUT [--serial S] FIELD=VALUE ...      guarded edit (never uploads)
+    mk2vsc qualify   FILE... --intent intent.json             check against intended values
+    mk2vsc fix       IN OUT                  recompute every checksum (forensic use only)
+    mk2vsc fields                            print the settings table
+    mk2vsc census    FILE...                 one line per file (block lengths, flags, form, assistant kind)
+    mk2vsc history   FILE...                 dated change log mined from a library of downloads (by system, by serial)
+    mk2vsc experimental graft BASELINE TEMPLATE OUT [--install-state] [--capacity-ah N] --i-accept-the-risk
+    mk2vsc experimental to-upload-form DEVICE OUT [--reference GUI_EXPORT] --i-accept-the-risk
+                                            assistant-injection experiments; never produced a running system
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from .sections import RvmsFile, RvmsParseError
+from .units import unit_blocks
+from .decode import decode_file, brief
+from .diff import diff_files, render as render_diff
+from .writer import set_settings_file, WriteRefused
+from .qualify import Intent, qualify_file, render as render_qual
+from .fields import FIELDS
+from .assistants import parse_assistant_area
+from .history import load_snapshots, changes as history_changes, render as render_history
+
+
+def _load(path):
+    try:
+        return RvmsFile.load(path)
+    except (RvmsParseError, OSError) as e:
+        print(f"{path}: {e}", file=sys.stderr)
+        return None
+
+
+def cmd_info(a):
+    rc = 0
+    for p in a.files:
+        try:
+            print(f"== {p}")
+            print(brief(decode_file(p)))
+        except Exception as e:  # noqa: BLE001
+            rc = 1
+            print(f"  ERROR {e}")
+    return rc
+
+
+def cmd_validate(a):
+    rc = 0
+    for p in a.files:
+        f = _load(p)
+        if f is None:
+            rc = 1
+            continue
+        for name, start, stored, computed, ok in f.checksum_report():
+            if not ok or a.verbose:
+                print(f"{'OK ' if ok else 'BAD'} {p} {name}@0x{start:x} stored={stored:08x} computed={computed:08x}")
+            if not ok:
+                rc = 1
+        if f.all_checksums_ok:
+            print(f"OK  {p}  ({len(f.sections)} sections, {len(f.unit_sections)} inverters)")
+    return rc
+
+
+def cmd_decode(a):
+    try:
+        d = decode_file(a.file, include_unknown=a.all)
+    except (RvmsParseError, OSError) as e:
+        print(f"{a.file}: {e}", file=sys.stderr)
+        return 1
+    if a.json:
+        print(json.dumps(d, indent=1, default=str))
+    else:
+        print(brief(d))
+        for u in d["units"]:
+            print(f"\n{u['serial']}: all named settings")
+            for s in u["settings"]:
+                if s.get("name") or a.all:
+                    v = s.get("value", s["raw"])
+                    print(f"  {s['id']:3d} {s['offset']} {s.get('name') or '-':30s} raw={s['raw']:6d} value={v!s:>9} "
+                          f"{s.get('unit','')} [{s['confidence']}]")
+    return 0
+
+
+def cmd_diff(a):
+    try:
+        d = diff_files(a.a, a.b)
+    except (RvmsParseError, OSError, ValueError) as e:
+        print(f"cannot diff: {e}", file=sys.stderr)
+        return 1
+    print(json.dumps(d.as_dict(), indent=1) if a.json else render_diff(d))
+    return 0 if (d.identical or d.only_bookkeeping) else 2
+
+
+def cmd_set(a):
+    changes = []
+    for kv in a.assignments:
+        if "=" not in kv:
+            print(f"bad assignment {kv!r}; expected FIELD=VALUE", file=sys.stderr)
+            return 2
+        k, v = kv.split("=", 1)
+        try:
+            val = int(v, 0) if v.strip().lstrip("-").isdigit() or v.startswith("0x") else float(v)
+        except ValueError:
+            print(f"bad value {v!r} for {k}; expected a number", file=sys.stderr)
+            return 2
+        changes.append((a.serial, k, val))
+    try:
+        edits = set_settings_file(a.inp, a.out, changes, allow_unverified=a.i_know_this_is_unverified,
+                                  allow_out_of_range=a.allow_out_of_range)
+    except KeyError as e:
+        print(f"REFUSED: unknown field {e}; see `mk2vsc fields`", file=sys.stderr)
+        return 1
+    except (WriteRefused, ValueError, RvmsParseError, OSError) as e:
+        print(f"REFUSED: {e}", file=sys.stderr)
+        return 1
+    for e in edits:
+        d = e.as_dict()
+        print(f"{d['serial']}  {d['field']}  {d['old']} -> {d['new']} {d['unit']}  ({d['block_offset']} / file {d['file_offset']})")
+    print(f"wrote {a.out}; verified: only the listed bytes and their section checksums changed")
+    return 0
+
+
+def cmd_qualify(a):
+    try:
+        intent = Intent.load(a.intent) if a.intent else Intent(settings={})
+    except (OSError, ValueError) as e:
+        print(f"cannot load intent {a.intent}: {e}", file=sys.stderr)
+        return 2
+    rc = 0
+    for p in a.files:
+        try:
+            ok, res = qualify_file(p, intent)
+        except KeyError as e:
+            print(f"{p}: intent names an unknown field {e}; see `mk2vsc fields`", file=sys.stderr)
+            return 2
+        print(render_qual(ok, res, p))
+        rc |= 0 if ok else 1
+    return rc
+
+
+def cmd_fix(a):
+    f = _load(a.inp)
+    if f is None:
+        return 1
+    out = f.fixed().to_bytes()
+    with open(a.out, "wb") as fh:
+        fh.write(out)
+    changed = sum(1 for s in f.sections if not s.checksum_ok)
+    print(f"wrote {a.out}: {changed} checksum(s) recomputed")
+    return 0
+
+
+def cmd_fields(a):
+    print(f"{'id':>3} {'offset':>7} {'name':30s} {'scale':>5} {'unit':6s} {'conf':9s} label")
+    for f in FIELDS:
+        print(f"{f.id:3d} +0x{f.offset:03x} {f.name:30s} {f.scale:>5g} {f.unit:6s} {f.confidence:9s} {f.label}")
+    return 0
+
+
+def cmd_census(a):
+    for p in a.files:
+        f = _load(p)
+        if f is None:
+            continue
+        cells = []
+        for u in unit_blocks(f):
+            asst = parse_assistant_area(u)
+            cells.append(f"{u.serial}:{u.length:#x}:{u.assistant_flag:02x}:{'U' if u.is_upload_form else 'd'}:{asst['kind']}")
+        print(f"{f.length:6d} {'ok ' if f.all_checksums_ok else 'BAD'} | {' '.join(cells)} | {p}")
+    return 0
+
+
+def cmd_history(a):
+    snaps, skipped = load_snapshots(a.files)
+    chs = history_changes(snaps)
+    if a.json:
+        print(json.dumps([{"system": c.system, "serial": c.serial, "what": c.what, "old": c.old, "new": c.new,
+                           "confidence": c.confidence, "after": c.after.when, "before": c.before.when,
+                           "file_after": c.after.path, "file_before": c.before.path} for c in chs], indent=1, default=str))
+    else:
+        print(render_history(snaps, chs, skipped))
+    return 0
+
+
+def cmd_experimental(a):
+    if not a.i_accept_the_risk:
+        print("experimental commands have never produced a running ESS system and have disrupted live systems; "
+              "read docs/ESS_INJECTION.md, then pass --i-accept-the-risk", file=sys.stderr)
+        return 2
+    from .experimental import graft, to_upload_form, GraftRefused, TransformRefused
+    try:
+        if a.what == "graft":
+            out, checks = graft(open(a.baseline, "rb").read(), open(a.template, "rb").read(),
+                                install_state=a.install_state, capacity_ah=a.capacity_ah)
+            for k, v in checks.items():
+                print(f"  {k}: {v}")
+        else:
+            ref = open(a.reference, "rb").read() if a.reference else None
+            out = to_upload_form(open(a.device, "rb").read(), reference=ref)
+        with open(a.out, "wb") as fh:
+            fh.write(out)
+        print(f"wrote {a.out} ({len(out)} bytes). EXPERIMENTAL: see docs/ESS_INJECTION.md before uploading.")
+        return 0
+    except (GraftRefused, TransformRefused, RvmsParseError, OSError) as e:
+        print(f"REFUSED: {e}", file=sys.stderr)
+        return 1
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="mk2vsc", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--version", action="version", version=__version__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("info"); s.add_argument("files", nargs="+"); s.set_defaults(fn=cmd_info)
+    s = sub.add_parser("validate"); s.add_argument("files", nargs="+"); s.add_argument("-v", "--verbose", action="store_true"); s.set_defaults(fn=cmd_validate)
+    s = sub.add_parser("decode"); s.add_argument("file"); s.add_argument("--json", action="store_true"); s.add_argument("--all", action="store_true", help="include unknown settings"); s.set_defaults(fn=cmd_decode)
+    s = sub.add_parser("diff"); s.add_argument("a"); s.add_argument("b"); s.add_argument("--json", action="store_true"); s.set_defaults(fn=cmd_diff)
+    s = sub.add_parser("set"); s.add_argument("inp"); s.add_argument("out"); s.add_argument("--serial", default=None, help="edit one inverter only (default: all)")
+    s.add_argument("--i-know-this-is-unverified", action="store_true", help="allow MEDIUM/LOW/UNKNOWN fields")
+    s.add_argument("--allow-out-of-range", action="store_true", help="skip the plausibility range and float<=absorption checks")
+    s.add_argument("assignments", nargs="+", metavar="FIELD=VALUE"); s.set_defaults(fn=cmd_set)
+    s = sub.add_parser("qualify"); s.add_argument("files", nargs="+"); s.add_argument("--intent", help="intent JSON"); s.set_defaults(fn=cmd_qualify)
+    s = sub.add_parser("fix"); s.add_argument("inp"); s.add_argument("out"); s.set_defaults(fn=cmd_fix)
+    s = sub.add_parser("fields"); s.set_defaults(fn=cmd_fields)
+    s = sub.add_parser("census"); s.add_argument("files", nargs="+"); s.set_defaults(fn=cmd_census)
+    s = sub.add_parser("history"); s.add_argument("files", nargs="+"); s.add_argument("--json", action="store_true"); s.set_defaults(fn=cmd_history)
+    x = sub.add_parser("experimental", help="assistant-injection experiments (read docs/ESS_INJECTION.md)")
+    xs = x.add_subparsers(dest="what", required=True)
+    g = xs.add_parser("graft"); g.add_argument("baseline"); g.add_argument("template"); g.add_argument("out")
+    g.add_argument("--install-state", action="store_true"); g.add_argument("--capacity-ah", type=int, default=None)
+    g.add_argument("--i-accept-the-risk", action="store_true"); g.set_defaults(fn=cmd_experimental)
+    t = xs.add_parser("to-upload-form"); t.add_argument("device"); t.add_argument("out"); t.add_argument("--reference")
+    t.add_argument("--i-accept-the-risk", action="store_true"); t.set_defaults(fn=cmd_experimental)
+
+    a = ap.parse_args(argv)
+    return a.fn(a)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

mk2vsc/decode.py

@@ -0,0 +1,79 @@
+"""
+Decode a ``.rvms`` into a plain dictionary: file structure, per-inverter identity, every setting with
+its label/confidence, and the assistant area summary.  JSON-serialisable.
+"""
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from .sections import RvmsFile
+from .units import unit_blocks, N_SETTINGS
+from .fields import BY_ID, UNKNOWN
+from .assistants import parse_assistant_area
+
+
+def decode_bytes(data: bytes, include_unknown: bool = True, include_raw_settings: bool = False) -> Dict:
+    f = RvmsFile.parse(data)
+    out: Dict = {
+        "length": f.length,
+        "sections": [
+            {"name": s.name.decode(), "start": s.start, "next": s.next_ptr, "payload_bytes": len(s.payload),
+             "checksum_stored": f"{s.stored_checksum:08x}", "checksum_computed": f"{s.computed_checksum:08x}",
+             "checksum_ok": s.checksum_ok}
+            for s in f.sections
+        ],
+        "all_checksums_ok": f.all_checksums_ok,
+        "units": [],
+    }
+    try:
+        mk = f.section(b"Mk2vscInfo")
+        vlen = int.from_bytes(mk.payload[4:6], "little")
+        out["format_version"] = mk.payload[6: 6 + vlen].decode()
+    except Exception:  # malformed or missing header
+        out["format_version"] = None
+
+    for u in unit_blocks(f):
+        d = u.summary()
+        settings = u.settings()
+        named: List[Dict] = []
+        for sid in range(N_SETTINGS):
+            raw = settings[sid]
+            fld = BY_ID.get(sid)
+            if fld is None:
+                if not include_unknown:
+                    continue
+                named.append({"id": sid, "offset": f"+0x{u.setting_offset(sid):03x}", "raw": raw,
+                              "name": None, "confidence": UNKNOWN})
+                continue
+            if fld.confidence == UNKNOWN and not include_unknown:
+                continue
+            entry = {"id": sid, "offset": f"+0x{u.setting_offset(sid):03x}", "raw": raw, "name": fld.name,
+                     "label": fld.label, "value": fld.decode(raw), "unit": fld.unit, "confidence": fld.confidence}
+            if fld.bits:
+                entry["bits_set"] = [fld.bits[b] for b in fld.bits if raw & (1 << b)]
+            named.append(entry)
+        d["settings"] = named
+        if include_raw_settings:
+            d["settings_raw"] = settings
+        d["assistant"] = parse_assistant_area(u)
+        out["units"].append(d)
+    return out
+
+
+def decode_file(path: str, **kw) -> Dict:
+    with open(path, "rb") as fh:
+        return decode_bytes(fh.read(), **kw)
+
+
+def brief(d: Dict) -> str:
+    """Human-readable one-screen summary of a decoded file."""
+    lines = [f"format {d.get('format_version')}  length {d['length']}  checksums {'OK' if d['all_checksums_ok'] else 'BAD'}"]
+    for u in d["units"]:
+        a = u["assistant"]
+        lines.append(f"  {u['serial']}  fw {u['firmware']}  form={u['form']}  flag={u['assistant_flag']}  "
+                     f"saved {u['save_time_utc']}  assistant: {a['summary']}")
+        for s in u["settings"]:
+            if s.get("name") and s["confidence"] in ("CONFIRMED", "HIGH"):
+                v = s["value"]
+                lines.append(f"      {s['name']:28s} {v!s:>8} {s['unit']:4s} [{s['confidence']}] ({s['offset']})")
+    return "\n".join(lines)

mk2vsc/diff.py

@@ -0,0 +1,155 @@
+"""
+Compare two ``.rvms`` files **by inverter serial** and classify every differing byte.
+
+Why by serial: the two blocks of a parallel pair swap file position between downloads of the same
+system.  A naive byte diff of two consecutive downloads shows ~44 differences; compared by serial it is
+exactly 6 bookkeeping bytes per block (next-pointer, save timestamp, checksum).  That distinction is
+the basis of change verification: after an upload, the re-download must differ from what you uploaded
+*only* in bookkeeping.
+
+Classification of a differing block byte:
+
+    bookkeeping   next-pointer (+0x0f..0x10), save timestamp (+0x4f..0x52 device form), checksum trailer
+    setting       inside the settings array -> reported as (id, name, old, new)
+    header        other bytes before the settings array (identity, flags, form blob)
+    assistant     bytes in the assistant area
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from .sections import RvmsFile
+from .units import units_by_serial, UnitBlock, OFF_NEXT_PTR
+from .fields import BY_ID
+
+
+@dataclass
+class UnitDiff:
+    serial: str
+    length_a: int          # raw section length (name start .. checksum), position independent
+    length_b: int
+    form_a: str
+    form_b: str
+    bookkeeping: List[int] = field(default_factory=list)
+    settings: List[Dict] = field(default_factory=list)
+    header: List[Dict] = field(default_factory=list)
+    assistant: int = 0
+    note: str = ""
+
+    @property
+    def only_bookkeeping(self) -> bool:
+        same_len = self.length_a == self.length_b or self.form_a != self.form_b
+        return not self.settings and not self.header and self.assistant == 0 and same_len
+
+
+@dataclass
+class FileDiff:
+    identical: bool
+    length_a: int
+    length_b: int
+    prologue_identical: bool
+    only_in_a: List[str]
+    only_in_b: List[str]
+    units: List[UnitDiff]
+
+    @property
+    def only_bookkeeping(self) -> bool:
+        return (self.prologue_identical and not self.only_in_a and not self.only_in_b
+                and all(u.only_bookkeeping for u in self.units))
+
+    def as_dict(self) -> Dict:
+        return {
+            "identical": self.identical, "only_bookkeeping": self.only_bookkeeping,
+            "length": [self.length_a, self.length_b], "prologue_identical": self.prologue_identical,
+            "only_in_a": self.only_in_a, "only_in_b": self.only_in_b,
+            "units": [{"serial": u.serial, "length": [u.length_a, u.length_b], "form": [u.form_a, u.form_b],
+                       "bookkeeping_bytes": [f"+0x{o:03x}" for o in u.bookkeeping],
+                       "settings": u.settings, "header": u.header, "assistant_bytes_differ": u.assistant,
+                       "note": u.note} for u in self.units],
+        }
+
+
+def _classify(a: UnitBlock, b: UnitBlock) -> UnitDiff:
+    d = UnitDiff(a.serial, len(a.raw), len(b.raw), "upload" if a.is_upload_form else "device",
+                 "upload" if b.is_upload_form else "device")
+    ra, rb = a.raw, b.raw
+    cross_form = a.is_upload_form != b.is_upload_form
+    if cross_form:
+        d.note = ("different forms (device vs upload): offsets shift by 10 after +0x45 and the GUI writes compact "
+                  "assistant records, so lengths differ by form; settings compared by id")
+    elif len(ra) != len(rb):
+        d.note = "block length differs (assistant area changed)"
+    # settings compared by id regardless of form
+    sa, sb = a.settings(), b.settings()
+    for sid, (x, y) in enumerate(zip(sa, sb)):
+        if x != y:
+            f = BY_ID.get(sid)
+            d.settings.append({"id": sid, "name": f.name if f else None, "old_raw": x, "new_raw": y,
+                               "old": f.decode(x) if f else x, "new": f.decode(y) if f else y,
+                               "confidence": f.confidence if f else "UNKNOWN"})
+    # header bytes before the settings array, compared positionally only when forms match
+    if a.is_upload_form == b.is_upload_form:
+        bk = {OFF_NEXT_PTR, OFF_NEXT_PTR + 1} | {a.settings_offset - 10 + i for i in range(4)}  # save ts
+        n = min(a.settings_offset, b.settings_offset)
+        for i in range(n):
+            if ra[i] != rb[i]:
+                if i in bk:
+                    d.bookkeeping.append(i)
+                else:
+                    d.header.append({"offset": f"+0x{i:03x}", "old": f"{ra[i]:02x}", "new": f"{rb[i]:02x}"})
+        # assistant area
+        aa, ab = a.assistant_area, b.assistant_area
+        if aa != ab:
+            d.assistant = sum(1 for x, y in zip(aa, ab) if x != y) + abs(len(aa) - len(ab))
+    else:
+        # cannot compare assistant bytes across forms (padding differs); compare record structure instead
+        from .assistants import parse_assistant_area
+        ka = [(r["marker"], r["subtype"]) for r in parse_assistant_area(a)["records"]]
+        kb = [(r["marker"], r["subtype"]) for r in parse_assistant_area(b)["records"]]
+        d.assistant = 0 if ka == kb else 1
+    # checksum trailer counted as bookkeeping when it differs
+    if ra[-4:] != rb[-4:]:
+        d.bookkeeping.extend(range(len(ra) - 4, len(ra)))
+    return d
+
+
+def diff_bytes(data_a: bytes, data_b: bytes) -> FileDiff:
+    fa, fb = RvmsFile.parse(data_a), RvmsFile.parse(data_b)
+    ua, ub = units_by_serial(fa), units_by_serial(fb)
+    pro_a = fa.magic_raw + b"".join(s.raw for s in fa.sections if not s.is_unit)
+    pro_b = fb.magic_raw + b"".join(s.raw for s in fb.sections if not s.is_unit)
+    # BareSettingInfo carries its own pointer to the first unit block, which is position independent; the
+    # Mk2vscInfo/BareSettingInfo payloads are what we call the prologue.
+    pro_same = [s.payload for s in fa.sections if not s.is_unit] == [s.payload for s in fb.sections if not s.is_unit]
+    units = [_classify(ua[s], ub[s]) for s in sorted(set(ua) & set(ub))]
+    return FileDiff(identical=(data_a == data_b), length_a=len(data_a), length_b=len(data_b),
+                    prologue_identical=pro_same, only_in_a=sorted(set(ua) - set(ub)),
+                    only_in_b=sorted(set(ub) - set(ua)), units=units)
+
+
+def diff_files(path_a: str, path_b: str) -> FileDiff:
+    with open(path_a, "rb") as fa, open(path_b, "rb") as fb:
+        return diff_bytes(fa.read(), fb.read())
+
+
+def render(d: FileDiff) -> str:
+    if d.identical:
+        return "identical"
+    lines = [f"lengths {d.length_a} -> {d.length_b}; prologue {'same' if d.prologue_identical else 'DIFFERS'}; "
+             f"verdict: {'ONLY BOOKKEEPING (settings verbatim)' if d.only_bookkeeping else 'CONTENT CHANGED'}"]
+    for s in d.only_in_a:
+        lines.append(f"  {s}: only in A")
+    for s in d.only_in_b:
+        lines.append(f"  {s}: only in B")
+    for u in d.units:
+        lines.append(f"  {u.serial}: len {u.length_a}->{u.length_b} form {u.form_a}->{u.form_b} "
+                     f"bookkeeping={len(u.bookkeeping)}B header={len(u.header)}B assistant={u.assistant}B"
+                     + (f"  [{u.note}]" if u.note else ""))
+        for s in u.settings:
+            lines.append(f"      setting {s['id']:3d} {s['name'] or '?':28s} {s['old']} -> {s['new']}  [{s['confidence']}]")
+        for h in u.header[:12]:
+            lines.append(f"      header {h['offset']}: {h['old']} -> {h['new']}")
+        if len(u.header) > 12:
+            lines.append(f"      ... {len(u.header)-12} more header bytes")
+    return "\n".join(lines)

mk2vsc/experimental/__init__.py

@@ -0,0 +1,11 @@
+"""
+Experimental: assistant (ESS) injection by file.
+
+Nothing in this package has produced a running ESS system.  It is published so the work can be picked
+up, reviewed, or corrected.  Read docs/ESS_INJECTION.md first.  The CLI exposes these only behind
+``mk2vsc experimental ... --i-accept-the-risk``.
+"""
+from .ess_graft import graft, GraftRefused, INSTALL_STATE
+from .upload_form import to_upload_form, compare_per_slot, TransformRefused, BLOB12
+
+__all__ = ["graft", "GraftRefused", "INSTALL_STATE", "to_upload_form", "compare_per_slot", "TransformRefused", "BLOB12"]