ledgdex 1.0.20__py3-none-any.whl

ledgdex/__init__.py

@@ -0,0 +1,2 @@
+"""Ledgdex: an agent-centric ledger with triple-entry accounting, kept in a dex. See architecture/SPEC.md."""
+__version__ = '1.0.20'

ledgdex/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

ledgdex/canon.py

@@ -0,0 +1,99 @@
+"""Canonical JSON (spec 1.2): the exact bytes that are hashed and signed."""
+import hashlib, json, re
+
+KEY = re.compile(r'[a-z][a-z0-9_]*\Z')
+ID_KEY = re.compile(r'[\x21-\x7e]+\Z')  # state output (spec 6.3) is keyed by ids: printable ASCII
+MAX_INT = 2 ** 53 - 1
+MAX_DEPTH = 32  # spec 1.2 rule 9: arrays and objects nest at most this deep
+
+
+class CanonError(ValueError):
+    pass
+
+
+def check(v, keys=KEY, depth=1):
+    """Raise CanonError unless v follows rules 1-3 and 9."""
+    if v is None or isinstance(v, bool):
+        return
+    if isinstance(v, int):
+        if not -MAX_INT <= v <= MAX_INT:
+            raise CanonError('integer out of range')
+        return
+    if isinstance(v, float):
+        raise CanonError('floats are not allowed')
+    if isinstance(v, str):
+        try:
+            v.encode('utf-8')
+        except UnicodeEncodeError:
+            raise CanonError('string is not valid Unicode')
+        return
+    if isinstance(v, (list, dict)) and depth > MAX_DEPTH:
+        raise CanonError('too deeply nested')
+    if isinstance(v, list):
+        for x in v:
+            check(x, keys, depth + 1)
+        return
+    if isinstance(v, dict):
+        for k, x in v.items():
+            if not isinstance(k, str) or not keys.match(k):
+                raise CanonError('bad key: ' + str(k))
+            check(x, keys, depth + 1)
+        return
+    raise CanonError('value of type ' + type(v).__name__ + ' is not allowed')
+
+
+def canon(v, keys=KEY):
+    """canon(x): canonical JSON bytes. keys=ID_KEY only for the state object, whose keys are ids."""
+    check(v, keys)
+    return json.dumps(v, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode('utf-8')
+
+
+def sha256(data):
+    return 'sha256:' + hashlib.sha256(data).hexdigest()
+
+
+def hash_(v):
+    """hash(x) = "sha256:" + hex(SHA-256(canon(x)))."""
+    return sha256(canon(v))
+
+
+def _pairs(pairs):
+    d = {}
+    for k, v in pairs:
+        if k in d:
+            raise CanonError('duplicate key: ' + k)
+        d[k] = v
+    return d
+
+
+def _no_float(s):
+    raise CanonError('floats are not allowed')
+
+
+def _int(s):
+    # refuse a long integer before converting it: converting a huge one takes quadratic time on older Pythons
+    if len(s.lstrip('-')) > 16:
+        raise CanonError('integer out of range')
+    return int(s)
+
+
+def _no_constant(s):
+    raise CanonError(s + ' is not allowed')
+
+
+def parse(data):
+    """Parse bytes that MUST already be canonical JSON. Returns the value or raises CanonError."""
+    if isinstance(data, str):
+        data = data.encode('utf-8')
+    try:
+        v = json.loads(data.decode('utf-8'), object_pairs_hook=_pairs,
+                       parse_float=_no_float, parse_int=_int, parse_constant=_no_constant)
+    except CanonError:
+        raise
+    except RecursionError:
+        raise CanonError('too deeply nested')
+    except ValueError as e:
+        raise CanonError('not JSON: ' + str(e))
+    if canon(v) != data:
+        raise CanonError('not canonical JSON')
+    return v

ledgdex/check.py

@@ -0,0 +1,352 @@
+"""Tamper checks over time (spec 3.4, 9): run by hand, from cron, as a worker, or in a GitHub workflow.
+
+A ledger that verifies is internally consistent and signed by its owner. These checks add what one verification
+cannot see: whether a ledger still extends the copy seen last time, whether every receipt still matches the ledger
+it came from, whether the same address still serves the same ledger, and (optionally) whether offer media still match
+their hashes. Errors mean tampering or equivocation and come with signed proof where there is one. Warnings mean
+something could not be checked (a site is down)."""
+import datetime, hashlib, json, os, re
+from .canon import hash_
+from .core import KEY_TYPES, SKEW, Invalid, Keys, Ledger, now, seconds
+from .dex import LEDGER, fetch, http_get, inside
+
+STATE = '.ledgdex-check'
+
+
+class Store:
+    """Last-seen copies of ledgers and which ledger each address served. Lives outside every dex's gen/."""
+
+    def __init__(self, root):
+        self.root = root
+        os.makedirs(os.path.join(root, 'seen'), exist_ok=True)
+        os.makedirs(os.path.join(root, 'proofs'), exist_ok=True)
+        self.urls_path = os.path.join(root, 'urls.json')
+        self.urls = {}
+        if os.path.exists(self.urls_path):
+            with open(self.urls_path, encoding='utf-8') as f:
+                self.urls = json.load(f)
+
+    def _seen(self, ledger_id):
+        return os.path.join(self.root, 'seen', ledger_id[7:] + '.jsonl')
+
+    def seen(self, ledger_id):
+        p = self._seen(ledger_id)
+        if not os.path.exists(p):
+            return None
+        with open(p, 'rb') as f:
+            return f.read()
+
+    def remember(self, led):
+        data = led.header_line + b'\n' + b''.join(line + b'\n' for line in led.lines)
+        old = self.seen(led.id)
+        if old is None or len(data) > len(old):
+            with open(self._seen(led.id), 'wb') as f:
+                f.write(data)
+
+    def proof(self, name, obj):
+        p = os.path.join(self.root, 'proofs', name + '.json')
+        with open(p, 'w', encoding='utf-8', newline='\n') as f:
+            f.write(json.dumps(obj, indent=2, ensure_ascii=False))
+        return p
+
+    def save(self):
+        with open(self.urls_path, 'w', encoding='utf-8', newline='\n') as f:
+            f.write(json.dumps(self.urls, indent=2, sort_keys=True))
+
+
+class Checker:
+    def __init__(self, state_dir=STATE, media=False, log=print, root=None):
+        self.store = Store(state_dir)
+        self.media = media
+        self.log = log
+        self.problems = []
+        self.root, self.root_error = None, None   # the root: needed for ledgers with a "recovered" entry
+        if root:
+            try:
+                self.root = fetch(root, cache=False)[0]
+            except Exception as e:  # the root is down: say so, and do not call ledgers that need it broken
+                self.root_error = str(e)
+                self.problem('warning', 'unreachable', root, 'the root: ' + str(e))
+        self.checked = []
+        self.copies = {}  # ledger id -> [(address, Ledger)] checked in this run
+
+    def problem(self, level, code, url, detail, ledger=None, proof=None):
+        p = {'level': level, 'code': code, 'url': url, 'detail': detail}
+        if ledger:
+            p['ledger'] = ledger
+        if proof:
+            p['proof'] = proof
+        self.problems.append(p)
+        self.log(level.upper() + ' ' + code + ': ' + url + ': ' + detail + (' (proof: ' + proof + ')' if proof else ''))
+
+    def load(self, src, level='error', remote=False):
+        # never the verification cache: a check verifies every entry, every time
+        try:
+            led, _ = fetch(src, cache=False, root=self.root, remote=remote)
+            return led
+        except Exception as e:  # unreachable, missing, not a ledger
+            self.problem(level, 'unreachable', src, str(e))
+            return None
+
+    def ledger(self, led, url):
+        """Checks for one ledger at one address."""
+        if led.header is None:
+            self.problem('error', 'broken', url, str(led.error))
+            return False
+        ok = True
+        if not led.whole and (self.root is None) and 'can only be verified with the root' in str(led.error):
+            # not tampering: this ledger was recovered through the root, and the root could not be read (or none was
+            # given); check it again with the root
+            self.problem('warning', 'needs_root', url, 'entry ' + str(led.broken_at) + ' needs the root to verify' +
+                         (' (the root could not be read)' if self.root_error else ': run with --root'), led.id)
+            return True
+        if not led.whole:
+            self.problem('error', 'broken', url, 'broken at seq ' + str(led.broken_at) + ': ' + str(led.error), led.id)
+            ok = False
+        if led.entries and seconds(led.entries[-1]['time']) > seconds(now()) + SKEW:
+            # an owner stamping entries ahead of time: every later entry must follow it, and it plays with any rule
+            # that compares times across ledgers
+            self.problem('warning', 'future_time', url, 'entries are timed up to ' + led.entries[-1]['time'] +
+                         ', ahead of now', led.id)
+        before = self.store.urls.get(url)
+        if before and before != led.id:
+            self.problem('error', 'ledger_changed', url, 'this address served ledger ' + before + ' before and now '
+                         'serves ' + led.id, led.id)
+            ok = False
+        old = self.store.seen(led.id)
+        if old is not None and not led.data.startswith(old):
+            ok = self.rewritten(led, url, old) and ok
+        elif ok:
+            self.store.remember(led)
+        if not before:
+            self.store.urls[url] = led.id
+        if self.media:
+            self.check_media(led, url)
+        self.checked.append({'url': url, 'ledger': led.id, 'entries': len(led.entries), 'whole': led.whole})
+        self.copies.setdefault(led.id, []).append((url, led))
+        return ok
+
+    def rewritten(self, led, url, old):
+        prev = Ledger(old, cache=False)
+        n = 0
+        while n < len(prev.lines) and n < len(led.lines) and prev.lines[n] == led.lines[n]:
+            n += 1
+        if n < len(prev.lines) and n < len(led.lines):
+            # two entries at one seq, both signed by the owner (spec 3.4)
+            proof = self.store.proof('equivocation-' + led.id[7:19] + '-' + str(n), {
+                'kind': 'equivocation', 'ledger': led.id, 'header': led.header, 'seq': n,
+                'seen': json.loads(prev.lines[n]), 'now': json.loads(led.lines[n]),
+                'note': 'Two entries at the same seq of one ledger, both signed by its owner key.'})
+            self.problem('error', 'equivocation', url, 'seq ' + str(n) + ' differs from the copy seen before', led.id,
+                         proof)
+            return False
+        # an exact start of the copy seen before: an old snapshot, a cache, or entries withheld. Not a rewrite: every
+        # entry served is one seen before, so nothing signed has changed.
+        self.problem('warning', 'stale', url, 'serves ' + str(n) + ' of the ' + str(len(prev.lines)) +
+                     ' entries seen before', led.id)
+        return True
+
+    def check_media(self, led, url):
+        for e in led.entries:
+            if e['msg']['type'] != 'offer':
+                continue
+            for m in e['msg']['body']['item']['media']:
+                try:
+                    got = 'sha256:' + hashlib.sha256(http_get(m['url'])).hexdigest()
+                except Exception as err:
+                    self.problem('warning', 'media_unavailable', m['url'], str(err), led.id)
+                    continue
+                if got != m['hash']:
+                    self.problem('error', 'media_changed', m['url'], 'hash is ' + got + ', the offer signed ' +
+                                 m['hash'], led.id)
+
+    def receipts(self, led, url):
+        """Every receipt in led must match every copy of the ledger it came from: the copies at the receipt's address
+        and the ledger's own dex address, copies already checked in this run, and the copy seen before."""
+        groups = {}
+        for e in led.entries:
+            if e['msg']['type'] == 'receipt':
+                b = e['msg']['body']
+                g = groups.setdefault(b['ledger'], {'urls': [], 'receipts': []})
+                if b['url'] not in g['urls']:
+                    g['urls'].append(b['url'])
+                g['receipts'].append(b)
+        for ledger_id, g in groups.items():
+            # one address per place: a local folder or file named relatively in a receipt is the same as its full path
+            urls = []
+            for u in g['urls']:
+                u = u if re.match(r'https?://', u) or ':' in u.split('/')[0] else os.path.abspath(u)
+                if u not in urls:
+                    urls.append(u)
+            for u in urls:
+                if any(u == c_url for c_url, _ in self.copies.get(ledger_id, [])):
+                    continue
+                other = self.load(u, level='warning', remote=True)
+                if other is None or other.header is None:
+                    continue
+                dex = other.dex
+                if re.match(r'https?://', dex) and dex not in urls:
+                    urls.append(dex)  # also check the ledger at its own dex address
+                if other.id != ledger_id:
+                    # the address in a receipt is written by the receipt's holder, not by the ledger's owner: it is
+                    # evidence against that address only if the address served this ledger before
+                    if self.store.urls.get(u) == ledger_id:
+                        self.problem('error', 'ledger_changed', u, 'this address served ledger ' + ledger_id +
+                                     ' before and now serves ' + other.id, ledger_id)
+                    else:
+                        self.problem('warning', 'receipt_url_mismatch', url, 'receipts from ledger ' + ledger_id +
+                                     ' name ' + u + ', which serves ' + other.id, ledger_id)
+                    continue
+                self.ledger(other, u)
+            copies = list(self.copies.get(ledger_id, []))
+            seen = self.store.seen(ledger_id)
+            if seen is not None:
+                copies.append(('last seen copy', Ledger(seen, cache=False)))
+            if not copies:
+                self.problem('warning', 'receipts_unchecked', url, 'could not reach ledger ' + ledger_id)
+                continue
+            reported = set()
+            for r in g['receipts']:
+                e, s = r['entry'], r['entry']['seq']
+                found = False
+                for c_url, c in copies:
+                    if s >= len(c.ids):
+                        continue
+                    found = True
+                    # every entry the receipt holds (its key chain too) is compared: a chain entry signed by the owner
+                    # that differs from this copy proves a rewrite as surely as the receipt entry itself
+                    for x in r.get('keys', []) + [e]:
+                        xs = x['seq']
+                        if xs >= len(c.ids) or c.ids[xs] == hash_(x) or (c_url, xs) in reported:
+                            continue
+                        reported.add((c_url, xs))
+                        if not self.signed_then(c, x):
+                            # signed by a key this ledger did not have at that seq (a revoked device, a forged
+                            # chain): not the owner's word, so not proof against the ledger
+                            self.problem('warning', 'receipt_bad_signer', url, 'the receipt\'s entry ' + str(xs) +
+                                         ' of ledger ' + ledger_id + ' is not signed by a key that ledger had then')
+                            continue
+                        proof = self.store.proof('receipt-' + ledger_id[7:19] + '-' + str(xs), {
+                            'kind': 'receipt_mismatch', 'ledger': ledger_id, 'url': c_url, 'receipt': r,
+                            'entry': x, 'now': json.loads(c.lines[xs]),
+                            'note': 'The receipt holds an entry at this seq signed by a key the ledger had then; this '
+                                    'copy has another.'})
+                        self.problem('error', 'receipt_mismatch', c_url, 'entry ' + str(xs) + ' differs from the '
+                                     'receipt held by ' + url, ledger_id, proof)
+                if not found:
+                    self.problem('warning', 'receipt_not_visible', url, 'no copy of ledger ' + ledger_id +
+                                 ' has entry ' + str(s) + ' yet')
+
+    @staticmethod
+    def signed_then(c, e):
+        """True if entry e is signed by a key ledger copy c had at e's seq (its full key history, not a receipt's)."""
+        keys = Keys(c.header['owner'])
+        for x in c.entries[:e['seq']]:
+            if x['msg']['type'] in KEY_TYPES:
+                keys.apply(x['msg'])
+        return keys.signed_by(e) is not None
+
+    def published(self, dex):
+        """For a dex folder with "publish" in config.json: the published ledger must be the start of the local one."""
+        cfg = os.path.join(dex, 'config.json')
+        if not os.path.exists(cfg):
+            return
+        with open(cfg, encoding='utf-8') as f:
+            if not isinstance(json.load(f).get('publish'), dict):
+                return
+        from dexweb import dexweb
+        from .render import dexweb_utf8
+        dexweb_utf8()
+        with inside(os.path.abspath(dex)):
+            pub = dexweb.Dexweb().published(LEDGER)
+        if pub is None:
+            return
+        with open(os.path.join(dex, LEDGER), 'rb') as f:
+            local = f.read()
+        pub = pub.encode('utf-8')
+        if local.startswith(pub):
+            return
+        if pub.startswith(local):
+            self.problem('warning', 'behind', dex, 'the published ledger has entries this copy does not: publish to '
+                         'catch up')
+        else:
+            self.problem('error', 'published_differs', dex, 'the published ledger is not the start of this one')
+
+    def run(self, sources, published=False):
+        loaded = []
+        for src in sources:
+            led = self.load(src)
+            if led is None:
+                continue
+            url = src if re.match(r'https?://', src) else os.path.abspath(src)
+            self.ledger(led, url)
+            loaded.append((led, url))
+            if published and os.path.isdir(src):
+                self.published(src)
+        for led, url in loaded:  # after every source is in, so receipts meet every copy
+            if led.header is not None:
+                self.receipts(led, url)
+        self.store.save()
+        errors = [p for p in self.problems if p['level'] == 'error']
+        return {'time': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+                'ok': not errors, 'errors': len(errors), 'warnings': len(self.problems) - len(errors),
+                'checked': self.checked, 'problems': self.problems}
+
+
+def check(sources, state_dir=STATE, media=False, published=False, log=print, root=None):
+    return Checker(state_dir, media, log, root).run(sources, published)
+
+
+WORKFLOW = '''# ledgdex check: made by "ledgdex workflow". Runs on every push, daily, and by hand.
+# A failed run means a ledger was tampered with or rewritten; the report and proofs are attached to the run.
+name: ledgdex check
+on:
+  push:
+  schedule:
+    - cron: "{cron}"
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: "3.12"
+      - run: pip install "ledgdex=={version}"
+      - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: .ledgdex-check
+          key: ledgdex-check-${{{{ github.run_id }}}}
+          restore-keys: ledgdex-check-
+      - run: ledgdex check {sources} --json ledgdex-check.json{media}
+      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        if: always()
+        with:
+          path: .ledgdex-check
+          key: ledgdex-check-${{{{ github.run_id }}}}
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        if: always()
+        with:
+          name: ledgdex-check
+          path: |
+            ledgdex-check.json
+            .ledgdex-check/proofs/
+          if-no-files-found: ignore
+'''
+
+
+def workflow(sources, cron='17 6 * * *', media=False, version=None):
+    """The workflow file. Sources are shell-quoted and the cron is checked, so neither can add steps; ledgdex is
+    installed from PyPI at this exact version, not from a moving branch."""
+    import shlex
+    from . import __version__
+    if not re.match(r'[0-9*/,-]+( [0-9*/,A-Za-z-]+){4}\Z', cron):
+        raise Invalid('not a cron schedule: ' + cron)
+    for src in sources:
+        if not src or src.startswith('-') or any(c in src for c in '\n\r'):
+            raise Invalid('not a ledger source: ' + repr(src))
+    return WORKFLOW.format(cron=cron, sources=' '.join(shlex.quote(x) for x in sources), media=' --media' if media else '',
+                           version=version or __version__)