loopmarket 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmarket/__init__.py +43 -0
- loopmarket/clearing.py +147 -0
- loopmarket/dimensions.py +147 -0
- loopmarket/federation.py +314 -0
- loopmarket/graph.py +198 -0
- loopmarket/matching.py +136 -0
- loopmarket/ontology.py +149 -0
- loopmarket/registry.py +286 -0
- loopmarket/schema.py +364 -0
- loopmarket/settlement.py +17 -0
- loopmarket/sigs.py +82 -0
- loopmarket/solver/__init__.py +3 -0
- loopmarket/solver/agent.py +104 -0
- loopmarket/spacetime.py +93 -0
- loopmarket-0.1.0.dist-info/METADATA +200 -0
- loopmarket-0.1.0.dist-info/RECORD +19 -0
- loopmarket-0.1.0.dist-info/WHEEL +5 -0
- loopmarket-0.1.0.dist-info/licenses/LICENSE +28 -0
- loopmarket-0.1.0.dist-info/top_level.txt +1 -0
loopmarket/__init__.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""loopmarket — a universal combinatorial marketplace over OntoDAG + recordstore + Swarm.
|
|
2
|
+
|
|
3
|
+
One uniform offer form; a shared OntoDAG catalogue in which meanings, minutes
|
|
4
|
+
and map regions are ordered by the same fits-within relation; a distributed,
|
|
5
|
+
versioned offer book over recordstore (Swarm-backed via BeeBytesStore +
|
|
6
|
+
SwarmFeedPointer); competing solver agents hunting profitable loops as
|
|
7
|
+
negative cycles; clearing that re-verifies everything and trusts no one.
|
|
8
|
+
|
|
9
|
+
Dependency direction (boundary B2, enforced by tests/test_boundaries.py):
|
|
10
|
+
|
|
11
|
+
loopmarket -> ontodag -> recordstore -> (Swarm, optional)
|
|
12
|
+
|
|
13
|
+
The core imports work with no network and no Bee node (boundary B1); Swarm
|
|
14
|
+
is a persistence backend chosen at the edges (registry.swarm_offer_book,
|
|
15
|
+
Ontology.persistent over a swarm_store), never a requirement of the model.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from .schema import (
|
|
19
|
+
ASK, BID, GIVE, WANT, GeoDisc, Offer, Thing, TimeWindow, Tokens,
|
|
20
|
+
ask, bid, give, want,
|
|
21
|
+
)
|
|
22
|
+
from .federation import Aggregator, Manifest, Omission, audit_manifest
|
|
23
|
+
from .ontology import Ontology
|
|
24
|
+
from .registry import OfferRegistry, PartialLoopError, swarm_offer_book
|
|
25
|
+
from .matching import Match, candidate_matches, check_match
|
|
26
|
+
from .sigs import maker_address, recover_maker, sign_offer, verify_offer_sig
|
|
27
|
+
from .dimensions import DimensionIndex, candidate_matches_indexed
|
|
28
|
+
from .graph import ExchangeGraph, Loop
|
|
29
|
+
from .clearing import LoopProposal, MockClearing, Receipt, Clearing
|
|
30
|
+
from .solver.agent import SolverAgent
|
|
31
|
+
|
|
32
|
+
__version__ = "0.1.0"
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"GIVE", "WANT", "ASK", "BID", "GeoDisc", "Offer", "Thing", "TimeWindow",
|
|
36
|
+
"Tokens", "give", "want", "ask", "bid",
|
|
37
|
+
"Aggregator", "Manifest", "Omission", "audit_manifest", "Ontology", "OfferRegistry",
|
|
38
|
+
"PartialLoopError", "swarm_offer_book",
|
|
39
|
+
"Match", "candidate_matches", "check_match",
|
|
40
|
+
"maker_address", "recover_maker", "sign_offer", "verify_offer_sig",
|
|
41
|
+
"DimensionIndex", "candidate_matches_indexed", "ExchangeGraph", "Loop",
|
|
42
|
+
"LoopProposal", "MockClearing", "Receipt", "Clearing", "SolverAgent",
|
|
43
|
+
]
|
loopmarket/clearing.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Clearing: where a proposed loop becomes a bundle of commitments.
|
|
2
|
+
|
|
3
|
+
Trust model (the one non-negotiable): clearing *never trusts the solver*.
|
|
4
|
+
A `LoopProposal` names the book root and ontology root it was solved
|
|
5
|
+
against; the clearing layer re-derives every leg with `check_match`, the
|
|
6
|
+
chaining, the product, and the not-already-filled status — cheap, linear in
|
|
7
|
+
the loop — before atomically marking every offer filled. Discovery is
|
|
8
|
+
expensive and competitive; verification is cheap and neutral.
|
|
9
|
+
|
|
10
|
+
`MockClearing` is the in-process stand-in: its "atomic stroke" is one
|
|
11
|
+
recordstore commit (all fills + the loop record land under a single new
|
|
12
|
+
root, or none do). The on-chain path it stands in for (roadmap P2) keeps
|
|
13
|
+
the same interface: a contract receives the loop plus *inclusion proofs*
|
|
14
|
+
that each offer is present under the pinned book root — recordstore's
|
|
15
|
+
canonical-trie `prove`/`verify_proof` (>= 0.16.0) is the primary route;
|
|
16
|
+
POT ForkPathProof is the conditional fallback only if the on-chain
|
|
17
|
+
verifier demands BMT-native proofs (docs/plans/proof-fabric.md). Batch
|
|
18
|
+
auctions across competing sealed proposals are P2 as well
|
|
19
|
+
(docs/plans/P2-batch-auction.md); the mock is first-valid-wins.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import time as _time
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from typing import Protocol
|
|
27
|
+
|
|
28
|
+
from .graph import Loop
|
|
29
|
+
from .matching import check_match
|
|
30
|
+
from .ontology import Ontology
|
|
31
|
+
from .registry import OfferRegistry
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True, slots=True)
|
|
35
|
+
class LoopProposal:
|
|
36
|
+
loop: Loop
|
|
37
|
+
book_root: str # the registry version the loop was solved against
|
|
38
|
+
ontology_root: str # the catalogue version subsumption was checked under
|
|
39
|
+
solver: str # who found it (fee/reputation address)
|
|
40
|
+
found_at: int
|
|
41
|
+
|
|
42
|
+
def to_record(self) -> dict:
|
|
43
|
+
return {
|
|
44
|
+
"loop_id": self.loop.loop_id,
|
|
45
|
+
"solver": self.solver,
|
|
46
|
+
"found_at": self.found_at,
|
|
47
|
+
"book_root": self.book_root,
|
|
48
|
+
"ontology_root": self.ontology_root,
|
|
49
|
+
"surplus": self.loop.surplus,
|
|
50
|
+
"nodes": list(self.loop.nodes),
|
|
51
|
+
"legs": [
|
|
52
|
+
{"give": m.give.offer_id, "want": m.want.offer_id, "rate": m.rate}
|
|
53
|
+
for m in self.loop.matches
|
|
54
|
+
],
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True, slots=True)
|
|
59
|
+
class Receipt:
|
|
60
|
+
accepted: bool
|
|
61
|
+
loop_id: str
|
|
62
|
+
reason: str = ""
|
|
63
|
+
book_root: str = "" # the new root, if accepted
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Clearing(Protocol):
|
|
67
|
+
def submit(self, proposal: LoopProposal) -> Receipt: ...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class MockClearing:
|
|
71
|
+
"""In-process clearing over the shared registry."""
|
|
72
|
+
|
|
73
|
+
#: Oracle types this clearing knows how to verify — the P3 refusal
|
|
74
|
+
#: gate (docs/plans/P3-guarantee-coupling.md, enforcement rule 1): a leg
|
|
75
|
+
#: naming a witness type outside this set never clears here, in U7's
|
|
76
|
+
#: shape — unknown fails closed rather than silently clearing with a
|
|
77
|
+
#: guarantee nobody can check. The mock declares exactly the P0
|
|
78
|
+
#: countersign semantics.
|
|
79
|
+
VERIFIABLE_ORACLES = frozenset({"countersign"})
|
|
80
|
+
|
|
81
|
+
def __init__(self, registry: OfferRegistry, ontology: Ontology, *,
|
|
82
|
+
min_surplus: float = 0.0, require_per_node: bool = True,
|
|
83
|
+
clock=_time.time, verifiable_oracles=VERIFIABLE_ORACLES):
|
|
84
|
+
self.registry = registry
|
|
85
|
+
self.ontology = ontology
|
|
86
|
+
self.min_surplus = min_surplus
|
|
87
|
+
self.require_per_node = require_per_node
|
|
88
|
+
self.clock = clock # injectable for tests / deterministic replay
|
|
89
|
+
self.verifiable_oracles = frozenset(verifiable_oracles)
|
|
90
|
+
|
|
91
|
+
def submit(self, proposal: LoopProposal) -> Receipt:
|
|
92
|
+
loop = proposal.loop
|
|
93
|
+
lid = loop.loop_id
|
|
94
|
+
now = int(self.clock())
|
|
95
|
+
|
|
96
|
+
def reject(reason: str) -> Receipt:
|
|
97
|
+
return Receipt(False, lid, reason)
|
|
98
|
+
|
|
99
|
+
# 0. pins — the rehearsal of U10's clearing half (full enforcement,
|
|
100
|
+
# with proofs, lands with P2): the proposal's catalogue pin must
|
|
101
|
+
# *equal* this clearing's own, refused before any leg work.
|
|
102
|
+
# Plain equality covers mismatch and absence in both directions:
|
|
103
|
+
# a pinned clearing refuses unpinned proposals, an unpinned
|
|
104
|
+
# (development) one refuses proposals claiming ground it cannot
|
|
105
|
+
# confirm; '' == '' keeps the in-memory flow working.
|
|
106
|
+
if proposal.ontology_root != self.ontology.root:
|
|
107
|
+
return reject("ontology pin mismatch")
|
|
108
|
+
|
|
109
|
+
# 1. every offer must exist in the *current* book, be unfilled, and
|
|
110
|
+
# name a witness type this clearing can actually verify
|
|
111
|
+
seen: set[str] = set()
|
|
112
|
+
for oid in loop.offer_ids:
|
|
113
|
+
if oid in seen:
|
|
114
|
+
return reject(f"offer used twice: {oid[:12]}")
|
|
115
|
+
seen.add(oid)
|
|
116
|
+
try:
|
|
117
|
+
offer = self.registry.get(oid)
|
|
118
|
+
except KeyError:
|
|
119
|
+
return reject(f"unknown offer: {oid[:12]}")
|
|
120
|
+
if self.registry.is_filled(oid):
|
|
121
|
+
return reject(f"already filled: {oid[:12]}")
|
|
122
|
+
if self.registry.is_withdrawn(oid):
|
|
123
|
+
return reject(f"withdrawn: {oid[:12]}")
|
|
124
|
+
if offer.oracle not in self.verifiable_oracles:
|
|
125
|
+
return reject(f"unverifiable oracle type: {offer.oracle}")
|
|
126
|
+
|
|
127
|
+
# 2. re-derive every leg — never trust the solver's matches
|
|
128
|
+
for m in loop.matches:
|
|
129
|
+
fresh_give = self.registry.get(m.give.offer_id)
|
|
130
|
+
fresh_want = self.registry.get(m.want.offer_id)
|
|
131
|
+
if check_match(fresh_give, fresh_want, self.ontology, now=now) is None:
|
|
132
|
+
return reject(
|
|
133
|
+
f"leg fails re-verification: {m.give.offer_id[:8]}"
|
|
134
|
+
f" -> {m.want.offer_id[:8]}"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# 3. the arithmetic
|
|
138
|
+
if loop.surplus < self.min_surplus - 1e-12:
|
|
139
|
+
return reject(f"surplus {loop.surplus:.4f} below minimum")
|
|
140
|
+
if self.require_per_node and not loop.all_divisible \
|
|
141
|
+
and not loop.per_node_ok:
|
|
142
|
+
return reject("indivisible legs without per-node surplus")
|
|
143
|
+
|
|
144
|
+
# 4. atomic commitment: all fills land under one new root, or none
|
|
145
|
+
self.registry.mark_filled(loop.offer_ids, lid, proposal.to_record())
|
|
146
|
+
root = self.registry.commit()
|
|
147
|
+
return Receipt(True, lid, book_root=root)
|
loopmarket/dimensions.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Candidate generation through ontodag parametric dimensions (>= 0.4.0).
|
|
2
|
+
|
|
3
|
+
The 2026-07-30 upgrade of the P1 candidate-generation plan (ARCHITECTURE.md
|
|
4
|
+
§3): instead of generated bucket/cell category chains, gives are filed under
|
|
5
|
+
*exact* parametric terms — the service window as one linear-interval value,
|
|
6
|
+
the service cell as one prefix value — and a want's candidates come from two
|
|
7
|
+
native catalogue queries:
|
|
8
|
+
|
|
9
|
+
- meaning: ``dag.get(wanted_concepts)`` — exact-necessary: an give whose
|
|
10
|
+
concepts satisfy the want is, by fits-within, inside every wanted cone;
|
|
11
|
+
- time: ``dag.get_overlapping(service-time(a..b))`` — *exact* for the
|
|
12
|
+
window-overlap gate, because the filed value IS the offer's window
|
|
13
|
+
(no buckets, no quantization error).
|
|
14
|
+
|
|
15
|
+
Geo stays with the exact check (``GeoDisc.intersects``), as in the baseline:
|
|
16
|
+
sibling geohash cells share no prefix, so a cell filter would lose recall
|
|
17
|
+
("cells are hints", ARCHITECTURE.md §3). The generator is therefore
|
|
18
|
+
recall-exact against the baseline give x want product — and clearing
|
|
19
|
+
re-verification never depends on it either way (invariant U3).
|
|
20
|
+
|
|
21
|
+
**The index is derived, local, and never shared.** Filing offers into the
|
|
22
|
+
shared catalogue would move its root under every offer that pins it, so
|
|
23
|
+
`DimensionIndex` works on a deepcopy: regenerable from book + catalogue,
|
|
24
|
+
per-solver, never merged — the same doctrine as every other index in this
|
|
25
|
+
stack. (Corollary: nothing here needs the dimensions registry version
|
|
26
|
+
pinned; that rule from ontodag's DIMENSIONS.md §10 applies when parametric
|
|
27
|
+
terms enter *shared* state, e.g. published region nodes in the catalogue.)
|
|
28
|
+
|
|
29
|
+
Two ontodag adoption rules are load-bearing here: an offer is filed under
|
|
30
|
+
exactly ONE value per dimension (an item sits in the INTERSECTION of its
|
|
31
|
+
parents — never fan a union of cells or buckets into parents), and the cell
|
|
32
|
+
value indexes the disc's *centre* cell only, which is fine because geo is
|
|
33
|
+
not used for pruning.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from datetime import datetime, timezone
|
|
39
|
+
from typing import Iterable, Iterator
|
|
40
|
+
|
|
41
|
+
from ontodag import dimensions as _dims
|
|
42
|
+
|
|
43
|
+
from .matching import Match, check_match
|
|
44
|
+
from .ontology import Ontology
|
|
45
|
+
from .schema import GIVE, WANT, Offer, TimeWindow
|
|
46
|
+
from .spacetime import cell_for
|
|
47
|
+
|
|
48
|
+
TIME_DIMENSION = "service-time"
|
|
49
|
+
CELL_DIMENSION = "service-cell"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _iso(t: int) -> str:
|
|
53
|
+
return datetime.fromtimestamp(t, tz=timezone.utc).strftime(
|
|
54
|
+
"%Y-%m-%dT%H:%M:%SZ")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def time_term(window: TimeWindow) -> str:
|
|
58
|
+
"""The window as one inclusive parametric value.
|
|
59
|
+
|
|
60
|
+
`TimeWindow` is half-open [start, end) in whole seconds; dimension
|
|
61
|
+
ranges are inclusive, so [start, end-1] represents exactly the same
|
|
62
|
+
set of service seconds — overlap is preserved exactly.
|
|
63
|
+
"""
|
|
64
|
+
return f"{TIME_DIMENSION}({_iso(window.start)}..{_iso(window.end - 1)})"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def cell_term(offer: Offer) -> str:
|
|
68
|
+
"""The offer's centre geohash cell as one prefix value (an index fact,
|
|
69
|
+
not a pruning gate — see the module docstring)."""
|
|
70
|
+
return f"{CELL_DIMENSION}({cell_for(offer.where)})"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class DimensionIndex:
|
|
74
|
+
"""Files gives into a derived catalogue copy; answers want candidates.
|
|
75
|
+
|
|
76
|
+
Build one per solve step, like a snapshot: it is cheap relative to the
|
|
77
|
+
O(gives x wants) product it replaces, and regenerating it is what keeps it
|
|
78
|
+
honest (derived state is never merged, never persisted).
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
def __init__(self, ontology: Ontology):
|
|
82
|
+
self.ontology = ontology # the exact-check ground truth
|
|
83
|
+
self._dag = ontology.dag.deepcopy() # derived: catalogue + offers
|
|
84
|
+
self._filed: set[str] = set()
|
|
85
|
+
self._declare()
|
|
86
|
+
|
|
87
|
+
def _declare(self) -> None:
|
|
88
|
+
for name, supers in [
|
|
89
|
+
(_dims.DIMENSION_ROOT, []),
|
|
90
|
+
(_dims.KIND_LINEAR, [_dims.DIMENSION_ROOT]),
|
|
91
|
+
(_dims.KIND_PREFIX, [_dims.DIMENSION_ROOT]),
|
|
92
|
+
(TIME_DIMENSION, [_dims.KIND_LINEAR]),
|
|
93
|
+
(CELL_DIMENSION, [_dims.KIND_PREFIX]),
|
|
94
|
+
]:
|
|
95
|
+
if name not in self._dag.nodes:
|
|
96
|
+
self._dag.put(name, supers)
|
|
97
|
+
|
|
98
|
+
def file(self, offer: Offer) -> bool:
|
|
99
|
+
"""Index an GIVE. Returns False (not filed) when its vocabulary is
|
|
100
|
+
unknown to the catalogue — the same fail-closed outcome the exact
|
|
101
|
+
check would reach (invariant U7)."""
|
|
102
|
+
if offer.kind != GIVE:
|
|
103
|
+
return False
|
|
104
|
+
if offer.offer_id in self._filed:
|
|
105
|
+
return True
|
|
106
|
+
if not all(self.ontology.known(c) for c in offer.thing.concepts):
|
|
107
|
+
return False
|
|
108
|
+
self._dag.put(
|
|
109
|
+
offer.offer_id,
|
|
110
|
+
list(offer.thing.concepts)
|
|
111
|
+
+ [time_term(offer.service), cell_term(offer)])
|
|
112
|
+
self._filed.add(offer.offer_id)
|
|
113
|
+
return True
|
|
114
|
+
|
|
115
|
+
def candidates(self, want_offer: Offer) -> set[str]:
|
|
116
|
+
"""Give offer-ids that can possibly match `want_offer`: inside every
|
|
117
|
+
wanted concept cone AND service windows overlapping. Recall-exact
|
|
118
|
+
for those two gates; every candidate still faces `check_match`."""
|
|
119
|
+
by_concept = {item.name
|
|
120
|
+
for item in self._dag.get(list(want_offer.thing.concepts))}
|
|
121
|
+
if not by_concept:
|
|
122
|
+
return set()
|
|
123
|
+
by_time = {item.name for item in
|
|
124
|
+
self._dag.get_overlapping(time_term(want_offer.service))}
|
|
125
|
+
return by_concept & by_time & self._filed
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def candidate_matches_indexed(
|
|
129
|
+
offers: Iterable[Offer], ontology: Ontology, *,
|
|
130
|
+
now: int, index: DimensionIndex | None = None) -> Iterator[Match]:
|
|
131
|
+
"""Drop-in for `matching.candidate_matches`, generating through a
|
|
132
|
+
`DimensionIndex` instead of the full give x want product. Yields exactly
|
|
133
|
+
the baseline's matches (the recall test in tests/test_dimensions.py is
|
|
134
|
+
the benchmark ARCHITECTURE.md §6 demands of smarter generators)."""
|
|
135
|
+
offers = list(offers)
|
|
136
|
+
index = index if index is not None else DimensionIndex(ontology)
|
|
137
|
+
gives_by_id: dict[str, Offer] = {}
|
|
138
|
+
for offer in offers:
|
|
139
|
+
if offer.kind == GIVE and index.file(offer):
|
|
140
|
+
gives_by_id[offer.offer_id] = offer
|
|
141
|
+
for want_offer in offers:
|
|
142
|
+
if want_offer.kind != WANT:
|
|
143
|
+
continue
|
|
144
|
+
for oid in sorted(index.candidates(want_offer)):
|
|
145
|
+
match = check_match(gives_by_id[oid], want_offer, ontology, now=now)
|
|
146
|
+
if match is not None:
|
|
147
|
+
yield match
|
loopmarket/federation.py
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""The federated book: per-maker books folded into one solver-speed view.
|
|
2
|
+
|
|
3
|
+
The production multi-writer shape (ARCHITECTURE §5 shape 3, ratified
|
|
4
|
+
2026-08-21): every book is single-writer at the source — each maker
|
|
5
|
+
publishes `offer/`, `sig/` and `withdraw/` keys under their own feed and
|
|
6
|
+
signer, clearing publishes `fill/` and `loop/` under its own — and
|
|
7
|
+
conflicts exist only at the fold. An **aggregator** folds announced books
|
|
8
|
+
with three-way merge under the loop-aware resolver, applies the U8 fold
|
|
9
|
+
rules per offer, records its decisions as attributed provenance, rebuilds
|
|
10
|
+
the derived index, and publishes the **manifest tuple**
|
|
11
|
+
`{book_root, provenance_root, index_root, announcement_root}`
|
|
12
|
+
(docs/plans/P1-federated-book.md §2).
|
|
13
|
+
|
|
14
|
+
The fold is *pure*: deterministic admission rules plus commutative merge
|
|
15
|
+
mean aggregators that saw the same inputs produce byte-identical
|
|
16
|
+
`book_root`s in any fold order — divergence between manifests is evidence,
|
|
17
|
+
not opinion, and omission is provable against `announcement_root` and the
|
|
18
|
+
announcement ground truth (threat register T14). Aggregators charge for
|
|
19
|
+
serving, never inclusion; an aggregator that folds selectively is a
|
|
20
|
+
censoring aggregator and is caught as one.
|
|
21
|
+
|
|
22
|
+
One assumption rides throughout: all books share one blob space — Swarm's
|
|
23
|
+
in deployment, one `MemoryBytesStore` in tests — so a root is enough to
|
|
24
|
+
reach any book's bytes. In memory, "feed ownership" is the declared
|
|
25
|
+
`owner` of an announced book; on Swarm it becomes the feed's owner
|
|
26
|
+
address, which is what makes U8's primary layer real (P1 §1).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
|
|
33
|
+
from recordstore import RecordStore
|
|
34
|
+
|
|
35
|
+
from .registry import (
|
|
36
|
+
FILL, LOOP, OFFER, SIG, WITHDRAW, OfferRegistry, index_offers,
|
|
37
|
+
or_set_resolver,
|
|
38
|
+
)
|
|
39
|
+
from .schema import Offer
|
|
40
|
+
|
|
41
|
+
#: Roles an announced book may carry: makers speak offers, signatures and
|
|
42
|
+
#: tombstones; a clearing instance speaks fills and loops. Every other
|
|
43
|
+
#: key class in a book is outside its writer's authority and is refused.
|
|
44
|
+
MAKER = "maker"
|
|
45
|
+
CLEARING = "clearing"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True, slots=True)
|
|
49
|
+
class Manifest:
|
|
50
|
+
"""What an aggregator publishes: four roots and its name.
|
|
51
|
+
|
|
52
|
+
`book_root` is the pure fold (byte-identical across honest aggregators
|
|
53
|
+
with the same inputs); `provenance_root` holds the aggregator's
|
|
54
|
+
attributed speech acts (`origin/`, `reject/`); `index_root` is derived
|
|
55
|
+
and regenerable (never merged); `announcement_root` commits to the
|
|
56
|
+
exact input set this fold consumed — the completeness handle (T14).
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
aggregator: str
|
|
60
|
+
book_root: str
|
|
61
|
+
provenance_root: str
|
|
62
|
+
index_root: str
|
|
63
|
+
announcement_root: str
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class Aggregator:
|
|
67
|
+
"""Folds announced books into one book a solver can read at speed.
|
|
68
|
+
|
|
69
|
+
`store_factory` returns a fresh writable RecordStore over the shared
|
|
70
|
+
blob space (in tests: ``lambda: RecordStore(blobs)``; on Swarm, a
|
|
71
|
+
store under the aggregator's own feed). Admission is by reference:
|
|
72
|
+
the aggregator folds the books it was told about and can un-announce
|
|
73
|
+
a flooder — there is no store-side rate limiting to game (P1 §8).
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
def __init__(self, store_factory, *, aggregator_id: str = "agg-0"):
|
|
77
|
+
self._new_store = store_factory
|
|
78
|
+
self.id = aggregator_id
|
|
79
|
+
self._announced: dict[str, tuple[str, object]] = {}
|
|
80
|
+
|
|
81
|
+
# -- inputs ----------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
def announce(self, owner: str, store, *, role: str = MAKER) -> None:
|
|
84
|
+
"""Register a book: "`owner`'s book is `store`" (one per owner).
|
|
85
|
+
|
|
86
|
+
In deployment the announcement arrives over GSOC or the registry
|
|
87
|
+
events and names (owner address, topic); here the store stands in
|
|
88
|
+
for the resolved feed. Re-announcing an owner replaces the entry;
|
|
89
|
+
un-announcing (admission-by-reference's teeth) is `retract`.
|
|
90
|
+
"""
|
|
91
|
+
if role not in (MAKER, CLEARING):
|
|
92
|
+
raise ValueError(f"unknown book role: {role!r}")
|
|
93
|
+
self._announced[owner] = (role, store)
|
|
94
|
+
|
|
95
|
+
def retract(self, owner: str) -> None:
|
|
96
|
+
"""Stop folding an owner's book (takes effect at the next fold)."""
|
|
97
|
+
self._announced.pop(owner, None)
|
|
98
|
+
|
|
99
|
+
# -- the fold ----------------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
def fold(self) -> Manifest:
|
|
102
|
+
"""Sanitize every announced book, merge, re-derive, publish.
|
|
103
|
+
|
|
104
|
+
Every step is deterministic in the announced (owner, root) set, so
|
|
105
|
+
the whole manifest — not just `book_root` — reproduces across
|
|
106
|
+
aggregators that saw the same inputs.
|
|
107
|
+
"""
|
|
108
|
+
provenance = self._new_store()
|
|
109
|
+
announcement = self._new_store()
|
|
110
|
+
|
|
111
|
+
staged_roots: list[str] = []
|
|
112
|
+
blobs = None
|
|
113
|
+
store_type = None
|
|
114
|
+
for owner in sorted(self._announced):
|
|
115
|
+
role, store = self._announced[owner]
|
|
116
|
+
root = store.root
|
|
117
|
+
announcement.put(f"announce/{owner}",
|
|
118
|
+
{"role": role, "root": root or ""})
|
|
119
|
+
if not root:
|
|
120
|
+
continue
|
|
121
|
+
blobs, store_type = store.blobs, type(store)
|
|
122
|
+
source = store_type.at(root, blobs)
|
|
123
|
+
staged = self._sanitize(owner, role, root, source, provenance)
|
|
124
|
+
staged_root = staged.commit()
|
|
125
|
+
if staged_root:
|
|
126
|
+
staged_roots.append(staged_root)
|
|
127
|
+
|
|
128
|
+
book_root = None
|
|
129
|
+
for staged_root in staged_roots:
|
|
130
|
+
book_root = staged_root if book_root is None else \
|
|
131
|
+
store_type.merge(blobs, None, book_root, staged_root,
|
|
132
|
+
resolver=or_set_resolver)
|
|
133
|
+
book_root = book_root or ""
|
|
134
|
+
|
|
135
|
+
index = self._new_store()
|
|
136
|
+
if book_root:
|
|
137
|
+
folded = OfferRegistry(store_type.at(book_root, blobs))
|
|
138
|
+
folded.verify_loop_atomicity() # U11, on every fold
|
|
139
|
+
index_offers(index, folded.offers())
|
|
140
|
+
|
|
141
|
+
return Manifest(
|
|
142
|
+
aggregator=self.id,
|
|
143
|
+
book_root=book_root,
|
|
144
|
+
provenance_root=provenance.commit() or "",
|
|
145
|
+
index_root=index.commit() or "",
|
|
146
|
+
announcement_root=announcement.commit() or "",
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# -- admission (the U8 fold rules) -------------------------------------------
|
|
150
|
+
|
|
151
|
+
def _sanitize(self, owner: str, role: str, root: str, source,
|
|
152
|
+
provenance) -> object:
|
|
153
|
+
"""One book's admissible speech, copied into a staging store.
|
|
154
|
+
|
|
155
|
+
Fail closed in U7's spirit: a record outside its writer's
|
|
156
|
+
authority, an unreadable or mis-keyed offer, a forged maker
|
|
157
|
+
without a valid detached signature — none of it enters the fold,
|
|
158
|
+
and every rejection is an attributed provenance record.
|
|
159
|
+
"""
|
|
160
|
+
staged = self._new_store()
|
|
161
|
+
|
|
162
|
+
def reject(key: str, reason: str) -> None:
|
|
163
|
+
provenance.put(f"reject/{owner}/{key}",
|
|
164
|
+
{"owner": owner, "reason": reason})
|
|
165
|
+
|
|
166
|
+
offers: dict[str, Offer] = {}
|
|
167
|
+
records = dict(source.items())
|
|
168
|
+
for key in sorted(records):
|
|
169
|
+
rec = records[key]
|
|
170
|
+
if role == CLEARING and not (key.startswith(FILL)
|
|
171
|
+
or key.startswith(LOOP)):
|
|
172
|
+
# a clearing book legitimately *contains* the fold it
|
|
173
|
+
# cleared on (it re-based via absorb); only its fills and
|
|
174
|
+
# loops are its own speech — the rest is silently not
|
|
175
|
+
# re-asserted, never "rejected": provenance records are
|
|
176
|
+
# accusations, and carrying your base is not an offense
|
|
177
|
+
continue
|
|
178
|
+
if key.startswith(OFFER):
|
|
179
|
+
if role != MAKER:
|
|
180
|
+
continue
|
|
181
|
+
oid = key[len(OFFER):]
|
|
182
|
+
try:
|
|
183
|
+
offer = Offer.from_record(rec)
|
|
184
|
+
except (ValueError, KeyError, TypeError):
|
|
185
|
+
reject(key, "unreadable offer record")
|
|
186
|
+
continue
|
|
187
|
+
if offer.offer_id != oid:
|
|
188
|
+
reject(key, "content address mismatch")
|
|
189
|
+
continue
|
|
190
|
+
if offer.maker != owner:
|
|
191
|
+
sig = records.get(SIG + oid)
|
|
192
|
+
if not self._sig_recovers(oid, sig, offer.maker):
|
|
193
|
+
reject(key, "foreign maker without valid signature")
|
|
194
|
+
continue
|
|
195
|
+
staged.put(SIG + oid, sig)
|
|
196
|
+
offers[oid] = offer
|
|
197
|
+
staged.put(key, rec)
|
|
198
|
+
provenance.put(f"origin/{oid}", {"owner": owner, "root": root})
|
|
199
|
+
elif key.startswith(WITHDRAW):
|
|
200
|
+
if role != MAKER:
|
|
201
|
+
reject(key, "tombstone outside a maker book")
|
|
202
|
+
continue
|
|
203
|
+
oid = key[len(WITHDRAW):]
|
|
204
|
+
offer = offers.get(oid) # offer/ sorts before withdraw/
|
|
205
|
+
if offer is None or offer.maker != owner:
|
|
206
|
+
reject(key, "tombstone for an offer this book cannot close")
|
|
207
|
+
continue
|
|
208
|
+
staged.put(key, rec)
|
|
209
|
+
elif key.startswith(SIG):
|
|
210
|
+
oid = key[len(SIG):]
|
|
211
|
+
offer = offers.get(oid) # offer/ sorts before sig/
|
|
212
|
+
if offer is None:
|
|
213
|
+
reject(key, "signature without an admitted offer")
|
|
214
|
+
elif offer.maker != owner:
|
|
215
|
+
pass # verified and staged alongside its foreign offer
|
|
216
|
+
elif self._sig_recovers(oid, rec, owner):
|
|
217
|
+
staged.put(key, rec)
|
|
218
|
+
# else: an own-maker signature that does not verify here
|
|
219
|
+
# (bad, or no crypto library) is dropped, not folded — feed
|
|
220
|
+
# ownership already authenticates the offer itself.
|
|
221
|
+
elif key.startswith(FILL) or key.startswith(LOOP):
|
|
222
|
+
if role != CLEARING:
|
|
223
|
+
reject(key, "clearing keys in a maker book")
|
|
224
|
+
continue
|
|
225
|
+
staged.put(key, rec)
|
|
226
|
+
else:
|
|
227
|
+
reject(key, "unknown keyspace")
|
|
228
|
+
return staged
|
|
229
|
+
|
|
230
|
+
@staticmethod
|
|
231
|
+
def _sig_recovers(offer_id: str, sig, maker: str) -> bool:
|
|
232
|
+
"""Fail closed: no signature, no crypto library, no entry."""
|
|
233
|
+
if not isinstance(sig, str):
|
|
234
|
+
return False
|
|
235
|
+
try:
|
|
236
|
+
from .sigs import recover_maker
|
|
237
|
+
return recover_maker(offer_id, sig) == maker
|
|
238
|
+
except Exception:
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# -- the cross-audit (T14) -----------------------------------------------------
|
|
243
|
+
|
|
244
|
+
@dataclass(frozen=True, slots=True)
|
|
245
|
+
class Omission:
|
|
246
|
+
"""One record a manifest claims to have consumed and did not carry.
|
|
247
|
+
|
|
248
|
+
`key` sits in `owner`'s book at `announced_root` (the root the
|
|
249
|
+
aggregator's own announcement record names), is absent from
|
|
250
|
+
`book_root`, and has no `reject/` record in `provenance_root`. `proof`
|
|
251
|
+
is recordstore's absence proof for `key` against `book_root`: anyone
|
|
252
|
+
can `verify_proof(proof, book_root)` with no store access, so the
|
|
253
|
+
accusation travels as bytes, not as trust in the auditor.
|
|
254
|
+
"""
|
|
255
|
+
|
|
256
|
+
owner: str
|
|
257
|
+
key: str
|
|
258
|
+
announced_root: str
|
|
259
|
+
proof: dict | None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def audit_manifest(manifest: Manifest, blobs, *,
|
|
263
|
+
store_type=RecordStore) -> list[Omission]:
|
|
264
|
+
"""(announced set) − (speech under `book_root`), as P1 §2 defines it.
|
|
265
|
+
|
|
266
|
+
An honest fold is total on the speech it admits: every `offer/` and
|
|
267
|
+
`withdraw/` record in an announced maker book either enters
|
|
268
|
+
`book_root` or earns an attributed `reject/`. Whatever does neither
|
|
269
|
+
was dropped silently — and an aggregator's `announcement_root` is
|
|
270
|
+
its own signed claim about which inputs, at which roots, it folded,
|
|
271
|
+
so the audit needs nothing but the manifest and the blob space.
|
|
272
|
+
Omission (including pay-to-be-indexed, and the nastier form: a
|
|
273
|
+
dropped tombstone resurrecting a withdrawn offer) becomes a proof,
|
|
274
|
+
never a suspicion — the T14 defence, computable by any reader.
|
|
275
|
+
|
|
276
|
+
Not audited: `sig/` (an own-maker signature that fails to verify is
|
|
277
|
+
dropped without a rejection by design — feed ownership already
|
|
278
|
+
authenticates the offer) and clearing books (their fills and loops
|
|
279
|
+
are checked by U11 at every fold instead).
|
|
280
|
+
"""
|
|
281
|
+
announced = store_type.at(manifest.announcement_root, blobs) \
|
|
282
|
+
if manifest.announcement_root else None
|
|
283
|
+
if announced is None:
|
|
284
|
+
return []
|
|
285
|
+
provenance = store_type.at(manifest.provenance_root, blobs) \
|
|
286
|
+
if manifest.provenance_root else None
|
|
287
|
+
book = store_type.at(manifest.book_root, blobs) \
|
|
288
|
+
if manifest.book_root else None
|
|
289
|
+
|
|
290
|
+
def in_store(store, key: str) -> bool:
|
|
291
|
+
if store is None:
|
|
292
|
+
return False
|
|
293
|
+
try:
|
|
294
|
+
store.get(key)
|
|
295
|
+
except KeyError:
|
|
296
|
+
return False
|
|
297
|
+
return True
|
|
298
|
+
|
|
299
|
+
omissions: list[Omission] = []
|
|
300
|
+
for ann_key in sorted(announced.keys("announce/")):
|
|
301
|
+
rec = announced.get(ann_key)
|
|
302
|
+
owner = ann_key[len("announce/"):]
|
|
303
|
+
if rec.get("role") != MAKER or not rec.get("root"):
|
|
304
|
+
continue
|
|
305
|
+
source = store_type.at(rec["root"], blobs)
|
|
306
|
+
for prefix in (OFFER, WITHDRAW):
|
|
307
|
+
for key in sorted(source.keys(prefix)):
|
|
308
|
+
if in_store(book, key):
|
|
309
|
+
continue
|
|
310
|
+
if in_store(provenance, f"reject/{owner}/{key}"):
|
|
311
|
+
continue
|
|
312
|
+
proof = book.prove(key) if book is not None else None
|
|
313
|
+
omissions.append(Omission(owner, key, rec["root"], proof))
|
|
314
|
+
return omissions
|