fleetmesh-node 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (338) hide show
  1. fleetmesh_node-0.2.0.dist-info/METADATA +57 -0
  2. fleetmesh_node-0.2.0.dist-info/RECORD +338 -0
  3. fleetmesh_node-0.2.0.dist-info/WHEEL +5 -0
  4. fleetmesh_node-0.2.0.dist-info/entry_points.txt +2 -0
  5. fleetmesh_node-0.2.0.dist-info/licenses/LICENSE +121 -0
  6. fleetmesh_node-0.2.0.dist-info/top_level.txt +2 -0
  7. node/RELEASE +1 -0
  8. node/__init__.py +35 -0
  9. node/app/__init__.py +49 -0
  10. node/app/commands.py +265 -0
  11. node/app/config.py +131 -0
  12. node/app/control.py +670 -0
  13. node/app/core.py +867 -0
  14. node/app/plugins.py +76 -0
  15. node/app/router.py +83 -0
  16. node/app/socket.py +431 -0
  17. node/app/systemd.py +36 -0
  18. node/app/views.py +204 -0
  19. node/channels/__init__.py +378 -0
  20. node/channels/bus.py +331 -0
  21. node/channels/journal.py +46 -0
  22. node/commands/__init__.py +140 -0
  23. node/commands/adapter.py +210 -0
  24. node/commands/base.py +278 -0
  25. node/commands/cluster_cmds.py +183 -0
  26. node/commands/compute_cmds.py +401 -0
  27. node/commands/core.py +474 -0
  28. node/commands/gossip_cmds.py +106 -0
  29. node/commands/nsite_cmds.py +141 -0
  30. node/commands/peering.py +402 -0
  31. node/commands/plugin_cmds.py +135 -0
  32. node/commands/sidecar_cmds.py +120 -0
  33. node/commands/sync.py +388 -0
  34. node/commands/transport_cmds.py +451 -0
  35. node/commands/video_cmds.py +99 -0
  36. node/compartment/__init__.py +124 -0
  37. node/compartment/base.py +868 -0
  38. node/compartment/command.py +458 -0
  39. node/compartment/context.py +163 -0
  40. node/compartment/every.py +108 -0
  41. node/compartment/handler.py +131 -0
  42. node/compartment/lifecycle.py +460 -0
  43. node/compartment/lint.py +388 -0
  44. node/compartment/need.py +158 -0
  45. node/compartment/problem.py +521 -0
  46. node/compartment/refusals.py +120 -0
  47. node/compartment/registry.py +790 -0
  48. node/compartment/schema.py +290 -0
  49. node/compartment/sidecar.py +499 -0
  50. node/compartment/stores.py +662 -0
  51. node/compartment/wire.py +225 -0
  52. node/compartment/worker.py +61 -0
  53. node/config/__init__.py +43 -0
  54. node/config/base.py +116 -0
  55. node/config/change.py +441 -0
  56. node/config/changestore.py +201 -0
  57. node/config/compartments.py +1144 -0
  58. node/config/node.py +318 -0
  59. node/config/parameter.py +563 -0
  60. node/config/protocols.py +274 -0
  61. node/config/records.py +171 -0
  62. node/examples/__init__.py +1 -0
  63. node/examples/demo.py +226 -0
  64. node/examples/follower.py +46 -0
  65. node/interfaces.py +235 -0
  66. node/keystore/__init__.py +27 -0
  67. node/keystore/core.py +103 -0
  68. node/keystore/enclave.py +688 -0
  69. node/meshnode.py +111 -0
  70. node/metrics/__init__.py +396 -0
  71. node/metrics/adapters.py +102 -0
  72. node/metrics/face.py +110 -0
  73. node/planes/__init__.py +21 -0
  74. node/planes/identity/__init__.py +20 -0
  75. node/planes/identity/announce/__init__.py +9 -0
  76. node/planes/identity/announce/compartment.py +447 -0
  77. node/planes/identity/announce/types.py +26 -0
  78. node/planes/identity/discovery/__init__.py +9 -0
  79. node/planes/identity/discovery/compartment.py +522 -0
  80. node/planes/identity/discovery/types.py +13 -0
  81. node/planes/identity/linked/__init__.py +1 -0
  82. node/planes/identity/linked/compartment.py +270 -0
  83. node/planes/identity/linked/types.py +25 -0
  84. node/planes/identity/presence/__init__.py +7 -0
  85. node/planes/identity/presence/compartment.py +409 -0
  86. node/planes/identity/presence/types.py +14 -0
  87. node/planes/operations/__init__.py +13 -0
  88. node/planes/operations/scheduler/__init__.py +9 -0
  89. node/planes/operations/scheduler/compartment.py +353 -0
  90. node/planes/operations/scheduler/types.py +21 -0
  91. node/planes/operations/updates/__init__.py +17 -0
  92. node/planes/operations/updates/core.py +310 -0
  93. node/planes/operations/updates/store.py +79 -0
  94. node/planes/operations/updates/types.py +20 -0
  95. node/planes/peering/__init__.py +28 -0
  96. node/planes/peering/budget/__init__.py +43 -0
  97. node/planes/peering/budget/chain.py +43 -0
  98. node/planes/peering/budget/core.py +537 -0
  99. node/planes/peering/budget/evaluation.py +80 -0
  100. node/planes/peering/budget/meter.py +100 -0
  101. node/planes/peering/budget/receipts.py +439 -0
  102. node/planes/peering/budget/shape.py +101 -0
  103. node/planes/peering/budget/tick.py +110 -0
  104. node/planes/peering/budget/types.py +67 -0
  105. node/planes/peering/budget/windows.py +346 -0
  106. node/planes/peering/capacity/__init__.py +7 -0
  107. node/planes/peering/capacity/compartment.py +342 -0
  108. node/planes/peering/capacity/types.py +24 -0
  109. node/planes/peering/peering/__init__.py +25 -0
  110. node/planes/peering/peering/core.py +464 -0
  111. node/planes/peering/peering/envelope.py +142 -0
  112. node/planes/peering/peering/handshake.py +198 -0
  113. node/planes/peering/peering/shape.py +80 -0
  114. node/planes/peering/peering/types.py +39 -0
  115. node/planes/peering/reporter/__init__.py +42 -0
  116. node/planes/peering/reporter/constants.py +29 -0
  117. node/planes/peering/reporter/core.py +299 -0
  118. node/planes/peering/reporter/finding.py +74 -0
  119. node/planes/peering/reporter/findings.py +171 -0
  120. node/planes/peering/reporter/publish.py +285 -0
  121. node/planes/peering/reporter/reading.py +193 -0
  122. node/planes/peering/reporter/shape.py +86 -0
  123. node/planes/peering/reporter/types.py +50 -0
  124. node/planes/peering/strain/__init__.py +9 -0
  125. node/planes/peering/strain/compartment.py +204 -0
  126. node/planes/peering/strain/types.py +21 -0
  127. node/planes/peering/trust/__init__.py +9 -0
  128. node/planes/peering/trust/compartment.py +239 -0
  129. node/planes/peering/trust/types.py +30 -0
  130. node/planes/services/__init__.py +33 -0
  131. node/planes/services/compute/__init__.py +160 -0
  132. node/planes/services/compute/common.py +134 -0
  133. node/planes/services/compute/compartment.py +681 -0
  134. node/planes/services/compute/executors/__init__.py +121 -0
  135. node/planes/services/compute/executors/agent.py +840 -0
  136. node/planes/services/compute/executors/bacalhau.py +605 -0
  137. node/planes/services/compute/executors/base.py +127 -0
  138. node/planes/services/compute/executors/harness.py +439 -0
  139. node/planes/services/compute/executors/mcp.py +454 -0
  140. node/planes/services/compute/executors/media.py +898 -0
  141. node/planes/services/compute/executors/nondet.py +283 -0
  142. node/planes/services/compute/executors/providers.py +499 -0
  143. node/planes/services/compute/executors/registry.py +140 -0
  144. node/planes/services/compute/executors/wasm.py +244 -0
  145. node/planes/services/compute/executors/zk.py +604 -0
  146. node/planes/services/compute/jobs.py +227 -0
  147. node/planes/services/compute/load.py +180 -0
  148. node/planes/services/compute/market.py +433 -0
  149. node/planes/services/compute/sandbox.py +282 -0
  150. node/planes/services/compute/types.py +84 -0
  151. node/planes/services/compute_wire/__init__.py +58 -0
  152. node/planes/services/compute_wire/compartment.py +256 -0
  153. node/planes/services/compute_wire/messages.py +320 -0
  154. node/planes/services/compute_wire/sealing.py +147 -0
  155. node/planes/services/compute_wire/session.py +572 -0
  156. node/planes/services/compute_wire/types.py +49 -0
  157. node/planes/services/compute_wire/verify.py +331 -0
  158. node/planes/services/dvm/__init__.py +24 -0
  159. node/planes/services/dvm/gateway.py +526 -0
  160. node/planes/services/dvm/protocol.py +35 -0
  161. node/planes/services/dvm/translate.py +15 -0
  162. node/planes/services/dvm/types.py +25 -0
  163. node/planes/services/gateway/__init__.py +40 -0
  164. node/planes/services/gateway/settle.py +279 -0
  165. node/planes/services/gateway/translate.py +644 -0
  166. node/planes/services/gossip/__init__.py +22 -0
  167. node/planes/services/gossip/core.py +929 -0
  168. node/planes/services/gossip/envelope.py +154 -0
  169. node/planes/services/gossip/mesh.py +127 -0
  170. node/planes/services/gossip/types.py +50 -0
  171. node/planes/services/nsite/__init__.py +7 -0
  172. node/planes/services/nsite/client.py +62 -0
  173. node/planes/services/nsite/compartment.py +752 -0
  174. node/planes/services/nsite/types.py +44 -0
  175. node/planes/services/paidfetch/__init__.py +7 -0
  176. node/planes/services/paidfetch/compartment.py +862 -0
  177. node/planes/services/paidfetch/types.py +99 -0
  178. node/planes/services/settle/__init__.py +27 -0
  179. node/planes/services/settle/compartment.py +384 -0
  180. node/planes/services/settle/engine.py +456 -0
  181. node/planes/services/settle/gateways.py +541 -0
  182. node/planes/services/settle/payment.py +512 -0
  183. node/planes/services/settle/store.py +74 -0
  184. node/planes/services/settle/types.py +58 -0
  185. node/planes/services/sidecars/__init__.py +8 -0
  186. node/planes/services/sidecars/compartment.py +461 -0
  187. node/planes/services/sidecars/manifest.py +304 -0
  188. node/planes/services/sidecars/manifests/10-grasp.json +37 -0
  189. node/planes/services/sidecars/manifests/20-ncontainer.json +31 -0
  190. node/planes/services/sidecars/manifests/30-magpie.json +28 -0
  191. node/planes/services/sidecars/types.py +43 -0
  192. node/planes/services/video/__init__.py +106 -0
  193. node/planes/services/video/barter.py +407 -0
  194. node/planes/services/video/cli.py +158 -0
  195. node/planes/services/video/compartment.py +932 -0
  196. node/planes/services/video/ingest.py +524 -0
  197. node/planes/services/video/ledger.py +705 -0
  198. node/planes/services/video/packager.py +847 -0
  199. node/planes/services/video/player.py +456 -0
  200. node/planes/services/video/timed_segments.py +124 -0
  201. node/planes/services/video/transit.py +478 -0
  202. node/planes/store/__init__.py +24 -0
  203. node/planes/store/blob_replication/__init__.py +7 -0
  204. node/planes/store/blob_replication/compartment.py +496 -0
  205. node/planes/store/blob_replication/types.py +35 -0
  206. node/planes/store/blob_wire/__init__.py +48 -0
  207. node/planes/store/blob_wire/client.py +57 -0
  208. node/planes/store/blob_wire/core.py +359 -0
  209. node/planes/store/blob_wire/intake.py +74 -0
  210. node/planes/store/blob_wire/messages.py +85 -0
  211. node/planes/store/projected/__init__.py +7 -0
  212. node/planes/store/projected/compartment.py +258 -0
  213. node/planes/store/projected/types.py +29 -0
  214. node/planes/store/retention/__init__.py +7 -0
  215. node/planes/store/retention/compartment.py +214 -0
  216. node/planes/store/retention/types.py +12 -0
  217. node/planes/store/sync/__init__.py +35 -0
  218. node/planes/store/sync/core.py +311 -0
  219. node/planes/store/sync/envelope.py +106 -0
  220. node/planes/store/sync/handover.py +206 -0
  221. node/planes/store/sync/initiator.py +128 -0
  222. node/planes/store/sync/responder.py +114 -0
  223. node/planes/store/sync/rules.py +179 -0
  224. node/planes/store/sync/shape.py +80 -0
  225. node/planes/store/sync/types.py +51 -0
  226. node/planes/transport/__init__.py +23 -0
  227. node/planes/transport/bridge/__init__.py +13 -0
  228. node/planes/transport/bridge/compartment.py +785 -0
  229. node/planes/transport/bridge/ratelimit.py +44 -0
  230. node/planes/transport/bridge/types.py +43 -0
  231. node/planes/transport/lan/__init__.py +9 -0
  232. node/planes/transport/lan/compartment.py +446 -0
  233. node/planes/transport/lan/types.py +25 -0
  234. node/planes/transport/link/__init__.py +7 -0
  235. node/planes/transport/link/compartment.py +257 -0
  236. node/planes/transport/link/types.py +27 -0
  237. node/planes/transport/mediation/__init__.py +45 -0
  238. node/planes/transport/mediation/compartment.py +238 -0
  239. node/planes/transport/mediation/leaf.py +107 -0
  240. node/planes/transport/mediation/messages.py +302 -0
  241. node/planes/transport/mediation/service.py +233 -0
  242. node/planes/transport/mediation/store.py +37 -0
  243. node/planes/transport/mediation/types.py +20 -0
  244. node/planes/transport/p2p/__init__.py +7 -0
  245. node/planes/transport/p2p/compartment.py +349 -0
  246. node/planes/transport/p2p/types.py +22 -0
  247. node/protocols/__init__.py +277 -0
  248. node/protocols/activitypub/__init__.py +56 -0
  249. node/protocols/activitypub/actor.py +230 -0
  250. node/protocols/activitypub/federation.py +490 -0
  251. node/protocols/activitypub/media.py +145 -0
  252. node/protocols/activitypub/models.py +286 -0
  253. node/protocols/activitypub/package.py +271 -0
  254. node/protocols/activitypub/store.py +564 -0
  255. node/protocols/arcade/__init__.py +26 -0
  256. node/protocols/atproto/__init__.py +13 -0
  257. node/protocols/atproto/firehose.py +126 -0
  258. node/protocols/atproto/metering.py +165 -0
  259. node/protocols/atproto/package.py +155 -0
  260. node/protocols/atproto/repo.py +250 -0
  261. node/protocols/atproto/xrpc.py +285 -0
  262. node/protocols/base.py +109 -0
  263. node/protocols/circles/__init__.py +29 -0
  264. node/protocols/dm/__init__.py +50 -0
  265. node/protocols/grasp/__init__.py +22 -0
  266. node/protocols/isotime.py +15 -0
  267. node/protocols/nostr/__init__.py +51 -0
  268. node/protocols/nostr/blobstore.py +383 -0
  269. node/protocols/nostr/blossom.py +627 -0
  270. node/protocols/nostr/client.py +377 -0
  271. node/protocols/nostr/descriptor.py +211 -0
  272. node/protocols/nostr/eventstore.py +304 -0
  273. node/protocols/nostr/package.py +182 -0
  274. node/protocols/nostr/peers.py +109 -0
  275. node/protocols/nostr/relayserver.py +656 -0
  276. node/protocols/p2panda/__init__.py +22 -0
  277. node/protocols/p2panda/identity.py +19 -0
  278. node/protocols/p2panda/package.py +236 -0
  279. node/protocols/p2panda/rung.py +152 -0
  280. node/protocols/p2panda/store.py +273 -0
  281. node/protocols/rhio/__init__.py +33 -0
  282. node/protocols/rhio/package.py +275 -0
  283. node/protocols/rhio/rung.py +136 -0
  284. node/protocols/rhio/store.py +1423 -0
  285. node/protocols/ssb/muxrpc.py +256 -0
  286. node/protocols/ssb/package.py +267 -0
  287. node/protocols/textile/__init__.py +111 -0
  288. node/protocols/textile/transducers.py +259 -0
  289. node/release.py +119 -0
  290. node/storage/__init__.py +88 -0
  291. node/storage/accounts.py +305 -0
  292. node/storage/adapter.py +161 -0
  293. node/storage/base.py +385 -0
  294. node/storage/dialect.py +209 -0
  295. node/storage/grants.py +317 -0
  296. node/storage/locator.py +150 -0
  297. node/storage/postgres.py +294 -0
  298. node/storage/projected.py +953 -0
  299. node/storage/sharding.py +448 -0
  300. node/storage/standing.py +568 -0
  301. node/testnet/__init__.py +9 -0
  302. node/testnet/cluster.py +731 -0
  303. node/testnet/fips_daemon.py +230 -0
  304. node/transport/__init__.py +94 -0
  305. node/transport/const.py +110 -0
  306. node/transport/didcomm_inbound.py +251 -0
  307. node/transport/dnstxt.py +169 -0
  308. node/transport/fips_rung/__init__.py +67 -0
  309. node/transport/fips_rung/core.py +666 -0
  310. node/transport/fips_rung/listener.py +96 -0
  311. node/transport/fips_rung/types.py +12 -0
  312. node/transport/host.py +334 -0
  313. node/transport/iroh_rung/__init__.py +29 -0
  314. node/transport/iroh_rung/core.py +477 -0
  315. node/transport/iroh_rung/quic.py +942 -0
  316. node/transport/iroh_rung/types.py +22 -0
  317. node/transport/ladder.py +145 -0
  318. node/transport/lineproto.py +49 -0
  319. node/transport/mdns.py +303 -0
  320. node/transport/netfetch.py +283 -0
  321. node/transport/relay_rung/__init__.py +23 -0
  322. node/transport/relay_rung/cooldown.py +87 -0
  323. node/transport/relay_rung/core.py +138 -0
  324. node/transport/relay_rung/inbound.py +161 -0
  325. node/transport/relay_rung/ledger.py +104 -0
  326. node/transport/relay_rung/listen.py +121 -0
  327. node/transport/relay_rung/outbound.py +181 -0
  328. node/transport/relay_rung/shape.py +71 -0
  329. node/transport/relay_rung/types.py +58 -0
  330. node/transport/relay_rung/wire.py +63 -0
  331. node/transport/router.py +761 -0
  332. node/transport/rung.py +276 -0
  333. node/transport/ssb_rung.py +166 -0
  334. node/transport/ws.py +502 -0
  335. sidecar/fleetmesh_announce.py +338 -0
  336. sidecar/fleetmesh_p2p.py +373 -0
  337. sidecar/fleetmesh_rhio.py +352 -0
  338. sidecar/fleetmesh_ssb.py +603 -0
@@ -0,0 +1,359 @@
1
+ """The blob_wire compartment: serve pulls for what this node holds, take
2
+ offers within the operator's caps, drop replicas when the grant ends.
3
+ """
4
+ from __future__ import annotations
5
+
6
+ import base64
7
+ import hashlib
8
+ from typing import Any, Callable, Mapping, Optional
9
+
10
+ from node.compartment import stores as st
11
+ from node.channels import Channel, INFO, TROUBLE
12
+ from node.metrics import COUNTER, Metric
13
+ from node.config import BlobWireConfig
14
+ from node.config.parameter import Parameter
15
+ from node.compartment import Command, Handler, Need, Outbound, WireProtocol
16
+ from .client import TransportBlobClient
17
+ from .intake import IntakeStore
18
+ from .messages import (CAP_DEFAULTS, DEFAULT_CHUNK_BYTES, HTTP_SCHEMES, MAX_CHUNK_BYTES, PIURI, PULL_TIMEOUT, blossom_url, make_chunk,
19
+ make_have, make_pull, make_refuse)
20
+
21
+
22
+ class BlobWire(WireProtocol):
23
+ """Answers have and pull for blobs this node holds; takes offers under policy."""
24
+
25
+ name = 'blob_wire'
26
+ piuris: tuple[str, ...] = (PIURI,)
27
+ reads = ('blob_wire',)
28
+ config_cls = BlobWireConfig
29
+ live = ('blob_wire',)
30
+ SCHEMA = IntakeStore.SCHEMA
31
+ STORE_CLS = IntakeStore # builds its own; the base opens nothing for it (B5)
32
+ CHANNELS = (
33
+ Channel('blob_served', 'Blob chunk or have served to peer', level=INFO, say='blob chunk served to peer {peer}', tags=('blob', 'wire')),
34
+ Channel('offer_refused', 'Blob offer refused under policy', level=TROUBLE, say='blob offer refused from peer {peer}: {reason}', tags=('blob', 'wire')),
35
+ )
36
+ METRICS = (
37
+ Metric(name='blob_wire_chunks_served', description='Times blob chunks were served.', kind=COUNTER, counts='blob_served', tags=('blob', 'wire')),
38
+ Metric(name='blob_wire_offers_refused', description='Times blob offers were refused.', kind=COUNTER, counts='offer_refused', tags=('blob', 'wire')),
39
+ )
40
+ STORES = (
41
+ st.Store(key='_inflight', description='concurrent blob transfers per peer',
42
+ kind=st.TABLE, shape=int, bound=1024, keyed_by='peer', evict=st.LRU),
43
+ )
44
+ COMMANDS = (
45
+ Command(
46
+ key='held',
47
+ name='Held Replicas',
48
+ description='replicas taken over the transport, per peer',
49
+ tags=('blob_wire', 'held'),
50
+ parameters=(
51
+ Parameter(key='peer', name='peer', description='peer pubkey to filter by', type=str, default=None, required=False),
52
+ ),
53
+ permission='blob_wire/held',
54
+ run='cmd_held',
55
+ reply='{peer: [{sha256, size, at}]}',
56
+ ),
57
+ Command(
58
+ key='drop',
59
+ name='Drop Peer Replicas',
60
+ description='drop every replica held for a peer now',
61
+ tags=('blob_wire', 'drop'),
62
+ parameters=(
63
+ Parameter(key='peer', name='peer', description='peer pubkey', type=str, default='', required=True),
64
+ ),
65
+ permission='blob_wire/drop',
66
+ run='cmd_drop',
67
+ reply='{peer, dropped}',
68
+ ),
69
+ )
70
+
71
+ HANDLERS = (
72
+ Handler('have', run='handle_have', reads=('blobs',), doc='answer have with held hashes'),
73
+ Handler('pull', run='handle_pull', reads=('blobs',), doc='serve requested chunk of held blob'),
74
+ Handler('offer', run='handle_offer', writes=('blobs',), doc='evaluate offer and pull if accepted'),
75
+ Handler('chunk', run='handle_chunk', doc='handle unprompted chunk'),
76
+ Handler('refuse', run='handle_refuse', doc='handle unprompted refuse'),
77
+ )
78
+
79
+ OUTBOUND = (
80
+ Outbound('have', doc='query peer for held hashes or reply with held hashes'),
81
+ Outbound('offer', reply=('have', 'refuse'), doc='offer blob to peer'),
82
+ Outbound('pull', reply=('chunk', 'refuse'), doc='request chunk from peer'),
83
+ Outbound('chunk', doc='serve chunk bytes to pull requester'),
84
+ Outbound('refuse', doc='refuse unheld or disallowed pull/offer'),
85
+ )
86
+
87
+ REPLICAS_DB: str = 'blob_replicas.sqlite'
88
+ NEEDS = (
89
+ Need('blobs', 'stores.blobs'),
90
+ Need('transport', 'transport'),
91
+ Need('db_path', path='REPLICAS_DB'),
92
+ Need('cfg', 'cfg', getter=True),
93
+ Need('grants', 'stores.grants'),
94
+ )
95
+
96
+ def __init__(
97
+ self,
98
+ blobs: Any,
99
+ transport: Any,
100
+ db_path: str,
101
+ cfg: Callable[[], Mapping[str, Any]],
102
+ grantees: Optional[Callable[[], list[str]]] = None,
103
+ clock: Optional[Callable[[], int]] = None,
104
+ node: Any = None,
105
+ grants: Optional[Any] = None,
106
+ ) -> None:
107
+ super().__init__(node, clock)
108
+ self.blobs = blobs
109
+ self.transport = transport
110
+ self.db = IntakeStore(db_path)
111
+ self._cfg = cfg
112
+ if grantees is None and grants is not None:
113
+ self._grantees = lambda: grants.peers()
114
+ else:
115
+ self._grantees = grantees or (lambda: [])
116
+ self.client = TransportBlobClient(self)
117
+ self._inflight: st.Table[str, int]
118
+ self.taken = 0
119
+ self.dropped = 0
120
+ self.refused = 0
121
+ self._closed = False
122
+
123
+ # ---- configuration ----------------------------------------------------------------
124
+ def rc(self) -> BlobWireConfig:
125
+ """The operator's `blob_wire` block as it is now; it reads live."""
126
+ return BlobWireConfig.from_dict((self._cfg() or {}).get('blob_wire') or {})
127
+
128
+ def settings(self) -> BlobWireConfig: # the Compartment hook, through the injected cfg
129
+ return self.rc()
130
+
131
+ def cap(self, name: str) -> int:
132
+ v = getattr(self.rc(), name)
133
+ return max(0, int(v)) if v is not None else CAP_DEFAULTS[name]
134
+
135
+ def chunk_bytes(self) -> int:
136
+ v = self.rc().chunk_bytes
137
+ return max(1, min(int(v), MAX_CHUNK_BYTES)) if v is not None else DEFAULT_CHUNK_BYTES
138
+
139
+ def accepts(self, peer: str) -> bool:
140
+ pol = self.rc().accept_from
141
+ if isinstance(pol, tuple):
142
+ return peer in pol
143
+ if pol == 'grantees':
144
+ return peer in self._grantees()
145
+ return False
146
+
147
+ def why_not(self, peer: str, size: int, now: int) -> Optional[str]:
148
+ """The first reason an offer from this peer may not be taken, or None."""
149
+ if not self.accepts(peer):
150
+ return 'not accepting replicas from this peer'
151
+ if size > self.cap('max_blob_bytes'):
152
+ return 'blob larger than max_blob_bytes'
153
+ if self.db.total_for(peer) + size > self.cap('max_total_bytes'):
154
+ return 'would exceed max_total_bytes for this peer'
155
+ if self.db.today(peer, now) + size > self.cap('bytes_per_day'):
156
+ return 'would exceed bytes_per_day for this peer'
157
+ if self._inflight.get(peer, 0) >= self.cap('max_concurrent'):
158
+ return 'max_concurrent pulls already in flight for this peer'
159
+ return None
160
+
161
+ # ---- serving: what I hold, on request ------------------------------------------------
162
+ def handle_have(self, inbound: Any, sender: str) -> tuple[Any, ...]:
163
+ m = getattr(inbound, 'message', inbound)
164
+ peer = str(sender).lower()
165
+ body = m.get('body', {})
166
+ held = [h for h in body.get('sha256', []) if self.blobs.has(h)]
167
+ self.wire_client.reply(inbound, make_have(held), sender=peer)
168
+ return ('blob_wire', 'have', peer[:16], len(held))
169
+
170
+ def handle_pull(self, inbound: Any, sender: str) -> tuple[Any, ...]:
171
+ return self._serve_pull(inbound, sender)
172
+
173
+ def handle_offer(self, inbound: Any, sender: str) -> tuple[Any, ...]:
174
+ return self._take_offer(inbound, sender)
175
+
176
+ def handle_chunk(self, inbound: Any, sender: str) -> tuple[Any, ...]:
177
+ return ('ignored', 'chunk')
178
+
179
+ def handle_refuse(self, inbound: Any, sender: str) -> tuple[Any, ...]:
180
+ return ('ignored', 'refuse')
181
+
182
+ def _serve_pull(self, inbound: Any, sender: str) -> tuple[Any, ...]:
183
+ m = getattr(inbound, 'message', inbound)
184
+ peer = str(sender).lower()
185
+ body = m['body']
186
+ h = str(body['sha256']).lower()
187
+ data = self.blobs.get(h)
188
+ if data is None:
189
+ self.wire_client.reply(inbound, make_refuse(h, 'not held'), sender=peer)
190
+ return ('blob_wire', 'pull refused', peer[:16], h[:16], 'not held')
191
+ off, ln = int(body['offset']), int(body['length'])
192
+ if off < 0 or ln <= 0 or off > len(data):
193
+ self.wire_client.reply(inbound, make_refuse(h, 'range out of bounds'), sender=peer)
194
+ return ('blob_wire', 'pull refused', peer[:16], h[:16], 'range')
195
+ ln = min(ln, self.chunk_bytes())
196
+ self.wire_client.reply(inbound, make_chunk(h, off, data[off:off + ln]), sender=peer)
197
+ return ('blob_wire', 'chunk', peer[:16], h[:16], off)
198
+
199
+ # ---- taking: the receiver's pull loop ---------------------------------------------------
200
+ def _take_offer(self, inbound: Any, sender: str) -> tuple[Any, ...]:
201
+ m = getattr(inbound, 'message', inbound)
202
+ peer = str(sender).lower()
203
+ body = m['body']
204
+ h = str(body['sha256']).lower()
205
+ size = int(body['size'])
206
+ now = self.clock()
207
+ if self.blobs.has(h):
208
+ # Already here, whoever brought it. Say so; pin it for this peer too.
209
+ self.blobs.pin(h)
210
+ if not self.db.holds(h):
211
+ self.db.took(h, peer, 0, now)
212
+ self.wire_client.reply(inbound, make_have([h]), sender=peer)
213
+ return ('blob_wire', 'already held', peer[:16], h[:16])
214
+ why = self.why_not(peer, size, now)
215
+ if why is not None:
216
+ self.refused += 1
217
+ self.refusals.refuse('blob_wire', 'offer refused', subject=peer, detail={'blob': h[:16], 'why': why}, at=now)
218
+ self.wire_client.reply(inbound, make_refuse(h, why), sender=peer)
219
+ return ('blob_wire', 'offer refused', peer[:16], h[:16], why)
220
+ chunk = max(1, min(int(body.get('chunk_bytes', DEFAULT_CHUNK_BYTES)), self.chunk_bytes()))
221
+ self._inflight[peer] = self._inflight.get(peer, 0) + 1
222
+ try:
223
+ data = self._pull(peer, h, size, chunk)
224
+ except Exception as e:
225
+ why = '%s: %s' % (type(e).__name__, str(e)[:120])
226
+ self.refused += 1
227
+ self.refusals.refuse('blob_wire', 'pull failed', subject=peer, detail={'blob': h[:16], 'why': why}, at=now)
228
+ self.wire_client.reply(inbound, make_refuse(h, why), sender=peer)
229
+ return ('blob_wire', 'pull failed', peer[:16], h[:16], why)
230
+ finally:
231
+ self._inflight[peer] -= 1
232
+ if self._inflight[peer] <= 0:
233
+ self._inflight.pop(peer, None)
234
+ self.blobs.put(data, content_type=str(body.get('type') or 'application/octet-stream'),
235
+ uploaded_by='replica:' + peer, now=now)
236
+ self.blobs.pin(h)
237
+ self.db.took(h, peer, size, now)
238
+ self.taken += 1
239
+ self.wire_client.reply(inbound, make_have([h]), sender=peer)
240
+ return ('blob_wire', 'took', peer[:16], h[:16], size)
241
+
242
+ def _pull(self, peer: str, h: str, size: int, chunk: int) -> bytes:
243
+ hasher = hashlib.sha256()
244
+ out = bytearray()
245
+ off = 0
246
+ while off < size:
247
+ want = min(chunk, size - off)
248
+ reply = self.wire_client.request(peer, make_pull(h, off, want), timeout=PULL_TIMEOUT)
249
+ if reply is None:
250
+ raise TimeoutError('no chunk at offset %d' % off)
251
+ rm = reply.message
252
+ k = self.wire_client.kind_of(rm)
253
+ if k == 'refuse':
254
+ raise ValueError('source refused: %s' % rm.get('body', {}).get('why', '?'))
255
+ if k != 'chunk':
256
+ raise ValueError('expected chunk, got %s' % (k or rm.get('type')))
257
+ rb = rm['body']
258
+ if str(rb.get('sha256', '')).lower() != h or int(rb.get('offset', -1)) != off:
259
+ raise ValueError('chunk for the wrong blob or offset')
260
+ piece = base64.b64decode(rb['data'], validate=True)
261
+ if not piece or len(piece) > want:
262
+ raise ValueError('chunk of the wrong size at offset %d' % off)
263
+ hasher.update(piece)
264
+ out += piece
265
+ off += len(piece)
266
+ if hasher.hexdigest() != h:
267
+ raise ValueError('sha256 mismatch after %d bytes' % off)
268
+ return bytes(out)
269
+
270
+ # ---- the compartment contract ---------------------------------------------------------------
271
+ def drop_peer(self, peer: str, now: Optional[int] = None) -> Mapping[str, Any]:
272
+ """Let go of every replica held for a peer: unpin, delete, forget."""
273
+ peer = peer.lower()
274
+ gone = self.db.forget_peer(peer)
275
+ for h in gone:
276
+ try:
277
+ self.blobs.unpin(h)
278
+ stat = self.blobs.stat(h)
279
+ if stat and str(stat.get('uploaded_by') or '') == 'replica:' + peer:
280
+ self.blobs.delete(h)
281
+ except Exception:
282
+ pass
283
+ self.dropped += len(gone)
284
+ return {'peer': peer, 'dropped': len(gone)}
285
+
286
+ def tick(self, now: int) -> list[tuple[Any, ...]]:
287
+ did: list[tuple[Any, ...]] = []
288
+ if self._closed:
289
+ return did
290
+ live = set(self._grantees())
291
+ for peer in self.db.peers():
292
+ if peer not in live:
293
+ r = self.drop_peer(peer, now)
294
+ if r['dropped']:
295
+ did.append(('blob_wire', 'grant ended, dropped', peer[:16], r['dropped']))
296
+ self.db.prune_days(now)
297
+ return did
298
+
299
+ def state(self) -> Mapping[str, Any]:
300
+ if self._closed:
301
+ return {'stopped': True, 'taken': self.taken, 'dropped': self.dropped, 'refused': self.refused}
302
+ peers = self.db.peers()
303
+ pol = self.rc().accept_from
304
+ return {'accept_from': list(pol) if isinstance(pol, tuple) else pol,
305
+ 'caps': {k: self.cap(k) for k in CAP_DEFAULTS},
306
+ 'peers': len(peers),
307
+ 'held_bytes': sum(self.db.total_for(p) for p in peers),
308
+ 'inflight': dict(self._inflight),
309
+ 'taken': self.taken, 'dropped': self.dropped, 'refused': self.refused}
310
+
311
+ def trouble(self) -> list[str]:
312
+ if self._closed:
313
+ return []
314
+ pol = self.rc().accept_from
315
+ if pol != 'none' and pol and not all(self.cap(k) for k in ('bytes_per_day', 'max_blob_bytes', 'max_total_bytes')):
316
+ return ['blob_wire.accept_from is set but a cap is 0, so every offer is refused: '
317
+ 'set bytes_per_day, max_blob_bytes, max_total_bytes and max_concurrent']
318
+ return []
319
+
320
+ def cmd_held(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
321
+ peers = [str(args['peer']).lower()] if args.get('peer') else self.db.peers()
322
+ return {p: self.db.held_for(p) for p in peers}
323
+
324
+ def cmd_drop(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
325
+ return self.drop_peer(str(args['peer']))
326
+
327
+ def stop(self) -> Any:
328
+ self._closed = True
329
+ try:
330
+ self.db.close()
331
+ except Exception:
332
+ pass
333
+ super().stop()
334
+ return None
335
+
336
+
337
+ def partner_transport_peers(node: Any) -> list[str]:
338
+ """`peer:<pk>` for every grantee that has no Blossom URL in its descriptor.
339
+
340
+ Peers with an HTTP endpoint take the Blossom path; this is for the rest.
341
+ """
342
+ out: list[str] = []
343
+ grants = getattr(node, 'grants', None)
344
+ peers = getattr(node, 'peers', None)
345
+ if grants is None or peers is None:
346
+ return out
347
+ for pk in grants.peers():
348
+ try:
349
+ d = peers.descriptor(pk)
350
+ except Exception:
351
+ d = None
352
+ has_http = False
353
+ for t in (d or {}).get('tags', []):
354
+ if t and t[0] == 'endpoint' and len(t) > 2 and t[1] in HTTP_SCHEMES and blossom_url(t[2]):
355
+ has_http = True
356
+ break
357
+ if not has_http:
358
+ out.append(TransportBlobClient.PREFIX + pk)
359
+ return out
@@ -0,0 +1,74 @@
1
+ """What this node has taken from whom: the replica ledger behind the caps."""
2
+ from __future__ import annotations
3
+
4
+ from typing import Any, Mapping
5
+
6
+ from node.compartment.schema import Column, Index, Table
7
+ from node.storage import SqliteStore
8
+ from .messages import DAY
9
+
10
+
11
+ class IntakeStore(SqliteStore):
12
+ """Replicas taken over the transport and the per-peer, per-day byte count.
13
+
14
+ Lives in the same file as blob_replication's ReplicaStore so one operator
15
+ backup carries both directions of the same fact: what I hold for others
16
+ and what others hold for me.
17
+ """
18
+
19
+ SCHEMA = (
20
+ Table('replicas_in', (
21
+ Column('sha256', 'TEXT', not_null=True),
22
+ Column('peer', 'TEXT', not_null=True),
23
+ Column('size', 'INTEGER', not_null=True),
24
+ Column('at', 'INTEGER', not_null=True),
25
+ ), primary_key=('sha256', 'peer'), indices=(
26
+ Index('idx_replicas_in_peer', ('peer',)),
27
+ )),
28
+ Table('intake_day', (
29
+ Column('peer', 'TEXT', not_null=True),
30
+ Column('day', 'INTEGER', not_null=True),
31
+ Column('bytes', 'INTEGER', not_null=True),
32
+ ), primary_key=('peer', 'day')),
33
+ )
34
+
35
+ def took(self, h: str, peer: str, size: int, now: int) -> None:
36
+ with self.transaction() as cur:
37
+ cur.execute('INSERT OR REPLACE INTO replicas_in (sha256, peer, size, at) VALUES (?, ?, ?, ?)',
38
+ (h, peer, int(size), int(now)))
39
+ cur.execute('INSERT INTO intake_day (peer, day, bytes) VALUES (?, ?, ?) '
40
+ 'ON CONFLICT(peer, day) DO UPDATE SET bytes = bytes + excluded.bytes',
41
+ (peer, int(now) // DAY, int(size)))
42
+
43
+ def holds(self, h: str) -> bool:
44
+ return self._one('SELECT 1 FROM replicas_in WHERE sha256 = ? LIMIT 1', (h,)) is not None
45
+
46
+ def held_for(self, peer: str) -> list[Mapping[str, Any]]:
47
+ rows = self._all('SELECT sha256, size, at FROM replicas_in WHERE peer = ? ORDER BY at', (peer,))
48
+ return [{'sha256': r[0], 'size': int(r[1]), 'at': int(r[2])} for r in rows]
49
+
50
+ def peers(self) -> list[str]:
51
+ return [r[0] for r in self._all('SELECT DISTINCT peer FROM replicas_in')]
52
+
53
+ def total_for(self, peer: str) -> int:
54
+ row = self._one('SELECT COALESCE(SUM(size), 0) FROM replicas_in WHERE peer = ?', (peer,))
55
+ return int(row[0]) if row else 0
56
+
57
+ def today(self, peer: str, now: int) -> int:
58
+ row = self._one('SELECT bytes FROM intake_day WHERE peer = ? AND day = ?', (peer, int(now) // DAY))
59
+ return int(row[0]) if row else 0
60
+
61
+ def forget(self, h: str, peer: str) -> None:
62
+ with self.transaction() as cur:
63
+ cur.execute('DELETE FROM replicas_in WHERE sha256 = ? AND peer = ?', (h, peer))
64
+
65
+ def forget_peer(self, peer: str) -> list[str]:
66
+ with self.transaction() as cur:
67
+ gone = [r['sha256'] for r in self.held_for(peer)]
68
+ cur.execute('DELETE FROM replicas_in WHERE peer = ?', (peer,))
69
+ cur.execute('DELETE FROM intake_day WHERE peer = ?', (peer,))
70
+ return gone
71
+
72
+ def prune_days(self, now: int, keep_days: int = 2) -> None:
73
+ with self.transaction() as cur:
74
+ cur.execute('DELETE FROM intake_day WHERE day < ?', (int(now) // DAY - keep_days,))
@@ -0,0 +1,85 @@
1
+ """blob/1.0 on the wire: the protocol URI, the five message kinds, the
2
+ numbers the window fixes (chunk sizes, caps) and the constructors.
3
+ """
4
+ from __future__ import annotations
5
+
6
+ import base64
7
+ import uuid
8
+ from typing import Any, Mapping, Optional, TypeVar
9
+
10
+ from fleetmesh_core.accounting import window
11
+ from fleetmesh_core.wire.wire_types import BlobChunkBody, BlobHaveBody, BlobOfferBody, BlobPullBody, BlobRefuseBody, Message
12
+
13
+ BlobBody = TypeVar('BlobBody', BlobHaveBody, BlobOfferBody, BlobPullBody, BlobChunkBody, BlobRefuseBody)
14
+
15
+ PIURI: str = 'https://fleetmesh.org/blob/1.0'
16
+ KINDS: tuple[str, ...] = ('have', 'offer', 'pull', 'chunk', 'refuse')
17
+
18
+ C: Mapping[str, Any] = window.C['blob_wire']
19
+ DEFAULT_CHUNK_BYTES: int = int(C['chunk_bytes']) # ~30 FIPS datagrams; a Nostr wrap holds it whole
20
+ MAX_CHUNK_BYTES: int = int(C['max_chunk_bytes']) # above this, the wrap outgrows relay limits
21
+ MAX_HAVE: int = int(C['max_have']) # hashes in one have
22
+ PULL_TIMEOUT: float = float(C['pull_timeout_seconds'])
23
+ DAY: int = 86400
24
+
25
+ # How a descriptor `endpoint` tag is read as a Blossom base URL. Shared by
26
+ # the Blossom path (blob_replication) and the transport path (core); lives
27
+ # in this leaf so neither side has to import the other to agree on it.
28
+ HTTP_SCHEMES: tuple[str, ...] = ('http', 'https', 'ws', 'wss')
29
+
30
+
31
+ def blossom_url(address: str) -> Optional[str]:
32
+ """An endpoint address as an http(s) Blossom base URL, or None."""
33
+ a = (address or '').strip().rstrip('/')
34
+ for pre, rep in (('wss://', 'https://'), ('ws://', 'http://')):
35
+ if a.startswith(pre):
36
+ return rep + a[len(pre):]
37
+ if a.startswith(('http://', 'https://')):
38
+ return a
39
+ return None
40
+
41
+ # What an operator may set under `blob_wire` in the config, and what each
42
+ # means when left alone. Zero is "no bytes" everywhere, not "unlimited".
43
+ CAP_DEFAULTS: dict[str, int] = {
44
+ 'bytes_per_day': int(C['bytes_per_day']), # per peer, taken over the transport path
45
+ 'max_concurrent': int(C['max_concurrent']), # pulls in flight per peer
46
+ 'max_blob_bytes': int(C['max_blob_bytes']), # one blob
47
+ 'max_total_bytes': int(C['max_total_bytes']), # all replicas held for one peer
48
+ }
49
+
50
+ # ---- messages -------------------------------------------------------------------
51
+ def _msg(kind: str, body: BlobBody, thid: Optional[str] = None) -> Message[BlobBody]:
52
+ m: Message[BlobBody] = {'id': str(uuid.uuid4()), 'type': '%s/%s' % (PIURI, kind), 'body': body}
53
+ if thid:
54
+ m['thid'] = thid
55
+ return m
56
+
57
+
58
+ def make_have(hashes: list[str], thid: Optional[str] = None) -> Message[BlobHaveBody]:
59
+ return _msg('have', BlobHaveBody(sha256= [h.lower() for h in hashes[:MAX_HAVE]]), thid)
60
+
61
+
62
+ def make_offer(sha256: str, size: int, content_type: str, chunk_bytes: int = DEFAULT_CHUNK_BYTES) -> Message[BlobOfferBody]:
63
+ return _msg('offer', BlobOfferBody(sha256=sha256.lower(), size=int(size), type=content_type or 'application/octet-stream',
64
+ chunk_bytes=max(1, min(int(chunk_bytes), MAX_CHUNK_BYTES))))
65
+
66
+
67
+ def make_pull(sha256: str, offset: int, length: int) -> Message[BlobPullBody]:
68
+ return _msg('pull', BlobPullBody(sha256=sha256.lower(), offset=int(offset), length=int(length)))
69
+
70
+
71
+ def make_chunk(sha256: str, offset: int, data: bytes, thid: Optional[str] = None) -> Message[BlobChunkBody]:
72
+ return _msg('chunk', BlobChunkBody(sha256=sha256.lower(), offset=int(offset),
73
+ data=base64.b64encode(data).decode('ascii')), thid)
74
+
75
+
76
+ def make_refuse(sha256: str, why: str, thid: Optional[str] = None) -> Message[BlobRefuseBody]:
77
+ return _msg('refuse', BlobRefuseBody(sha256=sha256.lower(), why=why[:200]), thid)
78
+
79
+
80
+ def kind_of(message: Mapping[str, Any]) -> Optional[str]:
81
+ t = str(message.get('type', ''))
82
+ if not t.startswith(PIURI + '/'):
83
+ return None
84
+ k = t[len(PIURI) + 1:]
85
+ return k if k in KINDS else None
@@ -0,0 +1,7 @@
1
+ """Co-transactional SQL projections over MeshStore events and P2Panda causal logs.
2
+
3
+ One-module compartment: everything is in compartment.py.
4
+ """
5
+ from .compartment import ProjectedDB, ProjectedDBConfig
6
+
7
+ __all__ = ['ProjectedDB', 'ProjectedDBConfig']