fleetmesh-node 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (338) hide show
  1. fleetmesh_node-0.2.0.dist-info/METADATA +57 -0
  2. fleetmesh_node-0.2.0.dist-info/RECORD +338 -0
  3. fleetmesh_node-0.2.0.dist-info/WHEEL +5 -0
  4. fleetmesh_node-0.2.0.dist-info/entry_points.txt +2 -0
  5. fleetmesh_node-0.2.0.dist-info/licenses/LICENSE +121 -0
  6. fleetmesh_node-0.2.0.dist-info/top_level.txt +2 -0
  7. node/RELEASE +1 -0
  8. node/__init__.py +35 -0
  9. node/app/__init__.py +49 -0
  10. node/app/commands.py +265 -0
  11. node/app/config.py +131 -0
  12. node/app/control.py +670 -0
  13. node/app/core.py +867 -0
  14. node/app/plugins.py +76 -0
  15. node/app/router.py +83 -0
  16. node/app/socket.py +431 -0
  17. node/app/systemd.py +36 -0
  18. node/app/views.py +204 -0
  19. node/channels/__init__.py +378 -0
  20. node/channels/bus.py +331 -0
  21. node/channels/journal.py +46 -0
  22. node/commands/__init__.py +140 -0
  23. node/commands/adapter.py +210 -0
  24. node/commands/base.py +278 -0
  25. node/commands/cluster_cmds.py +183 -0
  26. node/commands/compute_cmds.py +401 -0
  27. node/commands/core.py +474 -0
  28. node/commands/gossip_cmds.py +106 -0
  29. node/commands/nsite_cmds.py +141 -0
  30. node/commands/peering.py +402 -0
  31. node/commands/plugin_cmds.py +135 -0
  32. node/commands/sidecar_cmds.py +120 -0
  33. node/commands/sync.py +388 -0
  34. node/commands/transport_cmds.py +451 -0
  35. node/commands/video_cmds.py +99 -0
  36. node/compartment/__init__.py +124 -0
  37. node/compartment/base.py +868 -0
  38. node/compartment/command.py +458 -0
  39. node/compartment/context.py +163 -0
  40. node/compartment/every.py +108 -0
  41. node/compartment/handler.py +131 -0
  42. node/compartment/lifecycle.py +460 -0
  43. node/compartment/lint.py +388 -0
  44. node/compartment/need.py +158 -0
  45. node/compartment/problem.py +521 -0
  46. node/compartment/refusals.py +120 -0
  47. node/compartment/registry.py +790 -0
  48. node/compartment/schema.py +290 -0
  49. node/compartment/sidecar.py +499 -0
  50. node/compartment/stores.py +662 -0
  51. node/compartment/wire.py +225 -0
  52. node/compartment/worker.py +61 -0
  53. node/config/__init__.py +43 -0
  54. node/config/base.py +116 -0
  55. node/config/change.py +441 -0
  56. node/config/changestore.py +201 -0
  57. node/config/compartments.py +1144 -0
  58. node/config/node.py +318 -0
  59. node/config/parameter.py +563 -0
  60. node/config/protocols.py +274 -0
  61. node/config/records.py +171 -0
  62. node/examples/__init__.py +1 -0
  63. node/examples/demo.py +226 -0
  64. node/examples/follower.py +46 -0
  65. node/interfaces.py +235 -0
  66. node/keystore/__init__.py +27 -0
  67. node/keystore/core.py +103 -0
  68. node/keystore/enclave.py +688 -0
  69. node/meshnode.py +111 -0
  70. node/metrics/__init__.py +396 -0
  71. node/metrics/adapters.py +102 -0
  72. node/metrics/face.py +110 -0
  73. node/planes/__init__.py +21 -0
  74. node/planes/identity/__init__.py +20 -0
  75. node/planes/identity/announce/__init__.py +9 -0
  76. node/planes/identity/announce/compartment.py +447 -0
  77. node/planes/identity/announce/types.py +26 -0
  78. node/planes/identity/discovery/__init__.py +9 -0
  79. node/planes/identity/discovery/compartment.py +522 -0
  80. node/planes/identity/discovery/types.py +13 -0
  81. node/planes/identity/linked/__init__.py +1 -0
  82. node/planes/identity/linked/compartment.py +270 -0
  83. node/planes/identity/linked/types.py +25 -0
  84. node/planes/identity/presence/__init__.py +7 -0
  85. node/planes/identity/presence/compartment.py +409 -0
  86. node/planes/identity/presence/types.py +14 -0
  87. node/planes/operations/__init__.py +13 -0
  88. node/planes/operations/scheduler/__init__.py +9 -0
  89. node/planes/operations/scheduler/compartment.py +353 -0
  90. node/planes/operations/scheduler/types.py +21 -0
  91. node/planes/operations/updates/__init__.py +17 -0
  92. node/planes/operations/updates/core.py +310 -0
  93. node/planes/operations/updates/store.py +79 -0
  94. node/planes/operations/updates/types.py +20 -0
  95. node/planes/peering/__init__.py +28 -0
  96. node/planes/peering/budget/__init__.py +43 -0
  97. node/planes/peering/budget/chain.py +43 -0
  98. node/planes/peering/budget/core.py +537 -0
  99. node/planes/peering/budget/evaluation.py +80 -0
  100. node/planes/peering/budget/meter.py +100 -0
  101. node/planes/peering/budget/receipts.py +439 -0
  102. node/planes/peering/budget/shape.py +101 -0
  103. node/planes/peering/budget/tick.py +110 -0
  104. node/planes/peering/budget/types.py +67 -0
  105. node/planes/peering/budget/windows.py +346 -0
  106. node/planes/peering/capacity/__init__.py +7 -0
  107. node/planes/peering/capacity/compartment.py +342 -0
  108. node/planes/peering/capacity/types.py +24 -0
  109. node/planes/peering/peering/__init__.py +25 -0
  110. node/planes/peering/peering/core.py +464 -0
  111. node/planes/peering/peering/envelope.py +142 -0
  112. node/planes/peering/peering/handshake.py +198 -0
  113. node/planes/peering/peering/shape.py +80 -0
  114. node/planes/peering/peering/types.py +39 -0
  115. node/planes/peering/reporter/__init__.py +42 -0
  116. node/planes/peering/reporter/constants.py +29 -0
  117. node/planes/peering/reporter/core.py +299 -0
  118. node/planes/peering/reporter/finding.py +74 -0
  119. node/planes/peering/reporter/findings.py +171 -0
  120. node/planes/peering/reporter/publish.py +285 -0
  121. node/planes/peering/reporter/reading.py +193 -0
  122. node/planes/peering/reporter/shape.py +86 -0
  123. node/planes/peering/reporter/types.py +50 -0
  124. node/planes/peering/strain/__init__.py +9 -0
  125. node/planes/peering/strain/compartment.py +204 -0
  126. node/planes/peering/strain/types.py +21 -0
  127. node/planes/peering/trust/__init__.py +9 -0
  128. node/planes/peering/trust/compartment.py +239 -0
  129. node/planes/peering/trust/types.py +30 -0
  130. node/planes/services/__init__.py +33 -0
  131. node/planes/services/compute/__init__.py +160 -0
  132. node/planes/services/compute/common.py +134 -0
  133. node/planes/services/compute/compartment.py +681 -0
  134. node/planes/services/compute/executors/__init__.py +121 -0
  135. node/planes/services/compute/executors/agent.py +840 -0
  136. node/planes/services/compute/executors/bacalhau.py +605 -0
  137. node/planes/services/compute/executors/base.py +127 -0
  138. node/planes/services/compute/executors/harness.py +439 -0
  139. node/planes/services/compute/executors/mcp.py +454 -0
  140. node/planes/services/compute/executors/media.py +898 -0
  141. node/planes/services/compute/executors/nondet.py +283 -0
  142. node/planes/services/compute/executors/providers.py +499 -0
  143. node/planes/services/compute/executors/registry.py +140 -0
  144. node/planes/services/compute/executors/wasm.py +244 -0
  145. node/planes/services/compute/executors/zk.py +604 -0
  146. node/planes/services/compute/jobs.py +227 -0
  147. node/planes/services/compute/load.py +180 -0
  148. node/planes/services/compute/market.py +433 -0
  149. node/planes/services/compute/sandbox.py +282 -0
  150. node/planes/services/compute/types.py +84 -0
  151. node/planes/services/compute_wire/__init__.py +58 -0
  152. node/planes/services/compute_wire/compartment.py +256 -0
  153. node/planes/services/compute_wire/messages.py +320 -0
  154. node/planes/services/compute_wire/sealing.py +147 -0
  155. node/planes/services/compute_wire/session.py +572 -0
  156. node/planes/services/compute_wire/types.py +49 -0
  157. node/planes/services/compute_wire/verify.py +331 -0
  158. node/planes/services/dvm/__init__.py +24 -0
  159. node/planes/services/dvm/gateway.py +526 -0
  160. node/planes/services/dvm/protocol.py +35 -0
  161. node/planes/services/dvm/translate.py +15 -0
  162. node/planes/services/dvm/types.py +25 -0
  163. node/planes/services/gateway/__init__.py +40 -0
  164. node/planes/services/gateway/settle.py +279 -0
  165. node/planes/services/gateway/translate.py +644 -0
  166. node/planes/services/gossip/__init__.py +22 -0
  167. node/planes/services/gossip/core.py +929 -0
  168. node/planes/services/gossip/envelope.py +154 -0
  169. node/planes/services/gossip/mesh.py +127 -0
  170. node/planes/services/gossip/types.py +50 -0
  171. node/planes/services/nsite/__init__.py +7 -0
  172. node/planes/services/nsite/client.py +62 -0
  173. node/planes/services/nsite/compartment.py +752 -0
  174. node/planes/services/nsite/types.py +44 -0
  175. node/planes/services/paidfetch/__init__.py +7 -0
  176. node/planes/services/paidfetch/compartment.py +862 -0
  177. node/planes/services/paidfetch/types.py +99 -0
  178. node/planes/services/settle/__init__.py +27 -0
  179. node/planes/services/settle/compartment.py +384 -0
  180. node/planes/services/settle/engine.py +456 -0
  181. node/planes/services/settle/gateways.py +541 -0
  182. node/planes/services/settle/payment.py +512 -0
  183. node/planes/services/settle/store.py +74 -0
  184. node/planes/services/settle/types.py +58 -0
  185. node/planes/services/sidecars/__init__.py +8 -0
  186. node/planes/services/sidecars/compartment.py +461 -0
  187. node/planes/services/sidecars/manifest.py +304 -0
  188. node/planes/services/sidecars/manifests/10-grasp.json +37 -0
  189. node/planes/services/sidecars/manifests/20-ncontainer.json +31 -0
  190. node/planes/services/sidecars/manifests/30-magpie.json +28 -0
  191. node/planes/services/sidecars/types.py +43 -0
  192. node/planes/services/video/__init__.py +106 -0
  193. node/planes/services/video/barter.py +407 -0
  194. node/planes/services/video/cli.py +158 -0
  195. node/planes/services/video/compartment.py +932 -0
  196. node/planes/services/video/ingest.py +524 -0
  197. node/planes/services/video/ledger.py +705 -0
  198. node/planes/services/video/packager.py +847 -0
  199. node/planes/services/video/player.py +456 -0
  200. node/planes/services/video/timed_segments.py +124 -0
  201. node/planes/services/video/transit.py +478 -0
  202. node/planes/store/__init__.py +24 -0
  203. node/planes/store/blob_replication/__init__.py +7 -0
  204. node/planes/store/blob_replication/compartment.py +496 -0
  205. node/planes/store/blob_replication/types.py +35 -0
  206. node/planes/store/blob_wire/__init__.py +48 -0
  207. node/planes/store/blob_wire/client.py +57 -0
  208. node/planes/store/blob_wire/core.py +359 -0
  209. node/planes/store/blob_wire/intake.py +74 -0
  210. node/planes/store/blob_wire/messages.py +85 -0
  211. node/planes/store/projected/__init__.py +7 -0
  212. node/planes/store/projected/compartment.py +258 -0
  213. node/planes/store/projected/types.py +29 -0
  214. node/planes/store/retention/__init__.py +7 -0
  215. node/planes/store/retention/compartment.py +214 -0
  216. node/planes/store/retention/types.py +12 -0
  217. node/planes/store/sync/__init__.py +35 -0
  218. node/planes/store/sync/core.py +311 -0
  219. node/planes/store/sync/envelope.py +106 -0
  220. node/planes/store/sync/handover.py +206 -0
  221. node/planes/store/sync/initiator.py +128 -0
  222. node/planes/store/sync/responder.py +114 -0
  223. node/planes/store/sync/rules.py +179 -0
  224. node/planes/store/sync/shape.py +80 -0
  225. node/planes/store/sync/types.py +51 -0
  226. node/planes/transport/__init__.py +23 -0
  227. node/planes/transport/bridge/__init__.py +13 -0
  228. node/planes/transport/bridge/compartment.py +785 -0
  229. node/planes/transport/bridge/ratelimit.py +44 -0
  230. node/planes/transport/bridge/types.py +43 -0
  231. node/planes/transport/lan/__init__.py +9 -0
  232. node/planes/transport/lan/compartment.py +446 -0
  233. node/planes/transport/lan/types.py +25 -0
  234. node/planes/transport/link/__init__.py +7 -0
  235. node/planes/transport/link/compartment.py +257 -0
  236. node/planes/transport/link/types.py +27 -0
  237. node/planes/transport/mediation/__init__.py +45 -0
  238. node/planes/transport/mediation/compartment.py +238 -0
  239. node/planes/transport/mediation/leaf.py +107 -0
  240. node/planes/transport/mediation/messages.py +302 -0
  241. node/planes/transport/mediation/service.py +233 -0
  242. node/planes/transport/mediation/store.py +37 -0
  243. node/planes/transport/mediation/types.py +20 -0
  244. node/planes/transport/p2p/__init__.py +7 -0
  245. node/planes/transport/p2p/compartment.py +349 -0
  246. node/planes/transport/p2p/types.py +22 -0
  247. node/protocols/__init__.py +277 -0
  248. node/protocols/activitypub/__init__.py +56 -0
  249. node/protocols/activitypub/actor.py +230 -0
  250. node/protocols/activitypub/federation.py +490 -0
  251. node/protocols/activitypub/media.py +145 -0
  252. node/protocols/activitypub/models.py +286 -0
  253. node/protocols/activitypub/package.py +271 -0
  254. node/protocols/activitypub/store.py +564 -0
  255. node/protocols/arcade/__init__.py +26 -0
  256. node/protocols/atproto/__init__.py +13 -0
  257. node/protocols/atproto/firehose.py +126 -0
  258. node/protocols/atproto/metering.py +165 -0
  259. node/protocols/atproto/package.py +155 -0
  260. node/protocols/atproto/repo.py +250 -0
  261. node/protocols/atproto/xrpc.py +285 -0
  262. node/protocols/base.py +109 -0
  263. node/protocols/circles/__init__.py +29 -0
  264. node/protocols/dm/__init__.py +50 -0
  265. node/protocols/grasp/__init__.py +22 -0
  266. node/protocols/isotime.py +15 -0
  267. node/protocols/nostr/__init__.py +51 -0
  268. node/protocols/nostr/blobstore.py +383 -0
  269. node/protocols/nostr/blossom.py +627 -0
  270. node/protocols/nostr/client.py +377 -0
  271. node/protocols/nostr/descriptor.py +211 -0
  272. node/protocols/nostr/eventstore.py +304 -0
  273. node/protocols/nostr/package.py +182 -0
  274. node/protocols/nostr/peers.py +109 -0
  275. node/protocols/nostr/relayserver.py +656 -0
  276. node/protocols/p2panda/__init__.py +22 -0
  277. node/protocols/p2panda/identity.py +19 -0
  278. node/protocols/p2panda/package.py +236 -0
  279. node/protocols/p2panda/rung.py +152 -0
  280. node/protocols/p2panda/store.py +273 -0
  281. node/protocols/rhio/__init__.py +33 -0
  282. node/protocols/rhio/package.py +275 -0
  283. node/protocols/rhio/rung.py +136 -0
  284. node/protocols/rhio/store.py +1423 -0
  285. node/protocols/ssb/muxrpc.py +256 -0
  286. node/protocols/ssb/package.py +267 -0
  287. node/protocols/textile/__init__.py +111 -0
  288. node/protocols/textile/transducers.py +259 -0
  289. node/release.py +119 -0
  290. node/storage/__init__.py +88 -0
  291. node/storage/accounts.py +305 -0
  292. node/storage/adapter.py +161 -0
  293. node/storage/base.py +385 -0
  294. node/storage/dialect.py +209 -0
  295. node/storage/grants.py +317 -0
  296. node/storage/locator.py +150 -0
  297. node/storage/postgres.py +294 -0
  298. node/storage/projected.py +953 -0
  299. node/storage/sharding.py +448 -0
  300. node/storage/standing.py +568 -0
  301. node/testnet/__init__.py +9 -0
  302. node/testnet/cluster.py +731 -0
  303. node/testnet/fips_daemon.py +230 -0
  304. node/transport/__init__.py +94 -0
  305. node/transport/const.py +110 -0
  306. node/transport/didcomm_inbound.py +251 -0
  307. node/transport/dnstxt.py +169 -0
  308. node/transport/fips_rung/__init__.py +67 -0
  309. node/transport/fips_rung/core.py +666 -0
  310. node/transport/fips_rung/listener.py +96 -0
  311. node/transport/fips_rung/types.py +12 -0
  312. node/transport/host.py +334 -0
  313. node/transport/iroh_rung/__init__.py +29 -0
  314. node/transport/iroh_rung/core.py +477 -0
  315. node/transport/iroh_rung/quic.py +942 -0
  316. node/transport/iroh_rung/types.py +22 -0
  317. node/transport/ladder.py +145 -0
  318. node/transport/lineproto.py +49 -0
  319. node/transport/mdns.py +303 -0
  320. node/transport/netfetch.py +283 -0
  321. node/transport/relay_rung/__init__.py +23 -0
  322. node/transport/relay_rung/cooldown.py +87 -0
  323. node/transport/relay_rung/core.py +138 -0
  324. node/transport/relay_rung/inbound.py +161 -0
  325. node/transport/relay_rung/ledger.py +104 -0
  326. node/transport/relay_rung/listen.py +121 -0
  327. node/transport/relay_rung/outbound.py +181 -0
  328. node/transport/relay_rung/shape.py +71 -0
  329. node/transport/relay_rung/types.py +58 -0
  330. node/transport/relay_rung/wire.py +63 -0
  331. node/transport/router.py +761 -0
  332. node/transport/rung.py +276 -0
  333. node/transport/ssb_rung.py +166 -0
  334. node/transport/ws.py +502 -0
  335. sidecar/fleetmesh_announce.py +338 -0
  336. sidecar/fleetmesh_p2p.py +373 -0
  337. sidecar/fleetmesh_rhio.py +352 -0
  338. sidecar/fleetmesh_ssb.py +603 -0
node/meshnode.py ADDED
@@ -0,0 +1,111 @@
1
+ #!/usr/bin/env python3
2
+ """meshnode: the operator's command.
3
+
4
+ <!-- commands:begin -->
5
+ meshnode init DIR generate keys, write config, mine and publish the descriptor
6
+ meshnode run DIR start the node and tick until SIGTERM
7
+ meshnode status DIR what the running node last wrote, and its trouble
8
+ meshnode health DIR every error plane: relays, drops, rejections, failures
9
+ meshnode whoami DIR the node's identity: DID, npub, keys, descriptor
10
+ meshnode check DIR is this directory a node that can run? exits 1 if not
11
+ meshnode tick DIR one tick now, and what it did (needs the node running)
12
+ meshnode api DIR the control socket described by itself: protocol, commands, arguments, replies
13
+ meshnode peers DIR every peer: tiers, pending grant, last window, standing
14
+ meshnode grant DIR PEER both grants with a peer, their history, and what waits for the boundary
15
+ meshnode windows DIR PEER the windows this node priced for a peer
16
+ meshnode peer DIR PEER add a peer and shard to pull from
17
+ meshnode unpeer DIR PEER stop pulling from a peer; the grants stay live
18
+ meshnode trust DIR PEER inspect or set a peer's tier, which nothing else can raise
19
+ meshnode terminate DIR PEER end a peering: zero both grants and tell the peer (needs the node running)
20
+ meshnode retire DIR leave the mesh in order: hand over every asserted shard, withdraw the claims, stop
21
+ meshnode withdraw DIR PEER WINDOW withdraw a finding this node made about one window (needs the node running)
22
+ meshnode standing DIR PEER TARGET ask a peer what it knows about a third node (needs the node running)
23
+ meshnode reports DIR TARGET the kind 30802s about a node, weighed by this one (needs the node running)
24
+ meshnode strain DIR what this node has survived, by direction, and the reserves it sets
25
+ meshnode subsystems DIR every compartment of this node, and what each has refused
26
+ meshnode refusals DIR every no this node said, across every compartment
27
+ meshnode events DIR the node's recent bus events, from memory
28
+ meshnode subscribe DIR follow the node's bus live, one line per event
29
+ meshnode capacity DIR this node's own ceiling, the pressure on it, and what it is shedding
30
+ meshnode scheduled DIR every scheduled pull, throttles and holds
31
+ meshnode coverage DIR ACTION SHARD the shards this node claims to hold, as kind 30803
32
+ meshnode descriptor DIR the kind 11801 this node published
33
+ meshnode pull DIR PEER SHARD pull a shard from a peer now (needs the node running)
34
+ meshnode bridge DIR ACTION interact with the Universal Nostr Relay Bridge
35
+ meshnode compute DIR ACTION inspect or interact with the compute market and order book
36
+ meshnode dvm DIR ACTION interact with the NIP-90 Data Vending Machine Gateway
37
+ meshnode settle DIR ACTION inspect settlement rails, escrows, and execute live payments
38
+ meshnode gossip DIR ACTION gossip/1.0 group topics: create and publish rosters, flood sealed wraps
39
+ meshnode enclave DIR ACTION inspect and interact with hardware enclaves and passkeys
40
+ meshnode iroh DIR ACTION iroh P2P QUIC transport diagnostics and control
41
+ meshnode fips DIR ACTION FIPS Native Datagram API transport diagnostics and control
42
+ meshnode cluster DIR CLUSTER_ACTION manage a local multi-node testnet cluster
43
+ meshnode nsite DIR ACTION host static sites (NIP-5A): publish this node's site, manage custom domains
44
+ meshnode sidecars DIR ACTION fronted sidecars (grasp, ncontainer, magpie, ...): self-test results, or run the tests now
45
+ meshnode sidecar-manifest FILE write, lint or try a sidecar manifest with no node (for sidecar authors)
46
+ meshnode plugin-check REF hold a plugin class to the rules the node applies at startup, with no node (for plugin authors)
47
+ <!-- commands:end -->
48
+
49
+ Two kinds of command. The ones that read state read the node directory, so
50
+ they work whether the node is running or not. The ones that change what the
51
+ node is told to do - peer, unpeer, trust, coverage - write config.json and
52
+ nothing else; a running node re-reads it every tick. The ones that need the
53
+ daemon itself - pull, terminate, withdraw, standing, reports, tick, republish - go over
54
+ the control socket in the directory, and say so when no node is running.
55
+
56
+ PEER may be a pubkey, a did:nostr, an npub, or an unambiguous prefix of a peer this
57
+ node holds a grant with.
58
+
59
+ `trust` is the only thing that raises a peer's tier. The severity ladder
60
+ lowers one and never raises it, because standing is bought with observed
61
+ time rather than granted by arithmetic; seeding a tier for a node you know
62
+ out of band is a human decision, recorded as a signed grant and reversible.
63
+ A peer left alone opens at probation, whose reporter weight is zero.
64
+
65
+ DIR holds keys.json (0600), config.json, descriptor.json, events.sqlite,
66
+ grants.sqlite, accounts.sqlite, status.json and, while the node runs, the
67
+ control socket.
68
+
69
+ The command table above sits between marker comments and is regenerated from
70
+ the parser by `scripts/check-cli.py --write`; the checker fails when it drifts.
71
+ The commands themselves live in node/commands/*.py, one module per group; this
72
+ file only assembles the parser and is the `meshnode` entrypoint.
73
+ """
74
+ from __future__ import annotations
75
+
76
+ import argparse
77
+ import os
78
+ import sys
79
+ from typing import Sequence
80
+
81
+ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')))
82
+ from node.commands import PROVIDERS, base, plugin_cli_classes, register_plugin_cli
83
+
84
+ TIERS: tuple[str, ...] = base.TIERS
85
+
86
+
87
+ def build_parser(argv: Sequence[str] | None = None) -> argparse.ArgumentParser:
88
+ """The whole operator CLI. Separate from main() so a checker can walk it.
89
+
90
+ `argv` lets the plugins a node directory declares add their verbs
91
+ (node/commands `register_plugin_cli`); without it the parser is the core
92
+ CLI alone, which is what the documentation checker reads.
93
+ """
94
+ ap = argparse.ArgumentParser(prog='meshnode', description=__doc__.split('\n\n')[0])
95
+ sub = ap.add_subparsers(dest='cmd', required=True)
96
+ for provider in PROVIDERS:
97
+ provider.register(sub)
98
+ if argv is not None:
99
+ register_plugin_cli(sub, plugin_cli_classes(argv))
100
+ return ap
101
+
102
+
103
+ def main(argv: Sequence[str] | None = None) -> None:
104
+ a = build_parser(argv if argv is not None else sys.argv[1:]).parse_args(argv)
105
+ code = a.fn(a)
106
+ if isinstance(code, int) and code:
107
+ sys.exit(code)
108
+
109
+
110
+ if __name__ == '__main__':
111
+ main()
@@ -0,0 +1,396 @@
1
+ """What a compartment measures, declared where the compartment is.
2
+
3
+ The node already exposes Prometheus text, and it does it in one place for one
4
+ compartment: `relayserver.render_metrics` hand-builds the exposition with the
5
+ metric names, HELP and TYPE lines written inline, reaching into `self.store`,
6
+ `self._conns` and `self.blob_store` behind `hasattr` guards and bare excepts.
7
+ Meanwhile nineteen compartments keep a `counts` Counter and a `state()` dict
8
+ that no exporter has ever seen. So a node reports the relay face and is silent
9
+ about the budget, the scheduler, the grants and the bridge.
10
+
11
+ That is the same shape as the log before `Channel`: one hand-written path, no
12
+ declaration, and nothing a reader can enumerate. A `Metric` is the answer in
13
+ the same form - declared beside the compartment that measures it, with its
14
+ kind, its unit and its help text attached, so the exposition is rendered from
15
+ declarations rather than typed out.
16
+
17
+ `relayserver` already takes a `metrics_provider` callable and nothing ever
18
+ passed one. That seam is what this plugs into: the registry becomes the
19
+ provider, and the relay face's own numbers become declared metrics like
20
+ everybody else's.
21
+
22
+ ## The bus does the counting
23
+
24
+ A metric naming a channel in `counts` is incremented by that channel being
25
+ raised. `Log.say` tells the registry, so declaring
26
+
27
+ Channel(name='release.available', ...)
28
+ Metric(name='release_available', kind=COUNTER, counts='release.available')
29
+
30
+ is all a compartment does to get both a log line and a counter, and the two
31
+ cannot disagree about how many there were. Monitors ride the bus the log
32
+ rides, which is the point of putting them on it.
33
+
34
+ A gauge reads the world instead, through `sample`. A sampler that raises is
35
+ counted and skipped: an exporter that can be taken down by one bad reading
36
+ takes the monitoring down exactly when something is wrong.
37
+
38
+ ## Prometheus names are not Python names
39
+
40
+ Exposition names must match [a-zA-Z_:][a-zA-Z0-9_:]*, so a channel's dots
41
+ cannot appear in one; a counter conventionally ends `_total`; a unit belongs
42
+ in the name as `_bytes` or `_seconds`. All three are derived here and checked
43
+ at registration, because a name Prometheus rejects is a metric that silently
44
+ never arrives.
45
+ """
46
+ from __future__ import annotations
47
+
48
+ import collections
49
+ import dataclasses
50
+ import re
51
+ import threading
52
+ from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, TypedDict
53
+
54
+ COUNTER, GAUGE, HISTOGRAM = 'counter', 'gauge', 'histogram'
55
+ KINDS: tuple[str, ...] = (COUNTER, GAUGE, HISTOGRAM)
56
+
57
+ # Every metric this node exposes carries the project prefix, so a Prometheus
58
+ # server scraping several things can tell whose they are.
59
+ PREFIX: str = 'fleetmesh'
60
+
61
+ # Units that belong in the name, by Prometheus convention. A unit not listed
62
+ # here is refused rather than appended blindly: 'ms' is the classic mistake,
63
+ # and the convention is seconds.
64
+ UNITS: tuple[str, ...] = ('bytes', 'seconds', 'ratio', 'info')
65
+
66
+ _NAME = re.compile(r'[a-z_][a-z0-9_]*$')
67
+ _LABEL = re.compile(r'[a-z_][a-z0-9_]*$')
68
+
69
+ # Buckets for a latency histogram, in seconds. Declared so a compartment that
70
+ # wants the usual spread does not retype it, and so two compartments measuring
71
+ # latency are comparable.
72
+ LATENCY_BUCKETS: tuple[float, ...] = (0.005, 0.025, 0.1, 0.5, 1.0, 5.0, 30.0)
73
+
74
+
75
+ class MetricRecord(TypedDict, total=False):
76
+ """A metric as `meshnode metrics` and the docs read it."""
77
+ name: str
78
+ exposed: str
79
+ description: str
80
+ kind: str
81
+ unit: str
82
+ labels: list[str]
83
+ counts: str
84
+ tags: list[str]
85
+ compartment: str
86
+
87
+
88
+ class RegistryState(TypedDict):
89
+ """What the registry itself is doing, for the health block."""
90
+ metrics: int
91
+ channels_bound: int
92
+ rejected: list[str]
93
+ failures: dict[str, int]
94
+
95
+
96
+ class Reading(TypedDict, total=False):
97
+ """One value of one metric, with the labels it was recorded under."""
98
+ name: str
99
+ exposed: str
100
+ kind: str
101
+ labels: dict[str, str]
102
+ value: float
103
+ buckets: dict[str, int]
104
+ count: int
105
+ sum: float
106
+
107
+
108
+ @dataclasses.dataclass(frozen=True)
109
+ class Metric:
110
+ """One thing a compartment measures.
111
+
112
+ `counts` names a channel and makes this a counter the bus drives.
113
+ `sample` reads the world and makes it a gauge. A metric with neither is
114
+ incremented by hand through `Monitors.inc`, which is what a counter that
115
+ does not correspond to a single channel needs.
116
+ """
117
+ name: str
118
+ description: str
119
+ kind: str = GAUGE
120
+ unit: str = ''
121
+ labels: tuple[str, ...] = ()
122
+ counts: str = ''
123
+ sample: Optional[Callable[[], Any]] = None
124
+ buckets: tuple[float, ...] = ()
125
+ tags: tuple[str, ...] = ()
126
+
127
+ @property
128
+ def exposed(self) -> str:
129
+ """The Prometheus name: prefix, name, unit, and `_total` for a counter.
130
+
131
+ Derived rather than written, so a declaration cannot name one thing in
132
+ the docs and another in the exposition.
133
+ """
134
+ parts = [PREFIX, self.name]
135
+ if self.unit:
136
+ parts.append(self.unit)
137
+ out = '_'.join(parts)
138
+ if self.kind == COUNTER and not out.endswith('_total'):
139
+ out += '_total'
140
+ return out
141
+
142
+ def problems(self) -> list[str]:
143
+ """Everything wrong with this declaration, in sentences.
144
+
145
+ Collected rather than raised one at a time, as `config.Problems` does,
146
+ and checked at registration: a name Prometheus rejects is a metric
147
+ that silently never arrives, which is the failure mode worth paying a
148
+ check for.
149
+ """
150
+ out: list[str] = []
151
+ if not _NAME.match(self.name or ''):
152
+ out.append('%r must be lowercase letters, digits and underscores, '
153
+ 'starting with a letter or underscore' % self.name)
154
+ if self.kind not in KINDS:
155
+ out.append('%s: kind must be one of %s' % (self.name, ', '.join(KINDS)))
156
+ if self.unit and self.name.endswith('_' + self.unit):
157
+ # fleetmesh_updates_check_seconds_seconds, which is what the first
158
+ # declaration produced. The unit is appended from `unit`, so
159
+ # putting it in the name as well doubles it. Refused rather than
160
+ # quietly de-duplicated: the declaration is muddled and a silent
161
+ # fix leaves two spellings that look equally correct.
162
+ out.append('%s: the unit is appended from `unit`, so do not end the name '
163
+ 'with %r as well' % (self.name, self.unit))
164
+ if self.unit and self.unit not in UNITS:
165
+ out.append('%s: unit %r is not one of %s - milliseconds are a classic '
166
+ 'mistake and the convention is seconds'
167
+ % (self.name, self.unit, ', '.join(UNITS)))
168
+ for lab in self.labels:
169
+ if not _LABEL.match(lab):
170
+ out.append('%s: label %r must be lowercase letters, digits and '
171
+ 'underscores' % (self.name, lab))
172
+ if self.kind == HISTOGRAM and not self.buckets:
173
+ out.append('%s: a histogram needs buckets; LATENCY_BUCKETS is the house set'
174
+ % self.name)
175
+ if self.kind != HISTOGRAM and self.buckets:
176
+ out.append('%s: only a histogram has buckets' % self.name)
177
+ if self.counts and self.kind != COUNTER:
178
+ out.append('%s: a metric fed by a channel is a counter, not a %s'
179
+ % (self.name, self.kind))
180
+ if self.counts and self.sample is not None:
181
+ out.append('%s: counts and sample are two sources for one metric; pick one'
182
+ % self.name)
183
+ if self.kind == GAUGE and self.sample is None and not self.labels:
184
+ # A gauge nothing reads and nothing sets is a metric that will
185
+ # always report zero, which is worse than an absent one.
186
+ out.append('%s: a gauge needs a sample, or labels it is set under'
187
+ % self.name)
188
+ if self.buckets and list(self.buckets) != sorted(self.buckets):
189
+ out.append('%s: buckets must be ascending' % self.name)
190
+ return out
191
+
192
+ def describe(self) -> MetricRecord:
193
+ out: MetricRecord = {
194
+ 'name': self.name, 'exposed': self.exposed, 'description': self.description,
195
+ 'kind': self.kind, 'labels': list(self.labels), 'tags': list(self.tags),
196
+ }
197
+ if self.unit:
198
+ out['unit'] = self.unit
199
+ if self.counts:
200
+ out['counts'] = self.counts
201
+ return out
202
+
203
+
204
+ def declared(obj: Any) -> tuple[Metric, ...]:
205
+ """The metrics a compartment declares, or () for one not yet converted."""
206
+ return tuple(getattr(obj, 'METRICS', ()) or ())
207
+
208
+
209
+ def every_metric(compartments: Sequence[Any]) -> list[MetricRecord]:
210
+ """Every declared metric, the companion of every_channel and
211
+ every_parameter: what the node measures, as those are what it says and
212
+ what it can be told."""
213
+ out: list[MetricRecord] = []
214
+ for c in compartments:
215
+ for m in declared(c):
216
+ row = m.describe()
217
+ row['compartment'] = (getattr(c, 'name', '') or getattr(c, 'key', '')
218
+ or getattr(c, '__name__', str(c)))
219
+ out.append(row)
220
+ return out
221
+
222
+
223
+ def _key(labels: Mapping[str, str], names: Sequence[str]) -> tuple[str, ...]:
224
+ """The label values in declared order, so one series has one key."""
225
+ return tuple(str(labels.get(n, '')) for n in names)
226
+
227
+
228
+ class Monitors:
229
+ """The registry: declarations in, readings out, adapters on the end.
230
+
231
+ One per node. `Log` holds a reference and tells it when a channel is
232
+ raised, which is how a counter declared against a channel is fed without
233
+ the compartment doing anything twice.
234
+ """
235
+
236
+ def __init__(self) -> None:
237
+ self._lock = threading.RLock()
238
+ self.metrics: dict[str, Metric] = {}
239
+ self.owner: dict[str, str] = {}
240
+ self._counts: dict[str, dict[tuple[str, ...], float]] = collections.defaultdict(dict)
241
+ self._gauges: dict[str, dict[tuple[str, ...], float]] = collections.defaultdict(dict)
242
+ self._hist: dict[str, dict[tuple[str, ...], list[float]]] = collections.defaultdict(dict)
243
+ self._by_channel: dict[str, list[str]] = collections.defaultdict(list)
244
+ self.rejected: list[str] = []
245
+ self.failures: dict[str, int] = {}
246
+
247
+ # ---- declaring ------------------------------------------------------
248
+ def register(self, metrics: Iterable[Metric], owner: str = '') -> list[str]:
249
+ """Declare metrics. Returns the problems, and registers the rest.
250
+
251
+ A bad declaration is refused and reported rather than raised, so one
252
+ malformed metric cannot stop a node booting - but it is kept in
253
+ `rejected` and surfaced by `state()`, because a metric silently
254
+ missing from the exposition is the thing this checks for.
255
+ """
256
+ problems: list[str] = []
257
+ with self._lock:
258
+ for m in metrics:
259
+ bad = m.problems()
260
+ if m.exposed in self.metrics and self.metrics[m.exposed] != m:
261
+ bad.append('%s: %s is already declared by %s; one exposition name is '
262
+ 'one metric' % (m.name, m.exposed, self.owner.get(m.exposed, '?')))
263
+ if bad:
264
+ problems.extend(bad)
265
+ self.rejected.extend(bad)
266
+ continue
267
+ self.metrics[m.exposed] = m
268
+ self.owner[m.exposed] = owner
269
+ if m.counts:
270
+ self._by_channel[m.counts].append(m.exposed)
271
+ return problems
272
+
273
+ def register_from(self, compartments: Iterable[Any]) -> list[str]:
274
+ out: list[str] = []
275
+ for c in compartments:
276
+ name = (getattr(c, 'name', '') or getattr(c, 'key', '')
277
+ or getattr(c, '__name__', str(c)))
278
+ out.extend(self.register(declared(c), owner=name))
279
+ return out
280
+
281
+ # ---- recording ------------------------------------------------------
282
+ def observe(self, channel: str) -> None:
283
+ """A channel was raised. Increment every counter declared against it.
284
+
285
+ Called by `Log.say`, which is what puts the monitors on the same bus
286
+ as the log: one declaration feeds the line and the count, so they
287
+ cannot disagree about how many there were.
288
+ """
289
+ with self._lock:
290
+ for exposed in self._by_channel.get(channel, ()):
291
+ series = self._counts[exposed]
292
+ series[()] = series.get((), 0.0) + 1.0
293
+
294
+ def inc(self, name: str, n: float = 1.0, **labels: str) -> None:
295
+ """Increment a counter by hand, for one that is not a single channel."""
296
+ with self._lock:
297
+ m = self._find(name)
298
+ if m is None or m.kind != COUNTER:
299
+ self.failures[name] = self.failures.get(name, 0) + 1
300
+ return
301
+ k = _key(labels, m.labels)
302
+ self._counts[m.exposed][k] = self._counts[m.exposed].get(k, 0.0) + n
303
+
304
+ def set(self, name: str, value: float, **labels: str) -> None:
305
+ """Set a gauge that is pushed rather than sampled."""
306
+ with self._lock:
307
+ m = self._find(name)
308
+ if m is None or m.kind != GAUGE:
309
+ self.failures[name] = self.failures.get(name, 0) + 1
310
+ return
311
+ self._gauges[m.exposed][_key(labels, m.labels)] = float(value)
312
+
313
+ def observe_value(self, name: str, value: float, **labels: str) -> None:
314
+ """Record one observation into a histogram."""
315
+ with self._lock:
316
+ m = self._find(name)
317
+ if m is None or m.kind != HISTOGRAM:
318
+ self.failures[name] = self.failures.get(name, 0) + 1
319
+ return
320
+ self._hist[m.exposed].setdefault(_key(labels, m.labels), []).append(float(value))
321
+
322
+ def _find(self, name: str) -> Optional[Metric]:
323
+ if name in self.metrics:
324
+ return self.metrics[name]
325
+ for m in self.metrics.values():
326
+ if m.name == name:
327
+ return m
328
+ return None
329
+
330
+ # ---- reading --------------------------------------------------------
331
+ def read(self) -> list[Reading]:
332
+ """Every metric's current value, samplers called now.
333
+
334
+ A sampler that raises is counted and skipped. An exporter one bad
335
+ reading can take down is monitoring that fails exactly when something
336
+ is wrong, which is when it is wanted.
337
+ """
338
+ out: list[Reading] = []
339
+ with self._lock:
340
+ items = list(self.metrics.items())
341
+ for exposed, m in items:
342
+ if m.sample is not None:
343
+ try:
344
+ val = m.sample()
345
+ except Exception:
346
+ with self._lock:
347
+ self.failures[m.name] = self.failures.get(m.name, 0) + 1
348
+ continue
349
+ if isinstance(val, Mapping):
350
+ for lab, v in val.items():
351
+ labels = lab if isinstance(lab, tuple) else (lab,)
352
+ out.append({'name': m.name, 'exposed': exposed, 'kind': m.kind,
353
+ 'labels': dict(zip(m.labels, [str(x) for x in labels])),
354
+ 'value': float(v)})
355
+ else:
356
+ out.append({'name': m.name, 'exposed': exposed, 'kind': m.kind,
357
+ 'labels': {}, 'value': float(val)})
358
+ continue
359
+ with self._lock:
360
+ if m.kind == COUNTER:
361
+ series = dict(self._counts.get(exposed, {}))
362
+ elif m.kind == GAUGE:
363
+ series = dict(self._gauges.get(exposed, {}))
364
+ else:
365
+ series = {}
366
+ for k, vals in self._hist.get(exposed, {}).items():
367
+ buckets = {}
368
+ for b in m.buckets:
369
+ buckets['%g' % b] = sum(1 for v in vals if v <= b)
370
+ buckets['+Inf'] = len(vals)
371
+ out.append({'name': m.name, 'exposed': exposed, 'kind': m.kind,
372
+ 'labels': dict(zip(m.labels, k)), 'buckets': buckets,
373
+ 'count': len(vals), 'sum': float(sum(vals))})
374
+ for k, v in series.items():
375
+ out.append({'name': m.name, 'exposed': exposed, 'kind': m.kind,
376
+ 'labels': dict(zip(m.labels, k)), 'value': float(v)})
377
+ if m.kind == COUNTER and not series:
378
+ # A declared counter that has not fired yet reports zero. An
379
+ # absent series and a zero one mean different things to an
380
+ # alert, and "it has never happened" is the useful one.
381
+ out.append({'name': m.name, 'exposed': exposed, 'kind': m.kind,
382
+ 'labels': {}, 'value': 0.0})
383
+ return out
384
+
385
+ def state(self) -> 'RegistryState':
386
+ """What the registry itself is doing, for the health block."""
387
+ with self._lock:
388
+ out: RegistryState = {
389
+ 'metrics': len(self.metrics), 'channels_bound': len(self._by_channel),
390
+ 'rejected': list(self.rejected), 'failures': dict(self.failures)}
391
+ return out
392
+
393
+
394
+ __all__ = ['Metric', 'MetricRecord', 'Reading', 'Monitors', 'declared', 'every_metric',
395
+ 'COUNTER', 'GAUGE', 'HISTOGRAM', 'KINDS', 'UNITS', 'PREFIX', 'LATENCY_BUCKETS',
396
+ 'RegistryState']
@@ -0,0 +1,102 @@
1
+ """Rendering readings for whoever is asking.
2
+
3
+ An adapter turns `Monitors.read()` into one wire format. They are separate
4
+ from the registry for the reason the log's backends are separate from the
5
+ channels: what a compartment measures is not a decision about who scrapes it,
6
+ and a second consumer must not mean a second set of declarations.
7
+
8
+ prometheus the text exposition format, which is also OpenMetrics-
9
+ compatible enough for a Prometheus server to scrape
10
+ json the same readings as data, for the control socket and the
11
+ health block
12
+
13
+ `relayserver` already accepts a `metrics_provider` callable and nothing ever
14
+ passed one, so `provider(monitors)` is the function to hand it. The relay
15
+ face's hand-written exposition then becomes the fallback it already is rather
16
+ than the only path.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ from typing import Callable, Mapping, Sequence, TypedDict
21
+
22
+ from . import HISTOGRAM, Monitors, Reading, RegistryState
23
+
24
+
25
+ class Exposition(TypedDict):
26
+ """The readings as data: what the control socket and the health block get.
27
+
28
+ A named record for the reason check-dicts.py enforces it - this is a row
29
+ an operator reads by field name.
30
+ """
31
+ metrics: list[Reading]
32
+ registry: RegistryState
33
+
34
+
35
+ def _escape(v: str) -> str:
36
+ """A label value, escaped as the exposition format requires.
37
+
38
+ Backslash first, or the escapes introduced after it get escaped again.
39
+ """
40
+ return v.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
41
+
42
+
43
+ def _labels(labels: Mapping[str, str], extra: Sequence[tuple[str, str]] = ()) -> str:
44
+ items = [(k, str(v)) for k, v in labels.items() if v != ''] + list(extra)
45
+ if not items:
46
+ return ''
47
+ return '{%s}' % ','.join('%s="%s"' % (k, _escape(v)) for k, v in items)
48
+
49
+
50
+ def prometheus(monitors: Monitors) -> str:
51
+ """The text exposition format.
52
+
53
+ HELP and TYPE are emitted once per metric name and before its samples,
54
+ which the format requires: a repeated TYPE line makes a scrape fail, and
55
+ a scrape that fails is monitoring that is absent rather than wrong.
56
+ """
57
+ readings = monitors.read()
58
+ by_name: dict[str, list[Reading]] = {}
59
+ for r in readings:
60
+ by_name.setdefault(r['exposed'], []).append(r)
61
+
62
+ out: list[str] = []
63
+ for exposed, rows in by_name.items():
64
+ m = monitors.metrics.get(exposed)
65
+ if m is None:
66
+ continue
67
+ help_text = m.description.replace('\\', '\\\\').replace('\n', ' ')
68
+ out.append('# HELP %s %s' % (exposed, help_text))
69
+ out.append('# TYPE %s %s' % (exposed, m.kind))
70
+ for r in rows:
71
+ if m.kind == HISTOGRAM:
72
+ for le, n in (r.get('buckets') or {}).items():
73
+ out.append('%s_bucket%s %d' % (
74
+ exposed, _labels(r.get('labels', {}), [('le', le)]), n))
75
+ out.append('%s_sum%s %g' % (exposed, _labels(r.get('labels', {})),
76
+ r.get('sum', 0.0)))
77
+ out.append('%s_count%s %d' % (exposed, _labels(r.get('labels', {})),
78
+ r.get('count', 0)))
79
+ else:
80
+ out.append('%s%s %g' % (exposed, _labels(r.get('labels', {})),
81
+ r.get('value', 0.0)))
82
+ return '\n'.join(out) + ('\n' if out else '')
83
+
84
+
85
+ def json_readings(monitors: Monitors) -> 'Exposition':
86
+ """The readings as data, for the control socket and the health block.
87
+
88
+ The same readings the exposition is built from, so a dashboard reading the
89
+ socket and a Prometheus server scraping the face cannot see different
90
+ numbers - which is the failure the log had before Channel.
91
+ """
92
+ return {'metrics': monitors.read(), 'registry': monitors.state()}
93
+
94
+
95
+ def provider(monitors: Monitors) -> Callable[[], str]:
96
+ """The callable `relayserver(metrics_provider=...)` has always wanted."""
97
+ def render() -> str:
98
+ return prometheus(monitors)
99
+ return render
100
+
101
+
102
+ __all__ = ['prometheus', 'json_readings', 'provider', 'Exposition']