superlocalmemory 3.8.13 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/ATTRIBUTION.md +4 -4
  2. package/CHANGELOG.md +113 -121
  3. package/README.md +65 -63
  4. package/docs/pi-dev-integration.md +1 -1
  5. package/package.json +6 -1
  6. package/plugin/.claude-plugin/plugin.json +1 -1
  7. package/plugin/CLAUDE.md +3 -3
  8. package/plugin/agents/slm-governance-advisor.md +1 -1
  9. package/plugin/agents/slm-loop-runner.md +1 -1
  10. package/plugin/agents/slm-memory-advisor.md +1 -1
  11. package/plugin/agents/slm-optimize-advisor.md +1 -1
  12. package/plugin/requirements.txt +1 -1
  13. package/plugin/skills/slm-cache/SKILL.md +1 -1
  14. package/plugin/skills/slm-compress/SKILL.md +1 -1
  15. package/plugin/skills/slm-governance/SKILL.md +1 -1
  16. package/plugin/skills/slm-graph/SKILL.md +1 -1
  17. package/plugin/skills/slm-loop/SKILL.md +1 -1
  18. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  19. package/plugin/skills/slm-profile/SKILL.md +1 -1
  20. package/plugin/skills/slm-recall/SKILL.md +1 -1
  21. package/plugin/skills/slm-remember/SKILL.md +1 -1
  22. package/plugin/skills/slm-scope/SKILL.md +1 -1
  23. package/plugin/skills/slm-session/SKILL.md +1 -1
  24. package/plugin/skills/slm-status/SKILL.md +1 -1
  25. package/plugin-src/rules/AGENTS.md +1 -1
  26. package/plugin-src/skills/slm-cache/SKILL.md +1 -1
  27. package/plugin-src/skills/slm-compress/SKILL.md +1 -1
  28. package/plugin-src/skills/slm-governance/SKILL.md +248 -0
  29. package/plugin-src/skills/slm-graph/SKILL.md +1 -1
  30. package/plugin-src/skills/slm-loop/SKILL.md +99 -0
  31. package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
  32. package/plugin-src/skills/slm-profile/SKILL.md +148 -0
  33. package/plugin-src/skills/slm-recall/SKILL.md +1 -1
  34. package/plugin-src/skills/slm-remember/SKILL.md +1 -1
  35. package/plugin-src/skills/slm-scope/SKILL.md +176 -0
  36. package/plugin-src/skills/slm-session/SKILL.md +1 -1
  37. package/plugin-src/skills/slm-status/SKILL.md +1 -1
  38. package/pyproject.toml +11 -4
  39. package/src/superlocalmemory/__init__.py +1 -1
  40. package/src/superlocalmemory/cli/commands.py +125 -11
  41. package/src/superlocalmemory/cli/daemon.py +5 -1
  42. package/src/superlocalmemory/cli/main.py +35 -2
  43. package/src/superlocalmemory/cli/ops_cmd.py +281 -0
  44. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  45. package/src/superlocalmemory/compliance/audit.py +65 -0
  46. package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
  47. package/src/superlocalmemory/compliance/gdpr.py +416 -20
  48. package/src/superlocalmemory/compliance/retention.py +74 -22
  49. package/src/superlocalmemory/compliance/scheduler.py +78 -9
  50. package/src/superlocalmemory/core/actor_context.py +166 -0
  51. package/src/superlocalmemory/core/admission.py +549 -0
  52. package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
  53. package/src/superlocalmemory/core/config.py +202 -24
  54. package/src/superlocalmemory/core/consolidation_engine.py +13 -13
  55. package/src/superlocalmemory/core/context_cache.py +28 -0
  56. package/src/superlocalmemory/core/embeddings.py +64 -2
  57. package/src/superlocalmemory/core/engine.py +7 -2
  58. package/src/superlocalmemory/core/engine_ingestion.py +65 -3
  59. package/src/superlocalmemory/core/engine_wiring.py +36 -9
  60. package/src/superlocalmemory/core/ingest_policy.py +38 -0
  61. package/src/superlocalmemory/core/maintenance.py +255 -0
  62. package/src/superlocalmemory/core/modes.py +40 -13
  63. package/src/superlocalmemory/core/mutations.py +437 -44
  64. package/src/superlocalmemory/core/operation_policy.py +92 -0
  65. package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
  66. package/src/superlocalmemory/core/operation_request.py +127 -0
  67. package/src/superlocalmemory/core/ops_remediation.py +542 -0
  68. package/src/superlocalmemory/core/recall_pipeline.py +7 -0
  69. package/src/superlocalmemory/core/remember_runtime.py +202 -4
  70. package/src/superlocalmemory/core/remote_mode.py +20 -5
  71. package/src/superlocalmemory/core/store_pipeline.py +150 -0
  72. package/src/superlocalmemory/core/topic_signature.py +19 -4
  73. package/src/superlocalmemory/core/transactions/__init__.py +78 -0
  74. package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
  75. package/src/superlocalmemory/core/transactions/erasure.py +825 -0
  76. package/src/superlocalmemory/core/transactions/manifest.py +255 -0
  77. package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
  78. package/src/superlocalmemory/core/transactions/obligations.py +272 -0
  79. package/src/superlocalmemory/core/transactions/owners.py +114 -0
  80. package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
  81. package/src/superlocalmemory/core/transactions/service.py +330 -0
  82. package/src/superlocalmemory/core/worker_pool.py +33 -5
  83. package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
  84. package/src/superlocalmemory/encoding/emotional.py +75 -14
  85. package/src/superlocalmemory/encoding/scene_builder.py +115 -13
  86. package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
  87. package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
  88. package/src/superlocalmemory/evolution/evolution_store.py +244 -4
  89. package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
  90. package/src/superlocalmemory/evolution/model_selection.py +18 -3
  91. package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
  92. package/src/superlocalmemory/evolution/skill_activator.py +270 -0
  93. package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
  94. package/src/superlocalmemory/evolution/types.py +30 -8
  95. package/src/superlocalmemory/graph/cozo_backend.py +17 -9
  96. package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
  97. package/src/superlocalmemory/hooks/auto_recall.py +64 -30
  98. package/src/superlocalmemory/hooks/codex_assets.py +14 -1
  99. package/src/superlocalmemory/infra/backup.py +434 -7
  100. package/src/superlocalmemory/infra/process_reaper.py +18 -0
  101. package/src/superlocalmemory/infra/self_heal.py +401 -0
  102. package/src/superlocalmemory/learning/feedback.py +52 -9
  103. package/src/superlocalmemory/loops/engine.py +10 -0
  104. package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
  105. package/src/superlocalmemory/mcp/http_transport.py +30 -331
  106. package/src/superlocalmemory/mcp/profiles.py +5 -0
  107. package/src/superlocalmemory/mcp/resources.py +8 -0
  108. package/src/superlocalmemory/mcp/server.py +51 -4
  109. package/src/superlocalmemory/mcp/shared.py +19 -0
  110. package/src/superlocalmemory/mcp/tools_active.py +25 -4
  111. package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
  112. package/src/superlocalmemory/mcp/tools_context.py +50 -8
  113. package/src/superlocalmemory/mcp/tools_core.py +69 -21
  114. package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
  115. package/src/superlocalmemory/mcp/tools_learning.py +21 -10
  116. package/src/superlocalmemory/mcp/tools_loops.py +29 -18
  117. package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
  118. package/src/superlocalmemory/mcp/tools_ops.py +115 -0
  119. package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
  120. package/src/superlocalmemory/mcp/tools_v28.py +10 -3
  121. package/src/superlocalmemory/mcp/tools_v3.py +34 -14
  122. package/src/superlocalmemory/mcp/tools_v33.py +18 -33
  123. package/src/superlocalmemory/mesh/broker.py +124 -46
  124. package/src/superlocalmemory/mesh/broker_security.py +470 -0
  125. package/src/superlocalmemory/mesh/discovery.py +365 -0
  126. package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
  127. package/src/superlocalmemory/mesh/node_identity.py +97 -0
  128. package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
  129. package/src/superlocalmemory/mesh/remote_sync.py +511 -28
  130. package/src/superlocalmemory/mesh/state_sync.py +286 -0
  131. package/src/superlocalmemory/optimize/config/store.py +45 -0
  132. package/src/superlocalmemory/parameterization/cross_project.py +12 -0
  133. package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
  134. package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
  135. package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
  136. package/src/superlocalmemory/retrieval/ann_index.py +5 -0
  137. package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
  138. package/src/superlocalmemory/retrieval/engine.py +19 -4
  139. package/src/superlocalmemory/retrieval/fusion.py +4 -1
  140. package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
  141. package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
  142. package/src/superlocalmemory/retrieval/reranker.py +32 -1
  143. package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
  144. package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
  145. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
  146. package/src/superlocalmemory/retrieval/vector_store.py +214 -8
  147. package/src/superlocalmemory/server/api.py +5 -5
  148. package/src/superlocalmemory/server/egress_policy.py +258 -0
  149. package/src/superlocalmemory/server/rbac_enforce.py +32 -0
  150. package/src/superlocalmemory/server/route_mutations.py +20 -0
  151. package/src/superlocalmemory/server/routes/compliance.py +153 -7
  152. package/src/superlocalmemory/server/routes/data_io.py +43 -2
  153. package/src/superlocalmemory/server/routes/events.py +15 -0
  154. package/src/superlocalmemory/server/routes/memories.py +56 -3
  155. package/src/superlocalmemory/server/routes/mesh.py +82 -1
  156. package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
  157. package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
  158. package/src/superlocalmemory/server/routes/v3_api.py +50 -27
  159. package/src/superlocalmemory/server/routes/ws.py +86 -0
  160. package/src/superlocalmemory/server/ui.py +6 -6
  161. package/src/superlocalmemory/server/unified_daemon.py +942 -119
  162. package/src/superlocalmemory/storage/_migration_internals.py +568 -0
  163. package/src/superlocalmemory/storage/_schema_version.py +110 -0
  164. package/src/superlocalmemory/storage/database.py +329 -24
  165. package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
  166. package/src/superlocalmemory/storage/erasure_fence.py +45 -0
  167. package/src/superlocalmemory/storage/generation_fence.py +63 -0
  168. package/src/superlocalmemory/storage/migration_runner.py +140 -417
  169. package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
  170. package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
  171. package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
  172. package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
  173. package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
  174. package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
  175. package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
  176. package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
  177. package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
  178. package/src/superlocalmemory/storage/schema.py +67 -0
  179. package/src/superlocalmemory/storage/write_coordinator.py +125 -0
  180. package/src/superlocalmemory/trust/scorer.py +28 -4
  181. package/src/superlocalmemory/ui/index.html +14 -3
  182. package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
  183. package/src/superlocalmemory/ui/js/brain.js +6 -4
  184. package/src/superlocalmemory/ui/js/compliance.js +66 -12
  185. package/src/superlocalmemory/ui/js/dashboard.js +13 -3
  186. package/src/superlocalmemory/ui/js/feedback.js +8 -2
  187. package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
  188. package/src/superlocalmemory/ui/js/modal.js +272 -5
  189. package/src/superlocalmemory/ui/js/od-backup.js +9 -2
  190. package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
  191. package/src/superlocalmemory/ui/js/od-operations.js +154 -23
  192. package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
  193. package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
  194. package/src/superlocalmemory/ui/js/od-team.js +9 -2
  195. package/src/superlocalmemory/ui/js/optimize.js +13 -16
  196. package/src/superlocalmemory/ui/js/profiles.js +7 -3
  197. package/src/superlocalmemory/ui/js/settings.js +7 -1
  198. package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
  199. package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
  200. package/src/superlocalmemory/cli/post_install.py +0 -114
  201. package/src/superlocalmemory/core/clock_monitor.py +0 -45
  202. package/src/superlocalmemory/core/db_pool.py +0 -80
  203. package/src/superlocalmemory/core/error_catalog.py +0 -113
  204. package/src/superlocalmemory/core/loop_watchdog.py +0 -56
  205. package/src/superlocalmemory/core/priority_queue.py +0 -61
  206. package/src/superlocalmemory/core/pruning_engine.py +0 -216
  207. package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
  208. package/src/superlocalmemory/core/slmignore.py +0 -125
  209. package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
  210. package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
  211. package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
  212. package/src/superlocalmemory/storage/access_control.py +0 -182
@@ -9,16 +9,27 @@ Populates broker._remote_peers from the remote /mesh/peers endpoint.
9
9
  Proxies mesh_send to remote when the target peer lives on the remote machine.
10
10
  Optional mDNS discovery via zeroconf.
11
11
 
12
+ New in 3b-1: durable remote outbox — failed sends are enqueued and retried
13
+ with exponential back-off until delivered or TTL-expired.
14
+ New in 3b-3: TLS-pinned transport — opt-in SHA-256 cert pinning + custom CA.
15
+
12
16
  Environment variables:
13
- SLM_MESH_PEER_URL: Full URL of remote SLM (e.g. http://192.168.1.100:8765)
14
- SLM_MESH_SHARED_SECRET: Shared auth secret for remote SLM
15
- SLM_MESH_DISCOVERY: 'on'|'off' (default 'on') — enable mDNS discovery
17
+ SLM_MESH_PEER_URL: Full URL of remote SLM (e.g. http://192.168.1.100:8765)
18
+ SLM_MESH_SHARED_SECRET: Shared auth secret for remote SLM
19
+ SLM_MESH_DISCOVERY: 'on'|'off' (default 'on') — enable mDNS discovery
20
+ SLM_MESH_TLS: 'on'|'off' (default 'off') — use https:// for discovered peers
21
+ SLM_MESH_TLS_CA: Path to custom CA bundle (PEM) for TLS verification
22
+ SLM_MESH_TLS_PIN: Hex SHA-256 of peer leaf cert DER for cert pinning
16
23
  """
17
24
 
18
25
  from __future__ import annotations
19
26
 
27
+ import hashlib
28
+ import json
20
29
  import logging
21
30
  import os
31
+ import socket
32
+ import ssl
22
33
  import threading
23
34
  import time
24
35
  import ipaddress
@@ -28,6 +39,19 @@ import httpx
28
39
 
29
40
  logger = logging.getLogger("superlocalmemory.mesh.remote_sync")
30
41
 
42
+ _PRODUCTION_TRUTHY = frozenset({"1", "true", "yes", "on", "production", "prod"})
43
+
44
+
45
+ def _is_production_mode() -> bool:
46
+ """True when SLM_MESH_PRODUCTION signals a hardened deployment."""
47
+ return os.environ.get("SLM_MESH_PRODUCTION", "").strip().lower() in _PRODUCTION_TRUTHY
48
+
49
+
50
+ def _is_plaintext_url(url: str) -> bool:
51
+ """True for http:// and ws:// URLs (not TLS-protected)."""
52
+ lower = url.strip().lower()
53
+ return lower.startswith("http://") or lower.startswith("ws://")
54
+
31
55
 
32
56
  def _service_ip_addresses(info: Any) -> list[str]:
33
57
  """Return validated textual IPs from current and older Zeroconf APIs."""
@@ -57,7 +81,11 @@ def _service_ip_addresses(info: Any) -> list[str]:
57
81
 
58
82
 
59
83
  def _peer_url(host: str, port: int) -> str:
60
- """Format an IP literal safely for an HTTP authority."""
84
+ """Format an IP literal safely for an HTTP authority.
85
+
86
+ Always produces http:// — preserved exactly for backward compatibility.
87
+ Use _peer_url_with_scheme() when an https:// override is needed.
88
+ """
61
89
  address = ipaddress.ip_address(host)
62
90
  rendered = str(address)
63
91
  if address.version == 6:
@@ -67,6 +95,52 @@ def _peer_url(host: str, port: int) -> str:
67
95
  rendered = f"[{rendered}]"
68
96
  return f"http://{rendered}:{int(port)}"
69
97
 
98
+
99
+ def _peer_url_with_scheme(host: str, port: int, tls: bool = False) -> str:
100
+ """Format an IP literal safely for an HTTP or HTTPS authority.
101
+
102
+ Args:
103
+ host: IP address string (v4 or v6).
104
+ port: Port number.
105
+ tls: When True, emit https:// instead of http://.
106
+
107
+ Returns:
108
+ Scheme-correct URL string.
109
+ """
110
+ address = ipaddress.ip_address(host)
111
+ rendered = str(address)
112
+ if address.version == 6:
113
+ rendered = rendered.replace("%", "%25")
114
+ rendered = f"[{rendered}]"
115
+ scheme = "https" if tls else "http"
116
+ return f"{scheme}://{rendered}:{int(port)}"
117
+
118
+
119
+ def _get_cert_sha256(
120
+ host: str,
121
+ port: int,
122
+ ca_file: str | None = None,
123
+ timeout: int = 5,
124
+ ) -> str:
125
+ """Return hex SHA-256 of the server's leaf certificate DER bytes.
126
+
127
+ Opens a raw TLS connection, retrieves the peer cert in DER form, and
128
+ returns its SHA-256 digest as a lowercase hex string (no colons).
129
+
130
+ Raises:
131
+ ssl.SSLError: TLS negotiation failed or cert not trusted.
132
+ socket.error: Connection refused or timed out.
133
+ OSError: Any underlying network error.
134
+ """
135
+ ctx = ssl.create_default_context(cafile=ca_file)
136
+ with socket.create_connection((host, port), timeout=timeout) as raw_sock:
137
+ with ctx.wrap_socket(raw_sock, server_hostname=host) as tls_sock:
138
+ cert_der = tls_sock.getpeercert(binary_form=True)
139
+ if not cert_der:
140
+ raise ssl.SSLError("peer returned no certificate")
141
+ return hashlib.sha256(cert_der).hexdigest().lower()
142
+
143
+
70
144
  # Optional zeroconf for mDNS discovery
71
145
  try:
72
146
  from zeroconf import ServiceBrowser, ServiceInfo, Zeroconf
@@ -77,6 +151,19 @@ except ImportError:
77
151
  ServiceBrowser = None
78
152
  ServiceInfo = None
79
153
 
154
+ # Optional durable outbox (3b-1) — fail-open so missing module never breaks start()
155
+ try:
156
+ from .outbox_remote import RemoteOutbox
157
+ _OUTBOX_AVAILABLE = True
158
+ except ImportError:
159
+ RemoteOutbox = None # type: ignore[assignment,misc]
160
+ _OUTBOX_AVAILABLE = False
161
+
162
+ #: Wall-clock budget for a single outbox drain pass. Bounds how long a
163
+ #: slow/black-holing peer can monopolize the shared 30s sync thread (audit
164
+ #: P1); remaining due rows are deferred to the next cycle.
165
+ _DRAIN_BUDGET_SECONDS: float = 8.0
166
+
80
167
 
81
168
  class RemoteSyncClient:
82
169
  """HTTP-based sync client for multi-machine mesh coordination.
@@ -84,13 +171,20 @@ class RemoteSyncClient:
84
171
  Syncs remote peers from a peer SLM instance periodically.
85
172
  Proxies mesh_send to remote when target peer lives on remote machine.
86
173
  Optionally discovers remote SLM via mDNS.
174
+
175
+ 3b-1: Failed sends are stored in a durable SQLite outbox and retried
176
+ with exponential back-off so messages survive peer downtime.
177
+
178
+ 3b-3: Optional TLS cert pinning (SLM_MESH_TLS_PIN) and custom CA
179
+ (SLM_MESH_TLS_CA). Default behavior (no env) is byte-for-byte identical
180
+ to previous releases.
87
181
  """
88
182
 
89
183
  def __init__(self, broker: Any) -> None:
90
184
  """Initialize sync client.
91
185
 
92
186
  Args:
93
- broker: Reference to MeshBroker instance
187
+ broker: Reference to MeshBroker instance.
94
188
  """
95
189
  self._broker = broker
96
190
  self._peer_url: str | None = os.environ.get("SLM_MESH_PEER_URL") or None
@@ -116,9 +210,24 @@ class RemoteSyncClient:
116
210
  self._stop_event = threading.Event()
117
211
  self._zeroconf: Zeroconf | None = None
118
212
  self._last_peers: dict[str, dict] = {}
213
+ # 3b-1: lazily initialised durable outbox — None until first access
214
+ self._outbox: RemoteOutbox | None = None # type: ignore[type-arg]
119
215
 
120
216
  def start(self) -> None:
121
- """Start background sync and discovery threads."""
217
+ """Start background sync and discovery threads.
218
+
219
+ In production mode (SLM_MESH_PRODUCTION=1), plaintext peer URLs
220
+ (http:// or ws://) are rejected so credentials cannot be sent in
221
+ the clear and traffic cannot be intercepted.
222
+ """
223
+ if self._peer_url and _is_production_mode() and _is_plaintext_url(self._peer_url):
224
+ raise ValueError(
225
+ f"Production mesh requires TLS transport (https:// or wss://); "
226
+ f"got plaintext peer URL: {self._peer_url!r}. "
227
+ "Set SLM_MESH_PEER_URL to an https:// endpoint or "
228
+ "unset SLM_MESH_PRODUCTION to allow plaintext in dev/local mode."
229
+ )
230
+
122
231
  if not self._peer_url and not self._discovery_enabled:
123
232
  logger.debug(
124
233
  "RemoteSyncClient: no peer URL and discovery disabled, skipping"
@@ -157,14 +266,326 @@ class RemoteSyncClient:
157
266
  if self._discovery_thread:
158
267
  self._discovery_thread.join(timeout=2)
159
268
 
269
+ # ------------------------------------------------------------------
270
+ # TLS helpers (3b-3)
271
+ # ------------------------------------------------------------------
272
+
273
+ def _http_client(self, timeout: int) -> httpx.Client:
274
+ """Build an httpx.Client with appropriate TLS configuration.
275
+
276
+ Honors SLM_MESH_TLS_CA (custom CA bundle path). When unset, uses
277
+ system CAs (verify=True). The cert-pin check is done separately in
278
+ _check_cert_pin() before the first byte is sent.
279
+
280
+ Default (no env) produces httpx.Client(timeout=timeout) — identical
281
+ to the previous hard-coded behavior.
282
+ """
283
+ ca_path = os.environ.get("SLM_MESH_TLS_CA") or None
284
+ if ca_path:
285
+ return httpx.Client(verify=ca_path, timeout=timeout)
286
+ return httpx.Client(timeout=timeout)
287
+
288
+ def _check_cert_pin(self, peer_url: str) -> tuple[bool, str]:
289
+ """Pre-flight SHA-256 certificate pin check for an https:// URL.
290
+
291
+ Opens a brief raw TLS connection to retrieve the leaf certificate,
292
+ computes its SHA-256 hash, and compares it to the configured pin.
293
+ This happens BEFORE the actual HTTP request so no payload is sent
294
+ to a peer with a mismatched certificate.
295
+
296
+ Normalization: both actual and expected hex strings are lowercased
297
+ and colon-stripped before comparison to avoid case-sensitivity bugs.
298
+
299
+ Args:
300
+ peer_url: URL being targeted.
301
+
302
+ Returns:
303
+ (True, "") if pin matches or no pin is configured.
304
+ (False, reason) if pin is configured and does not match.
305
+ """
306
+ pin_env = os.environ.get("SLM_MESH_TLS_PIN") or None
307
+ if not pin_env:
308
+ return True, ""
309
+
310
+ parsed = httpx.URL(peer_url)
311
+ if parsed.scheme != "https":
312
+ # A pin is configured but the peer URL is not https. Pinning is
313
+ # impossible over plaintext, so FAIL CLOSED (audit P1) rather than
314
+ # silently sending in the clear under a false sense of pinning.
315
+ return False, (
316
+ f"SLM_MESH_TLS_PIN is set but peer URL scheme is "
317
+ f"{parsed.scheme!r}, not https — refusing to send unpinned "
318
+ "plaintext (set an https:// peer URL or unset the pin)"
319
+ )
320
+
321
+ host = parsed.host
322
+ port = parsed.port or 443
323
+ # Normalise: lowercase and strip colons (handles both plain hex and
324
+ # colon-separated fingerprint formats e.g. "AB:CD:...").
325
+ expected_pin = pin_env.strip().lower().replace(":", "")
326
+ ca_path = os.environ.get("SLM_MESH_TLS_CA") or None
327
+
328
+ try:
329
+ actual_pin = _get_cert_sha256(host, port, ca_file=ca_path)
330
+ except (ssl.SSLError, socket.error, OSError) as exc:
331
+ return False, f"cert pin check connection failed: {exc}"
332
+
333
+ if actual_pin != expected_pin:
334
+ return False, (
335
+ f"certificate pin mismatch for {host}:{port} "
336
+ f"(expected {expected_pin!r}, got {actual_pin!r})"
337
+ )
338
+ return True, ""
339
+
340
+ # ------------------------------------------------------------------
341
+ # Outbox helpers (3b-1)
342
+ # ------------------------------------------------------------------
343
+
344
+ def _get_outbox(self) -> RemoteOutbox | None: # type: ignore[return]
345
+ """Lazily initialise the RemoteOutbox using the broker's DB path.
346
+
347
+ Returns None (and logs once) if the outbox cannot be initialised
348
+ so the online send path is always unaffected.
349
+ """
350
+ if self._outbox is not None:
351
+ return self._outbox
352
+ if not _OUTBOX_AVAILABLE:
353
+ return None
354
+ db_path = getattr(self._broker, "_db_path", None)
355
+ if not db_path:
356
+ logger.debug("RemoteSyncClient: no db_path on broker — outbox disabled")
357
+ return None
358
+ try:
359
+ self._outbox = RemoteOutbox(db_path) # type: ignore[call-arg]
360
+ except Exception as exc:
361
+ logger.error("RemoteSyncClient: failed to init RemoteOutbox: %s", exc)
362
+ return self._outbox
363
+
364
+ def _enqueue_on_failure(
365
+ self,
366
+ peer_url: str,
367
+ to_peer: str,
368
+ payload: dict[str, Any],
369
+ headers: dict[str, str],
370
+ now: float,
371
+ ) -> None:
372
+ """Enqueue a failed send to the durable outbox.
373
+
374
+ Headers are stored for audit purposes only. The drain loop re-signs
375
+ fresh headers (new nonce + timestamp) before each retry attempt to
376
+ avoid stale-timestamp rejections at the receiving peer.
377
+ """
378
+ outbox = self._get_outbox()
379
+ if outbox is not None:
380
+ outbox.enqueue(peer_url, to_peer, payload, headers, now=now)
381
+
382
+ def _build_signed_headers(
383
+ self,
384
+ payload: dict[str, Any],
385
+ to_peer: str,
386
+ base_headers: dict[str, str],
387
+ ) -> dict[str, str]:
388
+ """Add fresh HMAC signing headers to base_headers if a secret is configured.
389
+
390
+ Generates a new nonce + current timestamp so each signing is unique
391
+ and replay-safe. Returns base_headers unchanged when signing is
392
+ not applicable.
393
+ """
394
+ if not (self._shared_secret and self._peer_url_trusted):
395
+ return dict(base_headers)
396
+
397
+ import secrets as _sec
398
+ from .broker_security import sign_mesh_message
399
+
400
+ from_peer = payload.get("from_peer", "")
401
+ content = payload.get("content", "")
402
+ nonce = _sec.token_hex(16)
403
+ ts = str(int(time.time()))
404
+ sig = sign_mesh_message(
405
+ self._shared_secret, from_peer, to_peer, content, nonce, ts,
406
+ )
407
+ return {
408
+ **base_headers,
409
+ "X-Mesh-Sig": sig,
410
+ "X-Mesh-Nonce": nonce,
411
+ "X-Mesh-Ts": ts,
412
+ }
413
+
414
+ def _drain_outbox(self) -> None:
415
+ """Re-attempt delivery for due outbox items (called from _sync_loop).
416
+
417
+ Processes up to _BATCH_LIMIT rows per call (bounded by outbox.due()).
418
+ Each row gets fresh signing headers. On success: deleted. On any
419
+ failure: mark_retry() with exponential back-off. Always prunes
420
+ expired rows at the end of the drain pass.
421
+
422
+ This method only runs when self._peer_url is set (guarded in
423
+ _sync_loop) — when no peer is configured the outbox is inert.
424
+ """
425
+ outbox = self._get_outbox()
426
+ if outbox is None:
427
+ return
428
+
429
+ now = time.time()
430
+ due_rows = outbox.due(now)
431
+ if not due_rows:
432
+ outbox.prune_expired(now)
433
+ return
434
+
435
+ for row in due_rows:
436
+ # Audit P1: bound wall-clock so a slow/black-holing peer can't
437
+ # monopolize the shared 30s sync thread. Remaining due rows stay
438
+ # queued and are retried next cycle.
439
+ if time.time() - now > _DRAIN_BUDGET_SECONDS:
440
+ logger.debug(
441
+ "RemoteOutbox: drain budget (%.0fs) exhausted; deferring "
442
+ "remaining rows to next cycle",
443
+ _DRAIN_BUDGET_SECONDS,
444
+ )
445
+ break
446
+ row_id: int = row["id"]
447
+ peer_url: str = row["peer_url"]
448
+ to_peer: str = row["to_peer"]
449
+
450
+ try:
451
+ payload: dict[str, Any] = json.loads(row["payload"])
452
+ except (json.JSONDecodeError, ValueError) as exc:
453
+ logger.debug(
454
+ "RemoteOutbox: corrupt payload in row %d — deleting: %s",
455
+ row_id, exc,
456
+ )
457
+ outbox.delete(row_id)
458
+ continue
459
+
460
+ # Pre-flight pin check (3b-3)
461
+ pin_ok, pin_err = self._check_cert_pin(peer_url)
462
+ if not pin_ok:
463
+ logger.debug(
464
+ "RemoteOutbox: pin check failed for row %d (%s): %s",
465
+ row_id, peer_url, pin_err,
466
+ )
467
+ outbox.mark_retry(row_id, now)
468
+ continue
469
+
470
+ # Rebuild fresh headers: auth bearer + fresh HMAC signature
471
+ base_headers = self._auth_headers()
472
+ full_headers = self._build_signed_headers(payload, to_peer, base_headers)
473
+
474
+ try:
475
+ with self._http_client(timeout=10) as client:
476
+ resp = client.post(
477
+ f"{peer_url}/mesh/send",
478
+ json=payload,
479
+ headers=full_headers,
480
+ timeout=10,
481
+ )
482
+ resp.raise_for_status()
483
+
484
+ outbox.delete(row_id)
485
+ logger.debug(
486
+ "RemoteOutbox: delivered row %d → %s (to_peer=%s)",
487
+ row_id, peer_url, to_peer,
488
+ )
489
+ except httpx.RequestError as exc:
490
+ logger.debug(
491
+ "RemoteOutbox: HTTP error for row %d: %s", row_id, exc
492
+ )
493
+ outbox.mark_retry(row_id, now)
494
+ except httpx.HTTPStatusError as exc:
495
+ logger.debug(
496
+ "RemoteOutbox: non-2xx for row %d: %s", row_id, exc
497
+ )
498
+ outbox.mark_retry(row_id, now)
499
+ except Exception as exc:
500
+ logger.debug(
501
+ "RemoteOutbox: unexpected error for row %d: %s", row_id, exc
502
+ )
503
+ outbox.mark_retry(row_id, now)
504
+
505
+ outbox.prune_expired(now)
506
+
507
+ # ------------------------------------------------------------------
508
+ # Core sync loop
509
+ # ------------------------------------------------------------------
510
+
511
+ def _sync_protocol_from_remote(self) -> None:
512
+ """3c: pull the peer's state + lock deltas and converge locally.
513
+
514
+ - State: deterministic LWW merge (StateSyncer.merge_remote).
515
+ - Locks: fencing-token resolution (LockCoordinator.resolve).
516
+
517
+ Fail-soft: any error is logged and never interrupts the sync loop.
518
+ Honors the SAME cert pin as peer-sync/send (fail-closed) so a MITM peer
519
+ cannot feed us forged deltas or harvest the bearer token. Scoped to the
520
+ "default" profile (single-profile mesh); per-profile sync is a
521
+ documented follow-up.
522
+ """
523
+ if not self._peer_url:
524
+ return
525
+ pin_ok, pin_err = self._check_cert_pin(self._peer_url)
526
+ if not pin_ok:
527
+ logger.debug("RemoteSyncClient: protocol-sync pin check failed: %s", pin_err)
528
+ return
529
+ try:
530
+ from .lock_protocol import LockCoordinator
531
+ from .state_sync import StateSyncer
532
+ except Exception as exc: # pragma: no cover — modules always present
533
+ logger.debug("RemoteSyncClient: protocol modules unavailable: %s", exc)
534
+ return
535
+
536
+ profile = "default"
537
+ headers = self._auth_headers()
538
+
539
+ # State delta → LWW merge.
540
+ try:
541
+ with self._http_client(timeout=5) as client:
542
+ resp = client.get(
543
+ f"{self._peer_url}/mesh/state/delta", headers=headers, timeout=5
544
+ )
545
+ resp.raise_for_status()
546
+ entries = resp.json().get("entries", [])
547
+ if entries:
548
+ StateSyncer(self._broker).merge_remote(profile, entries)
549
+ except httpx.RequestError as exc:
550
+ logger.debug("RemoteSyncClient: state-delta sync error: %s", exc)
551
+ except Exception as exc:
552
+ logger.debug("RemoteSyncClient: state-delta merge error: %s", exc)
553
+
554
+ # Lock delta → fencing-token resolution.
555
+ try:
556
+ with self._http_client(timeout=5) as client:
557
+ resp = client.get(
558
+ f"{self._peer_url}/mesh/lock/delta", headers=headers, timeout=5
559
+ )
560
+ resp.raise_for_status()
561
+ locks = resp.json().get("locks", [])
562
+ if locks:
563
+ LockCoordinator(self._broker).resolve(profile, locks)
564
+ except httpx.RequestError as exc:
565
+ logger.debug("RemoteSyncClient: lock-delta sync error: %s", exc)
566
+ except Exception as exc:
567
+ logger.debug("RemoteSyncClient: lock-delta resolve error: %s", exc)
568
+
160
569
  def _sync_loop(self) -> None:
161
- """Background thread: sync remote peers every 30s."""
570
+ """Background thread: sync remote peers every 30s, then drain outbox."""
162
571
  while not self._stop_event.is_set():
163
- try:
164
- if self._peer_url:
572
+ if self._peer_url:
573
+ try:
165
574
  self._sync_peers_from_remote()
166
- except Exception as exc:
167
- logger.debug("RemoteSyncClient: sync error: %s", exc)
575
+ except Exception as exc:
576
+ logger.debug("RemoteSyncClient: sync error: %s", exc)
577
+
578
+ # 3b-1: drain outbox after every peer sync
579
+ try:
580
+ self._drain_outbox()
581
+ except Exception as exc:
582
+ logger.debug("RemoteSyncClient: outbox drain error: %s", exc)
583
+
584
+ # 3c: converge remote state (LWW) + locks (fencing) each cycle
585
+ try:
586
+ self._sync_protocol_from_remote()
587
+ except Exception as exc:
588
+ logger.debug("RemoteSyncClient: protocol sync error: %s", exc)
168
589
 
169
590
  # Wait 30s before next sync
170
591
  if self._stop_event.wait(30):
@@ -183,8 +604,20 @@ class RemoteSyncClient:
183
604
  if not self._peer_url:
184
605
  return
185
606
 
607
+ # Audit P1: the peers-sync GET carries the bearer token — it MUST honor
608
+ # the same cert pin as send/drain, else a MITM (CA-valid wrong leaf)
609
+ # harvests the token here and poisons the remote-peer directory while
610
+ # /mesh/send is pin-blocked. Fail closed for this cycle on pin failure.
611
+ pin_ok, pin_err = self._check_cert_pin(self._peer_url)
612
+ if not pin_ok:
613
+ logger.debug(
614
+ "RemoteSyncClient: peer-sync pin check failed, skipping: %s",
615
+ pin_err,
616
+ )
617
+ return
618
+
186
619
  try:
187
- with httpx.Client(timeout=5) as client:
620
+ with self._http_client(timeout=5) as client:
188
621
  headers = self._auth_headers()
189
622
 
190
623
  resp = client.get(
@@ -221,47 +654,90 @@ class RemoteSyncClient:
221
654
  def send_to_remote(self, to_peer: str, message_data: dict) -> dict:
222
655
  """Proxy mesh_send to remote /mesh/send endpoint.
223
656
 
657
+ On success: returns the remote response dict (unchanged from pre-3b).
658
+ On any failure (RequestError, non-2xx, unexpected): enqueues the
659
+ message to the durable outbox for retry, then returns the same
660
+ {"ok": False, ...} dict as before. The return contract is unchanged.
661
+
224
662
  Args:
225
- to_peer: Target peer ID on remote machine
663
+ to_peer: Target peer ID on remote machine.
226
664
  message_data: Dict with from_peer, content, type, etc.
227
665
 
228
666
  Returns:
229
- Dict with {"ok": True, ...} or {"ok": False, "error": "..."}
667
+ Dict with {"ok": True, ...} or {"ok": False, "error": "..."}.
230
668
  """
231
669
  if not self._peer_url:
232
670
  return {"ok": False, "error": "no remote peer URL configured"}
233
671
 
672
+ from_peer = message_data.get("from_peer", "")
673
+ content = message_data.get("content", "")
674
+ payload = {
675
+ "from_peer": from_peer,
676
+ "to_peer": to_peer,
677
+ "content": content,
678
+ "type": message_data.get("type", "text"),
679
+ }
680
+ # Everything below (pin pre-flight, signing, POST) runs INSIDE the try
681
+ # so any error returns the stable {"ok": False, ...} contract instead
682
+ # of propagating into the broker / route handler (audit P1 — restores
683
+ # the pre-3b fail-soft behavior).
234
684
  try:
235
- with httpx.Client(timeout=10) as client:
236
- headers = self._auth_headers()
685
+ # 3b-3: pre-flight cert pin check before sending any data. A pin
686
+ # mismatch (or pin-set-without-https) is a PERMANENT config/attack
687
+ # condition — do NOT enqueue: retrying cannot fix it and would just
688
+ # fill the outbox with undeliverable rows (audit P2).
689
+ pin_ok, pin_err = self._check_cert_pin(self._peer_url)
690
+ if not pin_ok:
691
+ logger.warning(
692
+ "RemoteSyncClient: cert pin check failed for %s "
693
+ "(not enqueued): %s",
694
+ self._peer_url, pin_err,
695
+ )
696
+ return {"ok": False, "error": f"certificate pin failure: {pin_err}"}
237
697
 
238
- payload = {
239
- "from_peer": message_data.get("from_peer", ""),
240
- "to_peer": to_peer,
241
- "content": message_data.get("content", ""),
242
- "type": message_data.get("type", "text"),
243
- }
698
+ signed_headers = self._build_signed_headers(
699
+ payload, to_peer, self._auth_headers()
700
+ )
244
701
 
702
+ with self._http_client(timeout=10) as client:
245
703
  resp = client.post(
246
704
  f"{self._peer_url}/mesh/send",
247
705
  json=payload,
248
- headers=headers,
706
+ headers=signed_headers,
249
707
  timeout=10,
250
708
  )
251
709
  resp.raise_for_status()
252
710
  return resp.json()
711
+
253
712
  except httpx.RequestError as e:
254
713
  logger.debug(
255
714
  "RemoteSyncClient: HTTP error sending to remote peer %s: %s",
256
- to_peer,
257
- e,
715
+ to_peer, e,
716
+ )
717
+ # Audit P0: never persist auth headers (bearer secret) to disk —
718
+ # the drain re-signs from the in-memory secret. headers=None.
719
+ self._enqueue_on_failure(
720
+ self._peer_url, to_peer, payload, None, now=time.time()
258
721
  )
259
722
  return {"ok": False, "error": f"remote send failed: {e}"}
723
+
724
+ except httpx.HTTPStatusError as e:
725
+ logger.debug(
726
+ "RemoteSyncClient: non-2xx sending to remote peer %s: %s",
727
+ to_peer, e,
728
+ )
729
+ self._enqueue_on_failure(
730
+ self._peer_url, to_peer, payload, None, now=time.time()
731
+ )
732
+ return {"ok": False, "error": f"remote send non-2xx: {e}"}
733
+
260
734
  except Exception as e:
261
735
  logger.debug(
262
736
  "RemoteSyncClient: unexpected error sending to remote peer %s: %s",
263
- to_peer,
264
- e,
737
+ to_peer, e,
738
+ )
739
+ self._enqueue_on_failure(
740
+ self._peer_url, to_peer, payload, None, now=time.time()
265
741
  )
266
742
  return {"ok": False, "error": f"remote send error: {e}"}
267
743
 
@@ -320,6 +796,9 @@ class RemoteSyncClient:
320
796
  config is the source of truth and must not be hijacked by a spoofed
321
797
  mDNS announcement. A discovered peer is marked UNTRUSTED (the shared
322
798
  secret is withheld) unless SLM_MESH_TRUST_DISCOVERED is enabled (M05).
799
+
800
+ 3b-3: When SLM_MESH_TLS=on, discovered peers use https:// instead
801
+ of the default http://. Default OFF preserves today's behavior.
323
802
  """
324
803
  if self._peer_url_from_config:
325
804
  logger.debug(
@@ -328,7 +807,11 @@ class RemoteSyncClient:
328
807
  host, port,
329
808
  )
330
809
  return
331
- new_url = _peer_url(host, port)
810
+ tls_enabled = (
811
+ os.environ.get("SLM_MESH_TLS", "off").strip().lower()
812
+ in ("1", "on", "true", "yes")
813
+ )
814
+ new_url = _peer_url_with_scheme(host, port, tls=tls_enabled)
332
815
  if self._peer_url != new_url:
333
816
  self._peer_url = new_url
334
817
  self._peer_url_trusted = self._trust_discovered