fleetmesh-strfry 0.1.0__py3-none-any.whl

fleetmesh_strfry/__init__.py

@@ -0,0 +1,36 @@
+"""fleetmesh-strfry: the strfry relay bridge for a Fleetmesh node.
+
+Two ways to run it, both over the same classes:
+
+- in the node's process: `fleetmesh_strfry.plugin:StrfryBridgePlugin` in the
+  node's `plugins` config block;
+- beside the node: `python -m fleetmesh_strfry run --dir NODE_DIR`.
+
+This package imports `node.protocols.nostr` (`client`, `eventstore`),
+`node.storage.grants`, `node.keystore`, `node.interfaces`, `node.compartment`
+and `fleetmesh_core` at runtime: it runs on the node host with the meshnode
+and core checkouts on PYTHONPATH. It has no other dependency beyond the standard library.
+"""
+from __future__ import annotations
+
+from .bridge import StrfryRelayBridge
+from .pipe import StrfryPipe
+from .plugin import PLUGIN_REF, StrfryBridgePlugin, cmd_strfry, register_cli
+from .policy import (
+    DEFAULT_STRFRY_URL,
+    MESH_CONTROL_KINDS,
+    RateLimiter,
+    StrfryPluginRunner,
+    StrfryPolicyConfig,
+    StrfryPolicyEngine,
+    StrfryPolicyError,
+)
+
+__version__ = '0.1.0'
+
+__all__ = [
+    'DEFAULT_STRFRY_URL', 'MESH_CONTROL_KINDS', 'PLUGIN_REF', 'RateLimiter',
+    'StrfryBridgePlugin', 'StrfryPipe', 'StrfryPluginRunner', 'StrfryPolicyConfig',
+    'StrfryPolicyEngine', 'StrfryPolicyError', 'StrfryRelayBridge',
+    'cmd_strfry', 'register_cli', '__version__',
+]

fleetmesh_strfry/__main__.py

@@ -0,0 +1,122 @@
+"""`python -m fleetmesh_strfry`: the strfry bridge as its own service.
+
+    python -m fleetmesh_strfry run    --dir NODE_DIR [--url ws://127.0.0.1:7777] [--every 30]
+    python -m fleetmesh_strfry plugin --dir NODE_DIR          # strfry writePolicy stdio runner
+    python -m fleetmesh_strfry status|export|import|eval --dir NODE_DIR [flags]
+
+`run` opens the node directory's stores the way `meshnode strfry` did, runs
+a StrfryRelayBridge against the strfry URL, and drains the egress queue
+every `--every` seconds until SIGTERM/SIGINT. The other verbs are the
+former `meshnode strfry DIR ACTION` with `--dir` instead of a positional.
+
+Exit codes: 0 on a clean stop or an accepted `eval`; 1 when `eval` rejects;
+2 when the arguments are unusable (no such directory, no keys).
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import signal
+import sys
+import threading
+from typing import Optional, Sequence
+
+from .bridge import StrfryRelayBridge
+from .plugin import ACTIONS, STORE_FILE, add_strfry_arguments, cmd_strfry, open_stores
+from .policy import DEFAULT_STRFRY_URL, StrfryPolicyConfig, StrfryPolicyEngine
+
+EXIT_OK = 0
+EXIT_USAGE = 2
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(prog='python -m fleetmesh_strfry',
+                                 description=(__doc__ or '').split('\n\n')[0])
+    sub = ap.add_subparsers(dest='action', required=True)
+
+    run = sub.add_parser('run', help='bridge the node store and a strfry relay until stopped')
+    run.add_argument('--dir', required=True, help='the node directory (keys.json, store.sqlite, grants.sqlite)')
+    run.add_argument('--url', default=DEFAULT_STRFRY_URL, help='strfry WebSocket URL')
+    run.add_argument('--every', type=float, default=30.0, help='seconds between egress flushes')
+    run.add_argument('--store-file', default=STORE_FILE, help=argparse.SUPPRESS)
+    run.add_argument('--enforce-grants', action='store_true', help='enforce bilateral peering grants')
+    run.add_argument('--shadow', action='store_true', help='use shadowReject instead of reject')
+    run.add_argument('--iterations', type=int, default=0, help=argparse.SUPPRESS)   # tests: stop after N flushes
+
+    for action in ACTIONS:
+        p = sub.add_parser(action, help='the former `meshnode strfry DIR %s`' % action)
+        p.add_argument('--dir', required=True, help='the node directory')
+        p.add_argument('--store-file', default=STORE_FILE, help=argparse.SUPPRESS)
+        p.add_argument('--json', action='store_true', help='print the structure, for scripts')
+        add_strfry_arguments(p)
+    return ap
+
+
+def _usable(directory: str) -> Optional[str]:
+    if not os.path.isdir(directory):
+        return 'no such directory: %s' % directory
+    if not os.path.exists(os.path.join(directory, 'keys.json')):
+        return 'not a node directory (no keys.json): %s' % directory
+    return None
+
+
+def serve(directory: str, url: str, every: float, store_file: str = STORE_FILE,
+          enforce_grants: bool = False, shadow: bool = False, iterations: int = 0,
+          stop: Optional[threading.Event] = None) -> int:
+    """Run the bridge until `stop` is set (SIGTERM/SIGINT set it) or `iterations` flushes are done."""
+    st, gr = open_stores(directory, store_file)
+    engine = StrfryPolicyEngine(config=StrfryPolicyConfig(enforce_grants=enforce_grants, shadow_reject=shadow),
+                                store_backend=st, grant_store=gr)
+    bridge = StrfryRelayBridge(store_backend=st, strfry_url=url, policy_engine=engine)
+    stop = stop or threading.Event()
+
+    def _on_signal(signum: int, _frame: object) -> None:
+        stop.set()
+
+    main_thread = threading.current_thread() is threading.main_thread()
+    if main_thread:
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            signal.signal(sig, _on_signal)
+
+    print('fleetmesh-strfry: bridging %s <-> %s every %gs' % (directory, url, every), file=sys.stderr, flush=True)
+    bridge.start()
+    done = 0
+    try:
+        while not stop.is_set():
+            if stop.wait(every):
+                break
+            sent = bridge.flush_egress()
+            done += 1
+            if sent:
+                print('fleetmesh-strfry: flushed %d egress events' % sent, file=sys.stderr, flush=True)
+            if iterations and done >= iterations:
+                break
+    finally:
+        bridge.stop()
+        for s in (st, gr):
+            close = getattr(s, 'close', None)
+            if callable(close):
+                try:
+                    close()
+                except Exception:
+                    pass
+    print('fleetmesh-strfry: stopped; %s' % bridge.stats(), file=sys.stderr, flush=True)
+    return EXIT_OK
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    ap = build_parser()
+    a = ap.parse_args(argv)
+    why = _usable(a.dir)
+    if why is not None:
+        print('fleetmesh-strfry: %s' % why, file=sys.stderr)
+        return EXIT_USAGE
+    if a.action == 'run':
+        return serve(a.dir, a.url, a.every, store_file=a.store_file, enforce_grants=a.enforce_grants,
+                     shadow=a.shadow, iterations=a.iterations)
+    code = cmd_strfry(a)
+    return int(code) if isinstance(code, int) else EXIT_OK
+
+
+if __name__ == '__main__':
+    sys.exit(main())

fleetmesh_strfry/bridge.py

@@ -0,0 +1,236 @@
+"""Live Streaming Bridge (StrfryRelayBridge).
+
+Real-time bidirectional streaming between a local/remote strfry relay (via WebSocket)
+and the Fleetmesh node store and P2P gossip planes.
+
+Moved verbatim from meshnode's node/strfry_bridge.py; only the import paths changed.
+"""
+from __future__ import annotations
+
+import collections
+import threading
+import time
+from typing import Any, Callable, Deque, Optional, Sequence
+
+from node.protocols.nostr import client as relay
+from node.interfaces import NostrEvent
+
+from .policy import DEFAULT_STRFRY_URL, StrfryPolicyEngine
+
+
+class StrfryRelayBridge:
+    """Live bidirectional WebSocket streaming bridge connecting strfry and Fleetmesh."""
+
+    def __init__(
+        self,
+        store_backend: Any,
+        strfry_url: str = DEFAULT_STRFRY_URL,
+        policy_engine: Optional[StrfryPolicyEngine] = None,
+        kinds: Optional[Sequence[int]] = None,
+        clock: Optional[Callable[[], int]] = None,
+    ) -> None:
+        self.store: Any = store_backend
+        self.strfry_url: str = strfry_url
+        self.policy: StrfryPolicyEngine = policy_engine or StrfryPolicyEngine(store_backend=store_backend)
+        self.kinds: list[int] = list(kinds or [0, 1, 3, 6, 7, 1059, 11801, 11802, 11803, 30801, 30802, 30803])
+        self.clock: Callable[[], int] = clock or (lambda: int(time.time()))
+
+        self.connected: bool = False
+        self.last_connected_at: Optional[int] = None
+        self.ingress_count: int = 0
+        self.egress_count: int = 0
+        self.errors_count: int = 0
+        self.last_error: Optional[str] = None
+
+        self._relay: Optional[relay.Relay] = None
+        self._thread: Optional[threading.Thread] = None
+        self._stop_event: threading.Event = threading.Event()
+        self._egress_queue: Deque[NostrEvent] = collections.deque(maxlen=1000)
+        self._lock: threading.Lock = threading.Lock()
+
+    def enqueue(self, event: NostrEvent) -> None:
+        """Queue `event` for strfry; it goes out on the next flush_egress.
+
+        The queue holds the newest 1000: when full, the oldest is dropped."""
+        if isinstance(event, dict):
+            with self._lock:
+                self._egress_queue.append(event)
+
+    def start(self) -> None:
+        """Launch background streaming bridge worker."""
+        with self._lock:
+            if self._thread is not None and self._thread.is_alive():
+                return
+            self._stop_event.clear()
+            self._thread = threading.Thread(target=self._run_loop, name=f'strfry-bridge-{self.strfry_url}', daemon=True)
+            self._thread.start()
+
+    def stop(self) -> None:
+        """Stop background worker and close WebSocket."""
+        self._stop_event.set()
+        with self._lock:
+            if self._relay is not None:
+                try:
+                    self._relay.close()
+                except Exception:
+                    pass
+                self._relay = None
+            self.connected = False
+        if self._thread is not None and self._thread.is_alive():
+            self._thread.join(timeout=1.0)
+            self._thread = None
+
+    def publish_to_strfry(self, event: NostrEvent) -> tuple[bool, str]:
+        """Publish an outbound event to the strfry relay after egress policy check."""
+        # 1. Egress Policy Evaluation
+        verdict = self.policy.evaluate_egress(event)
+        act = verdict.get('action', 'allow')
+        if act == 'drop':
+            return False, f"egress dropped: {verdict.get('msg', 'policy')}"
+        elif act == 'shadowDrop':
+            return True, "shadow dropped"
+        elif act == 'redact':
+            # A redacted event is the same event with fewer tags; its id and sig
+            # are deliberately left as they were - the relay is expected to
+            # refuse it, and the audit trail shows what was attempted.
+            event = {**event, 'tags': verdict.get('redacted_tags', [])}
+
+        # 2. WebSocket publication via dedicated relay client to prevent socket contention
+        pub_client = relay.Relay(self.strfry_url, timeout=5.0)
+        try:
+            pub_client.connect()
+            ok_pub, reason = pub_client.publish(event)
+            if ok_pub:
+                self.egress_count += 1
+            return ok_pub, reason
+        except Exception as e:
+            self.errors_count += 1
+            self.last_error = str(e)
+            self.enqueue(event)
+            return False, str(e)
+        finally:
+            pub_client.close()
+
+    def _run_loop(self) -> None:
+        consecutive_errors = 0
+        while not self._stop_event.is_set():
+            try:
+                r = relay.Relay(self.strfry_url, timeout=10.0)
+                r.connect()
+                with self._lock:
+                    self._relay = r
+                    self.connected = True
+                    self.last_connected_at = int(time.time())
+                    self.last_error = None
+                consecutive_errors = 0
+
+                # Subscribe to strfry
+                sid = r.subscribe([{'kinds': self.kinds}], ready=False)
+
+                # Anything queued while strfry was away goes out now, not
+                # the next time strfry happens to send us something.
+                self.flush_egress()
+
+                # Process incoming stream
+                for ev in r.events(sid):
+                    if self._stop_event.is_set():
+                        break
+
+                    # Pass through writePolicy engine
+                    verdict = self.policy.evaluate_ingress({'type': 'new', 'event': ev})
+                    if verdict.get('action') == 'accept':
+                        self.ingress_count += 1
+                        if self.policy.store is not self.store and hasattr(self.store, 'insert_event'):
+                            self.store.insert_event(ev, self.clock())
+                        if hasattr(self.store, 'note_origin'):
+                            try:
+                                self.store.note_origin(ev.get('id', ''), getattr(self.store, 'LOCAL', 'local'))
+                            except Exception:
+                                pass
+
+                    # Flush pending egress
+                    self.flush_egress()
+
+            except Exception as e:
+                consecutive_errors += 1
+                self.errors_count += 1
+                self.last_error = f'{type(e).__name__}: {e}'
+                with self._lock:
+                    self.connected = False
+                    if self._relay is not None:
+                        try:
+                            self._relay.close()
+                        except Exception:
+                            pass
+                        self._relay = None
+
+            if not self._stop_event.is_set():
+                backoff = min(10.0, 0.5 * (2 ** min(consecutive_errors, 4)))
+                self._stop_event.wait(backoff)
+
+    def flush_egress(self) -> int:
+        """Send queued egress to strfry; returns how many went out.
+
+        Driven from three places: after every (re)connect of the read loop,
+        after every inbound event, and from the plugin's tick() - so a quiet
+        strfry that sends nothing still gets what was queued for it. What
+        cannot be sent because strfry is unreachable is put back at the
+        front of the queue for the next attempt, in order.
+        """
+        raw_items: list[NostrEvent] = []
+        with self._lock:
+            while self._egress_queue and len(raw_items) < 20:
+                raw_items.append(self._egress_queue.popleft())
+        if not raw_items:
+            return 0
+        items: list[NostrEvent] = []
+        for ev in raw_items:
+            verdict = self.policy.evaluate_egress(ev)
+            act = verdict.get('action', 'allow')
+            # A redacted event is a different event: its id and signature
+            # cover the tags, so strfry would refuse it. Withhold it instead.
+            if act in ('drop', 'shadowDrop', 'redact'):
+                continue
+            items.append(ev)
+        if not items:
+            return 0
+        sent = 0
+        pub_client = relay.Relay(self.strfry_url, timeout=5.0)
+        try:
+            pub_client.connect()
+            while items:
+                ev = items[0]
+                try:
+                    ok_p, _ = pub_client.publish(ev)
+                except Exception:
+                    break               # connection gone; keep this and the rest
+                items.pop(0)
+                if ok_p:
+                    self.egress_count += 1
+                    sent += 1
+                # a relay that says "false" has judged the event; retrying it is pointless
+        except Exception as e:
+            self.last_error = f'{type(e).__name__}: {e}'
+        finally:
+            pub_client.close()
+            if items:
+                with self._lock:
+                    self._egress_queue.extendleft(reversed(items))
+        return sent
+
+    def pending_egress(self) -> int:
+        with self._lock:
+            return len(self._egress_queue)
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            'strfry_url': self.strfry_url,
+            'connected': self.connected,
+            'last_connected_at': self.last_connected_at,
+            'ingress_count': self.ingress_count,
+            'egress_count': self.egress_count,
+            'errors_count': self.errors_count,
+            'last_error': self.last_error,
+            'pending_egress': self.pending_egress(),
+            'policy': self.policy.stats(),
+        }

fleetmesh_strfry/pipe.py

@@ -0,0 +1,78 @@
+"""High-throughput bulk stream pipe (StrfryPipe): JSONL export/import compatible with strfry.
+
+- Streaming JSONL export (`NodeStore` -> strfry export format).
+- Streaming JSONL import (`strfry export` -> `NodeStore`) with deduplication and origin tracking.
+
+Moved verbatim from meshnode's node/strfry_bridge.py; only the import paths changed.
+"""
+from __future__ import annotations
+
+import json
+import time
+from typing import Any, Callable, Optional, TextIO
+from fleetmesh_core.storage import store  # noqa: E402
+
+
+class StrfryPipe:
+    """High-throughput bulk streaming export and import compatible with strfry."""
+
+    @staticmethod
+    def export_jsonl(store_backend: Any, out_stream: TextIO) -> int:
+        """Stream events from NodeStore out to JSONL (strfry export format)."""
+        count = 0
+        if hasattr(store_backend, 'db') and hasattr(store_backend.db, 'execute'):
+            cur = store_backend.db.execute('SELECT json FROM events ORDER BY created_at ASC, id ASC')
+            for row in cur:
+                out_stream.write(row[0] + '\n')
+                count += 1
+        elif hasattr(store_backend, 'all'):
+            for ev in store_backend.all():
+                out_stream.write(json.dumps(ev, separators=(',', ':')) + '\n')
+                count += 1
+        out_stream.flush()
+        return count
+
+    @staticmethod
+    def import_jsonl(
+        store_backend: Any,
+        in_stream: TextIO,
+        verify: bool = True,
+        clock_fn: Optional[Callable[[], int]] = None,
+    ) -> tuple[int, int, int]:
+        """Stream events from JSONL (strfry export format) into NodeStore.
+
+        Returns (imported, duplicates, errors).
+        """
+        now = (clock_fn or (lambda: int(time.time())))()
+        imported = 0
+        duplicates = 0
+        errors = 0
+
+        for line in in_stream:
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                ev = json.loads(line)
+                if not isinstance(ev, dict) or 'id' not in ev:
+                    errors += 1
+                    continue
+
+                if verify and not store.verify_event(ev):
+                    errors += 1
+                    continue
+
+                res = store_backend.insert_event(ev, now)
+                if res == 'stored':
+                    imported += 1
+                    if hasattr(store_backend, 'note_origin'):
+                        store_backend.note_origin(ev['id'], getattr(store_backend, 'LOCAL', 'local'))
+                elif res == 'duplicate':
+                    duplicates += 1
+                else:
+                    errors += 1
+            except Exception:
+                errors += 1
+
+        return imported, duplicates, errors