yrb-lite-reliable 0.1.0

package/README.md

@@ -0,0 +1,90 @@
+# yrb-lite-reliable
+
+The **client core** for the [`yrb-lite`](https://github.com/jpcamara/yrb-lite)
+y-websocket protocol — everything a Yjs provider needs *except the transport*.
+Bring your own socket (ActionCable, AnyCable, raw WebSocket); this owns the
+protocol.
+
+Two layers, use whichever you need:
+
+- **`SyncEngine`** — batteries-included. Binds to a `Y.Doc` (+ optional
+  `Awareness`) and owns the y-protocols **message encode/decode**, the
+  **sync-step handshake** (SyncStep1 / SyncStep2 / Update), **awareness**, and
+  reliable delivery. Speaks raw `Uint8Array` frames; you only wire the socket.
+- **`ReliableSync`** — the zero-dependency reliable-delivery state machine on its
+  own: ack-tracked queue, **sync-since-last-ack** (the unacked tail merged into
+  one causally-complete delta), cumulative acks, retransmit + "server doesn't
+  support acks" fallback, and reconnect replay. Compose it yourself if you
+  already have your own framing.
+
+## Install
+
+```bash
+npm install yrb-lite-reliable
+```
+
+`SyncEngine` needs `yjs` and `y-protocols` (peers — your provider already has
+them). `ReliableSync` has **no dependencies**; import it on its own via
+`yrb-lite-reliable/reliable` if that's all you want.
+
+## SyncEngine
+
+```js
+import { SyncEngine, toBase64, fromBase64 } from "yrb-lite-reliable";
+import * as Y from "yjs";
+import { Awareness } from "y-protocols/awareness";
+
+const doc = new Y.Doc();
+const awareness = new Awareness(doc);
+
+const engine = new SyncEngine(doc, {
+  awareness,
+  // transmit one raw frame; `id` is set for reliable doc updates -> tag your envelope
+  send: (frame, id) => {
+    const payload = { update: toBase64(frame) };
+    if (id !== undefined) payload.id = id;
+    subscription.send(payload);
+  },
+});
+
+// wire your transport's callbacks:
+subscription.connected    = () => engine.onConnect();      // handshake + replay
+subscription.disconnected = () => engine.onDisconnect();   // pause + clear presence
+subscription.received = (msg) => {
+  if (msg.ack !== undefined) return engine.ack(msg.ack);   // reliable ack envelope
+  const reply = engine.receive(fromBase64(msg.update || msg.m)); // decode + apply
+  if (reply) subscription.send({ update: toBase64(reply) });     // e.g. answer a SyncStep1
+};
+// engine.synced -> caught up; engine.hasPending -> unacked edits in flight
+// engine.destroy() -> detach listeners + stop retransmits
+```
+
+Local document edits and awareness changes are picked up automatically from the
+doc's / awareness's `update` events — you never call anything for outbound edits.
+
+## ReliableSync (standalone)
+
+```js
+import { ReliableSync } from "yrb-lite-reliable/reliable"; // zero-dep
+import * as Y from "yjs";
+
+const rs = new ReliableSync({
+  send: (update, id) => { /* frame + transmit */ },
+  merge: Y.mergeUpdates,
+});
+
+rs.enqueue(update);  // a local document update
+rs.onAck(id);        // an { ack: id } arrived
+rs.onConnect();      // (re)connected — replay the tail, resume retransmits
+rs.onDisconnect();   // dropped — keep the queue, pause
+```
+
+## How it fits
+
+The server counterpart — ack *generation*, gap detection, record-before-distribute
+— is the `yrb-lite-actioncable` gem's `YrbLite::ActionCable::Sync`. This package
+is the client half of the same protocol.
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "yrb-lite-reliable",
+  "version": "0.1.0",
+  "description": "Client core for the yrb-lite y-websocket protocol: sync steps, message encode/decode, awareness, and reliable delivery (ack-tracked queue, sync-since-last-ack, retransmit + reconnect replay). Bring your own transport.",
+  "type": "module",
+  "main": "src/index.js",
+  "module": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./reliable": "./src/reliable_sync.js",
+    "./base64": "./src/base64.js"
+  },
+  "files": [
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "node --test"
+  },
+  "license": "MIT",
+  "author": "JP Camara <jp@jpcamara.com>",
+  "homepage": "https://github.com/jpcamara/yrb-lite/tree/main/packages/yrb-lite-reliable#readme",
+  "bugs": {
+    "url": "https://github.com/jpcamara/yrb-lite/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jpcamara/yrb-lite.git",
+    "directory": "packages/yrb-lite-reliable"
+  },
+  "keywords": [
+    "yjs",
+    "crdt",
+    "yrb-lite",
+    "reliable-delivery",
+    "actioncable",
+    "y-websocket"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "yjs": "^13.6.0",
+    "y-protocols": "^1.0.5"
+  },
+  "peerDependenciesMeta": {
+    "yjs": {
+      "optional": true
+    },
+    "y-protocols": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "yjs": "^13.6.0",
+    "y-protocols": "^1.0.5"
+  }
+}

package/src/base64.js

@@ -0,0 +1,9 @@
+// Convenience codecs for transports that carry binary frames as base64 strings
+// (e.g. ActionCable's JSON envelope). Optional -- a binary WebSocket transport
+// sends the raw frames directly and never needs these.
+
+/** @param {Uint8Array} bytes */
+export const toBase64 = (bytes) => btoa(Array.from(bytes, (b) => String.fromCharCode(b)).join(""));
+
+/** @param {string} str */
+export const fromBase64 = (str) => Uint8Array.from(atob(str), (c) => c.charCodeAt(0));

package/src/index.js

@@ -0,0 +1,9 @@
+// Zero-dependency reliable-delivery core. Safe to import on its own.
+export { ReliableSync } from "./reliable_sync.js";
+
+// Batteries-included protocol client (sync steps + encode/decode + awareness).
+// Requires `yjs` and `y-protocols` as peers.
+export { SyncEngine, MessageType } from "./sync_engine.js";
+
+// Optional base64 helpers for transports that carry frames as strings.
+export { toBase64, fromBase64 } from "./base64.js";

package/src/reliable_sync.js

@@ -0,0 +1,155 @@
+// Transport-agnostic reliable-delivery core for the yrb-lite y-websocket
+// protocol. This owns the "nuances" a provider would otherwise re-implement:
+// an ack-tracked queue of unacknowledged local updates, "sync since last ack"
+// (the unacked tail is sent as one MERGED, causally-complete delta so the server
+// never sees an internal gap), cumulative acks, periodic retransmit with a
+// "server doesn't support acks" fallback, and reconnect replay.
+//
+// It does NOT touch any transport, Yjs binding, or wire encoding. You inject:
+//   - send(update, id):  transmit one update. `update` is the raw merged update
+//                        bytes; `id` is the cumulative sequence (or undefined,
+//                        post-fallback). Frame + base64 + put it on your socket.
+//   - merge(updates):    merge an array of update byte-arrays into one
+//                        (typically Y.mergeUpdates from yjs).
+// and you drive it from your provider's lifecycle:
+//   - enqueue(update)         on every local document update (not server echoes)
+//   - onAck(id)               when an { ack: id } frame arrives
+//   - onConnect()/onDisconnect()  on transport (re)connect / drop
+//
+// Awareness/presence is intentionally out of scope -- it stays fire-and-forget
+// in the provider.
+
+const DEFAULTS = { resendInterval: 1000, maxUnconfirmedResends: 8 };
+
+export class ReliableSync {
+  /**
+   * @param {object} opts
+   * @param {(update: Uint8Array, id: number|undefined) => void} opts.send
+   * @param {(updates: Uint8Array[]) => Uint8Array} opts.merge
+   * @param {number} [opts.resendInterval=1000] ms between retransmits
+   * @param {number} [opts.maxUnconfirmedResends=8] resends with no ack before
+   *   deciding the server doesn't support reliable delivery and falling back
+   * @param {() => void} [opts.onFallback] called once if that fallback trips
+   * @param {(fn: () => void, ms: number) => any} [opts.setInterval]
+   * @param {(handle: any) => void} [opts.clearInterval]
+   */
+  constructor({
+    send,
+    merge,
+    resendInterval = DEFAULTS.resendInterval,
+    maxUnconfirmedResends = DEFAULTS.maxUnconfirmedResends,
+    onFallback,
+    setInterval: setIntervalFn,
+    clearInterval: clearIntervalFn,
+  } = {}) {
+    if (typeof send !== "function") throw new TypeError("ReliableSync requires a send(update, id) function");
+    if (typeof merge !== "function") throw new TypeError("ReliableSync requires a merge(updates) function");
+
+    this._send = send;
+    this._merge = merge;
+    this.resendInterval = resendInterval;
+    this.maxUnconfirmedResends = maxUnconfirmedResends;
+    this._onFallback = onFallback;
+    // Injectable timer hooks make the resend loop testable; default to globals.
+    this._setInterval = setIntervalFn || ((fn, ms) => setInterval(fn, ms));
+    this._clearInterval = clearIntervalFn || ((h) => clearInterval(h));
+
+    this.reliable = true; // flips false after the no-ack fallback
+    this.pending = []; // unacked local updates: [{ seq, update }], in order
+    this.nextSeq = 1;
+    this.everAcked = false;
+    this._resendsSinceProgress = 0;
+    this._connected = false;
+    this._timer = undefined;
+  }
+
+  /** True while there are unacknowledged local updates. */
+  get hasPending() {
+    return this.pending.length > 0;
+  }
+
+  /**
+   * Record a local document update. While reliable, it's queued and the unacked
+   * tail is flushed; once we've fallen back, it's sent fire-and-forget.
+   * @param {Uint8Array} update
+   */
+  enqueue(update) {
+    if (!this.reliable) {
+      this._send(update, undefined);
+      return;
+    }
+    this.pending.push({ seq: this.nextSeq++, update });
+    this.flush();
+  }
+
+  /**
+   * Send the whole unacked tail as one merged delta. The id is the highest seq
+   * in the batch, so a single { ack } cumulatively confirms everything up to it.
+   * No-op while disconnected (the tail is replayed on the next onConnect).
+   */
+  flush() {
+    if (!this._connected || this.pending.length === 0) return;
+    const updates = this.pending.map((p) => p.update);
+    const merged = updates.length === 1 ? updates[0] : this._merge(updates);
+    const id = this.pending[this.pending.length - 1].seq;
+    this._send(merged, id);
+  }
+
+  /**
+   * Confirm delivery up to `id`: prune every queued update with seq <= id.
+   * @param {number} id
+   */
+  onAck(id) {
+    this.everAcked = true;
+    this._resendsSinceProgress = 0;
+    this.pending = this.pending.filter((p) => p.seq > id);
+  }
+
+  /** Transport (re)connected: replay the unacked tail and resume retransmits. */
+  onConnect() {
+    this._connected = true;
+    this.flush();
+    this._startTimer();
+  }
+
+  /** Transport dropped: keep the queue (for reconnect replay), pause the timer. */
+  onDisconnect() {
+    this._connected = false;
+    this._stopTimer();
+  }
+
+  /**
+   * One retransmit tick. Exposed for deterministic testing; normally driven by
+   * the internal timer. If we keep resending on a live connection and never get
+   * an ack, the server doesn't support reliable delivery, so fall back to
+   * fire-and-forget (and stop tracking, since idempotent CRDT sync covers it).
+   */
+  onTick() {
+    if (!this._connected || this.pending.length === 0) return;
+    if (!this.everAcked && ++this._resendsSinceProgress > this.maxUnconfirmedResends) {
+      this.reliable = false;
+      this.pending = [];
+      this._stopTimer();
+      this._onFallback?.();
+      return;
+    }
+    this.flush();
+  }
+
+  /** Stop timers and drop references. Call when the provider is destroyed. */
+  destroy() {
+    this._stopTimer();
+    this.pending = [];
+  }
+
+  _startTimer() {
+    if (this._timer || !this.reliable) return;
+    this._timer = this._setInterval(() => this.onTick(), this.resendInterval);
+    if (this._timer && typeof this._timer.unref === "function") this._timer.unref();
+  }
+
+  _stopTimer() {
+    if (this._timer !== undefined) this._clearInterval(this._timer);
+    this._timer = undefined;
+  }
+}

package/src/sync_engine.js

@@ -0,0 +1,187 @@
+// A batteries-included client core for the yrb-lite y-websocket protocol.
+//
+// SyncEngine composes ReliableSync and additionally owns the parts a provider
+// would otherwise re-implement: the y-protocols message framing (encode/decode),
+// the sync-step handshake (SyncStep1 / SyncStep2 / Update), and awareness
+// encode/apply. It binds to a Y.Doc (and optional Awareness) and speaks in raw
+// Uint8Array frames -- you bring only the transport: base64 + the
+// `{ update, id }` / `{ ack }` envelope and the socket.
+//
+// Drive it from your transport:
+//   onConnect()          -> sends the opening handshake, replays the unacked tail
+//   onDisconnect()       -> pauses retransmits, clears remote presence
+//   ack(id)              -> a `{ ack: id }` envelope arrived
+//   const reply = receive(frame)  -> a binary protocol frame arrived; send `reply` if non-null
+// Local document edits and awareness changes are picked up automatically via the
+// doc's / awareness's "update" events.
+import * as Y from "yjs";
+import * as encoding from "lib0/encoding";
+import * as decoding from "lib0/decoding";
+import { readSyncMessage, writeSyncStep1, writeUpdate, messageYjsSyncStep2 } from "y-protocols/sync";
+import { encodeAwarenessUpdate, applyAwarenessUpdate, removeAwarenessStates } from "y-protocols/awareness";
+import { readAuthMessage } from "y-protocols/auth";
+import { ReliableSync } from "./reliable_sync.js";
+
+export const MessageType = { Sync: 0, Awareness: 1, Auth: 2, QueryAwareness: 3 };
+
+export class SyncEngine {
+  /**
+   * @param {Y.Doc} doc
+   * @param {object} opts
+   * @param {(frame: Uint8Array, id: number|undefined) => void} opts.send
+   *   transmit one raw protocol frame; `id` is set only for reliable document
+   *   updates (tag it onto your envelope so the server can ack).
+   * @param {import("y-protocols/awareness").Awareness} [opts.awareness]
+   * @param {boolean} [opts.reliable=true] use ack-tracked reliable delivery
+   * @param {number} [opts.resendInterval] forwarded to ReliableSync
+   * @param {number} [opts.maxUnconfirmedResends] forwarded to ReliableSync
+   * @param {() => void} [opts.onFallback] forwarded to ReliableSync
+   */
+  constructor(
+    doc,
+    {
+      send,
+      awareness = null,
+      reliable = true,
+      resendInterval,
+      maxUnconfirmedResends,
+      onFallback,
+      setInterval: setIntervalFn,
+      clearInterval: clearIntervalFn,
+    } = {}
+  ) {
+    if (!doc) throw new TypeError("SyncEngine requires a Y.Doc");
+    if (typeof send !== "function") throw new TypeError("SyncEngine requires a send(frame, id) function");
+
+    this.doc = doc;
+    this.awareness = awareness;
+    this.reliable = reliable;
+    this._send = send;
+    this._synced = false;
+
+    this._delivery = new ReliableSync({
+      merge: Y.mergeUpdates,
+      send: (update, id) => this._send(this._frameUpdate(update), id),
+      resendInterval,
+      maxUnconfirmedResends,
+      onFallback,
+      setInterval: setIntervalFn,
+      clearInterval: clearIntervalFn,
+    });
+
+    this._onDocUpdate = (update, origin) => {
+      if (origin === this) return; // applied from the server; don't echo it back
+      if (this.reliable && this._delivery.reliable) this._delivery.enqueue(update);
+      else this._send(this._frameUpdate(update), undefined);
+    };
+    this.doc.on("update", this._onDocUpdate);
+
+    if (this.awareness) {
+      this._onAwarenessUpdate = ({ added, updated, removed }) => {
+        const changed = added.concat(updated, removed);
+        this._send(this._frameAwareness(changed), undefined); // presence: fire-and-forget
+      };
+      this.awareness.on("update", this._onAwarenessUpdate);
+    }
+  }
+
+  /** True once we've received the server's SyncStep2 (the document is caught up). */
+  get synced() {
+    return this._synced;
+  }
+
+  /** True while there are unacknowledged local document updates in flight. */
+  get hasPending() {
+    return this._delivery.hasPending;
+  }
+
+  /** Transport connected: send the opening handshake and replay the unacked tail. */
+  onConnect() {
+    this._send(this._frameSyncStep1(), undefined);
+    if (this.awareness && this.awareness.getLocalState() !== null) {
+      this._send(this._frameAwareness([this.doc.clientID]), undefined);
+    }
+    if (this.reliable) this._delivery.onConnect();
+  }
+
+  /** Transport dropped: pause retransmits (queue kept) and clear remote presence. */
+  onDisconnect() {
+    this._synced = false;
+    this._delivery.onDisconnect();
+    if (this.awareness) {
+      const remote = [...this.awareness.getStates().keys()].filter((c) => c !== this.doc.clientID);
+      if (remote.length) removeAwarenessStates(this.awareness, remote, this);
+    }
+  }
+
+  /** A reliable-delivery `{ ack: id }` envelope arrived. */
+  ack(id) {
+    this._delivery.onAck(id);
+  }
+
+  /**
+   * Decode and apply one incoming binary protocol frame (document sync, awareness,
+   * query, or auth). Returns a reply frame to transmit (e.g. SyncStep2 answering a
+   * SyncStep1, or an awareness reply to a query), or null if there's nothing to send.
+   * @param {Uint8Array} frame
+   * @returns {Uint8Array|null}
+   */
+  receive(frame) {
+    const decoder = decoding.createDecoder(frame);
+    const encoder = encoding.createEncoder();
+    const type = decoding.readVarUint(decoder);
+    switch (type) {
+      case MessageType.Sync: {
+        encoding.writeVarUint(encoder, MessageType.Sync);
+        const syncType = readSyncMessage(decoder, encoder, this.doc, this);
+        if (!this._synced && syncType === messageYjsSyncStep2) this._synced = true;
+        break;
+      }
+      case MessageType.Awareness:
+        if (this.awareness) applyAwarenessUpdate(this.awareness, decoding.readVarUint8Array(decoder), this);
+        return null;
+      case MessageType.QueryAwareness:
+        if (!this.awareness) return null;
+        encoding.writeVarUint(encoder, MessageType.Awareness);
+        encoding.writeVarUint8Array(
+          encoder,
+          encodeAwarenessUpdate(this.awareness, [...this.awareness.getStates().keys()])
+        );
+        break;
+      case MessageType.Auth:
+        readAuthMessage(decoder, this.doc, (_doc, reason) => console.warn(`[yrb-lite] auth denied: ${reason}`));
+        return null;
+      default:
+        return null;
+    }
+    return encoding.length(encoder) > 1 ? encoding.toUint8Array(encoder) : null;
+  }
+
+  /** Detach doc/awareness listeners and stop retransmits. */
+  destroy() {
+    this.doc.off("update", this._onDocUpdate);
+    if (this.awareness && this._onAwarenessUpdate) this.awareness.off("update", this._onAwarenessUpdate);
+    this._delivery.destroy();
+  }
+
+  _frameSyncStep1() {
+    const e = encoding.createEncoder();
+    encoding.writeVarUint(e, MessageType.Sync);
+    writeSyncStep1(e, this.doc);
+    return encoding.toUint8Array(e);
+  }
+
+  _frameUpdate(update) {
+    const e = encoding.createEncoder();
+    encoding.writeVarUint(e, MessageType.Sync);
+    writeUpdate(e, update);
+    return encoding.toUint8Array(e);
+  }
+
+  _frameAwareness(clients) {
+    const e = encoding.createEncoder();
+    encoding.writeVarUint(e, MessageType.Awareness);
+    encoding.writeVarUint8Array(e, encodeAwarenessUpdate(this.awareness, clients));
+    return encoding.toUint8Array(e);
+  }
+}