state-sync-log 0.9.0

package/src/checkpoints.ts

@@ -0,0 +1,208 @@
+import * as Y from "yjs"
+import { ClientId } from "./ClientId"
+import { getFinalizedEpochAndCheckpoint } from "./checkpointUtils"
+import { ClientState } from "./clientState"
+import { failure } from "./error"
+import { JSONObject } from "./json"
+import { TxRecord } from "./TxRecord"
+import { TxTimestampKey } from "./txTimestamp"
+
+/**
+ * Watermarking for deduplication and pruning.
+ * - maxClock: All txs from this client with clock <= maxClock are FINALIZED.
+ * - maxWallClock: The last time we saw this client active (for pruning).
+ */
+type ClientWatermark = Readonly<{
+  maxClock: number
+  maxWallClock: number
+}>
+
+/**
+ * Watermarks for all clients.
+ */
+export type ClientWatermarks = Record<ClientId, ClientWatermark>
+
+/**
+ * A snapshot of the state at the end of a specific epoch.
+ */
+export type CheckpointRecord = {
+  state: JSONObject // The document state
+  watermarks: ClientWatermarks // Dedup/Pruning info
+  txCount: number // Tie-breaker for canonical selection
+  minWallClock: number // Reference time for this epoch (deterministic pruning)
+}
+
+/**
+ * Unique ID for a checkpoint.
+ * Format: `${epoch};${txCount};${clientId}`
+ */
+export type CheckpointKey = string
+
+/**
+ * Data extracted from a checkpoint key.
+ */
+export type CheckpointKeyData = {
+  epoch: number
+  txCount: number
+  clientId: ClientId
+}
+
+/**
+ * Converts checkpoint key data components to a key string.
+ */
+function checkpointKeyDataToKey(data: CheckpointKeyData): CheckpointKey {
+  return `${data.epoch};${data.txCount};${data.clientId}`
+}
+
+/**
+ * Helper to parse checkpoint keys.
+ * Checkpoint keys have format: `${epoch};${txCount};${clientId}`
+ * Throws if key is malformed.
+ */
+export function parseCheckpointKey(key: CheckpointKey): CheckpointKeyData {
+  const i1 = key.indexOf(";")
+  const i2 = key.indexOf(";", i1 + 1)
+
+  if (i1 === -1 || i2 === -1) {
+    failure(`Malformed checkpoint key: ${key}`)
+  }
+
+  return {
+    epoch: Number.parseInt(key.substring(0, i1), 10),
+    txCount: Number.parseInt(key.substring(i1 + 1, i2), 10),
+    clientId: key.substring(i2 + 1),
+  }
+}
+
+/**
+ * Called periodically (e.g. by a server or leader client) to finalize the epoch.
+ */
+export function createCheckpoint(
+  yTx: Y.Map<TxRecord>,
+  yCheckpoint: Y.Map<CheckpointRecord>,
+  clientState: ClientState,
+  activeEpoch: number,
+  currentState: JSONObject,
+  myClientId: string
+): void {
+  // 1. Start with previous watermarks (from finalized epoch = activeEpoch - 1)
+  const { checkpoint: prevCP } = getFinalizedEpochAndCheckpoint(yCheckpoint)
+  const newWatermarks = prevCP ? { ...prevCP.watermarks } : {}
+
+  // Get active txs using cached sorted order (filter by epoch)
+  // FILTER IS REQUIRED:
+  // Although we are finalizing 'activeEpoch', other peers may have already
+  // advanced to the next epoch and started syncing those txs.
+  // We must ensure this checkpoint ONLY contains txs from 'activeEpoch'.
+  // Using stateCalculator.getSortedTxs avoids redundant key parsing (timestamps are cached).
+  //
+  // OPTIMIZATION: Since sortedTxs is sorted by epoch (primary key) and past epochs
+  // are pruned, we only need to find the right boundary. Future epochs are rare,
+  // so a simple linear search from the right is efficient (typically 0-1 iterations).
+  const sortedTxs = clientState.stateCalculator.getSortedTxs()
+
+  // Find end boundary by searching from right (skip any future epoch entries)
+  let endIndex = sortedTxs.length
+  while (endIndex > 0 && sortedTxs[endIndex - 1].txTimestamp.epoch > activeEpoch) {
+    endIndex--
+  }
+
+  // Slice from start to endIndex (past epochs are pruned, so these are all activeEpoch)
+  const activeTxs = sortedTxs.slice(0, endIndex)
+
+  if (activeTxs.length === 0) {
+    return // Do nothing if no txs (prevents empty epochs)
+  }
+
+  // 2. Update watermarks based on OBSERVED active txs and calculate minWallClock
+  // NOTE: We cannot use activeTxs[0].txTimestamp.wallClock for minWallClock because
+  // txs are sorted by Lamport clock (epoch → clock → clientId), not by wallClock.
+  // A client may have a high Lamport clock but early wallClock due to clock drift
+  // or receiving many messages before emitting.
+  let minWallClock = Number.POSITIVE_INFINITY
+  let txCount = 0
+  for (const entry of activeTxs) {
+    const ts = entry.txTimestamp
+
+    // Track min wallClock for deterministic pruning reference
+    if (ts.wallClock < minWallClock) {
+      minWallClock = ts.wallClock
+    }
+
+    const newWm = newWatermarks[ts.clientId]
+      ? { ...newWatermarks[ts.clientId] }
+      : { maxClock: -1, maxWallClock: 0 }
+
+    if (ts.clock > newWm.maxClock) {
+      newWm.maxClock = ts.clock
+      newWm.maxWallClock = ts.wallClock
+    }
+    newWatermarks[ts.clientId] = newWm
+    txCount++
+  }
+
+  // 3. Prune Inactive Watermarks (Deterministic)
+  // Uses minWallClock so all clients agree on exactly who to prune.
+  for (const clientId in newWatermarks) {
+    if (minWallClock - newWatermarks[clientId].maxWallClock > clientState.retentionWindowMs) {
+      delete newWatermarks[clientId]
+    }
+  }
+
+  // 4. Save Checkpoint
+  const cpKey = checkpointKeyDataToKey({
+    epoch: activeEpoch,
+    txCount,
+    clientId: myClientId,
+  })
+  yCheckpoint.set(cpKey, {
+    state: currentState, // Responsibility for cloning is moved to the caller if needed
+    watermarks: newWatermarks,
+    txCount,
+    minWallClock,
+  })
+
+  // 5. Early tx pruning (Optimization)
+  // Delete all txs from the now-finalized epoch
+  // This reduces memory pressure instead of waiting for cleanupLog
+  const keysToDelete: TxTimestampKey[] = []
+  for (const entry of activeTxs) {
+    yTx.delete(entry.txTimestampKey)
+    keysToDelete.push(entry.txTimestampKey)
+  }
+  clientState.stateCalculator.removeTxs(keysToDelete)
+}
+
+/**
+ * Garbage collects old checkpoints.
+ * Should be called periodically to prevent unbounded growth of yCheckpoint.
+ *
+ * Keeps only the canonical checkpoint for the finalized epoch.
+ * Everything else is deleted (old epochs + non-canonical).
+ *
+ * Note: The active epoch never has checkpoints - creating a checkpoint
+ * for an epoch immediately makes it finalized.
+ */
+export function pruneCheckpoints(
+  yCheckpoint: Y.Map<CheckpointRecord>,
+  finalizedEpoch: number
+): void {
+  // Find the canonical checkpoint and its key in one pass
+  let canonicalKey: CheckpointKey | null = null
+  let bestTxCount = -1
+
+  for (const [key] of yCheckpoint.entries()) {
+    const { epoch, txCount } = parseCheckpointKey(key)
+    if (epoch === finalizedEpoch && txCount > bestTxCount) {
+      canonicalKey = key
+      bestTxCount = txCount
+    }
+  }
+
+  // Delete everything except the canonical checkpoint
+  for (const key of yCheckpoint.keys()) {
+    if (key !== canonicalKey) {
+      yCheckpoint.delete(key)
+    }
+  }
+}

package/src/clientState.ts

@@ -0,0 +1,37 @@
+import { JSONObject } from "./json"
+import { ValidateFn } from "./operations"
+import { StateCalculator } from "./StateCalculator"
+
+/**
+ * Client-side state including clocks and calculator for state management
+ */
+export interface ClientState {
+  // Lamport clocks (monotonic, never reset)
+  localClock: number
+
+  // Cached finalized epoch (null = not yet initialized, recalculated only when checkpoint map changes)
+  cachedFinalizedEpoch: number | null
+
+  // State calculator (manages sorted tx cache, state calculation, and invalidation)
+  stateCalculator: StateCalculator
+
+  /**
+   * Timestamp retention window in milliseconds.
+   */
+  retentionWindowMs: number
+}
+
+/**
+ * Factory to create an initial ClientState
+ */
+export function createClientState(
+  validateFn: ValidateFn<JSONObject> | undefined,
+  retentionWindowMs: number
+): ClientState {
+  return {
+    localClock: 0,
+    cachedFinalizedEpoch: null, // Will be recalculated on first run
+    stateCalculator: new StateCalculator(validateFn),
+    retentionWindowMs,
+  }
+}

package/src/createStateSyncLog.ts

@@ -0,0 +1,330 @@
+import * as Y from "yjs"
+import { CheckpointRecord, createCheckpoint } from "./checkpoints"
+import { createClientState } from "./clientState"
+import { failure } from "./error"
+import { JSONObject } from "./json"
+
+import { Op, ValidateFn } from "./operations"
+
+import { computeReconcileOps } from "./reconcile"
+import { SortedTxEntry } from "./SortedTxEntry"
+import { TxRecord } from "./TxRecord"
+import { appendTx, TxKeyChanges, updateState } from "./txLog"
+import { TxTimestampKey } from "./txTimestamp"
+import { generateID } from "./utils"
+
+export const getSortedTxsSymbol = Symbol("getSortedTxs")
+
+export interface StateSyncLogOptions<State extends JSONObject> {
+  /**
+   * The Y.js Document to bind to.
+   */
+  yDoc: Y.Doc
+
+  /**
+   * Name for the txs Y.Map.
+   * Default: "state-sync-log-tx"
+   */
+  yTxMapName?: string
+
+  /**
+   * Name for the checkpoint Y.Map.
+   * Default: "state-sync-log-checkpoint"
+   */
+  yCheckpointMapName?: string
+
+  /**
+   * Unique identifier for this client.
+   * If omitted, a random UUID (nanoid) will be generated.
+   * NOTE: If you need to resume a session (keep local clock/watermark), you MUST provide a stable ID.
+   * MUST NOT contain semicolons.
+   */
+  clientId?: string
+
+  /**
+   * Origin tag for Y.js txs created by this library.
+   */
+  yjsOrigin?: unknown
+
+  /**
+   * Optional validation function.
+   * Runs after each tx's ops are applied.
+   * If it returns false, the tx is rejected (state reverts).
+   * MUST be deterministic and consistent across all clients.
+   */
+  validate?: (state: State) => boolean
+
+  /**
+   * Timestamp retention window in milliseconds.
+   * Txs older than this window are considered "Ancient" and pruned.
+   *
+   * Default: Infinity (No pruning).
+   * Recommended: 14 days (1209600000 ms).
+   */
+  retentionWindowMs: number | undefined
+}
+
+export interface StateSyncLogController<State extends JSONObject> {
+  /**
+   * Returns the current state.
+   */
+  getState(): State
+
+  /**
+   * Subscribes to state changes.
+   */
+  subscribe(callback: (newState: State, getAppliedOps: () => readonly Op[]) => void): () => void
+
+  /**
+   * Emits a new tx (list of operations) to the log.
+   */
+  emit(ops: Op[]): void
+
+  /**
+   * Reconciles the current state with the target state.
+   */
+  reconcileState(targetState: State): void
+
+  /**
+   * Manually triggers epoch compaction (Checkpointing).
+   */
+  compact(): void
+
+  /**
+   * Cleans up observers and releases memory.
+   */
+  dispose(): void
+
+  // --- Observability & Stats ---
+
+  /**
+   * Returns the current active epoch number.
+   */
+  getActiveEpoch(): number
+
+  /**
+   * Returns the number of txs currently in the active epoch.
+   */
+  getActiveEpochTxCount(): number
+
+  /**
+   * Returns the wallClock timestamp of the first tx in the active epoch.
+   */
+  getActiveEpochStartTime(): number | undefined
+
+  /**
+   * Returns true if the log is completely empty.
+   */
+  isLogEmpty(): boolean
+
+  /**
+   * Internal/Testing: Returns all txs currently in the log, sorted.
+   */
+  [getSortedTxsSymbol](): readonly SortedTxEntry[]
+}
+
+/**
+ * Creates a StateSyncLog controller.
+ */
+export function createStateSyncLog<State extends JSONObject>(
+  options: StateSyncLogOptions<State>
+): StateSyncLogController<State> {
+  const {
+    yDoc,
+    yTxMapName = "state-sync-log-tx",
+    yCheckpointMapName = "state-sync-log-checkpoint",
+    clientId = generateID(),
+    yjsOrigin,
+    validate,
+    retentionWindowMs,
+  } = options
+
+  if (clientId.includes(";")) {
+    failure(`clientId MUST NOT contain semicolons: ${clientId}`)
+  }
+
+  const yTx = yDoc.getMap<TxRecord>(yTxMapName)
+  const yCheckpoint = yDoc.getMap<CheckpointRecord>(yCheckpointMapName)
+
+  // Cast validate to basic type to match internal ClientState
+  const clientState = createClientState(
+    validate as unknown as ValidateFn<JSONObject>,
+    retentionWindowMs ?? Number.POSITIVE_INFINITY
+  )
+
+  // Listeners
+  const subscribers = new Set<(state: State, getAppliedOps: () => readonly Op[]) => void>()
+
+  const notifySubscribers = (state: State, getAppliedOps: () => readonly Op[]) => {
+    for (const sub of subscribers) {
+      sub(state, getAppliedOps)
+    }
+  }
+
+  // Helper to extract key changes from YMapEvent
+  const extractTxChanges = (event: Y.YMapEvent<TxRecord>): TxKeyChanges => {
+    const added: TxTimestampKey[] = []
+    const deleted: TxTimestampKey[] = []
+
+    for (const [key, change] of event.changes.keys) {
+      if (change.action === "add") {
+        added.push(key)
+      } else if (change.action === "delete") {
+        deleted.push(key)
+      } else if (change.action === "update") {
+        deleted.push(key)
+        added.push(key)
+      }
+    }
+
+    return { added, deleted }
+  }
+
+  // Empty txChanges object for checkpoint observer (no tx keys changed)
+  const emptyTxChanges: TxKeyChanges = { added: [], deleted: [] }
+
+  // Update Logic with incremental changes
+  const runUpdate = (txChanges: TxKeyChanges | undefined) => {
+    const { state, getAppliedOps } = updateState(
+      yDoc,
+      yTx,
+      yCheckpoint,
+      clientId,
+      clientState,
+      txChanges
+    )
+    notifySubscribers(state as State, getAppliedOps)
+  }
+
+  // Tx observer
+  const txObserver = (event: Y.YMapEvent<TxRecord>, _transaction: Y.Transaction) => {
+    const txChanges = extractTxChanges(event)
+    runUpdate(txChanges)
+  }
+
+  // Checkpoint observer
+  const checkpointObserver = (
+    _event: Y.YMapEvent<CheckpointRecord>,
+    _transaction: Y.Transaction
+  ) => {
+    runUpdate(emptyTxChanges)
+  }
+
+  yCheckpoint.observe(checkpointObserver)
+  yTx.observe(txObserver)
+
+  // Initial run (full recompute, treat as checkpoint change to initialize epoch cache)
+  runUpdate(undefined)
+
+  // Track disposal state
+  let disposed = false
+
+  const assertNotDisposed = () => {
+    if (disposed) {
+      failure("StateSyncLog has been disposed and cannot be used")
+    }
+  }
+
+  const getActiveEpochInternal = () => {
+    if (clientState.cachedFinalizedEpoch === null) {
+      failure("cachedFinalizedEpoch is null - this should not happen after initialization")
+    }
+    return clientState.cachedFinalizedEpoch + 1
+  }
+
+  return {
+    getState(): State {
+      assertNotDisposed()
+      return (clientState.stateCalculator.getCachedState() ?? {}) as State
+    },
+
+    subscribe(callback: (newState: State, getAppliedOps: () => readonly Op[]) => void): () => void {
+      assertNotDisposed()
+      subscribers.add(callback)
+      return () => {
+        subscribers.delete(callback)
+      }
+    },
+
+    emit(ops: Op[]): void {
+      assertNotDisposed()
+      yDoc.transact(() => {
+        const activeEpoch = getActiveEpochInternal()
+        appendTx(ops, yTx, activeEpoch, clientId, clientState)
+      }, yjsOrigin)
+    },
+
+    reconcileState(targetState: State): void {
+      assertNotDisposed()
+      const currentState = (clientState.stateCalculator.getCachedState() ?? {}) as State
+      const ops = computeReconcileOps(currentState, targetState)
+      if (ops.length > 0) {
+        this.emit(ops)
+      }
+    },
+
+    compact(): void {
+      assertNotDisposed()
+      yDoc.transact(() => {
+        const activeEpoch = getActiveEpochInternal()
+        const currentState = clientState.stateCalculator.getCachedState() ?? {}
+        createCheckpoint(yTx, yCheckpoint, clientState, activeEpoch, currentState, clientId)
+      }, yjsOrigin)
+    },
+
+    dispose(): void {
+      if (disposed) return // Already disposed, no-op
+      disposed = true
+      yTx.unobserve(txObserver)
+      yCheckpoint.unobserve(checkpointObserver)
+      subscribers.clear()
+    },
+
+    getActiveEpoch(): number {
+      assertNotDisposed()
+      return getActiveEpochInternal()
+    },
+
+    getActiveEpochTxCount(): number {
+      assertNotDisposed()
+      const activeEpoch = getActiveEpochInternal()
+      let count = 0
+      // Only current or future epochs exist in sortedTxs (past epochs are pruned during updateState).
+      // Future epochs appear if we receive txs before the corresponding checkpoint.
+      for (const entry of clientState.stateCalculator.getSortedTxs()) {
+        const ts = entry.txTimestamp
+        if (ts.epoch === activeEpoch) {
+          count++
+        } else if (ts.epoch > activeEpoch) {
+          break // Optimization: sorted order means we can stop early
+        }
+      }
+      return count
+    },
+
+    getActiveEpochStartTime(): number | undefined {
+      assertNotDisposed()
+      const activeEpoch = getActiveEpochInternal()
+      // Only current or future epochs exist in sortedTxs (past epochs are pruned during updateState).
+      for (const entry of clientState.stateCalculator.getSortedTxs()) {
+        const ts = entry.txTimestamp
+        if (ts.epoch === activeEpoch) {
+          return ts.wallClock
+        } else if (ts.epoch > activeEpoch) {
+          break // Optimization: sorted order means we can stop early
+        }
+      }
+      return undefined
+    },
+
+    isLogEmpty(): boolean {
+      assertNotDisposed()
+      return yTx.size === 0 && yCheckpoint.size === 0
+    },
+
+    [getSortedTxsSymbol](): readonly SortedTxEntry[] {
+      assertNotDisposed()
+      return clientState.stateCalculator.getSortedTxs()
+    },
+  }
+}