state-sync-log 0.9.0

package/package.json

@@ -0,0 +1,94 @@
+{
+  "name": "state-sync-log",
+  "version": "0.9.0",
+  "description": "Validated Replicated State Machine built on Yjs. Combine CRDT offline capabilities with strict business logic validation, state reconciliation, and audit trails.",
+  "keywords": [
+    "state-sync",
+    "collaboration",
+    "crdt",
+    "log",
+    "yjs",
+    "yjs-middleware",
+    "replicated-state-machine",
+    "event-sourcing",
+    "validation",
+    "verification",
+    "audit-trail",
+    "offline-first",
+    "optimistic-ui",
+    "collaborative-state"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/xaviergonz/state-sync-log.git"
+  },
+  "bugs": {
+    "url": "https://github.com/xaviergonz/state-sync-log/issues"
+  },
+  "license": "MIT",
+  "author": "Javier González Garcés",
+  "source": "./src/index.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "script": "./dist/state-sync-log.umd.js",
+      "import": "./dist/state-sync-log.esm.mjs",
+      "require": "./dist/state-sync-log.umd.js",
+      "default": "./dist/state-sync-log.esm.mjs"
+    }
+  },
+  "esmodule": "./dist/state-sync-log.esm.js",
+  "module": "./dist/state-sync-log.esm.js",
+  "jsnext:main": "./dist/state-sync-log.esm.js",
+  "react-native": "./dist/state-sync-log.umd.js",
+  "umd:main": "./dist/state-sync-log.umd.js",
+  "unpkg": "./dist/state-sync-log.umd.js",
+  "jsdelivr": "./dist/state-sync-log.umd.js",
+  "main": "./dist/state-sync-log.umd.js",
+  "types": "./dist/types/index.d.ts",
+  "typings": "./dist/types/index.d.ts",
+  "sideEffects": false,
+  "files": [
+    "src",
+    "dist",
+    "LICENSE",
+    "CHANGELOG.md",
+    "README.md"
+  ],
+  "scripts": {
+    "quick-build": "tsc",
+    "quick-build-tests": "tsc -p test",
+    "copy-root-files": "shx cp ../../README.md . && shx cp ../../LICENSE . && shx cp ../../CHANGELOG.md .",
+    "build": "pnpm quick-build && pnpm copy-root-files && shx rm -rf dist && vite build && shx cp dist/state-sync-log.esm.mjs dist/state-sync-log.esm.js",
+    "test": "vitest run",
+    "test:ci": "vitest run --coverage --exclude **/fuzzySync.test.ts",
+    "test:fuzzy-loop": "./run_fuzzy_1000.sh",
+    "build-docs": "shx rm -rf api-docs && typedoc --options ./typedocconfig.js src/index.ts"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.3",
+    "@types/rfdc": "^1.2.0",
+    "@vitest/coverage-v8": "^4.0.15",
+    "shx": "^0.4.0",
+    "ts-node": "^10.9.2",
+    "typedoc": "^0.28.15",
+    "typescript": "^5.9.3",
+    "vite": "^7.2.6",
+    "vite-plugin-dts": "^4.5.4",
+    "vitest": "^4.0.15",
+    "yjs": "^13.6.27"
+  },
+  "peerDependencies": {
+    "yjs": "^13.0.0"
+  },
+  "dependencies": {
+    "fast-deep-equal": "^3.1.3",
+    "nanoid": "^5.1.6",
+    "rfdc": "^1.4.1",
+    "tslib": "^2.8.1"
+  },
+  "directories": {
+    "test": "test"
+  }
+}

package/src/ClientId.ts

@@ -0,0 +1 @@
+export type ClientId = string

package/src/SortedTxEntry.ts

@@ -0,0 +1,83 @@
+import type * as Y from "yjs"
+import { failure } from "./error"
+import { TxRecord } from "./TxRecord"
+import { parseTxTimestampKey, type TxTimestamp, type TxTimestampKey } from "./txTimestamp"
+
+/**
+ * A cached tx entry with lazy parsing and optional tx caching.
+ * The timestamp is parsed on first access and cached.
+ * The tx record can be fetched lazily and cached.
+ */
+export class SortedTxEntry {
+  private _txTimestamp?: TxTimestamp
+  private _originalTxTimestampKey?: TxTimestampKey | null
+  private _originalTxTimestamp?: TxTimestamp | null
+  private _txRecord?: TxRecord
+
+  constructor(
+    readonly txTimestampKey: TxTimestampKey,
+    private readonly _yTx: Y.Map<TxRecord>
+  ) {}
+
+  /**
+   * Gets the parsed timestamp, lazily parsing and caching on first access.
+   */
+  get txTimestamp(): TxTimestamp {
+    if (!this._txTimestamp) {
+      this._txTimestamp = parseTxTimestampKey(this.txTimestampKey)
+    }
+    return this._txTimestamp
+  }
+
+  /**
+   * Gets the original tx timestamp key, lazily and caching on first access.
+   */
+  get originalTxTimestampKey(): TxTimestampKey | null {
+    if (this._originalTxTimestampKey === undefined) {
+      const tx = this.txRecord
+      this._originalTxTimestampKey = tx.originalTxKey ?? null
+    }
+    return this._originalTxTimestampKey
+  }
+
+  /**
+   * Gets the parsed original tx timestamp, lazily parsing and caching on first access.
+   */
+  get originalTxTimestamp(): TxTimestamp | null {
+    if (this._originalTxTimestamp === undefined) {
+      const key = this.originalTxTimestampKey
+      this._originalTxTimestamp = key ? parseTxTimestampKey(key) : null
+    }
+    return this._originalTxTimestamp
+  }
+
+  /**
+   * Gets the logical (deduplicated) tx timestamp key.
+   * This is the original tx key if it exists, otherwise the physical key.
+   */
+  get dedupTxTimestampKey(): TxTimestampKey {
+    return this.originalTxTimestampKey ?? this.txTimestampKey
+  }
+
+  /**
+   * Gets the logical (deduplicated) parsed tx timestamp.
+   * This is the original tx timestamp if it exists, otherwise the physical timestamp.
+   */
+  get dedupTxTimestamp(): TxTimestamp {
+    return this.originalTxTimestamp ?? this.txTimestamp
+  }
+
+  /**
+   * Gets the tx record, lazily fetching and caching on first access.
+   * Returns undefined if the tx doesn't exist.
+   */
+  get txRecord(): TxRecord {
+    if (!this._txRecord) {
+      this._txRecord = this._yTx.get(this.txTimestampKey)
+      if (!this._txRecord) {
+        throw failure(`SortedTxEntry: TxRecord not found for key ${this.txTimestampKey}`)
+      }
+    }
+    return this._txRecord
+  }
+}

package/src/StateCalculator.ts

@@ -0,0 +1,407 @@
+import type * as Y from "yjs"
+import { type CheckpointRecord, type ClientWatermarks } from "./checkpoints"
+import { applyTxImmutable } from "./draft"
+import { JSONObject } from "./json"
+import { Op, ValidateFn } from "./operations"
+import { computeReconcileOps } from "./reconcile"
+import { SortedTxEntry } from "./SortedTxEntry"
+import { TxRecord } from "./TxRecord"
+import { compareTxTimestamps, type TxTimestamp, type TxTimestampKey } from "./txTimestamp"
+import { lazy } from "./utils"
+
+/**
+ * Checks if a transaction is covered by the checkpoint watermarks.
+ */
+export function isTransactionInCheckpoint(ts: TxTimestamp, watermarks: ClientWatermarks): boolean {
+  const wm = watermarks[ts.clientId]
+  if (!wm) return false
+  return ts.clock <= wm.maxClock
+}
+
+/**
+ * StateCalculator encapsulates the sorted transaction cache and state calculation logic.
+ *
+ * It maintains:
+ * - A sorted array of transaction entries
+ * - A map for O(1) lookup
+ * - A tracking index indicating up to which tx the state has been calculated
+ * - The cached calculated state
+ * - The base checkpoint
+ *
+ * The tracking index is invalidated (set to null) when:
+ * - A transaction is inserted before the already-calculated slice
+ * - A transaction is deleted from the already-calculated slice
+ * - The base checkpoint changes
+ */
+export class StateCalculator {
+  /** Sorted tx cache (ALL active/future txs, kept sorted by timestamp) */
+  private sortedTxs: SortedTxEntry[] = []
+
+  /** O(1) existence check and lookup */
+  private sortedTxsMap: Map<TxTimestampKey, SortedTxEntry> = new Map()
+
+  /**
+   * Index of the last transaction applied to cachedState.
+   * - null: state needs full recalculation from checkpoint
+   * - -1: no transactions have been applied yet (state === checkpoint state)
+   * - >= 0: transactions up to and including this index have been applied
+   */
+  private lastAppliedIndex: number | null = null
+
+  /** The cached calculated state */
+  private cachedState: JSONObject | null = null
+
+  /** The base checkpoint to calculate state from */
+  private baseCheckpoint: CheckpointRecord | null = null
+
+  /**
+   * Applied dedup keys - tracks which LOGICAL txs have been applied.
+   * This is the originalTxKey (or physical key if no original) for each applied tx.
+   * Used to properly deduplicate re-emits.
+   */
+  private appliedTxKeys: Set<TxTimestampKey> = new Set()
+
+  /** Max clock seen from any transaction (for Lamport clock updates) */
+  private maxSeenClock = 0
+
+  /** Validation function (optional) */
+  private validateFn?: ValidateFn<JSONObject>
+
+  constructor(validateFn?: ValidateFn<JSONObject>) {
+    this.validateFn = validateFn
+  }
+
+  /**
+   * Sets the base checkpoint. Invalidates cached state if checkpoint changed.
+   * @returns true if the checkpoint changed
+   */
+  setBaseCheckpoint(checkpoint: CheckpointRecord | null): boolean {
+    if (checkpoint === this.baseCheckpoint) {
+      return false
+    }
+
+    this.baseCheckpoint = checkpoint
+    this.invalidate()
+    return true
+  }
+
+  /**
+   * Gets the current base checkpoint.
+   */
+  getBaseCheckpoint(): CheckpointRecord | null {
+    return this.baseCheckpoint
+  }
+
+  /**
+   * Clears all transactions and rebuilds from yTx map.
+   * This is used when the checkpoint changes and we need a fresh start.
+   */
+  rebuildFromYjs(yTx: Y.Map<TxRecord>): void {
+    this.sortedTxs = []
+    this.sortedTxsMap.clear()
+
+    // Collect all entries, build the map and max clock
+    for (const key of yTx.keys()) {
+      const entry = new SortedTxEntry(key, yTx)
+      this.sortedTxs.push(entry)
+
+      this.sortedTxsMap.set(entry.txTimestampKey, entry)
+      if (entry.txTimestamp.clock > this.maxSeenClock) {
+        this.maxSeenClock = entry.txTimestamp.clock
+      }
+    }
+
+    // Sort once - O(n log n)
+    this.sortedTxs.sort((a, b) => compareTxTimestamps(a.txTimestamp, b.txTimestamp))
+
+    // Invalidate cached state since we rebuilt
+    this.invalidate()
+  }
+
+  /**
+   * Inserts a transaction into the sorted cache.
+   * Invalidates cached state if the transaction was inserted before the calculated slice.
+   *
+   * @returns true if this caused invalidation (out-of-order insert)
+   */
+  insertTx(key: TxTimestampKey, yTx: Y.Map<TxRecord>): boolean {
+    if (this.sortedTxsMap.has(key)) {
+      return false // Already exists
+    }
+
+    const entry = new SortedTxEntry(key, yTx)
+    const ts = entry.txTimestamp
+
+    // Update max seen clock for Lamport clock mechanism
+    if (ts.clock > this.maxSeenClock) {
+      this.maxSeenClock = ts.clock
+    }
+
+    const sortedTxs = this.sortedTxs
+
+    // Find insertion position (search from end since new txs typically have higher timestamps)
+    let insertIndex = sortedTxs.length // Default: append at end
+    for (let i = sortedTxs.length - 1; i >= 0; i--) {
+      const existingTs = sortedTxs[i].txTimestamp
+      if (compareTxTimestamps(ts, existingTs) >= 0) {
+        insertIndex = i + 1
+        break
+      }
+      if (i === 0) {
+        insertIndex = 0 // Insert at beginning
+      }
+    }
+
+    // Insert at the found position
+    sortedTxs.splice(insertIndex, 0, entry)
+    this.sortedTxsMap.set(key, entry)
+
+    // Check if this invalidates our cached state
+    // If we inserted before or at the last applied index, we need to recalculate
+    if (this.lastAppliedIndex !== null && insertIndex <= this.lastAppliedIndex) {
+      this.invalidate()
+      return true
+    }
+
+    return false
+  }
+
+  /**
+   * Removes multiple transactions from the sorted cache.
+   * @returns the number of keys that were actually removed
+   */
+  removeTxs(keys: readonly TxTimestampKey[]): number {
+    if (keys.length === 0) return 0
+
+    let removedCount = 0
+    let minRemovedIndex = Number.POSITIVE_INFINITY
+
+    // Build set of keys to delete and track their indices
+    const toDelete = new Set<TxTimestampKey>()
+    for (const key of keys) {
+      const entry = this.sortedTxsMap.get(key)
+      if (entry) {
+        this.sortedTxsMap.delete(key)
+        toDelete.add(key)
+
+        // Find index for invalidation check
+        const index = this.sortedTxs.indexOf(entry)
+        if (index !== -1 && index < minRemovedIndex) {
+          minRemovedIndex = index
+        }
+      }
+    }
+
+    if (toDelete.size === 0) return 0
+
+    // Single forward pass through sortedTxs, removing matching entries
+    const sortedTxs = this.sortedTxs
+    let i = 0
+    while (i < sortedTxs.length && toDelete.size > 0) {
+      if (toDelete.has(sortedTxs[i].txTimestampKey)) {
+        toDelete.delete(sortedTxs[i].txTimestampKey)
+        sortedTxs.splice(i, 1)
+        removedCount++
+      } else {
+        i++
+      }
+    }
+
+    // Check if this invalidates our cached state
+    if (this.lastAppliedIndex !== null && minRemovedIndex <= this.lastAppliedIndex) {
+      this.invalidate()
+    }
+
+    return removedCount
+  }
+
+  /**
+   * Checks if a transaction key exists in the cache.
+   */
+  hasTx(key: TxTimestampKey): boolean {
+    return this.sortedTxsMap.has(key)
+  }
+
+  /**
+   * Gets a transaction entry by key.
+   */
+  getTx(key: TxTimestampKey): SortedTxEntry | undefined {
+    return this.sortedTxsMap.get(key)
+  }
+
+  /**
+   * Gets all sorted transaction entries.
+   */
+  getSortedTxs(): readonly SortedTxEntry[] {
+    return this.sortedTxs
+  }
+
+  /**
+   * Gets the number of transactions in the cache.
+   */
+  get txCount(): number {
+    return this.sortedTxs.length
+  }
+
+  /**
+   * Returns true if the state needs full recalculation.
+   */
+  needsFullRecalculation(): boolean {
+    return this.lastAppliedIndex === null
+  }
+
+  /**
+   * Invalidates the cached state, forcing a full recalculation on next calculateState().
+   * Note: cachedState is kept so computeReconcileOps can diff old vs new state.
+   */
+  invalidate(): void {
+    this.lastAppliedIndex = null
+  }
+
+  /**
+   * Calculates and returns the current state, along with a lazy getter for ops that changed from the previous state.
+   *
+   * - If lastAppliedIndex is null: full recalculation from checkpoint
+   * - If lastAppliedIndex >= -1: incremental apply from lastAppliedIndex + 1
+   */
+  calculateState(): { state: JSONObject; getAppliedOps: () => readonly Op[] } {
+    const baseState: JSONObject = this.baseCheckpoint?.state ?? {}
+    const watermarks = this.baseCheckpoint?.watermarks ?? {}
+    const hasWatermarks = Object.keys(watermarks).length > 0
+
+    if (this.lastAppliedIndex === null) {
+      // SLOW PATH: Full recalculation
+      return this.fullRecalculation(baseState, watermarks, hasWatermarks)
+    }
+
+    // FAST PATH: Incremental apply
+    return this.incrementalApply(watermarks, hasWatermarks)
+  }
+
+  /**
+   * Full recalculation of state from the base checkpoint.
+   */
+  private fullRecalculation(
+    baseState: JSONObject,
+    watermarks: ClientWatermarks,
+    hasWatermarks: boolean
+  ): { state: JSONObject; getAppliedOps: () => readonly Op[] } {
+    const oldState = this.cachedState ?? {}
+
+    // Reset tracking for full recompute
+    this.appliedTxKeys.clear()
+    this.lastAppliedIndex = -1
+    this.cachedState = baseState
+
+    // Delegate to incremental apply to replay all transactions
+    // We ignore the returned ops because they represent the operations applied from the base state,
+    // whereas we want the diff from the *previous cached state*.
+    // We pass returnOps=false to avoid collecting ops during replay.
+    const { state } = this.incrementalApply(watermarks, hasWatermarks, false)
+
+    // Lazy load the reconciliation ops (expensive diff)
+    const getAppliedOps = lazy(() => computeReconcileOps(oldState, state))
+
+    return { state, getAppliedOps }
+  }
+
+  /**
+   * Incremental apply of transactions from lastAppliedIndex + 1.
+   * @param returnOps If true, collects applied transactions (to lazy compute ops). If false, skips collection.
+   */
+  private incrementalApply(
+    watermarks: ClientWatermarks,
+    hasWatermarks: boolean,
+    returnOps = true
+  ): { state: JSONObject; getAppliedOps: () => readonly Op[] } {
+    let state = this.cachedState as JSONObject
+    const appliedTxs: TxRecord[] = []
+    const sortedTxs = this.sortedTxs
+    const startIndex = this.lastAppliedIndex! + 1
+
+    for (let i = startIndex; i < sortedTxs.length; i++) {
+      const entry = sortedTxs[i]
+      const dedupKey = entry.dedupTxTimestampKey
+
+      // Skip if already applied (deduplication)
+      if (this.appliedTxKeys.has(dedupKey)) {
+        continue
+      }
+
+      // Skip if in checkpoint
+      if (hasWatermarks) {
+        const dedupTs = entry.dedupTxTimestamp
+        if (isTransactionInCheckpoint(dedupTs, watermarks)) {
+          this.appliedTxKeys.add(dedupKey)
+          continue
+        }
+      }
+
+      const tx = entry.txRecord
+
+      // Apply transaction 1-by-1 to avoid draft context pollution on validation failure
+      const newState = applyTxImmutable(state, tx, this.validateFn)
+
+      if (newState !== state) {
+        state = newState
+        if (returnOps) {
+          appliedTxs.push(tx)
+        }
+      }
+
+      this.appliedTxKeys.add(dedupKey)
+      this.lastAppliedIndex = i
+    }
+
+    // Update lastAppliedIndex to end even if all txs were skipped
+    // This ensures we don't re-process skipped txs on next incremental apply
+    if (sortedTxs.length > 0 && this.lastAppliedIndex! < sortedTxs.length - 1) {
+      this.lastAppliedIndex = sortedTxs.length - 1
+    }
+
+    this.cachedState = state
+
+    // Lazy getter for ops (flattens applied txs)
+    const getAppliedOps = lazy(() => {
+      const ops: Op[] = []
+      for (const tx of appliedTxs) {
+        ops.push(...tx.ops)
+      }
+      return ops
+    })
+
+    return { state, getAppliedOps }
+  }
+
+  /**
+   * Gets the max seen clock (for Lamport clock updates).
+   */
+  getMaxSeenClock(): number {
+    return this.maxSeenClock
+  }
+
+  /**
+   * Gets the current cached state without recalculating.
+   * Returns null if state has never been calculated.
+   */
+  getCachedState(): JSONObject | null {
+    return this.cachedState
+  }
+
+  /**
+   * Gets the last applied timestamp.
+   */
+  getLastAppliedTs(): TxTimestamp | null {
+    if (this.lastAppliedIndex === null || this.lastAppliedIndex < 0) {
+      return null
+    }
+    return this.sortedTxs[this.lastAppliedIndex]?.txTimestamp ?? null
+  }
+
+  /**
+   * Gets the last applied index (for debugging/tracking).
+   */
+  getLastAppliedIndex(): number | null {
+    return this.lastAppliedIndex
+  }
+}

package/src/TxRecord.ts

@@ -0,0 +1,15 @@
+import { Op } from "./operations"
+import type { TxTimestampKey } from "./txTimestamp"
+
+/**
+ * The immutable record stored in the Log.
+ */
+export type TxRecord = {
+  ops: readonly Op[]
+  /**
+   * If this is a re-emit of a missed transaction, this field holds the
+   * ORIGINAL key. Used for deduplication to prevent applying the same logical
+   * action twice.
+   */
+  originalTxKey?: TxTimestampKey
+}

package/src/checkpointUtils.ts

@@ -0,0 +1,44 @@
+import * as Y from "yjs"
+import { type CheckpointRecord, parseCheckpointKey } from "./checkpoints"
+
+/**
+ * Determines the finalized epoch and its canonical checkpoint in a single pass.
+ *
+ * Policy A: The finalized epoch is the most recent epoch with a checkpoint.
+ * Canonical checkpoint: The checkpoint with highest txCount for that epoch
+ * (tie-break: lowest clientId alphabetically).
+ *
+ * Returns { finalizedEpoch: -1, checkpoint: null } if no checkpoints exist.
+ */
+export function getFinalizedEpochAndCheckpoint(yCheckpoint: Y.Map<CheckpointRecord>): {
+  finalizedEpoch: number
+  checkpoint: CheckpointRecord | null
+} {
+  let maxEpoch = -1
+  let best: CheckpointRecord | null = null
+  let bestTxCount = -1
+  let bestClientId = ""
+
+  for (const [key, cp] of yCheckpoint.entries()) {
+    const { epoch, clientId } = parseCheckpointKey(key)
+
+    if (epoch > maxEpoch) {
+      // New highest epoch - reset best checkpoint tracking
+      maxEpoch = epoch
+      best = cp
+      bestTxCount = cp.txCount
+      bestClientId = clientId
+    } else if (epoch === maxEpoch) {
+      // Same epoch - check if this is a better canonical checkpoint
+      // Primary: higher txCount wins
+      // Secondary (tie-break): lower clientId (alphabetically) wins
+      if (cp.txCount > bestTxCount || (cp.txCount === bestTxCount && clientId < bestClientId)) {
+        best = cp
+        bestTxCount = cp.txCount
+        bestClientId = clientId
+      }
+    }
+  }
+
+  return { finalizedEpoch: maxEpoch, checkpoint: best }
+}