yaml-flow 8.6.3 → 8.7.0

package/examples/board-firestore/server/adapters/firestore-archive-factory.js

@@ -0,0 +1,59 @@
+/**
+ * firestore-archive-factory.js
+ *
+ * AsyncArchiveFactory backed by Firestore.
+ * Each named stream is a subcollection: boards/{boardId}/archive-stream-{name}
+ * Each named blob collection: boards/{boardId}/archive-blob-{name}
+ *
+ * NOTE: listStreams / listBlobs use Admin SDK's listCollections() — not available
+ * in the browser Firestore JS SDK. In the browser, skip or maintain an index manually.
+ *
+ * @param {import('@google-cloud/firestore').Firestore} db
+ * @param {string} boardId
+ * @returns {import('yaml-flow/cloud-storage').AsyncArchiveFactory}
+ */
+
+import { createFirestoreJournalStorage } from './firestore-journal-storage.js';
+import { createFirestoreBlobStorage } from './firestore-blob-storage.js';
+
+export function createFirestoreArchiveFactory(db, boardId) {
+  const boardDoc = db.collection('boards').doc(boardId);
+
+  return {
+    stream(name) {
+      return createFirestoreJournalStorage(boardDoc.collection(`archive-stream-${name}`));
+    },
+
+    blob(name) {
+      return createFirestoreBlobStorage(boardDoc.collection(`archive-blob-${name}`));
+    },
+
+    async listStreams(prefix = '') {
+      const cols = await boardDoc.listCollections();
+      const tag = `archive-stream-${prefix}`;
+      return cols
+        .map((c) => c.id)
+        .filter((id) => id.startsWith(tag))
+        .map((id) => id.slice('archive-stream-'.length));
+    },
+
+    async listBlobs(prefix = '') {
+      const cols = await boardDoc.listCollections();
+      const tag = `archive-blob-${prefix}`;
+      return cols
+        .map((c) => c.id)
+        .filter((id) => id.startsWith(tag))
+        .map((id) => id.slice('archive-blob-'.length));
+    },
+
+    config: {
+      async get(k) {
+        const snap = await boardDoc.collection('archive-config').doc('main').get();
+        return snap.data()?.[k] ?? null;
+      },
+      async set(k, v) {
+        await boardDoc.collection('archive-config').doc('main').set({ [k]: v }, { merge: true });
+      },
+    },
+  };
+}

package/examples/board-firestore/server/adapters/firestore-blob-storage.js

@@ -0,0 +1,82 @@
+/**
+ * firestore-blob-storage.js
+ *
+ * AsyncBlobStorage backed by Firestore for text/small blobs.
+ * Document shape: { k, content, contentType?, bytesBase64? }
+ *
+ * Binary blobs (readBytes/writeBytes) are stored as base64-encoded strings.
+ * For production use with files > 1 MB, replace writeBytes/readBytes with
+ * Firebase Storage (Cloud Storage for Firebase) and store the GCS path here.
+ *
+ * @param {import('@google-cloud/firestore').CollectionReference} col
+ * @returns {import('yaml-flow/cloud-storage').AsyncBlobStorage}
+ */
+
+function encodeDocId(key) {
+  return Buffer.from(key).toString('base64url');
+}
+
+function decodeDocId(docId) {
+  return Buffer.from(docId, 'base64url').toString();
+}
+
+export function createFirestoreBlobStorage(col) {
+  return {
+    async read(key) {
+      const snap = await col.doc(encodeDocId(key)).get();
+      if (!snap.exists) return null;
+      return snap.data()?.content ?? null;
+    },
+
+    async write(key, content) {
+      await col.doc(encodeDocId(key)).set({ k: key, content });
+    },
+
+    async exists(key) {
+      const snap = await col.doc(encodeDocId(key)).get();
+      return snap.exists;
+    },
+
+    async remove(key) {
+      await col.doc(encodeDocId(key)).delete();
+    },
+
+    async readBytes(key) {
+      const snap = await col.doc(encodeDocId(key)).get();
+      if (!snap.exists) return null;
+      const data = snap.data();
+      if (data?.bytesBase64) return Buffer.from(data.bytesBase64, 'base64');
+      if (data?.content) return Buffer.from(data.content);
+      return null;
+    },
+
+    async writeBytes(key, bytes) {
+      // NOTE: Firestore 1 MB doc limit. For larger files use Firebase Storage.
+      const bytesBase64 = Buffer.from(bytes).toString('base64');
+      await col.doc(encodeDocId(key)).set({ k: key, bytesBase64 });
+    },
+
+    async listKeys(prefix = '') {
+      let q = col;
+      if (prefix) {
+        q = col.where('k', '>=', prefix).where('k', '<', prefix + '\uf8ff');
+      }
+      const snap = await q.orderBy('k').get();
+      return snap.docs.map((d) => d.data().k ?? decodeDocId(d.id));
+    },
+
+    async stat(key) {
+      const snap = await col.doc(encodeDocId(key)).get();
+      if (!snap.exists) return null;
+      const data = snap.data();
+      const size = data?.bytesBase64
+        ? Math.floor((data.bytesBase64.length * 3) / 4)
+        : (data?.content?.length ?? 0);
+      return { key, size, contentType: data?.contentType ?? 'application/octet-stream' };
+    },
+
+    keyRef(key) {
+      return { kind: 'firestore-blob', value: `${col.path}/${encodeDocId(key)}` };
+    },
+  };
+}

package/examples/board-firestore/server/adapters/firestore-board-adapter.js

@@ -0,0 +1,127 @@
+/**
+ * firestore-board-adapter.js
+ *
+ * Composes all Firestore-backed storage primitives into an AsyncBoardPlatformAdapter
+ * using createHostedAsyncBoardPlatformAdapter from yaml-flow/cloud-storage.
+ *
+ * Firestore collection layout under boards/{boardId}/:
+ *   kv-{namespace}/          AsyncKVStorage per namespace
+ *   journal/                 single board journal
+ *   worker-queue/            board worker task queue
+ *   chat-queue/              chat agent dispatch queue
+ *   process-queue/           processAccumulated trigger queue
+ *   blobs-{namespace}/       AsyncBlobStorage per namespace
+ *   scratch/                 ephemeral scratch storage
+ *   archive-stream-{name}/   archive journal streams
+ *   archive-blob-{name}/     archive blob collections
+ *   archive-config/          archive config KV
+ *   locks/board-lock         AtomicRelayLock document
+ *
+ * @param {import('@google-cloud/firestore').Firestore} db
+ * @param {string} boardId
+ * @param {{ holderId?: string, requestProcessAccumulated?: () => void }} [options]
+ * @returns {import('yaml-flow/cloud-storage').AsyncBoardPlatformAdapter}
+ */
+
+import { createHash, randomUUID } from 'node:crypto';
+import { createHostedAsyncBoardPlatformAdapter } from 'yaml-flow/cloud-storage';
+
+import { createFirestoreKvStorage } from './firestore-kv-storage.js';
+import { createFirestoreJournalStorage } from './firestore-journal-storage.js';
+import { createFirestoreQueueStorage } from './firestore-queue-storage.js';
+import { createFirestoreBlobStorage } from './firestore-blob-storage.js';
+import { createFirestoreScratchStorage } from './firestore-scratch-storage.js';
+import { createFirestoreArchiveFactory } from './firestore-archive-factory.js';
+import { createFirestoreLock } from './firestore-lock.js';
+
+function parseKindValueRef(ref) {
+  const PREFIX = 'b64:';
+  if (!ref.startsWith(PREFIX)) return { kind: 'unknown', value: ref };
+  const b64 = ref.slice(PREFIX.length)
+    .replace(/-/g, '+').replace(/_/g, '/')
+    + '='.repeat((4 - (ref.length - PREFIX.length) % 4) % 4);
+  return JSON.parse(Buffer.from(b64, 'base64').toString());
+}
+
+function lexicalId() {
+  const ts = String(Date.now()).padStart(13, '0');
+  const rand = Math.random().toString(36).slice(2, 10).padEnd(8, '0');
+  return `${ts}-${rand}`;
+}
+
+export function createFirestoreBoardAdapter(db, boardId, options = {}) {
+  const boardDoc = db.collection('boards').doc(boardId);
+
+  return createHostedAsyncBoardPlatformAdapter({
+    boardId,
+
+    // ── KV storage ────────────────────────────────────────────────────────
+    kvStorage(namespace) {
+      return createFirestoreKvStorage(boardDoc.collection(`kv-${namespace || 'root'}`));
+    },
+    kvStorageForRef(ref) {
+      const parsed = parseKindValueRef(ref);
+      // ref.value is treated as a full Firestore collection path when kind='firestore'
+      const col = parsed.kind === 'firestore'
+        ? db.collection(parsed.value)
+        : boardDoc.collection(`kv-ref-${Buffer.from(ref).toString('base64url').slice(0, 16)}`);
+      return createFirestoreKvStorage(col);
+    },
+
+    // ── Blob storage ──────────────────────────────────────────────────────
+    blobStorage(namespace) {
+      return createFirestoreBlobStorage(boardDoc.collection(`blobs-${namespace || 'root'}`));
+    },
+
+    // ── Scratch storage ───────────────────────────────────────────────────
+    scratchStorage() {
+      return createFirestoreScratchStorage(boardDoc.collection('scratch'));
+    },
+    scratchStorageForRef(ref) {
+      const parsed = parseKindValueRef(ref);
+      const col = parsed.kind === 'firestore'
+        ? db.collection(parsed.value)
+        : boardDoc.collection('scratch');
+      return createFirestoreScratchStorage(col);
+    },
+
+    // ── Archive factory ───────────────────────────────────────────────────
+    archiveFactory() {
+      return createFirestoreArchiveFactory(db, boardId);
+    },
+    archiveFactoryForRef(ref) {
+      const parsed = parseKindValueRef(ref);
+      const altBoardId = parsed.kind === 'firestore' ? parsed.value : `${boardId}-archive`;
+      return createFirestoreArchiveFactory(db, altBoardId);
+    },
+
+    // ── Journal storage ───────────────────────────────────────────────────
+    journalStorage() {
+      return createFirestoreJournalStorage(boardDoc.collection('journal'));
+    },
+
+    // ── Queue storage ─────────────────────────────────────────────────────
+    queueStorage: createFirestoreQueueStorage(boardDoc.collection('worker-queue')),
+    chatAgentQueueStorage: createFirestoreQueueStorage(boardDoc.collection('chat-queue')),
+    processAccumulatedQueueStorage: createFirestoreQueueStorage(boardDoc.collection('process-queue')),
+
+    // ── Lock ──────────────────────────────────────────────────────────────
+    lock: createFirestoreLock(
+      boardDoc.collection('locks').doc('board-lock'),
+      { holderId: options.holderId ?? randomUUID() },
+    ),
+
+    // ── Utilities ─────────────────────────────────────────────────────────
+    hashFn(value) {
+      return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 16);
+    },
+
+    genId() {
+      return lexicalId();
+    },
+
+    requestProcessAccumulated: options.requestProcessAccumulated,
+    publishBoardChangeNotifications: options.publishBoardChangeNotifications,
+    onWarn: (msg) => console.warn(`[firestore-board-adapter:${boardId}] ${msg}`),
+  });
+}

package/examples/board-firestore/server/adapters/firestore-journal-storage.js

@@ -0,0 +1,54 @@
+/**
+ * firestore-journal-storage.js
+ *
+ * AsyncJournalStorage backed by a Firestore CollectionReference.
+ * Each entry is stored as: { id, createdAt, payload }
+ * IDs are lexicographically sortable (timestamp prefix) so ORDER BY id
+ * gives insertion order.
+ *
+ * @param {import('@google-cloud/firestore').CollectionReference} col
+ * @returns {import('yaml-flow/cloud-storage').AsyncJournalStorage}
+ */
+
+function lexicalId() {
+  const ts = String(Date.now()).padStart(13, '0');
+  const rand = Math.random().toString(36).slice(2, 10).padEnd(8, '0');
+  return `${ts}-${rand}`;
+}
+
+export function createFirestoreJournalStorage(col) {
+  return {
+    async append(payload) {
+      const id = lexicalId();
+      const doc = { id, createdAt: new Date().toISOString(), payload };
+      await col.doc(id).set(doc);
+      return { id, payload };
+    },
+
+    async readAll() {
+      const snap = await col.orderBy('id').get();
+      return snap.docs.map((d) => ({ id: d.data().id, payload: d.data().payload }));
+    },
+
+    async readAfter(cursor) {
+      let q = col.orderBy('id');
+      if (cursor) q = col.where('id', '>', cursor).orderBy('id');
+      const snap = await q.get();
+      const entries = snap.docs.map((d) => ({ id: d.data().id, payload: d.data().payload }));
+      return {
+        entries,
+        newCursor: entries.length > 0 ? entries[entries.length - 1].id : cursor,
+      };
+    },
+
+    async clear() {
+      const snap = await col.get();
+      const batchSize = 500;
+      for (let i = 0; i < snap.docs.length; i += batchSize) {
+        const batch = col.firestore.batch();
+        for (const doc of snap.docs.slice(i, i + batchSize)) batch.delete(doc.ref);
+        await batch.commit();
+      }
+    },
+  };
+}

package/examples/board-firestore/server/adapters/firestore-kv-storage.js

@@ -0,0 +1,47 @@
+/**
+ * firestore-kv-storage.js
+ *
+ * AsyncKVStorage backed by a Firestore CollectionReference.
+ * Each key is stored as a document with fields: { k, value }
+ * where k = original key string, value = arbitrary JSON value.
+ *
+ * Document ID: base64url(key)  — avoids '/' and other Firestore-forbidden chars.
+ *
+ * @param {import('@google-cloud/firestore').CollectionReference} col
+ * @returns {import('yaml-flow/cloud-storage').AsyncKVStorage}
+ */
+
+function encodeDocId(key) {
+  return Buffer.from(key).toString('base64url');
+}
+
+function decodeDocId(docId) {
+  return Buffer.from(docId, 'base64url').toString();
+}
+
+export function createFirestoreKvStorage(col) {
+  return {
+    async read(key) {
+      const snap = await col.doc(encodeDocId(key)).get();
+      if (!snap.exists) return null;
+      return snap.data()?.value ?? null;
+    },
+
+    async write(key, value) {
+      await col.doc(encodeDocId(key)).set({ k: key, value });
+    },
+
+    async delete(key) {
+      await col.doc(encodeDocId(key)).delete();
+    },
+
+    async listKeys(prefix = '') {
+      let q = col;
+      if (prefix) {
+        q = col.where('k', '>=', prefix).where('k', '<', prefix + '\uf8ff');
+      }
+      const snap = await q.orderBy('k').get();
+      return snap.docs.map((d) => d.data().k ?? decodeDocId(d.id));
+    },
+  };
+}

package/examples/board-firestore/server/adapters/firestore-lock.js

@@ -0,0 +1,62 @@
+/**
+ * firestore-lock.js
+ *
+ * AsyncAtomicRelayLock backed by a Firestore document + runTransaction.
+ *
+ * Document shape: { held, holderId, expiresAt, acquiredAt }
+ * Guarantees at-most-one holder via Firestore transaction atomicity.
+ *
+ * @param {import('@google-cloud/firestore').DocumentReference} lockDoc
+ * @param {{ holderId?: string, ttlMs?: number }} [options]
+ * @returns {import('yaml-flow/cloud-storage').AsyncAtomicRelayLock}
+ */
+
+import { randomUUID } from 'node:crypto';
+
+export function createFirestoreLock(lockDoc, options = {}) {
+  const holderId = options.holderId ?? randomUUID();
+  const ttlMs = options.ttlMs ?? 30_000;
+
+  return {
+    async tryAcquire() {
+      let released = false;
+
+      try {
+        await lockDoc.firestore.runTransaction(async (tx) => {
+          const snap = await tx.get(lockDoc);
+          const now = new Date();
+          if (snap.exists) {
+            const d = snap.data();
+            if (d.held && d.expiresAt > now.toISOString()) {
+              throw Object.assign(new Error('locked'), { code: 'locked' });
+            }
+          }
+          tx.set(lockDoc, {
+            held: true,
+            holderId,
+            expiresAt: new Date(now.getTime() + ttlMs).toISOString(),
+            acquiredAt: now.toISOString(),
+          });
+        });
+      } catch (err) {
+        if (err?.code === 'locked') return null;
+        throw err;
+      }
+
+      return async () => {
+        if (released) return;
+        released = true;
+        try {
+          await lockDoc.firestore.runTransaction(async (tx) => {
+            const snap = await tx.get(lockDoc);
+            if (snap.exists && snap.data().holderId === holderId) {
+              tx.update(lockDoc, { held: false, holderId: null });
+            }
+          });
+        } catch {
+          // Best-effort release — lock will expire via TTL
+        }
+      };
+    },
+  };
+}

package/examples/board-firestore/server/adapters/firestore-queue-storage.js

@@ -0,0 +1,186 @@
+/**
+ * firestore-queue-storage.js
+ *
+ * AsyncQueueStorage backed by Firestore with visibility-timeout leasing.
+ *
+ * Document shape:
+ *   { id, body, enqueuedAt, attempt, visibleAfter, leaseToken, leaseExpiresAt,
+ *     dead, deadReason, dedupKey? }
+ *
+ * Leasing is done via Firestore transactions: find visible messages,
+ * then atomically claim each one (optimistic concurrency — skip on race).
+ *
+ * Required Firestore composite index for the lease query:
+ *   Collection group: <your collection>
+ *   Fields: dead ASC, visibleAfter ASC
+ *
+ * @param {import('@google-cloud/firestore').CollectionReference} col
+ * @param {{ defaultVisibilityMs?: number }} [options]
+ * @returns {import('yaml-flow/cloud-storage').AsyncQueueStorage}
+ */
+
+import { randomUUID } from 'node:crypto';
+
+function lexicalId() {
+  const ts = String(Date.now()).padStart(13, '0');
+  const rand = Math.random().toString(36).slice(2, 10).padEnd(8, '0');
+  return `${ts}-${rand}`;
+}
+
+export function createFirestoreQueueStorage(col, options = {}) {
+  const defaultVisibilityMs = options.defaultVisibilityMs ?? 30_000;
+
+  return {
+    async enqueue(body) {
+      const id = lexicalId();
+      const now = new Date().toISOString();
+      await col.doc(id).set({
+        id,
+        body,
+        enqueuedAt: now,
+        attempt: 0,
+        visibleAfter: now,
+        leaseToken: null,
+        leaseExpiresAt: null,
+        dead: false,
+        deadReason: null,
+      });
+      return { id, body, enqueuedAt: now };
+    },
+
+    async enqueueIfAbsent(body, dedupKey) {
+      const existing = await col
+        .where('dedupKey', '==', dedupKey)
+        .where('dead', '==', false)
+        .limit(1)
+        .get();
+      if (!existing.empty) return null;
+      const id = lexicalId();
+      const now = new Date().toISOString();
+      await col.doc(id).set({
+        id,
+        body,
+        enqueuedAt: now,
+        attempt: 0,
+        visibleAfter: now,
+        leaseToken: null,
+        leaseExpiresAt: null,
+        dead: false,
+        deadReason: null,
+        dedupKey,
+      });
+      return { id, body, enqueuedAt: now };
+    },
+
+    async lease(opts = {}) {
+      const max = opts.max ?? 1;
+      const visibilityMs = opts.visibilityMs ?? defaultVisibilityMs;
+      const nowIso = new Date().toISOString();
+
+      // Find visible, non-dead messages — fetch extra to tolerate races
+      const snap = await col
+        .where('dead', '==', false)
+        .where('visibleAfter', '<=', nowIso)
+        .orderBy('visibleAfter')
+        .limit(max * 4)
+        .get();
+
+      const leased = [];
+      for (const doc of snap.docs) {
+        if (leased.length >= max) break;
+
+        const data = doc.data();
+        // Skip if currently leased and lease hasn't expired
+        if (data.leaseToken && data.leaseExpiresAt > nowIso) continue;
+
+        const leaseToken = randomUUID();
+        const leaseExpiresAt = new Date(Date.now() + visibilityMs).toISOString();
+
+        try {
+          let claimedAttempt = 0;
+          await col.firestore.runTransaction(async (tx) => {
+            const fresh = await tx.get(doc.ref);
+            if (!fresh.exists) throw Object.assign(new Error('gone'), { code: 'gone' });
+            const d = fresh.data();
+            if (d.dead) throw Object.assign(new Error('dead'), { code: 'dead' });
+            if (d.leaseToken && d.leaseExpiresAt > new Date().toISOString()) {
+              throw Object.assign(new Error('taken'), { code: 'taken' });
+            }
+            claimedAttempt = (d.attempt ?? 0) + 1;
+            tx.update(doc.ref, { leaseToken, leaseExpiresAt, attempt: claimedAttempt });
+          });
+          leased.push({
+            id: data.id,
+            body: data.body,
+            enqueuedAt: data.enqueuedAt,
+            attempt: claimedAttempt,
+            leaseToken,
+            leaseExpiresAt,
+          });
+        } catch {
+          // Race condition — skip this message
+        }
+      }
+      return leased;
+    },
+
+    async ack(messageId, leaseToken) {
+      try {
+        await col.firestore.runTransaction(async (tx) => {
+          const ref = col.doc(messageId);
+          const snap = await tx.get(ref);
+          if (!snap.exists) return;
+          if (snap.data().leaseToken !== leaseToken) throw new Error('token mismatch');
+          tx.delete(ref);
+        });
+        return true;
+      } catch {
+        return false;
+      }
+    },
+
+    async nack(messageId, leaseToken, opts = {}) {
+      try {
+        await col.firestore.runTransaction(async (tx) => {
+          const ref = col.doc(messageId);
+          const snap = await tx.get(ref);
+          if (!snap.exists) return;
+          if (snap.data().leaseToken !== leaseToken) throw new Error('token mismatch');
+          if (opts.dead) {
+            tx.update(ref, {
+              dead: true,
+              deadReason: opts.reason ?? 'nacked',
+              leaseToken: null,
+              leaseExpiresAt: null,
+            });
+          } else {
+            tx.update(ref, {
+              leaseToken: null,
+              leaseExpiresAt: null,
+              visibleAfter: new Date().toISOString(),
+            });
+          }
+        });
+        return true;
+      } catch {
+        return false;
+      }
+    },
+
+    async peekActive(prefix) {
+      const snap = await col.where('dead', '==', false).orderBy('enqueuedAt').get();
+      return snap.docs
+        .map((d) => d.data())
+        .filter((d) => !prefix || String(d.id).startsWith(prefix))
+        .map((d) => ({ id: d.id, body: d.body, enqueuedAt: d.enqueuedAt }));
+    },
+
+    async peekDeadLetter(prefix) {
+      const snap = await col.where('dead', '==', true).orderBy('enqueuedAt').get();
+      return snap.docs
+        .map((d) => d.data())
+        .filter((d) => !prefix || String(d.id).startsWith(prefix))
+        .map((d) => ({ id: d.id, body: d.body, enqueuedAt: d.enqueuedAt, reason: d.deadReason }));
+    },
+  };
+}

package/examples/board-firestore/server/adapters/firestore-scratch-storage.js

@@ -0,0 +1,50 @@
+/**
+ * firestore-scratch-storage.js
+ *
+ * AsyncScratchStorage backed by Firestore.
+ * Extends AsyncBlobStorage with ephemeral-key creation and a config sub-map.
+ *
+ * @param {import('@google-cloud/firestore').CollectionReference} col
+ * @returns {import('yaml-flow/cloud-storage').AsyncScratchStorage}
+ */
+
+import { createFirestoreBlobStorage } from './firestore-blob-storage.js';
+
+function lexicalId() {
+  const ts = String(Date.now()).padStart(13, '0');
+  const rand = Math.random().toString(36).slice(2, 10).padEnd(8, '0');
+  return `${ts}-${rand}`;
+}
+
+export function createFirestoreScratchStorage(col) {
+  const blob = createFirestoreBlobStorage(col);
+
+  return {
+    ...blob,
+
+    async getUniqueKey(prefix = 'scratch-', suffix = '') {
+      return `${prefix}${lexicalId()}${suffix}`;
+    },
+
+    async create(data, prefix = 'scratch-', suffix = '') {
+      const key = `${prefix}${lexicalId()}${suffix}`;
+      await blob.write(key, data);
+      return key;
+    },
+
+    keyRef(key) {
+      return { kind: 'firestore-blob', value: `${col.path}/${Buffer.from(key).toString('base64url')}` };
+    },
+
+    config: {
+      async get(k) {
+        const content = await blob.read(`__config__/${k}`);
+        if (content == null) return null;
+        try { return JSON.parse(content); } catch { return content; }
+      },
+      async set(k, v) {
+        await blob.write(`__config__/${k}`, JSON.stringify(v));
+      },
+    },
+  };
+}