taladb 0.8.3 → 0.9.0

package/dist/index.react-native.mjs

@@ -21,6 +21,103 @@ async function loadConfig(_configPath) {
   return {};
 }
 
+// src/sync.ts
+var CURSOR_COLLECTION = "__taladb_sync";
+async function resolveCollections(handle, options) {
+  const base = options.collections ?? await handle.listCollectionNames();
+  const excluded = new Set(options.exclude ?? []);
+  return base.filter((c) => !excluded.has(c) && !c.startsWith("_"));
+}
+function unsupportedSync(runtime) {
+  const err = () => new Error(
+    `TalaDB sync is not yet available on the ${runtime} runtime (Node.js is supported today; browser and React Native are in progress). Track it on the roadmap.`
+  );
+  return {
+    sync: () => Promise.reject(err()),
+    exportChanges: () => Promise.reject(err()),
+    importChanges: () => Promise.reject(err())
+  };
+}
+async function readCursor(cursorCol, target) {
+  const doc = await cursorCol.findOne({ target });
+  return { pushMs: doc?.pushMs ?? 0, pullMs: doc?.pullMs ?? 0 };
+}
+async function writeCursor(cursorCol, target, cursor) {
+  const updated = await cursorCol.updateOne({ target }, { $set: { ...cursor } });
+  if (!updated) {
+    await cursorCol.insert({ target, ...cursor });
+  }
+}
+async function runSync(handle, adapter, options) {
+  const direction = options.direction ?? "both";
+  const target = options.target ?? "default";
+  const doPush = direction === "push" || direction === "both";
+  const doPull = direction === "pull" || direction === "both";
+  if (doPull && !adapter.pull) {
+    throw new Error(`sync direction '${direction}' requires adapter.pull()`);
+  }
+  if (doPush && !adapter.push) {
+    throw new Error(`sync direction '${direction}' requires adapter.push()`);
+  }
+  const collections = await resolveCollections(handle, options);
+  const cursorCol = handle.collection(CURSOR_COLLECTION);
+  const cursor = await readCursor(cursorCol, target);
+  const local = doPush ? await handle.exportChanges(collections, 0) : "[]";
+  let pulled = 0;
+  if (doPull) {
+    const remote = await adapter.pull(0);
+    if (remote && remote !== "[]") {
+      pulled = await handle.importChanges(remote);
+    }
+  }
+  let pushed = 0;
+  if (doPush && local !== "[]") {
+    pushed = JSON.parse(local).length;
+    await adapter.push(local);
+  }
+  await writeCursor(cursorCol, target, {
+    pushMs: cursor.pushMs,
+    pullMs: cursor.pullMs
+  });
+  return { pushed, pulled, cursor: 0 };
+}
+
+// src/http-adapter.ts
+var HttpSyncAdapter = class {
+  constructor(options) {
+    this.endpoint = options.endpoint.replace(/\/$/, "");
+    this.headers = options.headers ?? {};
+    const f = options.fetch ?? globalThis.fetch?.bind(globalThis);
+    if (!f) {
+      throw new Error(
+        "HttpSyncAdapter: no fetch available. Pass options.fetch on runtimes without a global fetch."
+      );
+    }
+    this.fetchFn = f;
+    this.pushPath = options.paths?.push ?? "/push";
+    this.pullPath = options.paths?.pull ?? "/pull";
+  }
+  async push(changeset) {
+    const res = await this.fetchFn(`${this.endpoint}${this.pushPath}`, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...this.headers },
+      body: changeset
+    });
+    if (!res.ok) {
+      throw new Error(`HttpSyncAdapter push failed: ${res.status} ${res.statusText}`);
+    }
+  }
+  async pull(sinceMs) {
+    const url = `${this.endpoint}${this.pullPath}?since=${encodeURIComponent(String(sinceMs))}`;
+    const res = await this.fetchFn(url, { method: "GET", headers: this.headers });
+    if (!res.ok) {
+      throw new Error(`HttpSyncAdapter pull failed: ${res.status} ${res.statusText}`);
+    }
+    const body = (await res.text()).trim();
+    return body.length === 0 ? "[]" : body;
+  }
+};
+
 // src/index.ts
 var TalaDbValidationError = class extends Error {
   constructor(cause, context) {
@@ -116,28 +213,44 @@ var WorkerProxy = class {
     this.pending.clear();
   }
 };
-function makePoller(findFn, callback) {
+function makePoller(findFn, callback, onError) {
   let active = true;
   let lastJson = "";
+  let running = false;
+  let rerun = false;
   const poll = async () => {
     if (!active) return;
+    if (running) {
+      rerun = true;
+      return;
+    }
+    running = true;
     try {
       const docs = await findFn();
+      if (!active) return;
       const json = JSON.stringify(docs);
       if (json !== lastJson) {
         lastJson = json;
         callback(docs);
       }
-    } catch {
+    } catch (error) {
+      if (active) onError?.(error);
+    } finally {
+      running = false;
+      if (active) {
+        if (rerun) {
+          rerun = false;
+          void poll();
+        } else setTimeout(poll, 300);
+      }
     }
-    if (active) setTimeout(poll, 300);
   };
   poll();
   return () => {
     active = false;
   };
 }
-async function createBrowserDB(dbName, config) {
+async function createBrowserDB(dbName, config, passphrase) {
   const workerUrl = new URL("@taladb/web/worker/taladb.worker.js", "react-native://unreachable");
   const worker = new Worker(workerUrl, { type: "module", name: "taladb" });
   const proxy = new WorkerProxy(worker);
@@ -145,7 +258,13 @@ async function createBrowserDB(dbName, config) {
     proxy.abort(new Error(`taladb worker error: ${e.message ?? "unknown"}`));
   };
   const configJson = config !== void 0 ? JSON.stringify(config) : void 0;
-  await proxy.send("init", { dbName, configJson });
+  try {
+    await proxy.send("init", { dbName, configJson, passphrase });
+  } catch (e) {
+    proxy.abort(e instanceof Error ? e : new Error(String(e)));
+    worker.terminate();
+    throw e;
+  }
   const nudgeCallbacks = /* @__PURE__ */ new Set();
   let channel = null;
   if (typeof BroadcastChannel !== "undefined") {
@@ -197,8 +316,17 @@ async function createBrowserDB(dbName, config) {
         collection: name,
         filterJson: filter ? s(filter) : "null"
       }),
+      aggregate: async (pipeline) => {
+        const json = await proxy.send("aggregate", {
+          collection: name,
+          pipelineJson: s(pipeline)
+        });
+        return JSON.parse(json);
+      },
       createIndex: (field) => proxy.send("createIndex", { collection: name, field }),
       dropIndex: (field) => proxy.send("dropIndex", { collection: name, field }),
+      createCompoundIndex: (fields) => proxy.send("createCompoundIndex", { collection: name, fieldsJson: JSON.stringify(fields) }),
+      dropCompoundIndex: (fields) => proxy.send("dropCompoundIndex", { collection: name, fieldsJson: JSON.stringify(fields) }),
       createFtsIndex: (field) => proxy.send("createFtsIndex", { collection: name, field }),
       dropFtsIndex: (field) => proxy.send("dropFtsIndex", { collection: name, field }),
       createVectorIndex: (field, options) => {
@@ -229,12 +357,19 @@ async function createBrowserDB(dbName, config) {
         });
         return JSON.parse(json);
       },
-      subscribe: (filter, callback) => {
+      subscribe: (filter, callback, onError) => {
         let active = true;
         let lastJson = "";
         let timer = null;
+        let running = false;
+        let rerun = false;
         const poll = async () => {
           if (!active) return;
+          if (running) {
+            rerun = true;
+            return;
+          }
+          running = true;
           if (timer !== null) {
             clearTimeout(timer);
             timer = null;
@@ -244,13 +379,22 @@ async function createBrowserDB(dbName, config) {
               collection: name,
               filterJson: filter ? s(filter) : "null"
             });
+            if (!active) return;
             if (json !== lastJson) {
               lastJson = json;
               callback(JSON.parse(json));
             }
-          } catch {
+          } catch (error) {
+            if (active) onError?.(error);
+          } finally {
+            running = false;
+          }
+          if (active) {
+            if (rerun) {
+              rerun = false;
+              void poll();
+            } else timer = setTimeout(poll, 300);
           }
-          if (active) timer = setTimeout(poll, 300);
         };
         nudgeCallbacks.add(poll);
         poll();
@@ -266,9 +410,11 @@ async function createBrowserDB(dbName, config) {
     };
     return opts ? applySchema(wrapped, opts) : wrapped;
   }
-  return {
+  const handle = {
     collection: (name, opts) => wrapCollection(name, opts),
     compact: () => proxy.send("compact"),
+    syncStatus: async () => JSON.parse(await proxy.send("syncStatus")),
+    flushSync: (timeoutMs = 5e3) => proxy.send("flushSync", { timeoutMs }),
     close: async () => {
       channel?.close();
       try {
@@ -277,13 +423,22 @@ async function createBrowserDB(dbName, config) {
         worker.terminate();
         proxy.abort(new Error("taladb worker closed"));
       }
-    }
+    },
+    // All engine work (export scan, LWW merge) runs inside the worker, off the
+    // main thread — a sync pass never blocks rendering, whatever its size.
+    exportChanges: (collections, sinceMs) => proxy.send("exportChangeset", { collectionsJson: JSON.stringify(collections), sinceMs }),
+    importChanges: (changeset) => proxy.send("importChangeset", { changesetJson: changeset }),
+    listCollectionNames: async () => JSON.parse(await proxy.send("listCollections")),
+    sync: (adapter, options) => runSync(handle, adapter, options)
   };
+  return handle;
 }
-async function createNodeDB(dbName, config) {
-  const { TalaDBNode } = await import("./node-A4LKRSW5.mjs");
+async function createNodeDB(dbName, config, passphrase) {
+  const native = await import("./node-A4LKRSW5.mjs");
+  const TalaDBNode = native.TalaDbNode ?? native.TalaDBNode;
+  if (!TalaDBNode) throw new Error("@taladb/node loaded but exports no TalaDbNode class \u2014 rebuild the native module");
   const configJson = config !== void 0 ? JSON.stringify(config) : null;
-  const db = TalaDBNode.open(dbName, configJson);
+  const db = TalaDBNode.open(dbName, configJson, passphrase ?? null);
   function wrapCollection(name, opts) {
     const col = db.collection(name);
     const wrapped = {
@@ -296,8 +451,11 @@ async function createNodeDB(dbName, config) {
       deleteOne: async (filter) => col.deleteOneAsync ? col.deleteOneAsync(filter) : col.deleteOne(filter),
       deleteMany: async (filter) => col.deleteManyAsync ? col.deleteManyAsync(filter) : col.deleteMany(filter),
       count: async (filter) => col.count(filter ?? null),
+      aggregate: async (pipeline) => col.aggregate(pipeline),
       createIndex: async (field) => col.createIndex(field),
       dropIndex: async (field) => col.dropIndex(field),
+      createCompoundIndex: async (fields) => col.createCompoundIndex(fields),
+      dropCompoundIndex: async (fields) => col.dropCompoundIndex(fields),
       createFtsIndex: async (field) => col.createFtsIndex(field),
       dropFtsIndex: async (field) => col.dropFtsIndex(field),
       createVectorIndex: async (field, options) => col.createVectorIndex(field, options.dimensions, options.metric ?? null, options.indexType ?? null, options.hnswM ?? null, options.hnswEfConstruction ?? null),
@@ -311,16 +469,21 @@ async function createNodeDB(dbName, config) {
         const raw = await col.findNearest(field, vector, topK, filter ?? null);
         return raw;
       },
-      subscribe: (filter, callback) => makePoller(async () => col.find(filter ?? null), callback)
+      subscribe: (filter, callback, onError) => makePoller(async () => col.find(filter ?? null), callback, onError)
     };
     return opts ? applySchema(wrapped, opts) : wrapped;
   }
-  return {
+  const handle = {
     collection: (name, opts) => wrapCollection(name, opts),
     compact: async () => db.compact(),
     // Releases the native file handle/lock (no-op on older .node binaries).
-    close: async () => db.close?.()
+    close: async () => db.close?.(),
+    exportChanges: async (collections, sinceMs) => db.exportChanges(sinceMs, collections),
+    importChanges: async (changeset) => db.importChanges(changeset),
+    listCollectionNames: async () => db.listCollectionNames(),
+    sync: (adapter, options) => runSync(handle, adapter, options)
   };
+  return handle;
 }
 async function createNativeDB(_dbName) {
   const maybeNative = globalThis.__TalaDB__;
@@ -341,8 +504,11 @@ async function createNativeDB(_dbName) {
       deleteOne: async (filter) => native.deleteOne(name, filter),
       deleteMany: async (filter) => native.deleteMany(name, filter),
       count: async (filter) => native.count(name, filter ?? {}),
+      aggregate: async (pipeline) => native.aggregate(name, pipeline),
       createIndex: async (field) => native.createIndex(name, field),
       dropIndex: async (field) => native.dropIndex(name, field),
+      createCompoundIndex: async (fields) => native.createCompoundIndex(name, fields),
+      dropCompoundIndex: async (fields) => native.dropCompoundIndex(name, fields),
       createFtsIndex: async (field) => native.createFtsIndex(name, field),
       dropFtsIndex: async (field) => native.dropFtsIndex(name, field),
       createVectorIndex: async (field, options) => {
@@ -362,17 +528,35 @@ async function createNativeDB(_dbName) {
         const raw = native.findNearest(name, field, vector, topK, filter ?? null);
         return raw;
       },
-      subscribe: (filter, callback) => makePoller(async () => native.find(name, filter ?? {}), callback)
+      subscribe: (filter, callback, onError) => makePoller(async () => native.find(name, filter ?? {}), callback, onError)
     };
     return opts ? applySchema(wrapped, opts) : wrapped;
   }
+  const syncSurface = typeof native.exportChanges === "function" && typeof native.importChanges === "function" && typeof native.listCollectionNames === "function" ? (() => {
+    const handle = {
+      collection: (name, opts) => wrapCollection(name, opts),
+      exportChanges: async (collections, sinceMs) => native.exportChanges(collections, sinceMs),
+      importChanges: async (changeset) => native.importChanges(changeset),
+      listCollectionNames: async () => native.listCollectionNames(),
+      sync: (adapter, options) => runSync(handle, adapter, options)
+    };
+    return {
+      exportChanges: handle.exportChanges,
+      importChanges: handle.importChanges,
+      sync: handle.sync
+    };
+  })() : unsupportedSync("react-native");
   return {
     collection: (name, opts) => wrapCollection(name, opts),
     compact: async () => native.compact(),
-    close: async () => native.close()
+    close: async () => native.close(),
+    ...syncSurface
   };
 }
 async function openDB(dbName = "taladb.db", options) {
+  if (options?.passphrase !== void 0 && options.passphrase.length === 0) {
+    throw new Error("TalaDB encryption passphrase must not be empty");
+  }
   let resolvedConfig;
   if (options?.config !== void 0) {
     validateConfig(options.config);
@@ -383,14 +567,18 @@ async function openDB(dbName = "taladb.db", options) {
   const platform = detectPlatform();
   switch (platform) {
     case "browser":
-      return createBrowserDB(dbName, resolvedConfig);
+      return createBrowserDB(dbName, resolvedConfig, options?.passphrase);
     case "react-native":
+      if (options?.passphrase !== void 0) {
+        throw new Error("On React Native, pass the passphrase in the config JSON to TalaDBModule.initialize(); refusing to assume the already-open native database is encrypted");
+      }
       return createNativeDB(dbName);
     case "node":
-      return createNodeDB(dbName, resolvedConfig);
+      return createNodeDB(dbName, resolvedConfig, options?.passphrase);
   }
 }
 export {
+  HttpSyncAdapter,
   TalaDbValidationError,
   openDB
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "taladb",
-  "version": "0.8.3",
+  "version": "0.9.0",
   "description": "Local-first document and vector database for React, React Native, and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -42,7 +42,7 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/thinkgrid-labs/taladb.git",
-    "directory": "packages/taladb"
+    "directory": "packages/clients/taladb"
   },
   "homepage": "https://thinkgrid-labs.github.io/taladb/",
   "bugs": {