rete-graph 0.3.6 → 0.4.0

package/README.md

@@ -28,7 +28,7 @@ for (const row of g.query("SELECT ?s ?label WHERE { ?s rdfs:label ?label } LIMIT
 One self-contained file, engine included — via any npm CDN:
 
 ```html
-<script src="https://cdn.jsdelivr.net/npm/rete-graph@0.3.6/dist/rete-graph.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/rete-graph@0.4.0/dist/rete-graph.min.js"></script>
 <script>
   (async () => {
     const g = await rete.open(await rete.build("<urn:a> <urn:knows> <urn:b> ."));
@@ -49,7 +49,9 @@ g.stats(); // { fileLength, bytes, requests } — a LIMIT query fetches KBs–MB
 
 Remote opens use synchronous XHR range reads, so they work:
 
-- **in Node** (≥18) — out of the box, via a built-in sync-fetch bridge;
+- **in Node** (≥18) — out of the box, via a built-in sync-fetch bridge, which
+  also fetches the engine's batched reads in parallel (16 at once; set
+  `globalThis.reteReadMany = null` before the first `open()` to opt out);
 - **in browser web workers** — where sync binary XHR is allowed (this is how
   the playground runs its queries).
 
@@ -61,7 +63,8 @@ worker. The host serving the file must send CORS headers and honor `Range`.
 `query()` returns `{var: Term}` rows for SELECT, a boolean for ASK, and
 `[s, p, o]` Term triples for CONSTRUCT/DESCRIBE. A `Term` carries `.kind` /
 `.value` / `.datatype` / `.lang`, plus `.toJS()` (number/boolean/BigInt for
-common XSD types) and `.n3`. Also: `queryRaw`, `lastWarnings()` (why a FILTER silently dropped rows:
+common XSD types) and `.n3`. Also: `partitionPlan()` + `queryRaw(q, { partition })` (a long query as
+resumable chunks split by one variable), `queryRaw`, `lastWarnings()` (why a FILTER silently dropped rows:
 CONTAINS on an IRI, STRSTARTS on a number, … — see the docs), `query(q, {reason: true})`
 (OWL 2 QL entailment), `prefixSearch`, `textSearch`, `schema`, `graphNames`,
 `info`, `card()` / `examples()` (the file's embedded Dataset Card and its
@@ -164,4 +167,5 @@ npm install && npm test
 ```
 
 Releases are published to npm by `.github/workflows/js-client-publish.yml`
-on a `js-v*` tag. Docs: <https://caviri.github.io/rete/javascript.html>.
+on the `v*` tag that releases every rete channel.
+Docs: <https://caviri.github.io/rete/javascript.html>.

package/dist/index.js

@@ -1,4 +1,4 @@
-/*! rete-graph v0.3.6 | Apache-2.0 | https://github.com/caviri/rete */
+/*! rete-graph v0.4.0 | Apache-2.0 | https://github.com/caviri/rete */
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __esm = (fn, res) => function __init() {
@@ -16,14 +16,17 @@ __export(node_sync_xhr_exports, {
   install: () => install
 });
 import { MessageChannel, Worker, receiveMessageOnPort } from "node:worker_threads";
-function syncFetch(method, url, headers, body) {
+function fetchWorker() {
   if (!worker) {
     worker = new Worker(WORKER_SRC, { eval: true });
     worker.unref();
   }
+  return worker;
+}
+function syncFetch(method, url, headers, body) {
   const signal = new Int32Array(new SharedArrayBuffer(4));
   const { port1, port2 } = new MessageChannel();
-  worker.postMessage({ port: port2, signal, method, url, headers, body }, [port2]);
+  fetchWorker().postMessage({ port: port2, signal, method, url, headers, body }, [port2]);
   Atomics.wait(signal, 0, 0);
   const msg = receiveMessageOnPort(port1);
   port1.close();
@@ -31,12 +34,24 @@ function syncFetch(method, url, headers, body) {
   if (msg.message.error) throw new Error(`sync fetch failed: ${msg.message.error}`);
   return msg.message;
 }
+function readMany(url, offsets, lens) {
+  let total = 0;
+  for (const len of lens) total += len;
+  const data = new SharedArrayBuffer(total);
+  const signal = new Int32Array(new SharedArrayBuffer(4));
+  fetchWorker().postMessage({ op: "many", signal, url, offsets, lens, data, fanout: FANOUT });
+  Atomics.wait(signal, 0, 0);
+  return signal[0] === 1 ? new Uint8Array(data) : null;
+}
 function install() {
   if (typeof globalThis.XMLHttpRequest === "undefined") {
     globalThis.XMLHttpRequest = NodeSyncXMLHttpRequest;
   }
+  if (globalThis.XMLHttpRequest === NodeSyncXMLHttpRequest && globalThis.reteReadMany === void 0) {
+    globalThis.reteReadMany = readMany;
+  }
 }
-var WORKER_SRC, worker, NodeSyncXMLHttpRequest;
+var WORKER_SRC, FANOUT, worker, NodeSyncXMLHttpRequest;
 var init_node_sync_xhr = __esm({
   "src/node-sync-xhr.js"() {
     WORKER_SRC = `
@@ -88,7 +103,74 @@ function readFile(method, url, headers, result) {
   result.body = buf.buffer.slice(buf.byteOffset, buf.byteOffset + got);
 }
 
-parentPort.on("message", async ({ port, signal, method, url, headers, body }) => {
+// One batch for reteReadMany: every range read at once (at most \`fanout\` in
+// flight) and copied straight into the caller's shared buffer at its place in
+// the concatenation. A range that is not exactly what was asked for fails the
+// whole batch \u2014 the caller then returns null and the engine re-reads it one
+// range at a time, under its own validation.
+async function readMany({ url, offsets, lens, data, fanout }) {
+  const out = new Uint8Array(data);
+  const at = [];
+  let pos = 0;
+  for (const len of lens) {
+    at.push(pos);
+    pos += len;
+  }
+  if (url.startsWith("file:")) {
+    const fd = fdFor(fileURLToPath(url));
+    for (let i = 0; i < offsets.length; i++) {
+      let got = 0;
+      while (got < lens[i]) {
+        const n = fs.readSync(fd, out, at[i] + got, lens[i] - got, offsets[i] + got);
+        if (n <= 0) throw new Error("short read at offset " + offsets[i]);
+        got += n;
+      }
+    }
+    return;
+  }
+  let next = 0;
+  let failure = null;
+  // Each lane pulls the next range until the batch is done or one has failed;
+  // waiting for every lane (not just the first rejection) means nothing is
+  // still writing into the buffer once the caller is told the outcome.
+  const lane = async () => {
+    while (failure === null && next < offsets.length) {
+      const i = next++;
+      const offset = offsets[i];
+      const len = lens[i];
+      if (len === 0) continue;
+      try {
+        const resp = await fetch(url, { headers: { Range: "bytes=" + offset + "-" + (offset + len - 1) } });
+        const range = resp.headers.get("content-range") || "";
+        if (resp.status !== 206 || !range.startsWith("bytes " + offset + "-")) {
+          await resp.body?.cancel();
+          throw new Error("status " + resp.status + " (" + range + ") for offset " + offset);
+        }
+        const body = new Uint8Array(await resp.arrayBuffer());
+        if (body.length < len) throw new Error("short range at offset " + offset);
+        out.set(body.subarray(0, len), at[i]);
+      } catch (e) {
+        failure ??= e;
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(fanout, offsets.length) }, lane));
+  if (failure) throw failure;
+}
+
+parentPort.on("message", async (msg) => {
+  if (msg.op === "many") {
+    let outcome = 1;
+    try {
+      await readMany(msg);
+    } catch {
+      outcome = 2;
+    }
+    Atomics.store(msg.signal, 0, outcome);
+    Atomics.notify(msg.signal, 0);
+    return;
+  }
+  const { port, signal, method, url, headers, body } = msg;
   const result = { status: 0, headers: {}, body: null, error: null };
   try {
     if (url.startsWith("file:")) {
@@ -108,6 +190,7 @@ parentPort.on("message", async ({ port, signal, method, url, headers, body }) =>
   Atomics.notify(signal, 0);
 });
 `;
+    FANOUT = 16;
     worker = null;
     NodeSyncXMLHttpRequest = class {
       #method;
@@ -317,6 +400,38 @@ var Graph = class {
     GraphFinalization.register(this, this.__wbg_ptr, this);
     return this;
   }
+  /**
+   * Plan running `query` as chunks split by the values of `var` — see
+   * `rete_core::partition_plan`. JSON `{var, form, keys_query,
+   * keys_from_where, offset, limit, dedupe}`, or an error saying why the
+   * split would change the answer.
+   * @param {string} query
+   * @param {string} _var
+   * @returns {string}
+   */
+  partition_plan(query2, _var) {
+    let deferred4_0;
+    let deferred4_1;
+    try {
+      const ptr0 = passStringToWasm0(query2, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len0 = WASM_VECTOR_LEN;
+      const ptr1 = passStringToWasm0(_var, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len1 = WASM_VECTOR_LEN;
+      const ret = wasm.graph_partition_plan(this.__wbg_ptr, ptr0, len0, ptr1, len1);
+      var ptr3 = ret[0];
+      var len3 = ret[1];
+      if (ret[3]) {
+        ptr3 = 0;
+        len3 = 0;
+        throw takeFromExternrefTable0(ret[2]);
+      }
+      deferred4_0 = ptr3;
+      deferred4_1 = len3;
+      return getStringFromWasm0(ptr3, len3);
+    } finally {
+      wasm.__wbindgen_free(deferred4_0, deferred4_1, 1);
+    }
+  }
   /**
    * See [`prefix_search`].
    * @param {string} prefix
@@ -482,6 +597,44 @@ var Graph = class {
       wasm.__wbindgen_free(deferred4_0, deferred4_1, 1);
     }
   }
+  /**
+   * One chunk of a split query: the solutions whose `var` is one of `keys`
+   * (term tokens, as the plan's keys query returned them), without the
+   * query's top-level OFFSET/LIMIT. Same envelope as [`Graph::query`].
+   * @param {string} query
+   * @param {string} format
+   * @param {boolean} reason
+   * @param {string} _var
+   * @param {string[]} keys
+   * @returns {string}
+   */
+  query_partition(query2, format, reason2, _var, keys) {
+    let deferred6_0;
+    let deferred6_1;
+    try {
+      const ptr0 = passStringToWasm0(query2, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len0 = WASM_VECTOR_LEN;
+      const ptr1 = passStringToWasm0(format, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len1 = WASM_VECTOR_LEN;
+      const ptr2 = passStringToWasm0(_var, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len2 = WASM_VECTOR_LEN;
+      const ptr3 = passArrayJsValueToWasm0(keys, wasm.__wbindgen_malloc);
+      const len3 = WASM_VECTOR_LEN;
+      const ret = wasm.graph_query_partition(this.__wbg_ptr, ptr0, len0, ptr1, len1, reason2, ptr2, len2, ptr3, len3);
+      var ptr5 = ret[0];
+      var len5 = ret[1];
+      if (ret[3]) {
+        ptr5 = 0;
+        len5 = 0;
+        throw takeFromExternrefTable0(ret[2]);
+      }
+      deferred6_0 = ptr5;
+      deferred6_1 = len5;
+      return getStringFromWasm0(ptr5, len5);
+    } finally {
+      wasm.__wbindgen_free(deferred6_0, deferred6_1, 1);
+    }
+  }
   /**
    * As [`query`], with OWL 2 QL entailment on (`rdfs:subClassOf` /
    * `subPropertyOf` / `domain` / `range` reasoning by query rewriting).
@@ -1018,6 +1171,36 @@ var RemoteGraph = class {
     RemoteGraphFinalization.register(this, this.__wbg_ptr, this);
     return this;
   }
+  /**
+   * See [`Graph::partition_plan`]. Reads the dictionary entries of the
+   * query's constants and the resident tile directory — no tile data.
+   * @param {string} query
+   * @param {string} _var
+   * @returns {string}
+   */
+  partition_plan(query2, _var) {
+    let deferred4_0;
+    let deferred4_1;
+    try {
+      const ptr0 = passStringToWasm0(query2, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len0 = WASM_VECTOR_LEN;
+      const ptr1 = passStringToWasm0(_var, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len1 = WASM_VECTOR_LEN;
+      const ret = wasm.remotegraph_partition_plan(this.__wbg_ptr, ptr0, len0, ptr1, len1);
+      var ptr3 = ret[0];
+      var len3 = ret[1];
+      if (ret[3]) {
+        ptr3 = 0;
+        len3 = 0;
+        throw takeFromExternrefTable0(ret[2]);
+      }
+      deferred4_0 = ptr3;
+      deferred4_1 = len3;
+      return getStringFromWasm0(ptr3, len3);
+    } finally {
+      wasm.__wbindgen_free(deferred4_0, deferred4_1, 1);
+    }
+  }
   /**
    * See [`prefix_search`] — over the resident, cached remote handle. Faults the
    * pyramid (where the label index lives) on the first call, then serves the
@@ -1143,6 +1326,42 @@ var RemoteGraph = class {
       wasm.__wbindgen_free(deferred4_0, deferred4_1, 1);
     }
   }
+  /**
+   * See [`Graph::query_partition`].
+   * @param {string} query
+   * @param {string} format
+   * @param {boolean} reason
+   * @param {string} _var
+   * @param {string[]} keys
+   * @returns {string}
+   */
+  query_partition(query2, format, reason2, _var, keys) {
+    let deferred6_0;
+    let deferred6_1;
+    try {
+      const ptr0 = passStringToWasm0(query2, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len0 = WASM_VECTOR_LEN;
+      const ptr1 = passStringToWasm0(format, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len1 = WASM_VECTOR_LEN;
+      const ptr2 = passStringToWasm0(_var, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+      const len2 = WASM_VECTOR_LEN;
+      const ptr3 = passArrayJsValueToWasm0(keys, wasm.__wbindgen_malloc);
+      const len3 = WASM_VECTOR_LEN;
+      const ret = wasm.remotegraph_query_partition(this.__wbg_ptr, ptr0, len0, ptr1, len1, reason2, ptr2, len2, ptr3, len3);
+      var ptr5 = ret[0];
+      var len5 = ret[1];
+      if (ret[3]) {
+        ptr5 = 0;
+        len5 = 0;
+        throw takeFromExternrefTable0(ret[2]);
+      }
+      deferred6_0 = ptr5;
+      deferred6_1 = len5;
+      return getStringFromWasm0(ptr5, len5);
+    } finally {
+      wasm.__wbindgen_free(deferred6_0, deferred6_1, 1);
+    }
+  }
   /**
    * As [`query`], with OWL 2 QL entailment on (reason over the ontology while
    * reading only the bytes the rewritten query touches).
@@ -3193,13 +3412,29 @@ var Graph2 = class {
    * The engine's raw JSON result envelope. Carries a `warnings` array when the
    * query raised type errors (see {@link Graph#lastWarnings}).
    */
-  queryRaw(query2, { reason: reason2 = false } = {}) {
+  queryRaw(query2, { reason: reason2 = false, partition } = {}) {
     this.#warnings = [];
-    const s = reason2 ? this.#g.query_reasoned(query2, "json") : this.#g.query(query2, "json");
+    const s = partition ? this.#g.query_partition(query2, "json", reason2, partition.var, partition.keys) : reason2 ? this.#g.query_reasoned(query2, "json") : this.#g.query(query2, "json");
     const env = JSON.parse(s);
     this.#warnings = env.warnings ?? [];
     return env;
   }
+  /**
+   * Plan running `query` as chunks split by the values of `variable` (`?w`
+   * or `w`) — so a long extraction can go in short pieces, each retried or
+   * resumed on its own. Throws, saying why, when the chunks' answers together
+   * would not be the query's: ORDER BY, an aggregate not grouped by the
+   * variable, a variable some row leaves unbound, ASK, DESCRIBE.
+   *
+   * `{var, form, keys_query, keys_from_where, offset, limit, dedupe}`: run
+   * `keys_query` for the values, then each chunk of them with
+   * `queryRaw(query, {partition: {var, keys}})`. The chunks' rows in key
+   * order, deduplicated when `dedupe`, then sliced by `offset` / `limit`, are
+   * the query's rows.
+   */
+  partitionPlan(query2, variable) {
+    return JSON.parse(this.#g.partition_plan(query2, variable));
+  }
   /**
    * Diagnostics from the most recent `query()` / `queryRaw()` on this graph:
    * the SPARQL type errors a FILTER silently turned into `false` (CONTAINS on