dirsql 0.4.60__tar.gz → 0.4.61__tar.gz

{dirsql-0.4.60 → dirsql-0.4.61}/Cargo.lock

@@ -552,7 +552,7 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
 [[package]]
 name = "dirsql"
-version = "0.4.60"
+version = "0.4.61"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -585,7 +585,6 @@ dependencies = [
  "tokio-stream",
  "toml",
  "typed-path",
- "walkdir",
 ]
 
 [[package]]
@@ -601,7 +600,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.60"
+version = "0.4.61"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.60 → dirsql-0.4.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.60
+Version: 0.4.61
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.60/packages/rust → dirsql-0.4.61}/docs/reference/config.md

@@ -111,6 +111,7 @@ its `embed()` function exactly this way.
 | `command` | yes (non-empty) | The worker command. Argv-split with the same no-shell quoting rules as [command hooks](./hooks.md#argv-not-a-shell); runs in the config file's directory. |
 | `deterministic` | no (default `false`) | When `true`, the function is registered with `SQLITE_DETERMINISTIC`, letting SQLite cache and reuse results for identical arguments within a query. Only set it when the worker really is a pure function of its arguments. |
 | `timeout` | no | Per-**call** time bound: a positive integer is whole seconds (`timeout = 600`), a string is an integer suffixed `s` or `ms` (`"600s"`, `"250ms"`). When absent, the function mechanism's own 30-second default applies. |
+| `batch` | no | A positive integer: the most calls one **batched request** carries. When set, a statement's values are gathered and sent to the worker in bulk (see [Batching](#batching)); the worker must then speak the batched protocol shape as well as the single-call one. When absent, the worker only ever sees single calls. |
 
 ```toml
 [[dirsql.function]]
@@ -173,6 +174,42 @@ request line in, one response line out, per call:
 - **stderr passes through** to `dirsql`'s stderr, so a worker's progress
   bars and download logs reach the terminal.
 
+### Batching
+
+One round trip per row is the wrong shape for a worker whose cost is
+per-request rather than per-value — an embedding model encodes a list of
+texts in little more time than one. A function declaring `batch = N` is sent
+values in bulk instead:
+
+- **Request:** `{"calls": [[<arg>, ...], ...]}` — up to `N` calls, each
+  encoded exactly as a single call's argument list.
+- **Response:** `{"results": [<response>, ...]}` — one single-call response
+  (`{"ok": ...}` with its optional `"meta"`, or `{"err": ...}`) per call, in
+  order. A per-call `{"err": ...}` fails the query the way it would on a
+  single call. A top-level `{"err": "message"}` instead fails every call in
+  the request.
+- The round trip is bounded by the per-call `timeout` **times the number of
+  calls** in the request.
+
+A batching worker must still answer `{"call": [...]}`: `dirsql` gathers a
+statement's values by running it twice. The first run makes one ordinary
+round trip for the function's first value, binds that reply in place of every
+later distinct argument tuple, and queues those tuples into batched requests
+of at most `N`; the second run is served from the replies. A statement that
+never calls a batched function runs once, exactly as before. Values are
+deduplicated along the way, so the [progress line](./cli.md#progress-reporting)
+counts distinct calls, whichever request shape carried them.
+
+```toml
+[[dirsql.function]]
+name          = "embed"
+args          = [1, 2]
+command       = "dirsql-plugin-embeddings worker"
+deterministic = true
+timeout       = "600s"
+batch         = 256
+```
+
 ## `[[table]]`
 
 Each entry maps a glob pattern to a SQL table. A table's columns are exactly

{dirsql-0.4.60 → dirsql-0.4.61}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.60"
+version = "0.4.61"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

{dirsql-0.4.60 → dirsql-0.4.61/packages/python}/docs/reference/config.md

@@ -111,6 +111,7 @@ its `embed()` function exactly this way.
 | `command` | yes (non-empty) | The worker command. Argv-split with the same no-shell quoting rules as [command hooks](./hooks.md#argv-not-a-shell); runs in the config file's directory. |
 | `deterministic` | no (default `false`) | When `true`, the function is registered with `SQLITE_DETERMINISTIC`, letting SQLite cache and reuse results for identical arguments within a query. Only set it when the worker really is a pure function of its arguments. |
 | `timeout` | no | Per-**call** time bound: a positive integer is whole seconds (`timeout = 600`), a string is an integer suffixed `s` or `ms` (`"600s"`, `"250ms"`). When absent, the function mechanism's own 30-second default applies. |
+| `batch` | no | A positive integer: the most calls one **batched request** carries. When set, a statement's values are gathered and sent to the worker in bulk (see [Batching](#batching)); the worker must then speak the batched protocol shape as well as the single-call one. When absent, the worker only ever sees single calls. |
 
 ```toml
 [[dirsql.function]]
@@ -173,6 +174,42 @@ request line in, one response line out, per call:
 - **stderr passes through** to `dirsql`'s stderr, so a worker's progress
   bars and download logs reach the terminal.
 
+### Batching
+
+One round trip per row is the wrong shape for a worker whose cost is
+per-request rather than per-value — an embedding model encodes a list of
+texts in little more time than one. A function declaring `batch = N` is sent
+values in bulk instead:
+
+- **Request:** `{"calls": [[<arg>, ...], ...]}` — up to `N` calls, each
+  encoded exactly as a single call's argument list.
+- **Response:** `{"results": [<response>, ...]}` — one single-call response
+  (`{"ok": ...}` with its optional `"meta"`, or `{"err": ...}`) per call, in
+  order. A per-call `{"err": ...}` fails the query the way it would on a
+  single call. A top-level `{"err": "message"}` instead fails every call in
+  the request.
+- The round trip is bounded by the per-call `timeout` **times the number of
+  calls** in the request.
+
+A batching worker must still answer `{"call": [...]}`: `dirsql` gathers a
+statement's values by running it twice. The first run makes one ordinary
+round trip for the function's first value, binds that reply in place of every
+later distinct argument tuple, and queues those tuples into batched requests
+of at most `N`; the second run is served from the replies. A statement that
+never calls a batched function runs once, exactly as before. Values are
+deduplicated along the way, so the [progress line](./cli.md#progress-reporting)
+counts distinct calls, whichever request shape carried them.
+
+```toml
+[[dirsql.function]]
+name          = "embed"
+args          = [1, 2]
+command       = "dirsql-plugin-embeddings worker"
+deterministic = true
+timeout       = "600s"
+batch         = 256
+```
+
 ## `[[table]]`
 
 Each entry maps a glob pattern to a SQL table. A table's columns are exactly

dirsql-0.4.61/packages/python/e2e-attestations/claude-1201-single-walk.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1790945578,
+  "exit_code": 0,
+  "commit": "c74aacb34e2aa97f26338176180e7e063f83465b",
+  "branch": "claude/1201-single-walk"
+}

dirsql-0.4.61/packages/python/e2e-attestations/claude-1204-embed-batch.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1790942055,
+  "exit_code": 0,
+  "commit": "b9f0529e785dd54fa49496fb3fa48bbcdb14764d",
+  "branch": "claude/1204-embed-batch"
+}

{dirsql-0.4.60 → dirsql-0.4.61}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.60"
+version = "0.4.61"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly
@@ -69,7 +69,7 @@ globset = "0.4"
 # library consumer never pulls it.
 hex = { version = "0.4", optional = true }
 # Hierarchical `.gitignore` matching for path-table scans (the fd/ripgrep
-# semantics), without adopting its walker: the scan keeps `walkdir`.
+# semantics), without adopting its walker: the scan has its own.
 ignore = "0.4"
 indicatif = "0.18.6"
 notify = "7"
@@ -87,7 +87,6 @@ tokio.workspace = true
 tokio-stream = { version = "0.1", features = ["sync"], optional = true }
 toml = "0.8"
 typed-path = "0.12"
-walkdir = "2"
 
 [lints.rust]
 # `deny`, not `forbid`: rusqlite's API imposes five unavoidable `unsafe`

{dirsql-0.4.60/packages/python → dirsql-0.4.61/packages/rust}/docs/reference/config.md

@@ -111,6 +111,7 @@ its `embed()` function exactly this way.
 | `command` | yes (non-empty) | The worker command. Argv-split with the same no-shell quoting rules as [command hooks](./hooks.md#argv-not-a-shell); runs in the config file's directory. |
 | `deterministic` | no (default `false`) | When `true`, the function is registered with `SQLITE_DETERMINISTIC`, letting SQLite cache and reuse results for identical arguments within a query. Only set it when the worker really is a pure function of its arguments. |
 | `timeout` | no | Per-**call** time bound: a positive integer is whole seconds (`timeout = 600`), a string is an integer suffixed `s` or `ms` (`"600s"`, `"250ms"`). When absent, the function mechanism's own 30-second default applies. |
+| `batch` | no | A positive integer: the most calls one **batched request** carries. When set, a statement's values are gathered and sent to the worker in bulk (see [Batching](#batching)); the worker must then speak the batched protocol shape as well as the single-call one. When absent, the worker only ever sees single calls. |
 
 ```toml
 [[dirsql.function]]
@@ -173,6 +174,42 @@ request line in, one response line out, per call:
 - **stderr passes through** to `dirsql`'s stderr, so a worker's progress
   bars and download logs reach the terminal.
 
+### Batching
+
+One round trip per row is the wrong shape for a worker whose cost is
+per-request rather than per-value — an embedding model encodes a list of
+texts in little more time than one. A function declaring `batch = N` is sent
+values in bulk instead:
+
+- **Request:** `{"calls": [[<arg>, ...], ...]}` — up to `N` calls, each
+  encoded exactly as a single call's argument list.
+- **Response:** `{"results": [<response>, ...]}` — one single-call response
+  (`{"ok": ...}` with its optional `"meta"`, or `{"err": ...}`) per call, in
+  order. A per-call `{"err": ...}` fails the query the way it would on a
+  single call. A top-level `{"err": "message"}` instead fails every call in
+  the request.
+- The round trip is bounded by the per-call `timeout` **times the number of
+  calls** in the request.
+
+A batching worker must still answer `{"call": [...]}`: `dirsql` gathers a
+statement's values by running it twice. The first run makes one ordinary
+round trip for the function's first value, binds that reply in place of every
+later distinct argument tuple, and queues those tuples into batched requests
+of at most `N`; the second run is served from the replies. A statement that
+never calls a batched function runs once, exactly as before. Values are
+deduplicated along the way, so the [progress line](./cli.md#progress-reporting)
+counts distinct calls, whichever request shape carried them.
+
+```toml
+[[dirsql.function]]
+name          = "embed"
+args          = [1, 2]
+command       = "dirsql-plugin-embeddings worker"
+deterministic = true
+timeout       = "600s"
+batch         = 256
+```
+
 ## `[[table]]`
 
 Each entry maps a glob pattern to a SQL table. A table's columns are exactly

{dirsql-0.4.60 → dirsql-0.4.61}/packages/rust/src/config.rs

@@ -45,6 +45,9 @@ pub enum ConfigError {
     )]
     InvalidFunctionTimeout { name: String, value: String },
 
+    #[error("[[dirsql.function]] '{name}': 'batch' must be a positive integer, got {value}")]
+    InvalidFunctionBatch { name: String, value: String },
+
     #[error("Field '{0}' must not be empty")]
     EmptyField(&'static str),
 
@@ -212,6 +215,11 @@ pub struct FunctionSpec {
     /// Optional per-round-trip-call timeout, overriding the function
     /// mechanism's own 30-second default.
     pub timeout: Option<Duration>,
+    /// Optional cap on the calls one batched request carries. When set, a
+    /// statement's values are sent to the worker in bulk (see
+    /// `dirsql::functions`); when absent the worker only ever sees single
+    /// calls.
+    pub batch: Option<usize>,
 }
 
 /// Configuration for a single table.
@@ -267,6 +275,9 @@ struct RawFunction {
     command: Option<String>,
     deterministic: Option<bool>,
     timeout: Option<RawFunctionTimeout>,
+    // Any shape, so a non-integer hits the same actionable error as a
+    // non-positive one.
+    batch: Option<toml::Value>,
 }
 
 /// The `timeout` key accepts positive whole seconds (`timeout = 600`) or a
@@ -417,12 +428,27 @@ fn parse_function(raw: RawFunction) -> Result<FunctionSpec> {
         Some(raw_timeout) => Some(parse_function_timeout(&name, &raw_timeout)?),
     };
 
+    let batch = match raw.batch {
+        None => None,
+        Some(value) => Some(
+            value
+                .as_integer()
+                .and_then(|n| usize::try_from(n).ok())
+                .filter(|n| *n > 0)
+                .ok_or_else(|| ConfigError::InvalidFunctionBatch {
+                    name: name.clone(),
+                    value: value.to_string(),
+                })?,
+        ),
+    };
+
     Ok(FunctionSpec {
         name,
         args,
         command,
         deterministic: raw.deterministic.unwrap_or(false),
         timeout,
+        batch,
     })
 }
 
@@ -1162,6 +1188,7 @@ args = [1, 2]
 command = "dirsql-plugin-embeddings worker"
 deterministic = true
 timeout = "600s"
+batch = 64
 "#;
         let config = load_config_str(toml).unwrap();
         assert_eq!(
@@ -1172,6 +1199,7 @@ timeout = "600s"
                 command: "dirsql-plugin-embeddings worker".to_string(),
                 deterministic: true,
                 timeout: Some(Duration::from_secs(600)),
+                batch: Some(64),
             }]
         );
     }
@@ -1187,6 +1215,36 @@ command = "worker"
         let config = load_config_str(toml).unwrap();
         assert!(!config.functions[0].deterministic);
         assert!(config.functions[0].timeout.is_none());
+        assert!(config.functions[0].batch.is_none());
+    }
+
+    /// `batch = 0` would send empty requests, a negative cap means nothing,
+    /// and a string is not a count; each is the one actionable error.
+    #[test]
+    fn function_batch_rejects_anything_but_a_positive_integer() {
+        for value in ["0", "-4", "\"many\"", "2.5"] {
+            let toml = format!(
+                r#"
+[[dirsql.function]]
+name = "f"
+args = [1]
+command = "worker"
+batch = {value}
+"#
+            );
+            let err = load_config_str(&toml).unwrap_err();
+            match &err {
+                ConfigError::InvalidFunctionBatch { name, value: got } => {
+                    assert_eq!(name, "f");
+                    assert_eq!(got, value);
+                }
+                other => panic!("batch = {value}: expected InvalidFunctionBatch, got {other:?}"),
+            }
+            assert_eq!(
+                err.to_string(),
+                format!("[[dirsql.function]] 'f': 'batch' must be a positive integer, got {value}")
+            );
+        }
     }
 
     #[test]