dirsql 0.4.101__tar.gz → 0.4.102__tar.gz

{dirsql-0.4.101 → dirsql-0.4.102}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.101"
+version = "0.4.102"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.101"
+version = "0.4.102"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.101 → dirsql-0.4.102}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.101
+Version: 0.4.102
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.101/packages/rust → dirsql-0.4.102}/docs/reference/hooks.md

@@ -161,7 +161,9 @@ on-file command `python3 extract.py {path}` uses `{path}`, but an on-file comman
 Every platform caps the bytes one process may receive as arguments. When a
 table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
-the rows it prints, in order. The command cannot tell: each run is an
+the rows it prints. Runs execute concurrently, one per CPU at most, like
+`xargs -P`, so rows keep their order within a run but not across runs; SQL
+never promised row order, so `ORDER BY` what you need. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

{dirsql-0.4.101 → dirsql-0.4.102}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.101"
+version = "0.4.102"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.102/packages/python/changelog.d/2026-10-07-parallel-hooks.md

@@ -0,0 +1 @@
+**Changed** An `on-file` command's runs now execute concurrently, one per CPU at most, when a table's paths outgrow one argument list, and each run's output is parsed as it is written instead of after it exits. Rows keep their order within a run but not across runs. A 2M-row JSONL ingest through `jq` runs within 5% of the equivalent `jq | awk` pipeline, and a frontmatter parse over thousands of files runs in a quarter of the time of `find -exec awk +`. The scratch database no longer syncs to disk.

{dirsql-0.4.101 → dirsql-0.4.102/packages/python}/docs/reference/hooks.md

@@ -161,7 +161,9 @@ on-file command `python3 extract.py {path}` uses `{path}`, but an on-file comman
 Every platform caps the bytes one process may receive as arguments. When a
 table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
-the rows it prints, in order. The command cannot tell: each run is an
+the rows it prints. Runs execute concurrently, one per CPU at most, like
+`xargs -P`, so rows keep their order within a run but not across runs; SQL
+never promised row order, so `ORDER BY` what you need. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

dirsql-0.4.102/packages/python/e2e-attestations/bot-1429-parallel-hooks.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/declared_table_name_test.py tests/e2e/fanout_test.py tests/e2e/extension_package_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/plugin_extension_discovery_test.py tests/e2e/on_file_query_test.py tests/e2e/sigint_interrupt_test.py tests/e2e/use_case_notebook_test.py tests/e2e/server_speed_test.py tests/e2e/query_contract_test.py tests/e2e/plugin_discovery_test.py tests/e2e/use_case_transcripts_test.py  -q",
+  "ran_at": 1791407652,
+  "exit_code": 0,
+  "commit": "8d6801314e86177dc46926aedb94bf63e47e87bb",
+  "branch": "bot/1429-parallel-hooks"
+}

dirsql-0.4.102/packages/python/migrations.d/2026-10-07-parallel-hooks.md

@@ -0,0 +1,41 @@
+### `on-file` runs execute concurrently
+
+#### Summary
+
+When a table's matched paths outgrow one argument list, dirsql splits them
+into runs and now executes the runs concurrently, one per CPU at most. Rows
+keep their order within a run, but not across runs.
+
+#### Required changes
+
+_None._
+
+#### Deprecations removed
+
+_None._
+
+#### Behavior changes without code changes
+
+- A query that relied on rows coming back in path order across runs must add
+  an `ORDER BY`. A table small enough for one run is unchanged.
+- A hook that holds memory or a GPU may now have several copies running at
+  once on a very large table.
+
+#### Verification
+
+```bash
+cd "$(mktemp -d)"
+for i in $(seq 1 5000); do : > "a-long-file-name-to-fill-the-argument-list-$i.txt"; done
+cat > count.sh <<'SH'
+echo "{\"n\":$#}"
+SH
+cat > .dirsql.toml <<'TOML'
+[[table]]
+name    = "t"
+ddl     = "CREATE TABLE t (n INTEGER)"
+glob    = "*.txt"
+on-file = "sh count.sh"
+TOML
+dirsql query "SELECT count(*) AS runs, sum(n) AS paths FROM t" -c .dirsql.toml
+# expected: one row whose paths is 5000 and whose runs is greater than 1
+```

{dirsql-0.4.101 → dirsql-0.4.102}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.101"
+version = "0.4.102"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.101/packages/python → dirsql-0.4.102/packages/rust}/docs/reference/hooks.md

@@ -161,7 +161,9 @@ on-file command `python3 extract.py {path}` uses `{path}`, but an on-file comman
 Every platform caps the bytes one process may receive as arguments. When a
 table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
-the rows it prints, in order. The command cannot tell: each run is an
+the rows it prints. Runs execute concurrently, one per CPU at most, like
+`xargs -P`, so rows keep their order within a run but not across runs; SQL
+never promised row order, so `ORDER BY` what you need. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

{dirsql-0.4.101 → dirsql-0.4.102}/packages/rust/src/command.rs

@@ -256,21 +256,85 @@ pub(crate) fn run_argv_stdout(
     let stdout = stdout_thread.join().unwrap_or_default();
     let stderr = stderr_thread.join().unwrap_or_default();
 
-    if !status.success() {
-        let code = status
-            .code()
-            .map(|c| c.to_string())
-            .unwrap_or_else(|| "signal".to_string());
-        return Err(CommandError::NonZeroExit {
-            command: command.to_string(),
-            code,
-            stderr_tail: stderr_tail(&stderr),
-        });
-    }
+    check_exit(command, status, &stderr)?;
 
     Ok(String::from_utf8_lossy(&stdout).into_owned())
 }
 
+fn check_exit(
+    command: &str,
+    status: std::process::ExitStatus,
+    stderr: &[u8],
+) -> Result<(), CommandError> {
+    if status.success() {
+        return Ok(());
+    }
+    let code = status
+        .code()
+        .map(|c| c.to_string())
+        .unwrap_or_else(|| "signal".to_string());
+    Err(CommandError::NonZeroExit {
+        command: command.to_string(),
+        code,
+        stderr_tail: stderr_tail(stderr),
+    })
+}
+
+/// Bytes read from a child's stdout per `read` call.
+const READ_BYTES: usize = 262_144;
+
+/// Run an already-built `argv` and hand its stdout to `on_block` as it
+/// arrives, in blocks that each hold whole lines (the last block may lack its
+/// trailing newline). The child runs with stdin closed. A non-zero exit is an
+/// error after every block has been delivered.
+pub(crate) fn run_argv_blocks(
+    command: &str,
+    argv: &[String],
+    cwd: &Path,
+    on_block: &mut dyn FnMut(&str),
+) -> Result<(), CommandError> {
+    let mut cmd = Command::new(&argv[0]);
+    push_args(&mut cmd, &argv[1..]);
+    cmd.current_dir(cwd)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .stdin(Stdio::null());
+    let mut child = cmd
+        .spawn()
+        .map_err(|source| spawn_error(&argv[0], source, cwd))?;
+
+    let mut stderr_pipe = child.stderr.take().expect("stderr piped");
+    let stderr_thread = std::thread::spawn(move || {
+        let mut buf = Vec::new();
+        let _ = stderr_pipe.read_to_end(&mut buf);
+        buf
+    });
+    let mut stdout_pipe = child.stdout.take().expect("stdout piped");
+    let mut pending: Vec<u8> = Vec::new();
+    let mut buf = vec![0u8; READ_BYTES];
+    loop {
+        let n = match stdout_pipe.read(&mut buf) {
+            Ok(0) | Err(_) => break,
+            Ok(n) => n,
+        };
+        pending.extend_from_slice(&buf[..n]);
+        if let Some(end) = pending.iter().rposition(|&b| b == b'\n') {
+            on_block(&String::from_utf8_lossy(&pending[..=end]));
+            pending.drain(..=end);
+        }
+    }
+    if !pending.is_empty() {
+        on_block(&String::from_utf8_lossy(&pending));
+    }
+
+    let status = child.wait().map_err(|source| CommandError::Io {
+        command: command.to_string(),
+        source,
+    })?;
+    let stderr = stderr_thread.join().unwrap_or_default();
+    check_exit(command, status, &stderr)
+}
+
 // One fn with cfg blocks rather than two cfg'd fns: cargo-mutants mutates
 // the uncompiled twin too, and no test on a Linux gate can kill that mutant.
 fn push_args(cmd: &mut Command, args: &[String]) {
@@ -736,6 +800,75 @@ mod tests {
         );
     }
 
+    #[test]
+    fn run_argv_blocks_delivers_whole_lines_and_the_unterminated_tail() {
+        let argv = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "printf 'a\\nb\\nc'".to_string(),
+        ];
+        let mut seen = String::new();
+        let mut blocks = 0;
+        run_argv_blocks("t", &argv, &cwd(), &mut |block| {
+            seen.push_str(block);
+            blocks += 1;
+        })
+        .unwrap();
+        assert_eq!(seen, "a\nb\nc");
+        assert!(blocks >= 1);
+    }
+
+    #[test]
+    fn run_argv_blocks_cuts_every_block_at_a_line_boundary() {
+        let argv = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "yes abcdefghijklmnopqrstuvwxyz | head -n 100000".to_string(),
+        ];
+        let mut lines = 0;
+        let mut blocks = 0;
+        run_argv_blocks("t", &argv, &cwd(), &mut |block| {
+            blocks += 1;
+            assert!(block.ends_with('\n'), "block cut mid-line");
+            lines += block.lines().count();
+        })
+        .unwrap();
+        assert_eq!(lines, 100_000);
+        assert!(blocks > 1, "2.7 MB arrives in more than one read");
+    }
+
+    #[test]
+    fn run_argv_blocks_reports_a_non_zero_exit_after_delivering_output() {
+        let argv = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "echo partial; echo oops >&2; exit 3".to_string(),
+        ];
+        let mut seen = String::new();
+        let err =
+            run_argv_blocks("t", &argv, &cwd(), &mut |block| seen.push_str(block)).unwrap_err();
+        assert_eq!(seen, "partial\n");
+        match err {
+            CommandError::NonZeroExit {
+                code, stderr_tail, ..
+            } => {
+                assert_eq!(code, "3");
+                assert!(stderr_tail.contains("oops"));
+            }
+            other => panic!("expected NonZeroExit, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn run_argv_blocks_names_a_missing_program() {
+        let argv = vec!["definitely-not-a-real-binary-xyzzy".to_string()];
+        let err = run_argv_blocks("t", &argv, &cwd(), &mut |_| {}).unwrap_err();
+        assert!(
+            matches!(err, CommandError::NotOnPath { .. }),
+            "got: {err:?}"
+        );
+    }
+
     #[test]
     fn run_command_writes_stdin_payload_to_the_child() {
         let out = run_command("cat", &[], &cwd(), Some(b"hello-stdin")).unwrap();

{dirsql-0.4.101 → dirsql-0.4.102}/packages/rust/src/db.rs

@@ -368,6 +368,9 @@ impl Db {
     /// tmpfs mount.
     pub fn new() -> Result<Self> {
         let conn = Connection::open("")?;
+        // The database is a private temp file that dies with the process, so
+        // durability buys nothing.
+        conn.pragma_update(None, "synchronous", "OFF")?;
         ensure_internal_rows_table(&conn)?;
         let scope = StatementScope::new();
         vtab::load_module(&conn, Arc::clone(&scope))?;
@@ -2066,6 +2069,17 @@ mod tests {
         assert_eq!(mode, "delete");
     }
 
+    #[test]
+    fn new_skips_fsync() {
+        let db = Db::new().unwrap();
+        let pragma = |name: &str| -> i64 {
+            db.conn()
+                .query_row(&format!("PRAGMA {name}"), [], |row| row.get(0))
+                .unwrap()
+        };
+        assert_eq!(pragma("synchronous"), 0);
+    }
+
     #[test]
     fn get_rows_by_file_returns_rows_in_row_index_order() {
         let db = Db::new().unwrap();

{dirsql-0.4.101 → dirsql-0.4.102}/packages/rust/src/infer.rs

@@ -110,13 +110,19 @@ impl<'de> serde::Deserialize<'de> for JsonRow {
 /// Parse a parser command's output — one JSON object per line (NDJSON), blank
 /// lines skipped — into rows that remember their key order.
 pub fn parse_rows(output: &str) -> Result<Vec<JsonRow>, String> {
+    parse_rows_at(output, 0)
+}
+
+/// [`parse_rows`] for a block that begins after `skipped` earlier lines of the
+/// same output, so an error names its line in the whole output.
+pub(crate) fn parse_rows_at(output: &str, skipped: usize) -> Result<Vec<JsonRow>, String> {
     let mut rows = Vec::new();
     for (index, line) in output.lines().enumerate() {
         let line = line.trim();
         if line.is_empty() {
             continue;
         }
-        let number = index + 1;
+        let number = skipped + index + 1;
         if line.starts_with('[') {
             return Err(format!(
                 "line {number} is a JSON array; print one JSON object per line instead"
@@ -255,6 +261,12 @@ mod tests {
         assert_eq!(SqlType::Real.as_str(), "REAL");
     }
 
+    #[test]
+    fn parse_rows_at_numbers_lines_after_the_skipped_ones() {
+        let err = parse_rows_at("{\"a\":1}\nnope", 40).unwrap_err();
+        assert!(err.starts_with("line 42 "), "got: {err}");
+    }
+
     #[test]
     fn parse_rows_preserves_key_order_rather_than_sorting() {
         let parsed = parse_rows(r#"{"zeta":1,"alpha":2}"#).unwrap();

{dirsql-0.4.101 → dirsql-0.4.102}/packages/rust/src/on_file.rs

@@ -9,9 +9,11 @@
 //! fails the whole table.
 
 use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
 use crate::command::{self, Placeholder};
-use crate::infer::{JsonRow, parse_rows};
+use crate::infer::{JsonRow, parse_rows_at};
 
 /// Bytes of trailing arguments one invocation may carry: GNU xargs's ceiling,
 /// comfortably under every platform's argv limit with the environment and the
@@ -52,9 +54,9 @@ pub(crate) fn run(
     Ok(rows)
 }
 
-/// [`run`], handing each invocation's rows to `sink` as they parse. The
-/// invocations still run one after another; parsing one invocation's output
-/// overlaps the next invocation, the way a shell pipe would.
+/// [`run`], handing rows to `sink` as they parse. Invocations run
+/// concurrently, like `xargs -P`, each decoding its own output as it arrives;
+/// rows keep their order within an invocation but not across invocations.
 pub(crate) fn run_streaming(
     command: &str,
     cwd: &Path,
@@ -71,35 +73,89 @@ pub(crate) fn run_streaming(
         .iter()
         .map(|path| command::non_verbatim(&path.to_string_lossy()))
         .collect();
+    let runs = chunks(&args, ARG_BUDGET.saturating_sub(argv_bytes(&argv)));
+    let workers = std::thread::available_parallelism()
+        .map_or(1, std::num::NonZero::get)
+        .min(runs.len());
 
+    let next = AtomicUsize::new(0);
+    let aborted = AtomicBool::new(false);
+    let failures = Mutex::new(Vec::<(usize, String)>::new());
     std::thread::scope(|scope| {
-        let (payloads, received) = std::sync::mpsc::channel::<String>();
-        let parser = scope.spawn(move || -> Result<(), String> {
-            for payload in received {
-                let rows = parse_rows(&payload).map_err(|message| {
-                    format!("on-file output was not one JSON object per line: {message}")
-                })?;
-                sink(rows);
-            }
-            Ok(())
-        });
-        let spawned = (|| -> Result<(), String> {
-            for chunk in chunks(&args, ARG_BUDGET.saturating_sub(argv_bytes(&argv))) {
-                let mut full = argv.clone();
-                full.extend(chunk.iter().cloned());
-                let output =
-                    command::run_argv_stdout(command, &full, cwd, None).map_err(spawn_failure)?;
-                if payloads.send(output).is_err() {
-                    break;
+        let (rows_tx, rows_rx) = std::sync::mpsc::sync_channel::<Vec<JsonRow>>(16);
+        for _ in 0..workers {
+            let rows_tx = rows_tx.clone();
+            let (argv, runs) = (&argv, &runs);
+            let (next, aborted, failures) = (&next, &aborted, &failures);
+            scope.spawn(move || {
+                loop {
+                    let index = next.fetch_add(1, Ordering::SeqCst);
+                    if index >= runs.len() || aborted.load(Ordering::SeqCst) {
+                        return;
+                    }
+                    let mut full = argv.clone();
+                    full.extend(runs[index].iter().cloned());
+                    let outcome = run_invocation(command, &full, cwd, &|rows| {
+                        let _ = rows_tx.send(rows);
+                    });
+                    if let Err(message) = outcome {
+                        aborted.store(true, Ordering::SeqCst);
+                        failures
+                            .lock()
+                            .expect("failures lock")
+                            .push((index, message));
+                    }
+                }
+            });
+        }
+        drop(rows_tx);
+        for rows in rows_rx {
+            sink(rows);
+        }
+    });
+    // The earliest invocation's failure outranks a later one's.
+    match failures
+        .into_inner()
+        .expect("failures lock")
+        .into_iter()
+        .min()
+    {
+        Some((_, message)) => Err(message),
+        None => Ok(()),
+    }
+}
+
+/// One invocation: stream its stdout through the parser, `send`ing each
+/// block's rows. A non-zero exit outranks bad output, and bad output is
+/// drained rather than cut short so the child never dies on a broken pipe.
+fn run_invocation(
+    command: &str,
+    full: &[String],
+    cwd: &Path,
+    send: &dyn Fn(Vec<JsonRow>),
+) -> Result<(), String> {
+    let mut bad_output: Option<String> = None;
+    let mut lines_seen = 0usize;
+    command::run_argv_blocks(command, full, cwd, &mut |block| {
+        if bad_output.is_some() {
+            return;
+        }
+        match parse_rows_at(block, lines_seen) {
+            Ok(rows) => {
+                lines_seen += block.bytes().filter(|&b| b == b'\n').count();
+                if !rows.is_empty() {
+                    send(rows);
                 }
             }
-            Ok(())
-        })();
-        drop(payloads);
-        // An earlier invocation's bad output outranks a later one's failure.
-        parser.join().expect("parser thread panicked")?;
-        spawned
+            Err(message) => {
+                bad_output = Some(format!(
+                    "on-file output was not one JSON object per line: {message}"
+                ));
+            }
+        }
     })
+    .map_err(spawn_failure)?;
+    bad_output.map_or(Ok(()), Err)
 }
 
 fn spawn_failure(error: command::CommandError) -> String {
@@ -256,6 +312,63 @@ mod tests {
         assert_eq!(rows[0].get("n").unwrap().as_i64().unwrap(), 300);
     }
 
+    fn long_paths(count: usize) -> Vec<PathBuf> {
+        (0..count)
+            .map(|i| {
+                PathBuf::from(format!(
+                    "/some/long/directory/name/for/the/budget/f-{i:06}.json"
+                ))
+            })
+            .collect()
+    }
+
+    #[test]
+    fn run_gives_every_path_to_some_invocation_when_the_table_needs_several() {
+        let paths = long_paths(6000);
+        let rows = run(
+            r#"sh -c 'echo "{\"n\":$#}"' sh"#,
+            Path::new("."),
+            Path::new("."),
+            &paths,
+        )
+        .unwrap();
+        assert!(
+            rows.len() > 1,
+            "{} paths outgrow one argument list",
+            paths.len()
+        );
+        let total: i64 = rows
+            .iter()
+            .map(|r| r.get("n").unwrap().as_i64().unwrap())
+            .sum();
+        assert_eq!(total, 6000);
+    }
+
+    #[test]
+    fn run_names_the_line_of_bad_output_after_earlier_blocks() {
+        let err = run(
+            r#"sh -c 'yes "{\"a\":1}" | head -n 100000; echo nope' sh"#,
+            Path::new("."),
+            Path::new("."),
+            &[PathBuf::from("/x")],
+        )
+        .unwrap_err();
+        assert!(err.contains("line 100001 "), "got: {err}");
+        assert!(err.contains("one JSON object per line"), "got: {err}");
+    }
+
+    #[test]
+    fn run_prefers_a_non_zero_exit_to_the_bad_output_before_it() {
+        let err = run(
+            "sh -c 'echo nope; echo boom >&2; exit 4' sh",
+            Path::new("."),
+            Path::new("."),
+            &[PathBuf::from("/x")],
+        )
+        .unwrap_err();
+        assert!(err.contains("boom"), "got: {err}");
+    }
+
     #[test]
     fn run_over_no_paths_spawns_nothing() {
         let rows = run(