dirsql 0.4.101__tar.gz → 0.4.103__tar.gz

{dirsql-0.4.101 → dirsql-0.4.103}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.101"
+version = "0.4.103"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.101"
+version = "0.4.103"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.101 → dirsql-0.4.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.101
+Version: 0.4.103
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.101/packages/rust → dirsql-0.4.103}/docs/reference/hooks.md

@@ -161,7 +161,9 @@ on-file command `python3 extract.py {path}` uses `{path}`, but an on-file comman
 Every platform caps the bytes one process may receive as arguments. When a
 table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
-the rows it prints, in order. The command cannot tell: each run is an
+the rows it prints. Runs execute concurrently, one per CPU at most, like
+`xargs -P`, so rows keep their order within a run but not across runs; SQL
+never promised row order, so `ORDER BY` what you need. A table whose paths fit one run is also split, by file size, into up to one run per CPU once it holds at least 4 MiB of file data per run, so a few large files do not serialize behind one process. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

{dirsql-0.4.101 → dirsql-0.4.103}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.101"
+version = "0.4.103"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.103/packages/python/changelog.d/2026-10-07-parallel-hooks.md

@@ -0,0 +1 @@
+**Changed** An `on-file` command's runs now execute concurrently, one per CPU at most, when a table's paths outgrow one argument list, and each run's output is parsed as it is written instead of after it exits. Rows keep their order within a run but not across runs. A 2M-row JSONL ingest through `jq` runs within 5% of the equivalent `jq | awk` pipeline, and a frontmatter parse over thousands of files runs in a quarter of the time of `find -exec awk +`. The scratch database no longer syncs to disk.

dirsql-0.4.103/packages/python/changelog.d/2026-10-07-spread-few-large-files.md

@@ -0,0 +1 @@
+**Changed** An `on-file` table with a few large files now runs its command over up to one run per CPU, split by file size at 4 MiB of data per run, instead of one run over all of them. Small tables still use one run.

{dirsql-0.4.101 → dirsql-0.4.103/packages/python}/docs/reference/hooks.md

@@ -161,7 +161,9 @@ on-file command `python3 extract.py {path}` uses `{path}`, but an on-file comman
 Every platform caps the bytes one process may receive as arguments. When a
 table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
-the rows it prints, in order. The command cannot tell: each run is an
+the rows it prints. Runs execute concurrently, one per CPU at most, like
+`xargs -P`, so rows keep their order within a run but not across runs; SQL
+never promised row order, so `ORDER BY` what you need. A table whose paths fit one run is also split, by file size, into up to one run per CPU once it holds at least 4 MiB of file data per run, so a few large files do not serialize behind one process. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

dirsql-0.4.103/packages/python/e2e-attestations/bot-1429-parallel-hooks.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/declared_table_name_test.py tests/e2e/fanout_test.py tests/e2e/extension_package_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/plugin_extension_discovery_test.py tests/e2e/on_file_query_test.py tests/e2e/sigint_interrupt_test.py tests/e2e/use_case_notebook_test.py tests/e2e/server_speed_test.py tests/e2e/query_contract_test.py tests/e2e/plugin_discovery_test.py tests/e2e/use_case_transcripts_test.py  -q",
+  "ran_at": 1791407652,
+  "exit_code": 0,
+  "commit": "8d6801314e86177dc46926aedb94bf63e47e87bb",
+  "branch": "bot/1429-parallel-hooks"
+}

dirsql-0.4.103/packages/python/e2e-attestations/bot-1454-split-few-files.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/declared_table_name_test.py tests/e2e/fanout_test.py tests/e2e/extension_package_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/plugin_extension_discovery_test.py tests/e2e/on_file_query_test.py tests/e2e/sigint_interrupt_test.py tests/e2e/use_case_notebook_test.py tests/e2e/server_speed_test.py tests/e2e/query_contract_test.py tests/e2e/plugin_discovery_test.py tests/e2e/use_case_transcripts_test.py  -q",
+  "ran_at": 1791411055,
+  "exit_code": 0,
+  "commit": "7a93d0b97fe085a269b90d2b75c2a0aa22511c18",
+  "branch": "bot/1454-split-few-files"
+}

dirsql-0.4.103/packages/python/migrations.d/2026-10-07-parallel-hooks.md

@@ -0,0 +1,41 @@
+### `on-file` runs execute concurrently
+
+#### Summary
+
+When a table's matched paths outgrow one argument list, dirsql splits them
+into runs and now executes the runs concurrently, one per CPU at most. Rows
+keep their order within a run, but not across runs.
+
+#### Required changes
+
+_None._
+
+#### Deprecations removed
+
+_None._
+
+#### Behavior changes without code changes
+
+- A query that relied on rows coming back in path order across runs must add
+  an `ORDER BY`. A table small enough for one run is unchanged.
+- A hook that holds memory or a GPU may now have several copies running at
+  once on a very large table.
+
+#### Verification
+
+```bash
+cd "$(mktemp -d)"
+for i in $(seq 1 5000); do : > "a-long-file-name-to-fill-the-argument-list-$i.txt"; done
+cat > count.sh <<'SH'
+echo "{\"n\":$#}"
+SH
+cat > .dirsql.toml <<'TOML'
+[[table]]
+name    = "t"
+ddl     = "CREATE TABLE t (n INTEGER)"
+glob    = "*.txt"
+on-file = "sh count.sh"
+TOML
+dirsql query "SELECT count(*) AS runs, sum(n) AS paths FROM t" -c .dirsql.toml
+# expected: one row whose paths is 5000 and whose runs is greater than 1
+```

dirsql-0.4.103/packages/python/migrations.d/2026-10-07-spread-few-large-files.md

@@ -0,0 +1,39 @@
+### `on-file` splits large tables by file size
+
+#### Summary
+
+A table whose paths fit one argument list is now also split into several
+concurrent runs when its files hold at least 4 MiB in total per run, up
+to one run per CPU. Small tables are unchanged.
+
+#### Required changes
+
+_None._
+
+#### Deprecations removed
+
+_None._
+
+#### Behavior changes without code changes
+
+- A hook may now see a subset of the table's paths even when the table is
+  small in path count. A hook must already not assume it sees every path;
+  do cross-file work in SQL.
+- Rows from different runs have no defined order; add an `ORDER BY`.
+
+#### Verification
+
+```bash
+cd "$(mktemp -d)"
+for i in 1 2 3 4; do head -c 5000000 /dev/zero | tr '\0' x > "f$i.txt"; done
+printf 'echo "{\\"n\\":$#}"\n' > count.sh
+cat > .dirsql.toml <<'TOML'
+[[table]]
+name    = "t"
+ddl     = "CREATE TABLE t (n INTEGER)"
+glob    = "*.txt"
+on-file = "sh count.sh"
+TOML
+dirsql query "SELECT count(*) AS runs, sum(n) AS paths FROM t" -c .dirsql.toml
+# expected: paths is 4; runs is the smaller of 4 and the CPU count
+```

{dirsql-0.4.101 → dirsql-0.4.103}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.101"
+version = "0.4.103"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.101/packages/python → dirsql-0.4.103/packages/rust}/docs/reference/hooks.md

@@ -161,7 +161,9 @@ on-file command `python3 extract.py {path}` uses `{path}`, but an on-file comman
 Every platform caps the bytes one process may receive as arguments. When a
 table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
-the rows it prints, in order. The command cannot tell: each run is an
+the rows it prints. Runs execute concurrently, one per CPU at most, like
+`xargs -P`, so rows keep their order within a run but not across runs; SQL
+never promised row order, so `ORDER BY` what you need. A table whose paths fit one run is also split, by file size, into up to one run per CPU once it holds at least 4 MiB of file data per run, so a few large files do not serialize behind one process. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

{dirsql-0.4.101 → dirsql-0.4.103}/packages/rust/src/command.rs

@@ -256,21 +256,85 @@ pub(crate) fn run_argv_stdout(
     let stdout = stdout_thread.join().unwrap_or_default();
     let stderr = stderr_thread.join().unwrap_or_default();
 
-    if !status.success() {
-        let code = status
-            .code()
-            .map(|c| c.to_string())
-            .unwrap_or_else(|| "signal".to_string());
-        return Err(CommandError::NonZeroExit {
-            command: command.to_string(),
-            code,
-            stderr_tail: stderr_tail(&stderr),
-        });
-    }
+    check_exit(command, status, &stderr)?;
 
     Ok(String::from_utf8_lossy(&stdout).into_owned())
 }
 
+fn check_exit(
+    command: &str,
+    status: std::process::ExitStatus,
+    stderr: &[u8],
+) -> Result<(), CommandError> {
+    if status.success() {
+        return Ok(());
+    }
+    let code = status
+        .code()
+        .map(|c| c.to_string())
+        .unwrap_or_else(|| "signal".to_string());
+    Err(CommandError::NonZeroExit {
+        command: command.to_string(),
+        code,
+        stderr_tail: stderr_tail(stderr),
+    })
+}
+
+/// Bytes read from a child's stdout per `read` call.
+const READ_BYTES: usize = 262_144;
+
+/// Run an already-built `argv` and hand its stdout to `on_block` as it
+/// arrives, in blocks that each hold whole lines (the last block may lack its
+/// trailing newline). The child runs with stdin closed. A non-zero exit is an
+/// error after every block has been delivered.
+pub(crate) fn run_argv_blocks(
+    command: &str,
+    argv: &[String],
+    cwd: &Path,
+    on_block: &mut dyn FnMut(&str),
+) -> Result<(), CommandError> {
+    let mut cmd = Command::new(&argv[0]);
+    push_args(&mut cmd, &argv[1..]);
+    cmd.current_dir(cwd)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .stdin(Stdio::null());
+    let mut child = cmd
+        .spawn()
+        .map_err(|source| spawn_error(&argv[0], source, cwd))?;
+
+    let mut stderr_pipe = child.stderr.take().expect("stderr piped");
+    let stderr_thread = std::thread::spawn(move || {
+        let mut buf = Vec::new();
+        let _ = stderr_pipe.read_to_end(&mut buf);
+        buf
+    });
+    let mut stdout_pipe = child.stdout.take().expect("stdout piped");
+    let mut pending: Vec<u8> = Vec::new();
+    let mut buf = vec![0u8; READ_BYTES];
+    loop {
+        let n = match stdout_pipe.read(&mut buf) {
+            Ok(0) | Err(_) => break,
+            Ok(n) => n,
+        };
+        pending.extend_from_slice(&buf[..n]);
+        if let Some(end) = pending.iter().rposition(|&b| b == b'\n') {
+            on_block(&String::from_utf8_lossy(&pending[..=end]));
+            pending.drain(..=end);
+        }
+    }
+    if !pending.is_empty() {
+        on_block(&String::from_utf8_lossy(&pending));
+    }
+
+    let status = child.wait().map_err(|source| CommandError::Io {
+        command: command.to_string(),
+        source,
+    })?;
+    let stderr = stderr_thread.join().unwrap_or_default();
+    check_exit(command, status, &stderr)
+}
+
 // One fn with cfg blocks rather than two cfg'd fns: cargo-mutants mutates
 // the uncompiled twin too, and no test on a Linux gate can kill that mutant.
 fn push_args(cmd: &mut Command, args: &[String]) {
@@ -736,6 +800,75 @@ mod tests {
         );
     }
 
+    #[test]
+    fn run_argv_blocks_delivers_whole_lines_and_the_unterminated_tail() {
+        let argv = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "printf 'a\\nb\\nc'".to_string(),
+        ];
+        let mut seen = String::new();
+        let mut blocks = 0;
+        run_argv_blocks("t", &argv, &cwd(), &mut |block| {
+            seen.push_str(block);
+            blocks += 1;
+        })
+        .unwrap();
+        assert_eq!(seen, "a\nb\nc");
+        assert!(blocks >= 1);
+    }
+
+    #[test]
+    fn run_argv_blocks_cuts_every_block_at_a_line_boundary() {
+        let argv = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "yes abcdefghijklmnopqrstuvwxyz | head -n 100000".to_string(),
+        ];
+        let mut lines = 0;
+        let mut blocks = 0;
+        run_argv_blocks("t", &argv, &cwd(), &mut |block| {
+            blocks += 1;
+            assert!(block.ends_with('\n'), "block cut mid-line");
+            lines += block.lines().count();
+        })
+        .unwrap();
+        assert_eq!(lines, 100_000);
+        assert!(blocks > 1, "2.7 MB arrives in more than one read");
+    }
+
+    #[test]
+    fn run_argv_blocks_reports_a_non_zero_exit_after_delivering_output() {
+        let argv = vec![
+            "sh".to_string(),
+            "-c".to_string(),
+            "echo partial; echo oops >&2; exit 3".to_string(),
+        ];
+        let mut seen = String::new();
+        let err =
+            run_argv_blocks("t", &argv, &cwd(), &mut |block| seen.push_str(block)).unwrap_err();
+        assert_eq!(seen, "partial\n");
+        match err {
+            CommandError::NonZeroExit {
+                code, stderr_tail, ..
+            } => {
+                assert_eq!(code, "3");
+                assert!(stderr_tail.contains("oops"));
+            }
+            other => panic!("expected NonZeroExit, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn run_argv_blocks_names_a_missing_program() {
+        let argv = vec!["definitely-not-a-real-binary-xyzzy".to_string()];
+        let err = run_argv_blocks("t", &argv, &cwd(), &mut |_| {}).unwrap_err();
+        assert!(
+            matches!(err, CommandError::NotOnPath { .. }),
+            "got: {err:?}"
+        );
+    }
+
     #[test]
     fn run_command_writes_stdin_payload_to_the_child() {
         let out = run_command("cat", &[], &cwd(), Some(b"hello-stdin")).unwrap();

{dirsql-0.4.101 → dirsql-0.4.103}/packages/rust/src/db.rs

@@ -368,6 +368,9 @@ impl Db {
     /// tmpfs mount.
     pub fn new() -> Result<Self> {
         let conn = Connection::open("")?;
+        // The database is a private temp file that dies with the process, so
+        // durability buys nothing.
+        conn.pragma_update(None, "synchronous", "OFF")?;
         ensure_internal_rows_table(&conn)?;
         let scope = StatementScope::new();
         vtab::load_module(&conn, Arc::clone(&scope))?;
@@ -2066,6 +2069,17 @@ mod tests {
         assert_eq!(mode, "delete");
     }
 
+    #[test]
+    fn new_skips_fsync() {
+        let db = Db::new().unwrap();
+        let pragma = |name: &str| -> i64 {
+            db.conn()
+                .query_row(&format!("PRAGMA {name}"), [], |row| row.get(0))
+                .unwrap()
+        };
+        assert_eq!(pragma("synchronous"), 0);
+    }
+
     #[test]
     fn get_rows_by_file_returns_rows_in_row_index_order() {
         let db = Db::new().unwrap();

{dirsql-0.4.101 → dirsql-0.4.103}/packages/rust/src/infer.rs

@@ -110,13 +110,19 @@ impl<'de> serde::Deserialize<'de> for JsonRow {
 /// Parse a parser command's output — one JSON object per line (NDJSON), blank
 /// lines skipped — into rows that remember their key order.
 pub fn parse_rows(output: &str) -> Result<Vec<JsonRow>, String> {
+    parse_rows_at(output, 0)
+}
+
+/// [`parse_rows`] for a block that begins after `skipped` earlier lines of the
+/// same output, so an error names its line in the whole output.
+pub(crate) fn parse_rows_at(output: &str, skipped: usize) -> Result<Vec<JsonRow>, String> {
     let mut rows = Vec::new();
     for (index, line) in output.lines().enumerate() {
         let line = line.trim();
         if line.is_empty() {
             continue;
         }
-        let number = index + 1;
+        let number = skipped + index + 1;
         if line.starts_with('[') {
             return Err(format!(
                 "line {number} is a JSON array; print one JSON object per line instead"
@@ -255,6 +261,12 @@ mod tests {
         assert_eq!(SqlType::Real.as_str(), "REAL");
     }
 
+    #[test]
+    fn parse_rows_at_numbers_lines_after_the_skipped_ones() {
+        let err = parse_rows_at("{\"a\":1}\nnope", 40).unwrap_err();
+        assert!(err.starts_with("line 42 "), "got: {err}");
+    }
+
     #[test]
     fn parse_rows_preserves_key_order_rather_than_sorting() {
         let parsed = parse_rows(r#"{"zeta":1,"alpha":2}"#).unwrap();