dirsql 0.4.102__tar.gz → 0.4.103__tar.gz

{dirsql-0.4.102 → dirsql-0.4.103}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.102"
+version = "0.4.103"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.102"
+version = "0.4.103"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.102 → dirsql-0.4.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.102
+Version: 0.4.103
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.102/packages/rust → dirsql-0.4.103}/docs/reference/hooks.md

@@ -163,7 +163,7 @@ table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
 the rows it prints. Runs execute concurrently, one per CPU at most, like
 `xargs -P`, so rows keep their order within a run but not across runs; SQL
-never promised row order, so `ORDER BY` what you need. The command cannot tell: each run is an
+never promised row order, so `ORDER BY` what you need. A table whose paths fit one run is also split, by file size, into up to one run per CPU once it holds at least 4 MiB of file data per run, so a few large files do not serialize behind one process. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

{dirsql-0.4.102 → dirsql-0.4.103}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.102"
+version = "0.4.103"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.103/packages/python/changelog.d/2026-10-07-spread-few-large-files.md

@@ -0,0 +1 @@
+**Changed** An `on-file` table with a few large files now runs its command over up to one run per CPU, split by file size at 4 MiB of data per run, instead of one run over all of them. Small tables still use one run.

{dirsql-0.4.102 → dirsql-0.4.103/packages/python}/docs/reference/hooks.md

@@ -163,7 +163,7 @@ table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
 the rows it prints. Runs execute concurrently, one per CPU at most, like
 `xargs -P`, so rows keep their order within a run but not across runs; SQL
-never promised row order, so `ORDER BY` what you need. The command cannot tell: each run is an
+never promised row order, so `ORDER BY` what you need. A table whose paths fit one run is also split, by file size, into up to one run per CPU once it holds at least 4 MiB of file data per run, so a few large files do not serialize behind one process. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

dirsql-0.4.103/packages/python/e2e-attestations/bot-1454-split-few-files.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/declared_table_name_test.py tests/e2e/fanout_test.py tests/e2e/extension_package_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/plugin_extension_discovery_test.py tests/e2e/on_file_query_test.py tests/e2e/sigint_interrupt_test.py tests/e2e/use_case_notebook_test.py tests/e2e/server_speed_test.py tests/e2e/query_contract_test.py tests/e2e/plugin_discovery_test.py tests/e2e/use_case_transcripts_test.py  -q",
+  "ran_at": 1791411055,
+  "exit_code": 0,
+  "commit": "7a93d0b97fe085a269b90d2b75c2a0aa22511c18",
+  "branch": "bot/1454-split-few-files"
+}

dirsql-0.4.103/packages/python/migrations.d/2026-10-07-spread-few-large-files.md

@@ -0,0 +1,39 @@
+### `on-file` splits large tables by file size
+
+#### Summary
+
+A table whose paths fit one argument list is now also split into several
+concurrent runs when its files hold at least 4 MiB in total per run, up
+to one run per CPU. Small tables are unchanged.
+
+#### Required changes
+
+_None._
+
+#### Deprecations removed
+
+_None._
+
+#### Behavior changes without code changes
+
+- A hook may now see a subset of the table's paths even when the table is
+  small in path count. A hook must already not assume it sees every path;
+  do cross-file work in SQL.
+- Rows from different runs have no defined order; add an `ORDER BY`.
+
+#### Verification
+
+```bash
+cd "$(mktemp -d)"
+for i in 1 2 3 4; do head -c 5000000 /dev/zero | tr '\0' x > "f$i.txt"; done
+printf 'echo "{\\"n\\":$#}"\n' > count.sh
+cat > .dirsql.toml <<'TOML'
+[[table]]
+name    = "t"
+ddl     = "CREATE TABLE t (n INTEGER)"
+glob    = "*.txt"
+on-file = "sh count.sh"
+TOML
+dirsql query "SELECT count(*) AS runs, sum(n) AS paths FROM t" -c .dirsql.toml
+# expected: paths is 4; runs is the smaller of 4 and the CPU count
+```

{dirsql-0.4.102 → dirsql-0.4.103}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.102"
+version = "0.4.103"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.102/packages/python → dirsql-0.4.103/packages/rust}/docs/reference/hooks.md

@@ -163,7 +163,7 @@ table's paths outgrow that cap, dirsql splits them into the fewest
 consecutive runs that fit, runs the command once per run, and concatenates
 the rows it prints. Runs execute concurrently, one per CPU at most, like
 `xargs -P`, so rows keep their order within a run but not across runs; SQL
-never promised row order, so `ORDER BY` what you need. The command cannot tell: each run is an
+never promised row order, so `ORDER BY` what you need. A table whose paths fit one run is also split, by file size, into up to one run per CPU once it holds at least 4 MiB of file data per run, so a few large files do not serialize behind one process. The command cannot tell: each run is an
 ordinary invocation with a subset of the paths. A command must therefore not
 assume one invocation sees every path — a count, a cross-file join, or a
 dedupe over `sys.argv[1:]` is per run, not per table. Do that work in SQL.

{dirsql-0.4.102 → dirsql-0.4.103}/packages/rust/src/on_file.rs

@@ -73,10 +73,12 @@ pub(crate) fn run_streaming(
         .iter()
         .map(|path| command::non_verbatim(&path.to_string_lossy()))
         .collect();
-    let runs = chunks(&args, ARG_BUDGET.saturating_sub(argv_bytes(&argv)));
-    let workers = std::thread::available_parallelism()
-        .map_or(1, std::num::NonZero::get)
-        .min(runs.len());
+    let cpus = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
+    let mut runs = chunks(&args, ARG_BUDGET.saturating_sub(argv_bytes(&argv)));
+    if runs.len() == 1 {
+        runs = spread(&args, &file_sizes(paths), cpus);
+    }
+    let workers = cpus.min(runs.len());
 
     let next = AtomicUsize::new(0);
     let aborted = AtomicBool::new(false);
@@ -202,6 +204,45 @@ fn chunks(args: &[String], budget: usize) -> Vec<&[String]> {
     out
 }
 
+/// Bytes of file data worth a process of its own: below this, spawning
+/// another invocation costs more than it saves.
+const MIN_RUN_BYTES: u64 = 4 * 1024 * 1024;
+
+/// Each path's size in bytes; a path that cannot be read counts as empty.
+fn file_sizes(paths: &[PathBuf]) -> Vec<u64> {
+    paths
+        .iter()
+        .map(|path| std::fs::metadata(path).map_or(0, |meta| meta.len()))
+        .collect()
+}
+
+/// Split `args` into at most `parts` consecutive runs holding about equal
+/// bytes of file data, preserving order, and never more runs than
+/// [`MIN_RUN_BYTES`] of data justify. `sizes` runs parallel to `args`.
+fn spread<'a>(args: &'a [String], sizes: &[u64], parts: usize) -> Vec<&'a [String]> {
+    let total: u64 = sizes.iter().sum();
+    let parts = parts
+        .min(args.len())
+        .min(usize::try_from(total / MIN_RUN_BYTES).unwrap_or(usize::MAX));
+    if parts <= 1 {
+        return vec![args];
+    }
+    let mut out = Vec::new();
+    let mut start = 0;
+    let mut seen = 0u64;
+    for (i, size) in sizes.iter().enumerate() {
+        seen += size;
+        let cut = u64::try_from(out.len() + 1).unwrap_or(u64::MAX);
+        let last = i + 1 == args.len();
+        if !last && out.len() + 1 < parts && seen * parts as u64 >= total * cut {
+            out.push(&args[start..=i]);
+            start = i + 1;
+        }
+    }
+    out.push(&args[start..]);
+    out
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -344,6 +385,88 @@ mod tests {
         assert_eq!(total, 6000);
     }
 
+    fn files_of(dir: &Path, count: usize, bytes: usize) -> Vec<PathBuf> {
+        (0..count)
+            .map(|i| {
+                let path = dir.join(format!("f{i}.jsonl"));
+                std::fs::write(&path, vec![b'x'; bytes]).unwrap();
+                path
+            })
+            .collect()
+    }
+
+    fn invocations_over(paths: &[PathBuf]) -> usize {
+        run(
+            r#"sh -c 'echo "{\"n\":$#}"' sh"#,
+            Path::new("."),
+            Path::new("."),
+            paths,
+        )
+        .unwrap()
+        .len()
+    }
+
+    #[test]
+    fn spread_cuts_equal_bytes_in_order() {
+        let args = args(&["a", "b", "c", "d"]);
+        let mib = 1024 * 1024;
+        let runs = spread(&args, &[8 * mib, 8 * mib, 8 * mib, 8 * mib], 2);
+        assert_eq!(lens(&runs), [2, 2]);
+        assert_eq!(runs[1], ["c", "d"]);
+    }
+
+    #[test]
+    fn spread_never_makes_more_runs_than_the_data_justifies() {
+        let args = args(&["a", "b", "c", "d"]);
+        let mib = 1024 * 1024;
+        assert_eq!(lens(&spread(&args, &[5 * mib; 4], 16)), [1, 1, 1, 1]);
+        assert_eq!(lens(&spread(&args, &[mib; 4], 16)), [4]);
+    }
+
+    #[test]
+    fn spread_isolates_a_dominant_file() {
+        let args = args(&["a", "b", "c"]);
+        let mib = 1024 * 1024;
+        assert_eq!(lens(&spread(&args, &[100 * mib, mib, mib], 2)), [1, 2]);
+    }
+
+    #[test]
+    fn spread_cuts_at_each_even_share_of_the_bytes() {
+        let args = args(&["a", "b", "c", "d", "e"]);
+        let mib = 1024 * 1024;
+        let sizes = [5 * mib, 0, 5 * mib, 5 * mib, 0];
+        assert_eq!(lens(&spread(&args, &sizes, 3)), [1, 2, 2]);
+    }
+
+    #[test]
+    fn spread_never_ends_with_an_empty_run() {
+        let args = args(&["a", "b", "c"]);
+        let mib = 1024 * 1024;
+        let sizes = [20 * mib, 0, 20 * mib];
+        assert_eq!(lens(&spread(&args, &sizes, 3)), [1, 2]);
+    }
+
+    #[test]
+    fn spread_gives_unreadable_files_no_weight() {
+        let args = args(&["a", "b"]);
+        assert_eq!(lens(&spread(&args, &[0, 0], 4)), [2]);
+    }
+
+    #[test]
+    fn run_spreads_a_few_large_files_across_the_available_workers() {
+        let dir = tempfile::tempdir().unwrap();
+        let paths = files_of(dir.path(), 4, 4 * 1024 * 1024);
+        let runs = invocations_over(&paths);
+        assert!((2..=4).contains(&runs), "got {runs} invocations");
+    }
+
+    #[test]
+    fn run_keeps_a_few_small_files_in_one_invocation() {
+        let dir = tempfile::tempdir().unwrap();
+        let paths = files_of(dir.path(), 4, 100);
+        assert_eq!(invocations_over(&paths), 1);
+    }
+
     #[test]
     fn run_names_the_line_of_bad_output_after_earlier_blocks() {
         let err = run(