dirsql 0.4.108__tar.gz → 0.4.109__tar.gz

{dirsql-0.4.108 → dirsql-0.4.109}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.108"
+version = "0.4.109"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.108"
+version = "0.4.109"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.108 → dirsql-0.4.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.108
+Version: 0.4.109
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.108 → dirsql-0.4.109}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.108"
+version = "0.4.109"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.109/packages/python/e2e-attestations/bot-1416-watch-speed-1008.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/watch_speed_test.py tests/e2e/on_file_query_test.py tests/e2e/server_speed_test.py -q -p no:cacheprovider",
+  "ran_at": 1791548609,
+  "exit_code": 0,
+  "commit": "ea38a739b990bc5135cc32ba9b61193fd0a2c1e1",
+  "branch": "bot/1416-watch-speed-1008"
+}

dirsql-0.4.109/packages/python/e2e-attestations/bot-1462-paired-speed-harness-1008.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/glob_speed_one_level_test.py -x -q -rA",
+  "ran_at": 1791492173,
+  "exit_code": 0,
+  "commit": "ce22059009267d7759e47aa47cba10e28fd6543c",
+  "branch": "bot/1462-paired-speed-harness-1008"
+}

dirsql-0.4.109/packages/python/tests/e2e/speed_of_light.py

@@ -0,0 +1,205 @@
+"""Harness for the use-case speed-of-light tests.
+
+Each use-case test asks one question of a generated corpus twice: once through
+the fastest native pipeline (find, sort, join, awk, a single parser process)
+and once through the real `dirsql` launcher. The rows must be identical and
+dirsql's wall time, after subtracting the launcher's measured startup, must be
+within ten percent of native. The bar is a ratio, never a millisecond number,
+so it holds on any machine.
+
+Native and dirsql run as alternating pairs (native first, then dirsql first,
+and so on) after one discarded warm-up pair, so drift in machine speed hits
+both sides alike. The verdict is the median of the per-pair ratios. Startup is
+the median of `dirsql query 'SELECT 1'` runs with the first dropped, and
+native's own is the median of `bash -c true` the same way; each is subtracted
+from its side. A corpus too small to time is grown by doubling until the
+native pipeline takes a second, unless the corpus ceiling stops it sooner.
+Output is captured as bytes and decoded after the clock stops.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import statistics
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass
+from typing import NamedTuple
+
+ONE_SECOND = 1.0
+TOLERANCE = 1.1
+STARTUP_PROBES = 11
+PAIRS = 100
+HOPELESS_STREAK = 3
+NATIVE_ENV = {**os.environ, "LC_ALL": "C"}
+
+
+class Startup(NamedTuple):
+    dirsql: float
+    native: float
+
+
+@dataclass
+class Paired:
+    native_rows: object
+    dirsql_rows: object
+    native_seconds: list
+    dirsql_seconds: list
+
+
+def cli() -> str:
+    dirsql = shutil.which("dirsql")
+    assert dirsql is not None, (
+        "`dirsql` console script not on PATH -- run `uv run maturin develop`"
+    )
+    return dirsql
+
+
+def timed(argv, cwd, timeout, env=None):
+    started = time.perf_counter()
+    proc = subprocess.run(
+        argv,
+        cwd=str(cwd),
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        timeout=timeout,
+        env=env,
+        check=False,
+    )
+    seconds = time.perf_counter() - started
+    proc.stdout = proc.stdout.decode()
+    proc.stderr = proc.stderr.decode()
+    return proc, seconds
+
+
+def timed_native(argv, cwd, timeout=600):
+    return timed(argv, cwd, timeout, env=NATIVE_ENV)
+
+
+def median_after_first(samples):
+    return statistics.median(samples[1:])
+
+
+def probe_seconds(argv, cwd, env=None) -> float:
+    samples = []
+    for _ in range(STARTUP_PROBES):
+        proc, seconds = timed(argv, cwd, timeout=60, env=env)
+        assert proc.returncode == 0, proc.stderr
+        samples.append(seconds)
+    return median_after_first(samples)
+
+
+def startup_seconds(argv=None, cwd=None) -> Startup:
+    argv = [cli(), "query", "SELECT 1"] if argv is None else argv
+    with tempfile.TemporaryDirectory() as empty:
+        dirsql = probe_seconds(argv, cwd or empty)
+        native = probe_seconds(["bash", "-c", "true"], empty, NATIVE_ENV)
+    return Startup(dirsql, native)
+
+
+def dirsql_rows(proc, columns):
+    assert proc.returncode == 0, (
+        f"dirsql failed: stdout={proc.stdout!r} stderr={proc.stderr!r}"
+    )
+    return [tuple(row[column] for column in columns) for row in json.loads(proc.stdout)]
+
+
+def grow_until_native_takes_a_second(build, native, start, ceiling):
+    """Grow the corpus by doubling until `native` runs for a second.
+
+    `build(lo, hi)` adds items `lo` through `hi - 1` to the corpus already
+    holding `lo` items, so each doubling writes only the new half.
+    """
+    built, size = 0, start
+    while True:
+        build(built, size)
+        os.sync()
+        built = size
+        _, seconds = native()
+        if seconds >= ONE_SECOND or size >= ceiling:
+            return size
+        size *= 2
+
+
+def dirsql_timeout(native_seconds: float) -> float:
+    return max(60.0, 30 * native_seconds)
+
+
+def pair_ratios(native_seconds, dirsql_seconds, startup):
+    ratios = []
+    for native, dirsql in zip(native_seconds, dirsql_seconds):
+        net = native - startup.native
+        assert net > 0, (
+            f"native {native:.3f}s is under its startup {startup.native:.3f}s"
+        )
+        ratios.append((dirsql - startup.dirsql) / net)
+    return ratios
+
+
+def is_hopeless(native, dirsql, startup):
+    return dirsql - startup.dirsql > 10 * TOLERANCE * (native - startup.native)
+
+
+def run_or_fail(run, name):
+    try:
+        return run()
+    except subprocess.TimeoutExpired as expired:
+        raise AssertionError(
+            f"{name} did not finish within {expired.timeout:.0f}s"
+        ) from None
+
+
+def paired(native, dirsql, startup, pairs=PAIRS):
+    """Alternate `native()` and `dirsql(timeout)`, native first on even pairs.
+
+    Both return `(rows, seconds)`. The first pair is a discarded warm-up that
+    also sets dirsql's timeout. A run that raises `TimeoutExpired` fails the
+    test at once. Three hopeless pairs in a row end the series early: noise
+    does not account for an order of magnitude, so more pairs cannot bring
+    the median under the bar.
+    """
+    native_rows, native_warm = run_or_fail(native, "native")
+    timeout = dirsql_timeout(native_warm)
+
+    def run_dirsql():
+        return run_or_fail(lambda: dirsql(timeout), "dirsql")
+
+    dirsql_rows_, _ = run_dirsql()
+    native_times, dirsql_times, streak = [], [], 0
+    for index in range(pairs):
+        if index % 2 == 0:
+            native_rows, native_seconds = run_or_fail(native, "native")
+            dirsql_rows_, dirsql_seconds = run_dirsql()
+        else:
+            dirsql_rows_, dirsql_seconds = run_dirsql()
+            native_rows, native_seconds = run_or_fail(native, "native")
+        native_times.append(native_seconds)
+        dirsql_times.append(dirsql_seconds)
+        streak = (
+            streak + 1 if is_hopeless(native_seconds, dirsql_seconds, startup) else 0
+        )
+        if streak == HOPELESS_STREAK:
+            break
+    return Paired(native_rows, dirsql_rows_, native_times, dirsql_times)
+
+
+def summarize(ratios):
+    low, high = min(ratios), max(ratios)
+    median = statistics.median(ratios)
+    return low, median, high, (high - low) / median
+
+
+def assert_speed_of_light(name, result, startup):
+    ratios = pair_ratios(result.native_seconds, result.dirsql_seconds, startup)
+    low, median, high, spread = summarize(ratios)
+    line = (
+        f"{name}: {len(ratios)} pairs, ratio min {low:.3f} median {median:.3f}"
+        f" max {high:.3f} spread {spread:.0%}, native median"
+        f" {statistics.median(result.native_seconds):.3f}s, startup"
+        f" dirsql {startup.dirsql:.3f}s native {startup.native:.3f}s"
+    )
+    print(f"SPEED {line}")
+    assert median <= TOLERANCE, f"{line}; median is over {TOLERANCE}x"

{dirsql-0.4.108 → dirsql-0.4.109}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.108"
+version = "0.4.109"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.108 → dirsql-0.4.109}/packages/rust/src/db.rs

@@ -779,7 +779,7 @@ impl Db {
         };
         let mut stored: Vec<(i64, Vec<Value>)> = Vec::new();
         {
-            let mut stmt = self.conn.prepare(&select)?;
+            let mut stmt = self.conn.prepare_cached(&select)?;
             let mut cursor = stmt.query([])?;
             while let Some(row) = cursor.next()? {
                 let cells = (0..columns.len())

{dirsql-0.4.108 → dirsql-0.4.109}/packages/rust/src/lib.rs

@@ -655,15 +655,31 @@ impl DirSQL {
     /// the TypeScript binding).
     #[doc(hidden)]
     pub fn apply_file_events(&self, events: Vec<FileEvent>) -> Vec<RowEvent> {
+        self.apply_file_events_draining(events, Vec::new)
+    }
+
+    /// [`apply_file_events`](Self::apply_file_events), asking `late` for events
+    /// that arrived while the batch was being routed, until it returns none,
+    /// before any per-table hook runs. One write reaches the watcher as
+    /// several events; a hook run for the first would run again for the rest.
+    fn apply_file_events_draining(
+        &self,
+        events: Vec<FileEvent>,
+        mut late: impl FnMut() -> Vec<FileEvent>,
+    ) -> Vec<RowEvent> {
         let mut pending = PendingRefresh::default();
         let mut out = Vec::new();
         let mut rescan = false;
-        for fe in events {
-            if fe == FileEvent::Rescan {
-                rescan = true;
-                continue;
+        let mut batch = events;
+        while !batch.is_empty() {
+            for fe in batch {
+                if fe == FileEvent::Rescan {
+                    rescan = true;
+                    continue;
+                }
+                out.extend(self.route_file_event(fe, &mut pending));
             }
-            out.extend(self.route_file_event(fe, &mut pending));
+            batch = late();
         }
         if rescan {
             out.extend(self.rescan(&mut pending));
@@ -710,7 +726,17 @@ impl DirSQL {
             events
         };
 
-        Ok(self.apply_file_events(file_events))
+        Ok(self.apply_file_events_draining(file_events, || self.queued_file_events()))
+    }
+
+    /// The events the watcher already holds, without waiting for more.
+    fn queued_file_events(&self) -> Vec<FileEvent> {
+        self.inner
+            .watcher
+            .lock()
+            .ok()
+            .and_then(|guard| guard.as_ref().map(Watcher::try_recv_all))
+            .unwrap_or_default()
     }
 
     /// Process a single [`FileEvent`], mutating the DB and cache as needed.
@@ -4135,6 +4161,18 @@ mod internal_tests {
         assert!(e2.to_string().contains("watch() is active"), "got: {e2}");
     }
 
+    #[test]
+    fn queued_file_events_hands_over_what_the_watcher_holds_without_waiting() {
+        let (dir, db) = simple_db();
+        assert!(db.queued_file_events().is_empty(), "no watcher yet");
+        db.start_watching().unwrap();
+        std::fs::write(dir.path().join("a.none"), "x").unwrap();
+        let mut held = Vec::new();
+        while !held.iter().any(|e| matches!(e, FileEvent::Created(_))) {
+            held.extend(db.queued_file_events());
+        }
+    }
+
     #[test]
     fn poll_events_locks_out_watch() {
         let (_dir, db) = simple_db();
@@ -5503,6 +5541,31 @@ mod internal_tests {
         assert_eq!(row_names(&db), vec!["a.txt", "b.txt"]);
     }
 
+    #[test]
+    fn events_arriving_while_a_batch_is_routed_share_its_per_table_rerun() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let a = root.join("a.txt");
+        let b = root.join("b.txt");
+        let fake = FakeFs::default().with_subtree(
+            root.clone(),
+            vec![(a.clone(), "items".into()), (b.clone(), "items".into())],
+        );
+        let (items, calls) = recording_per_table("items", "**/*.txt");
+        let db =
+            DirSQL::with_ignore_and_fs(&root, vec![items], Vec::<String>::new(), Arc::new(fake))
+                .unwrap();
+
+        let mut arrivals = vec![vec![FileEvent::Modified(b.clone())]];
+        let events = db.apply_file_events_draining(vec![FileEvent::Created(a.clone())], || {
+            arrivals.pop().unwrap_or_default()
+        });
+
+        assert_eq!(*calls.lock().unwrap(), vec![vec![a, b]]);
+        assert_eq!(events.len(), 2);
+        assert_eq!(row_names(&db), vec!["a.txt", "b.txt"]);
+    }
+
     #[test]
     fn apply_file_events_refreshes_a_per_table_hook_without_walking_the_tree() {
         let dir = TempDir::new().unwrap();

{dirsql-0.4.108 → dirsql-0.4.109}/packages/rust/src/on_file.rs

@@ -73,12 +73,16 @@ pub(crate) fn run_streaming(
         .iter()
         .map(|path| command::non_verbatim(&path.to_string_lossy()))
         .collect();
-    let cpus = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
     let mut runs = chunks(&args, ARG_BUDGET.saturating_sub(argv_bytes(&argv)));
     if runs.len() == 1 {
-        runs = spread(&args, &file_sizes(paths), cpus);
+        runs = spread_runs(&args, &file_sizes(paths), cpus);
     }
-    let workers = cpus.min(runs.len());
+    if let [only] = runs.as_slice() {
+        let mut full = argv;
+        full.extend(only.iter().cloned());
+        return run_invocation(command, &full, cwd, &mut |rows| sink(rows));
+    }
+    let workers = cpus().min(runs.len());
 
     let next = AtomicUsize::new(0);
     let aborted = AtomicBool::new(false);
@@ -97,7 +101,7 @@ pub(crate) fn run_streaming(
                     }
                     let mut full = argv.clone();
                     full.extend(runs[index].iter().cloned());
-                    let outcome = run_invocation(command, &full, cwd, &|rows| {
+                    let outcome = run_invocation(command, &full, cwd, &mut |rows| {
                         let _ = rows_tx.send(rows);
                     });
                     if let Err(message) = outcome {
@@ -134,7 +138,7 @@ fn run_invocation(
     command: &str,
     full: &[String],
     cwd: &Path,
-    send: &dyn Fn(Vec<JsonRow>),
+    send: &mut dyn FnMut(Vec<JsonRow>),
 ) -> Result<(), String> {
     let mut bad_output: Option<String> = None;
     let mut lines_seen = 0usize;
@@ -208,6 +212,24 @@ fn chunks(args: &[String], budget: usize) -> Vec<&[String]> {
 /// another invocation costs more than it saves.
 const MIN_RUN_BYTES: u64 = 4 * 1024 * 1024;
 
+fn cpus() -> usize {
+    std::thread::available_parallelism().map_or(1, std::num::NonZero::get)
+}
+
+/// [`spread`] over the machine's CPUs, asking for the count only when the
+/// data could fill more than one run: the query reads cgroup files each time.
+fn spread_runs<'a>(
+    args: &'a [String],
+    sizes: &[u64],
+    cpus: impl FnOnce() -> usize,
+) -> Vec<&'a [String]> {
+    let by_data = usize::try_from(sizes.iter().sum::<u64>() / MIN_RUN_BYTES).unwrap_or(usize::MAX);
+    if by_data.min(args.len()) <= 1 {
+        return vec![args];
+    }
+    spread(args, sizes, cpus())
+}
+
 /// Each path's size in bytes; a path that cannot be read counts as empty.
 fn file_sizes(paths: &[PathBuf]) -> Vec<u64> {
     paths
@@ -423,6 +445,22 @@ mod tests {
         assert_eq!(lens(&spread(&args, &[mib; 4], 16)), [4]);
     }
 
+    #[test]
+    fn spread_runs_does_not_count_cpus_for_data_that_fits_one_run() {
+        let args = args(&["a", "b", "c", "d"]);
+        let mib = 1024 * 1024;
+        let runs = spread_runs(&args, &[mib; 4], || panic!("counted the cpus"));
+        assert_eq!(lens(&runs), [4]);
+    }
+
+    #[test]
+    fn spread_runs_counts_cpus_once_the_data_fills_more_than_one_run() {
+        let args = args(&["a", "b", "c", "d"]);
+        let mib = 1024 * 1024;
+        let runs = spread_runs(&args, &[8 * mib; 4], || 2);
+        assert_eq!(lens(&runs), [2, 2]);
+    }
+
     #[test]
     fn spread_isolates_a_dominant_file() {
         let args = args(&["a", "b", "c"]);

dirsql-0.4.108/packages/python/tests/e2e/speed_of_light.py

@@ -1,124 +0,0 @@
-"""Harness for the use-case speed-of-light tests.
-
-Each use-case test asks one question of a generated corpus twice: once through
-the fastest native pipeline (find, sort, join, awk, a single parser process)
-and once through the real `dirsql` launcher. The rows must be identical and
-dirsql's wall time, after subtracting the launcher's measured startup, must be
-within ten percent of native. The bar is a ratio, never a millisecond number,
-so it holds on any machine.
-
-Startup is the mean of ten `dirsql --help` runs; `--help` and `SELECT 1` cost
-the same, so it stands in for the fixed per-invocation cost. A corpus too
-small to time is grown by doubling until the native pipeline takes at least a
-second, so the comparison is above the noise floor. Each side is run three
-times and the fastest run counts.
-"""
-
-from __future__ import annotations
-
-import json
-import shutil
-import subprocess
-import time
-
-ONE_SECOND = 1.0
-TOLERANCE = 1.1
-STARTUP_PROBES = 10
-BEST_OF = 3
-
-
-def cli() -> str:
-    dirsql = shutil.which("dirsql")
-    assert dirsql is not None, (
-        "`dirsql` console script not on PATH -- run `uv run maturin develop`"
-    )
-    return dirsql
-
-
-def timed(argv, cwd, timeout, env=None):
-    started = time.perf_counter()
-    proc = subprocess.run(
-        argv,
-        cwd=str(cwd),
-        stdin=subprocess.DEVNULL,
-        capture_output=True,
-        text=True,
-        timeout=timeout,
-        env=env,
-        check=False,
-    )
-    return proc, time.perf_counter() - started
-
-
-def startup_seconds(argv=None, cwd=None) -> float:
-    argv = [cli(), "--help"] if argv is None else argv
-    samples = []
-    for _ in range(STARTUP_PROBES):
-        proc, seconds = timed(argv, cwd or ".", timeout=60)
-        assert proc.returncode == 0, proc.stderr
-        samples.append(seconds)
-    return sum(samples) / len(samples)
-
-
-def dirsql_rows(proc, columns):
-    assert proc.returncode == 0, (
-        f"dirsql failed: stdout={proc.stdout!r} stderr={proc.stderr!r}"
-    )
-    return [tuple(row[column] for column in columns) for row in json.loads(proc.stdout)]
-
-
-def grow_until_native_takes_a_second(build, native, start, ceiling):
-    """Grow the corpus by doubling until `native` runs for a second.
-
-    `build(lo, hi)` adds items `lo` through `hi - 1` to the corpus already
-    holding `lo` items, so each doubling writes only the new half.
-    """
-    built, size = 0, start
-    while True:
-        build(built, size)
-        built = size
-        _, seconds = native()
-        if seconds >= ONE_SECOND or size >= ceiling:
-            return size
-        size *= 2
-
-
-def best_of(run, name, hopeless=None):
-    """Run `run` three times; return the rows and wall time of the fastest.
-
-    `run` returns `(rows, seconds)`. A run that raises `TimeoutExpired` fails
-    the test immediately with the timeout it blew through. A run slower than
-    `hopeless` seconds ends the series early: noise does not account for an
-    order of magnitude, so no retry could bring the best under the bar.
-    """
-    fastest = None
-    for _ in range(BEST_OF):
-        try:
-            rows, seconds = run()
-        except subprocess.TimeoutExpired as expired:
-            raise AssertionError(
-                f"{name} did not finish within {expired.timeout:.0f}s"
-            ) from None
-        if fastest is None or seconds < fastest[1]:
-            fastest = (rows, seconds)
-        if hopeless is not None and seconds > hopeless:
-            break
-    return fastest
-
-
-def hopeless_seconds(native_seconds: float, startup: float) -> float:
-    return startup + 10 * TOLERANCE * native_seconds
-
-
-def dirsql_timeout(native_seconds: float) -> float:
-    return max(60.0, 30 * native_seconds)
-
-
-def assert_speed_of_light(name, native_seconds, dirsql_seconds, startup):
-    measured = dirsql_seconds - startup
-    budget = TOLERANCE * native_seconds
-    assert measured <= budget, (
-        f"{name}: dirsql {dirsql_seconds:.3f}s minus startup {startup:.3f}s"
-        f" = {measured:.3f}s, over {TOLERANCE}x native {native_seconds:.3f}s"
-        f" = {budget:.3f}s"
-    )