dirsql 0.4.107__tar.gz → 0.4.109__tar.gz

{dirsql-0.4.107 → dirsql-0.4.109}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.107"
+version = "0.4.109"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.107"
+version = "0.4.109"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.107 → dirsql-0.4.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.107
+Version: 0.4.109
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.107 → dirsql-0.4.109}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.107"
+version = "0.4.109"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.109/packages/python/e2e-attestations/bot-1416-watch-speed-1008.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/watch_speed_test.py tests/e2e/on_file_query_test.py tests/e2e/server_speed_test.py -q -p no:cacheprovider",
+  "ran_at": 1791548609,
+  "exit_code": 0,
+  "commit": "ea38a739b990bc5135cc32ba9b61193fd0a2c1e1",
+  "branch": "bot/1416-watch-speed-1008"
+}

dirsql-0.4.109/packages/python/e2e-attestations/bot-1461-row-footprint-1008.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/query_contract_test.py tests/e2e/on_file_query_test.py tests/e2e/stat_speed_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/use_case_dossier_test.py -x -q",
+  "ran_at": 1791489188,
+  "exit_code": 0,
+  "commit": "fe75eb79358f5324a5af3395d1e2383e6233c98f",
+  "branch": "bot/1461-row-footprint-1008"
+}

dirsql-0.4.109/packages/python/e2e-attestations/bot-1462-paired-speed-harness-1008.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/glob_speed_one_level_test.py -x -q -rA",
+  "ran_at": 1791492173,
+  "exit_code": 0,
+  "commit": "ce22059009267d7759e47aa47cba10e28fd6543c",
+  "branch": "bot/1462-paired-speed-harness-1008"
+}

dirsql-0.4.109/packages/python/tests/e2e/speed_of_light.py

@@ -0,0 +1,205 @@
+"""Harness for the use-case speed-of-light tests.
+
+Each use-case test asks one question of a generated corpus twice: once through
+the fastest native pipeline (find, sort, join, awk, a single parser process)
+and once through the real `dirsql` launcher. The rows must be identical and
+dirsql's wall time, after subtracting the launcher's measured startup, must be
+within ten percent of native. The bar is a ratio, never a millisecond number,
+so it holds on any machine.
+
+Native and dirsql run as alternating pairs (native first, then dirsql first,
+and so on) after one discarded warm-up pair, so drift in machine speed hits
+both sides alike. The verdict is the median of the per-pair ratios. Startup is
+the median of `dirsql query 'SELECT 1'` runs with the first dropped, and
+native's own is the median of `bash -c true` the same way; each is subtracted
+from its side. A corpus too small to time is grown by doubling until the
+native pipeline takes a second, unless the corpus ceiling stops it sooner.
+Output is captured as bytes and decoded after the clock stops.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import statistics
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass
+from typing import NamedTuple
+
+ONE_SECOND = 1.0
+TOLERANCE = 1.1
+STARTUP_PROBES = 11
+PAIRS = 100
+HOPELESS_STREAK = 3
+NATIVE_ENV = {**os.environ, "LC_ALL": "C"}
+
+
+class Startup(NamedTuple):
+    dirsql: float
+    native: float
+
+
+@dataclass
+class Paired:
+    native_rows: object
+    dirsql_rows: object
+    native_seconds: list
+    dirsql_seconds: list
+
+
+def cli() -> str:
+    dirsql = shutil.which("dirsql")
+    assert dirsql is not None, (
+        "`dirsql` console script not on PATH -- run `uv run maturin develop`"
+    )
+    return dirsql
+
+
+def timed(argv, cwd, timeout, env=None):
+    started = time.perf_counter()
+    proc = subprocess.run(
+        argv,
+        cwd=str(cwd),
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        timeout=timeout,
+        env=env,
+        check=False,
+    )
+    seconds = time.perf_counter() - started
+    proc.stdout = proc.stdout.decode()
+    proc.stderr = proc.stderr.decode()
+    return proc, seconds
+
+
+def timed_native(argv, cwd, timeout=600):
+    return timed(argv, cwd, timeout, env=NATIVE_ENV)
+
+
+def median_after_first(samples):
+    return statistics.median(samples[1:])
+
+
+def probe_seconds(argv, cwd, env=None) -> float:
+    samples = []
+    for _ in range(STARTUP_PROBES):
+        proc, seconds = timed(argv, cwd, timeout=60, env=env)
+        assert proc.returncode == 0, proc.stderr
+        samples.append(seconds)
+    return median_after_first(samples)
+
+
+def startup_seconds(argv=None, cwd=None) -> Startup:
+    argv = [cli(), "query", "SELECT 1"] if argv is None else argv
+    with tempfile.TemporaryDirectory() as empty:
+        dirsql = probe_seconds(argv, cwd or empty)
+        native = probe_seconds(["bash", "-c", "true"], empty, NATIVE_ENV)
+    return Startup(dirsql, native)
+
+
+def dirsql_rows(proc, columns):
+    assert proc.returncode == 0, (
+        f"dirsql failed: stdout={proc.stdout!r} stderr={proc.stderr!r}"
+    )
+    return [tuple(row[column] for column in columns) for row in json.loads(proc.stdout)]
+
+
+def grow_until_native_takes_a_second(build, native, start, ceiling):
+    """Grow the corpus by doubling until `native` runs for a second.
+
+    `build(lo, hi)` adds items `lo` through `hi - 1` to the corpus already
+    holding `lo` items, so each doubling writes only the new half.
+    """
+    built, size = 0, start
+    while True:
+        build(built, size)
+        os.sync()
+        built = size
+        _, seconds = native()
+        if seconds >= ONE_SECOND or size >= ceiling:
+            return size
+        size *= 2
+
+
+def dirsql_timeout(native_seconds: float) -> float:
+    return max(60.0, 30 * native_seconds)
+
+
+def pair_ratios(native_seconds, dirsql_seconds, startup):
+    ratios = []
+    for native, dirsql in zip(native_seconds, dirsql_seconds):
+        net = native - startup.native
+        assert net > 0, (
+            f"native {native:.3f}s is under its startup {startup.native:.3f}s"
+        )
+        ratios.append((dirsql - startup.dirsql) / net)
+    return ratios
+
+
+def is_hopeless(native, dirsql, startup):
+    return dirsql - startup.dirsql > 10 * TOLERANCE * (native - startup.native)
+
+
+def run_or_fail(run, name):
+    try:
+        return run()
+    except subprocess.TimeoutExpired as expired:
+        raise AssertionError(
+            f"{name} did not finish within {expired.timeout:.0f}s"
+        ) from None
+
+
+def paired(native, dirsql, startup, pairs=PAIRS):
+    """Alternate `native()` and `dirsql(timeout)`, native first on even pairs.
+
+    Both return `(rows, seconds)`. The first pair is a discarded warm-up that
+    also sets dirsql's timeout. A run that raises `TimeoutExpired` fails the
+    test at once. Three hopeless pairs in a row end the series early: noise
+    does not account for an order of magnitude, so more pairs cannot bring
+    the median under the bar.
+    """
+    native_rows, native_warm = run_or_fail(native, "native")
+    timeout = dirsql_timeout(native_warm)
+
+    def run_dirsql():
+        return run_or_fail(lambda: dirsql(timeout), "dirsql")
+
+    dirsql_rows_, _ = run_dirsql()
+    native_times, dirsql_times, streak = [], [], 0
+    for index in range(pairs):
+        if index % 2 == 0:
+            native_rows, native_seconds = run_or_fail(native, "native")
+            dirsql_rows_, dirsql_seconds = run_dirsql()
+        else:
+            dirsql_rows_, dirsql_seconds = run_dirsql()
+            native_rows, native_seconds = run_or_fail(native, "native")
+        native_times.append(native_seconds)
+        dirsql_times.append(dirsql_seconds)
+        streak = (
+            streak + 1 if is_hopeless(native_seconds, dirsql_seconds, startup) else 0
+        )
+        if streak == HOPELESS_STREAK:
+            break
+    return Paired(native_rows, dirsql_rows_, native_times, dirsql_times)
+
+
+def summarize(ratios):
+    low, high = min(ratios), max(ratios)
+    median = statistics.median(ratios)
+    return low, median, high, (high - low) / median
+
+
+def assert_speed_of_light(name, result, startup):
+    ratios = pair_ratios(result.native_seconds, result.dirsql_seconds, startup)
+    low, median, high, spread = summarize(ratios)
+    line = (
+        f"{name}: {len(ratios)} pairs, ratio min {low:.3f} median {median:.3f}"
+        f" max {high:.3f} spread {spread:.0%}, native median"
+        f" {statistics.median(result.native_seconds):.3f}s, startup"
+        f" dirsql {startup.dirsql:.3f}s native {startup.native:.3f}s"
+    )
+    print(f"SPEED {line}")
+    assert median <= TOLERANCE, f"{line}; median is over {TOLERANCE}x"

{dirsql-0.4.107 → dirsql-0.4.109}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.107"
+version = "0.4.109"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.107 → dirsql-0.4.109}/packages/rust/src/db.rs

@@ -779,7 +779,7 @@ impl Db {
         };
         let mut stored: Vec<(i64, Vec<Value>)> = Vec::new();
         {
-            let mut stmt = self.conn.prepare(&select)?;
+            let mut stmt = self.conn.prepare_cached(&select)?;
             let mut cursor = stmt.query([])?;
             while let Some(row) = cursor.next()? {
                 let cells = (0..columns.len())

{dirsql-0.4.107 → dirsql-0.4.109}/packages/rust/src/lib.rs

@@ -655,15 +655,31 @@ impl DirSQL {
     /// the TypeScript binding).
     #[doc(hidden)]
     pub fn apply_file_events(&self, events: Vec<FileEvent>) -> Vec<RowEvent> {
+        self.apply_file_events_draining(events, Vec::new)
+    }
+
+    /// [`apply_file_events`](Self::apply_file_events), asking `late` for events
+    /// that arrived while the batch was being routed, until it returns none,
+    /// before any per-table hook runs. One write reaches the watcher as
+    /// several events; a hook run for the first would run again for the rest.
+    fn apply_file_events_draining(
+        &self,
+        events: Vec<FileEvent>,
+        mut late: impl FnMut() -> Vec<FileEvent>,
+    ) -> Vec<RowEvent> {
         let mut pending = PendingRefresh::default();
         let mut out = Vec::new();
         let mut rescan = false;
-        for fe in events {
-            if fe == FileEvent::Rescan {
-                rescan = true;
-                continue;
+        let mut batch = events;
+        while !batch.is_empty() {
+            for fe in batch {
+                if fe == FileEvent::Rescan {
+                    rescan = true;
+                    continue;
+                }
+                out.extend(self.route_file_event(fe, &mut pending));
             }
-            out.extend(self.route_file_event(fe, &mut pending));
+            batch = late();
         }
         if rescan {
             out.extend(self.rescan(&mut pending));
@@ -710,7 +726,17 @@ impl DirSQL {
             events
         };
 
-        Ok(self.apply_file_events(file_events))
+        Ok(self.apply_file_events_draining(file_events, || self.queued_file_events()))
+    }
+
+    /// The events the watcher already holds, without waiting for more.
+    fn queued_file_events(&self) -> Vec<FileEvent> {
+        self.inner
+            .watcher
+            .lock()
+            .ok()
+            .and_then(|guard| guard.as_ref().map(Watcher::try_recv_all))
+            .unwrap_or_default()
     }
 
     /// Process a single [`FileEvent`], mutating the DB and cache as needed.
@@ -4135,6 +4161,18 @@ mod internal_tests {
         assert!(e2.to_string().contains("watch() is active"), "got: {e2}");
     }
 
+    #[test]
+    fn queued_file_events_hands_over_what_the_watcher_holds_without_waiting() {
+        let (dir, db) = simple_db();
+        assert!(db.queued_file_events().is_empty(), "no watcher yet");
+        db.start_watching().unwrap();
+        std::fs::write(dir.path().join("a.none"), "x").unwrap();
+        let mut held = Vec::new();
+        while !held.iter().any(|e| matches!(e, FileEvent::Created(_))) {
+            held.extend(db.queued_file_events());
+        }
+    }
+
     #[test]
     fn poll_events_locks_out_watch() {
         let (_dir, db) = simple_db();
@@ -5503,6 +5541,31 @@ mod internal_tests {
         assert_eq!(row_names(&db), vec!["a.txt", "b.txt"]);
     }
 
+    #[test]
+    fn events_arriving_while_a_batch_is_routed_share_its_per_table_rerun() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let a = root.join("a.txt");
+        let b = root.join("b.txt");
+        let fake = FakeFs::default().with_subtree(
+            root.clone(),
+            vec![(a.clone(), "items".into()), (b.clone(), "items".into())],
+        );
+        let (items, calls) = recording_per_table("items", "**/*.txt");
+        let db =
+            DirSQL::with_ignore_and_fs(&root, vec![items], Vec::<String>::new(), Arc::new(fake))
+                .unwrap();
+
+        let mut arrivals = vec![vec![FileEvent::Modified(b.clone())]];
+        let events = db.apply_file_events_draining(vec![FileEvent::Created(a.clone())], || {
+            arrivals.pop().unwrap_or_default()
+        });
+
+        assert_eq!(*calls.lock().unwrap(), vec![vec![a, b]]);
+        assert_eq!(events.len(), 2);
+        assert_eq!(row_names(&db), vec!["a.txt", "b.txt"]);
+    }
+
     #[test]
     fn apply_file_events_refreshes_a_per_table_hook_without_walking_the_tree() {
         let dir = TempDir::new().unwrap();

{dirsql-0.4.107 → dirsql-0.4.109}/packages/rust/src/on_file.rs

@@ -73,12 +73,16 @@ pub(crate) fn run_streaming(
         .iter()
         .map(|path| command::non_verbatim(&path.to_string_lossy()))
         .collect();
-    let cpus = std::thread::available_parallelism().map_or(1, std::num::NonZero::get);
     let mut runs = chunks(&args, ARG_BUDGET.saturating_sub(argv_bytes(&argv)));
     if runs.len() == 1 {
-        runs = spread(&args, &file_sizes(paths), cpus);
+        runs = spread_runs(&args, &file_sizes(paths), cpus);
     }
-    let workers = cpus.min(runs.len());
+    if let [only] = runs.as_slice() {
+        let mut full = argv;
+        full.extend(only.iter().cloned());
+        return run_invocation(command, &full, cwd, &mut |rows| sink(rows));
+    }
+    let workers = cpus().min(runs.len());
 
     let next = AtomicUsize::new(0);
     let aborted = AtomicBool::new(false);
@@ -97,7 +101,7 @@ pub(crate) fn run_streaming(
                     }
                     let mut full = argv.clone();
                     full.extend(runs[index].iter().cloned());
-                    let outcome = run_invocation(command, &full, cwd, &|rows| {
+                    let outcome = run_invocation(command, &full, cwd, &mut |rows| {
                         let _ = rows_tx.send(rows);
                     });
                     if let Err(message) = outcome {
@@ -134,7 +138,7 @@ fn run_invocation(
     command: &str,
     full: &[String],
     cwd: &Path,
-    send: &dyn Fn(Vec<JsonRow>),
+    send: &mut dyn FnMut(Vec<JsonRow>),
 ) -> Result<(), String> {
     let mut bad_output: Option<String> = None;
     let mut lines_seen = 0usize;
@@ -208,6 +212,24 @@ fn chunks(args: &[String], budget: usize) -> Vec<&[String]> {
 /// another invocation costs more than it saves.
 const MIN_RUN_BYTES: u64 = 4 * 1024 * 1024;
 
+fn cpus() -> usize {
+    std::thread::available_parallelism().map_or(1, std::num::NonZero::get)
+}
+
+/// [`spread`] over the machine's CPUs, asking for the count only when the
+/// data could fill more than one run: the query reads cgroup files each time.
+fn spread_runs<'a>(
+    args: &'a [String],
+    sizes: &[u64],
+    cpus: impl FnOnce() -> usize,
+) -> Vec<&'a [String]> {
+    let by_data = usize::try_from(sizes.iter().sum::<u64>() / MIN_RUN_BYTES).unwrap_or(usize::MAX);
+    if by_data.min(args.len()) <= 1 {
+        return vec![args];
+    }
+    spread(args, sizes, cpus())
+}
+
 /// Each path's size in bytes; a path that cannot be read counts as empty.
 fn file_sizes(paths: &[PathBuf]) -> Vec<u64> {
     paths
@@ -423,6 +445,22 @@ mod tests {
         assert_eq!(lens(&spread(&args, &[mib; 4], 16)), [4]);
     }
 
+    #[test]
+    fn spread_runs_does_not_count_cpus_for_data_that_fits_one_run() {
+        let args = args(&["a", "b", "c", "d"]);
+        let mib = 1024 * 1024;
+        let runs = spread_runs(&args, &[mib; 4], || panic!("counted the cpus"));
+        assert_eq!(lens(&runs), [4]);
+    }
+
+    #[test]
+    fn spread_runs_counts_cpus_once_the_data_fills_more_than_one_run() {
+        let args = args(&["a", "b", "c", "d"]);
+        let mib = 1024 * 1024;
+        let runs = spread_runs(&args, &[8 * mib; 4], || 2);
+        assert_eq!(lens(&runs), [2, 2]);
+    }
+
     #[test]
     fn spread_isolates_a_dominant_file() {
         let args = args(&["a", "b", "c"]);

{dirsql-0.4.107 → dirsql-0.4.109}/packages/rust/src/vtab.rs

@@ -130,7 +130,7 @@ fn read_contents(rows: &[&FileRow], read: &(dyn Fn(&FileRow) -> Option<String> +
         .filter(|row| row.content.get().is_none())
         .collect();
     share_out(&unread, READERS, &|row| {
-        row.content.get_or_init(|| read(row));
+        row.content.get_or_init(|| Box::new(read(row)));
     });
 }
 
@@ -146,7 +146,7 @@ fn stat_rows(rows: &[&FileRow], stat: &(dyn Fn(&FileRow) -> StatFacts + Sync)) {
         .collect();
     let workers = thread::available_parallelism().map_or(1, usize::from);
     share_out(&unstatted, workers, &|row| {
-        row.facts.get_or_init(|| stat(row));
+        row.facts.get_or_init(|| Box::new(stat(row)));
     });
 }
 
@@ -196,11 +196,11 @@ struct FileRow {
     spans: PathSpans,
     /// Stat'ed at most once per statement, and only when the statement reads
     /// a stat column or the content.
-    facts: OnceLock<StatFacts>,
+    facts: OnceLock<Box<StatFacts>>,
     /// The file's text once a statement has asked for it, read at most once
     /// per statement: ahead for every row when the statement names the
     /// column, or on demand for the row being emitted.
-    content: OnceLock<Option<String>>,
+    content: OnceLock<Box<Option<String>>>,
 }
 
 impl FileRow {
@@ -217,15 +217,15 @@ impl FileRow {
     }
 
     fn basename(&self) -> Option<&str> {
-        self.spans.name_at.map(|at| &self.path[at..])
+        self.spans.name_at.map(|at| &self.path[at as usize..])
     }
 
     fn dir(&self) -> Option<&str> {
-        self.spans.dir_len.map(|len| &self.path[..len])
+        self.spans.dir_len.map(|len| &self.path[..len as usize])
     }
 
     fn ext(&self) -> Option<&str> {
-        self.spans.ext_at.map(|at| &self.path[at..])
+        self.spans.ext_at.map(|at| &self.path[at as usize..])
     }
 }
 
@@ -236,19 +236,20 @@ impl FileRow {
 /// one length each locates them.
 #[derive(Debug, PartialEq, Eq)]
 struct PathSpans {
-    name_at: Option<usize>,
-    dir_len: Option<usize>,
-    ext_at: Option<usize>,
+    name_at: Option<u32>,
+    dir_len: Option<u32>,
+    ext_at: Option<u32>,
 }
 
 impl PathSpans {
     fn of(path: &str) -> Self {
         let p = Path::new(path);
-        let suffix_at = |s: &std::ffi::OsStr| path.len() - s.len();
+        let span = |at: usize| u32::try_from(at).ok();
+        let suffix_at = |s: &std::ffi::OsStr| span(path.len() - s.len());
         Self {
-            name_at: p.file_name().map(suffix_at),
-            dir_len: p.parent().map(|d| d.as_os_str().len()),
-            ext_at: p.extension().map(suffix_at),
+            name_at: p.file_name().and_then(suffix_at),
+            dir_len: p.parent().and_then(|d| span(d.as_os_str().len())),
+            ext_at: p.extension().and_then(suffix_at),
         }
     }
 }
@@ -374,7 +375,7 @@ impl ScanSpec {
 
     fn facts(&self, row: &FileRow) -> StatFacts {
         if let Some(facts) = row.facts.get() {
-            return *facts;
+            return **facts;
         }
         let current = self
             .current
@@ -383,7 +384,7 @@ impl ScanSpec {
         if let Some(rows) = current.upgrade() {
             stat_rows(&rows.iter().collect::<Vec<_>>(), &|row| self.stat(row));
         }
-        *row.facts.get_or_init(|| self.stat(row))
+        **row.facts.get_or_init(|| Box::new(self.stat(row)))
     }
 
     fn read(&self, row: &FileRow) -> Option<String> {
@@ -438,7 +439,7 @@ impl TableSource for ScanSpec {
             Some(value) => ctx.set_result(&ToSqlOutput::Borrowed(value)),
             // The one effectful read, reached only when a query names the
             // column: this is where laziness actually lives.
-            None => ctx.set_result(row.content.get_or_init(|| self.read(row))),
+            None => ctx.set_result(&**row.content.get_or_init(|| Box::new(self.read(row)))),
         }
     }
 
@@ -674,7 +675,7 @@ mod tests {
 
     fn sized_row(rel: &str, size: i64) -> FileRow {
         let row = FileRow::new(Path::new(""), PathBuf::from(rel));
-        row.facts.set(facts(size)).unwrap();
+        row.facts.set(Box::new(facts(size))).unwrap();
         row
     }
 
@@ -721,7 +722,10 @@ mod tests {
 
         spec.prefetch(&rows);
 
-        assert_eq!(row.content.get(), Some(&Some("contents".to_owned())));
+        assert_eq!(
+            row.content.get().map(|c| &**c),
+            Some(&Some("contents".to_owned()))
+        );
     }
 
     fn facts(size: i64) -> StatFacts {
@@ -873,7 +877,7 @@ mod tests {
     #[test]
     fn stat_rows_stats_each_unstatted_row_once() {
         let rows: Vec<FileRow> = (0..40).map(|n| row_for("", &format!("f{n}.md"))).collect();
-        rows[3].facts.set(facts(1)).unwrap();
+        rows[3].facts.set(Box::new(facts(1))).unwrap();
         let stats = AtomicUsize::new(0);
         let stat = |row: &FileRow| {
             stats.fetch_add(1, Ordering::Relaxed);
@@ -883,8 +887,8 @@ mod tests {
         stat_rows(&rows.iter().collect::<Vec<_>>(), &stat);
 
         assert_eq!(stats.load(Ordering::Relaxed), 39);
-        assert_eq!(rows[3].facts.get(), Some(&facts(1)));
-        assert_eq!(rows[7].facts.get(), Some(&facts(7)));
+        assert_eq!(rows[3].facts.get().map(|f| **f), Some(facts(1)));
+        assert_eq!(rows[7].facts.get().map(|f| **f), Some(facts(7)));
         stat_rows(&[], &|_| panic!("nothing to stat"));
     }
 
@@ -1002,7 +1006,10 @@ mod tests {
     #[test]
     fn read_contents_reads_each_unread_row_once() {
         let rows: Vec<FileRow> = (0..40).map(|n| row_for("", &format!("f{n}.md"))).collect();
-        rows[3].content.set(Some("already".to_string())).unwrap();
+        rows[3]
+            .content
+            .set(Box::new(Some("already".to_string())))
+            .unwrap();
         let reads = std::sync::Mutex::new(Vec::new());
         let read = |row: &FileRow| {
             reads.lock().unwrap().push(row.path.clone());
@@ -1033,13 +1040,13 @@ mod tests {
     fn read_contents_keeps_a_null_read() {
         let rows = [row_for("", "a.md")];
         read_contents(&[&rows[0]], &|_| None);
-        assert_eq!(rows[0].content.get(), Some(&None));
+        assert_eq!(rows[0].content.get().map(|c| &**c), Some(&None));
     }
 
     #[test]
     fn read_contents_reads_nothing_when_every_row_is_read() {
         let rows = [row_for("", "a.md")];
-        rows[0].content.set(None).unwrap();
+        rows[0].content.set(Box::new(None)).unwrap();
         read_contents(&[&rows[0]], &|_| {
             panic!("a row read already is not read again")
         });
@@ -1142,4 +1149,28 @@ mod tests {
         assert_eq!(epoch_secs(Some(t)), None);
         assert_eq!(epoch_secs(None), None);
     }
+
+    #[test]
+    fn a_row_not_yet_stat_or_read_stays_compact() {
+        let size = std::mem::size_of::<FileRow>();
+        assert!(size <= 112, "a FileRow is {size} bytes");
+    }
+    #[test]
+    fn a_content_column_reads_the_file_through_the_module() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(dir.path().join("a.md"), "hi").unwrap();
+        let conn = Connection::open_in_memory().unwrap();
+        load_module(&conn, StatementScope::new()).unwrap();
+        conn.execute_batch(&format!(
+            "CREATE VIRTUAL TABLE t USING dirsql_path('{}', '*.md', '', 'no-gitignore')",
+            dir.path().display()
+        ))
+        .unwrap();
+
+        let content: String = conn
+            .query_row("SELECT content FROM t", [], |row| row.get(0))
+            .unwrap();
+
+        assert_eq!(content, "hi");
+    }
 }