dirsql 0.4.18__tar.gz → 0.4.19__tar.gz

{dirsql-0.4.18 → dirsql-0.4.19}/Cargo.lock

@@ -499,7 +499,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.18"
+version = "0.4.19"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.18 → dirsql-0.4.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.18
+Version: 0.4.19
 Requires-Dist: tomli>=2 ; python_full_version < '3.11'
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory

{dirsql-0.4.18/packages/rust → dirsql-0.4.19}/docs/howto/persist.md

@@ -55,6 +55,19 @@ can't be trusted at all — the table/ignore configuration changed, or the
 automatically. You never need to delete it by hand; a full rebuild costs
 exactly what a non-persistent startup does.
 
+A run that changes nothing writes nothing: the cache file is read and left
+exactly as it was.
+
+This covers both kinds of table. Declared tables keep their indexed rows, and
+a [path-table](../reference/path-tables.md) parsed with `--on-file` keeps each
+file's parser output — so the second run over an unchanged tree spawns no
+parser process at all. A path-table's cached rows are keyed by the tree, the
+glob, the parser command and the `dirsql` version together, so changing any of
+them parses afresh rather than serving rows the current command never
+produced. A *stat* path-table caches nothing, because there is nothing to
+save: its columns are the metadata the scan already collected, and `content`
+is read live by design.
+
 Persistence is a startup-time optimization, not a change in meaning: the
 database remains a derived view of your files, and queries return the same
 rows either way ([how `dirsql` thinks](../explanation.md)).

{dirsql-0.4.18/packages/rust → dirsql-0.4.19}/docs/reference/path-tables.md

@@ -140,6 +140,12 @@ the table:
   scan continues. The schema is inferred from the files that did parse.
 - **The skip rules still apply.** A parsed scan honors the same `node_modules`
   /`.git`/`ignore` rules a stat scan does (see below).
+- **`--persist` skips the parser for unchanged files.** With a
+  [persistent cache](/howto/persist), each file's parser output is stored
+  against its stat metadata, so a later run over an unchanged tree serves the
+  rows from the cache and spawns no process. Change the file, the glob, the
+  parser command, or the dirsql version and that file (or that whole table) is
+  parsed again.
 
 `--on-file` applies to **every** path-table in the query and may be given **at
 most once**. For different parsers per file set, define named tables in a
@@ -160,15 +166,17 @@ deleted *after* the scan finds it but *before* its `content` is read yields
 than failing the query. This is an accepted consequence of reading live, not a
 bug to design around.
 
-Path-tables are per-connection and are never written to a persistent cache, so
-they cannot leak into `sqlite_master` or survive a restart. The reserved
-top-level `.dirsql/` directory is excluded from the scan, as everywhere else.
+The table itself is per-connection: it lives in `temp`, so it cannot leak into
+`sqlite_master` or survive a restart. Under `--persist` a *parsed* table's rows
+outlive the connection in the cache (above), but the table is still minted
+fresh each run and the scan still decides what exists. The reserved top-level
+`.dirsql/` directory is excluded from the scan, as everywhere else.
 
 ### When to promote to a declared table
 
-Every query re-scans the filesystem: a path-table has no index, no watcher, and
-no persistent cache. That is the right trade for a hundreds-of-files, run-it-once
-question. When the same tree is queried repeatedly, or is large, declare a
+Every query re-scans the filesystem: a path-table has no index and no watcher.
+That is the right trade for a hundreds-of-files, run-it-once question. When the
+same tree is queried repeatedly, or is large, declare a
 [table](/reference/config) for it instead — a declared table is indexed on
 build, kept fresh by the watcher, and (with `--persist`) survives restarts, so
 its rows are read from SQLite rather than re-walked each time.

{dirsql-0.4.18 → dirsql-0.4.19}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.18"
+version = "0.4.19"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

{dirsql-0.4.18 → dirsql-0.4.19}/packages/python/docs/howto/persist.md

@@ -55,6 +55,19 @@ can't be trusted at all — the table/ignore configuration changed, or the
 automatically. You never need to delete it by hand; a full rebuild costs
 exactly what a non-persistent startup does.
 
+A run that changes nothing writes nothing: the cache file is read and left
+exactly as it was.
+
+This covers both kinds of table. Declared tables keep their indexed rows, and
+a [path-table](../reference/path-tables.md) parsed with `--on-file` keeps each
+file's parser output — so the second run over an unchanged tree spawns no
+parser process at all. A path-table's cached rows are keyed by the tree, the
+glob, the parser command and the `dirsql` version together, so changing any of
+them parses afresh rather than serving rows the current command never
+produced. A *stat* path-table caches nothing, because there is nothing to
+save: its columns are the metadata the scan already collected, and `content`
+is read live by design.
+
 Persistence is a startup-time optimization, not a change in meaning: the
 database remains a derived view of your files, and queries return the same
 rows either way ([how `dirsql` thinks](../explanation.md)).

{dirsql-0.4.18 → dirsql-0.4.19}/packages/python/docs/reference/path-tables.md

@@ -140,6 +140,12 @@ the table:
   scan continues. The schema is inferred from the files that did parse.
 - **The skip rules still apply.** A parsed scan honors the same `node_modules`
   /`.git`/`ignore` rules a stat scan does (see below).
+- **`--persist` skips the parser for unchanged files.** With a
+  [persistent cache](/howto/persist), each file's parser output is stored
+  against its stat metadata, so a later run over an unchanged tree serves the
+  rows from the cache and spawns no process. Change the file, the glob, the
+  parser command, or the dirsql version and that file (or that whole table) is
+  parsed again.
 
 `--on-file` applies to **every** path-table in the query and may be given **at
 most once**. For different parsers per file set, define named tables in a
@@ -160,15 +166,17 @@ deleted *after* the scan finds it but *before* its `content` is read yields
 than failing the query. This is an accepted consequence of reading live, not a
 bug to design around.
 
-Path-tables are per-connection and are never written to a persistent cache, so
-they cannot leak into `sqlite_master` or survive a restart. The reserved
-top-level `.dirsql/` directory is excluded from the scan, as everywhere else.
+The table itself is per-connection: it lives in `temp`, so it cannot leak into
+`sqlite_master` or survive a restart. Under `--persist` a *parsed* table's rows
+outlive the connection in the cache (above), but the table is still minted
+fresh each run and the scan still decides what exists. The reserved top-level
+`.dirsql/` directory is excluded from the scan, as everywhere else.
 
 ### When to promote to a declared table
 
-Every query re-scans the filesystem: a path-table has no index, no watcher, and
-no persistent cache. That is the right trade for a hundreds-of-files, run-it-once
-question. When the same tree is queried repeatedly, or is large, declare a
+Every query re-scans the filesystem: a path-table has no index and no watcher.
+That is the right trade for a hundreds-of-files, run-it-once question. When the
+same tree is queried repeatedly, or is large, declare a
 [table](/reference/config) for it instead — a declared table is indexed on
 build, kept fresh by the watcher, and (with `--persist`) survives restarts, so
 its rows are read from SQLite rather than re-walked each time.

dirsql-0.4.19/packages/python/e2e-attestations/claude-github-issue-825-e4mhhp.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/ -x -q",
+  "ran_at": 1786485396,
+  "exit_code": 0,
+  "commit": "31831d69029df1e191afbab33cb50818e43dc7df",
+  "branch": "claude/github-issue-825-e4mhhp"
+}

dirsql-0.4.19/packages/rust/changelog.d/2026-08-11-persist-parsed-path-tables.md

@@ -0,0 +1,4 @@
+**Fixed**
+
+- **`--persist` now caches a parsed path-table's rows, so a second run over an unchanged tree re-runs the `on-file` parser for no file.** Previously the persistent cache only covered declared `[[table]]` entries: a path-table (`SELECT ... FROM './**/*.json' --on-file '<command>'`) re-scanned, re-read and re-parsed every file on every run, which made `--persist` pure overhead on the surface a configless `dirsql --persist` actually queries — measured 2.7s cold and 2.7s warm over a 2,000-file tree, versus 2.7s → 0.04s now. Each file's parser output is stored in the cache against its stat tuple (`_dirsql_parsed_rows`, reconciled by the same racy-stat rule declared tables use), keyed by the tree, glob, parser command and `dirsql` version together, so changed files re-parse and a changed command or glob starts fresh. A cache that cannot be updated costs the run its speedup, not its rows: the failure is a one-line stderr warning and the next run re-parses. A *stat* path-table is unaffected either way: its columns are the scan's own metadata and `content` stays a live read. (#825)
+- **A run that changes nothing no longer rewrites the cache file.** The `_dirsql_meta` block was rewritten on every startup, so `cache.db` changed even when the tree had not; it is now written only when the meta actually differs, leaving an unchanged warm start byte-for-byte identical. (#825)

{dirsql-0.4.18 → dirsql-0.4.19/packages/rust}/docs/howto/persist.md

@@ -55,6 +55,19 @@ can't be trusted at all — the table/ignore configuration changed, or the
 automatically. You never need to delete it by hand; a full rebuild costs
 exactly what a non-persistent startup does.
 
+A run that changes nothing writes nothing: the cache file is read and left
+exactly as it was.
+
+This covers both kinds of table. Declared tables keep their indexed rows, and
+a [path-table](../reference/path-tables.md) parsed with `--on-file` keeps each
+file's parser output — so the second run over an unchanged tree spawns no
+parser process at all. A path-table's cached rows are keyed by the tree, the
+glob, the parser command and the `dirsql` version together, so changing any of
+them parses afresh rather than serving rows the current command never
+produced. A *stat* path-table caches nothing, because there is nothing to
+save: its columns are the metadata the scan already collected, and `content`
+is read live by design.
+
 Persistence is a startup-time optimization, not a change in meaning: the
 database remains a derived view of your files, and queries return the same
 rows either way ([how `dirsql` thinks](../explanation.md)).

{dirsql-0.4.18 → dirsql-0.4.19/packages/rust}/docs/reference/path-tables.md

@@ -140,6 +140,12 @@ the table:
   scan continues. The schema is inferred from the files that did parse.
 - **The skip rules still apply.** A parsed scan honors the same `node_modules`
   /`.git`/`ignore` rules a stat scan does (see below).
+- **`--persist` skips the parser for unchanged files.** With a
+  [persistent cache](/howto/persist), each file's parser output is stored
+  against its stat metadata, so a later run over an unchanged tree serves the
+  rows from the cache and spawns no process. Change the file, the glob, the
+  parser command, or the dirsql version and that file (or that whole table) is
+  parsed again.
 
 `--on-file` applies to **every** path-table in the query and may be given **at
 most once**. For different parsers per file set, define named tables in a
@@ -160,15 +166,17 @@ deleted *after* the scan finds it but *before* its `content` is read yields
 than failing the query. This is an accepted consequence of reading live, not a
 bug to design around.
 
-Path-tables are per-connection and are never written to a persistent cache, so
-they cannot leak into `sqlite_master` or survive a restart. The reserved
-top-level `.dirsql/` directory is excluded from the scan, as everywhere else.
+The table itself is per-connection: it lives in `temp`, so it cannot leak into
+`sqlite_master` or survive a restart. Under `--persist` a *parsed* table's rows
+outlive the connection in the cache (above), but the table is still minted
+fresh each run and the scan still decides what exists. The reserved top-level
+`.dirsql/` directory is excluded from the scan, as everywhere else.
 
 ### When to promote to a declared table
 
-Every query re-scans the filesystem: a path-table has no index, no watcher, and
-no persistent cache. That is the right trade for a hundreds-of-files, run-it-once
-question. When the same tree is queried repeatedly, or is large, declare a
+Every query re-scans the filesystem: a path-table has no index and no watcher.
+That is the right trade for a hundreds-of-files, run-it-once question. When the
+same tree is queried repeatedly, or is large, declare a
 [table](/reference/config) for it instead — a declared table is indexed on
 build, kept fresh by the watcher, and (with `--persist`) survives restarts, so
 its rows are read from SQLite rather than re-walked each time.

dirsql-0.4.19/packages/rust/migrations.d/2026-08-11-persist-parsed-path-tables.md

@@ -0,0 +1,58 @@
+### Persisted path-tables reuse a parser's output across runs
+
+#### Summary
+
+Under `--persist`, a path-table parsed with `--on-file` now serves an unchanged
+file's rows from the cache instead of re-running the parser (dirsql#825). No
+API changed and no call site moves; what changes is how often your parser
+command runs. A parser whose output is a function of the file it is handed —
+the documented `on-file` contract, and what declared `[[table]]` hooks have
+always assumed under `--persist` — sees no difference beyond speed. A parser
+that returns something different for the same unchanged file (it reads the
+clock, a network service, or an environment variable) will now be observed
+returning the *previous* run's answer until the file itself changes.
+
+#### Required changes
+
+_None._ Nothing to update: the fix restores the behavior
+`docs/howto/persist.md` already documented.
+
+| Surface | Before | After |
+| ------- | ------ | ----- |
+| `dirsql query "… FROM './**/*.json'" --on-file '<cmd>' --persist` | the parser ran once per file on every run | the parser runs only for files whose stat metadata moved |
+
+#### Deprecations removed
+
+_None._
+
+#### Behavior changes without code changes
+
+- **Parsed path-tables under `--persist`**: previously the parser ran once per
+  matched file on every run; now a file whose size/mtime/ctime/inode/device
+  tuple matches the cache serves its cached rows and the process is not
+  spawned. Cached rows are keyed by the scan root, the glob, the parser command
+  and the `dirsql` version together, so changing any of them re-parses. A
+  parser that is not a pure function of its input file should either be run
+  without `--persist` or made to vary with the file.
+- **The cache file on an unchanged run**: previously every startup rewrote the
+  `_dirsql_meta` block, so `cache.db` changed even when nothing about the tree
+  had; now a run that changes nothing writes nothing and the file is left
+  byte-for-byte identical. Tooling that watched the cache's mtime as a
+  "dirsql ran" signal needs a different signal.
+- **The internal `dirsql_parsed` vtab module** takes a fifth fixed argument
+  (the cache path, empty for none) between the gitignore switch and the ignore
+  patterns. It is `#[doc(hidden)]` plumbing that `dirsql` mints for itself; a
+  hand-written `CREATE VIRTUAL TABLE … USING dirsql_parsed(…)` must add the
+  argument.
+
+#### Verification
+
+```bash
+mkdir -p /tmp/dirsql-825 && cd /tmp/dirsql-825
+printf '[{"id":1}]' > a.json
+dirsql query "SELECT id FROM './*.json'" --on-file 'cat {path}' --persist
+md5sum .dirsql/cache.db
+time dirsql query "SELECT id FROM './*.json'" --on-file 'cat {path}' --persist
+md5sum .dirsql/cache.db
+# expected: the same rows, the second run near-instant, the same md5 both times
+```

{dirsql-0.4.18 → dirsql-0.4.19}/packages/rust/src/db.rs

@@ -114,12 +114,18 @@ fn quote_identifier(s: &str) -> String {
 /// the parsed module instead: its rows and schema come from the parser, not
 /// the stat columns, and the `path_prefix` is irrelevant (a parser wanting a
 /// path emits it). Both forms carry the same ignore rules.
+///
+/// A parsed table also carries `cache` — the persistent cache path when the
+/// index has one — so it can serve an unchanged file's rows without re-running
+/// the parser. The stat module takes no such argument: its columns are the
+/// stat tuple the scan already has, so there is nothing to save.
 fn path_table_ddl(
     name: &str,
     table: &PathTable,
     ignore: &[String],
     gitignore: bool,
     parser: Option<&str>,
+    cache: Option<&Path>,
 ) -> String {
     let (module, mut args) = match parser {
         Some(command) => (
@@ -144,6 +150,13 @@ fn path_table_ddl(
     } else {
         scanner::NO_GITIGNORE_ARG
     }));
+    if parser.is_some() {
+        args.push(quote_literal(
+            &cache
+                .map(|p| p.to_string_lossy().into_owned())
+                .unwrap_or_default(),
+        ));
+    }
     args.extend(ignore.iter().map(|p| quote_literal(p)));
 
     format!(
@@ -222,6 +235,9 @@ pub struct Db {
     /// parser instead of the stat columns: its rows and schema come from the
     /// command's output. `None` keeps the stat path-table behavior.
     path_table_parser: Option<String>,
+    /// The persistent cache a parsed path-table reuses across runs. `None` for
+    /// an ephemeral index, which has nowhere to cache to.
+    path_table_cache: Option<PathBuf>,
 }
 
 /// The skip rules a fresh `Db` starts with.
@@ -256,6 +272,7 @@ impl Db {
             path_table_ignore: default_path_table_ignore(),
             path_table_gitignore: true,
             path_table_parser: None,
+            path_table_cache: None,
         })
     }
 
@@ -277,6 +294,7 @@ impl Db {
             path_table_ignore: default_path_table_ignore(),
             path_table_gitignore: true,
             path_table_parser: None,
+            path_table_cache: None,
         })
     }
 
@@ -313,6 +331,14 @@ impl Db {
         self.path_table_parser = Some(command);
     }
 
+    /// Point every parsed path-table minted on this connection at the
+    /// persistent cache, so an unchanged file's rows are served from it
+    /// instead of re-running the parser. Set only when the index persists;
+    /// an ephemeral index has nowhere to cache to.
+    pub fn set_path_table_cache(&mut self, path: PathBuf) {
+        self.path_table_cache = Some(path);
+    }
+
     /// Borrow the underlying SQLite connection. Internal use only — exposed
     /// to the `persist` module so it can manage the sidecar tables.
     #[doc(hidden)]
@@ -783,6 +809,7 @@ impl Db {
             &self.path_table_ignore,
             self.path_table_gitignore,
             self.path_table_parser.as_deref(),
+            self.path_table_cache.as_deref(),
         ))?;
         Ok(())
     }
@@ -2167,7 +2194,7 @@ mod tests {
 
     #[test]
     fn path_table_ddl_creates_in_temp_if_not_exists() {
-        let ddl = path_table_ddl("./docs/*.md", &docs_path_table(), &[], true, None);
+        let ddl = path_table_ddl("./docs/*.md", &docs_path_table(), &[], true, None, None);
         assert_eq!(
             ddl,
             "CREATE VIRTUAL TABLE IF NOT EXISTS temp.\"./docs/*.md\" \
@@ -2183,16 +2210,16 @@ mod tests {
             path_prefix: "/var/log".to_string(),
         };
         assert!(
-            path_table_ddl("/var/log/*.log", &table, &[], true, None)
+            path_table_ddl("/var/log/*.log", &table, &[], true, None, None)
                 .contains("'/var/log', '*.log', '/var/log', 'gitignore')"),
             "got: {}",
-            path_table_ddl("/var/log/*.log", &table, &[], true, None)
+            path_table_ddl("/var/log/*.log", &table, &[], true, None, None)
         );
     }
 
     #[test]
     fn path_table_ddl_carries_the_no_gitignore_switch() {
-        let ddl = path_table_ddl("./", &docs_path_table(), &[], false, None);
+        let ddl = path_table_ddl("./", &docs_path_table(), &[], false, None, None);
         assert!(
             ddl.ends_with("'', 'no-gitignore')"),
             "gitignore off must emit the no-gitignore switch, got: {ddl}"
@@ -2207,6 +2234,7 @@ mod tests {
             &["node_modules/**".to_string(), "*.tmp".to_string()],
             true,
             None,
+            None,
         );
         assert!(
             ddl.ends_with("'', 'gitignore', 'node_modules/**', '*.tmp')"),
@@ -2222,11 +2250,44 @@ mod tests {
             &[],
             true,
             Some("cat {path}"),
+            None,
         );
         assert_eq!(
             ddl,
             "CREATE VIRTUAL TABLE IF NOT EXISTS temp.\"./docs/*.md\" \
-             USING dirsql_parsed('/root', 'docs/*.md', 'cat {path}', 'gitignore')"
+             USING dirsql_parsed('/root', 'docs/*.md', 'cat {path}', 'gitignore', '')"
+        );
+    }
+
+    #[test]
+    fn path_table_ddl_parser_form_carries_the_cache_path() {
+        let ddl = path_table_ddl(
+            "./docs/*.md",
+            &docs_path_table(),
+            &[],
+            true,
+            Some("cat {path}"),
+            Some(Path::new("/cache/dirsql.db")),
+        );
+        assert!(
+            ddl.ends_with("'cat {path}', 'gitignore', '/cache/dirsql.db')"),
+            "a persisted index points the parsed module at its cache, got: {ddl}"
+        );
+    }
+
+    #[test]
+    fn path_table_ddl_stat_form_takes_no_cache_path() {
+        let ddl = path_table_ddl(
+            "./docs/*.md",
+            &docs_path_table(),
+            &[],
+            true,
+            None,
+            Some(Path::new("/cache/dirsql.db")),
+        );
+        assert!(
+            ddl.ends_with("'', 'gitignore')"),
+            "the stat form has nothing to cache, got: {ddl}"
         );
     }
 
@@ -2243,11 +2304,12 @@ mod tests {
             &["node_modules/**".to_string()],
             true,
             Some("parse.py {path}"),
+            None,
         );
         assert!(
             ddl.ends_with(
                 "USING dirsql_parsed('/var/log', '*.log', 'parse.py {path}', 'gitignore', \
-                 'node_modules/**')"
+                 '', 'node_modules/**')"
             ),
             "the parser form drops the path prefix and keeps ignore rules, got: {ddl}"
         );
@@ -2255,7 +2317,14 @@ mod tests {
 
     #[test]
     fn path_table_ddl_parser_form_quotes_a_command_with_a_quote() {
-        let ddl = path_table_ddl("./", &docs_path_table(), &[], true, Some("sh -c 'echo hi'"));
+        let ddl = path_table_ddl(
+            "./",
+            &docs_path_table(),
+            &[],
+            true,
+            Some("sh -c 'echo hi'"),
+            None,
+        );
         assert!(
             ddl.contains("'sh -c ''echo hi'''"),
             "an embedded quote must be doubled, got: {ddl}"
@@ -2269,6 +2338,17 @@ mod tests {
         assert_eq!(db.path_table_parser.as_deref(), Some("cat {path}"));
     }
 
+    #[test]
+    fn set_path_table_cache_points_parsed_tables_at_the_cache() {
+        let mut db = Db::new().unwrap();
+        assert_eq!(db.path_table_cache, None, "an ephemeral index has no cache");
+        db.set_path_table_cache(PathBuf::from("/cache/dirsql.db"));
+        assert_eq!(
+            db.path_table_cache.as_deref(),
+            Some(Path::new("/cache/dirsql.db"))
+        );
+    }
+
     #[test]
     fn path_table_gitignore_defaults_on() {
         let db = Db::new().unwrap();