dirsql 0.4.41__tar.gz → 0.4.42__tar.gz

{dirsql-0.4.41 → dirsql-0.4.42}/Cargo.lock

@@ -540,7 +540,7 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
 [[package]]
 name = "dirsql"
-version = "0.4.41"
+version = "0.4.42"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -583,7 +583,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.41"
+version = "0.4.42"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.41 → dirsql-0.4.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.41
+Version: 0.4.42
 Requires-Dist: tomli>=2 ; python_full_version < '3.11'
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory

{dirsql-0.4.41 → dirsql-0.4.42}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.41"
+version = "0.4.42"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.42/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1789515494,
+  "exit_code": 0,
+  "commit": "51555d8d0ad35095d69c1568e4478f34feb36e09",
+  "branch": "claude/1080-share-vtab-scaffolding"
+}

dirsql-0.4.42/packages/python/e2e-attestations/claude-1109-watch-moved-out.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1789595962,
+  "exit_code": 0,
+  "commit": "c645cfe8052bdca311060ca475f6ee102246650b",
+  "branch": "claude/1109-watch-moved-out"
+}

{dirsql-0.4.41 → dirsql-0.4.42}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.41"
+version = "0.4.42"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.41 → dirsql-0.4.42}/packages/rust/src/db.rs

@@ -645,6 +645,19 @@ impl Db {
         Ok(out)
     }
 
+    /// Every distinct `(table_name, file_path)` whose file sits beneath the
+    /// directory `dir`, ordered by table then path.
+    pub fn files_under(&self, dir: &str) -> Result<Vec<(String, String)>> {
+        let prefix = format!("{dir}{}", std::path::MAIN_SEPARATOR);
+        let mut stmt = self.conn.prepare(
+            "SELECT DISTINCT table_name, file_path FROM _dirsql_internal_rows \
+             WHERE substr(file_path, 1, length(?1)) = ?1 \
+             ORDER BY table_name, file_path",
+        )?;
+        let rows = stmt.query_map([prefix], |r| Ok((r.get(0)?, r.get(1)?)))?;
+        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+    }
+
     /// Helper to execute delete_rows_by_file statements on any connection.
     /// Called from delete_rows_by_file() either inside a caller-supplied
     /// transaction or inside a transaction opened by delete_rows_by_file().
@@ -1700,6 +1713,45 @@ mod tests {
         assert_eq!(rows[0].2, 42, "rowid_ref must be the user-declared rowid");
     }
 
+    #[test]
+    fn files_under_lists_each_file_beneath_the_directory_once_across_tables() {
+        let db = Db::new().unwrap();
+        db.create_table("t1", "CREATE TABLE t1 (id TEXT)").unwrap();
+        db.create_table("t2", "CREATE TABLE t2 (id TEXT)").unwrap();
+        let row = HashMap::from([("id".into(), Value::Text("x".into()))]);
+        for (table, file, idx) in [
+            ("t1", "dir/a.jsonl", 0),
+            ("t1", "dir/a.jsonl", 1),
+            ("t1", "dir/sub/b.jsonl", 0),
+            ("t2", "dir/c.jsonl", 0),
+            ("t1", "dir2/x.jsonl", 0),
+            ("t1", "other.jsonl", 0),
+        ] {
+            db.insert_row(table, &row, file, idx).unwrap();
+        }
+
+        assert_eq!(
+            db.files_under("dir").unwrap(),
+            vec![
+                ("t1".to_string(), "dir/a.jsonl".to_string()),
+                ("t1".to_string(), "dir/sub/b.jsonl".to_string()),
+                ("t2".to_string(), "dir/c.jsonl".to_string()),
+            ],
+            "a multi-row file is listed once; `dir2/` is not beneath `dir`"
+        );
+        assert!(db.files_under("nothing").unwrap().is_empty());
+    }
+
+    #[test]
+    fn files_under_surfaces_sql_failure() {
+        let db = Db::new().unwrap();
+        db.conn
+            .execute("DROP TABLE _dirsql_internal_rows", [])
+            .unwrap();
+        let err = db.files_under("dir").unwrap_err();
+        assert!(matches!(err, DbError::Sqlite(_)), "got: {err:?}");
+    }
+
     #[test]
     fn delete_rows_by_file_removes_mapping() {
         let db = Db::new().unwrap();
@@ -2340,7 +2392,7 @@ mod tests {
         let mut db = Db::new().unwrap();
         db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root"));
 
-        let rows = db.query("SELECT path FROM './'").unwrap();
+        let rows = db.query("SELECT path FROM './' LIMIT 1").unwrap();
 
         assert!(rows.is_empty(), "an empty root yields no rows: {rows:?}");
     }
@@ -2361,7 +2413,7 @@ mod tests {
         db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root"));
 
         let rows = db
-            .query("SELECT path FROM '/nonexistent-dirsql-dir/*.log'")
+            .query("SELECT path FROM '/nonexistent-dirsql-dir/*.log' LIMIT 1")
             .unwrap();
 
         assert!(

{dirsql-0.4.41 → dirsql-0.4.42}/packages/rust/src/lib.rs

@@ -39,6 +39,7 @@ pub mod scanner;
 pub mod sql_literal;
 #[doc(hidden)]
 pub mod vtab;
+mod vtab_scaffold;
 #[doc(hidden)]
 pub mod watcher;
 
@@ -557,9 +558,6 @@ impl DirSQL {
         // unspecified. An `on_file` failure produces an error event for that
         // table only; the other matching tables still process the event.
         let matches = self.inner.matcher.match_all(&rel_path_buf);
-        if matches.is_empty() {
-            return Vec::new();
-        }
         let rel_path = rel_path_buf.to_string_lossy().to_string();
 
         let mut events = Vec::new();
@@ -573,6 +571,33 @@ impl DirSQL {
                 }
             }
         }
+        // The mirror of the walk above: a directory that leaves the tree
+        // (`rm -r`, or a rename out of or within the root) arrives as one
+        // event naming the directory and none for the files beneath it. It is
+        // already gone, so the rows are the only record of what it held.
+        if matches!(event, FileEvent::Deleted(_)) {
+            events.extend(self.delete_subtree(&rel_path));
+        }
+        events
+    }
+
+    /// Delete the rows of every file recorded beneath `rel_dir`, in every
+    /// table, as if each file had its own delete event.
+    fn delete_subtree(&self, rel_dir: &str) -> Vec<RowEvent> {
+        let files = {
+            let db = match self.inner.db.lock() {
+                Ok(db) => db,
+                Err(e) => return vec![error_event(None, rel_dir, e.to_string())],
+            };
+            match db.files_under(rel_dir) {
+                Ok(files) => files,
+                Err(e) => return vec![error_event(None, rel_dir, e.to_string())],
+            }
+        };
+        let mut events = Vec::new();
+        for (table, file_path) in files {
+            events.extend(self.handle_delete(&table, &file_path));
+        }
         events
     }
 
@@ -2753,6 +2778,153 @@ mod internal_tests {
         );
     }
 
+    /// A one-table db whose fake fs stats every file in `files`, each already
+    /// upserted so the db holds a row per file.
+    fn populated_db(root: &Path, glob: &str, files: &[&str]) -> DirSQL {
+        let mut fake = FakeFs::default();
+        for f in files {
+            fake.stats.insert(root.join(f), fake_stat());
+        }
+        let db = DirSQL::with_ignore_and_fs(
+            root,
+            vec![Table::new(
+                "items",
+                "CREATE TABLE items (name TEXT)",
+                glob,
+                |path| {
+                    vec![Row::from_iter([(
+                        "name".to_string(),
+                        Value::Text(path.to_string()),
+                    )])]
+                },
+            )],
+            Vec::<String>::new(),
+            Arc::new(fake),
+        )
+        .unwrap();
+        for f in files {
+            let events = db.handle_upsert("items", &root.join(f), f);
+            assert!(
+                matches!(&events[..], [RowEvent::Insert { .. }]),
+                "got: {events:?}"
+            );
+        }
+        db
+    }
+
+    fn row_names(db: &DirSQL) -> Vec<String> {
+        let mut names: Vec<String> = db
+            .query("SELECT name FROM items")
+            .unwrap()
+            .into_iter()
+            .filter_map(|r| match r.get("name") {
+                Some(Value::Text(n)) => Some(n.clone()),
+                _ => None,
+            })
+            .collect();
+        names.sort();
+        names
+    }
+
+    fn deleted_paths(events: &[RowEvent]) -> Vec<String> {
+        assert!(
+            events.iter().all(|e| matches!(e, RowEvent::Delete { .. })),
+            "expected only Delete events: {events:?}"
+        );
+        let mut paths: Vec<String> = events
+            .iter()
+            .filter_map(|e| match e {
+                RowEvent::Delete { file_path, .. } => Some(file_path.clone()),
+                _ => None,
+            })
+            .collect();
+        paths.sort();
+        paths
+    }
+
+    /// A deleted directory matches no file glob, yet every row recorded
+    /// beneath it goes, with one Delete per file. A sibling whose name merely
+    /// starts with the directory's (`moved2/`) is untouched.
+    #[test]
+    fn process_file_event_deleted_directory_removes_the_rows_beneath_it() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let files = ["moved/a.txt", "moved/one/deep.txt", "moved2/x.txt"];
+        let db = populated_db(&root, "**/*.txt", &files);
+
+        let events = db.process_file_event(FileEvent::Deleted(root.join("moved")));
+
+        assert_eq!(
+            deleted_paths(&events),
+            vec!["moved/a.txt".to_string(), "moved/one/deep.txt".to_string()]
+        );
+        assert_eq!(
+            row_names(&db),
+            vec![root.join("moved2/x.txt").to_string_lossy().to_string()]
+        );
+    }
+
+    /// With a glob that also matches the directory itself (`**/*`), the
+    /// exact-path delete finds no row and the subtree delete still runs.
+    #[test]
+    fn process_file_event_deleted_directory_matching_a_glob_removes_the_rows_beneath_it() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let db = populated_db(&root, "**/*", &["moved/a.txt", "keep.txt"]);
+
+        let events = db.process_file_event(FileEvent::Deleted(root.join("moved")));
+
+        assert_eq!(deleted_paths(&events), vec!["moved/a.txt".to_string()]);
+        assert_eq!(
+            row_names(&db),
+            vec![root.join("keep.txt").to_string_lossy().to_string()]
+        );
+    }
+
+    /// Only a *deleted* directory empties its subtree; a metadata change on
+    /// one says nothing about its files.
+    #[test]
+    fn process_file_event_keeps_the_rows_beneath_a_modified_directory() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let db = populated_db(&root, "**/*.txt", &["moved/a.txt"]);
+
+        let events = db.process_file_event(FileEvent::Modified(root.join("moved")));
+
+        assert!(events.is_empty(), "got: {events:?}");
+        assert_eq!(
+            row_names(&db),
+            vec![root.join("moved/a.txt").to_string_lossy().to_string()]
+        );
+    }
+
+    #[test]
+    fn delete_subtree_surfaces_db_poison() {
+        let (_dir, db, _abs, _rel) = upsert_fixture();
+        poison(&db.inner.db);
+        let events = db.delete_subtree("moved");
+        assert_single_lock_error(&events);
+    }
+
+    #[test]
+    fn delete_subtree_surfaces_db_failure() {
+        let (_dir, db, _abs, _rel) = upsert_fixture();
+        db.inner
+            .db
+            .lock()
+            .unwrap()
+            .conn()
+            .execute("DROP TABLE _dirsql_internal_rows", [])
+            .unwrap();
+        let events = db.delete_subtree("moved");
+        assert_eq!(events.len(), 1, "expected one error event: {events:?}");
+        assert!(
+            matches!(&events[0], RowEvent::Error { table: None, file_path, error }
+                if file_path == Path::new("moved") && error.contains("no such table")),
+            "got: {events:?}"
+        );
+    }
+
     /// Building with a **relative** root canonicalizes `watch_root` to an
     /// absolute path while leaving `root` exactly as the caller supplied it,
     /// so `notify` never sees `.`.
@@ -3538,7 +3710,7 @@ mod internal_tests {
         let db = DirSQL::builder().root(dir.path()).build().unwrap();
 
         assert!(
-            db.query("SELECT path FROM './'").is_ok(),
+            db.query("SELECT path FROM './' LIMIT 1").is_ok(),
             "the path-table fallback must be armed on an ephemeral db"
         );
     }
@@ -3579,7 +3751,7 @@ mod internal_tests {
             .unwrap();
 
         assert!(
-            db.query("SELECT path FROM './'").is_ok(),
+            db.query("SELECT path FROM './' LIMIT 1").is_ok(),
             "the persist branch must arm the fallback too"
         );
     }

{dirsql-0.4.41 → dirsql-0.4.42}/packages/rust/src/parsed_vtab.rs

@@ -23,13 +23,11 @@
 use std::collections::{HashMap, HashSet};
 use std::ffi::c_int;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use globset::GlobSet;
-use rusqlite::vtab::{
-    Context, CreateVTab, IndexInfo, VTab, VTabConnection, VTabCursor, VTabKind, Values,
-    read_only_module,
-};
-use rusqlite::{Connection, Error, Result, ffi};
+use rusqlite::vtab::Context;
+use rusqlite::{Connection, Error, Result};
 
 use crate::Value;
 use crate::command::{Placeholder, run_command};
@@ -38,16 +36,15 @@ use crate::matcher::TableMatcher;
 use crate::parsed_cache::{self, CachedParse, Entry, RowCache, SqliteRowCache};
 use crate::path_table;
 use crate::persist::{FileStat, hash_file, now_ns};
-use crate::scanner::{self, scan_glob};
-use crate::sql_literal::unquote;
+use crate::scanner::scan_glob;
+use crate::vtab_scaffold::{self, TableSource};
 
 /// SQL module name a parsed path-table is created with.
 pub const MODULE_NAME: &str = "dirsql_parsed";
 
 /// Register the parsed path-table module on `conn`.
 pub fn load_module(conn: &Connection) -> Result<()> {
-    let aux: Option<()> = None;
-    conn.create_module(MODULE_NAME, read_only_module::<ParsedTab>(), aux)
+    vtab_scaffold::load_module::<ParsedTable>(conn)
 }
 
 /// Number of module arguments that are not ignore patterns.
@@ -67,43 +64,32 @@ struct ModuleArgs {
     ignore: TableMatcher,
 }
 
-/// Compile the ignore patterns a parsed path-table scan applies.
-fn compile_ignore(patterns: &[String]) -> Result<TableMatcher> {
-    let refs: Vec<&str> = patterns.iter().map(String::as_str).collect();
-    TableMatcher::new(&[], &refs).map_err(|e| Error::ModuleError(e.to_string()))
-}
-
 /// Parse a parsed path-table's `CREATE VIRTUAL TABLE` arguments. `args[0..3]`
 /// are the module, database and table names; the module's own follow — root,
 /// glob, parser, the gitignore switch, the cache path, then any ignore
 /// patterns.
 fn parse_module_args(args: &[&[u8]]) -> Result<ModuleArgs> {
-    let user_args: Vec<String> = args
-        .iter()
-        .skip(3)
-        .map(|a| unquote(&String::from_utf8_lossy(a)).to_string())
-        .collect();
+    let user_args = vtab_scaffold::user_args(args);
 
     let [root, pattern, command, gitignore, cache, ignore @ ..] = user_args.as_slice() else {
-        return Err(Error::ModuleError(format!(
-            "{MODULE_NAME} takes at least {FIXED_ARGS} arguments \
-             (root, glob, parser, gitignore switch, cache path), got {}",
-            user_args.len()
-        )));
+        return Err(vtab_scaffold::arity_error(
+            MODULE_NAME,
+            FIXED_ARGS,
+            "root, glob, parser, gitignore switch, cache path",
+            user_args.len(),
+        ));
     };
 
-    let glob = scanner::compile_glob(pattern).map_err(|e| Error::ModuleError(e.to_string()))?;
-
     Ok(ModuleArgs {
         root: PathBuf::from(root),
         pattern: pattern.clone(),
-        glob,
+        glob: vtab_scaffold::compile_glob(pattern)?,
         command: command.clone(),
-        gitignore: scanner::parse_gitignore_arg(gitignore).map_err(Error::ModuleError)?,
+        gitignore: vtab_scaffold::parse_gitignore(gitignore)?,
         // The empty string is how "no cache" is spelled: a module argument
         // cannot be absent without shifting every argument after it.
         cache: Some(PathBuf::from(cache)).filter(|p| !p.as_os_str().is_empty()),
-        ignore: compile_ignore(ignore)?,
+        ignore: vtab_scaffold::compile_ignore(ignore)?,
     })
 }
 
@@ -321,37 +307,21 @@ fn column_name(names: &[String], i: c_int) -> Option<&str> {
     names.get(index).map(String::as_str)
 }
 
-/// Whether the cursor has run past the last row.
-fn at_eof(index: usize, row_count: usize) -> bool {
-    index >= row_count
-}
-
-/// Rowid for a cursor position, saturating rather than wrapping on a row count
-/// no filesystem will produce.
-fn rowid_of(index: usize) -> i64 {
-    i64::try_from(index).unwrap_or(i64::MAX)
-}
-
-#[repr(C)]
-struct ParsedTab {
-    /// Base class. Must be first.
-    base: ffi::sqlite3_vtab,
+/// The table a `CREATE` produced: the inferred column names and the rows the
+/// parsers emitted, both fixed for the life of the table.
+struct ParsedTable {
     /// Only the names survive registration: the types live in the schema
     /// SQLite already holds, and a cursor addresses columns by index.
     column_names: Vec<String>,
-    rows: Vec<JsonRow>,
+    rows: Arc<Vec<JsonRow>>,
 }
 
-#[expect(unsafe_code, reason = "rusqlite requires VTab to be an unsafe trait")]
-unsafe impl<'vtab> VTab<'vtab> for ParsedTab {
-    type Aux = ();
-    type Cursor = ParsedTabCursor;
+impl TableSource for ParsedTable {
+    type Row = JsonRow;
+
+    const NAME: &'static str = MODULE_NAME;
 
-    fn connect(
-        _db: &mut VTabConnection,
-        _aux: Option<&()>,
-        args: &[&[u8]],
-    ) -> Result<(String, Self)> {
+    fn connect(args: &[&[u8]]) -> Result<(String, Self)> {
         let ModuleArgs {
             root,
             pattern,
@@ -384,81 +354,25 @@ unsafe impl<'vtab> VTab<'vtab> for ParsedTab {
         }
 
         let schema = declared_schema(&columns);
-        let vtab = Self {
-            base: ffi::sqlite3_vtab::default(),
+        let table = Self {
             column_names: columns.into_iter().map(|c| c.name).collect(),
-            rows,
+            rows: Arc::new(rows),
         };
-        Ok((schema, vtab))
+        Ok((schema, table))
     }
 
-    fn best_index(&self, info: &mut IndexInfo) -> Result<()> {
-        info.set_estimated_cost(1000.);
-        Ok(())
+    fn rows(&self) -> Arc<Vec<JsonRow>> {
+        // Materialized once at CREATE, so every statement reads the same rows
+        // rather than re-spawning a parser per file.
+        Arc::clone(&self.rows)
     }
 
-    fn open(&'vtab mut self) -> Result<ParsedTabCursor> {
-        Ok(ParsedTabCursor {
-            base: ffi::sqlite3_vtab_cursor::default(),
-            column_names: self.column_names.clone(),
-            rows: self.rows.clone(),
-            index: 0,
-        })
-    }
-}
-
-impl<'vtab> CreateVTab<'vtab> for ParsedTab {
-    const KIND: VTabKind = VTabKind::Default;
-}
-
-#[repr(C)]
-struct ParsedTabCursor {
-    /// Base class. Must be first: `rust_open` hands this pointer straight to
-    /// SQLite as a `sqlite3_vtab_cursor`, so anything ahead of it gets
-    /// overwritten.
-    base: ffi::sqlite3_vtab_cursor,
-    column_names: Vec<String>,
-    rows: Vec<JsonRow>,
-    index: usize,
-}
-
-#[expect(
-    unsafe_code,
-    reason = "rusqlite requires VTabCursor to be an unsafe trait"
-)]
-unsafe impl VTabCursor for ParsedTabCursor {
-    fn filter(
-        &mut self,
-        _idx_num: c_int,
-        _idx_str: Option<&str>,
-        _args: &Values<'_>,
-    ) -> Result<()> {
-        self.index = 0;
-        Ok(())
-    }
-
-    fn next(&mut self) -> Result<()> {
-        self.index += 1;
-        Ok(())
-    }
-
-    fn eof(&self) -> bool {
-        at_eof(self.index, self.rows.len())
-    }
-
-    fn column(&self, ctx: &mut Context, i: c_int) -> Result<()> {
-        let value = self
-            .rows
-            .get(self.index)
-            .zip(column_name(&self.column_names, i))
-            .map(|(row, name)| cell(row, name))
+    fn column(&self, row: &JsonRow, ctx: &mut Context, i: c_int) -> Result<()> {
+        let value = column_name(&self.column_names, i)
+            .map(|name| cell(row, name))
             .unwrap_or(Value::Null);
         ctx.set_result(&value)
     }
-
-    fn rowid(&self) -> Result<i64> {
-        Ok(rowid_of(self.index))
-    }
 }
 
 #[cfg(test)]
@@ -729,34 +643,6 @@ mod tests {
         assert_eq!(column_name(&names(), -1), None);
     }
 
-    #[test]
-    fn at_eof_is_false_while_rows_remain() {
-        assert!(!at_eof(0, 2));
-        assert!(!at_eof(1, 2));
-    }
-
-    #[test]
-    fn at_eof_is_true_once_past_the_last_row() {
-        assert!(at_eof(2, 2));
-        assert!(at_eof(3, 2));
-    }
-
-    #[test]
-    fn at_eof_is_true_for_an_empty_row_set() {
-        assert!(at_eof(0, 0));
-    }
-
-    #[test]
-    fn rowid_of_tracks_the_cursor_position() {
-        assert_eq!(rowid_of(0), 0);
-        assert_eq!(rowid_of(7), 7);
-    }
-
-    #[test]
-    fn rowid_of_saturates_rather_than_wrapping() {
-        assert_eq!(rowid_of(usize::MAX), i64::MAX);
-    }
-
     // The run_parser tests spawn a real `sh`. Their test code statically
     // references only `super::` items (plus pure std), matching the pattern
     // `command.rs` uses for the runner underneath.