dirsql 0.4.41__tar.gz → 0.4.43__tar.gz

{dirsql-0.4.41 → dirsql-0.4.43}/Cargo.lock

@@ -350,6 +350,18 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
 
+[[package]]
+name = "console"
+version = "0.16.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.4.2"
@@ -540,12 +552,13 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
 [[package]]
 name = "dirsql"
-version = "0.4.41"
+version = "0.4.43"
 dependencies = [
  "assert_cmd",
  "axum",
  "blake3",
  "clap",
+ "console",
  "criterion",
  "eventsource-client",
  "futures",
@@ -554,6 +567,7 @@ dependencies = [
  "futures-util",
  "globset",
  "ignore",
+ "indicatif",
  "libc",
  "notify",
  "predicates",
@@ -583,7 +597,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.41"
+version = "0.4.43"
 dependencies = [
  "dirsql",
  "pyo3",
@@ -630,6 +644,12 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
@@ -1294,6 +1314,19 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.18.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
+dependencies = [
+ "console",
+ "portable-atomic",
+ "unicode-width",
+ "unit-prefix",
+ "web-time",
+]
+
 [[package]]
 name = "inotify"
 version = "0.10.2"
@@ -2902,6 +2935,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"

{dirsql-0.4.41 → dirsql-0.4.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.41
+Version: 0.4.43
 Requires-Dist: tomli>=2 ; python_full_version < '3.11'
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory

{dirsql-0.4.41 → dirsql-0.4.43}/docs/reference/cli.md

@@ -536,6 +536,11 @@ Override with `DIRSQL_PROGRESS`:
 Values are case-insensitive and surrounding whitespace is ignored; anything
 unrecognized reads as `auto`, so a typo cannot stop a scan from running.
 
+Each redraw clears the line and pads the new one out to the full width (the
+terminal's, or 80 columns when there is none), so a capture of an `always` run
+carries trailing spaces. It carries no escape sequences, and the summary that
+survives the phase is a whole, newline-terminated line.
+
 The setting is read by the **core**, not the CLI, so it governs an index built
 from any SDK as well — a Python or TypeScript program that builds a `DirSQL`
 with a terminal attached gets the same two phases on stderr, and the same

{dirsql-0.4.41 → dirsql-0.4.43}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.41"
+version = "0.4.43"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

{dirsql-0.4.41/packages/rust → dirsql-0.4.43/packages/python}/docs/reference/cli.md

@@ -536,6 +536,11 @@ Override with `DIRSQL_PROGRESS`:
 Values are case-insensitive and surrounding whitespace is ignored; anything
 unrecognized reads as `auto`, so a typo cannot stop a scan from running.
 
+Each redraw clears the line and pads the new one out to the full width (the
+terminal's, or 80 columns when there is none), so a capture of an `always` run
+carries trailing spaces. It carries no escape sequences, and the summary that
+survives the phase is a whole, newline-terminated line.
+
 The setting is read by the **core**, not the CLI, so it governs an index built
 from any SDK as well — a Python or TypeScript program that builds a `DirSQL`
 with a terminal attached gets the same two phases on stderr, and the same

dirsql-0.4.43/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1789515494,
+  "exit_code": 0,
+  "commit": "51555d8d0ad35095d69c1568e4478f34feb36e09",
+  "branch": "claude/1080-share-vtab-scaffolding"
+}

dirsql-0.4.43/packages/python/e2e-attestations/claude-1081-indicatif-progress.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1789666612,
+  "exit_code": 0,
+  "commit": "991bcb921ba010ac18968740a79fd1d6fce05e11",
+  "branch": "claude/1081-indicatif-progress"
+}

dirsql-0.4.43/packages/python/e2e-attestations/claude-1109-watch-moved-out.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1789595962,
+  "exit_code": 0,
+  "commit": "c645cfe8052bdca311060ca475f6ee102246650b",
+  "branch": "claude/1109-watch-moved-out"
+}

{dirsql-0.4.41 → dirsql-0.4.43}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.41"
+version = "0.4.43"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly
@@ -59,12 +59,14 @@ cli = [
 axum = { version = "0.8", optional = true }
 blake3 = "1"
 clap = { version = "4", features = ["derive"], optional = true }
+console = "0.16.6"
 futures = { version = "0.3", optional = true }
 futures-channel.workspace = true
 globset = "0.4"
 # Hierarchical `.gitignore` matching for path-table scans (the fd/ripgrep
 # semantics), without adopting its walker: the scan keeps `walkdir`.
 ignore = "0.4"
+indicatif = "0.18.6"
 notify = "7"
 # The REPL's interactive line editor (history, Ctrl+R, multi-line entry).
 # CLI-only: a library consumer never pulls crossterm.

{dirsql-0.4.41/packages/python → dirsql-0.4.43/packages/rust}/docs/reference/cli.md

@@ -536,6 +536,11 @@ Override with `DIRSQL_PROGRESS`:
 Values are case-insensitive and surrounding whitespace is ignored; anything
 unrecognized reads as `auto`, so a typo cannot stop a scan from running.
 
+Each redraw clears the line and pads the new one out to the full width (the
+terminal's, or 80 columns when there is none), so a capture of an `always` run
+carries trailing spaces. It carries no escape sequences, and the summary that
+survives the phase is a whole, newline-terminated line.
+
 The setting is read by the **core**, not the CLI, so it governs an index built
 from any SDK as well — a Python or TypeScript program that builds a `DirSQL`
 with a terminal attached gets the same two phases on stderr, and the same

{dirsql-0.4.41 → dirsql-0.4.43}/packages/rust/src/db.rs

@@ -645,6 +645,19 @@ impl Db {
         Ok(out)
     }
 
+    /// Every distinct `(table_name, file_path)` whose file sits beneath the
+    /// directory `dir`, ordered by table then path.
+    pub fn files_under(&self, dir: &str) -> Result<Vec<(String, String)>> {
+        let prefix = format!("{dir}{}", std::path::MAIN_SEPARATOR);
+        let mut stmt = self.conn.prepare(
+            "SELECT DISTINCT table_name, file_path FROM _dirsql_internal_rows \
+             WHERE substr(file_path, 1, length(?1)) = ?1 \
+             ORDER BY table_name, file_path",
+        )?;
+        let rows = stmt.query_map([prefix], |r| Ok((r.get(0)?, r.get(1)?)))?;
+        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+    }
+
     /// Helper to execute delete_rows_by_file statements on any connection.
     /// Called from delete_rows_by_file() either inside a caller-supplied
     /// transaction or inside a transaction opened by delete_rows_by_file().
@@ -1700,6 +1713,45 @@ mod tests {
         assert_eq!(rows[0].2, 42, "rowid_ref must be the user-declared rowid");
     }
 
+    #[test]
+    fn files_under_lists_each_file_beneath_the_directory_once_across_tables() {
+        let db = Db::new().unwrap();
+        db.create_table("t1", "CREATE TABLE t1 (id TEXT)").unwrap();
+        db.create_table("t2", "CREATE TABLE t2 (id TEXT)").unwrap();
+        let row = HashMap::from([("id".into(), Value::Text("x".into()))]);
+        for (table, file, idx) in [
+            ("t1", "dir/a.jsonl", 0),
+            ("t1", "dir/a.jsonl", 1),
+            ("t1", "dir/sub/b.jsonl", 0),
+            ("t2", "dir/c.jsonl", 0),
+            ("t1", "dir2/x.jsonl", 0),
+            ("t1", "other.jsonl", 0),
+        ] {
+            db.insert_row(table, &row, file, idx).unwrap();
+        }
+
+        assert_eq!(
+            db.files_under("dir").unwrap(),
+            vec![
+                ("t1".to_string(), "dir/a.jsonl".to_string()),
+                ("t1".to_string(), "dir/sub/b.jsonl".to_string()),
+                ("t2".to_string(), "dir/c.jsonl".to_string()),
+            ],
+            "a multi-row file is listed once; `dir2/` is not beneath `dir`"
+        );
+        assert!(db.files_under("nothing").unwrap().is_empty());
+    }
+
+    #[test]
+    fn files_under_surfaces_sql_failure() {
+        let db = Db::new().unwrap();
+        db.conn
+            .execute("DROP TABLE _dirsql_internal_rows", [])
+            .unwrap();
+        let err = db.files_under("dir").unwrap_err();
+        assert!(matches!(err, DbError::Sqlite(_)), "got: {err:?}");
+    }
+
     #[test]
     fn delete_rows_by_file_removes_mapping() {
         let db = Db::new().unwrap();
@@ -2340,7 +2392,7 @@ mod tests {
         let mut db = Db::new().unwrap();
         db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root"));
 
-        let rows = db.query("SELECT path FROM './'").unwrap();
+        let rows = db.query("SELECT path FROM './' LIMIT 1").unwrap();
 
         assert!(rows.is_empty(), "an empty root yields no rows: {rows:?}");
     }
@@ -2361,7 +2413,7 @@ mod tests {
         db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root"));
 
         let rows = db
-            .query("SELECT path FROM '/nonexistent-dirsql-dir/*.log'")
+            .query("SELECT path FROM '/nonexistent-dirsql-dir/*.log' LIMIT 1")
             .unwrap();
 
         assert!(

{dirsql-0.4.41 → dirsql-0.4.43}/packages/rust/src/lib.rs

@@ -39,6 +39,7 @@ pub mod scanner;
 pub mod sql_literal;
 #[doc(hidden)]
 pub mod vtab;
+mod vtab_scaffold;
 #[doc(hidden)]
 pub mod watcher;
 
@@ -557,9 +558,6 @@ impl DirSQL {
         // unspecified. An `on_file` failure produces an error event for that
         // table only; the other matching tables still process the event.
         let matches = self.inner.matcher.match_all(&rel_path_buf);
-        if matches.is_empty() {
-            return Vec::new();
-        }
         let rel_path = rel_path_buf.to_string_lossy().to_string();
 
         let mut events = Vec::new();
@@ -573,6 +571,33 @@ impl DirSQL {
                 }
             }
         }
+        // The mirror of the walk above: a directory that leaves the tree
+        // (`rm -r`, or a rename out of or within the root) arrives as one
+        // event naming the directory and none for the files beneath it. It is
+        // already gone, so the rows are the only record of what it held.
+        if matches!(event, FileEvent::Deleted(_)) {
+            events.extend(self.delete_subtree(&rel_path));
+        }
+        events
+    }
+
+    /// Delete the rows of every file recorded beneath `rel_dir`, in every
+    /// table, as if each file had its own delete event.
+    fn delete_subtree(&self, rel_dir: &str) -> Vec<RowEvent> {
+        let files = {
+            let db = match self.inner.db.lock() {
+                Ok(db) => db,
+                Err(e) => return vec![error_event(None, rel_dir, e.to_string())],
+            };
+            match db.files_under(rel_dir) {
+                Ok(files) => files,
+                Err(e) => return vec![error_event(None, rel_dir, e.to_string())],
+            }
+        };
+        let mut events = Vec::new();
+        for (table, file_path) in files {
+            events.extend(self.handle_delete(&table, &file_path));
+        }
         events
     }
 
@@ -2753,6 +2778,153 @@ mod internal_tests {
         );
     }
 
+    /// A one-table db whose fake fs stats every file in `files`, each already
+    /// upserted so the db holds a row per file.
+    fn populated_db(root: &Path, glob: &str, files: &[&str]) -> DirSQL {
+        let mut fake = FakeFs::default();
+        for f in files {
+            fake.stats.insert(root.join(f), fake_stat());
+        }
+        let db = DirSQL::with_ignore_and_fs(
+            root,
+            vec![Table::new(
+                "items",
+                "CREATE TABLE items (name TEXT)",
+                glob,
+                |path| {
+                    vec![Row::from_iter([(
+                        "name".to_string(),
+                        Value::Text(path.to_string()),
+                    )])]
+                },
+            )],
+            Vec::<String>::new(),
+            Arc::new(fake),
+        )
+        .unwrap();
+        for f in files {
+            let events = db.handle_upsert("items", &root.join(f), f);
+            assert!(
+                matches!(&events[..], [RowEvent::Insert { .. }]),
+                "got: {events:?}"
+            );
+        }
+        db
+    }
+
+    fn row_names(db: &DirSQL) -> Vec<String> {
+        let mut names: Vec<String> = db
+            .query("SELECT name FROM items")
+            .unwrap()
+            .into_iter()
+            .filter_map(|r| match r.get("name") {
+                Some(Value::Text(n)) => Some(n.clone()),
+                _ => None,
+            })
+            .collect();
+        names.sort();
+        names
+    }
+
+    fn deleted_paths(events: &[RowEvent]) -> Vec<String> {
+        assert!(
+            events.iter().all(|e| matches!(e, RowEvent::Delete { .. })),
+            "expected only Delete events: {events:?}"
+        );
+        let mut paths: Vec<String> = events
+            .iter()
+            .filter_map(|e| match e {
+                RowEvent::Delete { file_path, .. } => Some(file_path.clone()),
+                _ => None,
+            })
+            .collect();
+        paths.sort();
+        paths
+    }
+
+    /// A deleted directory matches no file glob, yet every row recorded
+    /// beneath it goes, with one Delete per file. A sibling whose name merely
+    /// starts with the directory's (`moved2/`) is untouched.
+    #[test]
+    fn process_file_event_deleted_directory_removes_the_rows_beneath_it() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let files = ["moved/a.txt", "moved/one/deep.txt", "moved2/x.txt"];
+        let db = populated_db(&root, "**/*.txt", &files);
+
+        let events = db.process_file_event(FileEvent::Deleted(root.join("moved")));
+
+        assert_eq!(
+            deleted_paths(&events),
+            vec!["moved/a.txt".to_string(), "moved/one/deep.txt".to_string()]
+        );
+        assert_eq!(
+            row_names(&db),
+            vec![root.join("moved2/x.txt").to_string_lossy().to_string()]
+        );
+    }
+
+    /// With a glob that also matches the directory itself (`**/*`), the
+    /// exact-path delete finds no row and the subtree delete still runs.
+    #[test]
+    fn process_file_event_deleted_directory_matching_a_glob_removes_the_rows_beneath_it() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let db = populated_db(&root, "**/*", &["moved/a.txt", "keep.txt"]);
+
+        let events = db.process_file_event(FileEvent::Deleted(root.join("moved")));
+
+        assert_eq!(deleted_paths(&events), vec!["moved/a.txt".to_string()]);
+        assert_eq!(
+            row_names(&db),
+            vec![root.join("keep.txt").to_string_lossy().to_string()]
+        );
+    }
+
+    /// Only a *deleted* directory empties its subtree; a metadata change on
+    /// one says nothing about its files.
+    #[test]
+    fn process_file_event_keeps_the_rows_beneath_a_modified_directory() {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().to_path_buf();
+        let db = populated_db(&root, "**/*.txt", &["moved/a.txt"]);
+
+        let events = db.process_file_event(FileEvent::Modified(root.join("moved")));
+
+        assert!(events.is_empty(), "got: {events:?}");
+        assert_eq!(
+            row_names(&db),
+            vec![root.join("moved/a.txt").to_string_lossy().to_string()]
+        );
+    }
+
+    #[test]
+    fn delete_subtree_surfaces_db_poison() {
+        let (_dir, db, _abs, _rel) = upsert_fixture();
+        poison(&db.inner.db);
+        let events = db.delete_subtree("moved");
+        assert_single_lock_error(&events);
+    }
+
+    #[test]
+    fn delete_subtree_surfaces_db_failure() {
+        let (_dir, db, _abs, _rel) = upsert_fixture();
+        db.inner
+            .db
+            .lock()
+            .unwrap()
+            .conn()
+            .execute("DROP TABLE _dirsql_internal_rows", [])
+            .unwrap();
+        let events = db.delete_subtree("moved");
+        assert_eq!(events.len(), 1, "expected one error event: {events:?}");
+        assert!(
+            matches!(&events[0], RowEvent::Error { table: None, file_path, error }
+                if file_path == Path::new("moved") && error.contains("no such table")),
+            "got: {events:?}"
+        );
+    }
+
     /// Building with a **relative** root canonicalizes `watch_root` to an
     /// absolute path while leaving `root` exactly as the caller supplied it,
     /// so `notify` never sees `.`.
@@ -3538,7 +3710,7 @@ mod internal_tests {
         let db = DirSQL::builder().root(dir.path()).build().unwrap();
 
         assert!(
-            db.query("SELECT path FROM './'").is_ok(),
+            db.query("SELECT path FROM './' LIMIT 1").is_ok(),
             "the path-table fallback must be armed on an ephemeral db"
         );
     }
@@ -3579,7 +3751,7 @@ mod internal_tests {
             .unwrap();
 
         assert!(
-            db.query("SELECT path FROM './'").is_ok(),
+            db.query("SELECT path FROM './' LIMIT 1").is_ok(),
             "the persist branch must arm the fallback too"
         );
     }