dirsql 0.4.105__tar.gz → 0.4.106__tar.gz

{dirsql-0.4.105 → dirsql-0.4.106}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.105"
+version = "0.4.106"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.105"
+version = "0.4.106"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.105 → dirsql-0.4.106}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.105
+Version: 0.4.106
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.105 → dirsql-0.4.106}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.105"
+version = "0.4.106"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.106/packages/python/e2e-attestations/bot-1459-rescan-on-overflow-1008.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/on_file_query_test.py tests/e2e/query_contract_test.py tests/e2e/use_case_notebook_test.py -x -q",
+  "ran_at": 1791464863,
+  "exit_code": 0,
+  "commit": "480d3e491818207f55eb2d8dc6c95498dc27afab",
+  "branch": "bot/1459-rescan-on-overflow-1008"
+}

{dirsql-0.4.105 → dirsql-0.4.106}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.105"
+version = "0.4.106"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.105 → dirsql-0.4.106}/packages/rust/src/db.rs

@@ -991,6 +991,22 @@ impl Db {
             .collect()
     }
 
+    /// Every distinct `(table_name, file_path)` that owns rows, ordered by
+    /// table then path. A per-table hook's rows have no file owner and are
+    /// not listed.
+    pub fn owned_files(&self) -> Result<Vec<(String, String)>> {
+        let mut stmt = self.conn.prepare(
+            "SELECT DISTINCT table_name, file_path FROM ( \
+                SELECT table_name, file_path FROM _dirsql_internal_rows \
+                UNION ALL \
+                SELECT table_name, file_path FROM _dirsql_internal_ranges) \
+             WHERE file_path != '' \
+             ORDER BY table_name, file_path",
+        )?;
+        let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?;
+        Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
+    }
+
     /// Every distinct `(table_name, file_path)` whose file sits beneath the
     /// directory `dir`, ordered by table then path.
     pub fn files_under(&self, dir: &str) -> Result<Vec<(String, String)>> {
@@ -2496,6 +2512,42 @@ mod tests {
         assert!(db.files_under("nothing").unwrap().is_empty());
     }
 
+    #[test]
+    fn owned_files_lists_each_file_once_and_skips_the_batch_owner() {
+        let db = Db::new().unwrap();
+        db.create_table("t1", "CREATE TABLE t1 (id TEXT)").unwrap();
+        db.create_table("t2", "CREATE TABLE t2 (id TEXT)").unwrap();
+        let row = HashMap::from([("id".into(), Value::Text("x".into()))]);
+        for (table, file, idx) in [
+            ("t2", "c.jsonl", 0),
+            ("t1", "a.jsonl", 0),
+            ("t1", "a.jsonl", 1),
+            ("t1", "d/b.jsonl", 0),
+            ("t1", "", 0),
+        ] {
+            db.insert_row(table, &row, file, idx).unwrap();
+        }
+
+        assert_eq!(
+            db.owned_files().unwrap(),
+            vec![
+                ("t1".to_string(), "a.jsonl".to_string()),
+                ("t1".to_string(), "d/b.jsonl".to_string()),
+                ("t2".to_string(), "c.jsonl".to_string()),
+            ]
+        );
+    }
+
+    #[test]
+    fn owned_files_surfaces_sql_failure() {
+        let db = Db::new().unwrap();
+        db.conn
+            .execute("DROP TABLE _dirsql_internal_rows", [])
+            .unwrap();
+        let err = db.owned_files().unwrap_err();
+        assert!(matches!(err, DbError::Sqlite(_)), "got: {err:?}");
+    }
+
     #[test]
     fn files_under_surfaces_sql_failure() {
         let db = Db::new().unwrap();

{dirsql-0.4.105 → dirsql-0.4.106}/packages/rust/src/lib.rs

@@ -658,9 +658,17 @@ impl DirSQL {
     pub fn apply_file_events(&self, events: Vec<FileEvent>) -> Vec<RowEvent> {
         let mut pending = PendingRefresh::default();
         let mut out = Vec::new();
+        let mut rescan = false;
         for fe in events {
+            if fe == FileEvent::Rescan {
+                rescan = true;
+                continue;
+            }
             out.extend(self.route_file_event(fe, &mut pending));
         }
+        if rescan {
+            out.extend(self.rescan(&mut pending));
+        }
         out.extend(self.refresh_pending(pending));
         out
     }
@@ -718,6 +726,7 @@ impl DirSQL {
     fn route_file_event(&self, event: FileEvent, pending: &mut PendingRefresh) -> Vec<RowEvent> {
         let abs_path = match &event {
             FileEvent::Created(p) | FileEvent::Modified(p) | FileEvent::Deleted(p) => p.clone(),
+            FileEvent::Rescan => return Vec::new(),
         };
         let mut events = Vec::new();
         for (group, rel_path_buf) in self.groups_containing(&abs_path) {
@@ -767,6 +776,7 @@ impl DirSQL {
                     FileEvent::Created(_) | FileEvent::Modified(_) => {
                         events.extend(self.handle_upsert(&m.table_name, &abs_path, &rel_path));
                     }
+                    FileEvent::Rescan => {}
                 }
             }
             // The mirror of the walk above: a directory that leaves the tree
@@ -931,6 +941,43 @@ impl DirSQL {
         events
     }
 
+    /// Reconcile every table with the tree after the OS dropped events: walk
+    /// each group's root again, drop the rows of files that are gone, re-read
+    /// the files that are there, and rebuild each per-table hook's file list.
+    fn rescan(&self, pending: &mut PendingRefresh) -> Vec<RowEvent> {
+        let owned = match self.inner.db.lock() {
+            Ok(db) => db.owned_files(),
+            Err(e) => return vec![error_event(None, "", e.to_string())],
+        };
+        let owned = match owned {
+            Ok(owned) => owned,
+            Err(e) => return vec![error_event(None, "", e.to_string())],
+        };
+        let mut events = Vec::new();
+        for group in &self.inner.groups {
+            let in_group = |table: &str| group.tables.iter().any(|name| name == table);
+            for (table, file_path) in &owned {
+                if in_group(table)
+                    && !self
+                        .inner
+                        .fs
+                        .is_file(&group.watch_anchor.join(file_path))
+                        .unwrap_or(false)
+                {
+                    events.extend(self.handle_delete(table, file_path));
+                }
+            }
+            for table in group.tables.iter().filter(|table| self.is_per_table(table)) {
+                if let Ok(mut batch_files) = self.inner.batch_files.lock() {
+                    batch_files.remove(table);
+                }
+                pending.mark(table, ".");
+            }
+            events.extend(self.index_subtree(group, &group.watch_anchor, pending));
+        }
+        events
+    }
+
     /// Index every file beneath `dir` as if each had its own create event,
     /// using the scanner's walk so the watch path and the initial scan share
     /// one definition of what counts as a row.
@@ -5563,6 +5610,114 @@ mod internal_tests {
         assert_eq!(row_names(&db), Vec::<String>::new());
     }
 
+    fn rescan_fixture(per_table: bool) -> (TempDir, PathBuf, DirSQL, Calls) {
+        let dir = TempDir::new().unwrap();
+        let root = dir.path().canonicalize().unwrap();
+        let (items, calls) = recording_per_table("items", "**/*.txt");
+        let table = if per_table {
+            items
+        } else {
+            Table::new(
+                "items",
+                "CREATE TABLE items (name TEXT)",
+                "**/*.txt",
+                |path| {
+                    vec![Row::from_iter([(
+                        "name".to_string(),
+                        Value::Text(
+                            Path::new(path)
+                                .file_name()
+                                .unwrap()
+                                .to_string_lossy()
+                                .into_owned(),
+                        ),
+                    )])]
+                },
+            )
+        };
+        let db =
+            DirSQL::with_ignore_and_fs(&root, vec![table], Vec::<String>::new(), Arc::new(RealFs))
+                .unwrap();
+        for rel in ["a.txt", "sub/b.txt"] {
+            let path = root.join(rel);
+            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+            std::fs::write(&path, "x").unwrap();
+            db.apply_file_events(vec![FileEvent::Created(path)]);
+        }
+        (dir, root, db, calls)
+    }
+
+    fn drift(root: &Path) {
+        std::fs::remove_file(root.join("a.txt")).unwrap();
+        std::fs::write(root.join("sub/c.txt"), "x").unwrap();
+        std::fs::write(root.join("d.txt"), "x").unwrap();
+    }
+
+    fn name_of(row: &Row) -> String {
+        match &row["name"] {
+            Value::Text(n) => n.clone(),
+            other => format!("{other:?}"),
+        }
+    }
+
+    fn rescan_summary(events: &[RowEvent]) -> Vec<String> {
+        let mut out: Vec<String> = events
+            .iter()
+            .map(|event| match event {
+                RowEvent::Insert { row, .. } => format!("insert {}", name_of(row)),
+                RowEvent::Delete { row, .. } => format!("delete {}", name_of(row)),
+                other => format!("{other:?}"),
+            })
+            .collect();
+        out.sort();
+        out
+    }
+
+    #[test]
+    fn a_rescan_reconciles_a_per_file_table_with_the_tree() {
+        let (_dir, root, db, _) = rescan_fixture(false);
+        drift(&root);
+
+        let events = db.apply_file_events(vec![FileEvent::Rescan]);
+
+        assert_eq!(
+            rescan_summary(&events),
+            vec!["delete a.txt", "insert c.txt", "insert d.txt"]
+        );
+        assert_eq!(row_names(&db), vec!["b.txt", "c.txt", "d.txt"]);
+    }
+
+    #[test]
+    fn a_rescan_rebuilds_a_per_table_hooks_files_and_reconciles_its_rows() {
+        let (_dir, root, db, calls) = rescan_fixture(true);
+        drift(&root);
+
+        let events = db.apply_file_events(vec![FileEvent::Rescan]);
+
+        assert_eq!(
+            calls.lock().unwrap().last().unwrap(),
+            &vec![
+                root.join("d.txt"),
+                root.join("sub/b.txt"),
+                root.join("sub/c.txt")
+            ]
+        );
+        assert_eq!(
+            rescan_summary(&events),
+            vec![
+                "Update { table: \"items\", old_row: {\"name\": Text(\"a.txt\")}, new_row: {\"name\": Text(\"d.txt\")}, file_path: \".\" }",
+                "insert c.txt"
+            ]
+        );
+        assert_eq!(row_names(&db), vec!["b.txt", "c.txt", "d.txt"]);
+    }
+
+    #[test]
+    fn a_rescan_of_a_tree_that_did_not_drift_emits_nothing() {
+        let (_dir, _root, db, _) = rescan_fixture(false);
+        assert_eq!(db.apply_file_events(vec![FileEvent::Rescan]), vec![]);
+    }
+
     fn anchored_table(name: &str, glob: &str, anchor: &str) -> Table {
         Table::new(
             name,

{dirsql-0.4.105 → dirsql-0.4.106}/packages/rust/src/watcher.rs

@@ -19,6 +19,9 @@ pub enum FileEvent {
     Created(PathBuf),
     Modified(PathBuf),
     Deleted(PathBuf),
+    /// The OS dropped events (an inotify queue overflow); the index may have
+    /// diverged from disk and must be rebuilt from a walk.
+    Rescan,
 }
 
 /// A tree to watch, and the skip rules the scan applies within it.
@@ -151,6 +154,19 @@ fn forward(
                     let _ = tolerate_missing(watcher.watch(&dir, RecursiveMode::NonRecursive));
                 }
             }
+            if fe == FileEvent::Rescan && PER_DIRECTORY {
+                let Some(watcher) = watcher.upgrade() else {
+                    return;
+                };
+                let mut watcher = watcher.lock().expect("watching a directory does not panic");
+                // Directories created while events were being dropped have no
+                // watch yet.
+                for scope in scopes {
+                    for dir in dirs_to_watch(scopes, &scope.root, &walk_scope) {
+                        let _ = tolerate_missing(watcher.watch(&dir, RecursiveMode::NonRecursive));
+                    }
+                }
+            }
             if tx.send(fe).is_err() {
                 return;
             }
@@ -258,6 +274,10 @@ fn translate_event(event: &Event) -> Vec<FileEvent> {
         }
     }
 
+    if event.need_rescan() {
+        results.push(FileEvent::Rescan);
+    }
+
     results
 }
 
@@ -347,6 +367,16 @@ fn access_event(paths: Vec<PathBuf>) -> Event {
     }
 }
 
+#[cfg(test)]
+fn rescan_event() -> Event {
+    Event {
+        kind: EventKind::Other,
+        paths: Vec::new(),
+        attrs: Default::default(),
+    }
+    .set_flag(notify::event::Flag::Rescan)
+}
+
 #[cfg(test)]
 fn io_error(raw_os_error: i32) -> notify::Error {
     notify::Error::io(std::io::Error::from_raw_os_error(raw_os_error))
@@ -563,6 +593,11 @@ mod tests {
         );
     }
 
+    #[test]
+    fn translate_event_turns_a_rescan_flag_into_a_rescan() {
+        assert_eq!(translate_event(&rescan_event()), vec![FileEvent::Rescan]);
+    }
+
     #[test]
     fn translate_event_ignores_access_events() {
         let results = translate_event(&access_event(vec![PathBuf::from("/tmp/read.txt")]));