dirsql 0.4.38__tar.gz → 0.4.40__tar.gz

{dirsql-0.4.38 → dirsql-0.4.40}/Cargo.lock

@@ -540,7 +540,7 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
 [[package]]
 name = "dirsql"
-version = "0.4.38"
+version = "0.4.40"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -583,7 +583,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.38"
+version = "0.4.40"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.38 → dirsql-0.4.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.38
+Version: 0.4.40
 Requires-Dist: tomli>=2 ; python_full_version < '3.11'
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory

{dirsql-0.4.38 → dirsql-0.4.40}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.38"
+version = "0.4.40"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.40/packages/python/e2e-attestations/claude-1096-watch-moved-dir.json

@@ -0,0 +1,7 @@
+{
+  "command": "just test-e2e",
+  "ran_at": 1788980718,
+  "exit_code": 0,
+  "commit": "f1d452531516e25204fbc4a2bdc3233d036a00f5",
+  "branch": "claude/1096-watch-moved-dir"
+}

{dirsql-0.4.38 → dirsql-0.4.40}/packages/python/testing-conventions.toml

@@ -9,9 +9,6 @@
 [e2e]
 extra_scope = ["packages/rust/src", "packages/python/src"]
 
-[python]
-one_function_per_file = { max_lines = 12 }
-
 # Same floor as the root file, which this lane no longer reads.
 [python.coverage]
 fail_under = 100

{dirsql-0.4.38 → dirsql-0.4.40}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.38"
+version = "0.4.40"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.38 → dirsql-0.4.40}/packages/rust/src/lib.rs

@@ -538,6 +538,18 @@ impl DirSQL {
             return Vec::new();
         }
 
+        // A directory that appears whole (`mkdir`, or a populated directory
+        // renamed into the tree) arrives as one event naming the directory and
+        // none for the files already inside it, so the subtree is walked here.
+        // Only on creation: a metadata change on a directory says nothing
+        // about its files, and walking on every one would re-run `on_file`
+        // across the subtree for a `chmod`.
+        if matches!(event, FileEvent::Created(_))
+            && self.inner.fs.is_dir(&abs_path).unwrap_or(false)
+        {
+            return self.index_subtree(&abs_path);
+        }
+
         // Fan-out: dispatch the event to every table whose glob matches, and
         // concatenate the resulting row events. Cross-table event order is
         // unspecified. An `on_file` failure produces an error event for that
@@ -562,6 +574,27 @@ impl DirSQL {
         events
     }
 
+    /// Index every file beneath `dir` as if each had its own create event,
+    /// using the scanner's walk so the watch path and the initial scan share
+    /// one definition of what counts as a row.
+    fn index_subtree(&self, dir: &Path) -> Vec<RowEvent> {
+        let base = if dir.starts_with(&self.inner.watch_root) {
+            &self.inner.watch_root
+        } else {
+            &self.inner.root
+        };
+        let mut events = Vec::new();
+        for (path, table) in self.inner.fs.scan_subtree(base, dir, &self.inner.matcher) {
+            let rel_path = path
+                .strip_prefix(base)
+                .unwrap_or(&path)
+                .to_string_lossy()
+                .to_string();
+            events.extend(self.handle_upsert(&table, &path, &rel_path));
+        }
+        events
+    }
+
     fn handle_delete(&self, table: &str, rel_path: &str) -> Vec<RowEvent> {
         // Snapshot the file's rows before deleting them — the Delete events
         // carry the old-row payloads. Read and delete under one lock
@@ -1571,6 +1604,15 @@ trait FileSystem: Send + Sync {
     fn stat(&self, path: &Path) -> std::io::Result<FileStat>;
     /// Whether `path` is a regular file. `Err(NotFound)` when it doesn't exist.
     fn is_file(&self, path: &Path) -> std::io::Result<bool>;
+    /// Whether `path` is a directory. `Err(NotFound)` when it doesn't exist.
+    fn is_dir(&self, path: &Path) -> std::io::Result<bool>;
+    /// The scanner's walk over the subtree at `dir`, matched relative to `root`.
+    fn scan_subtree(
+        &self,
+        root: &Path,
+        dir: &Path,
+        matcher: &TableMatcher,
+    ) -> Vec<(PathBuf, String)>;
     /// BLAKE3-hash a file's contents.
     fn hash(&self, path: &Path) -> std::io::Result<[u8; 32]>;
     /// Canonicalize the watch root, falling back to the literal path.
@@ -1588,6 +1630,19 @@ impl FileSystem for RealFs {
         std::fs::metadata(path).map(|m| m.is_file())
     }
 
+    fn is_dir(&self, path: &Path) -> std::io::Result<bool> {
+        std::fs::metadata(path).map(|m| m.is_dir())
+    }
+
+    fn scan_subtree(
+        &self,
+        root: &Path,
+        dir: &Path,
+        matcher: &TableMatcher,
+    ) -> Vec<(PathBuf, String)> {
+        scanner::scan_subtree(root, dir, matcher)
+    }
+
     fn hash(&self, path: &Path) -> std::io::Result<[u8; 32]> {
         hash_file(path)
     }
@@ -2213,6 +2268,8 @@ mod internal_tests {
         hashes: StdHashMap<PathBuf, [u8; 32]>,
         canonical_roots: StdHashMap<PathBuf, String>,
         dirs: StdHashSet<PathBuf>,
+        subtrees: StdHashMap<PathBuf, Vec<(PathBuf, String)>>,
+        subtree_walks: Mutex<Vec<PathBuf>>,
     }
 
     impl FakeFs {
@@ -2228,6 +2285,22 @@ mod internal_tests {
             self
         }
 
+        /// Register a directory together with what the scanner's walk of it
+        /// would return; each file is also stat-able as a regular file.
+        fn with_subtree(mut self, dir: impl Into<PathBuf>, found: Vec<(PathBuf, String)>) -> Self {
+            let dir = dir.into();
+            self.dirs.insert(dir.clone());
+            for (path, _) in &found {
+                self.stats.insert(path.clone(), fake_stat());
+            }
+            self.subtrees.insert(dir, found);
+            self
+        }
+
+        fn subtree_walks(&self) -> Vec<PathBuf> {
+            self.subtree_walks.lock().unwrap().clone()
+        }
+
         fn set_hash(&mut self, path: impl Into<PathBuf>, hash: [u8; 32]) {
             self.hashes.insert(path.into(), hash);
         }
@@ -2263,6 +2336,29 @@ mod internal_tests {
             ))
         }
 
+        fn is_dir(&self, path: &Path) -> std::io::Result<bool> {
+            if self.dirs.contains(path) {
+                return Ok(true);
+            }
+            if self.stats.contains_key(path) {
+                return Ok(false);
+            }
+            Err(std::io::Error::new(
+                std::io::ErrorKind::NotFound,
+                "fake: no such file",
+            ))
+        }
+
+        fn scan_subtree(
+            &self,
+            _root: &Path,
+            dir: &Path,
+            _matcher: &TableMatcher,
+        ) -> Vec<(PathBuf, String)> {
+            self.subtree_walks.lock().unwrap().push(dir.to_path_buf());
+            self.subtrees.get(dir).cloned().unwrap_or_default()
+        }
+
         fn hash(&self, path: &Path) -> std::io::Result<[u8; 32]> {
             self.hashes.get(path).copied().ok_or_else(|| {
                 std::io::Error::new(std::io::ErrorKind::NotFound, "fake: no such file")
@@ -2517,6 +2613,144 @@ mod internal_tests {
         assert_eq!(events.len(), 1, "non-ignored path must produce one event");
     }
 
+    fn subtree_db(fake: FakeFs, root: &Path, ignore: Vec<&str>) -> DirSQL {
+        DirSQL::with_ignore_and_fs(
+            root,
+            vec![Table::new(
+                "items",
+                "CREATE TABLE items (name TEXT)",
+                "**/*.txt",
+                |path| {
+                    vec![Row::from_iter([(
+                        "name".to_string(),
+                        Value::Text(path.to_string()),
+                    )])]
+                },
+            )],
+            ignore,
+            Arc::new(fake),
+        )
+        .unwrap()
+    }
+
+    /// A created directory is indexed by walking it: every file the scanner
+    /// reports beneath it becomes rows, as if each had its own create event.
+    #[test]
+    fn process_file_event_indexes_the_files_beneath_a_created_directory() {
+        let dir = TempDir::new().unwrap();
+        let moved = dir.path().join("moved");
+        let a = moved.join("a.txt");
+        let deep = moved.join("one").join("deep.txt");
+        let fake = FakeFs::default().with_subtree(
+            moved.clone(),
+            vec![(a.clone(), "items".into()), (deep.clone(), "items".into())],
+        );
+        let db = subtree_db(fake, dir.path(), Vec::new());
+
+        let events = db.process_file_event(FileEvent::Created(moved));
+
+        assert_eq!(events.len(), 2, "one insert per file found: {events:?}");
+        assert!(
+            events.iter().all(|e| matches!(e, RowEvent::Insert { .. })),
+            "got: {events:?}"
+        );
+        let paths: Vec<String> = events
+            .iter()
+            .filter_map(|e| match e {
+                RowEvent::Insert { file_path, .. } => Some(file_path.clone()),
+                _ => None,
+            })
+            .collect();
+        assert_eq!(
+            paths,
+            vec!["moved/a.txt".to_string(), "moved/one/deep.txt".to_string()],
+            "file paths are root-relative"
+        );
+    }
+
+    /// The walk is matched against the root the event was stripped with, so
+    /// the rows it writes carry the same relative paths a create event would.
+    #[test]
+    fn process_file_event_walks_a_created_directory_under_the_watch_root() {
+        let root = PathBuf::from("/ws");
+        let moved = root.join("moved");
+        let fake = FakeFs::default()
+            .with_canonical_root(&root, "/ws")
+            .with_subtree(moved.clone(), vec![(moved.join("a.txt"), "items".into())]);
+        let db = subtree_db(fake, &root, Vec::new());
+
+        let events = db.process_file_event(FileEvent::Created(moved.clone()));
+
+        assert!(
+            matches!(&events[..], [RowEvent::Insert { file_path, .. }] if file_path == "moved/a.txt"),
+            "got: {events:?}"
+        );
+    }
+
+    /// Only a *created* directory is walked. A metadata change on one says
+    /// nothing about its files, and walking on every such event would re-run
+    /// `on_file` across the whole subtree for a `chmod`.
+    #[test]
+    fn process_file_event_does_not_walk_a_modified_directory() {
+        let dir = TempDir::new().unwrap();
+        let sub = dir.path().join("sub");
+        let fake =
+            FakeFs::default().with_subtree(sub.clone(), vec![(sub.join("a.txt"), "items".into())]);
+        let fake = Arc::new(fake);
+        let db = DirSQL::with_ignore_and_fs(
+            dir.path(),
+            vec![Table::new(
+                "items",
+                "CREATE TABLE items (name TEXT)",
+                "**/*.txt",
+                |_| vec![],
+            )],
+            Vec::<String>::new(),
+            fake.clone(),
+        )
+        .unwrap();
+
+        let events = db.process_file_event(FileEvent::Modified(sub.clone()));
+
+        assert!(events.is_empty(), "got: {events:?}");
+        assert!(
+            fake.subtree_walks().is_empty(),
+            "a modified directory must not be walked; walked: {:?}",
+            fake.subtree_walks()
+        );
+    }
+
+    /// An ignored directory is never walked, matching the initial scan.
+    #[test]
+    fn process_file_event_does_not_walk_an_ignored_created_directory() {
+        let dir = TempDir::new().unwrap();
+        let skip = dir.path().join("skip");
+        let fake = Arc::new(
+            FakeFs::default()
+                .with_subtree(skip.clone(), vec![(skip.join("a.txt"), "items".into())]),
+        );
+        let db = DirSQL::with_ignore_and_fs(
+            dir.path(),
+            vec![Table::new(
+                "items",
+                "CREATE TABLE items (name TEXT)",
+                "**/*.txt",
+                |_| vec![],
+            )],
+            vec!["skip"],
+            fake.clone(),
+        )
+        .unwrap();
+
+        let events = db.process_file_event(FileEvent::Created(skip));
+
+        assert!(events.is_empty(), "got: {events:?}");
+        assert!(
+            fake.subtree_walks().is_empty(),
+            "ignored directory was walked"
+        );
+    }
+
     /// Building with a **relative** root canonicalizes `watch_root` to an
     /// absolute path while leaving `root` exactly as the caller supplied it,
     /// so `notify` never sees `.`.
@@ -2773,6 +3007,19 @@ mod internal_tests {
             fs.is_file(&missing).is_err(),
             "is_file of a missing path must error"
         );
+        assert!(
+            fs.is_dir(dir.path()).unwrap(),
+            "a directory must report is_dir=true"
+        );
+        assert!(
+            fs.is_dir(&missing).is_err(),
+            "is_dir of a missing path must error"
+        );
+        let matcher = TableMatcher::new(&[("**/*", "files")], &[]).unwrap();
+        assert!(
+            fs.scan_subtree(dir.path(), dir.path(), &matcher).is_empty(),
+            "an empty directory walks to nothing"
+        );
         // A nonexistent path can't canonicalize, so the literal fallback runs.
         assert_eq!(
             fs.canonical_root(&missing),

{dirsql-0.4.38 → dirsql-0.4.40}/packages/rust/src/scanner.rs

@@ -34,11 +34,31 @@ pub fn scan_directory_reporting(
     root: &Path,
     matcher: &TableMatcher,
     on_file: &mut dyn FnMut(u64),
+) -> Vec<(PathBuf, String)> {
+    scan_below(root, root, matcher, on_file)
+}
+
+/// [`scan_directory`] restricted to the subtree at `dir`, a directory beneath
+/// `root`. Paths are still matched and ignored relative to `root`, so the
+/// result is exactly the slice of a full scan that falls under `dir`.
+///
+/// This is how the live watcher indexes a directory that appears whole — a
+/// `mkdir` or a rename into the tree — since the OS reports one event for the
+/// directory and none for the files already inside it.
+pub fn scan_subtree(root: &Path, dir: &Path, matcher: &TableMatcher) -> Vec<(PathBuf, String)> {
+    scan_below(root, dir, matcher, &mut |_| {})
+}
+
+fn scan_below(
+    root: &Path,
+    start: &Path,
+    matcher: &TableMatcher,
+    on_file: &mut dyn FnMut(u64),
 ) -> Vec<(PathBuf, String)> {
     let mut results = Vec::new();
     let mut seen: u64 = 0;
 
-    for entry in walk(root, matcher, Path::new(""), false) {
+    for entry in walk(root, start, matcher, Path::new(""), false) {
         let path = entry.path();
 
         // Match against relative path so globs like "comments/**/*.jsonl" work
@@ -92,7 +112,7 @@ pub fn scan_glob(
 ) -> Vec<PathBuf> {
     let mut results = Vec::new();
 
-    for entry in walk(root, ignore, ignore_base, gitignore) {
+    for entry in walk(root, root, ignore, ignore_base, gitignore) {
         if !entry.file_type().is_file() {
             continue;
         }
@@ -145,6 +165,7 @@ pub(crate) fn parse_gitignore_arg(arg: &str) -> Result<bool, String> {
 /// entries a `.gitignore` in force ignores are pruned/skipped too.
 fn walk<'a>(
     root: &Path,
+    start: &Path,
     ignore: &'a TableMatcher,
     ignore_base: &'a Path,
     gitignore: bool,
@@ -154,13 +175,16 @@ fn walk<'a>(
     // first. `filter_entry` visits a directory before its children and the
     // children before the next sibling, so a depth-keyed stack tracks scope.
     let mut frames: Vec<GitignoreFrame> = Vec::new();
-    WalkDir::new(&root)
+    WalkDir::new(start)
         .into_iter()
         .filter_entry(move |entry| {
             let rel_path = entry.path().strip_prefix(&root).unwrap_or(entry.path());
+            // Depth below `root`, not below `start`: the reserved-directory
+            // rule is about the tree's top level wherever the walk begins.
+            let depth = rel_path.components().count();
             let is_dir = entry.file_type().is_dir();
             if !should_descend(
-                entry.depth(),
+                depth,
                 is_dir,
                 entry.file_name(),
                 rel_path,
@@ -172,7 +196,7 @@ fn walk<'a>(
             if !gitignore {
                 return true;
             }
-            while frames.last().is_some_and(|f| f.depth >= entry.depth()) {
+            while frames.last().is_some_and(|f| f.depth >= depth) {
                 frames.pop();
             }
             if is_gitignored(&frames, entry.path(), rel_path, is_dir, ignore_base) {
@@ -180,7 +204,7 @@ fn walk<'a>(
             }
             if is_dir && let Some(matcher) = load_gitignore(entry.path()) {
                 frames.push(GitignoreFrame {
-                    depth: entry.depth(),
+                    depth,
                     dir: rel_path.to_path_buf(),
                     matcher,
                 });

{dirsql-0.4.38 → dirsql-0.4.40}/packages/rust/src/watcher.rs

@@ -79,6 +79,13 @@ fn translate_event(event: &Event) -> Vec<FileEvent> {
             EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
                 Some(FileEvent::Deleted(path.clone()))
             }
+            // inotify follows a matched `From`/`To` pair with a `Both` naming
+            // the same two paths; both sides were already translated above.
+            EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => None,
+            // A rename INTO the tree puts a path here that did not exist a
+            // moment ago — a creation, and for a directory one the watcher must
+            // walk, since the OS reports nothing for the files already inside.
+            EventKind::Modify(ModifyKind::Name(_)) => Some(FileEvent::Created(path.clone())),
             EventKind::Modify(_) => Some(FileEvent::Modified(path.clone())),
             EventKind::Remove(_) => Some(FileEvent::Deleted(path.clone())),
             _ => None,
@@ -135,6 +142,39 @@ fn rename_from_event(paths: Vec<PathBuf>) -> Event {
     }
 }
 
+#[cfg(test)]
+fn rename_to_event(paths: Vec<PathBuf>) -> Event {
+    Event {
+        kind: EventKind::Modify(notify::event::ModifyKind::Name(
+            notify::event::RenameMode::To,
+        )),
+        paths,
+        attrs: Default::default(),
+    }
+}
+
+#[cfg(test)]
+fn rename_any_event(paths: Vec<PathBuf>) -> Event {
+    Event {
+        kind: EventKind::Modify(notify::event::ModifyKind::Name(
+            notify::event::RenameMode::Any,
+        )),
+        paths,
+        attrs: Default::default(),
+    }
+}
+
+#[cfg(test)]
+fn rename_both_event(paths: Vec<PathBuf>) -> Event {
+    Event {
+        kind: EventKind::Modify(notify::event::ModifyKind::Name(
+            notify::event::RenameMode::Both,
+        )),
+        paths,
+        attrs: Default::default(),
+    }
+}
+
 #[cfg(test)]
 fn access_event(paths: Vec<PathBuf>) -> Event {
     Event {
@@ -196,6 +236,39 @@ mod tests {
         );
     }
 
+    #[test]
+    fn translate_event_maps_rename_to_as_create() {
+        // inotify emits `Modify(Name(To))` when a path is renamed INTO the
+        // watched tree; it is a creation so a moved-in directory gets walked.
+        let results = translate_event(&rename_to_event(vec![PathBuf::from("/tmp/arrived")]));
+        assert_eq!(
+            results,
+            vec![FileEvent::Created(PathBuf::from("/tmp/arrived"))]
+        );
+    }
+
+    #[test]
+    fn translate_event_maps_rename_any_as_create() {
+        // kqueue / FSEvents cannot tell which side of a rename they saw; a
+        // creation of a path that turns out not to exist is a harmless no-op.
+        let results = translate_event(&rename_any_event(vec![PathBuf::from("/tmp/either")]));
+        assert_eq!(
+            results,
+            vec![FileEvent::Created(PathBuf::from("/tmp/either"))]
+        );
+    }
+
+    #[test]
+    fn translate_event_ignores_rename_both() {
+        // `Both` restates a `From`/`To` pair inotify already delivered; acting
+        // on it would index the destination twice.
+        let results = translate_event(&rename_both_event(vec![
+            PathBuf::from("/tmp/old"),
+            PathBuf::from("/tmp/new"),
+        ]));
+        assert!(results.is_empty());
+    }
+
     #[test]
     fn translate_event_multiple_paths() {
         let results = translate_event(&create_event(vec![