dirsql 0.4.115__tar.gz → 0.4.116__tar.gz

{dirsql-0.4.115 → dirsql-0.4.116}/Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql"
-version = "0.4.115"
+version = "0.4.116"
 dependencies = [
  "assert_cmd",
  "axum",
@@ -625,7 +625,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.4.115"
+version = "0.4.116"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.4.115 → dirsql-0.4.116}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.4.115
+Version: 0.4.116
 Requires-Dist: bin-shim>=0.1
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index

{dirsql-0.4.115 → dirsql-0.4.116}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.4.115"
+version = "0.4.116"
 # abi3 note: the `abi3-py310` pyo3 feature (below) builds ONE stable-ABI
 # `cp310-abi3` wheel per platform that loads on every CPython >= 3.10
 # (matching `pyproject.toml`'s `requires-python = ">=3.10"`), instead of

dirsql-0.4.116/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/query_contract_test.py -q",
+  "ran_at": 1792158925,
+  "exit_code": 0,
+  "commit": "e01da64bd1c48112f77059fab894961c5dc241ce",
+  "branch": "bot/1480-matcher-speed-1015"
+}

{dirsql-0.4.115 → dirsql-0.4.116}/packages/rust/Cargo.toml

@@ -6,7 +6,7 @@ description = "Ephemeral SQL index over a local directory"
 # version stays for the binding crates (publish = false). The literal
 # intentionally lags the published version: putitoutthere rewrites it to
 # the planned release version at build time and never commits it back.
-version = "0.4.115"
+version = "0.4.116"
 edition.workspace = true
 # Literal `license` (not `license.workspace = true`) because
 # putitoutthere's preflight check reads `[package].license` directly

{dirsql-0.4.115 → dirsql-0.4.116}/packages/rust/src/listing.rs

@@ -100,6 +100,13 @@ impl Listing {
         });
     }
 
+    /// Whether an entry is named `name`; the listing must be sorted.
+    pub(crate) fn holds(&self, name: &OsStr) -> bool {
+        self.entries
+            .binary_search_by(|listed| self.name(listed).cmp(name))
+            .is_ok()
+    }
+
     pub(crate) fn name(&self, listed: &Listed) -> &OsStr {
         self.names.get(listed.start, listed.end)
     }
@@ -138,46 +145,17 @@ fn open_dir(dir: &Path) -> rustix::io::Result<rustix::fd::OwnedFd> {
     rustix::fs::open(dir, flags, Mode::empty())
 }
 
-#[cfg(target_os = "linux")]
-thread_local! {
-    /// Where a walk thread reads its directories' entries: big enough that a
-    /// directory of millions takes few `getdents` calls, and reused so a deep
-    /// tree of small directories pays for it once.
-    static ENTRY_BUFFER: std::cell::RefCell<Vec<u8>> =
-        std::cell::RefCell::new(Vec::with_capacity(1 << 20));
-}
-
-/// Reads entries straight out of the kernel's buffer, where `Dir` would copy
-/// each name into its own allocation.
-#[cfg(target_os = "linux")]
+/// Uses `Dir`, not `RawDir`: on a deep tree of small directories the walk ran
+/// about twice as slow reading through `RawDir`, with no gain on a flat
+/// directory of a million entries.
+#[cfg(unix)]
 fn read_into(dir: &Path, listing: &mut Listing) {
-    use rustix::fs::RawDir;
-    use std::os::unix::ffi::OsStrExt;
-    let Ok(fd) = open_dir(dir) else {
-        return;
-    };
-    ENTRY_BUFFER.with_borrow_mut(|buffer| {
-        let mut entries = RawDir::new(&fd, buffer.spare_capacity_mut());
-        while let Some(Ok(entry)) = entries.next() {
-            let name = entry.file_name().to_bytes();
-            if name == b"." || name == b".." {
-                continue;
-            }
-            listing.push(OsStr::from_bytes(name), seen(entry.file_type()));
-        }
-    });
-}
-
-/// `read_into` for Unix systems without `RawDir`. Built under test too, so the
-/// Linux unit run covers it.
-#[cfg(all(unix, any(test, not(target_os = "linux"))))]
-fn read_dir_with_copies_into(dir: &Path, listing: &mut Listing) {
     use rustix::fs::Dir;
     use std::os::unix::ffi::OsStrExt;
     let Ok(fd) = open_dir(dir) else {
         return;
     };
-    let Ok(mut entries) = Dir::read_from(&fd) else {
+    let Ok(mut entries) = Dir::new(fd) else {
         return;
     };
     while let Some(Ok(entry)) = entries.read() {
@@ -214,8 +192,6 @@ fn seen_flags(symlink: bool, dir: bool, file: bool) -> Seen {
 
 #[cfg(not(unix))]
 use read_dir_into as read_into;
-#[cfg(all(unix, not(target_os = "linux")))]
-use read_dir_with_copies_into as read_into;
 
 /// A name's first eight bytes as a big-endian word, which orders names as
 /// their bytes do as far as those bytes go.
@@ -271,6 +247,19 @@ mod tests {
         assert_eq!(names(&listing), ["first", "second"]);
     }
 
+    #[test]
+    fn a_sorted_listing_holds_exactly_the_names_pushed() {
+        let mut listing = Listing::default();
+        for name in [".gitignore", "b", ".git", "a"] {
+            listing.push(OsStr::new(name), Seen::File);
+        }
+        listing.sort();
+        assert!(listing.holds(OsStr::new(".git")));
+        assert!(listing.holds(OsStr::new("a")));
+        assert!(!listing.holds(OsStr::new(".gi")));
+        assert!(!Listing::default().holds(OsStr::new(".git")));
+    }
+
     #[test]
     fn name_prefix_orders_names_as_their_bytes_do() {
         let mut names = ["abcdefghZ", "abcdefgh", "abcdefghA", "ab", "b", "abc", ""];
@@ -344,16 +333,6 @@ mod tests {
         assert_eq!(read_with(read_into, dir.path()), fixture_entries());
     }
 
-    #[cfg(unix)]
-    #[test]
-    fn read_dir_with_copies_into_lists_the_entries_without_dot_and_dot_dot() {
-        let dir = fixture();
-        assert_eq!(
-            read_with(read_dir_with_copies_into, dir.path()),
-            fixture_entries()
-        );
-    }
-
     #[test]
     fn read_dir_into_classifies_what_std_reports() {
         let dir = fixture();

{dirsql-0.4.115 → dirsql-0.4.116}/packages/rust/src/scanner.rs

@@ -126,6 +126,7 @@ pub fn scan_dirs(root: &Path, start: &Path, ignore: &TableMatcher) -> Vec<PathBu
         root,
         start,
         walker,
+        &|_| true,
         &|_| false,
         &|rel_path, parent| parent.join(rel_path.file_name().unwrap_or_default()),
         &mut |dir| dirs.push(dir),
@@ -163,6 +164,7 @@ fn scan_below(
         root,
         start,
         walker,
+        &|_| true,
         &|rel_path| !matcher.is_ignored(rel_path),
         // Matching and the relative path are made on the worker that found
         // the file: the calling thread hands out every file in turn, so any
@@ -267,6 +269,7 @@ pub(crate) fn scan_glob_rows<T: Send>(
         root,
         root,
         walker,
+        &|name| glob.may_match_name(name),
         &|rel_path| is_wanted(glob, ignore, rel_path),
         &|rel_path, _| row(rel_path),
         visit,
@@ -304,6 +307,8 @@ pub struct PathGlob {
     components: Vec<Component>,
     /// Where each word's components begin.
     starts: Vec<usize>,
+    /// Where each word's closing `End` sits.
+    ends: Vec<usize>,
     /// Whether any word spells a `.` or `..` component, which the walk has to
     /// enter itself: a directory listing never names them.
     spells_relative: bool,
@@ -362,6 +367,18 @@ impl PathGlob {
                 .all(|name| self.spells_dot_name(name))
     }
 
+    /// Whether a file named `name` may match: false when no word could match
+    /// it, since only `**` crosses a `/` and so a word's last component, unless
+    /// it is `**`, has to match the name alone.
+    pub(crate) fn may_match_name(&self, name: &OsStr) -> bool {
+        self.ends
+            .iter()
+            .any(|&end| match &self.components[end - 1] {
+                Component::Name(p) | Component::DotName(p) => p.is_match(Path::new(name)),
+                Component::AnyDepth | Component::End => true,
+            })
+    }
+
     fn has_components(&self) -> bool {
         !self.components.is_empty()
     }
@@ -465,10 +482,12 @@ pub fn compile_globs(patterns: &[&str]) -> Result<PathGlob, GlobError> {
         .collect();
     let mut components = Vec::new();
     let mut starts = Vec::new();
+    let mut ends = Vec::new();
     for word in &words {
         let word_components = compile_components(word)?;
         starts.push(components.len());
         components.extend(word_components);
+        ends.push(components.len());
         components.push(Component::End);
     }
     Ok(PathGlob {
@@ -476,6 +495,7 @@ pub fn compile_globs(patterns: &[&str]) -> Result<PathGlob, GlobError> {
         spelled_dot_names,
         components,
         starts,
+        ends,
         spells_relative,
         named,
     })
@@ -547,6 +567,7 @@ fn walk<L: Send>(
     root: &Path,
     start: &Path,
     walker: Walk<'_>,
+    keep_name: &(dyn Fn(&OsStr) -> bool + Sync),
     keep: &(dyn Fn(&Path) -> bool + Sync),
     leaf: &(dyn Fn(PathBuf, &Path) -> L + Sync),
     visit: &mut dyn FnMut(L),
@@ -566,7 +587,7 @@ fn walk<L: Send>(
     };
     walk_in_order(
         Box::new(place),
-        &|place: Box<Place<'_>>| (*place).explore(keep, leaf),
+        &|place: Box<Place<'_>>| (*place).explore(keep_name, keep, leaf),
         visit,
     );
 }
@@ -585,9 +606,11 @@ struct Place<'a> {
 impl<'a> Place<'a> {
     /// The directory's entries in walk order: each file `keep` takes, made
     /// into a `leaf` with the directory it sits in, and each directory to
-    /// enter.
+    /// enter. A regular file whose name `keep_name` rejects is dropped before
+    /// its path is built.
     fn explore<L: Send>(
         self,
+        keep_name: &(dyn Fn(&OsStr) -> bool + Sync),
         keep: &(dyn Fn(&Path) -> bool + Sync),
         leaf: &(dyn Fn(PathBuf, &Path) -> L + Sync),
     ) -> Vec<Step<Box<Place<'a>>, L>> {
@@ -598,37 +621,35 @@ impl<'a> Place<'a> {
             states,
             linked,
         } = self;
-        if walk.gitignore && holds_git(&dir) {
+        let listing = Listing::read(&dir);
+        if walk.gitignore && listing.holds(OsStr::new(".git")) && holds_git(&dir) {
             walk.in_repo = true;
             walk.frames.clear();
         }
         if walk.in_repo
+            && listing.holds(OsStr::new(".gitignore"))
             && let Some(frame) = Frame::load(&dir)
         {
             walk.frames.push(Arc::new(frame));
         }
-        let listing = Listing::read(&dir);
         let taken = judge_all(listing.entries(), &|listed| {
-            walk.take(
-                &dir,
-                listing.name(listed),
-                listed.seen,
-                &rel,
-                &states,
-                linked,
-            )
-            .and_then(|taken| match taken {
-                Taken::File(child) => keep(&child).then(|| Taken::File(leaf(child, &dir))),
-                Taken::Dir {
-                    child,
-                    next,
-                    linked,
-                } => Some(Taken::Dir {
-                    child,
-                    next,
-                    linked,
-                }),
-            })
+            let name = listing.name(listed);
+            if listed.seen == Seen::File && !keep_name(name) {
+                return None;
+            }
+            walk.take(&dir, name, listed.seen, &rel, &states, linked)
+                .and_then(|taken| match taken {
+                    Taken::File(child) => keep(&child).then(|| Taken::File(leaf(child, &dir))),
+                    Taken::Dir {
+                        child,
+                        next,
+                        linked,
+                    } => Some(Taken::Dir {
+                        child,
+                        next,
+                        linked,
+                    }),
+                })
         });
         let mut steps: Vec<Step<Box<Place<'a>>, L>> = if walk.dirs {
             taken
@@ -1235,7 +1256,7 @@ mod tests {
 
     fn explored_dirs(place: Place<'_>) -> Vec<Place<'_>> {
         place
-            .explore(&|_| true, &|_, _| ())
+            .explore(&|_| true, &|_| true, &|_, _| ())
             .into_iter()
             .filter_map(|step| match step {
                 Step::Dir(child) => Some(*child),
@@ -1269,6 +1290,65 @@ mod tests {
         );
     }
 
+    #[test]
+    fn explore_drops_a_file_whose_name_is_rejected_before_judging_its_path() {
+        let root = tempfile::tempdir().unwrap();
+        fs::write(root.path().join("a.md"), "").unwrap();
+        fs::write(root.path().join("b.md"), "").unwrap();
+        fs::create_dir(root.path().join("d")).unwrap();
+        let ignore = TableMatcher::new(&[], &[]).unwrap();
+        let top = Place {
+            walk: walk_with(&ignore, None, Vec::new()),
+            dir: root.path().to_path_buf(),
+            rel: PathBuf::new(),
+            states: Vec::new(),
+            linked: false,
+        };
+        let judged = std::sync::Mutex::new(Vec::new());
+        let steps = top.explore(
+            &|name| name != OsStr::new("b.md"),
+            &|rel| {
+                judged.lock().unwrap().push(rel.to_path_buf());
+                true
+            },
+            &|rel, _| rel,
+        );
+        let leaves: Vec<PathBuf> = steps
+            .into_iter()
+            .filter_map(|step| match step {
+                Step::Leaf(rel) => Some(rel),
+                Step::Dir(_) => None,
+            })
+            .collect();
+        assert_eq!(leaves, vec![PathBuf::from("a.md")]);
+        assert_eq!(judged.into_inner().unwrap(), vec![PathBuf::from("a.md")]);
+    }
+
+    #[test]
+    fn explore_outside_gitignore_mode_ignores_a_repo_it_enters() {
+        let root = tempfile::tempdir().unwrap();
+        fs::create_dir(root.path().join(".git")).unwrap();
+        fs::write(root.path().join(".gitignore"), "x.md\n").unwrap();
+        fs::write(root.path().join("x.md"), "").unwrap();
+        let ignore = TableMatcher::new(&[], &[]).unwrap();
+        let top = Place {
+            walk: walk_with(&ignore, None, Vec::new()),
+            dir: root.path().to_path_buf(),
+            rel: PathBuf::new(),
+            states: Vec::new(),
+            linked: false,
+        };
+        let leaves: Vec<PathBuf> = top
+            .explore(&|_| true, &|_| true, &|rel, _| rel)
+            .into_iter()
+            .filter_map(|step| match step {
+                Step::Leaf(rel) => Some(rel),
+                Step::Dir(_) => None,
+            })
+            .collect();
+        assert!(leaves.contains(&PathBuf::from("x.md")), "{leaves:?}");
+    }
+
     #[test]
     fn explore_enters_a_dirsql_directory_below_the_top_level() {
         let root = tempfile::tempdir().unwrap();
@@ -1686,6 +1766,21 @@ mod tests {
         })
     }
 
+    #[test]
+    fn a_name_may_match_only_when_some_word_ends_in_a_component_matching_it() {
+        let glob = compile_glob("./**/{a,b}*.md").unwrap();
+        assert!(glob.may_match_name(OsStr::new("a1.md")));
+        assert!(glob.may_match_name(OsStr::new("b1.md")));
+        assert!(!glob.may_match_name(OsStr::new("c1.md")));
+        assert!(!glob.may_match_name(OsStr::new("a1.txt")));
+    }
+
+    #[test]
+    fn a_word_ending_in_a_double_star_may_match_any_name() {
+        let glob = compile_glob("{docs/**,*.md}").unwrap();
+        assert!(glob.may_match_name(OsStr::new("a.txt")));
+    }
+
     #[test]
     fn a_double_star_starts_matched_or_not_yet_consumed() {
         assert_eq!(compile_glob("**/*.md").unwrap().start(), vec![0, 1]);