rgapi 0.2.1__tar.gz → 0.2.2__tar.gz

{rgapi-0.2.1 → rgapi-0.2.2}/Cargo.lock

@@ -11,6 +11,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "bit-set"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+
 [[package]]
 name = "bstr"
 version = "1.13.1"
@@ -18,7 +33,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
 dependencies = [
  "memchr",
- "regex-automata",
  "serde_core",
 ]
 
@@ -91,6 +105,17 @@ dependencies = [
  "encoding_rs",
 ]
 
+[[package]]
+name = "fancy-regex"
+version = "0.19.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912"
+dependencies = [
+ "bit-set",
+ "regex-automata",
+ "regex-syntax",
+]
+
 [[package]]
 name = "globset"
 version = "0.4.20"
@@ -113,19 +138,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "grep-regex"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef"
-dependencies = [
- "bstr",
- "grep-matcher",
- "log",
- "regex-automata",
- "regex-syntax",
-]
-
 [[package]]
 name = "grep-searcher"
 version = "0.1.17"
@@ -308,24 +320,24 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "rgapi"
-version = "0.2.1"
+version = "0.2.2"
 dependencies = [
  "crc32fast",
+ "fancy-regex",
  "globset",
  "grep-matcher",
- "grep-regex",
  "grep-searcher",
  "ignore",
+ "regex-syntax",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "rgapi-py"
-version = "0.2.1"
+version = "0.2.2"
 dependencies = [
  "grep-matcher",
- "grep-regex",
  "pyo3",
  "pyo3-build-config",
  "rgapi",

{rgapi-0.2.1 → rgapi-0.2.2}/Cargo.toml

@@ -2,12 +2,11 @@
 members = ["py"]
 
 [workspace.package]
-version = "0.2.1"
+version = "0.2.2"
 edition = "2024"
 
 [workspace.dependencies]
 grep-matcher = ">=0.1.8"
-grep-regex = ">=0.1.14"
 
 [package]
 name = "rgapi"
@@ -25,11 +24,12 @@ name = "rgapi"
 
 [dependencies]
 crc32fast = "1"
+fancy-regex = "0.19"
 globset = ">=0.4.18"
 grep-matcher.workspace = true
-grep-regex.workspace = true
 grep-searcher = ">=0.1.16"
 ignore = ">=0.4.26"
+regex-syntax = "0.8"
 serde = { version = "=1", features = ["derive"] }
 serde_json = "=1"
 

{rgapi-0.2.1 → rgapi-0.2.2}/DEV.md

@@ -5,8 +5,9 @@
 ## Layout
 
 ```text
-src/walk.rs       ignore/globset/grep-regex-backed path walking and filtering
-src/search.rs     grep-regex/grep-searcher-backed searching
+src/matcher.rs    shared fancy-regex engine, smart-case policy, and grep-matcher adapter
+src/walk.rs       ignore/globset-backed path walking and filtering
+src/search.rs     grep-searcher-backed searching
 src/block.rs      blank-line-delimited block grouping, matching, and block context
 py/src/lib.rs     PyO3 classes and private core functions
 python/rgapi/     public Python wrappers over `rgapi._core`, plus the `rgapi-nbrg` CLI
@@ -38,6 +39,10 @@ The GitHub workflow builds wheels for Python 3.10-3.13 on Linux and macOS and pu
 
 ## Design notes
 
+All user-supplied regexes use the shared `RegexMatcher` in `src/matcher.rs`, backed by `fancy-regex`. There is no separate fallback engine or native C dependency. The adapter implements `grep_matcher::Matcher` so `grep-searcher` continues to handle file scanning, line numbers, context, and binary detection. `compile_regex` returns rgapi's own exported `RegexMatcher` type.
+
+The matcher centralizes smart-case analysis and line-ending policy. Line-safe patterns can search a whole buffer; patterns that can consume line endings or inspect neighbouring lines are evaluated on individual lines. Notebook multiline matching instead uses the whole cell. Filename/path regexes match their whole input. Backtracking-limit errors propagate through the existing result channels, including directory-pruning errors; cancellation must not hide a queued error from async consumers.
+
 Python discovery and `paths=True` results contain absolute `pathlib.Path` objects. Structured search rows hold base-relative string labels with `/` separators. Traversal uses `ignore::WalkParallel`, so result order is not part of the API contract. Search results are structured rows; collected result lists use rg-style `str()` and notebook display. `SearchLine.lnhash` is computed with the same CRC-32-based line-content hash format as exhash (`lineno|hash|`, low 12 bits of CRC-32 over the line's UTF-8 bytes, encoded as two Base64url characters); `lnhashs=True` only changes row display, not `line_number` or matching behavior. Path regexes filter returned/searched paths; `skip_dir` and `skip_dir_re` prune traversal through `ignore::WalkBuilder::filter_entry`. Depth, size, filesystem, hidden, and ignore options use `ignore::WalkBuilder` settings. The `ignore` walker follows every root it is given. Discovery checks each root with `symlink_metadata`. The worker sends each unfollowed link root as a result before walking the other roots. This also supports dangling roots, which the underlying walker would reject. `ls` sets `walk_root_links`, which gives a root link to a directory to the walker. The walker then reports paths under the link. Other discovery roots use absolute paths without canonicalizing. Content searches retain canonical root resolution. `rg_iter` exposes the same parallel search stream that `rg` collects by default; `paths=True` and `count=True` consume that stream with different reducers. Text search skips binary files and invalid UTF-8 content.
 
 Streaming engine: `walk.rs` owns the generic machinery. `StreamIter<T>` is the worker-thread-plus-bounded-channel iterator (`sync_channel(8192)`, so producers block rather than buffer without limit when a consumer lags), and `spawn_walk` owns the shared scaffold: walker config, panic catching, cancel flag, and worker thread. `rg_iter` (`T = SearchLine`), `block_iter` (`T = SearchBlock`), `nb_iter` (`T = NbCell`), and `find_iter` (`T = PathBuf`, the path walk) plug entry closures into that engine. Block search reads each file once, searches it once, groups nonblank lines into blocks, maps matching lines to their blocks, and expands context by block index. Each `SearchBlock` carries numeric boundaries plus hashes for its first and last source lines. Python keeps both and chooses the displayed address without another file read.

{rgapi-0.2.1 → rgapi-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rgapi
-Version: 0.2.1
+Version: 0.2.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Dist: fastcore>=2.2.29
@@ -23,7 +23,7 @@ Project-URL: Repository, https://github.com/AnswerDotAI/rgapi
 
 `rgapi` provides `fd`-style file discovery and `rg`-style text search from Python without starting a shell command.
 
-It uses the same `ignore`, `grep-regex`, and `grep-searcher` crates that ripgrep uses for walking, regex matching, and file scanning. Walking and searching run in parallel by default. Most expensive work stays in Rust.
+It uses ripgrep's `ignore` and `grep-searcher` crates for walking and file scanning, with the pure-Rust `fancy-regex` engine for regex matching. Walking and searching run in parallel by default. Most expensive work stays in Rust.
 
 ## Overview
 
@@ -108,6 +108,16 @@ Set `min_depth` and `max_depth` to bound recursion. `max_filesize` skips files a
 
 Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behaviour. Use `case_sensitive=False` to force case-insensitive matching.
 
+Patterns support backreferences and lookaround everywhere, including filename and path filters:
+
+```python
+rg(r"\b(\w+)\s+\1\b", ".")  # repeated words
+rg(r"(?<=TODO: )\w+", ".")   # word following TODO:
+fd(".", pattern=r"^(\w+)_\1\.py$")  # names such as utils_utils.py
+```
+
+All regex parameters use `fancy-regex`; no engine-selection flag or C library is required. Ordinary patterns use its non-backtracking engine. Patterns requiring backtracking have a bounded backtracking budget; exceeding it raises `ValueError`, rather than silently skipping a file or path.
+
 Each `SearchLine` has these fields:
 
 ```text
@@ -127,7 +137,9 @@ Pass `lnhashs=True` to `rg` or `rg_iter` to display hash addresses instead of li
 
 For other result forms, use `rg(..., paths=True)` to return unique matched paths or `rg(..., count=True)` to count match spans. `paths` and `count` cannot both be set.
 
-`^` and `$` match at the start and end of each line. A line ends at `\n` or `\r\n`. `$` also matches before a `\r` that has no `\n` after it. `rg`, `rgstr`, and `nbrg` without `multiline=True` raise `ValueError` for a pattern that contains a literal `\n` or `\r`.
+`^` and `$` match at the start and end of each line. A line ends at `\n` or `\r\n`. `$` also matches before a `\r` that has no `\n` after it. `rg`, `rgstr`, and `compile` reject patterns containing a literal `\n` or `\r`. `nbrg` allows them with `multiline=True`.
+
+Content searches and compiled content matchers are confined to individual lines: whitespace classes, backreferences and lookaround cannot cross a line ending, and `\A`/`\z` refer to each line. With `nbrg(..., multiline=True)`, `\A`/`\z` refer to the whole cell, while `^`/`$` still anchor lines. Filename and path filters match the full name or path instead.
 
 ### Path results
 
@@ -159,6 +171,8 @@ Set `timeout_ms` on `rg`, `fd`, `walk`, or `ls` to stop at a deadline and return
 
 `complete` is true exactly when `stop_reason` is `None`. `count=True` returns a plain integer without a completion flag. It cannot be combined with `timeout_ms`.
 
+Deadlines stop result collection and request worker cancellation. Workers cannot interrupt a regex evaluation or filesystem call already in progress; a timeout is not a hard per-match execution limit.
+
 ### Context lines
 
 `before_context`, `after_context`, and `context` correspond to `rg -B`, `rg -A`, and `rg -C`. Files containing NUL bytes or invalid UTF-8 are skipped.
@@ -190,7 +204,9 @@ nbrg("read_csv", ".")                  # cells whose source matches, across all
 nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
 ```
 
-Notebook discovery, parsing, and matching run together in one parallel Rust pass. Matching uses `rg`'s regex engine with the same `case_sensitive` and `smart_case` behaviour. `nbrg` accepts the discovery filters from `fd` and `rg`, including `include`, `exclude`, `glob`, `hidden`, `max_depth`, and `skip_dir`.
+Notebook discovery, parsing, and matching run together in one parallel Rust pass. Matching uses the same `fancy-regex` engine as `rgapi.rg`, including backreferences and lookaround, with the same `case_sensitive` and `smart_case` behaviour. `nbrg` accepts the discovery filters from `fd` and `rg`, including `include`, `exclude`, `glob`, `hidden`, `max_depth`, and `skip_dir`.
+
+Matching is line-oriented by default. Use `nbrg(r"(foo)\n\1", ".", multiline=True)` to match across lines within a cell. Matches never cross cell boundaries.
 
 `nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
 
@@ -248,7 +264,7 @@ async for row in rga_iter("TODO", "."): ...
 
 Walking and searching use Rust threads, without `asyncio.to_thread` or the event loop's executor. A callback uses `loop.call_soon_threadsafe` to complete the awaited future or supply results to the generator's queue. The event loop remains unblocked, with normal `contextvars` behaviour.
 
-Cancellation stops the Rust workers within about one row. This includes timeouts from `asyncio.wait_for` or `asyncio.timeout`. It also includes task cancellation, such as starlette cancelling a disconnected client's request.
+Cancellation is checked between rows and files; it does not interrupt a regex evaluation or filesystem call already in progress. This includes timeouts from `asyncio.wait_for` or `asyncio.timeout`. It also includes task cancellation, such as starlette cancelling a disconnected client's request.
 
 Wrap an async iterator in `contextlib.aclosing` when leaving its loop early. `break` alone delays generator finalization until garbage collection. The context manager provides prompt cleanup:
 

{rgapi-0.2.1 → rgapi-0.2.2}/README.md

@@ -2,7 +2,7 @@
 
 `rgapi` provides `fd`-style file discovery and `rg`-style text search from Python without starting a shell command.
 
-It uses the same `ignore`, `grep-regex`, and `grep-searcher` crates that ripgrep uses for walking, regex matching, and file scanning. Walking and searching run in parallel by default. Most expensive work stays in Rust.
+It uses ripgrep's `ignore` and `grep-searcher` crates for walking and file scanning, with the pure-Rust `fancy-regex` engine for regex matching. Walking and searching run in parallel by default. Most expensive work stays in Rust.
 
 ## Overview
 
@@ -87,6 +87,16 @@ Set `min_depth` and `max_depth` to bound recursion. `max_filesize` skips files a
 
 Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behaviour. Use `case_sensitive=False` to force case-insensitive matching.
 
+Patterns support backreferences and lookaround everywhere, including filename and path filters:
+
+```python
+rg(r"\b(\w+)\s+\1\b", ".")  # repeated words
+rg(r"(?<=TODO: )\w+", ".")   # word following TODO:
+fd(".", pattern=r"^(\w+)_\1\.py$")  # names such as utils_utils.py
+```
+
+All regex parameters use `fancy-regex`; no engine-selection flag or C library is required. Ordinary patterns use its non-backtracking engine. Patterns requiring backtracking have a bounded backtracking budget; exceeding it raises `ValueError`, rather than silently skipping a file or path.
+
 Each `SearchLine` has these fields:
 
 ```text
@@ -106,7 +116,9 @@ Pass `lnhashs=True` to `rg` or `rg_iter` to display hash addresses instead of li
 
 For other result forms, use `rg(..., paths=True)` to return unique matched paths or `rg(..., count=True)` to count match spans. `paths` and `count` cannot both be set.
 
-`^` and `$` match at the start and end of each line. A line ends at `\n` or `\r\n`. `$` also matches before a `\r` that has no `\n` after it. `rg`, `rgstr`, and `nbrg` without `multiline=True` raise `ValueError` for a pattern that contains a literal `\n` or `\r`.
+`^` and `$` match at the start and end of each line. A line ends at `\n` or `\r\n`. `$` also matches before a `\r` that has no `\n` after it. `rg`, `rgstr`, and `compile` reject patterns containing a literal `\n` or `\r`. `nbrg` allows them with `multiline=True`.
+
+Content searches and compiled content matchers are confined to individual lines: whitespace classes, backreferences and lookaround cannot cross a line ending, and `\A`/`\z` refer to each line. With `nbrg(..., multiline=True)`, `\A`/`\z` refer to the whole cell, while `^`/`$` still anchor lines. Filename and path filters match the full name or path instead.
 
 ### Path results
 
@@ -138,6 +150,8 @@ Set `timeout_ms` on `rg`, `fd`, `walk`, or `ls` to stop at a deadline and return
 
 `complete` is true exactly when `stop_reason` is `None`. `count=True` returns a plain integer without a completion flag. It cannot be combined with `timeout_ms`.
 
+Deadlines stop result collection and request worker cancellation. Workers cannot interrupt a regex evaluation or filesystem call already in progress; a timeout is not a hard per-match execution limit.
+
 ### Context lines
 
 `before_context`, `after_context`, and `context` correspond to `rg -B`, `rg -A`, and `rg -C`. Files containing NUL bytes or invalid UTF-8 are skipped.
@@ -169,7 +183,9 @@ nbrg("read_csv", ".")                  # cells whose source matches, across all
 nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
 ```
 
-Notebook discovery, parsing, and matching run together in one parallel Rust pass. Matching uses `rg`'s regex engine with the same `case_sensitive` and `smart_case` behaviour. `nbrg` accepts the discovery filters from `fd` and `rg`, including `include`, `exclude`, `glob`, `hidden`, `max_depth`, and `skip_dir`.
+Notebook discovery, parsing, and matching run together in one parallel Rust pass. Matching uses the same `fancy-regex` engine as `rgapi.rg`, including backreferences and lookaround, with the same `case_sensitive` and `smart_case` behaviour. `nbrg` accepts the discovery filters from `fd` and `rg`, including `include`, `exclude`, `glob`, `hidden`, `max_depth`, and `skip_dir`.
+
+Matching is line-oriented by default. Use `nbrg(r"(foo)\n\1", ".", multiline=True)` to match across lines within a cell. Matches never cross cell boundaries.
 
 `nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
 
@@ -227,7 +243,7 @@ async for row in rga_iter("TODO", "."): ...
 
 Walking and searching use Rust threads, without `asyncio.to_thread` or the event loop's executor. A callback uses `loop.call_soon_threadsafe` to complete the awaited future or supply results to the generator's queue. The event loop remains unblocked, with normal `contextvars` behaviour.
 
-Cancellation stops the Rust workers within about one row. This includes timeouts from `asyncio.wait_for` or `asyncio.timeout`. It also includes task cancellation, such as starlette cancelling a disconnected client's request.
+Cancellation is checked between rows and files; it does not interrupt a regex evaluation or filesystem call already in progress. This includes timeouts from `asyncio.wait_for` or `asyncio.timeout`. It also includes task cancellation, such as starlette cancelling a disconnected client's request.
 
 Wrap an async iterator in `contextlib.aclosing` when leaving its loop early. `break` alone delays generator finalization until garbage collection. The context manager provides prompt cleanup:
 

{rgapi-0.2.1 → rgapi-0.2.2}/py/Cargo.toml

@@ -11,7 +11,6 @@ test = false
 [dependencies]
 rgapi = { path = ".." }
 grep-matcher.workspace = true
-grep-regex.workspace = true
 pyo3 = { version = ">=0.29", features = ["extension-module"] }
 
 [build-dependencies]

{rgapi-0.2.1 → rgapi-0.2.2}/py/src/lib.rs

@@ -179,7 +179,7 @@ struct RegexPy {
     case_sensitive: Option<bool>,
     #[pyo3(get)]
     smart_case: bool,
-    matcher: grep_regex::RegexMatcher,
+    matcher: rgapi::RegexMatcher,
 }
 #[pymethods]
 impl RegexPy {

{rgapi-0.2.1 → rgapi-0.2.2}/python/rgapi/__init__.py

@@ -17,7 +17,10 @@ def compile(
     case_sensitive:bool|None=None, # True/False forces case; None allows `smart_case`
     smart_case:bool=False # Match `rg --smart-case` behavior
 ) -> Regex:
-    "Compile a regex matcher for `search_text`, `search_path`, and direct matching."
+    """Compile a line-oriented fancy-regex matcher, supporting backreferences and lookaround.
+
+    `is_match` and `finditer` search each line independently; match offsets are UTF-8 bytes.
+    Compilation or backtracking-limit failures raise `ValueError`."""
     return _core.compile(pattern, case_sensitive=case_sensitive, smart_case=smart_case)
 
 class _Results(list):
@@ -159,7 +162,9 @@ def fd(
     timeout_ms:int|None=None, # Cancel the walk after this long and return partial results
     **kwargs
 ) -> PathResults:
-    "Find absolute Paths with fd-style filters and gitignore support."
+    """Find absolute Paths with fd-style filters and gitignore support.
+
+    Filename/path regexes support backreferences and lookaround, matching the full name/path. Regex errors raise `ValueError`."""
     return _find(root, pattern, files, dirs, timeout_ms, show_target, **kwargs)
 
 
@@ -306,6 +311,9 @@ def rg(
 ):
     """Search files and return `SearchResults`, matched paths, or a count; `lnhashs=True` shows exhash-style addresses.
 
+    Patterns support backreferences and lookaround, confined to individual lines. Compilation or backtracking-limit failures
+    raise `ValueError`. Timeouts request cancellation but do not interrupt a regex evaluation already in progress.
+
     `summary=True` returns one `SearchBlock` per block of lines separated by blank or whitespace-only lines, however many matches
     it holds, and `context` then counts blocks. Summary mode can't combine with `paths` or `count`, and with `lnhashs=True` shows
     each block's boundary addresses. `^` and `$` anchor each line, and a line ends at LF or CRLF. Patterns containing a newline or

{rgapi-0.2.1 → rgapi-0.2.2}/python/rgapi/_cli.py

@@ -22,7 +22,7 @@ def nbrg_cli(
     max_depth:int=None, # Maximum directory depth to search
     timeout_ms:int=None, # Stop searching after this many milliseconds
 ):
-    "Search notebook cell sources and report stable cell IDs."
+    "Search notebook cell sources with backreferences/lookaround and report stable cell IDs."
     print(nbrg(pattern, roots or '.', cell_context=cell_context, multiline=multiline, smart_case=smart_case,
         case_sensitive=True if case else None, paths=paths, count=count, max_results=max_results, maxlen=maxlen,
         glob=glob, exclude=exclude, hidden=hidden, max_depth=max_depth, timeout_ms=timeout_ms))

{rgapi-0.2.1 → rgapi-0.2.2}/python/rgapi/nb.py

@@ -64,7 +64,7 @@ def search_nb(
     display_path:str|Path|None=None, # Path stored in results; defaults to `path`
     maxlen:int=MAXLEN,                # Maximum source characters per displayed cell
 ) -> NbResults:
-    "Search one `.ipynb` file's cell sources, returning matched cells."
+    "Search one `.ipynb` file's cell sources with line-local backreferences and lookaround, returning matched cells."
     disp = _display_path(path if display_path is None else display_path)
     rows = _core.nb_search_file(pattern, _fs_path(path), disp, case_sensitive=case_sensitive,
         smart_case=smart_case, cell_context=cell_context, multiline=False)
@@ -100,7 +100,9 @@ def nbrg(
 ):
     """Search `.ipynb` cell sources under `root` in parallel, returning matched cells, paths, or a count.
 
-    Only cell sources are searched, never metadata or outputs. With `multiline=True`, `^` and `$` still anchor lines within a cell.
+    Only cell sources are searched, never metadata or outputs. Backreferences and lookaround are line-local by default.
+    `multiline=True` allows matching across lines within a cell, never across cells; `^` and `$` still anchor individual lines.
+    Compilation or backtracking-limit failures raise `ValueError`; timeouts cannot interrupt an in-progress regex evaluation.
     A match preview starts at the first matched line and marks omitted earlier lines as `…[Ln]` (1-based), but keeps a leading
     directive, as in `#| export…[L4]needle here`."""
     assert not (count and max_results), "count and max_results are mutually exclusive"

{rgapi-0.2.1 → rgapi-0.2.2}/python/rgapi/skill.py

@@ -1,6 +1,6 @@
 """Find files, search text, and search notebook cell sources from Python with ripgrep semantics and structured results. Use for fd-style discovery, regex searches, and notebook searches returning stable cell IDs rather than escaped JSON.
 
-Built on ripgrep's `ignore`, `grep-regex`, and `grep-searcher` crates: ignore files, hidden files, globs, extensions, and regex syntax behave as in ripgrep. Prefer these to shell parsing or manual file scans: `fd`/`ls` for listings, `rg` for files, `nbrg` for notebook cells, `rgstr` for text already in hand.
+Built on ripgrep's `ignore` and `grep-searcher` crates with pure-Rust `fancy-regex`: ignore files, hidden files, globs and extensions follow ripgrep conventions. All regex parameters support backreferences and lookaround without an engine flag. Content matching is line-local unless `nbrg(multiline=True)` is used; filename/path regexes match their full input. Regex compilation or backtracking-limit failures raise `ValueError`; cancellation cannot interrupt an in-progress regex evaluation. Prefer these to shell parsing or manual file scans: `fd`/`ls` for listings, `rg` for files, `nbrg` for notebook cells, `rgstr` for text already in hand.
 
 Orient with `rg(summary=True)`: one row per block (a paragraph of prose/config) with boundary addresses, where line results show fragments that may need another view. Switch to line results where needed; add `lnhashs=True` when edits may follow. Display results bare; narrow oversized results with parameters, not by joining, slicing, or reformatting.
 

{rgapi-0.2.1 → rgapi-0.2.2}/src/block.rs

@@ -3,11 +3,12 @@ use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::Ordering;
 
-use grep_regex::RegexMatcher;
+use crate::RegexMatcher;
 use ignore::{DirEntry, WalkState};
 
 use crate::RgApiError;
-use crate::search::{RgOptions, SearchLine, compile_regex, format_lnhash, search_text};
+use crate::search::{RgOptions, SearchLine, format_lnhash, search_text};
+use crate::compile_regex;
 use crate::walk::{PathFilters, StreamIter, entry_err, rel_path, resolve_roots, spawn_walk};
 
 /// One blank-line-delimited block containing a match, or context for one.
@@ -97,7 +98,7 @@ fn block_entry(
     let Some(ft) = dent.file_type() else { return Ok(Vec::new()); };
     if !ft.is_file() { return Ok(Vec::new()); }
     let rel = rel_path(base, path);
-    if !filters.path_allowed(Path::new(&rel)) { return Ok(Vec::new()); }
+    if !filters.path_allowed(Path::new(&rel))? { return Ok(Vec::new()); }
     let bytes = match std::fs::read(path) { Ok(bytes) => bytes, Err(_) => return Ok(Vec::new()) };
     process_file(rel, &bytes, matcher, before_context, after_context)
 }

{rgapi-0.2.1 → rgapi-0.2.2}/src/lib.rs

@@ -2,12 +2,14 @@
 
 mod block;
 mod nb;
+mod matcher;
 mod search;
 mod walk;
 
 pub use block::{BlockIter, SearchBlock, block_iter};
 pub use nb::{CellRefs, NbCell, NbIter, NbOptions, ancestor_indices, cell_refs, heading_level, nb_iter, nb_search, nb_search_file, section_range};
-pub use search::{MatchSpan, RgIter, RgOptions, SearchError, SearchKind, SearchLine, compile_regex, rg, rg_iter, search_path, search_text, spans_for};
+pub use matcher::{RegexMatcher, compile_regex};
+pub use search::{MatchSpan, RgIter, RgOptions, SearchError, SearchKind, SearchLine, rg, rg_iter, search_path, search_text, spans_for};
 pub use walk::{FindIter, FindOptions, StreamIter, WalkOptions, find, find_iter, find_iter_with, resolve_roots};
 
 #[derive(Debug, Clone)]

rgapi-0.2.2/src/matcher.rs

@@ -0,0 +1,142 @@
+use fancy_regex::{Assertion, Expr, Regex, RegexBuilder};
+use grep_matcher::{LineTerminator, Match, Matcher, NoCaptures};
+use regex_syntax::{ast::{self, Ast, Visitor}, hir::{Class, Hir, HirKind}};
+
+use crate::RgApiError;
+
+/// A fancy-regex matcher for grep-searcher. Ordinary line-safe patterns search whole
+/// buffers; other line-oriented patterns search individual lines, without terminators.
+/// Both paths use the same regex engine and return UTF-8 byte offsets.
+#[derive(Debug, Clone)]
+pub struct RegexMatcher { regex: Regex, line_oriented: bool, whole_buffer: bool }
+
+impl RegexMatcher {
+    pub(crate) fn path(pattern: &str, smart_case: bool) -> Result<Self, RgApiError> { Self::build(pattern, None, smart_case, false, false) }
+
+    fn build(pattern: &str, case_sensitive: Option<bool>, smart_case: bool, line_oriented: bool, line_anchors: bool) -> Result<Self, RgApiError> {
+        // Parse with the same anchor defaults as the compiled regex. Fancy's resolved
+        // tree excludes group names, escape names and comments from smart-case analysis.
+        let parsed = if line_anchors { format!("(?mR){pattern}") } else { pattern.to_string() };
+        let tree = Expr::parse_tree(&parsed).map_err(regex_error)?;
+        let mut analysis = Analysis::default();
+        let mut stack = vec![&tree.expr];
+        while let Some(expr) = stack.pop() {
+            analysis.expression(expr)?;
+            stack.extend(expr.children_iter());
+        }
+        if line_oriented && analysis.literal_newline { return Err(RgApiError::new("a literal newline or carriage return is not allowed in a regex without multiline=True")); }
+        let insensitive = case_sensitive.map_or(smart_case && analysis.literal && !analysis.uppercase, |s| !s);
+        let regex = RegexBuilder::new(pattern).multi_line(line_anchors).crlf(line_anchors).case_insensitive(insensitive).build().map_err(regex_error)?;
+        Ok(Self { regex, line_oriented, whole_buffer: !line_oriented || !analysis.line_local })
+    }
+
+    pub(crate) fn for_each_match(&self, text: &str, mut matched: impl FnMut(usize, usize)) -> Result<(), fancy_regex::Error> {
+        let mut scan = |part: &str, offset| {
+            for m in self.regex.find_iter(part) { let m = m?; matched(offset + m.start(), offset + m.end()); }
+            Ok(())
+        };
+        if self.whole_buffer { return scan(text, 0); }
+        let mut offset = 0;
+        for line in text.split_inclusive('\n') {
+            let body = line.strip_suffix('\n').unwrap_or(line);
+            scan(body.strip_suffix('\r').unwrap_or(body), offset)?;
+            offset += line.len();
+        }
+        if text.is_empty() { scan(text, 0)?; }
+        Ok(())
+    }
+}
+
+/// Compile a content regex, including backreferences and lookaround.
+/// With `multiline=false`, matching is confined to individual lines and literal
+/// line endings are rejected. With `multiline=true`, the whole input is searched.
+pub fn compile_regex(pattern: &str, case_sensitive: Option<bool>, smart_case: bool, multiline: bool) -> Result<RegexMatcher, RgApiError> {
+    if pattern.is_empty() { return Err(RgApiError::new("pattern may not be empty")); }
+    RegexMatcher::build(pattern, case_sensitive, smart_case, !multiline, true)
+}
+
+fn regex_error(error: impl std::fmt::Display) -> RgApiError { RgApiError::new(error.to_string()) }
+
+impl Matcher for RegexMatcher {
+    type Captures = NoCaptures;
+    type Error = fancy_regex::Error;
+
+    fn new_captures(&self) -> Result<NoCaptures, Self::Error> { Ok(NoCaptures::new()) }
+    fn line_terminator(&self) -> Option<LineTerminator> { self.line_oriented.then(LineTerminator::crlf) }
+
+    fn find_at(&self, haystack: &[u8], mut at: usize) -> Result<Option<Match>, Self::Error> {
+        if at > haystack.len() { return Ok(None); }
+        // grep-matcher advances empty matches by one byte; fancy's VM must never
+        // start its next search in the middle of a UTF-8 character.
+        while at < haystack.len() && haystack[at] & 0xC0 == 0x80 { at += 1; }
+        if self.whole_buffer { return self.regex.find_from_pos(haystack, at).map(|m| m.map(|m| Match::new(m.start(), m.end()))); }
+        // Keep the beginning of the current line for anchors and lookbehind when
+        // grep-matcher asks for another match part-way through it.
+        let mut start = haystack[..at].iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1);
+        loop {
+            if start > 0 && start == haystack.len() { return Ok(None); }
+            let end = haystack[start..].iter().position(|&b| b == b'\n').map_or(haystack.len(), |i| start + i);
+            let line = &haystack[start..end];
+            let line = line.strip_suffix(b"\r").unwrap_or(line);
+            let pos = at.saturating_sub(start);
+            if pos <= line.len() && let Some(m) = self.regex.find_from_pos(line, pos)? { return Ok(Some(Match::new(start + m.start(), start + m.end()))); }
+            if end == haystack.len() { return Ok(None); }
+            start = end + 1;
+        }
+    }
+}
+
+#[derive(Default)]
+struct Analysis { literal: bool, uppercase: bool, literal_newline: bool, line_local: bool }
+
+impl Analysis {
+    fn literal(&mut self, c: char) { self.literal = true; self.uppercase |= c.is_uppercase(); }
+
+    fn expression(&mut self, expr: &Expr) -> Result<(), RgApiError> {
+        match expr {
+            Expr::Literal { val, .. } => {
+                for c in val.chars() { self.literal(c); }
+                self.literal_newline |= val.contains(['\r', '\n']);
+            }
+            Expr::Delegate { inner, .. } => {
+                let ast = ast::parse::Parser::new().parse(inner).map_err(regex_error)?;
+                let literals = ast::visit(&ast, ClassLiterals::default()).unwrap();
+                self.literal |= literals.literal;
+                self.uppercase |= literals.uppercase;
+                let hir = regex_syntax::Parser::new().parse(inner).map_err(regex_error)?;
+                self.line_local |= contains_newline(&hir);
+                if let HirKind::Literal(lit) = hir.kind() { self.literal_newline |= lit.0.iter().any(|b| matches!(b, b'\r' | b'\n')); }
+            }
+            Expr::Any { newline, crlf } => self.line_local |= *newline || !crlf,
+            Expr::Assertion(Assertion::StartText | Assertion::EndText | Assertion::EndTextIgnoreTrailingNewlines { .. }) => self.line_local = true,
+            Expr::Empty | Expr::Assertion(_) | Expr::Concat(_) | Expr::Alt(_) | Expr::Group(_) | Expr::Repeat { .. } => {}
+            // Backreferences, lookaround and other fancy constructs use line-local
+            // input so captures and assertions cannot inspect neighbouring lines.
+            _ => self.line_local = true,
+        }
+        Ok(())
+    }
+}
+
+fn contains_newline(hir: &Hir) -> bool {
+    match hir.kind() {
+        HirKind::Literal(lit) => lit.0.iter().any(|b| matches!(b, b'\r' | b'\n')),
+        HirKind::Class(Class::Unicode(c)) => c.ranges().iter().any(|r| ['\r', '\n'].iter().any(|&c| r.start() <= c && c <= r.end())),
+        HirKind::Class(Class::Bytes(c)) => c.ranges().iter().any(|r| [b'\r', b'\n'].iter().any(|&c| r.start() <= c && c <= r.end())),
+        _ => hir.kind().subs().iter().any(contains_newline),
+    }
+}
+
+#[derive(Default)]
+struct ClassLiterals { literal: bool, uppercase: bool }
+impl ClassLiterals { fn add(&mut self, c: char) { self.literal = true; self.uppercase |= c.is_uppercase(); } }
+impl Visitor for ClassLiterals {
+    type Output = Self;
+    type Err = std::convert::Infallible;
+    fn finish(self) -> Result<Self, Self::Err> { Ok(self) }
+    fn visit_pre(&mut self, ast: &Ast) -> Result<(), Self::Err> { if let Ast::Literal(lit) = ast { self.add(lit.c); } Ok(()) }
+    fn visit_class_set_item_pre(&mut self, item: &ast::ClassSetItem) -> Result<(), Self::Err> {
+        match item { ast::ClassSetItem::Literal(lit) => self.add(lit.c), ast::ClassSetItem::Range(r) => { self.add(r.start.c); self.add(r.end.c); } _ => {} }
+        Ok(())
+    }
+}

{rgapi-0.2.1 → rgapi-0.2.2}/src/nb.rs

@@ -3,21 +3,22 @@ use std::path::Path;
 use std::sync::atomic::Ordering;
 use std::sync::{Arc, LazyLock};
 
-use grep_matcher::{Captures, Matcher};
-use grep_regex::RegexMatcher;
+use fancy_regex::Regex;
+use crate::RegexMatcher;
 use ignore::{DirEntry, WalkState};
 use serde::Deserialize;
 
 use crate::RgApiError;
-use crate::search::{SearchLine, compile_regex, search_text};
+use crate::search::{SearchLine, search_text};
+use crate::compile_regex;
 use crate::walk::{PathFilters, StreamIter, WalkOptions, entry_err, rel_path, resolve_roots, spawn_walk};
 
-static HEADING_RE: LazyLock<RegexMatcher> = LazyLock::new(|| RegexMatcher::new(r"^#{1,6} \w").unwrap());
+static HEADING_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^#{1,6} \w").unwrap());
 
 /// Heading level of the first nonblank line after skipping `#|` directives; zero unless it matches `^#{1,6} \w`.
 pub fn heading_level(source: &str) -> usize {
     let line = source.lines().find(|l| !l.trim().is_empty() && !l.starts_with("#|")).unwrap_or("");
-    if !HEADING_RE.is_match(line.as_bytes()).unwrap_or(false) { return 0; }
+    if !HEADING_RE.is_match(line).unwrap_or(false) { return 0; }
     line.bytes().take_while(|&b| b == b'#').count()
 }
 
@@ -44,12 +45,8 @@ pub fn ancestor_indices(levels: &[usize], idx: usize) -> Vec<usize> {
 
 /// Group 1 of every `` sigil`body` `` match in `text`, in order of appearance. `sigil` and `body` are regex fragments.
 fn sigil_caps(text: &str, sigil: &str, body: &str) -> Vec<String> {
-    let re = RegexMatcher::new(&format!("{sigil}`({body})`")).expect("sigil pattern compiles");
-    let mut caps = re.new_captures().expect("captures allocate");
-    let mut res = Vec::new();
-    re.captures_iter(text.as_bytes(), &mut caps, |c| { if let Some(m) = c.get(1) { res.push(text[m.start()..m.end()].to_string()); } true })
-    .expect("regex search is infallible");
-    res
+    let re = Regex::new(&format!("{sigil}`({body})`")).expect("sigil pattern compiles");
+    re.captures_iter(text).map(|c| c.expect("internal regex search succeeds").get(1).unwrap().as_str().to_string()).collect()
 }
 
 /// Names written as `` &`name` `` or `` &`[a, b]` `` in `text`. A name holds word characters and dots.
@@ -175,14 +172,6 @@ fn process_file(disp: String, bytes: &[u8], matcher: &RegexMatcher, cell_context
     Ok(out)
 }
 
-fn compile_nb_regex(pattern: &str, case_sensitive: Option<bool>, smart_case: bool, multiline: bool) -> Result<RegexMatcher, RgApiError> {
-    compile_regex(pattern, case_sensitive, smart_case, multiline).map_err(|e| {
-        if !multiline && e.to_string().contains("not allowed in a regex") {
-            RgApiError::new(format!("{e}; pass multiline=True to let the pattern match across lines within a cell"))
-        } else { e }
-    })
-}
-
 pub fn nb_search_file(
     path: &Path,
     display_path: String,
@@ -192,7 +181,7 @@ pub fn nb_search_file(
     cell_context: usize,
     multiline: bool,
 ) -> Result<Vec<NbCell>, RgApiError> {
-    let matcher = compile_nb_regex(pattern, case_sensitive, smart_case, multiline)?;
+    let matcher = compile_regex(pattern, case_sensitive, smart_case, multiline)?;
     let bytes = match std::fs::read(path) { Ok(b) => b, Err(_) => return Ok(Vec::new()) };
     process_file(display_path, &bytes, &matcher, cell_context, multiline)
 }
@@ -211,7 +200,7 @@ fn nb_entry(
     let Some(ft) = dent.file_type() else { return Ok(Vec::new()); };
     if !ft.is_file() { return Ok(Vec::new()); }
     let rel = rel_path(base, path);
-    if !filters.path_allowed(Path::new(&rel)) { return Ok(Vec::new()); }
+    if !filters.path_allowed(Path::new(&rel))? { return Ok(Vec::new()); }
     let bytes = match std::fs::read(path) { Ok(b) => b, Err(_) => return Ok(Vec::new()) };
     process_file(rel, &bytes, matcher, cell_context, multiline)
 }
@@ -221,7 +210,7 @@ pub type NbIter = StreamIter<NbCell>;
 pub fn nb_iter(opts: &NbOptions) -> Result<NbIter, RgApiError> {
     let (roots, base) = resolve_roots(&opts.walk, true, false)?;
     let filters = Arc::new(PathFilters::new(&opts.walk)?);
-    let matcher = compile_nb_regex(&opts.pattern, opts.case_sensitive, opts.smart_case, opts.multiline)?;
+    let matcher = compile_regex(&opts.pattern, opts.case_sensitive, opts.smart_case, opts.multiline)?;
     let (cell_context, multiline, max_depth) = (opts.cell_context, opts.multiline, opts.walk.max_depth);
     Ok(spawn_walk(roots, base, &opts.walk, filters, Vec::new(), move |dent, base, filters, tx, cancel| {
         match nb_entry(dent, base, filters, &matcher, cell_context, multiline, max_depth) {

{rgapi-0.2.1 → rgapi-0.2.2}/src/search.rs

@@ -5,12 +5,12 @@ use std::sync::{
     mpsc::SyncSender,
 };
 
-use grep_matcher::{LineTerminator, Matcher};
-use grep_regex::{RegexMatcher, RegexMatcherBuilder};
+use grep_matcher::LineTerminator;
+use crate::matcher::RegexMatcher;
 use grep_searcher::{BinaryDetection, SearcherBuilder, Sink, SinkContext, SinkContextKind, SinkError, SinkMatch};
 use ignore::{DirEntry, WalkState};
 
-use crate::RgApiError;
+use crate::{RgApiError, compile_regex};
 use crate::walk::{PathFilters, StreamIter, WalkOptions, entry_err, rel_path, resolve_roots, spawn_walk};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -84,7 +84,7 @@ fn search_entry(
     let Some(ft) = dent.file_type() else { return WalkState::Continue; };
     if !ft.is_file() { return WalkState::Continue; }
     let rel = rel_path(base, path);
-    if !filters.path_allowed(Path::new(&rel)) { return WalkState::Continue; }
+    match filters.path_allowed(Path::new(&rel)) { Ok(true) => {}, Ok(false) => return WalkState::Continue, Err(err) => return send_search_error(tx, err) }
     match search_path_cancelable(path, rel, matcher.clone(), before_context, after_context, Some(cancel.clone())) {
         Ok(lines) => {
             if is_cancelled(cancel) { return WalkState::Quit; }
@@ -99,27 +99,6 @@ fn is_cancelled(cancel: &Arc<AtomicBool>) -> bool { cancel.load(Ordering::Relaxe
 
 fn send_search_error(tx: &SyncSender<Result<SearchLine, RgApiError>>, err: RgApiError) -> WalkState { let _ = tx.send(Err(err)); WalkState::Quit }
 
-pub fn compile_regex(pattern: &str, case_sensitive: Option<bool>, smart_case: bool, multiline: bool) -> Result<RegexMatcher, RgApiError> {
-    if pattern.is_empty() { return Err(RgApiError::new("pattern may not be empty")); }
-    let mut builder = RegexMatcherBuilder::new();
-    builder.multi_line(true).crlf(true);
-    if multiline { builder.line_terminator(None); }
-    match case_sensitive {
-        Some(true) => {
-            builder.case_insensitive(false);
-            builder.case_smart(false);
-        }
-        Some(false) => {
-            builder.case_insensitive(true);
-            builder.case_smart(false);
-        }
-        None => {
-            builder.case_smart(smart_case);
-        }
-    }
-    builder.build(pattern).map_err(|e| RgApiError::new(e.to_string()))
-}
-
 pub(crate) fn format_lnhash(lineno: u64, line: &str) -> String {
     const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
     let hash = crc32fast::hash(line.as_bytes()) as usize;
@@ -244,8 +223,7 @@ fn bytes_to_line(bytes: &[u8]) -> Result<String, SearchError> {
 /// Return byte offsets for every match in a line.
 pub fn spans_for(matcher: &RegexMatcher, bytes: &[u8]) -> Result<Vec<MatchSpan>, SearchError> {
     let mut spans = Vec::new();
-    matcher
-        .find_iter(bytes, |m| { spans.push(MatchSpan { start: m.start(), end: m.end() }); true })
-        .map_err(|e| SearchError::Message(e.to_string()))?;
+    let text = std::str::from_utf8(bytes).map_err(|_| SearchError::InvalidUtf8)?;
+    matcher.for_each_match(text, |start, end| spans.push(MatchSpan { start, end })).map_err(|e| SearchError::Message(e.to_string()))?;
     Ok(spans)
 }

{rgapi-0.2.1 → rgapi-0.2.2}/src/walk.rs

@@ -6,10 +6,9 @@ use std::sync::{Arc, Mutex, mpsc};
 
 use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
 use grep_matcher::Matcher;
-use grep_regex::{RegexMatcher, RegexMatcherBuilder};
 use ignore::{DirEntry, WalkBuilder, WalkState};
 
-use crate::RgApiError;
+use crate::{RegexMatcher, RgApiError};
 
 /// Where to walk and which paths to keep. `FindOptions`, `RgOptions` and `NbOptions` each hold one.
 #[derive(Debug, Clone)]
@@ -83,10 +82,13 @@ pub fn find_iter_with(opts: &FindOptions, walk_root_links: bool) -> Result<FindI
     let walk = &opts.walk;
     let (roots, base) = resolve_roots(walk, false, walk_root_links)?;
     let filters = Arc::new(PathFilters::new(walk)?);
-    let pattern = opts.pattern.as_deref().map(build_fd_re).transpose()?;
+    let pattern = opts.pattern.as_deref().map(|p| RegexMatcher::path(p, true)).transpose()?;
     // The walker follows every root it is given. A link root returned as itself must not reach it.
     let (links, roots): (Vec<_>, Vec<_>) = roots.into_iter().partition(|r| r.is_symlink() && !walk.follow_links && !(walk_root_links && r.is_dir()));
-    let ready = if walk.min_depth.unwrap_or(0) > 0 { Vec::new() } else { links.iter().map(|r| relative_path(&base, r)).filter(|rel| find_matches(rel, &filters, pattern.as_ref())).map(Path::to_path_buf).collect() };
+    let mut ready = Vec::new();
+    if walk.min_depth.unwrap_or(0) == 0 {
+        for link in links { let rel = relative_path(&base, &link); if find_matches(rel, &filters, pattern.as_ref())? { ready.push(rel.to_path_buf()); } }
+    }
     let (files, dirs, special_files, panic_probe, max_depth) = (opts.files, opts.dirs, opts.special_files, opts.panic_probe, walk.max_depth);
     Ok(spawn_walk(roots, base, walk, filters, ready, move |dent, base, filters, tx, cancel| {
         if panic_probe { panic!("rgapi: deliberate panic for tests (panic_probe)"); }
@@ -196,7 +198,7 @@ where
             if worker_cancel.load(Ordering::Relaxed) { return; }
             let mut walker = WalkBuilder::new(root);
             configure_walker(&mut walker, root, &walk);
-            filter_dirs(&mut walker, &base, filters.clone());
+            filter_dirs(&mut walker, &base, filters.clone(), tx.clone(), worker_cancel.clone());
             walker.build_parallel().run(|| {
                 let (tx, base, filters, cancel, entry, seen) = (tx.clone(), base.clone(), filters.clone(), worker_cancel.clone(), entry.clone(), seen.clone());
                 Box::new(move |dent| {
@@ -230,7 +232,7 @@ fn find_entry(
         Err(err) => {
             if let Some(path) = dangling_link(&err) {
                 let rel = relative_path(base, path);
-                return Ok(find_matches(rel, filters, pattern).then(|| rel.to_path_buf()));
+                return Ok(find_matches(rel, filters, pattern)?.then(|| rel.to_path_buf()));
             }
             return entry_err(err, max_depth).map_or(Ok(None), Err);
         }
@@ -242,7 +244,7 @@ fn find_entry(
     if ft.is_dir() && !dirs { return Ok(None); }
     if !ft.is_file() && !ft.is_dir() && !ft.is_symlink() && !special_files { return Ok(None); }
     let rel = relative_path(base, path);
-    Ok(find_matches(rel, filters, pattern).then(|| rel.to_path_buf()))
+    Ok(find_matches(rel, filters, pattern)?.then(|| rel.to_path_buf()))
 }
 
 // With `follow_links` the walker reports a dangling link as an error; it is a dangling link when the path is a symlink whose target is missing.
@@ -254,8 +256,8 @@ fn dangling_link(err: &ignore::Error) -> Option<&Path> {
     }
 }
 
-fn find_matches(path: &Path, filters: &PathFilters, pattern: Option<&RegexMatcher>) -> bool {
-    if let Some(pattern) = pattern { if !re_match(pattern, &path.file_name().unwrap_or_default().to_string_lossy()) { return false; } }
+fn find_matches(path: &Path, filters: &PathFilters, pattern: Option<&RegexMatcher>) -> Result<bool, RgApiError> {
+    if let Some(pattern) = pattern { if !re_match(pattern, &path.file_name().unwrap_or_default().to_string_lossy())? { return Ok(false); } }
     filters.path_allowed(path)
 }
 
@@ -318,9 +320,12 @@ fn configure_walker(walker: &mut WalkBuilder, root: &Path, walk: &WalkOptions) {
     walker.same_file_system(walk.same_file_system);
 }
 
-fn filter_dirs(walker: &mut WalkBuilder, base: &Path, filters: Arc<PathFilters>) {
+fn filter_dirs<T: Send + 'static>(walker: &mut WalkBuilder, base: &Path, filters: Arc<PathFilters>, tx: mpsc::SyncSender<Result<T, RgApiError>>, cancel: Arc<AtomicBool>) {
     let base = base.to_path_buf();
-    walker.filter_entry(move |entry| filters.entry_allowed(&base, entry));
+    walker.filter_entry(move |entry| if cancel.load(Ordering::Relaxed) { false } else { match filters.entry_allowed(&base, entry) {
+        Ok(allowed) => allowed,
+        Err(err) => { let _ = tx.send(Err(err)); false }
+    } });
 }
 
 pub(crate) struct PathFilters {
@@ -346,37 +351,37 @@ impl PathFilters {
         })
     }
 
-    pub(crate) fn path_allowed(&self, path: &Path) -> bool {
+    pub(crate) fn path_allowed(&self, path: &Path) -> Result<bool, RgApiError> {
         if let Some(excludes) = &self.excludes
             && excludes.is_match(path)
-        { return false; }
+        { return Ok(false); }
         if let Some(skip_path_re) = &self.skip_path_re
-            && re_match(skip_path_re, &path_label(path))
-        { return false; }
+            && re_match(skip_path_re, &path_label(path))?
+        { return Ok(false); }
         if let Some(path_re) = &self.path_re
-            && !re_match(path_re, &path_label(path))
-        { return false; }
+            && !re_match(path_re, &path_label(path))?
+        { return Ok(false); }
         if let Some(exts) = &self.exts
             && !exts.is_match(path)
-        { return false; }
-        if let Some(includes) = &self.includes { return includes.is_match(path); }
-        true
+        { return Ok(false); }
+        if let Some(includes) = &self.includes { return Ok(includes.is_match(path)); }
+        Ok(true)
     }
 
-    fn entry_allowed(&self, base: &Path, dent: &DirEntry) -> bool {
+    fn entry_allowed(&self, base: &Path, dent: &DirEntry) -> Result<bool, RgApiError> {
         let path = dent.path();
-        if dent.depth() == 0 { return true; }
-        let Some(ft) = dent.file_type() else { return true; };
-        if !ft.is_dir() { return true; }
+        if dent.depth() == 0 { return Ok(true); }
+        let Some(ft) = dent.file_type() else { return Ok(true); };
+        if !ft.is_dir() { return Ok(true); }
         let rel = relative_path(base, path);
-        if self.excludes.as_ref().is_some_and(|globs| globs.is_match(rel)) { return false; }
+        if self.excludes.as_ref().is_some_and(|globs| globs.is_match(rel)) { return Ok(false); }
         if let Some(skip_dirs) = &self.skip_dirs
             && skip_dirs.is_match(rel)
-        { return false; }
+        { return Ok(false); }
         if let Some(skip_dir_re) = &self.skip_dir_re
-            && re_match(skip_dir_re, &path_label(rel))
-        { return false; }
-        true
+            && re_match(skip_dir_re, &path_label(rel))?
+        { return Ok(false); }
+        Ok(true)
     }
 }
 
@@ -393,17 +398,11 @@ fn add_glob(builder: &mut GlobSetBuilder, glob: &str) -> Result<(), RgApiError>
     Ok(())
 }
 
-fn build_fd_re(pattern: &str) -> Result<RegexMatcher, RgApiError> {
-    let mut builder = RegexMatcherBuilder::new();
-    builder.case_smart(true);
-    builder.build(pattern).map_err(|e| RgApiError::new(e.to_string()))
-}
-
 fn build_path_re(pattern: Option<&str>) -> Result<Option<RegexMatcher>, RgApiError> {
-    pattern.map(|pattern| RegexMatcherBuilder::new().build(pattern).map_err(|e| RgApiError::new(e.to_string()))).transpose()
+    pattern.map(|pattern| RegexMatcher::path(pattern, false)).transpose()
 }
 
-fn re_match(matcher: &RegexMatcher, rel: &str) -> bool { matcher.is_match(rel.as_bytes()).unwrap_or(false) }
+fn re_match(matcher: &RegexMatcher, rel: &str) -> Result<bool, RgApiError> { matcher.is_match(rel.as_bytes()).map_err(|e| RgApiError::new(e.to_string())) }
 
 #[cfg(test)]
 mod tests {

{rgapi-0.2.1 → rgapi-0.2.2}/tests/test_async.py

@@ -72,6 +72,16 @@ def test_fda_iter(tmp_path):
     assert run_sync(first_then_close()) in fd(tmp_path)
 
 
+def test_async_pruning_regex_errors(tmp_path):
+    sub = tmp_path/("ab"*26)
+    sub.mkdir()
+    (sub/"a.txt").write_text("TODO")
+    pattern = r"(a|b|ab)*(?>c)"
+    with pytest.raises(ValueError, match="backtrack"): run_sync(fda(tmp_path, skip_dir_re=pattern))
+    async def rows(): return [r async for r in rga_iter("TODO", tmp_path, skip_dir_re=pattern)]
+    with pytest.raises(ValueError, match="backtrack"): run_sync(rows())
+
+
 def test_nbrga_paths(tmp_path):
     from rgapi import nbrg, nbrga
     from test_rgapi import _cell, write_nb

{rgapi-0.2.1 → rgapi-0.2.2}/tests/test_rgapi.py

@@ -290,12 +290,14 @@ def test_worker_panic_surfaces_as_error_not_truncation(tmp_path):
     with pytest.raises(Exception): _core.panic_probe(str(tmp_path), walk=True)  # walk workers
 
 
-def test_search_path_skips_binary_and_invalid_utf8(tmp_path):
+@pytest.mark.parametrize("pattern", ["TODO", r"(TO)D(?!X)O"])
+def test_search_path_skips_binary_and_invalid_utf8(tmp_path, pattern):
     (tmp_path / "bin.dat").write_bytes(b"TODO before\n\0TODO after\n")
-    (tmp_path / "bad.txt").write_bytes(b"TODO\xff\n")
-    matcher = compile("TODO")
+    matcher = compile(pattern)
     assert search_path(matcher, tmp_path / "bin.dat") == []
-    assert search_path(matcher, tmp_path / "bad.txt") == []
+    for data in [b"TODO\xff\n", b"\xffTODO\n"]:
+        (tmp_path / "bad.txt").write_bytes(data)
+        assert search_path(matcher, tmp_path / "bad.txt") == []
 
 
 def test_rg_keyboard_interrupt_cancels(tmp_path):
@@ -345,6 +347,56 @@ def test_rgstr():
     assert str(res[0]) == "1-zero"
 
 
+def test_backreferences_and_lookaround(tmp_path):
+    pattern = r"\b(\w+)\s+\1\b"
+    text = "before\né é, foo foo\nfoo\nfoo\nafter\n"
+    p = tmp_path/"echo_echo.txt"
+    p.write_text(text)
+    matcher = compile(pattern)
+    assert matcher.finditer(text) == [(7, 12), (14, 21)]
+    assert [(r.line_number, r.matches) for r in search_text(matcher, text)] == [(2, [(0, 5), (7, 14)])]
+    assert [(r.line_number, r.matches) for r in search_path(matcher, p)] == [(2, [(0, 5), (7, 14)])]
+    assert rg(pattern, p, count=True) == 2
+    assert [b.source for b in rg(pattern, p, summary=True)] == [text.rstrip()]
+    assert compile(r"(?<=é )é(?=,)").finditer(text) == [(10, 12)]
+    assert compile(r"(?=.)").finditer("éλ") == [(0, 0), (2, 2)]
+    assert [r.matches for r in rgstr(r"(?=$)", "é\n")] == [[(2, 2)]]
+    assert [p.name for p in fd(tmp_path, r"^(\w+)_\1\.txt$")] == ["echo_echo.txt"]
+    assert fd(tmp_path, path_re=r"(?<=echo_)echo") == [p]
+    assert fd(tmp_path, skip_path_re=r"(?<=echo_)echo") == []
+    sub = tmp_path/"skip_skip"
+    sub.mkdir()
+    (sub/"a.txt").write_text("foo foo")
+    assert fd(tmp_path, skip_dir_re=r"^(\w+)_\1$") == [p]
+
+
+def test_line_local_regexes_and_smart_case():
+    for pattern in [r"foo\s+foo", r"foo[^x]+foo", r"(?s:foo.+foo)", r"foo(?=\s+foo)"]:
+        assert rgstr(pattern, "foo\nfoo\n") == []
+    assert [r.matches for r in rgstr(r"(?s:foo.*|bar)", "foo\nbar\n")] == [[(0, 3)], [(0, 3)]]
+    assert compile(r"\Afoo\z").finditer("foo\nfoo\n") == [(0, 3), (4, 7)]
+    empty = compile(r"\A\z")
+    for text, spans in [("foo\n", []), ("", [(0, 0)]), ("\n", [(0, 0)]), ("foo\n\n", [(4, 4)])]:
+        assert empty.finditer(text) == spans
+        assert empty.is_match(text) == bool(spans)
+    assert compile(r"(?P<UPPER>foo)\s+(?P=UPPER)", smart_case=True).is_match("FOO foo")
+    assert compile(r"foo\S", smart_case=True).is_match("FOOx")
+    assert compile("(?x)foo # UPPER comment", smart_case=True).is_match("FOO")
+    assert not compile(r"foo[A-Z]", smart_case=True).is_match("FOOX")
+    for pattern in [r"foo\nbar", r"foo\x0Dbar"]:
+        with pytest.raises(ValueError, match="multiline=True"): compile(pattern)
+
+
+def test_regex_runtime_errors(tmp_path):
+    pattern, text = r"(a|b|ab)*(?>c)", "ab"*26
+    p = tmp_path/text
+    p.mkdir()
+    (tmp_path/(text+".txt")).write_text(text)
+    for search in [lambda: compile(pattern).is_match(text), lambda: rgstr(pattern, text),
+                   lambda: fd(tmp_path, pattern), lambda: fd(tmp_path, path_re=pattern), lambda: fd(tmp_path, skip_dir_re=pattern)]:
+        with pytest.raises(ValueError, match="backtrack"): search()
+
+
 def write_nb(path, cells):
     nb = dict(cells=cells, metadata={}, nbformat=4, nbformat_minor=5)
     path.write_text(json.dumps(nb))
@@ -358,6 +410,15 @@ def _cell(cell_type, source, cid=None, outputs=None):
     return c
 
 
+def test_notebook_backreferences(tmp_path):
+    from rgapi import nbrg
+    p = tmp_path/"echo.ipynb"
+    write_nb(p, [_cell("code", "é é\nfoo\nfoo\n", cid="c1")])
+    assert nbrg(r"(\w+) \1", p)[0].matches[0].matches == [(0, 5)]
+    assert nbrg(r"(foo)\s+\1", p) == []
+    assert nbrg(r"(foo)\s+\1", p, multiline=True)[0].matches[0].matches == [(0, 7)]
+
+
 def test_nbrg_source_only_with_cells(tmp_path):
     from rgapi import nbrg
     write_nb(tmp_path / "nb.ipynb", [

{rgapi-0.2.1 → rgapi-0.2.2}/tools/bench.py

@@ -6,7 +6,7 @@ rgapi's in-process search. Import and fixture creation time are outside the time
 sections.
 """
 
-import shutil, subprocess, tempfile, timeit
+import argparse, shutil, subprocess, tempfile, timeit
 from pathlib import Path
 
 from rgapi import rg as rgapi_rg
@@ -48,49 +48,48 @@ def make_tiny_dir(root):
     return d
 
 
-def rg_cli(root):
-    cmd = ["rg", "--color", "never", "--no-heading", "--line-number", PATTERN, str(root)]
+def rg_cli(root, pattern):
+    cmd = ["rg", "--color", "never", "--no-heading", "--line-number", pattern, str(root)]
     return subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True).stdout
 
 
-def rgapi(root): return rgapi_rg(PATTERN, str(root))
+def rgapi(root, pattern): return rgapi_rg(pattern, str(root))
 
 
-def bench_one(name, func, root, number=1):
-    timer = timeit.Timer(lambda: func(root))
+def bench_one(name, func, root, pattern, number=1):
+    timer = timeit.Timer(lambda: func(root, pattern))
     times = timer.repeat(repeat=REPEATS, number=number)
     best = min(times) / number
     avg = sum(times) / len(times) / number
     print(f"{name:24} best {best * 1000:8.2f} ms   avg {avg * 1000:8.2f} ms")
 
 
-def bench(root):
+def bench(root, pattern):
     large = make_large_dir(root)
     many_small = make_many_small_dir(root)
     tiny = make_tiny_dir(root)
 
     # Warm imports, dynamic libraries, and disk caches before timing.
-    rgapi(large)
-    rg_cli(large)
+    rgapi(large, pattern)
+    rg_cli(large, pattern)
 
     print(f"fixture: {root}")
+    print(f"pattern: {pattern}")
     print(f"large files: {LARGE_FILES} x {LARGE_FILE_BYTES:,} bytes")
     print(f"many small files: {SMALL_FILES} x {SMALL_FILE_BYTES:,} bytes")
     print(f"repeats: {REPEATS}\n")
 
-    bench_one("rg large", rg_cli, large)
-    bench_one("rgapi large", rgapi, large)
-    print()
-    bench_one("rg many-small", rg_cli, many_small)
-    bench_one("rgapi many-small", rgapi, many_small)
-    print()
-    bench_one("rg tiny x30", rg_cli, tiny, number=SMALL_REPEATS_PER_TIMING)
-    bench_one("rgapi tiny x30", rgapi, tiny, number=SMALL_REPEATS_PER_TIMING)
+    for label, fixture, number in [("large", large, 1), ("many-small", many_small, 1), ("tiny x30", tiny, SMALL_REPEATS_PER_TIMING)]:
+        for name, func in [("rg", rg_cli), ("rgapi", rgapi)]: bench_one(f"{name} {label}", func, fixture, pattern, number)
+        print()
 
 
 def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--pattern", default=PATTERN, help="Regex to search; fixtures contain 'needle_rgapi_bench only here'")
+    args = parser.parse_args()
     if shutil.which("rg") is None: raise SystemExit("rg executable not found")
-    with tempfile.TemporaryDirectory(prefix="rgapi-bench-") as d: bench(Path(d))
+    with tempfile.TemporaryDirectory(prefix="rgapi-bench-") as d: bench(Path(d), args.pattern)
 
 
 if __name__ == "__main__": main()