rgapi 0.1.24__tar.gz → 0.1.26__tar.gz

{rgapi-0.1.24 → rgapi-0.1.26}/Cargo.lock

@@ -28,11 +28,17 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
+[[package]]
+name = "core_detect"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
+
 [[package]]
 name = "crc32fast"
-version = "1.5.1"
+version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
 dependencies = [
  "cfg-if",
 ]
@@ -64,11 +70,17 @@ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.35"
+version = "0.8.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a"
 dependencies = [
  "cfg-if",
+ "core_detect",
+ "multiversion",
+ "multiversion_no_op",
+ "rustversion",
+ "scopeguard",
+ "simdutf8",
 ]
 
 [[package]]
@@ -185,6 +197,33 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "multiversion"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c"
+dependencies = [
+ "multiversion-macros",
+]
+
+[[package]]
+name = "multiversion-macros"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "multiversion_no_op"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
+
 [[package]]
 name = "once_cell"
 version = "1.21.4"
@@ -291,7 +330,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "rgapi"
-version = "0.1.24"
+version = "0.1.26"
 dependencies = [
  "crc32fast",
  "globset",
@@ -304,6 +343,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -313,6 +358,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "serde"
 version = "1.0.229"
@@ -356,6 +407,12 @@ dependencies = [
  "zmij",
 ]
 
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
 [[package]]
 name = "syn"
 version = "2.0.119"

{rgapi-0.1.24 → rgapi-0.1.26}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rgapi"
-version = "0.1.24"
+version = "0.1.26"
 edition = "2024"
 rust-version = "1.91"
 license = "Apache-2.0"

{rgapi-0.1.24 → rgapi-0.1.26}/DEV.md

@@ -48,4 +48,8 @@ Truncation is recorded on collected results: `max_results` sets `stop_reason="ma
 
 Path results are `FileEntry` rows: a `str` subclass carrying the walk root, so paths stay plain strings for compatibility while stat info loads lazily (one cached `os.lstat` per entry, read only on attribute access). The wrapping happens at result construction on the Python side; Rust still streams plain strings. `PathResults.__repr__` shows an `ls -l`-style listing capped at `MAX_REPR` rows, so a huge result never stats everything, while `str()` stays one plain path per line. `ls` is `fd` with shell-style defaults (one level, dirs, ignore rules off), re-sorted with `stop_reason` preserved.
 
+Rust callers can consume a `StreamIter` with `cancel_and_join()` to cancel, drain queued results, and wait for the walk's workers to finish. `Drop` remains nonblocking. Since filesystem calls already in progress must return before joining completes, keep `cancel_and_join()` off async executors.
+
+Notebook hierarchy uses `heading_level(source)`, `section_range(levels, idx)` and `ancestor_indices(levels, idx)`. Callers supply zero for non-heading cells. Heading detection skips blank lines and lines starting with `#|`, then checks the first remaining line against `^#{1,6} \w`. It does not search past ordinary text. Section ranges include the addressed cell and end before the next equal-or-higher heading; non-headings select themselves. Ancestors exclude the addressed cell and are returned outermost first. These are calculations over the supplied levels, without retained outline state. Rustygate uses them for its cell selectors.
+
 This package intentionally has no CLI. Python is the interface.

rgapi-0.1.26/PKG-INFO

@@ -0,0 +1,255 @@
+Metadata-Version: 2.4
+Name: rgapi
+Version: 0.1.26
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Requires-Dist: fastcore>=1.14.6
+Requires-Dist: fastship>=0.0.12 ; extra == 'dev'
+Requires-Dist: maturin>=1.0,<2.0 ; extra == 'dev'
+Requires-Dist: pytest ; extra == 'dev'
+Provides-Extra: dev
+License-File: LICENSE
+Summary: Python API for ripgrep-style file walking and searching
+Home-Page: https://github.com/AnswerDotAI/rgapi
+Author-email: Jeremy Howard <j@fast.ai>
+License: Apache-2.0
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/AnswerDotAI/rgapi
+Project-URL: Issues, https://github.com/AnswerDotAI/rgapi/issues
+Project-URL: Repository, https://github.com/AnswerDotAI/rgapi
+
+# rgapi
+
+`rgapi` provides `fd`-style file discovery and `rg`-style text search from Python without starting a shell command.
+
+It uses the same `ignore`, `grep-regex`, and `grep-searcher` crates that ripgrep uses for walking, regex matching, and file scanning. Walking and searching run in parallel by default. Most expensive work stays in Rust.
+
+## Overview
+
+For common file discovery and search:
+
+```python
+from rgapi import fd, ls, rg, rg_iter
+
+fd(".", ext="py", exclude="test_*.py")
+ls("src")
+for row in rg_iter("TODO", ".", include="*.py", context=2): print(row.asdict())
+rg("TODO", ".", ext="py", skip_dir=".venv", paths=True)
+```
+
+For cell-aware search of Jupyter notebooks (see [Notebooks](#notebooks)):
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".", cell_context=1)
+```
+
+Walk and search functions have async versions. Streaming versions yield results as the search finds them. See [Async](#async):
+
+```python
+from rgapi import fda, rga, rga_iter, nbrga, nbrga_iter
+
+await rga("TODO", ".", ext="py", timeout_ms=200)
+async for row in rga_iter("TODO", "."): print(row)
+```
+
+Use the lower-level functions to compile a regex, search text or a single file, or walk a directory:
+
+```python
+from rgapi import compile, search_path, search_text, walk
+
+matcher = compile("TODO")
+matcher.is_match("TODO")
+matcher.finditer("TODO TODO")
+
+walk(".")
+search_text(matcher, "alpha\nTODO\nomega\n", path="memory.txt", context=1)
+search_path(matcher, "src/lib.rs", display_path="src/lib.rs")
+```
+
+## Install
+
+```bash
+pip install rgapi
+```
+
+## File discovery
+
+`fd` and `walk` return slash-separated paths relative to `root`. Pass `root` as a `str` or `pathlib.Path`. The sync and async APIs expand `~` and accept `.`, `./`, and paths containing `..`.
+
+Discovery uses the `ignore` crate with ripgrep's default filters. It reads `.gitignore`, `.ignore`, and `.rgignore` files. `.rgignore` takes precedence over `.gitignore`. Pass `ignore=False` to disable all ignore-file filtering, including `.rgignore`.
+
+Hidden files are skipped unless `hidden=True`. Symlinks are followed only with `follow_links=True`. Use `same_file_system=True` to avoid crossing filesystem boundaries.
+
+Traversal runs in parallel without guaranteed result order. Use `sorted(...)` when order matters.
+
+`fd` adds filename filters to `walk`. Its `pattern` is a smart-case regex matched against each basename. Lowercase patterns match case-insensitively. A pattern containing uppercase letters is case-sensitive. Use `path_re` to match the slash-separated relative path instead.
+
+`include` and `exclude` use glob syntax. `glob=` is an alias for `include=`. A basename glob such as `*.py` also matches nested paths such as `src/app.py`.
+
+Filter extensions with `ext="py"` or `ext=["py", "rs"]`. Extension and glob filters must both match. For example, `include="src/*", ext="py"` requires `src/*` and `*.py`, like combining `rg -g` with `-t`.
+
+Set `min_depth` and `max_depth` to bound recursion. `max_filesize` skips files above a byte limit.
+
+`ls` follows the shell command's listing conventions. It uses `fd` with `max_depth=1`, includes directories, disables ignore rules, and sorts by name. Set `hidden=True` for `ls -a` behaviour. All `fd` filters remain available.
+
+`fd_iter` yields `FileEntry` paths as the walk finds them. It accepts every `fd` filter. Stopping iteration ends the walk. It does not accept `timeout_ms`.
+
+`path_re` and `skip_path_re` filter slash-separated relative paths using regexes. They select returned paths or searched files without changing traversal. To skip entire subtrees, use `skip_dir` with a glob or `skip_dir_re` with a regex.
+
+## Text search
+
+`rg` and `rg_iter` return structured `SearchLine` rows. They accept the same filters as `fd`: `include`, `exclude`, `glob`, `ext`, `path_re`, `skip_path_re`, `skip_dir`, `skip_dir_re`, `min_depth`, `max_depth`, `max_filesize`, `follow_links`, and `same_file_system`.
+
+Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behaviour. Use `case_sensitive=False` to force case-insensitive matching.
+
+Each `SearchLine` has these fields:
+
+```text
+kind         'match', 'before', 'after', or 'context'
+path         path relative to root
+line_number  1-based line number
+lnhash       exhash-style `lineno|hash|` address for the line
+line         line text without the trailing newline
+matches      list of (start, end) byte offsets for match rows
+```
+
+`rg`, `search_text`, and `search_path` return `SearchResults` by default. This list subclass displays as rg-style multiline text in `str()` and notebook pretty output. `rg_iter` yields rows lazily.
+
+`SearchLine` has a structured `repr` and an rg-style `str`. The string display truncates `line` to 180 characters with a trailing `…`. `repr` and `asdict()` retain the full line. `SearchLine.asdict()` returns the fields as a plain Python dictionary.
+
+Pass `lnhashs=True` to `rg` or `rg_iter` to display hash addresses instead of line numbers. The `line_number` field remains available.
+
+For other result forms, use `rg(..., paths=True)` to return unique matched paths or `rg(..., count=True)` to count match spans. `paths` and `count` cannot both be set.
+
+### Path results
+
+`fd`, `walk`, and `ls` return `PathResults`. So do `rg` and `nbrg` with `paths=True`. This is a list of `FileEntry` rows.
+
+A `FileEntry` is a `str` subclass containing a relative path. Its `stat` property calls `os.lstat` on first access and caches the result. It returns `None` if the path has vanished. `size`, `mtime`, and `is_dir` use the cached stat result. The object also supports ordinary string operations.
+
+`PathResults` displays as an `ls -l`-style listing of at most `rgapi.MAX_REPR` rows. A final `… N more` line reports omitted rows. Only displayed rows need stat calls. `str()` returns one plain path per line. `list(res)` also displays plain paths.
+
+### Limits and timeouts
+
+Set `timeout_ms` on `rg`, `fd`, `walk`, or `ls` to stop at a deadline and return the results collected so far. Their async versions accept it too. Results report why the operation stopped:
+
+- `stop_reason=None` means the result is complete.
+- `stop_reason="max_results"` means `max_results` truncated the result.
+- `stop_reason="timeout"` means the deadline was reached.
+
+`complete` is true exactly when `stop_reason` is `None`. `count=True` returns a plain integer without a completion flag. It cannot be combined with `timeout_ms`.
+
+### Context lines
+
+`before_context`, `after_context`, and `context` correspond to `rg -B`, `rg -A`, and `rg -C`. Files containing NUL bytes or invalid UTF-8 are skipped.
+
+### Block summaries
+
+`rg(..., summary=True)` returns one row per blank-line-delimited block instead of one row per matching line. Empty and whitespace-only lines delimit blocks. A block containing several matching lines appears once and keeps every matching `SearchLine` in `matches`.
+
+```python
+rg("TODO", ".", summary=True, context=1, maxlen=120)
+```
+
+The result is `BlockResults`, a list of `SearchBlock` objects. Each block has `path`, `block_index`, `start_line`, `end_line`, `start_lnhash`, `end_lnhash`, `kind`, full `source`, and `matches`.
+
+Matches display as `path:start-end:source`. Context displays as `path:start-end-source`. With `lnhashs=True`, the range uses copyable boundary addresses such as `path:4|a3f2|,6|b1c3|:source`. Newline runs display as `¶`. `maxlen` limits the displayed text without changing `source` or `asdict()`.
+
+In summary mode, `before_context`, `after_context`, and `context` count neighbouring blocks. `max_results` counts matching blocks and retains their context. `summary=True` cannot be combined with `paths` or `count`. It can be combined with `lnhash` for copyable block boundaries.
+
+## Notebooks
+
+`nbrg` searches cell source in Jupyter `.ipynb` files and returns matching cells. Each result identifies the cell by its nbformat cell/message id, which stays stable across edits.
+
+Plain `rg` searches escaped notebook JSON, including outputs and metadata. Its line numbers refer to that JSON file. `nbrg` searches the reconstructed cell source and identifies the cell you would edit.
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".")                  # cells whose source matches, across all notebooks under "."
+nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
+```
+
+Notebook discovery, parsing, and matching run together in one parallel Rust pass. Matching uses `rg`'s regex engine with the same `case_sensitive` and `smart_case` behaviour. `nbrg` accepts the discovery filters from `fd` and `rg`, including `include`, `exclude`, `glob`, `hidden`, `max_depth`, and `skip_dir`.
+
+`nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
+
+```text
+path         notebook path relative to root
+cell_index   0-based position of the cell in the notebook
+cell_id      nbformat cell id (falls back to the cell index for notebooks without ids)
+cell_type    'code', 'markdown', or 'raw'
+kind         'match' or 'context'
+source       full cell source
+matches      list of SearchLine rows for the matched lines within the cell
+```
+
+`NbCell.asdict()` returns these fields as a plain dictionary. Its `matches` field contains `SearchLine` dictionaries.
+
+`str()` and pretty display show one line per cell. Matches use `path:cell_id:source`. Context uses `path:cell_id-source`. Newline runs display as `¶`.
+
+A matching cell's display starts at its first matched line. Earlier lines are replaced by `…[Ln]`, where `n` is the matched line's one-based number within the cell. A leading `#|` directive is retained, as in `#| export…[L4]needle here`.
+
+`maxlen` limits displayed source and defaults to 120. The full source remains in `source` and `asdict()`. Each cell appears once even when it has multiple matches. All hits remain in `matches`.
+
+`cell_context=N` includes the `N` cells before and after each match as `kind="context"` rows. Context cells are deduplicated within each notebook.
+
+`nbrg_iter` yields `NbCell` rows as notebooks are parsed. For collected results, `nbrg` accepts these limits and result options:
+
+- `max_results` returns at most that many cells after sorting by path and cell index.
+- `count=True` returns the number of matching cells.
+- `timeout_ms` applies a deadline with the same `stop_reason` values as `rg`.
+
+The parser reads only each cell's `id`, `cell_type`, and `source`. It skips outputs and metadata without loading embedded images or plots. `search_nb(pattern, path, ...)` searches a single notebook file in the same way.
+
+`rgapi-nbrg` exposes notebook search without requiring a Python kernel:
+
+```bash
+rgapi-nbrg 'read_csv' .
+rgapi-nbrg 'read_csv' . --cell-context 1
+rgapi-nbrg 'read_csv' nbs --glob '*.ipynb' --max-results 20
+```
+
+Run `rgapi-nbrg --help` for its discovery, matching, and output options.
+
+## Async
+
+`fda`, `rga`, and `nbrga` are async versions of `fd`, `rg`, and `nbrg`. The async generators `fda_iter`, `rga_iter`, and `nbrga_iter` yield rows as the search finds them. Async functions accept the same arguments and return the same types as their synchronous equivalents.
+
+```python
+from rgapi import fda, rga, rga_iter
+
+await fda(".", ext="py")
+res = await rga("TODO", ".", timeout_ms=200)
+if not res.complete: print(f"partial results: {res.stop_reason}")
+async for row in rga_iter("TODO", "."): ...
+```
+
+Walking and searching use Rust threads, without `asyncio.to_thread` or the event loop's executor. A callback uses `loop.call_soon_threadsafe` to complete the awaited future or supply results to the generator's queue. The event loop remains unblocked, with normal `contextvars` behaviour.
+
+Cancellation stops the Rust workers within about one row. This includes timeouts from `asyncio.wait_for` or `asyncio.timeout`. It also includes task cancellation, such as starlette cancelling a disconnected client's request.
+
+Wrap an async iterator in `contextlib.aclosing` when leaving its loop early. `break` alone delays generator finalization until garbage collection. The context manager provides prompt cleanup:
+
+```python
+async with aclosing(rga_iter("TODO", ".")) as it:
+    async for row in it:
+        if enough(row): break
+```
+
+Use streaming results for incremental display, such as sending batches to a browser as they arrive. Collected results with `timeout_ms` return what was found before the deadline. Use `asyncio.wait_for` when a timeout should raise instead.
+
+
+## Benchmarks
+
+`tools/bench.py` compares the `rg` CLI with in-process `rgapi`. Run it against a release build. One run on this machine, using best time from seven repeats:
+
+| fixture | rg | rgapi |
+| --- | ---: | ---: |
+| 6 x 2 MB files, 2 matches | 6.54 ms | 1.44 ms |
+| 800 x 1.5 KB files, 2 matches | 13.90 ms | 10.94 ms |
+| tiny dir, repeated 30x | 5.92 ms | 2.14 ms |
+

rgapi-0.1.26/README.md

@@ -0,0 +1,233 @@
+# rgapi
+
+`rgapi` provides `fd`-style file discovery and `rg`-style text search from Python without starting a shell command.
+
+It uses the same `ignore`, `grep-regex`, and `grep-searcher` crates that ripgrep uses for walking, regex matching, and file scanning. Walking and searching run in parallel by default. Most expensive work stays in Rust.
+
+## Overview
+
+For common file discovery and search:
+
+```python
+from rgapi import fd, ls, rg, rg_iter
+
+fd(".", ext="py", exclude="test_*.py")
+ls("src")
+for row in rg_iter("TODO", ".", include="*.py", context=2): print(row.asdict())
+rg("TODO", ".", ext="py", skip_dir=".venv", paths=True)
+```
+
+For cell-aware search of Jupyter notebooks (see [Notebooks](#notebooks)):
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".", cell_context=1)
+```
+
+Walk and search functions have async versions. Streaming versions yield results as the search finds them. See [Async](#async):
+
+```python
+from rgapi import fda, rga, rga_iter, nbrga, nbrga_iter
+
+await rga("TODO", ".", ext="py", timeout_ms=200)
+async for row in rga_iter("TODO", "."): print(row)
+```
+
+Use the lower-level functions to compile a regex, search text or a single file, or walk a directory:
+
+```python
+from rgapi import compile, search_path, search_text, walk
+
+matcher = compile("TODO")
+matcher.is_match("TODO")
+matcher.finditer("TODO TODO")
+
+walk(".")
+search_text(matcher, "alpha\nTODO\nomega\n", path="memory.txt", context=1)
+search_path(matcher, "src/lib.rs", display_path="src/lib.rs")
+```
+
+## Install
+
+```bash
+pip install rgapi
+```
+
+## File discovery
+
+`fd` and `walk` return slash-separated paths relative to `root`. Pass `root` as a `str` or `pathlib.Path`. The sync and async APIs expand `~` and accept `.`, `./`, and paths containing `..`.
+
+Discovery uses the `ignore` crate with ripgrep's default filters. It reads `.gitignore`, `.ignore`, and `.rgignore` files. `.rgignore` takes precedence over `.gitignore`. Pass `ignore=False` to disable all ignore-file filtering, including `.rgignore`.
+
+Hidden files are skipped unless `hidden=True`. Symlinks are followed only with `follow_links=True`. Use `same_file_system=True` to avoid crossing filesystem boundaries.
+
+Traversal runs in parallel without guaranteed result order. Use `sorted(...)` when order matters.
+
+`fd` adds filename filters to `walk`. Its `pattern` is a smart-case regex matched against each basename. Lowercase patterns match case-insensitively. A pattern containing uppercase letters is case-sensitive. Use `path_re` to match the slash-separated relative path instead.
+
+`include` and `exclude` use glob syntax. `glob=` is an alias for `include=`. A basename glob such as `*.py` also matches nested paths such as `src/app.py`.
+
+Filter extensions with `ext="py"` or `ext=["py", "rs"]`. Extension and glob filters must both match. For example, `include="src/*", ext="py"` requires `src/*` and `*.py`, like combining `rg -g` with `-t`.
+
+Set `min_depth` and `max_depth` to bound recursion. `max_filesize` skips files above a byte limit.
+
+`ls` follows the shell command's listing conventions. It uses `fd` with `max_depth=1`, includes directories, disables ignore rules, and sorts by name. Set `hidden=True` for `ls -a` behaviour. All `fd` filters remain available.
+
+`fd_iter` yields `FileEntry` paths as the walk finds them. It accepts every `fd` filter. Stopping iteration ends the walk. It does not accept `timeout_ms`.
+
+`path_re` and `skip_path_re` filter slash-separated relative paths using regexes. They select returned paths or searched files without changing traversal. To skip entire subtrees, use `skip_dir` with a glob or `skip_dir_re` with a regex.
+
+## Text search
+
+`rg` and `rg_iter` return structured `SearchLine` rows. They accept the same filters as `fd`: `include`, `exclude`, `glob`, `ext`, `path_re`, `skip_path_re`, `skip_dir`, `skip_dir_re`, `min_depth`, `max_depth`, `max_filesize`, `follow_links`, and `same_file_system`.
+
+Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behaviour. Use `case_sensitive=False` to force case-insensitive matching.
+
+Each `SearchLine` has these fields:
+
+```text
+kind         'match', 'before', 'after', or 'context'
+path         path relative to root
+line_number  1-based line number
+lnhash       exhash-style `lineno|hash|` address for the line
+line         line text without the trailing newline
+matches      list of (start, end) byte offsets for match rows
+```
+
+`rg`, `search_text`, and `search_path` return `SearchResults` by default. This list subclass displays as rg-style multiline text in `str()` and notebook pretty output. `rg_iter` yields rows lazily.
+
+`SearchLine` has a structured `repr` and an rg-style `str`. The string display truncates `line` to 180 characters with a trailing `…`. `repr` and `asdict()` retain the full line. `SearchLine.asdict()` returns the fields as a plain Python dictionary.
+
+Pass `lnhashs=True` to `rg` or `rg_iter` to display hash addresses instead of line numbers. The `line_number` field remains available.
+
+For other result forms, use `rg(..., paths=True)` to return unique matched paths or `rg(..., count=True)` to count match spans. `paths` and `count` cannot both be set.
+
+### Path results
+
+`fd`, `walk`, and `ls` return `PathResults`. So do `rg` and `nbrg` with `paths=True`. This is a list of `FileEntry` rows.
+
+A `FileEntry` is a `str` subclass containing a relative path. Its `stat` property calls `os.lstat` on first access and caches the result. It returns `None` if the path has vanished. `size`, `mtime`, and `is_dir` use the cached stat result. The object also supports ordinary string operations.
+
+`PathResults` displays as an `ls -l`-style listing of at most `rgapi.MAX_REPR` rows. A final `… N more` line reports omitted rows. Only displayed rows need stat calls. `str()` returns one plain path per line. `list(res)` also displays plain paths.
+
+### Limits and timeouts
+
+Set `timeout_ms` on `rg`, `fd`, `walk`, or `ls` to stop at a deadline and return the results collected so far. Their async versions accept it too. Results report why the operation stopped:
+
+- `stop_reason=None` means the result is complete.
+- `stop_reason="max_results"` means `max_results` truncated the result.
+- `stop_reason="timeout"` means the deadline was reached.
+
+`complete` is true exactly when `stop_reason` is `None`. `count=True` returns a plain integer without a completion flag. It cannot be combined with `timeout_ms`.
+
+### Context lines
+
+`before_context`, `after_context`, and `context` correspond to `rg -B`, `rg -A`, and `rg -C`. Files containing NUL bytes or invalid UTF-8 are skipped.
+
+### Block summaries
+
+`rg(..., summary=True)` returns one row per blank-line-delimited block instead of one row per matching line. Empty and whitespace-only lines delimit blocks. A block containing several matching lines appears once and keeps every matching `SearchLine` in `matches`.
+
+```python
+rg("TODO", ".", summary=True, context=1, maxlen=120)
+```
+
+The result is `BlockResults`, a list of `SearchBlock` objects. Each block has `path`, `block_index`, `start_line`, `end_line`, `start_lnhash`, `end_lnhash`, `kind`, full `source`, and `matches`.
+
+Matches display as `path:start-end:source`. Context displays as `path:start-end-source`. With `lnhashs=True`, the range uses copyable boundary addresses such as `path:4|a3f2|,6|b1c3|:source`. Newline runs display as `¶`. `maxlen` limits the displayed text without changing `source` or `asdict()`.
+
+In summary mode, `before_context`, `after_context`, and `context` count neighbouring blocks. `max_results` counts matching blocks and retains their context. `summary=True` cannot be combined with `paths` or `count`. It can be combined with `lnhash` for copyable block boundaries.
+
+## Notebooks
+
+`nbrg` searches cell source in Jupyter `.ipynb` files and returns matching cells. Each result identifies the cell by its nbformat cell/message id, which stays stable across edits.
+
+Plain `rg` searches escaped notebook JSON, including outputs and metadata. Its line numbers refer to that JSON file. `nbrg` searches the reconstructed cell source and identifies the cell you would edit.
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".")                  # cells whose source matches, across all notebooks under "."
+nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
+```
+
+Notebook discovery, parsing, and matching run together in one parallel Rust pass. Matching uses `rg`'s regex engine with the same `case_sensitive` and `smart_case` behaviour. `nbrg` accepts the discovery filters from `fd` and `rg`, including `include`, `exclude`, `glob`, `hidden`, `max_depth`, and `skip_dir`.
+
+`nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
+
+```text
+path         notebook path relative to root
+cell_index   0-based position of the cell in the notebook
+cell_id      nbformat cell id (falls back to the cell index for notebooks without ids)
+cell_type    'code', 'markdown', or 'raw'
+kind         'match' or 'context'
+source       full cell source
+matches      list of SearchLine rows for the matched lines within the cell
+```
+
+`NbCell.asdict()` returns these fields as a plain dictionary. Its `matches` field contains `SearchLine` dictionaries.
+
+`str()` and pretty display show one line per cell. Matches use `path:cell_id:source`. Context uses `path:cell_id-source`. Newline runs display as `¶`.
+
+A matching cell's display starts at its first matched line. Earlier lines are replaced by `…[Ln]`, where `n` is the matched line's one-based number within the cell. A leading `#|` directive is retained, as in `#| export…[L4]needle here`.
+
+`maxlen` limits displayed source and defaults to 120. The full source remains in `source` and `asdict()`. Each cell appears once even when it has multiple matches. All hits remain in `matches`.
+
+`cell_context=N` includes the `N` cells before and after each match as `kind="context"` rows. Context cells are deduplicated within each notebook.
+
+`nbrg_iter` yields `NbCell` rows as notebooks are parsed. For collected results, `nbrg` accepts these limits and result options:
+
+- `max_results` returns at most that many cells after sorting by path and cell index.
+- `count=True` returns the number of matching cells.
+- `timeout_ms` applies a deadline with the same `stop_reason` values as `rg`.
+
+The parser reads only each cell's `id`, `cell_type`, and `source`. It skips outputs and metadata without loading embedded images or plots. `search_nb(pattern, path, ...)` searches a single notebook file in the same way.
+
+`rgapi-nbrg` exposes notebook search without requiring a Python kernel:
+
+```bash
+rgapi-nbrg 'read_csv' .
+rgapi-nbrg 'read_csv' . --cell-context 1
+rgapi-nbrg 'read_csv' nbs --glob '*.ipynb' --max-results 20
+```
+
+Run `rgapi-nbrg --help` for its discovery, matching, and output options.
+
+## Async
+
+`fda`, `rga`, and `nbrga` are async versions of `fd`, `rg`, and `nbrg`. The async generators `fda_iter`, `rga_iter`, and `nbrga_iter` yield rows as the search finds them. Async functions accept the same arguments and return the same types as their synchronous equivalents.
+
+```python
+from rgapi import fda, rga, rga_iter
+
+await fda(".", ext="py")
+res = await rga("TODO", ".", timeout_ms=200)
+if not res.complete: print(f"partial results: {res.stop_reason}")
+async for row in rga_iter("TODO", "."): ...
+```
+
+Walking and searching use Rust threads, without `asyncio.to_thread` or the event loop's executor. A callback uses `loop.call_soon_threadsafe` to complete the awaited future or supply results to the generator's queue. The event loop remains unblocked, with normal `contextvars` behaviour.
+
+Cancellation stops the Rust workers within about one row. This includes timeouts from `asyncio.wait_for` or `asyncio.timeout`. It also includes task cancellation, such as starlette cancelling a disconnected client's request.
+
+Wrap an async iterator in `contextlib.aclosing` when leaving its loop early. `break` alone delays generator finalization until garbage collection. The context manager provides prompt cleanup:
+
+```python
+async with aclosing(rga_iter("TODO", ".")) as it:
+    async for row in it:
+        if enough(row): break
+```
+
+Use streaming results for incremental display, such as sending batches to a browser as they arrive. Collected results with `timeout_ms` return what was found before the deadline. Use `asyncio.wait_for` when a timeout should raise instead.
+
+
+## Benchmarks
+
+`tools/bench.py` compares the `rg` CLI with in-process `rgapi`. Run it against a release build. One run on this machine, using best time from seven repeats:
+
+| fixture | rg | rgapi |
+| --- | ---: | ---: |
+| 6 x 2 MB files, 2 matches | 6.54 ms | 1.44 ms |
+| 800 x 1.5 KB files, 2 matches | 13.90 ms | 10.94 ms |
+| tiny dir, repeated 30x | 5.92 ms | 2.14 ms |

{rgapi-0.1.24 → rgapi-0.1.26}/src/lib.rs

@@ -9,7 +9,7 @@ mod walk;
 mod python;
 
 pub use block::{BlockIter, SearchBlock, block_iter};
-pub use nb::{NbCell, NbIter, NbOptions, nb_iter, nb_search, nb_search_file};
+pub use nb::{NbCell, NbIter, NbOptions, ancestor_indices, heading_level, nb_iter, nb_search, nb_search_file, section_range};
 pub use search::{MatchSpan, RgIter, RgOptions, SearchKind, SearchLine, compile_regex, rg, rg_iter, search_path, search_text};
 pub use walk::{FindIter, FindOptions, StreamIter, find, find_iter};
 

{rgapi-0.1.24 → rgapi-0.1.26}/src/nb.rs

@@ -1,8 +1,9 @@
 use std::collections::{BTreeMap, HashMap};
 use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::sync::atomic::Ordering;
 
+use grep_matcher::Matcher;
 use grep_regex::RegexMatcher;
 use ignore::{DirEntry, WalkState};
 use serde::Deserialize;
@@ -11,6 +12,36 @@ use crate::RgApiError;
 use crate::search::{SearchLine, compile_regex, search_text};
 use crate::walk::{PathFilters, StreamIter, entry_err, file_root_flags, normalize_root, rel_path, spawn_walk};
 
+static HEADING_RE: LazyLock<RegexMatcher> = LazyLock::new(|| RegexMatcher::new(r"^#{1,6} \w").unwrap());
+
+/// Heading level of the first nonblank line after skipping `#|` directives; zero unless it matches `^#{1,6} \w`.
+pub fn heading_level(source: &str) -> usize {
+    let line = source.lines().find(|l| !l.trim().is_empty() && !l.starts_with("#|")).unwrap_or("");
+    if !HEADING_RE.is_match(line.as_bytes()).unwrap_or(false) { return 0; }
+    line.bytes().take_while(|&b| b == b'#').count()
+}
+
+/// A heading and its descendants; a non-heading selects itself. Zero levels represent non-heading cells.
+pub fn section_range(levels: &[usize], idx: usize) -> std::ops::Range<usize> {
+    if levels[idx] == 0 { return idx..idx + 1; }
+    let end = (idx + 1..levels.len()).find(|&i| levels[i] > 0 && levels[i] <= levels[idx]).unwrap_or(levels.len());
+    idx..end
+}
+
+/// Enclosing heading indices, outermost first, excluding the addressed cell.
+pub fn ancestor_indices(levels: &[usize], idx: usize) -> Vec<usize> {
+    let mut level = if levels[idx] == 0 { 7 } else { levels[idx] };
+    let mut parents = Vec::new();
+    for i in (0..idx).rev() {
+        if levels[i] > 0 && levels[i] < level {
+            parents.push(i);
+            level = levels[i];
+        }
+    }
+    parents.reverse();
+    parents
+}
+
 #[derive(Debug, Clone)]
 pub struct NbOptions {
     pub root: PathBuf,

{rgapi-0.1.24 → rgapi-0.1.26}/src/walk.rs

@@ -104,13 +104,23 @@ pub fn find_iter(opts: &FindOptions) -> Result<FindIter, RgApiError> {
     ))
 }
 
-pub struct StreamIter<T> { rx: mpsc::Receiver<Result<T, RgApiError>>, cancel: Arc<AtomicBool>, _worker: std::thread::JoinHandle<()> }
+pub struct StreamIter<T> { rx: mpsc::Receiver<Result<T, RgApiError>>, cancel: Arc<AtomicBool>, worker: Option<std::thread::JoinHandle<()>> }
 
 impl<T> StreamIter<T> {
     pub fn cancel(&self) { self.cancel.store(true, Ordering::Relaxed); }
 
     pub fn cancel_flag(&self) -> Arc<AtomicBool> { self.cancel.clone() }
 
+    /// Cancel and wait for the walk's workers to finish. Drain queued sends before
+    /// joining so a full result channel cannot deadlock shutdown. Unlike Drop,
+    /// this guarantees no background walk remains when it returns. Filesystem
+    /// calls already in progress must return first; run this off an async executor.
+    pub fn cancel_and_join(mut self) -> Result<(), RgApiError> {
+        self.cancel();
+        while self.rx.recv().is_ok() {}
+        self.worker.take().expect("stream owns its worker").join().map_err(|_| RgApiError::new("search worker panicked"))
+    }
+
     pub fn next_timeout(&mut self, timeout: std::time::Duration) -> Result<Result<T, RgApiError>, mpsc::RecvTimeoutError> { self.rx.recv_timeout(timeout) }
 
     /// Collect all items, stopping at `timeout_ms`; the bool is true when the deadline stopped it.
@@ -138,6 +148,33 @@ impl<T> Iterator for StreamIter<T> {
 
 impl<T> Drop for StreamIter<T> { fn drop(&mut self) { self.cancel(); } }
 
+#[cfg(test)]
+mod close_tests {
+    use super::*;
+
+    #[test]
+    fn cancel_and_join_drains_full_channel_and_waits_for_worker() {
+        let (tx, rx) = mpsc::sync_channel(1);
+        let (started, ready) = mpsc::channel();
+        let cancel = Arc::new(AtomicBool::new(false));
+        let worker_cancel = cancel.clone();
+        let done = Arc::new(AtomicBool::new(false));
+        let worker_done = done.clone();
+        let worker = std::thread::spawn(move || {
+            tx.send(Ok(1)).unwrap();
+            started.send(()).unwrap();
+            // This blocks while the channel is full; close must drain, not just join.
+            tx.send(Ok(2)).unwrap();
+            assert!(worker_cancel.load(Ordering::Acquire));
+            worker_done.store(true, Ordering::Release);
+        });
+        ready.recv().unwrap();
+        StreamIter { rx, cancel: cancel.clone(), worker: Some(worker) }.cancel_and_join().unwrap();
+        assert!(cancel.load(Ordering::Acquire));
+        assert!(done.load(Ordering::Acquire));
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 pub fn spawn_walk<T, F>(
     root: PathBuf,
@@ -181,7 +218,7 @@ where
             })
         });
     });
-    StreamIter { rx, cancel, _worker: worker }
+    StreamIter { rx, cancel, worker: Some(worker) }
 }
 
 fn find_entry(
@@ -365,7 +402,7 @@ mod tests {
                 if tx.send(Ok(i)).is_err() { return; }
             }
         });
-        StreamIter { rx, cancel, _worker: worker }
+        StreamIter { rx, cancel, worker: Some(worker) }
     }
 
     #[test]

rgapi-0.1.24/PKG-INFO

@@ -1,201 +0,0 @@
-Metadata-Version: 2.4
-Name: rgapi
-Version: 0.1.24
-Classifier: Programming Language :: Rust
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Requires-Dist: fastcore>=1.14.6
-Requires-Dist: fastship>=0.0.12 ; extra == 'dev'
-Requires-Dist: maturin>=1.0,<2.0 ; extra == 'dev'
-Requires-Dist: pytest ; extra == 'dev'
-Provides-Extra: dev
-License-File: LICENSE
-Summary: Python API for ripgrep-style file walking and searching
-Home-Page: https://github.com/AnswerDotAI/rgapi
-Author-email: Jeremy Howard <j@fast.ai>
-License: Apache-2.0
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-Project-URL: Homepage, https://github.com/AnswerDotAI/rgapi
-Project-URL: Issues, https://github.com/AnswerDotAI/rgapi/issues
-Project-URL: Repository, https://github.com/AnswerDotAI/rgapi
-
-# rgapi
-
-`rgapi` is a Python API for ripgrep-style walking and search. It is meant for Python code that wants `fd`-style file discovery or `rg`-style searching without shelling out.
-
-It uses the same `ignore`, `grep-regex`, and `grep-searcher` crates that ripgrep uses for walking, regex matching, and file scanning. Walking and searching run in parallel by default. Most expensive work stays in Rust.
-
-## Overview
-
-For common file discovery and search:
-
-```python
-from rgapi import fd, ls, rg, rg_iter
-
-fd(".", ext="py", exclude="test_*.py")
-ls("src")
-for row in rg_iter("TODO", ".", include="*.py", context=2): print(row.asdict())
-rg("TODO", ".", ext="py", skip_dir=".venv", paths=True)
-```
-
-For cell-aware search of Jupyter notebooks (see [Notebooks](#notebooks)):
-
-```python
-from rgapi import nbrg
-
-nbrg("read_csv", ".", cell_context=1)
-```
-
-Every walk and search has an async twin, plus streaming forms that yield results as they are found (see [Async](#async)):
-
-```python
-from rgapi import fda, rga, rga_iter, nbrga, nbrga_iter
-
-await rga("TODO", ".", ext="py", timeout_ms=200)
-async for row in rga_iter("TODO", "."): print(row)
-```
-
-For direct access to the regex, search, and walk pieces:
-
-```python
-from rgapi import compile, search_path, search_text, walk
-
-matcher = compile("TODO")
-matcher.is_match("TODO")
-matcher.finditer("TODO TODO")
-
-walk(".")
-search_text(matcher, "alpha\nTODO\nomega\n", path="memory.txt", context=1)
-search_path(matcher, "src/lib.rs", display_path="src/lib.rs")
-```
-
-## Install
-
-```bash
-pip install rgapi
-```
-
-## Semantics
-
-`fd` and `walk` return slash-separated paths relative to `root`. They use the `ignore` crate, so `.gitignore`, `.ignore`, and the usual ripgrep filters apply by default. `.rgignore` files are also honored and take precedence over `.gitignore`. Hidden files are skipped unless `hidden=True`. Pass `ignore=False` to disable all ignore filtering (including `.rgignore`). Symlinks are not followed unless `follow_links=True`; `same_file_system=True` avoids crossing filesystem boundaries. Traversal is parallel, and result order is not guaranteed; use `sorted(...)` if order matters. `root` arguments accept `str` or `pathlib.Path` and expand `~`; `.`, `./`, and paths containing `..` work across the sync and async APIs.
-
-`fd` adds fd-like filtering on top of `walk`: `pattern` is a smart-case regex matched against each basename, and `include`/`exclude` use glob syntax. Lowercase patterns match case-insensitively; a pattern containing uppercase letters is case-sensitive. Use `path_re` when matching the slash-separated relative path instead. `glob=` is accepted as an alias for `include=`. A basename glob such as `*.py` also matches recursively, so it finds `src/app.py`. Use `ext="py"` or `ext=["py", "rs"]` for extension filters, which compose as AND with `include`/`glob` (so `include="src/*", ext="py"` means `src/*` *and* `*.py`, like combining `rg -g` with `-t`); use `min_depth=`/`max_depth=` to bound recursion, and `max_filesize=` to skip files above a byte limit.
-
-`ls` lists like the shell command: it is `fd` with defaults flipped to one level (`max_depth=1`), directories included, ignore rules off, and results sorted by name. `hidden=True` is `ls -a`, and every `fd` filter still applies.
-
-`fd_iter` is the lazy form of `fd`, yielding `FileEntry` paths as the walk finds them, and takes every `fd` filter. It has no `timeout_ms`, since a consumer that stops asking for paths ends the walk itself.
-
-`path_re` and `skip_path_re` are regex filters on slash-separated relative paths. They filter returned paths or searched files, but do not control traversal. `skip_dir` uses glob syntax to prune matching directory subtrees, and `skip_dir_re` does the same with regex.
-
-`rg` and `rg_iter` return structured rows rather than raw CLI text. They accept the same `include`, `exclude`, `glob`, `ext`, `path_re`, `skip_path_re`, `skip_dir`, `skip_dir_re`, `min_depth`, `max_depth`, `max_filesize`, `follow_links`, and `same_file_system` filters as `fd`. Each row is a `SearchLine` with:
-
-```text
-kind         'match', 'before', 'after', or 'context'
-path         path relative to root
-line_number  1-based line number
-lnhash       exhash-style `lineno|hash|` address for the line
-line         line text without the trailing newline
-matches      list of (start, end) byte offsets for match rows
-```
-
-`rg`, `search_text`, and `search_path` return `SearchResults` by default, a list subclass whose `str()` and notebook pretty display are rg-style multiline text. `rg_iter` yields rows lazily.
-
-`SearchLine` has a structured `repr`, an rg-style `str` (the `line` is truncated to 180 chars with a trailing `…` for display; `repr` and `asdict()` keep the full line), and `SearchLine.asdict()` returns row fields as a plain Python dict. Pass `rg(..., lnhashs=True)` or `rg_iter(..., lnhashs=True)` to show `lnhash` addresses instead of line numbers in row display while keeping `line_number` available. `rg(..., paths=True)` returns unique matched paths, and `rg(..., count=True)` returns the total number of match spans. `paths` and `count` cannot both be set.
-
-`fd`, `walk`, `ls`, and `rg`/`nbrg` with `paths=True` return `PathResults`, a list of `FileEntry` rows. A `FileEntry` is a `str` subclass holding the relative path, so all string uses keep working, and it stats itself lazily on first access: `stat` is a cached `os.lstat` result (`None` if the path has vanished), with `size`, `mtime`, and `is_dir` derived from it. A `PathResults` displays as an `ls -l`-style long listing, capped at `rgapi.MAX_REPR` rows with a final `… N more` line, so stats are read only for displayed rows; `str()` is still one plain path per line, and `list(res)` shows plain paths. `rg(..., timeout_ms=200)` and `fd(..., timeout_ms=200)` stop at the deadline and return whatever was collected by then; `walk`, `ls`, and the async forms take it too. Results record how they ended: `stop_reason` is `None` for a complete result, `"max_results"` when truncated by `max_results`, or `"timeout"` when a deadline hit, and `complete` is true when `stop_reason` is `None`. `count=True` returns a plain int, which cannot carry the flag, so it rejects `timeout_ms`.
-
-`before_context`, `after_context`, and `context` are like `rg -B`, `rg -A`, and `rg -C`. Files containing NUL bytes or invalid UTF-8 are skipped.
-
-### Block summaries
-
-`rg(..., summary=True)` returns one row per blank-line-delimited block instead of one row per matching line. Empty and whitespace-only lines delimit blocks. A block containing several matching lines appears once and keeps every matching `SearchLine` in `matches`.
-
-```python
-rg("TODO", ".", summary=True, context=1, maxlen=120)
-```
-
-The result is `BlockResults`, a list of `SearchBlock` objects. Each block has `path`, `block_index`, `start_line`, `end_line`, `start_lnhash`, `end_lnhash`, `kind`, full `source`, and `matches`. Its display is `path:start-end:source` for matches and `path:start-end-source` for context. With `lnhashs=True`, the numeric range becomes copyable boundary addresses such as `path:4|a3f2|,6|b1c3|:source`. Embedded newline runs are shown as `¶`; `maxlen` limits displayed source without changing `source` or `asdict()`.
-
-In summary mode, `before_context`, `after_context`, and `context` count neighbouring blocks. `max_results` counts matching blocks and retains their block context. `summary=True` cannot be combined with `paths` or `count`; it can be combined with `lnhash` when copyable block boundaries are useful.
-
-Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behavior, or `case_sensitive=False` to force case-insensitive matching.
-
-## Notebooks
-
-`nbrg` searches Jupyter `.ipynb` files cell-by-cell, so results are *cells* rather than raw JSON lines, and each match is identified by its **cell id** (the nbformat cell/message id) rather than a line number. Searching a notebook with plain `rg` matches the escaped JSON text (including outputs and metadata) and reports meaningless JSON line numbers; `nbrg` instead searches each cell's reconstructed **source** and reports the cell id, which is stable across edits and points at the actual unit you work with.
-
-```python
-from rgapi import nbrg
-
-nbrg("read_csv", ".")                  # cells whose source matches, across all notebooks under "."
-nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
-```
-
-Notebooks are walked, parsed, and matched together in one parallel Rust pass, using the same regex engine as `rg`, so regex behaviour and the `case_sensitive`/`smart_case` flags match `rg`. Only cell `source` is searched, not outputs or metadata. `nbrg` accepts the same discovery filters as `fd`/`rg` (`include`, `exclude`, `glob`, `hidden`, `max_depth`, `skip_dir`, …).
-
-`nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
-
-```text
-path         notebook path relative to root
-cell_index   0-based position of the cell in the notebook
-cell_id      nbformat cell id (falls back to the cell index for notebooks without ids)
-cell_type    'code', 'markdown', or 'raw'
-kind         'match' or 'context'
-source       full cell source
-matches      list of SearchLine rows for the matched lines within the cell
-```
-
-`NbCell.asdict()` returns those fields as a plain dict (with `matches` as `SearchLine` dicts). `str()` and pretty display show one line per cell, newline runs shown as `¶`, keyed by `cell_id`: `path:cell_id:source` for matches and `path:cell_id-source` for context. A match row starts at its first matched line: earlier lines display as `…[Ln]` (`n` the matched line's 1-based number in the cell), except that a leading `#|` directive line is kept, e.g. `#| export…[L4]needle here`. `maxlen` controls the displayed source length and defaults to 120; the full source remains in `source` and `asdict()`. A cell with several matches appears once, with every hit collected in `matches`.
-
-`cell_context=N` includes the `N` cells before and after each matching cell as `kind="context"` rows (deduplicated per notebook).
-
-`nbrg_iter` yields `NbCell` rows lazily as notebooks are parsed. `nbrg` also accepts `max_results` (at most that many cells, after sorting by path and cell index), `count=True` (number of matching cells), and `timeout_ms=` with the same `stop_reason` semantics as `rg`.
-
-Notebook walking, parsing, and matching all happen in parallel in Rust, in the same pass as the file walk. Parsing uses a lean model that reads only each cell's `id`, `cell_type`, and `source` and skips outputs and metadata, so large embedded outputs (images, plots) are never materialized. `search_nb(pattern, path, ...)` searches a single notebook file the same way.
-
-`rgapi-nbrg` exposes notebook search without requiring a Python kernel:
-
-```bash
-rgapi-nbrg 'read_csv' .
-rgapi-nbrg 'read_csv' . --cell-context 1
-rgapi-nbrg 'read_csv' nbs --glob '*.ipynb' --max-results 20
-```
-
-Run `rgapi-nbrg --help` for its discovery, matching, and output options.
-
-## Async
-
-`fda`, `rga`, and `nbrga` are awaitable twins of `fd`, `rg`, and `nbrg`, and `fda_iter`, `rga_iter` and `nbrga_iter` are async generators that yield rows as the search finds them. All take the same arguments and return the same types as their sync counterparts.
-
-```python
-from rgapi import fda, rga, rga_iter
-
-await fda(".", ext="py")
-res = await rga("TODO", ".", timeout_ms=200)
-if not res.complete: print(f"partial results: {res.stop_reason}")
-async for row in rga_iter("TODO", "."): ...
-```
-
-None of this uses `asyncio.to_thread` or the loop's executor. The walk and search run on Rust threads, and a single callback settles the awaited future (or feeds the generator's queue) through `loop.call_soon_threadsafe`, so the event loop never blocks and contextvars behave normally.
-
-Cancellation cleans up the Rust workers automatically. Wrapping a call in `asyncio.wait_for` or `asyncio.timeout`, cancelling the task (as starlette does when a client disconnects), or leaving an `async for` early all stop the search within about one row. One caveat comes from the language rather than the library: `break` inside `async for` only finalizes the generator at GC time, so for prompt cleanup wrap the iterator in `contextlib.aclosing`:
-
-```python
-async with aclosing(rga_iter("TODO", ".")) as it:
-    async for row in it:
-        if enough(row): break
-```
-
-The streaming forms suit incremental display, such as pushing each batch of results to a browser as it arrives. The collected forms with `timeout_ms` give the best results available within a budget, and `asyncio.wait_for` gives timeout-as-failure. Pick per call site.
-
-
-## Benchmarks
-
-`tools/bench.py` compares the `rg` CLI with in-process `rgapi`. Run it against a release build. One run on this machine, using best time from seven repeats:
-
-| fixture | rg | rgapi |
-| --- | ---: | ---: |
-| 6 x 2 MB files, 2 matches | 6.54 ms | 1.44 ms |
-| 800 x 1.5 KB files, 2 matches | 13.90 ms | 10.94 ms |
-| tiny dir, repeated 30x | 5.92 ms | 2.14 ms |
-

rgapi-0.1.24/README.md

@@ -1,179 +0,0 @@
-# rgapi
-
-`rgapi` is a Python API for ripgrep-style walking and search. It is meant for Python code that wants `fd`-style file discovery or `rg`-style searching without shelling out.
-
-It uses the same `ignore`, `grep-regex`, and `grep-searcher` crates that ripgrep uses for walking, regex matching, and file scanning. Walking and searching run in parallel by default. Most expensive work stays in Rust.
-
-## Overview
-
-For common file discovery and search:
-
-```python
-from rgapi import fd, ls, rg, rg_iter
-
-fd(".", ext="py", exclude="test_*.py")
-ls("src")
-for row in rg_iter("TODO", ".", include="*.py", context=2): print(row.asdict())
-rg("TODO", ".", ext="py", skip_dir=".venv", paths=True)
-```
-
-For cell-aware search of Jupyter notebooks (see [Notebooks](#notebooks)):
-
-```python
-from rgapi import nbrg
-
-nbrg("read_csv", ".", cell_context=1)
-```
-
-Every walk and search has an async twin, plus streaming forms that yield results as they are found (see [Async](#async)):
-
-```python
-from rgapi import fda, rga, rga_iter, nbrga, nbrga_iter
-
-await rga("TODO", ".", ext="py", timeout_ms=200)
-async for row in rga_iter("TODO", "."): print(row)
-```
-
-For direct access to the regex, search, and walk pieces:
-
-```python
-from rgapi import compile, search_path, search_text, walk
-
-matcher = compile("TODO")
-matcher.is_match("TODO")
-matcher.finditer("TODO TODO")
-
-walk(".")
-search_text(matcher, "alpha\nTODO\nomega\n", path="memory.txt", context=1)
-search_path(matcher, "src/lib.rs", display_path="src/lib.rs")
-```
-
-## Install
-
-```bash
-pip install rgapi
-```
-
-## Semantics
-
-`fd` and `walk` return slash-separated paths relative to `root`. They use the `ignore` crate, so `.gitignore`, `.ignore`, and the usual ripgrep filters apply by default. `.rgignore` files are also honored and take precedence over `.gitignore`. Hidden files are skipped unless `hidden=True`. Pass `ignore=False` to disable all ignore filtering (including `.rgignore`). Symlinks are not followed unless `follow_links=True`; `same_file_system=True` avoids crossing filesystem boundaries. Traversal is parallel, and result order is not guaranteed; use `sorted(...)` if order matters. `root` arguments accept `str` or `pathlib.Path` and expand `~`; `.`, `./`, and paths containing `..` work across the sync and async APIs.
-
-`fd` adds fd-like filtering on top of `walk`: `pattern` is a smart-case regex matched against each basename, and `include`/`exclude` use glob syntax. Lowercase patterns match case-insensitively; a pattern containing uppercase letters is case-sensitive. Use `path_re` when matching the slash-separated relative path instead. `glob=` is accepted as an alias for `include=`. A basename glob such as `*.py` also matches recursively, so it finds `src/app.py`. Use `ext="py"` or `ext=["py", "rs"]` for extension filters, which compose as AND with `include`/`glob` (so `include="src/*", ext="py"` means `src/*` *and* `*.py`, like combining `rg -g` with `-t`); use `min_depth=`/`max_depth=` to bound recursion, and `max_filesize=` to skip files above a byte limit.
-
-`ls` lists like the shell command: it is `fd` with defaults flipped to one level (`max_depth=1`), directories included, ignore rules off, and results sorted by name. `hidden=True` is `ls -a`, and every `fd` filter still applies.
-
-`fd_iter` is the lazy form of `fd`, yielding `FileEntry` paths as the walk finds them, and takes every `fd` filter. It has no `timeout_ms`, since a consumer that stops asking for paths ends the walk itself.
-
-`path_re` and `skip_path_re` are regex filters on slash-separated relative paths. They filter returned paths or searched files, but do not control traversal. `skip_dir` uses glob syntax to prune matching directory subtrees, and `skip_dir_re` does the same with regex.
-
-`rg` and `rg_iter` return structured rows rather than raw CLI text. They accept the same `include`, `exclude`, `glob`, `ext`, `path_re`, `skip_path_re`, `skip_dir`, `skip_dir_re`, `min_depth`, `max_depth`, `max_filesize`, `follow_links`, and `same_file_system` filters as `fd`. Each row is a `SearchLine` with:
-
-```text
-kind         'match', 'before', 'after', or 'context'
-path         path relative to root
-line_number  1-based line number
-lnhash       exhash-style `lineno|hash|` address for the line
-line         line text without the trailing newline
-matches      list of (start, end) byte offsets for match rows
-```
-
-`rg`, `search_text`, and `search_path` return `SearchResults` by default, a list subclass whose `str()` and notebook pretty display are rg-style multiline text. `rg_iter` yields rows lazily.
-
-`SearchLine` has a structured `repr`, an rg-style `str` (the `line` is truncated to 180 chars with a trailing `…` for display; `repr` and `asdict()` keep the full line), and `SearchLine.asdict()` returns row fields as a plain Python dict. Pass `rg(..., lnhashs=True)` or `rg_iter(..., lnhashs=True)` to show `lnhash` addresses instead of line numbers in row display while keeping `line_number` available. `rg(..., paths=True)` returns unique matched paths, and `rg(..., count=True)` returns the total number of match spans. `paths` and `count` cannot both be set.
-
-`fd`, `walk`, `ls`, and `rg`/`nbrg` with `paths=True` return `PathResults`, a list of `FileEntry` rows. A `FileEntry` is a `str` subclass holding the relative path, so all string uses keep working, and it stats itself lazily on first access: `stat` is a cached `os.lstat` result (`None` if the path has vanished), with `size`, `mtime`, and `is_dir` derived from it. A `PathResults` displays as an `ls -l`-style long listing, capped at `rgapi.MAX_REPR` rows with a final `… N more` line, so stats are read only for displayed rows; `str()` is still one plain path per line, and `list(res)` shows plain paths. `rg(..., timeout_ms=200)` and `fd(..., timeout_ms=200)` stop at the deadline and return whatever was collected by then; `walk`, `ls`, and the async forms take it too. Results record how they ended: `stop_reason` is `None` for a complete result, `"max_results"` when truncated by `max_results`, or `"timeout"` when a deadline hit, and `complete` is true when `stop_reason` is `None`. `count=True` returns a plain int, which cannot carry the flag, so it rejects `timeout_ms`.
-
-`before_context`, `after_context`, and `context` are like `rg -B`, `rg -A`, and `rg -C`. Files containing NUL bytes or invalid UTF-8 are skipped.
-
-### Block summaries
-
-`rg(..., summary=True)` returns one row per blank-line-delimited block instead of one row per matching line. Empty and whitespace-only lines delimit blocks. A block containing several matching lines appears once and keeps every matching `SearchLine` in `matches`.
-
-```python
-rg("TODO", ".", summary=True, context=1, maxlen=120)
-```
-
-The result is `BlockResults`, a list of `SearchBlock` objects. Each block has `path`, `block_index`, `start_line`, `end_line`, `start_lnhash`, `end_lnhash`, `kind`, full `source`, and `matches`. Its display is `path:start-end:source` for matches and `path:start-end-source` for context. With `lnhashs=True`, the numeric range becomes copyable boundary addresses such as `path:4|a3f2|,6|b1c3|:source`. Embedded newline runs are shown as `¶`; `maxlen` limits displayed source without changing `source` or `asdict()`.
-
-In summary mode, `before_context`, `after_context`, and `context` count neighbouring blocks. `max_results` counts matching blocks and retains their block context. `summary=True` cannot be combined with `paths` or `count`; it can be combined with `lnhash` when copyable block boundaries are useful.
-
-Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behavior, or `case_sensitive=False` to force case-insensitive matching.
-
-## Notebooks
-
-`nbrg` searches Jupyter `.ipynb` files cell-by-cell, so results are *cells* rather than raw JSON lines, and each match is identified by its **cell id** (the nbformat cell/message id) rather than a line number. Searching a notebook with plain `rg` matches the escaped JSON text (including outputs and metadata) and reports meaningless JSON line numbers; `nbrg` instead searches each cell's reconstructed **source** and reports the cell id, which is stable across edits and points at the actual unit you work with.
-
-```python
-from rgapi import nbrg
-
-nbrg("read_csv", ".")                  # cells whose source matches, across all notebooks under "."
-nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
-```
-
-Notebooks are walked, parsed, and matched together in one parallel Rust pass, using the same regex engine as `rg`, so regex behaviour and the `case_sensitive`/`smart_case` flags match `rg`. Only cell `source` is searched, not outputs or metadata. `nbrg` accepts the same discovery filters as `fd`/`rg` (`include`, `exclude`, `glob`, `hidden`, `max_depth`, `skip_dir`, …).
-
-`nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
-
-```text
-path         notebook path relative to root
-cell_index   0-based position of the cell in the notebook
-cell_id      nbformat cell id (falls back to the cell index for notebooks without ids)
-cell_type    'code', 'markdown', or 'raw'
-kind         'match' or 'context'
-source       full cell source
-matches      list of SearchLine rows for the matched lines within the cell
-```
-
-`NbCell.asdict()` returns those fields as a plain dict (with `matches` as `SearchLine` dicts). `str()` and pretty display show one line per cell, newline runs shown as `¶`, keyed by `cell_id`: `path:cell_id:source` for matches and `path:cell_id-source` for context. A match row starts at its first matched line: earlier lines display as `…[Ln]` (`n` the matched line's 1-based number in the cell), except that a leading `#|` directive line is kept, e.g. `#| export…[L4]needle here`. `maxlen` controls the displayed source length and defaults to 120; the full source remains in `source` and `asdict()`. A cell with several matches appears once, with every hit collected in `matches`.
-
-`cell_context=N` includes the `N` cells before and after each matching cell as `kind="context"` rows (deduplicated per notebook).
-
-`nbrg_iter` yields `NbCell` rows lazily as notebooks are parsed. `nbrg` also accepts `max_results` (at most that many cells, after sorting by path and cell index), `count=True` (number of matching cells), and `timeout_ms=` with the same `stop_reason` semantics as `rg`.
-
-Notebook walking, parsing, and matching all happen in parallel in Rust, in the same pass as the file walk. Parsing uses a lean model that reads only each cell's `id`, `cell_type`, and `source` and skips outputs and metadata, so large embedded outputs (images, plots) are never materialized. `search_nb(pattern, path, ...)` searches a single notebook file the same way.
-
-`rgapi-nbrg` exposes notebook search without requiring a Python kernel:
-
-```bash
-rgapi-nbrg 'read_csv' .
-rgapi-nbrg 'read_csv' . --cell-context 1
-rgapi-nbrg 'read_csv' nbs --glob '*.ipynb' --max-results 20
-```
-
-Run `rgapi-nbrg --help` for its discovery, matching, and output options.
-
-## Async
-
-`fda`, `rga`, and `nbrga` are awaitable twins of `fd`, `rg`, and `nbrg`, and `fda_iter`, `rga_iter` and `nbrga_iter` are async generators that yield rows as the search finds them. All take the same arguments and return the same types as their sync counterparts.
-
-```python
-from rgapi import fda, rga, rga_iter
-
-await fda(".", ext="py")
-res = await rga("TODO", ".", timeout_ms=200)
-if not res.complete: print(f"partial results: {res.stop_reason}")
-async for row in rga_iter("TODO", "."): ...
-```
-
-None of this uses `asyncio.to_thread` or the loop's executor. The walk and search run on Rust threads, and a single callback settles the awaited future (or feeds the generator's queue) through `loop.call_soon_threadsafe`, so the event loop never blocks and contextvars behave normally.
-
-Cancellation cleans up the Rust workers automatically. Wrapping a call in `asyncio.wait_for` or `asyncio.timeout`, cancelling the task (as starlette does when a client disconnects), or leaving an `async for` early all stop the search within about one row. One caveat comes from the language rather than the library: `break` inside `async for` only finalizes the generator at GC time, so for prompt cleanup wrap the iterator in `contextlib.aclosing`:
-
-```python
-async with aclosing(rga_iter("TODO", ".")) as it:
-    async for row in it:
-        if enough(row): break
-```
-
-The streaming forms suit incremental display, such as pushing each batch of results to a browser as it arrives. The collected forms with `timeout_ms` give the best results available within a budget, and `asyncio.wait_for` gives timeout-as-failure. Pick per call site.
-
-
-## Benchmarks
-
-`tools/bench.py` compares the `rg` CLI with in-process `rgapi`. Run it against a release build. One run on this machine, using best time from seven repeats:
-
-| fixture | rg | rgapi |
-| --- | ---: | ---: |
-| 6 x 2 MB files, 2 matches | 6.54 ms | 1.44 ms |
-| 800 x 1.5 KB files, 2 matches | 13.90 ms | 10.94 ms |
-| tiny dir, repeated 30x | 5.92 ms | 2.14 ms |