rgapi 0.1.8__tar.gz → 0.1.9__tar.gz

{rgapi-0.1.8 → rgapi-0.1.9}/Cargo.lock

@@ -13,13 +13,13 @@ dependencies = [
 
 [[package]]
 name = "bstr"
-version = "1.12.1"
+version = "1.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"
+checksum = "a4e308e67087593070530a666f7fe8815cbd2e61d8f5b7313b71b7d721d1dfde"
 dependencies = [
  "memchr",
  "regex-automata",
- "serde",
+ "serde_core",
 ]
 
 [[package]]
@@ -276,7 +276,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "rgapi"
-version = "0.1.8"
+version = "0.1.9"
 dependencies = [
  "globset",
  "grep-matcher",
@@ -295,15 +295,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "serde"
-version = "1.0.228"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
-dependencies = [
- "serde_core",
-]
-
 [[package]]
 name = "serde_core"
 version = "1.0.228"

{rgapi-0.1.8 → rgapi-0.1.9}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rgapi"
-version = "0.1.8"
+version = "0.1.9"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "Python API for ripgrep-style file walking and searching"

{rgapi-0.1.8 → rgapi-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rgapi
-Version: 0.1.8
+Version: 0.1.9
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Requires-Dist: fastship>=0.0.12 ; extra == 'dev'
@@ -35,6 +35,14 @@ for row in rg_iter("TODO", ".", include="*.py", context=2): print(row.asdict())
 rg("TODO", ".", ext="py", skip_dir=".venv", paths=True)
 ```
 
+For cell-aware search of Jupyter notebooks (see [Notebooks](#notebooks)):
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".", cell_context=1)
+```
+
 For direct access to the regex, search, and walk pieces:
 
 ```python
@@ -82,6 +90,35 @@ matches      list of (start, end) byte offsets for match rows
 
 Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behavior, or `case_sensitive=False` to force case-insensitive matching.
 
+## Notebooks
+
+`nbrg` searches Jupyter `.ipynb` files cell-by-cell, so results are *cells* rather than raw JSON lines. Searching a notebook with plain `rg` matches the escaped JSON text (including outputs and metadata) and reports JSON line numbers; `nbrg` instead searches each cell's reconstructed **source** and returns the cells that matched.
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".")                  # cells whose source matches, across all notebooks under "."
+nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
+```
+
+It finds notebooks with the fast parallel Rust walker (`fd`), then matches each cell's source through the same Rust engine as `rg` (via `search_text`), so regex behaviour and the `case_sensitive`/`smart_case` flags match `rg`. Only cell `source` is searched, not outputs or metadata. `nbrg` accepts the same discovery filters as `fd`/`rg` (`include`, `exclude`, `glob`, `hidden`, `max_depth`, `skip_dir`, …).
+
+`nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
+
+```text
+path         notebook path relative to root
+cell_index   0-based position of the cell in the notebook
+cell_id      nbformat cell id (falls back to the cell index for notebooks without ids)
+cell_type    'code', 'markdown', or 'raw'
+kind         'match' or 'context'
+source       full cell source
+matches      list of SearchLine rows for the matched lines within the cell
+```
+
+`NbCell.asdict()` returns those fields as a plain dict (with `matches` as `SearchLine` dicts). `str()`/pretty display is one truncated, newline-escaped line per cell: `path:cell_id:source` for matches and `path:cell_id-source` for context cells. A cell with several matches appears once, with every hit collected in `matches`.
+
+`cell_context=N` includes the `N` cells before and after each matching cell as `kind="context"` rows (deduplicated per notebook). `prefilter=True` first narrows candidate notebooks with `rg` before the per-cell search; it is faster on notebook-heavy trees but can miss matches whose pattern is affected by JSON escaping (quotes, backslashes, regex metacharacters straddling escapes), so it is off by default.
+
 ## Benchmarks
 
 `tools/bench.py` compares the `rg` CLI with in-process `rgapi`. Run it against a release build. One run on this machine, using best time from seven repeats:

{rgapi-0.1.8 → rgapi-0.1.9}/README.md

@@ -16,6 +16,14 @@ for row in rg_iter("TODO", ".", include="*.py", context=2): print(row.asdict())
 rg("TODO", ".", ext="py", skip_dir=".venv", paths=True)
 ```
 
+For cell-aware search of Jupyter notebooks (see [Notebooks](#notebooks)):
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".", cell_context=1)
+```
+
 For direct access to the regex, search, and walk pieces:
 
 ```python
@@ -63,6 +71,35 @@ matches      list of (start, end) byte offsets for match rows
 
 Search is case-sensitive by default, matching `rg`. Use `smart_case=True` for `rg --smart-case` behavior, or `case_sensitive=False` to force case-insensitive matching.
 
+## Notebooks
+
+`nbrg` searches Jupyter `.ipynb` files cell-by-cell, so results are *cells* rather than raw JSON lines. Searching a notebook with plain `rg` matches the escaped JSON text (including outputs and metadata) and reports JSON line numbers; `nbrg` instead searches each cell's reconstructed **source** and returns the cells that matched.
+
+```python
+from rgapi import nbrg
+
+nbrg("read_csv", ".")                  # cells whose source matches, across all notebooks under "."
+nbrg("read_csv", ".", cell_context=1)  # also include neighbouring cells as context
+```
+
+It finds notebooks with the fast parallel Rust walker (`fd`), then matches each cell's source through the same Rust engine as `rg` (via `search_text`), so regex behaviour and the `case_sensitive`/`smart_case` flags match `rg`. Only cell `source` is searched, not outputs or metadata. `nbrg` accepts the same discovery filters as `fd`/`rg` (`include`, `exclude`, `glob`, `hidden`, `max_depth`, `skip_dir`, …).
+
+`nbrg` returns `NbResults`, a list of `NbCell`. Each `NbCell` has:
+
+```text
+path         notebook path relative to root
+cell_index   0-based position of the cell in the notebook
+cell_id      nbformat cell id (falls back to the cell index for notebooks without ids)
+cell_type    'code', 'markdown', or 'raw'
+kind         'match' or 'context'
+source       full cell source
+matches      list of SearchLine rows for the matched lines within the cell
+```
+
+`NbCell.asdict()` returns those fields as a plain dict (with `matches` as `SearchLine` dicts). `str()`/pretty display is one truncated, newline-escaped line per cell: `path:cell_id:source` for matches and `path:cell_id-source` for context cells. A cell with several matches appears once, with every hit collected in `matches`.
+
+`cell_context=N` includes the `N` cells before and after each matching cell as `kind="context"` rows (deduplicated per notebook). `prefilter=True` first narrows candidate notebooks with `rg` before the per-cell search; it is faster on notebook-heavy trees but can miss matches whose pattern is affected by JSON escaping (quotes, backslashes, regex metacharacters straddling escapes), so it is off by default.
+
 ## Benchmarks
 
 `tools/bench.py` compares the `rg` CLI with in-process `rgapi`. Run it against a release build. One run on this machine, using best time from seven repeats:

{rgapi-0.1.8 → rgapi-0.1.9}/python/rgapi/__init__.py

@@ -196,4 +196,6 @@ def search_path(
     return SearchResults(_core.search_path(matcher, _fs_path(path), _display_path(display_path), before_context, after_context))
 
 
-__all__ = [ "RgIter", "fd", "rg", "rg_iter" ]
+from .nb import NbCell, NbResults, nbrg, search_nb
+
+__all__ = [ "RgIter", "fd", "rg", "rg_iter", "nbrg" ]

rgapi-0.1.9/python/rgapi/nb.py

@@ -0,0 +1,96 @@
+"Notebook-aware search: run `rg`-style search over `.ipynb` files, returning matched cells instead of raw JSON lines."
+
+import json
+from pathlib import Path
+from fastcore.meta import delegates
+
+from . import compile, fd, rg, search_text
+
+
+def _cell_source(cell):
+    src = cell.get("source", "")
+    return "".join(src) if isinstance(src, list) else src
+
+def _preview(text, width=120):
+    text = text.rstrip("\n").replace("\n", "\\n")
+    return text if len(text) <= width else text[:width] + "…"
+
+
+class NbCell:
+    "A notebook cell that matched (or provides context for) a search."
+    def __init__(self, path, cell_index, cell_id, cell_type, kind, source, matches):
+        self.path,self.cell_index,self.cell_id = path,cell_index,cell_id
+        self.cell_type,self.kind,self.source,self.matches = cell_type,kind,source,matches
+
+    def asdict(self):
+        return dict(path=self.path, cell_index=self.cell_index, cell_id=self.cell_id,
+            cell_type=self.cell_type, kind=self.kind, source=self.source,
+            matches=[m.asdict() for m in self.matches])
+
+    def __repr__(self):
+        return (f"NbCell(path={self.path!r}, cell_id={self.cell_id!r}, cell_type={self.cell_type!r}, "
+                f"kind={self.kind!r}, matches={len(self.matches)})")
+
+    def __str__(self):
+        sep = ":" if self.kind == "match" else "-"
+        return f"{self.path}:{self.cell_id}{sep}{_preview(self.source)}"
+
+    def _repr_pretty_(self, p, cycle): p.text("..." if cycle else str(self))
+
+
+class NbResults(list):
+    "List of `NbCell` rows with rg-style text display."
+    def __str__(self): return "\n".join(map(str, self))
+    def _repr_pretty_(self, p, cycle): p.text("..." if cycle else str(self))
+
+
+def search_nb(
+    matcher,                 # Compiled `Regex` from `compile()`
+    path,                    # Notebook file to search (expands `~`)
+    cell_context:int=0,      # Cells of context to include before/after each matching cell
+    display_path:str=None    # Path stored in results; defaults to `path`
+) -> NbResults:
+    "Search one `.ipynb` file's cell sources with a compiled matcher."
+    disp = display_path if display_path is not None else str(path)
+    try: nb = json.loads(Path(path).expanduser().read_text())
+    except (ValueError, OSError, UnicodeDecodeError): return NbResults()
+    cells = nb.get("cells")
+    if not isinstance(cells, list): return NbResults()
+    info,matched = {},{}
+    for i,cell in enumerate(cells):
+        if not isinstance(cell, dict): continue
+        src = _cell_source(cell)
+        info[i] = (str(cell.get("id", i)), cell.get("cell_type", ""), src)
+        hits = [m for m in search_text(matcher, src, path=disp) if m.kind == "match"]
+        if hits: matched[i] = hits
+    if not matched: return NbResults()
+    emit = {i:"match" for i in matched}
+    if cell_context:
+        for i in matched:
+            for j in range(max(0, i-cell_context), min(len(cells), i+cell_context+1)): emit.setdefault(j, "context")
+    res = NbResults()
+    for i in sorted(emit):
+        if i not in info: continue
+        cid,ctype,src = info[i]
+        res.append(NbCell(disp, i, cid, ctype, emit[i], src, matched.get(i, [])))
+    return res
+
+
+@delegates(fd, but=["ext", "files", "dirs"])
+def nbrg(
+    pattern:str,                  # Regex pattern to search for
+    root:str=".",                 # Directory to search (expands `~`)
+    prefilter:bool=False,         # Pre-narrow notebooks with `rg` before per-cell search (may miss escape-affected patterns)
+    cell_context:int=0,           # Cells of context to include before/after each matching cell
+    case_sensitive:bool|None=None,# True/False forces case; None allows `smart_case`
+    smart_case:bool=False,        # Match `rg --smart-case` behavior
+    **kwargs
+) -> NbResults:
+    "Search `.ipynb` cell sources under `root`, returning matched cells."
+    matcher = compile(pattern, case_sensitive=case_sensitive, smart_case=smart_case)
+    if prefilter: paths = rg(pattern, root, ext="ipynb", case_sensitive=case_sensitive, smart_case=smart_case, paths=True, **kwargs)
+    else: paths = fd(root, ext="ipynb", **kwargs)
+    base = Path(root).expanduser()
+    res = NbResults()
+    for rel in sorted(paths): res += search_nb(matcher, base/rel, cell_context=cell_context, display_path=rel)
+    return res

{rgapi-0.1.8 → rgapi-0.1.9}/python/rgapi/skill.py

@@ -5,6 +5,7 @@ rgapi wraps the same `ignore`, `grep-regex`, and `grep-searcher` crates ripgrep
 Core APIs:
 - `fd(root=".", ...)` finds paths with fd-style filters (`pattern` substring, `include`/`exclude`/`glob`, `ext`); returns slash-separated relative paths.
 - `rg(pattern, root=".", ...)` searches and returns `SearchResults` (or `paths=True` for unique matched paths, `count=True` for a match-span total). NB: The `SearchResults` repr shows an rg-style multiline string, which is usually the most ergonomic approach.
+- `nbrg(pattern, root=".", cell_context=0, ...)` searches Jupyter `.ipynb` files (cell source only) and returns matched cells as `NbResults`/`NbCell`, which have a nice repr like rg(). Use this for notebooks rather than `rg`, to avoid escaping problems, and to get back message IDs.
 
 SearchLine rows:
   kind         'match', 'before', 'after', or 'context'
@@ -14,11 +15,20 @@ SearchLine rows:
   matches      list of (start, end) byte offsets, for 'match' rows
   asdict()     returns the row fields as a plain dict
 
+NbCell rows (from `nbrg`):
+  path/cell_index/cell_id/cell_type    locate the cell ('code'/'markdown'/'raw')
+  kind         'match' or 'context'
+  source       full cell source
+  matches      list of SearchLine rows for the matched lines within the cell
+  asdict()     returns the cell fields as a plain dict
+`cell_context=N` adds N neighbour cells as 'context'. `prefilter=True` narrows notebooks with `rg` first (faster, but can miss patterns affected by JSON escaping; off by default).
+
 Important:
 Traversal is parallel and result order is NOT guaranteed; wrap in `sorted(...)` if you need stable order. `path_re`/`skip_path_re` filter the returned/searched paths but do not prune traversal; use `skip_dir`/`skip_dir_re` to prune whole subtrees for speed. Run `doc(func)` for full parameter docments.
 """
 
 from . import Regex, RgIter, SearchLine, SearchResults, compile, fd, rg, rg_iter, search_path, search_text, walk
+from . import NbCell, NbResults, nbrg, search_nb
 
-__all__ = ["Regex", "RgIter", "SearchLine", "SearchResults", "compile", "fd", "rg", "rg_iter", "search_path", "search_text", "walk"]
+__all__ = [ "RgIter", "fd", "rg", "rg_iter", "nbrg" ]
 

{rgapi-0.1.8 → rgapi-0.1.9}/tests/test_rgapi.py

@@ -1,4 +1,4 @@
-import _thread, threading
+import _thread, json, threading
 
 import pytest
 
@@ -188,3 +188,107 @@ def test_direct_regex_and_search_apis(tmp_path):
     assert [(r.kind, r.path, r.line_number, r.line, r.matches) for r in path_res] == [
         ("match", "display.py", 2, "TODO here", [(0, 4)])]
     assert path_res[0].asdict() == dict(kind="match", path="display.py", line_number=2, line="TODO here", matches=[(0, 4)])
+
+
+def write_nb(path, cells):
+    nb = {"cells": cells, "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+    path.write_text(json.dumps(nb))
+
+def _cell(cell_type, source, cid=None, outputs=None):
+    c = {"cell_type": cell_type, "metadata": {}, "source": source}
+    if cell_type == "code":
+        c["execution_count"] = None
+        c["outputs"] = outputs or []
+    if cid is not None: c["id"] = cid
+    return c
+
+
+def test_nbrg_source_only_with_cells(tmp_path):
+    from rgapi import nbrg
+    write_nb(tmp_path / "nb.ipynb", [
+        _cell("code", ["import os\n", "def foo():\n", "    return 1\n"], cid="c1",
+              outputs=[{"output_type": "stream", "name": "stdout", "text": ["foo ran\n"]}]),
+        _cell("code", ["print('hi')\n"], cid="c2",
+              outputs=[{"output_type": "stream", "name": "stdout", "text": ["foo in output\n"]}]),
+        _cell("markdown", ["# Title\n", "use foo here\n"], cid="m1"),
+    ])
+    res = nbrg("foo", str(tmp_path))
+    # c2 has 'foo' only in its output, so source-only search must skip it
+    assert {c.cell_id for c in res} == {"c1", "m1"}
+    c1 = next(c for c in res if c.cell_id == "c1")
+    assert c1.cell_type == "code" and c1.kind == "match"
+    assert [m.line_number for m in c1.matches] == [2]
+    assert c1.matches[0].line == "def foo():"
+    assert c1.matches[0].matches == [(4, 7)]
+    m1 = next(c for c in res if c.cell_id == "m1")
+    assert m1.cell_type == "markdown"
+    assert [m.line for m in m1.matches] == ["use foo here"]
+
+
+def test_nbrg_string_source_and_missing_id(tmp_path):
+    from rgapi import nbrg
+    write_nb(tmp_path / "nb.ipynb", [_cell("code", "x = 1\nfoo = 2\n")])
+    res = nbrg("foo", str(tmp_path))
+    assert len(res) == 1
+    assert res[0].cell_id == "0"
+    assert res[0].matches[0].line == "foo = 2"
+    assert res[0].matches[0].line_number == 2
+
+
+def test_nbrg_multiple_matches_single_cell_appears_once(tmp_path):
+    from rgapi import nbrg
+    write_nb(tmp_path / "nb.ipynb", [
+        _cell("code", ["foo = 1\n", "bar = 2\n", "baz = foo + foo\n"], cid="c1"),
+    ])
+    res = nbrg("foo", str(tmp_path))
+    assert len(res) == 1                                     # cell appears once, not once per match
+    assert [c.cell_id for c in res] == ["c1"]
+    cell = res[0]
+    assert [m.line_number for m in cell.matches] == [1, 3]   # both matching lines kept
+    assert cell.matches[1].matches == [(6, 9), (12, 15)]     # two spans on the same line
+
+
+def test_nbrg_cell_context(tmp_path):
+    from rgapi import nbrg
+    write_nb(tmp_path / "nb.ipynb", [
+        _cell("code", ["a = 1\n"], cid="c0"),
+        _cell("code", ["target = 2\n"], cid="c1"),
+        _cell("code", ["b = 3\n"], cid="c2"),
+        _cell("code", ["c = 4\n"], cid="c3"),
+    ])
+    res = nbrg("target", str(tmp_path), cell_context=1)
+    kinds = {c.cell_id: c.kind for c in res}
+    assert kinds == {"c0": "context", "c1": "match", "c2": "context"}
+
+
+def test_nbrg_skips_bad_json_and_prefilter_parity(tmp_path):
+    from rgapi import nbrg
+    write_nb(tmp_path / "good.ipynb", [_cell("code", ["read_csv(x)\n"], cid="g1")])
+    (tmp_path / "bad.ipynb").write_text("{not valid json")
+    a = nbrg("read_csv", str(tmp_path))
+    b = nbrg("read_csv", str(tmp_path), prefilter=True)
+    assert {c.cell_id for c in a} == {"g1"}
+    assert {c.cell_id for c in b} == {"g1"}
+
+
+def test_search_nb_single_and_asdict_str(tmp_path):
+    from rgapi import compile, search_nb
+    p = tmp_path / "nb.ipynb"
+    write_nb(p, [_cell("code", ["def foo():\n", "    return 1\n"], cid="c1")])
+    res = search_nb(compile("foo"), p, display_path="nb.ipynb")
+    assert len(res) == 1
+    cell = res[0]
+    assert str(cell) == "nb.ipynb:c1:def foo():\\n    return 1"   # cell-oriented: whole cell, newlines escaped
+    d = cell.asdict()
+    assert d["cell_id"] == "c1" and d["cell_type"] == "code" and d["kind"] == "match"
+    assert d["matches"][0]["line"] == "def foo():"
+
+
+def test_nbcell_str_truncates_and_escapes(tmp_path):
+    from rgapi import compile, search_nb
+    p = tmp_path / "nb.ipynb"
+    write_nb(p, [_cell("code", ["x = '" + "a" * 500 + "'  # foo\n"], cid="c1")])
+    s = str(search_nb(compile("foo"), p, display_path="nb.ipynb")[0])
+    assert "\n" not in s            # newlines escaped to a single display line
+    assert s.endswith("…")          # long cell is truncated
+    assert s.startswith("nb.ipynb:c1:")