rbtr-lang-rst 2026.9.0.dev1__tar.gz

rbtr_lang_rst-2026.9.0.dev1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Alejandro Giacometti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rbtr_lang_rst-2026.9.0.dev1/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: rbtr-lang-rst
+Version: 2026.9.0.dev1
+Summary: rbtr — reStructuredText language plugin
+Keywords: code-search,code-index,tree-sitter,static-analysis,semantic-search,developer-tools,rst,restructuredtext
+Author: Alejandro Giacometti
+Author-email: Alejandro Giacometti <alejandro.giacometti@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Typing :: Typed
+Requires-Dist: rbtr==2026.9.0.dev1
+Requires-Dist: tree-sitter-rst
+Requires-Python: >=3.13
+Project-URL: Homepage, https://github.com/janrito/rbtr
+Project-URL: Repository, https://github.com/janrito/rbtr
+Project-URL: Documentation, https://github.com/janrito/rbtr/tree/main/packages/rbtr-lang-rst#readme
+Project-URL: Issues, https://github.com/janrito/rbtr/issues
+Project-URL: Changelog, https://github.com/janrito/rbtr/releases
+Description-Content-Type: text/markdown
+
+# rbtr-lang-rst
+
+reStructuredText support for [rbtr]. A **default** plugin — installed
+with rbtr itself (`pip install rbtr`).
+
+[rbtr]: https://github.com/janrito/rbtr/tree/main/packages/rbtr#readme
+
+## What it ingests
+
+A custom chunker: it reconstructs the section hierarchy from the *order of
+adornment characters* (RST has no fixed heading levels), each section a chunk
+scoped by its ancestors. Cross-references are extracted as imports.
+
+- **Sections** — headings (underline and overline adornments) → doc sections,
+  with `scope` the `::` chain of enclosing sections. A document with no
+  heading at all — a README, a changelog — gives each top-level block its own
+  section instead: paragraphs, lists, directives, literal blocks, link
+  targets.
+- **References** — the `:func:` / `:class:` / `:meth:` / `:mod:` / `:doc:`
+  roles, `` `text <target>`_ `` references, and `.. toctree::` entries →
+  imports. External URLs and same-file anchors are skipped.
+
+## Chunks produced
+
+```rst
+Title              doc_section "Title"
+=====
+Section            doc_section "Section", scope "Title"
+-------
+See :func:`helper` and :doc:`api/index`.
+                   import {names: helper}; import {module: api/index}
+```
+
+## Embedded / injected chunks
+
+None. RST does not embed other languages.
+
+## Grammar & dependencies
+
+Uses the `tree-sitter-rst` grammar. No runtime dependency on other language
+plugins; the test suite dev-depends on `rbtr-lang-python` to extract the
+sample's `helpers.py` and snapshot the `.rst → .py` reference edges.

rbtr_lang_rst-2026.9.0.dev1/README.md

@@ -0,0 +1,42 @@
+# rbtr-lang-rst
+
+reStructuredText support for [rbtr]. A **default** plugin — installed
+with rbtr itself (`pip install rbtr`).
+
+[rbtr]: https://github.com/janrito/rbtr/tree/main/packages/rbtr#readme
+
+## What it ingests
+
+A custom chunker: it reconstructs the section hierarchy from the *order of
+adornment characters* (RST has no fixed heading levels), each section a chunk
+scoped by its ancestors. Cross-references are extracted as imports.
+
+- **Sections** — headings (underline and overline adornments) → doc sections,
+  with `scope` the `::` chain of enclosing sections. A document with no
+  heading at all — a README, a changelog — gives each top-level block its own
+  section instead: paragraphs, lists, directives, literal blocks, link
+  targets.
+- **References** — the `:func:` / `:class:` / `:meth:` / `:mod:` / `:doc:`
+  roles, `` `text <target>`_ `` references, and `.. toctree::` entries →
+  imports. External URLs and same-file anchors are skipped.
+
+## Chunks produced
+
+```rst
+Title              doc_section "Title"
+=====
+Section            doc_section "Section", scope "Title"
+-------
+See :func:`helper` and :doc:`api/index`.
+                   import {names: helper}; import {module: api/index}
+```
+
+## Embedded / injected chunks
+
+None. RST does not embed other languages.
+
+## Grammar & dependencies
+
+Uses the `tree-sitter-rst` grammar. No runtime dependency on other language
+plugins; the test suite dev-depends on `rbtr-lang-python` to extract the
+sample's `helpers.py` and snapshot the `.rst → .py` reference edges.

rbtr_lang_rst-2026.9.0.dev1/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = ["uv_build>=0.11.26,<1"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "rbtr_lang_rst"
+
+[tool.uv.sources.rbtr]
+workspace = true
+
+[tool.uv.sources.rbtr-lang-python]
+workspace = true
+
+[project]
+name = "rbtr-lang-rst"
+version = "2026.9.0-dev1"
+description = "rbtr — reStructuredText language plugin"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = [
+    "code-search",
+    "code-index",
+    "tree-sitter",
+    "static-analysis",
+    "semantic-search",
+    "developer-tools",
+    "rst",
+    "restructuredtext",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Text Processing :: Indexing",
+    "Typing :: Typed",
+]
+requires-python = ">=3.13"
+dependencies = [
+    "rbtr==2026.9.0-dev1",
+    "tree-sitter-rst",
+]
+
+[[project.authors]]
+name = "Alejandro Giacometti"
+email = "alejandro.giacometti@gmail.com"
+
+[project.urls]
+Homepage = "https://github.com/janrito/rbtr"
+Repository = "https://github.com/janrito/rbtr"
+Documentation = "https://github.com/janrito/rbtr/tree/main/packages/rbtr-lang-rst#readme"
+Issues = "https://github.com/janrito/rbtr/issues"
+Changelog = "https://github.com/janrito/rbtr/releases"
+
+[project.entry-points."rbtr.languages"]
+rst = "rbtr_lang_rst.plugin:rst"
+
+[dependency-groups]
+dev = [
+    "rbtr[test]",
+    "rbtr-lang-python",
+]

rbtr_lang_rst-2026.9.0.dev1/pyproject.toml.orig

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["uv_build>=0.11.26,<1"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "rbtr_lang_rst"
+# No source-exclude: the plugin's tests, samples, and snapshots ship with
+# the wheel (they are the language's golden record), and references.scm
+# ships as package data.
+
+[project]
+name = "rbtr-lang-rst"
+version = "2026.9.0-dev1"
+description = "rbtr — reStructuredText language plugin"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+  { name = "Alejandro Giacometti", email = "alejandro.giacometti@gmail.com" },
+]
+keywords = [
+  "code-search",
+  "code-index",
+  "tree-sitter",
+  "static-analysis",
+  "semantic-search",
+  "developer-tools",
+  "rst",
+  "restructuredtext",
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Libraries",
+  "Topic :: Text Processing :: Indexing",
+  "Typing :: Typed",
+]
+requires-python = ">=3.13"
+dependencies = [
+  "rbtr==2026.9.0-dev1",
+  "tree-sitter-rst",
+]
+
+[project.urls]
+Homepage = "https://github.com/janrito/rbtr"
+Repository = "https://github.com/janrito/rbtr"
+Documentation = "https://github.com/janrito/rbtr/tree/main/packages/rbtr-lang-rst#readme"
+Issues = "https://github.com/janrito/rbtr/issues"
+Changelog = "https://github.com/janrito/rbtr/releases"
+
+[project.entry-points."rbtr.languages"]
+# The value resolves to the language's `LanguageRegistration` (named by its
+# id) — see the rbtr ARCHITECTURE "External plugins" note.
+rst = "rbtr_lang_rst.plugin:rst"
+
+[tool.uv.sources]
+rbtr = { workspace = true }
+# The sample project includes a helpers.py the docs cross-reference, so the
+# sample test needs the python plugin to extract it and snapshot the edges.
+rbtr-lang-python = { workspace = true }
+
+[dependency-groups]
+dev = ["rbtr[test]", "rbtr-lang-python"]

rbtr_lang_rst-2026.9.0.dev1/src/rbtr_lang_rst/__init__.py

@@ -0,0 +1 @@
+"""reStructuredText language plugin package."""

rbtr_lang_rst-2026.9.0.dev1/src/rbtr_lang_rst/plugin.py

@@ -0,0 +1,359 @@
+"""RST language plugin.
+
+Splits reStructuredText by heading hierarchy using tree-sitter.
+RST sections are flat in the AST (title + adornment only), so
+the chunker reconstructs hierarchy from adornment character
+order (first seen = level 1, second = level 2, etc.).
+
+Extracted chunks::
+
+    Title                           → doc_section "Title", scope ""
+    =====                             (content: heading through next heading)
+
+    Section A                       → doc_section "Section A", scope "Title"
+    ---------                         (adornment char "-" = level 2)
+
+    First paragraph.                → doc_section "", scope ""  (no heading)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
+
+from tree_sitter import Parser, Query, QueryCursor
+
+from rbtr.domain.models import Chunk, ChunkKind, ImportMeta
+from rbtr.languages.chunks import last_line
+from rbtr.languages.registration import LanguageRegistration, load_query
+
+if TYPE_CHECKING:
+    from tree_sitter import Language, Node, Range
+
+# URL schemes that indicate external links.
+_EXTERNAL_SCHEMES = ("http://", "https://", "mailto:", "ftp://")
+
+# RST roles that reference a code symbol by name.
+_SYMBOL_ROLES = frozenset({":func:", ":class:", ":meth:", ":attr:"})
+# RST roles that reference a document/module by path.
+_PATH_ROLES = frozenset({":doc:", ":mod:"})
+
+# ── Chunking ─────────────────────────────────────────────────────────
+
+
+def _rst_title_text(section: Node) -> str:
+    """Extract the title text from an RST section node."""
+    for child in section.children:
+        if child.type == "title" and child.text:
+            return child.text.decode("utf-8", errors="replace")
+    return ""
+
+
+def _rst_adornment_char(section: Node) -> str:
+    """Return the adornment character of an RST section."""
+    for child in section.children:
+        if child.type == "adornment" and child.text:
+            return child.text.decode()[0]
+    return ""
+
+
+def chunk_rst(
+    file_path: str,
+    blob_sha: str,
+    content: str,
+    grammar: Language,
+    ranges: list[Range] | None = None,
+) -> Iterator[Chunk]:
+    """Split RST into sections, then extract its cross-references.
+
+    A document with headings is split by hierarchy, reconstructed from
+    adornment characters (first character seen = level 1, second = level
+    2, ...), each section running from its heading to the next
+    same-or-higher-level heading. A document with no heading has one
+    section per top-level block instead. References are extracted from
+    either.
+    """
+    if not content.strip():
+        return
+
+    content_bytes = content.encode("utf-8")
+    parser = Parser(grammar)
+    if ranges is not None:
+        parser.included_ranges = ranges
+    tree = parser.parse(content_bytes)
+    root = tree.root_node
+
+    children = [c for c in root.children if c.is_named]
+
+    if any(c.type == "section" for c in children):
+        yield from _sectioned_chunks(children, root, content_bytes, file_path, blob_sha)
+    else:
+        yield from _block_chunks(children, content_bytes, file_path, blob_sha)
+
+    yield from _extract_references(root, grammar, content_bytes, file_path, blob_sha)
+
+
+def _block_chunks(
+    children: list[Node],
+    content_bytes: bytes,
+    file_path: str,
+    blob_sha: str,
+) -> Iterator[Chunk]:
+    """One section per top-level block, for a document with no headings.
+
+    Every block a document can open with is content: a paragraph, a
+    bullet or enumerated list, a definition list, a directive, a literal
+    block, a comment, a footnote, a link target. None carries a heading
+    to be named by, so each is anonymous and located by its lines.
+    """
+    for child in children:
+        text = (
+            content_bytes[child.start_byte : child.end_byte]
+            .decode("utf-8", errors="replace")
+            .strip()
+        )
+        if not text:
+            continue
+        yield Chunk(
+            blob_sha=blob_sha,
+            file_path=file_path,
+            kind=ChunkKind.DOC_SECTION,
+            name="",
+            scope="",
+            content=text,
+            line_start=child.start_point[0] + 1,
+            line_end=last_line(child),
+        )
+
+
+def _sectioned_chunks(
+    children: list[Node],
+    root: Node,
+    content_bytes: bytes,
+    file_path: str,
+    blob_sha: str,
+) -> Iterator[Chunk]:
+    """One section per heading, scoped by the adornment hierarchy."""
+    # Build heading level map from adornment character order.
+    adornment_levels: dict[str, int] = {}
+    for child in children:
+        if child.type == "section":
+            char = _rst_adornment_char(child)
+            if char and char not in adornment_levels:
+                adornment_levels[char] = len(adornment_levels) + 1
+
+    # Walk siblings: associate content with preceding section.
+    chunks: list[Chunk] = []
+    scope_stack: list[tuple[str, int]] = []
+    current_title = ""
+    current_start = 0
+    current_line_start = 1
+    in_section = False
+
+    for child in children:
+        if child.type == "section":
+            # Close previous section if open.
+            if in_section:
+                text = (
+                    content_bytes[current_start : child.start_byte]
+                    .decode("utf-8", errors="replace")
+                    .strip()
+                )
+                if text:
+                    # Exclude the section being closed (the stack top) so
+                    # scope is the enclosing path only — never self. The
+                    # final-section close below does the same via [:-1].
+                    chunks.append(
+                        Chunk.model_validate(
+                            {
+                                "blob_sha": blob_sha,
+                                "file_path": file_path,
+                                "kind": ChunkKind.DOC_SECTION,
+                                "name": current_title,
+                                "scope": [t for t, _ in scope_stack[:-1]],
+                                "content": text,
+                                "line_start": current_line_start,
+                                "line_end": child.start_point[0],
+                            }
+                        )
+                    )
+
+            # Start new section.
+            title = _rst_title_text(child)
+            char = _rst_adornment_char(child)
+            level = adornment_levels.get(char, 0)
+
+            # Pop scope stack to parent level.
+            while scope_stack and scope_stack[-1][1] >= level:
+                scope_stack.pop()
+
+            current_title = title
+            current_start = child.start_byte
+            current_line_start = child.start_point[0] + 1
+            in_section = True
+
+            scope_stack.append((title, level))
+
+    # Close final section and yield all accumulated chunks.
+    if in_section:
+        text = content_bytes[current_start:].decode("utf-8", errors="replace").strip()
+        if text:
+            scope_stack_for_scope = scope_stack[:-1]
+            chunks.append(
+                Chunk.model_validate(
+                    {
+                        "blob_sha": blob_sha,
+                        "file_path": file_path,
+                        "kind": ChunkKind.DOC_SECTION,
+                        "name": current_title,
+                        "scope": [t for t, _ in scope_stack_for_scope],
+                        "content": text,
+                        "line_start": current_line_start,
+                        "line_end": last_line(root),
+                    }
+                )
+            )
+
+    yield from chunks
+
+
+# ── Reference extraction ─────────────────────────────────────────────
+
+_REF_QUERY = load_query(__package__, "references")
+
+
+def _strip_rst_target(role: str, raw: str) -> str:
+    """Clean an RST role target: strip backticks and ~ prefix.
+
+    For symbol roles (:func:, :class:, :meth:), `~` means
+    "show only the last component" — we extract the last
+    component as the symbol name.
+    """
+    target = raw.strip("`")
+    if target.startswith("~"):
+        target = target[1:]
+        # ~module.Class.method → method (last component)
+        if "." in target:
+            target = target.rsplit(".", 1)[1]
+    return target
+
+
+def _extract_references(
+    root: Node,
+    grammar: Language,
+    content_bytes: bytes,
+    file_path: str,
+    blob_sha: str,
+) -> Iterator[Chunk]:
+    """Extract RST cross-references as IMPORT chunks.
+
+    Uses tree-sitter queries on the already-parsed tree for:
+    - Domain roles (:func:, :class:, :doc:, etc.)
+    - Toctree directives
+    - Hyperlink references (`text <url>`_)
+    """
+    query = Query(grammar, _REF_QUERY)
+    for _, captures in QueryCursor(query).matches(root):
+        # Domain roles: :func:`name`, :doc:`path`, etc.
+        if "ref" in captures:
+            role_nodes = captures.get("role", [])
+            ref_nodes = captures["ref"]
+            if not role_nodes or not ref_nodes:
+                continue
+            role_text = role_nodes[0].text.decode() if role_nodes[0].text else ""
+            ref_node = ref_nodes[0]
+
+            # Get inner interpreted_text child (the target).
+            target_text = ""
+            for child in ref_node.children:
+                if child.type == "interpreted_text" and child is not role_nodes[0]:
+                    target_text = child.text.decode().strip("`") if child.text else ""
+                    break
+            if not target_text:
+                continue
+
+            target = _strip_rst_target(role_text, target_text)
+            meta = ImportMeta()
+            if role_text in _SYMBOL_ROLES:
+                meta.names = target
+            elif role_text in _PATH_ROLES:
+                meta.module = target
+            else:
+                continue
+
+            yield Chunk(
+                blob_sha=blob_sha,
+                file_path=file_path,
+                kind=ChunkKind.IMPORT,
+                name=f"{role_text}`{target_text}`",
+                scope="",
+                content=ref_node.text.decode() if ref_node.text else "",
+                metadata=meta,
+                line_start=ref_node.start_point[0] + 1,
+                line_end=last_line(ref_node),
+            )
+
+        # Toctree directives.
+        elif "directive" in captures:
+            type_nodes = captures.get("dir_type", [])
+            content_nodes = captures.get("content", [])
+            if not type_nodes or not content_nodes:
+                continue
+            dtype = type_nodes[0].text.decode() if type_nodes[0].text else ""
+            if dtype != "toctree":
+                continue
+            raw_content = content_nodes[0].text.decode() if content_nodes[0].text else ""
+            for line in raw_content.splitlines():
+                entry = line.strip()
+                if not entry:
+                    continue
+                yield Chunk(
+                    blob_sha=blob_sha,
+                    file_path=file_path,
+                    kind=ChunkKind.IMPORT,
+                    name=f"toctree:{entry}",
+                    scope="",
+                    content=entry,
+                    metadata=ImportMeta(module=entry),
+                    line_start=content_nodes[0].start_point[0] + 1,
+                    line_end=last_line(content_nodes[0]),
+                )
+
+        # Hyperlink references: `text <url>`_
+        elif "hyperlink" in captures:
+            for ref_node in captures["hyperlink"]:
+                if ref_node.text is None:
+                    continue
+                text = ref_node.text.decode()
+                # Extract URL from between < and > in node text.
+                lt = text.rfind("<")
+                gt = text.rfind(">")
+                if lt == -1 or gt == -1 or gt <= lt:
+                    continue
+                url = text[lt + 1 : gt]
+                if any(url.startswith(s) for s in _EXTERNAL_SCHEMES):
+                    continue
+                yield Chunk(
+                    blob_sha=blob_sha,
+                    file_path=file_path,
+                    kind=ChunkKind.IMPORT,
+                    name=text,
+                    scope="",
+                    content=text,
+                    metadata=ImportMeta(module=url),
+                    line_start=ref_node.start_point[0] + 1,
+                    line_end=last_line(ref_node),
+                )
+
+
+# ── Plugin ───────────────────────────────────────────────────────────
+
+
+rst = LanguageRegistration(
+    id="rst",
+    extensions=frozenset({".rst"}),
+    grammar_module="tree_sitter_rst",
+    extraction_serial=4,
+)
+
+rst.chunker(chunk_rst)

rbtr_lang_rst-2026.9.0.dev1/src/rbtr_lang_rst/references.scm

@@ -0,0 +1,5 @@
+(interpreted_text (role) @role) @ref
+
+(directive (type) @dir_type (body (content) @content)) @directive
+
+(reference) @hyperlink

rbtr_lang_rst-2026.9.0.dev1/src/rbtr_lang_rst/tests/__snapshots__/test_samples/test_edges_match_snapshot.json

@@ -0,0 +1,8 @@
+[
+  "rst.rst:::class:`Greeter` -> helpers.py::Greeter [documents]",
+  "rst.rst:::doc:`config` -> config.rst::Config [documents]",
+  "rst.rst:::doc:`topics/intro` -> topics/intro.rst::Intro [documents]",
+  "rst.rst:::func:`format_greeting` -> helpers.py::format_greeting [documents]",
+  "rst.rst::`the guide <guide.rst>`_ -> guide.rst::Guide [documents]",
+  "topics/intro.rst:::doc:`detail` -> topics/detail.rst::Detail [documents]"
+]