sphinx-codelinks 1.3.0__tar.gz → 1.4.0__tar.gz

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sphinx-codelinks
-Version: 1.3.0
+Version: 1.4.0
 Summary: Fast Source Code Traceability for Sphinx-Needs
 Author-email: team useblocks <info@useblocks.com>
 Maintainer-email: Marco Heinemann <marco.heinemann@useblocks.com>, Jui-Wen Chen <jui-wen.chen@useblocks.com>
@@ -9,7 +9,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: comment-parser>=1.2.4
 Requires-Dist: ignore-python>=0.3.3
-Requires-Dist: typer>=0.16.0
+Requires-Dist: typer>=0.16.0,<0.26.8
 Requires-Dist: click < 8.2
 Requires-Dist: jsonschema
 Requires-Dist: sphinx>=7.4,<10
@@ -26,6 +26,9 @@ Requires-Dist: tree-sitter-yaml>=0.7.1
 Requires-Dist: tree-sitter-rust>=0.23.0
 Requires-Dist: tree-sitter-go>=0.23.0
 Requires-Dist: tree-sitter-json>=0.24.8
+Requires-Dist: tree-sitter-bash>=0.25.1
+Requires-Dist: libclang>=16 ; extra == "libclang"
+Provides-Extra: libclang
 
 # Sphinx CodeLinks
 

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sphinx-codelinks"
-version = "1.3.0"
+version = "1.4.0"
 description = "Fast Source Code Traceability for Sphinx-Needs"
 authors = [{ name = "team useblocks", email = "info@useblocks.com" }]
 maintainers = [
@@ -13,7 +13,7 @@ requires-python = ">= 3.12"
 dependencies = [
   "comment-parser>=1.2.4",
   "ignore-python>=0.3.3",
-  "typer>=0.16.0",
+  "typer>=0.16.0,<0.26.8",      # 0.26.8 removed rich_utils.STYLE_METAVAR, still imported by sphinxcontrib-typer (docs)
   "click < 8.2",                # click 8.2.* produces empty errors if no args are given
   "jsonschema",
   "sphinx>=7.4,<10",
@@ -32,8 +32,19 @@ dependencies = [
   "tree-sitter-rust>=0.23.0",
   "tree-sitter-go>=0.23.0",
   "tree-sitter-json>=0.24.8",
+  "tree-sitter-bash>=0.25.1",
 ]
 
+[project.optional-dependencies]
+# libclang enables the preprocessor-aware C/C++ engine (skips comments inside
+# inactive #if/#ifdef regions). Plain tree-sitter C/C++ extraction needs none of
+# this, so libclang stays optional. Named after the backend, not "cpp": the
+# engine handles C and Objective-C too, and tree-sitter already covers C++.
+# Floor 16 (not 18): the extraction snapshots are byte-identical on libclang 16,
+# 17 and 18 (the engine only uses get_all_skipped_ranges + comment tokenization,
+# stable since long before 16), so support older toolchains too. No upper bound.
+libclang = ["libclang>=16"]
+
 [build-system]
 requires = ["flit_core >=3.4,<4"]
 build-backend = "flit_core.buildapi"
@@ -48,13 +59,14 @@ testing = [
   "moto ~= 5.0",
   "toml>=0.10.2",
   "furo>=2024.5.6",
+  "libclang>=18",
 ]
 docs = [
   "furo>=2024.5.6",
   "myst-parser>=4.0.0",
   "sphinx-design>=0.6.1",
   "sphinx-code-tabs>=0.5.5",
-  "sphinxcontrib-typer>=0.5.1",
+  "sphinxcontrib-typer>=0.5.1,<0.9.1", # 0.9.1 imports typer.rich_utils.STYLE_TYPES, absent in the capped typer <0.26.8, breaking the docs build
   "sphinxcontrib-video>=0.4.1",
 ]
 mypy = [
@@ -106,8 +118,11 @@ force-sort-within-sections = true
 "**/tests/*" = [
   "ARG001",  # unused-function-argument - fixtures
   "ARG005",  # unused-lambda-argument - monkeypatches
+  "E402",    # module-import-not-at-top - pytest.importorskip guard pattern
+  "PLC0415", # import-outside-top-level - inline imports in test bodies
   "PLR2004", # magic-value-comparison - valueable for tests
   "S101",    # assert - needed for tests
+  "SIM300",  # yoda-conditions - assert order in tests is intentional
 ]
 "src/sphinx_codelinks/sphinx_extension/debug.py" = [
   "T201",  # print - used for output
@@ -140,7 +155,8 @@ plugins = ["pydantic.mypy"]
 mypy_path = "typings"
 
 [[tool.mypy.overrides]]
-module = ["licensing.*", "tomlkit.*"]
+# clang.cindex ships no type stubs / py.typed; the libclang engine wraps it.
+module = ["licensing.*", "tomlkit.*", "clang.*"]
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/analyse/analyse.py

@@ -2,7 +2,7 @@ from collections.abc import Generator
 from dataclasses import dataclass
 import json
 from pathlib import Path
-from typing import Any, TypedDict
+from typing import Any, TypedDict, cast
 
 from tree_sitter import Node as TreeSitterNode
 
@@ -26,6 +26,7 @@ from sphinx_codelinks.config import (
     SourceAnalyseConfig,
 )
 from sphinx_codelinks.logger import get_logger
+from sphinx_codelinks.source_discover.config import CommentType
 
 logger = get_logger(__name__)
 
@@ -78,10 +79,11 @@ class SourceAnalyse:
         self.git_commit_rev: str | None = (
             utils.get_current_rev(self.git_root) if self.git_root else None
         )
-        self.project_path: Path = (
-            self.git_root if self.git_root else self.analyse_config.src_dir
-        )
+        self.project_path: Path = self.git_root or self.analyse_config.src_dir
         self.oneline_warnings: list[AnalyseWarning] = []
+        # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
+        # database is read once per run instead of once per source file.
+        self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
 
     def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]:  # type: ignore[explicit-any]
         """Load source files and extract their content."""
@@ -112,6 +114,127 @@ class SourceAnalyse:
             self.src_files.append(src_file)
             self.src_comments.extend(src_comments)
 
+    def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
+        """Return the parsed compile_commands.json for ``db_path``, memoized.
+
+        The database is read and parsed once per run instead of once per source
+        file (O(files x entries) -> O(entries)). A present-but-malformed or
+        unreadable database is warned once and cached as ``None`` so callers fall
+        back to the configured defines without re-reading or re-warning per file.
+        """
+        from sphinx_codelinks.analyse.preproc import compile_db  # noqa: PLC0415
+
+        if db_path in self._flags_map_cache:
+            return self._flags_map_cache[db_path]
+        try:
+            flags = compile_db.load_flags_map(db_path)
+        except (OSError, ValueError, TypeError) as exc:
+            logger.warning(
+                f"codelinks: failed to read {db_path} ({exc}); "
+                f"falling back to the configured defines"
+            )
+            self._flags_map_cache[db_path] = None
+            return None
+        self._flags_map_cache[db_path] = flags
+        return flags
+
+    def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
+        from sphinx_codelinks.analyse.preproc import compile_db  # noqa: PLC0415
+
+        # `run()` calls this (via create_src_objects_libclang) only when
+        # `preprocessor is not None`, but keep the guard: it narrows the type for
+        # the checker and is a cheap defense (an `assert` would trip bandit S101).
+        preproc = self.analyse_config.preprocessor
+        if preproc is None:
+            return []
+        db_path = preproc.compile_commands
+        if db_path is None:
+            db_path = compile_db.find_compile_db(src_path, self.project_path)
+        if db_path is not None and db_path.is_file():
+            flags = self._flags_map_for(db_path)
+            if flags is None:
+                # Present-but-malformed/unreadable DB (warned once in the helper):
+                # fall back to the configured defines so extraction still runs.
+                return compile_db.defines_to_args(
+                    preproc.defines, preproc.includes, preproc.std
+                )
+            args = flags.get(src_path.absolute().resolve())
+            if args is not None:
+                return args
+            # Absent from the DB. compile_commands.json lists only compiled
+            # translation units, never headers — so a header here is parsed
+            # standalone with the global defines (one run = one variant). A
+            # compiled source absent from the build is skipped (spec §3.3).
+            if compile_db.is_translation_unit_source(src_path):
+                return None
+            return compile_db.defines_to_args(
+                preproc.defines, preproc.includes, preproc.std
+            )
+        # No readable DB. `find_compile_db` only ever returns a real file, so a
+        # non-None `db_path` that reaches here is an explicit `compile_commands`
+        # path that is not a readable file (typo/missing): warn and fall back
+        # rather than skip silently. A genuinely absent DB (db_path is None) just
+        # falls back to the manual defines applied globally.
+        if db_path is not None:
+            logger.warning(
+                f"codelinks: compile_commands path {db_path} is not a readable "
+                f"file; falling back to the configured defines"
+            )
+        return compile_db.defines_to_args(
+            preproc.defines, preproc.includes, preproc.std
+        )
+
+    # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
+    def create_src_objects_libclang(self) -> None:
+        from sphinx_codelinks.analyse.preproc import (  # noqa: PLC0415
+            libclang_parser,
+            loader,
+        )
+
+        # Resolve the exception via the loader (never a direct ``import
+        # clang.cindex``) so a missing ``libclang`` extra still surfaces the
+        # loader's friendly install hint rather than a bare ImportError.
+        translation_unit_load_error = (
+            loader.load_clang_cindex().TranslationUnitLoadError
+        )
+
+        for src_path in self.analyse_config.src_files:
+            if not utils.is_text_file(src_path):
+                continue
+            args = self._resolve_preproc_args(src_path)
+            if args is None:
+                logger.debug(
+                    f"codelinks: skipping {src_path} — not found in compile_commands.json"
+                )
+                continue
+            try:
+                comments = libclang_parser.extract_active_comments(src_path, args)
+            except translation_unit_load_error:
+                # Last-resort guard. Standalone parses pin ``-x`` to match the
+                # ``-std`` (see defines_to_args), so the common case — a ``.h``/
+                # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
+                # markers extract. This only fires if libclang still cannot load
+                # the file as a translation unit at all; skip it rather than
+                # aborting the whole run. Surfaced as a ``warning`` so the silent
+                # data-loss (a file's markers dropped) is visible — accepting that
+                # this fails ``sphinx-build -W``.
+                logger.warning(
+                    f"codelinks: skipping {src_path} — libclang could not load it "
+                    f"as a translation unit"
+                )
+                continue
+            if not comments:
+                continue
+            # ``c`` is a LibclangComment duck-typing the tree-sitter Node
+            # interface SourceComment reads (``.text`` / ``.start_point.row``);
+            # the Node-only path (find_associated_scope) is guarded by
+            # ``is_libclang`` so it never runs on these.
+            src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments]
+            src_file = SourceFile(src_path.absolute())
+            src_file.add_comments(src_comments)
+            self.src_files.append(src_file)
+            self.src_comments.extend(src_comments)
+
     def extract_marker(
         self,
         text: str,
@@ -327,9 +450,12 @@ class SourceAnalyse:
             )
             if not filepath:
                 continue
-            tagged_scope: TreeSitterNode | None = utils.find_associated_scope(
-                src_comment.node, self.analyse_config.comment_type
-            )
+            if getattr(src_comment.node, "is_libclang", False):
+                tagged_scope: TreeSitterNode | None = None
+            else:
+                tagged_scope = utils.find_associated_scope(
+                    src_comment.node, self.analyse_config.comment_type
+                )
             if self.analyse_config.get_need_id_refs:
                 anchors = self.extract_anchors(
                     text, filepath, tagged_scope, src_comment
@@ -372,7 +498,13 @@ class SourceAnalyse:
             json.dump(to_dump, f)
 
     def run(self) -> None:
-        self.create_src_objects()
+        if (
+            self.analyse_config.preprocessor is not None
+            and self.analyse_config.comment_type == CommentType.cpp
+        ):
+            self.create_src_objects_libclang()
+        else:
+            self.create_src_objects()
         self.extract_marked_content()
         self.merge_marked_content()
         self._log_summary()

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/analyse/oneline_parser.py

@@ -43,6 +43,19 @@ def oneline_parser(  # noqa: PLR0912, PLR0911 # handel warnings
         # start or end sequences do not exist
         return None
 
+    # A marker whose end sequence is the newline extends to the end of the
+    # line, so an unanchored start sequence would swallow trailing prose. Anchor
+    # such a marker to the start of the comment content: everything preceding
+    # the start sequence must be comment decoration (`//`, `#`, `*`, ...) and
+    # whitespace. A word character before it means the start sequence is part of
+    # free-form prose (e.g. `// see @author, ...`) and the line is ignored
+    # (issue #88). Explicitly-bounded markers (e.g. `[[ ... ]]`) are
+    # self-delimiting and may appear anywhere, so they are exempt.
+    if oneline_config.end_sequence == UNIX_NEWLINE and any(
+        char.isalnum() for char in oneline[:start_idx]
+    ):
+        return None
+
     # extract the string wrapped by start and end
     start_idx = start_idx + len(oneline_config.start_sequence)
     string = oneline[start_idx:end_idx].strip()

sphinx_codelinks-1.4.0/src/sphinx_codelinks/analyse/preproc/__init__.py

@@ -0,0 +1,5 @@
+"""libclang-based, preprocessor-aware extraction engine (opt-in)."""
+
+from sphinx_codelinks.analyse.preproc import compile_db, libclang_parser, loader
+
+__all__ = ["compile_db", "libclang_parser", "loader"]

sphinx_codelinks-1.4.0/src/sphinx_codelinks/analyse/preproc/compile_db.py

@@ -0,0 +1,156 @@
+"""Discover, read, and filter compile_commands.json for libclang."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import shlex
+
+from sphinx_codelinks.config import DEFAULT_CPP_STD
+
+_DB_NAME = "compile_commands.json"
+_BOUNDARY_MARKERS = (".git", "ubproject.toml", "pyproject.toml")
+
+# Flags that take a following value we must also drop.
+_DROP_WITH_VALUE = {"-o", "-MF", "-MT", "-MQ"}
+# Exact flags to drop.
+_DROP_EXACT = {"-c", "-MMD", "-MD", "-MG", "-MP"}
+# Separate-form flags whose following VALUE token must be kept verbatim (never
+# treated as the input source and stripped by basename).
+_KEEP_WITH_VALUE = {
+    "-include",
+    "-isystem",
+    "-iquote",
+    "-idirafter",
+    "-isysroot",
+    "-x",
+    "-I",
+}
+# Minimum length for joined-form flags like -MFdep.d (prefix length = 3)
+_MIN_JOINED_FLAG_LEN = 3
+
+# Suffixes a compiler builds as a translation unit. A discovered C/C++ file
+# absent from compile_commands.json is skipped only when it is a TU source
+# (build-excluded); all other discovered files (headers) are parsed standalone.
+TU_SOURCE_SUFFIXES = {".c", ".cpp", ".cc", ".cxx"}
+
+
+# @Discover compile_commands.json by walking up from the source file, IMPL_PREPROC_3, impl, [FE_PREPROC]
+def find_compile_db(start: Path, project_root: Path | None = None) -> Path | None:
+    """Walk up from ``start`` looking for compile_commands.json.
+
+    Stops at (inclusive) the directory that contains the db, or at
+    ``project_root`` / a directory containing a boundary marker / fs root.
+    """
+    current = start if start.is_dir() else start.parent
+    current = current.resolve()
+    root = project_root.resolve() if project_root else None
+    while True:
+        candidate = current / _DB_NAME
+        if candidate.is_file():
+            return candidate
+        if root is not None and current == root:
+            return None
+        if any((current / m).exists() for m in _BOUNDARY_MARKERS):
+            return None
+        if current.parent == current:  # filesystem root
+            return None
+        current = current.parent
+
+
+def filter_args(argv: list[str], input_file: str) -> list[str]:
+    """Keep only flags libclang needs; drop the compiler, -c/-o, depfiles, input."""
+    out: list[str] = []
+    skip_next = False
+    keep_next_value = False
+    input_base = Path(input_file).name
+    # Skip leading non-flag tokens. argv[0] is the compiler, but a build may prefix
+    # it with a launcher (ccache/sccache/distcc), so drop every leading non-flag
+    # token up to the first flag — otherwise the real compiler leaks in as a
+    # positional and libclang treats it as a second input (a NULL TU that silently
+    # drops the file). The input source (also a non-flag positional) is stripped by
+    # name below.
+    in_leading = True
+    for arg in argv:
+        if in_leading:
+            if not arg.startswith("-"):
+                continue
+            in_leading = False
+        if skip_next:
+            skip_next = False
+            continue
+        if keep_next_value:
+            # Value of a separate-form -include/-isystem/-I/... — keep verbatim.
+            keep_next_value = False
+            out.append(arg)
+            continue
+        if arg in _DROP_WITH_VALUE:
+            skip_next = True
+            continue
+        if arg in _DROP_EXACT:
+            continue
+        if arg.startswith(("-MF", "-MT", "-MQ")) and len(arg) > _MIN_JOINED_FLAG_LEN:
+            continue  # joined form, e.g. -MFdep.d
+        if arg in _KEEP_WITH_VALUE:
+            out.append(arg)
+            keep_next_value = True
+            continue
+        # Strip the TU source positional (exact match, or same basename).
+        if not arg.startswith("-") and (
+            arg in (input_file, input_base) or Path(arg).name == input_base
+        ):
+            continue
+        out.append(arg)
+    return out
+
+
+# @Resolve per-file compiler flags from the compilation database, IMPL_PREPROC_4, impl, [FE_PREPROC]
+def load_flags_map(db_path: Path) -> dict[Path, list[str]]:
+    """Parse compile_commands.json -> {absolute file path: filtered args}."""
+    entries = json.loads(db_path.read_text())
+    flags: dict[Path, list[str]] = {}
+    for entry in entries:
+        if "file" not in entry or "directory" not in entry:
+            continue  # malformed entry: skip, keep going
+        if "arguments" in entry:
+            argv = list(entry["arguments"])
+        elif "command" in entry:
+            argv = shlex.split(entry["command"])
+        else:
+            continue
+        directory = Path(entry["directory"])
+        file_field = entry["file"]
+        abs_file = (directory / file_field).resolve()
+        flags[abs_file] = filter_args(argv, file_field)
+    return flags
+
+
+# @Parse headers standalone from configured defines and includes, IMPL_PREPROC_5, impl, [FE_PREPROC]
+def defines_to_args(
+    defines: list[str], includes: list[Path], std: str = DEFAULT_CPP_STD
+) -> list[str]:
+    """Build a global flag list for parsing a file standalone.
+
+    Used for headers and files absent from a compile DB. libclang infers the
+    language from the file extension, so a C-inferred file (``.h`` / ``.c`` /
+    ``.inc``) handed a C++ ``-std`` makes clang reject the combination and return
+    a NULL translation unit (surfacing as ``TranslationUnitLoadError``). Pin the
+    language to match ``std`` with ``-x`` so such files — e.g. a ``.h`` header
+    carrying oneline need markers — parse and extract instead of failing. (C
+    sources parse as C++ well enough for comment/marker extraction.)
+    """
+    lang = "c++" if std.startswith(("c++", "gnu++")) else "c"
+    args = ["-x", lang, f"-std={std}"]
+    args += [f"-D{d}" for d in defines]
+    args += [f"-I{inc}" for inc in includes]
+    return args
+
+
+def is_translation_unit_source(path: Path) -> bool:
+    """True if ``path`` is a compiled translation-unit source (not a header).
+
+    compile_commands.json lists one entry per compiled TU; headers are never
+    entries. So a discovered file absent from the DB is skipped only when it is
+    a TU source; header-like files are parsed standalone (see _resolve_preproc_args).
+    """
+    return path.suffix.lower() in TU_SOURCE_SUFFIXES

sphinx_codelinks-1.4.0/src/sphinx_codelinks/analyse/preproc/libclang_parser.py

@@ -0,0 +1,102 @@
+"""Parse a C/C++ TU via libclang and yield ACTIVE comment tokens only."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import os
+from pathlib import Path
+
+from sphinx_codelinks.analyse.preproc import loader
+from sphinx_codelinks.analyse.preproc.loader import SkippedRange
+
+
+@dataclass
+class _Point:
+    row: int
+
+
+class LibclangComment:
+    """Minimal stand-in for a tree-sitter comment node.
+
+    Exposes exactly the two attributes the existing extract_* chain reads:
+    ``.text`` (bytes) and ``.start_point.row`` (0-indexed). ``is_libclang``
+    lets extract_marked_content skip tree-sitter scope association.
+    """
+
+    is_libclang = True
+
+    def __init__(self, text: bytes, row: int) -> None:
+        self.text: bytes = text
+        self.start_point = _Point(row)
+
+
+def _group_skipped(skipped: list[SkippedRange]) -> dict[str, list[tuple[int, int]]]:
+    """Group skipped ranges by normalised file path, built once per TU.
+
+    A per-comment membership test then scans only its own file's ranges instead
+    of every range in the translation unit (headers included). Normalising the
+    key (``os.path.normpath``) also fixes clang spelling the same file two ways
+    (``./x`` vs ``x``, ``a/../b``), which a naive compare would miss.
+    """
+    grouped: dict[str, list[tuple[int, int]]] = {}
+    for sr in skipped:
+        if sr.file is not None:
+            grouped.setdefault(os.path.normpath(str(sr.file)), []).append(
+                (sr.start_line, sr.end_line)
+            )
+    return grouped
+
+
+def _is_in_skipped(
+    file_path: str, line: int, grouped: dict[str, list[tuple[int, int]]]
+) -> bool:
+    return any(
+        start <= line <= end
+        for start, end in grouped.get(os.path.normpath(file_path), ())
+    )
+
+
+# @Drop comments inside inactive preprocessor branches, IMPL_PREPROC_2, impl, [FE_PREPROC]
+def extract_active_comments(file_path: Path, args: list[str]) -> list[LibclangComment]:
+    """Return one LibclangComment per ACTIVE comment token in ``file_path``.
+
+    Comments inside preprocessor-skipped (inactive) ranges are dropped.
+    """
+    cx = loader.load_clang_cindex()
+    index = cx.Index.create()
+    tu = index.parse(str(file_path), args=args, options=loader.PARSE_OPTIONS)
+    skipped = _group_skipped(loader.get_all_skipped_ranges(tu))
+
+    # Read the raw source bytes once. We derive the token extent from them
+    # (lossily, so a non-UTF-8 byte can't raise and abort the run) AND slice each
+    # comment's text out of them by byte offset below — never via ``tok.spelling``,
+    # which decodes the comment as strict UTF-8 and raises UnicodeDecodeError on a
+    # non-UTF-8 byte inside a comment.
+    raw = file_path.read_bytes()
+    line_count = len(raw.decode("utf-8", errors="replace").splitlines())
+    main = tu.get_file(str(file_path))
+    extent = cx.SourceRange.from_locations(
+        cx.SourceLocation.from_position(tu, main, 1, 1),
+        cx.SourceLocation.from_position(tu, main, line_count + 1, 1),
+    )
+
+    out: list[LibclangComment] = []
+    for tok in tu.get_tokens(extent=extent):
+        if tok.kind != cx.TokenKind.COMMENT:
+            continue
+        loc = tok.location
+        if loc.file is None:
+            continue
+        if _is_in_skipped(str(loc.file.name), loc.line, skipped):
+            continue  # inactive branch -> excluded
+        # Slice the comment text from the raw bytes by offset and decode lossily
+        # (not via tok.spelling, which raises on a non-UTF-8 byte in the comment).
+        # Then normalize CRLF/CR -> LF, matching get_src_strings on the tree-sitter
+        # path: a multi-line block comment (e.g. a reST block) from a CRLF-saved
+        # file otherwise carries embedded \r into the extracted marker text.
+        text = raw[tok.extent.start.offset : tok.extent.end.offset].decode(
+            "utf-8", errors="replace"
+        )
+        spelling = text.replace("\r\n", "\n").replace("\r", "\n")
+        out.append(LibclangComment(spelling.encode("utf-8"), loc.line - 1))
+    return out

sphinx_codelinks-1.4.0/src/sphinx_codelinks/analyse/preproc/loader.py

@@ -0,0 +1,95 @@
+"""Import guard for clang.cindex + ctypes shim for clang_getAllSkippedRanges.
+
+clang.cindex (shipped by the PyPI ``libclang`` wheel, which bundles the native
+library — no compiler required) does not expose clang_getAllSkippedRanges, so
+we bind it via ctypes, exactly as the design concept's _common.py did.
+"""
+
+from __future__ import annotations
+
+import ctypes
+from pathlib import Path
+from typing import Any, NamedTuple
+
+_INSTALL_HINT = (
+    "The libclang engine requires clang.cindex. Install the extra:\n"
+    "    pip install 'sphinx-codelinks[libclang]'"
+)
+
+
+# @Guard the optional libclang dependency with an install hint, IMPL_PREPROC_6, impl, [FE_PREPROC]
+def load_clang_cindex() -> Any:  # type: ignore[explicit-any]
+    """Return the clang.cindex module or raise a clear install error."""
+    try:
+        import clang.cindex as cx  # noqa: PLC0415
+    except ImportError as exc:  # pragma: no cover - exercised via patched import
+        raise ImportError(_INSTALL_HINT) from exc
+    return cx
+
+
+# Production parse flags (design "Parse flags codelinks should use").
+def _parse_options() -> int:
+    cx = load_clang_cindex()
+    # ``cx`` is untyped (clang ships no stubs), so the bit-or is ``Any``; coerce
+    # back to ``int`` to satisfy the declared return type.
+    return int(
+        cx.TranslationUnit.PARSE_INCOMPLETE
+        | cx.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
+        | cx.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
+    )
+
+
+PARSE_OPTIONS: int = _parse_options()
+
+
+class SkippedRange(NamedTuple):
+    file: Path | None
+    start_line: int
+    start_col: int
+    end_line: int
+    end_col: int
+
+
+class _CXSourceRangeList(ctypes.Structure):
+    pass
+
+
+_BOUND = False
+
+
+def _bind() -> None:
+    global _BOUND  # noqa: PLW0603
+    if _BOUND:
+        return
+    cx = load_clang_cindex()
+    _CXSourceRangeList._fields_ = [
+        ("count", ctypes.c_uint),
+        ("ranges", ctypes.POINTER(cx.SourceRange)),
+    ]
+    lib = cx.conf.lib
+    lib.clang_getAllSkippedRanges.argtypes = [ctypes.c_void_p]
+    lib.clang_getAllSkippedRanges.restype = ctypes.POINTER(_CXSourceRangeList)
+    lib.clang_disposeSourceRangeList.argtypes = [ctypes.POINTER(_CXSourceRangeList)]
+    lib.clang_disposeSourceRangeList.restype = None
+    _BOUND = True
+
+
+def get_all_skipped_ranges(tu: Any) -> list[SkippedRange]:  # type: ignore[explicit-any]
+    """Return every source range the preprocessor skipped in this TU."""
+    _bind()
+    cx = load_clang_cindex()
+    ptr = cx.conf.lib.clang_getAllSkippedRanges(tu.obj)
+    if not ptr:
+        return []
+    try:
+        rl = ptr.contents
+        out: list[SkippedRange] = []
+        for i in range(rl.count):
+            r = rl.ranges[i]
+            start = r.start
+            end = r.end
+            f = Path(start.file.name) if start.file else None
+            out.append(SkippedRange(f, start.line, start.column, end.line, end.column))
+        return out
+    finally:
+        cx.conf.lib.clang_disposeSourceRangeList(ptr)

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/analyse/utils.py

@@ -44,6 +44,8 @@ SCOPE_NODE_TYPES = {
         "type_declaration",
         "type_spec",
     },
+    # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
+    CommentType.bash: {"function_definition"},
 }
 
 logger = get_logger(__name__)
@@ -74,6 +76,8 @@ GO_QUERY = """
     (comment) @comment
 """
 JSONC_QUERY = """(comment) @comment"""
+# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
+BASH_QUERY = """(comment) @comment"""
 
 # JSON value node types that can be associated with a comment.
 JSON_STRUCTURE_TYPES = {
@@ -103,7 +107,7 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
         return False
 
 
-# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC]
+# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
 def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
     if comment_type == CommentType.cpp:
         import tree_sitter_cpp  # noqa: PLC0415
@@ -140,6 +144,11 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
 
         parsed_language = Language(tree_sitter_json.language())
         query = Query(parsed_language, JSONC_QUERY)
+    elif comment_type == CommentType.bash:
+        import tree_sitter_bash  # noqa: PLC0415
+
+        parsed_language = Language(tree_sitter_bash.language())
+        query = Query(parsed_language, BASH_QUERY)
     else:
         raise ValueError(f"Unsupported comment style: {comment_type}")
     parser = Parser(parsed_language)

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/cmd.py

@@ -12,6 +12,7 @@ from sphinx_codelinks.config import (
     CodeLinksConfig,
     CodeLinksConfigType,
     CodeLinksProjectConfigType,
+    anchor_preproc_paths,
     generate_project_configs,
 )
 from sphinx_codelinks.logger import configure_cli, logger
@@ -150,6 +151,13 @@ def analyse(  # noqa: PLR0912   # for CLI, so it needs the branches
                 config.parent / analyse_config.git_root
             ).resolve()
 
+        # preprocessor compile_commands / include dirs are relative to the config
+        # file's location too (like src_dir / git_root).
+        if analyse_config.preprocessor is not None:
+            analyse_config.preprocessor = anchor_preproc_paths(
+                analyse_config.preprocessor, config.parent
+            )
+
         analyse_errors = analyse_config.check_fields_configuration()
         errors.extend(analyse_errors)
         if errors:

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/config.py

@@ -1,5 +1,5 @@
 from collections import deque
-from dataclasses import MISSING, dataclass, field, fields
+from dataclasses import MISSING, dataclass, field, fields, replace
 from enum import Enum
 from pathlib import Path
 from typing import Any, Literal, TypedDict, cast
@@ -29,6 +29,10 @@ COMMENT_MARKERS = {
 }
 ESCAPE = "\\"
 
+# Default C/C++ standard for the standalone/defines parse path when no
+# compile_commands.json entry supplies one.
+DEFAULT_CPP_STD = "c++17"
+
 
 class CommentCategory(str, Enum):
     comment = "comment"
@@ -120,6 +124,53 @@ class MarkedRstConfig:
         return self.check_schema() + self.check_sequence_mutually_exclusive()
 
 
+@dataclass
+class PreprocessorConfig:
+    """Opt-in libclang engine config. Presence => libclang engine for C/C++."""
+
+    compile_commands: Path | None = field(
+        default=None, metadata={"schema": {"type": ["string", "null"]}}
+    )
+    """Explicit path to compile_commands.json. If None, walk-up auto-discovery."""
+
+    defines: list[str] = field(
+        default_factory=list,
+        metadata={"schema": {"type": "array", "items": {"type": "string"}}},
+    )
+    """Fallback -D defines applied globally when no compile_commands.json applies."""
+
+    includes: list[Path] = field(
+        default_factory=list,
+        metadata={"schema": {"type": "array", "items": {"type": "string"}}},
+    )
+    """Fallback -I include dirs for the defines path."""
+
+    std: str = field(
+        default=DEFAULT_CPP_STD,
+        metadata={"schema": {"type": "string"}},
+    )
+    """C/C++ standard for the standalone/defines parse path (e.g. ``c++17``,
+    ``c++20``, ``c11``); libclang pins ``-x`` to match it. Files resolved from a
+    ``compile_commands.json`` entry use that entry's own ``-std`` instead."""
+
+
+def anchor_preproc_paths(preproc: PreprocessorConfig, base: Path) -> PreprocessorConfig:
+    """Resolve a preprocessor config's ``compile_commands`` and ``includes``
+    against ``base`` (the config file's directory), so a relative path resolves
+    against the TOML file rather than the process CWD — matching ``src_dir`` /
+    ``git_root``. Absolute paths are left unchanged.
+    """
+    return replace(
+        preproc,
+        compile_commands=(
+            (base / preproc.compile_commands).resolve()
+            if preproc.compile_commands is not None
+            else None
+        ),
+        includes=[(base / inc).resolve() for inc in preproc.includes],
+    )
+
+
 class FieldConfig(TypedDict, total=False):
     name: str
     type: Literal["str", "list[str]"]
@@ -329,6 +380,7 @@ class AnalyseSectionConfigType(TypedDict, total=False):
     need_id_refs: NeedIdRefsConfigType
     marked_rst: MarkedRstConfigType
     oneline_comment_style: OneLineCommentStyleType
+    preprocessor: dict[str, object]
 
 
 class SourceAnalyseConfigType(TypedDict, total=False):
@@ -344,6 +396,7 @@ class SourceAnalyseConfigType(TypedDict, total=False):
     need_id_refs_config: NeedIdRefsConfig
     marked_rst_config: MarkedRstConfig
     oneline_comment_style: OneLineCommentStyle
+    preprocessor: PreprocessorConfig | None
 
 
 class ProjectsAnalyseConfigType(TypedDict, total=False):
@@ -400,6 +453,17 @@ class SourceAnalyseConfig:
     )
     """Configuration for extracting oneline needs from comments."""
 
+    preprocessor: PreprocessorConfig | None = field(default=None)
+    """Opt-in libclang preprocessor engine. None => tree-sitter (default).
+
+    No flat ``metadata["schema"]`` here: this is a nested dataclass, like the
+    sibling ``need_id_refs_config`` / ``marked_rst_config`` /
+    ``oneline_comment_style`` fields. ``check_schema`` only validates fields that
+    declare a flat schema; giving this field one made it validate the constructed
+    ``PreprocessorConfig`` instance against JSON type ``object`` and fail at
+    ``config-inited``. Its structure is enforced by ``convert_analyse_config``.
+    """
+
     @classmethod
     def get_schema(cls, name: str) -> dict[str, Any] | None:  # type: ignore[explicit-any]
         _field = next(_field for _field in fields(cls) if _field.name is name)
@@ -785,6 +849,35 @@ def convert_src_discovery_config(
     return src_discover_config
 
 
+def _validate_preprocessor_dict(preproc: dict[str, object]) -> None:
+    """Validate the schema-less ``[preprocessor]`` TOML section.
+
+    The section has no ``TypedDict``, so a mistyped scalar would otherwise be
+    coerced into garbage instead of reported: e.g. ``defines = "X"`` (a bare
+    string) becomes ``list("X") == ["X"]`` — or worse, ``defines = "cpp17"``
+    becomes ``["c", "p", "p", "1", "7"]`` → five bogus ``-D`` flags. Fail loud.
+
+    :param preproc: the raw ``[preprocessor]`` mapping from TOML.
+    :raises TypeError: if a key has the wrong type.
+    """
+    for key in ("defines", "includes"):
+        value = preproc.get(key)
+        if value is not None and (
+            not isinstance(value, list) or not all(isinstance(x, str) for x in value)
+        ):
+            raise TypeError(
+                f"[preprocessor] {key} must be a list of strings, "
+                f"got {type(value).__name__}: {value!r}"
+            )
+    for key in ("compile_commands", "std"):
+        value = preproc.get(key)
+        if value is not None and not isinstance(value, str):
+            raise TypeError(
+                f"[preprocessor] {key} must be a string, "
+                f"got {type(value).__name__}: {value!r}"
+            )
+
+
 def convert_analyse_config(
     config_dict: AnalyseSectionConfigType | None,
     src_discover: SourceDiscover | None = None,
@@ -792,7 +885,12 @@ def convert_analyse_config(
     analyse_config_dict: SourceAnalyseConfigType = {}
     if config_dict:
         for k, v in config_dict.items():
-            if k not in {"online_comment_style", "need_id_refs", "marked_rst"}:
+            if k not in {
+                "online_comment_style",
+                "need_id_refs",
+                "marked_rst",
+                "preprocessor",
+            }:
                 # Convert string paths to Path objects
                 if k in {"src_dir", "git_root"} and isinstance(v, str):
                     analyse_config_dict[k] = Path(v)  # type: ignore[literal-required]
@@ -823,6 +921,24 @@ def convert_analyse_config(
         analyse_config_dict["marked_rst_config"] = marked_rst_config
         analyse_config_dict["oneline_comment_style"] = oneline_comment_style
 
+        preprocessor_dict = config_dict.get("preprocessor")
+        if preprocessor_dict is not None:
+            # The preprocessor section has no TypedDict; its values are dynamic
+            # TOML (typed ``object``), so validate the shapes up front (a mistyped
+            # scalar would otherwise coerce into garbage flags) and keep the
+            # targeted ignores matching the concrete errors mypy reports.
+            _validate_preprocessor_dict(preprocessor_dict)
+            analyse_config_dict["preprocessor"] = PreprocessorConfig(
+                compile_commands=(
+                    Path(str(preprocessor_dict["compile_commands"]))
+                    if preprocessor_dict.get("compile_commands")
+                    else None
+                ),
+                defines=list(preprocessor_dict.get("defines", [])),  # type: ignore[call-overload]
+                includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])],  # type: ignore[attr-defined]
+                std=str(preprocessor_dict.get("std", DEFAULT_CPP_STD)),
+            )
+
     if src_discover:
         analyse_config_dict["src_files"] = src_discover.source_paths
         analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/source_discover/config.py

@@ -13,6 +13,12 @@ COMMENT_FILETYPE = {
     "rust": ["rs"],
     "go": ["go"],
     "jsonc": ["jsonc", "json"],
+    # Bash uses `#` line comments; zsh and ksh share the same comment syntax and
+    # are scanned with the bash grammar. Fish is intentionally excluded: it is
+    # not POSIX-compatible and no tree-sitter-fish distribution is published on
+    # PyPI, so it cannot be wired in here. Track fish separately if a PyPI
+    # grammar becomes available.
+    "bash": ["sh", "bash", "zsh", "ksh"],
 }
 
 
@@ -27,6 +33,8 @@ class CommentType(str, Enum):
     go = "go"
     # @Support JSONC style comments, IMPL_JSONC_1, impl, [FE_JSONC];
     jsonc = "jsonc"
+    # @Support Bash style comments, IMPL_BASH_1, impl, [FE_BASH];
+    bash = "bash"
 
 
 class SourceDiscoverSectionConfigType(TypedDict, total=False):

{sphinx_codelinks-1.3.0 → sphinx_codelinks-1.4.0}/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py

@@ -16,6 +16,7 @@ from sphinx_codelinks.analyse.models import OneLineNeed
 from sphinx_codelinks.config import (
     CodeLinksConfig,
     CodeLinksProjectConfigType,
+    anchor_preproc_paths,
     file_lineno_href,
 )
 from sphinx_codelinks.source_discover.config import SourceDiscoverConfig
@@ -113,19 +114,33 @@ class SourceTracingDirective(SphinxDirective):
         # time, forcing a full re-read. Build a per-directive copy instead so the
         # stored config value stays equal to what ``generate_project_configs`` yields.
         base_analyse_config = src_trace_conf["analyse_config"]
+        # Resolve the config file's directory once (used for git_root + preproc).
+        conf_dir = Path(self.env.app.confdir)
+        if src_trace_sphinx_config.config_from_toml:
+            src_trace_toml_path = Path(src_trace_sphinx_config.config_from_toml)
+            conf_dir = conf_dir / src_trace_toml_path.parent
         # git_root shall be relative to the config file's location (if provided)
         git_root = base_analyse_config.git_root
         if git_root:
-            conf_dir = Path(self.env.app.confdir)
-            if src_trace_sphinx_config.config_from_toml:
-                src_trace_toml_path = Path(src_trace_sphinx_config.config_from_toml)
-                conf_dir = conf_dir / src_trace_toml_path.parent
             git_root = (conf_dir / git_root).resolve()
+        # preprocessor compile_commands / include dirs are relative to the config
+        # file's location too (like src_dir / git_root).
+        preprocessor = base_analyse_config.preprocessor
+        if preprocessor is not None:
+            preprocessor = anchor_preproc_paths(preprocessor, conf_dir)
+            # Editing an explicitly-configured compile_commands.json changes the
+            # flags (hence which #if branches are active, hence the extracted
+            # markers), so register it as a build dependency to trigger a rebuild.
+            # (Auto-discovered databases are located per-file inside the analysis
+            # and are not tracked here.)
+            if preprocessor.compile_commands is not None:
+                self.env.note_dependency(str(preprocessor.compile_commands))
         analyse_config = replace(
             base_analyse_config,
             src_dir=src_dir,
             src_files=source_files,
             git_root=git_root,
+            preprocessor=preprocessor,
         )
         src_analyse = SourceAnalyse(analyse_config, name=project)
         src_analyse.run()