code-symbol-index 0.3.2__tar.gz → 0.3.4__tar.gz

{code_symbol_index-0.3.2 → code_symbol_index-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-symbol-index
-Version: 0.3.2
+Version: 0.3.4
 Summary: Tree-sitter backed symbol search and inspection for codebases.
 Author-email: hit9 <hit9@icloud.com>
 License-Expression: MIT

{code_symbol_index-0.3.2 → code_symbol_index-0.3.4}/code_symbol_index.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-symbol-index
-Version: 0.3.2
+Version: 0.3.4
 Summary: Tree-sitter backed symbol search and inspection for codebases.
 Author-email: hit9 <hit9@icloud.com>
 License-Expression: MIT

{code_symbol_index-0.3.2 → code_symbol_index-0.3.4}/code_symbol_index.py

@@ -22,7 +22,7 @@ from tree_sitter import Node
 from tree_sitter_language_pack import get_parser
 
 
-__version__ = "0.3.2"
+__version__ = "0.3.4"
 SCHEMA_VERSION = 5
 DEFAULT_INDEX_DIR = ".code-symbol-index"
 DEFAULT_INDEX_DB = "index.sqlite"
@@ -2516,7 +2516,7 @@ def _parse_file(
     except (OSError, UnicodeDecodeError, BinaryFileError):
         return None
 
-    tree = _parser_for_language(spec.name).parse(source_text)
+    tree = _parse_source(_parser_for_language(spec.name), source_text)
     root_node = tree.root_node() if callable(tree.root_node) else tree.root_node
     source_bytes = source_text.encode("utf-8")
     symbols, references = _extract_symbols_and_references(
@@ -2579,6 +2579,16 @@ def _file_contains_bytes(path: Path, needle: bytes) -> bool:
 _PARSER_TLS = threading.local()
 
 
+def _parse_source(parser, source: str):
+    # tree-sitter's parse() signature varies across versions/builds: some accept str, others
+    # require bytes (raising "source must be a bytestring or a callable, not str"). Try str first,
+    # then fall back to encoded bytes so both bindings work. Byte offsets are identical either way.
+    try:
+        return parser.parse(source)
+    except TypeError:
+        return parser.parse(source.encode("utf-8"))
+
+
 def _parser_for_language(language: str):
     if language not in LANGUAGE_BY_NAME:
         raise UnsupportedLanguageError(f"Unsupported language: {language}")
@@ -2751,8 +2761,8 @@ def _extract_symbols_and_references(
                     name=_node_text(source, node),
                     language=language.name,
                     path=path,
-                    range=_node_range(node),
-                    context=_line_context(lines, node),
+                    range=_node_range(source, node),
+                    context=_line_context(source, lines, node),
                     reference_kind=_classify_reference(node, parent, grandparent, ctx, language),
                 )
             )
@@ -2782,7 +2792,7 @@ def _python_top_level_symbols(source: bytes, path: Path, language: LanguageSpec,
         if not name or not _looks_like_symbol_name(name):
             continue
         kind = "constant" if name.isupper() else "variable"
-        range_ = _node_range(name_node)
+        range_ = _node_range(source, name_node)
         symbols.append(
             Symbol(
                 id=_symbol_id(language.name, path, kind, name, range_.start_byte),
@@ -2821,7 +2831,7 @@ def _python_dict_key_symbols(
         key_name, key_node = _python_dict_key_name(source, language, key)
         if not key_name or not _looks_like_symbol_name(key_name):
             continue
-        range_ = _node_range(key_node)
+        range_ = _node_range(source, key_node)
         symbols.append(
             Symbol(
                 id=_symbol_id(language.name, path, "dict_key", key_name, range_.start_byte),
@@ -2864,7 +2874,7 @@ def _symbol_from_node(
     name = _node_text(source, name_node)
     if not name or not _looks_like_symbol_name(name):
         return None
-    range_ = _node_range(name_node)
+    range_ = _node_range(source, name_node)
     return Symbol(
         id=_symbol_id(language.name, path, kind, name, range_.start_byte),
         name=name,
@@ -2913,25 +2923,28 @@ def _signature(source: bytes, node: Node) -> str:
     return first_line[:240]
 
 
-def _node_range(node: Node) -> Range:
-    start_line, start_column = _point(_node_start_point(node))
-    end_line, end_column = _point(_node_end_point(node))
+def _node_range(source: bytes, node: Node) -> Range:
+    start_byte = _node_start_byte(node)
+    end_byte = _node_end_byte(node)
+    start_line, start_column = _byte_position(source, start_byte)
+    end_line, end_column = _byte_position(source, end_byte)
     return Range(
         start=Position(line=start_line, column=start_column),
         end=Position(line=end_line, column=end_column),
-        start_byte=_node_start_byte(node),
-        end_byte=_node_end_byte(node),
+        start_byte=start_byte,
+        end_byte=end_byte,
     )
 
 
-def _point(point: Any) -> tuple[int, int]:
-    if hasattr(point, "row"):
-        return point.row, point.column
-    return point[0], point[1]
+def _byte_position(source: bytes, offset: int) -> tuple[int, int]:
+    prefix = source[:offset]
+    line = prefix.count(b"\n")
+    line_start = prefix.rfind(b"\n") + 1
+    return line, len(prefix) - line_start
 
 
-def _line_context(lines: list[str], node: Node) -> str:
-    line, _ = _point(_node_start_point(node))
+def _line_context(source: bytes, lines: list[str], node: Node) -> str:
+    line, _ = _byte_position(source, _node_start_byte(node))
     if 0 <= line < len(lines):
         return lines[line].strip()
     return ""
@@ -3685,13 +3698,14 @@ def _definition_ranges_for_symbols(
     spec = _spec_for_path(path, repo.languages)
     if spec is None:
         return {}
+    source_bytes = source.encode("utf-8")
 
     wanted: dict[tuple[str, int], Symbol] = {
         (symbol.kind, symbol.range.start_byte): symbol
         for symbol in symbols
     }
     ranges: dict[str, Range] = {}
-    tree = _parser_for_language(spec.name).parse(source)
+    tree = _parse_source(_parser_for_language(spec.name), source)
     root_node = tree.root_node() if callable(tree.root_node) else tree.root_node
 
     def walk(node: Node) -> None:
@@ -3703,7 +3717,7 @@ def _definition_ranges_for_symbols(
             if name_node is not None:
                 symbol = wanted.get((kind, _node_start_byte(name_node)))
                 if symbol is not None:
-                    ranges[symbol.id] = _node_range(node)
+                    ranges[symbol.id] = _node_range(source_bytes, node)
         for child in _node_children(node):
             walk(child)
 

{code_symbol_index-0.3.2 → code_symbol_index-0.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "code-symbol-index"
-version = "0.3.2"
+version = "0.3.4"
 description = "Tree-sitter backed symbol search and inspection for codebases."
 readme = "README.md"
 requires-python = ">=3.10"

{code_symbol_index-0.3.2 → code_symbol_index-0.3.4}/tests/test_code_symbol_index.py

@@ -41,6 +41,42 @@ value = helper()
     assert "def helper" in (inspection.source_preview or "")
 
 
+def test_python_ranges_do_not_depend_on_tree_sitter_points(monkeypatch) -> None:
+    source = (
+        "def caller():\n"
+        "    return target()\n"
+        "\n"
+        + ("# padding\n" * 32)
+        + "    def target():\n"
+        "        return 1\n"
+    )
+    source_bytes = source.encode("utf-8")
+
+    def fail_point_access(*_args):
+        raise AssertionError("tree-sitter point access should not be used")
+
+    monkeypatch.setattr(code_symbol_index, "_node_start_point", fail_point_access)
+    monkeypatch.setattr(code_symbol_index, "_node_end_point", fail_point_access)
+
+    parser = code_symbol_index._parser_for_language("python")
+    tree = parser.parse(source_bytes)
+    root_node = tree.root_node() if callable(tree.root_node) else tree.root_node
+    symbols, references = code_symbol_index._extract_symbols_and_references(
+        source=source_bytes,
+        root_node=root_node,
+        path=Path("app.py"),
+        language=code_symbol_index.LANGUAGE_BY_NAME["python"],
+        include_references=True,
+    )
+
+    target = next(symbol for symbol in symbols if symbol.name == "target")
+    target_line = source.splitlines().index("    def target():")
+    assert target.range.start.line == target_line
+    assert target.range.start.column == 8
+    assert target.range.start_byte == source_bytes.rindex(b"target")
+    assert any(reference.name == "target" and reference.context == "return target()" for reference in references)
+
+
 def test_indexes_python_top_level_constants_and_dict_keys(tmp_path: Path, capsys) -> None:
     (tmp_path / "settings.py").write_text(
         "MODEL_NAME = 'ask-syft'\n"