victor-codegraph 0.1.1__tar.gz → 0.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/PKG-INFO +13 -15
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/README.md +10 -12
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/pyproject.toml +8 -3
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/tests/test_adapter.py +39 -5
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/tests/test_line_base_contract.py +4 -1
- victor_codegraph-0.8.1/tests/test_manifest.py +127 -0
- victor_codegraph-0.8.1/tests/test_sizing_overlap.py +132 -0
- victor_codegraph-0.8.1/tests/test_symbol_oid_contract.py +81 -0
- victor_codegraph-0.8.1/tests/test_symbol_oid_parity.py +77 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/tests/test_treesitter_jsts.py +4 -1
- victor_codegraph-0.8.1/tests/test_treesitter_languages.py +326 -0
- victor_codegraph-0.8.1/tests/test_treesitter_relations.py +182 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/__init__.py +14 -2
- victor_codegraph-0.8.1/victor_codegraph/adapter.py +213 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/config.py +5 -1
- victor_codegraph-0.8.1/victor_codegraph/manifest.py +109 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/model.py +31 -1
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/parser.py +3 -1
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/python_parser.py +5 -22
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/repo.py +44 -7
- victor_codegraph-0.8.1/victor_codegraph/resolution.py +68 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/sizing.py +24 -7
- victor_codegraph-0.8.1/victor_codegraph/treesitter_parser.py +666 -0
- victor_codegraph-0.8.1/victor_codegraph/treesitter_tables.py +70 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph.egg-info/PKG-INFO +13 -15
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph.egg-info/SOURCES.txt +9 -0
- victor_codegraph-0.1.1/victor_codegraph/adapter.py +0 -99
- victor_codegraph-0.1.1/victor_codegraph/treesitter_parser.py +0 -268
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/setup.cfg +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/tests/test_python_parser.py +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/tests/test_repo.py +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/tests/test_sizing_and_chunk.py +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph/languages.py +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph.egg-info/dependency_links.txt +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph.egg-info/requires.txt +0 -0
- {victor_codegraph-0.1.1 → victor_codegraph-0.8.1}/victor_codegraph.egg-info/top_level.txt +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: victor-codegraph
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.1
|
|
4
4
|
Summary: Code->CPG chunker: tree-sitter symbol + relation extraction, size-capped chunks, ProximaRecord projection. Shared by Victor, ProximaDB SDK, and AnvaiOps.
|
|
5
5
|
Author-email: Vijaykumar Singh <singhvjd@gmail.com>
|
|
6
6
|
License: Apache-2.0
|
|
7
|
-
Project-URL: Homepage, https://github.com/
|
|
8
|
-
Project-URL: Repository, https://github.com/
|
|
7
|
+
Project-URL: Homepage, https://github.com/anvai-labs/victor
|
|
8
|
+
Project-URL: Repository, https://github.com/anvai-labs/victor
|
|
9
9
|
Keywords: code-graph,cpg,chunking,tree-sitter,ast,embeddings,rag
|
|
10
10
|
Classifier: Development Status :: 3 - Alpha
|
|
11
11
|
Classifier: Intended Audience :: Developers
|
|
@@ -53,20 +53,18 @@ gaps:
|
|
|
53
53
|
|
|
54
54
|
## Install
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
(
|
|
58
|
-
|
|
56
|
+
Published on PyPI as [`victor-codegraph`](https://pypi.org/project/victor-codegraph/)
|
|
57
|
+
(current release **0.1.2**). Consumers pin the PyPI release — the ProximaDB SDK's
|
|
58
|
+
`proximadb[codegraph]` extra and AnvaiOps both require `victor-codegraph>=0.1.2`.
|
|
59
59
|
|
|
60
60
|
```bash
|
|
61
|
-
#
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
# minimal: Python-only (stdlib ast) path, zero native deps
|
|
65
|
-
pip install -e ./victor-codegraph
|
|
61
|
+
# from PyPI
|
|
62
|
+
pip install victor-codegraph # Python-only (stdlib ast) path, zero native deps
|
|
63
|
+
pip install "victor-codegraph[treesitter]" # + multi-language tree-sitter grammars
|
|
66
64
|
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
|
|
65
|
+
# dev (monorepo): editable, with tree-sitter grammars + test tooling
|
|
66
|
+
make -C victor-codegraph dev # = pip install -e ../victor-contracts && pip install -e ".[dev]"
|
|
67
|
+
pip install -e ./victor-codegraph # minimal editable install
|
|
70
68
|
```
|
|
71
69
|
|
|
72
70
|
### Releasing
|
|
@@ -104,6 +102,6 @@ records = to_proxima_records(parsed, repo_graph_id="myrepo", branch_id="main",
|
|
|
104
102
|
|
|
105
103
|
## Status
|
|
106
104
|
|
|
107
|
-
`0.1.
|
|
105
|
+
`0.1.2` (PyPI). Python (stdlib `ast`) is the primary, fully-offline path.
|
|
108
106
|
Multi-language extraction is best-effort via tree-sitter; deeper per-language relation
|
|
109
107
|
extraction (the donor parsers' Rust/Go/Java specifics) lands incrementally.
|
|
@@ -23,20 +23,18 @@ gaps:
|
|
|
23
23
|
|
|
24
24
|
## Install
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
(
|
|
28
|
-
|
|
26
|
+
Published on PyPI as [`victor-codegraph`](https://pypi.org/project/victor-codegraph/)
|
|
27
|
+
(current release **0.1.2**). Consumers pin the PyPI release — the ProximaDB SDK's
|
|
28
|
+
`proximadb[codegraph]` extra and AnvaiOps both require `victor-codegraph>=0.1.2`.
|
|
29
29
|
|
|
30
30
|
```bash
|
|
31
|
-
#
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
# minimal: Python-only (stdlib ast) path, zero native deps
|
|
35
|
-
pip install -e ./victor-codegraph
|
|
31
|
+
# from PyPI
|
|
32
|
+
pip install victor-codegraph # Python-only (stdlib ast) path, zero native deps
|
|
33
|
+
pip install "victor-codegraph[treesitter]" # + multi-language tree-sitter grammars
|
|
36
34
|
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
|
|
35
|
+
# dev (monorepo): editable, with tree-sitter grammars + test tooling
|
|
36
|
+
make -C victor-codegraph dev # = pip install -e ../victor-contracts && pip install -e ".[dev]"
|
|
37
|
+
pip install -e ./victor-codegraph # minimal editable install
|
|
40
38
|
```
|
|
41
39
|
|
|
42
40
|
### Releasing
|
|
@@ -74,6 +72,6 @@ records = to_proxima_records(parsed, repo_graph_id="myrepo", branch_id="main",
|
|
|
74
72
|
|
|
75
73
|
## Status
|
|
76
74
|
|
|
77
|
-
`0.1.
|
|
75
|
+
`0.1.2` (PyPI). Python (stdlib `ast`) is the primary, fully-offline path.
|
|
78
76
|
Multi-language extraction is best-effort via tree-sitter; deeper per-language relation
|
|
79
77
|
extraction (the donor parsers' Rust/Go/Java specifics) lands incrementally.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "victor-codegraph"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.8.1"
|
|
8
8
|
description = "Code->CPG chunker: tree-sitter symbol + relation extraction, size-capped chunks, ProximaRecord projection. Shared by Victor, ProximaDB SDK, and AnvaiOps."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -55,8 +55,8 @@ dev = [
|
|
|
55
55
|
]
|
|
56
56
|
|
|
57
57
|
[project.urls]
|
|
58
|
-
Homepage = "https://github.com/
|
|
59
|
-
Repository = "https://github.com/
|
|
58
|
+
Homepage = "https://github.com/anvai-labs/victor"
|
|
59
|
+
Repository = "https://github.com/anvai-labs/victor"
|
|
60
60
|
|
|
61
61
|
[tool.setuptools.packages.find]
|
|
62
62
|
where = ["."]
|
|
@@ -66,6 +66,11 @@ include = ["victor_codegraph*"]
|
|
|
66
66
|
line-length = 100
|
|
67
67
|
target-version = "py310"
|
|
68
68
|
|
|
69
|
+
# Pin the rule set explicitly: the CI workflow installs the latest ruff, whose
|
|
70
|
+
# evolving defaults must not be able to red a path-filtered pipeline overnight.
|
|
71
|
+
[tool.ruff.lint]
|
|
72
|
+
select = ["E4", "E7", "E9", "F"]
|
|
73
|
+
|
|
69
74
|
# Self-contained pytest config so running `pytest` from this package directory uses
|
|
70
75
|
# THIS file as the rootdir/inifile and does NOT inherit the victor monorepo root's
|
|
71
76
|
# `addopts = --cov=victor` (pytest-cov isn't a dep here -> "unrecognized arguments").
|
|
@@ -4,14 +4,14 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from victor_codegraph import parse, to_proxima_records
|
|
6
6
|
|
|
7
|
-
SAMPLE =
|
|
7
|
+
SAMPLE = """\
|
|
8
8
|
def a():
|
|
9
9
|
return b()
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
def b():
|
|
13
13
|
return 1
|
|
14
|
-
|
|
14
|
+
"""
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def test_symbol_records_shape():
|
|
@@ -45,12 +45,46 @@ def test_edge_records_reference_node_oids():
|
|
|
45
45
|
|
|
46
46
|
def test_embedder_populates_embedding_cell():
|
|
47
47
|
parsed = parse(SAMPLE, file_path="m.py")
|
|
48
|
-
records = to_proxima_records(
|
|
49
|
-
parsed, repo_graph_id="repo1", embedder=lambda text: [0.0] * 384
|
|
50
|
-
)
|
|
48
|
+
records = to_proxima_records(parsed, repo_graph_id="repo1", embedder=lambda text: [0.0] * 384)
|
|
51
49
|
node = next(r for r in records if "graph_node" in r["labels"])
|
|
52
50
|
assert len(node["embeddings"]) == 1
|
|
53
51
|
cell = node["embeddings"][0]
|
|
54
52
|
assert cell["modality"] == "code"
|
|
55
53
|
assert cell["dim"] == 384
|
|
56
54
|
assert len(cell["values"]) == 384
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_batch_embedder_single_call_populates_all_nodes():
|
|
58
|
+
parsed = parse(SAMPLE, file_path="m.py")
|
|
59
|
+
calls: list[list[str]] = []
|
|
60
|
+
|
|
61
|
+
def batch(texts: list[str]) -> list[list[float]]:
|
|
62
|
+
calls.append(texts)
|
|
63
|
+
return [[0.5] * 8 for _ in texts]
|
|
64
|
+
|
|
65
|
+
records = to_proxima_records(parsed, repo_graph_id="repo1", batch_embedder=batch, dim=8)
|
|
66
|
+
nodes = [r for r in records if "graph_node" in r["labels"]]
|
|
67
|
+
|
|
68
|
+
assert len(calls) == 1 # ONE call for the whole file
|
|
69
|
+
assert len(calls[0]) == len(nodes)
|
|
70
|
+
for n in nodes:
|
|
71
|
+
assert len(n["embeddings"]) == 1
|
|
72
|
+
assert n["embeddings"][0]["dim"] == 8
|
|
73
|
+
assert n["embeddings"][0]["values"] == [0.5] * 8
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_batch_embedder_wins_over_per_symbol_embedder():
|
|
77
|
+
parsed = parse(SAMPLE, file_path="m.py")
|
|
78
|
+
per_symbol_calls: list[str] = []
|
|
79
|
+
|
|
80
|
+
records = to_proxima_records(
|
|
81
|
+
parsed,
|
|
82
|
+
repo_graph_id="repo1",
|
|
83
|
+
embedder=lambda text: per_symbol_calls.append(text) or [0.0] * 8,
|
|
84
|
+
batch_embedder=lambda texts: [[1.0] * 8 for _ in texts],
|
|
85
|
+
dim=8,
|
|
86
|
+
)
|
|
87
|
+
nodes = [r for r in records if "graph_node" in r["labels"]]
|
|
88
|
+
|
|
89
|
+
assert per_symbol_calls == [] # per-symbol path not used
|
|
90
|
+
assert all(n["embeddings"][0]["values"] == [1.0] * 8 for n in nodes)
|
|
@@ -16,7 +16,10 @@ def test_line_base_is_one():
|
|
|
16
16
|
|
|
17
17
|
def test_parse_emits_1_based_lines():
|
|
18
18
|
# The def is on the FIRST line → start_line must be 1 (not 0).
|
|
19
|
-
parsed = parse(
|
|
19
|
+
parsed = parse(
|
|
20
|
+
"def first():\n return second()\n\n\ndef second():\n return 1\n",
|
|
21
|
+
file_path="m.py",
|
|
22
|
+
)
|
|
20
23
|
by_name = {s.simple_name: s for s in parsed.symbols}
|
|
21
24
|
assert by_name["first"].location.start_line == 1
|
|
22
25
|
assert by_name["second"].location.start_line == 5 # 1-based line of the 2nd def
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Incremental manifest contract + encoding fallback (offline, tmp_path only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from victor_codegraph import (
|
|
6
|
+
build_manifest,
|
|
7
|
+
diff_manifest,
|
|
8
|
+
iter_changed_files,
|
|
9
|
+
parse_path,
|
|
10
|
+
read_source_text,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _seed(tmp_path):
|
|
15
|
+
(tmp_path / "pkg").mkdir()
|
|
16
|
+
(tmp_path / "pkg" / "a.py").write_text("def a():\n return 1\n")
|
|
17
|
+
(tmp_path / "pkg" / "b.py").write_text("def b():\n return 2\n")
|
|
18
|
+
(tmp_path / "README.md").write_text("# not source\n") # unrecognized: excluded
|
|
19
|
+
return tmp_path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_build_manifest_relative_posix_keys(tmp_path):
|
|
23
|
+
manifest = build_manifest(_seed(tmp_path))
|
|
24
|
+
assert set(manifest) == {"pkg/a.py", "pkg/b.py"}
|
|
25
|
+
assert all("\\" not in k for k in manifest)
|
|
26
|
+
assert all(len(v) == 64 for v in manifest.values()) # sha-256 hex
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_manifest_hash_matches_parse_path(tmp_path):
|
|
30
|
+
root = _seed(tmp_path)
|
|
31
|
+
manifest = build_manifest(root)
|
|
32
|
+
parsed = parse_path(root / "pkg" / "a.py")
|
|
33
|
+
assert parsed is not None
|
|
34
|
+
assert manifest["pkg/a.py"] == parsed.content_hash
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_diff_manifest_classifies_all_states(tmp_path):
|
|
38
|
+
root = _seed(tmp_path)
|
|
39
|
+
previous = build_manifest(root)
|
|
40
|
+
|
|
41
|
+
(root / "pkg" / "a.py").write_text("def a():\n return 42\n") # changed
|
|
42
|
+
(root / "pkg" / "b.py").unlink() # removed
|
|
43
|
+
(root / "pkg" / "c.py").write_text("def c():\n return 3\n") # added
|
|
44
|
+
|
|
45
|
+
current = build_manifest(root)
|
|
46
|
+
diff = diff_manifest(current, previous)
|
|
47
|
+
assert diff.added == ["pkg/c.py"]
|
|
48
|
+
assert diff.changed == ["pkg/a.py"]
|
|
49
|
+
assert diff.removed == ["pkg/b.py"]
|
|
50
|
+
assert diff.unchanged == []
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_diff_manifest_unchanged(tmp_path):
|
|
54
|
+
root = _seed(tmp_path)
|
|
55
|
+
manifest = build_manifest(root)
|
|
56
|
+
diff = diff_manifest(manifest, dict(manifest))
|
|
57
|
+
assert diff.added == diff.changed == diff.removed == []
|
|
58
|
+
assert set(diff.unchanged) == {"pkg/a.py", "pkg/b.py"}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_iter_changed_files_none_previous_yields_all(tmp_path):
|
|
62
|
+
root = _seed(tmp_path)
|
|
63
|
+
changed = list(iter_changed_files(root, None))
|
|
64
|
+
assert {p.name for p, _ in changed} == {"a.py", "b.py"}
|
|
65
|
+
# Yielded hashes ARE the next manifest's values.
|
|
66
|
+
assert {h for _, h in changed} == set(build_manifest(root).values())
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_iter_changed_files_skips_unchanged(tmp_path):
|
|
70
|
+
root = _seed(tmp_path)
|
|
71
|
+
previous = build_manifest(root)
|
|
72
|
+
assert list(iter_changed_files(root, previous)) == []
|
|
73
|
+
|
|
74
|
+
(root / "pkg" / "a.py").write_text("def a():\n return 99\n")
|
|
75
|
+
changed = list(iter_changed_files(root, previous))
|
|
76
|
+
assert [p.name for p, _ in changed] == ["a.py"]
|
|
77
|
+
# The yielded hash reflects the NEW content.
|
|
78
|
+
assert changed[0][1] == build_manifest(root)["pkg/a.py"]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_single_file_root(tmp_path):
|
|
82
|
+
f = tmp_path / "solo.py"
|
|
83
|
+
f.write_text("def solo():\n return 0\n")
|
|
84
|
+
manifest = build_manifest(f)
|
|
85
|
+
assert set(manifest) == {"solo.py"}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# ── encoding fallback ────────────────────────────────────────────────────────
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_latin1_file_no_longer_skipped(tmp_path):
|
|
92
|
+
f = tmp_path / "legacy.py"
|
|
93
|
+
f.write_bytes("# caf\xe9\ndef caf():\n return 1\n".encode("latin-1"))
|
|
94
|
+
text = read_source_text(f)
|
|
95
|
+
assert text is not None
|
|
96
|
+
assert "caf\xe9" in text
|
|
97
|
+
|
|
98
|
+
parsed = parse_path(f)
|
|
99
|
+
assert parsed is not None
|
|
100
|
+
assert [s.simple_name for s in parsed.symbols] == ["caf"]
|
|
101
|
+
|
|
102
|
+
# And it participates in manifests deterministically.
|
|
103
|
+
manifest = build_manifest(tmp_path)
|
|
104
|
+
assert manifest["legacy.py"] == parsed.content_hash
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_utf8_bom_stripped(tmp_path):
|
|
108
|
+
f = tmp_path / "bom.py"
|
|
109
|
+
f.write_bytes(b"\xef\xbb\xbf" + b"def bom():\n return 1\n")
|
|
110
|
+
text = read_source_text(f)
|
|
111
|
+
assert text is not None
|
|
112
|
+
assert not text.startswith("")
|
|
113
|
+
parsed = parse_path(f)
|
|
114
|
+
assert parsed is not None
|
|
115
|
+
assert [s.simple_name for s in parsed.symbols] == ["bom"]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_fallback_disabled_restores_skip_behavior(tmp_path):
|
|
119
|
+
f = tmp_path / "legacy.py"
|
|
120
|
+
f.write_bytes("# caf\xe9\n".encode("latin-1"))
|
|
121
|
+
assert read_source_text(f, encoding_fallback=False) is None
|
|
122
|
+
assert parse_path(f, encoding_fallback=False) is None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_unreadable_file_returns_none(tmp_path):
|
|
126
|
+
assert read_source_text(tmp_path / "missing.py") is None
|
|
127
|
+
assert parse_path(tmp_path / "missing.py") is None
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Overlap correctness and no-data-loss guarantees for body-splitting.
|
|
2
|
+
|
|
3
|
+
Pins the fix for the minified-line hard-cut: a single line over the char budget
|
|
4
|
+
used to be truncated (``ln[:max_chars]``), silently dropping the remainder.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from victor_codegraph.config import ChunkConfig
|
|
10
|
+
from victor_codegraph.model import CodeSymbol, CodeSymbolType, SourceLocation
|
|
11
|
+
from victor_codegraph.sizing import chunks_for_symbol
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _cfg(max_tokens: int = 40, overlap_tokens: int = 8) -> ChunkConfig:
|
|
15
|
+
# Small budgets so tests exercise splitting with tiny fixtures.
|
|
16
|
+
return ChunkConfig(max_chunk_tokens=max_tokens, chunk_overlap_tokens=overlap_tokens)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _symbol(source: str, start_line: int = 10) -> CodeSymbol:
|
|
20
|
+
line_count = source.count("\n") + (0 if source.endswith("\n") else 1)
|
|
21
|
+
return CodeSymbol(
|
|
22
|
+
id="sym1",
|
|
23
|
+
symbol_type=CodeSymbolType.FUNCTION,
|
|
24
|
+
fully_qualified_name="f.py::big",
|
|
25
|
+
simple_name="big",
|
|
26
|
+
location=SourceLocation(
|
|
27
|
+
file_path="f.py",
|
|
28
|
+
start_line=start_line,
|
|
29
|
+
end_line=start_line + line_count - 1,
|
|
30
|
+
),
|
|
31
|
+
source_code=source,
|
|
32
|
+
language="python",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _multiline_source(n_lines: int = 120) -> str:
|
|
37
|
+
return "".join(f" value_{i} = compute_{i}(x)\n" for i in range(n_lines))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_no_chunk_exceeds_budget():
|
|
41
|
+
cfg = _cfg()
|
|
42
|
+
chunks = chunks_for_symbol(_symbol(_multiline_source()), cfg)
|
|
43
|
+
assert len(chunks) > 1
|
|
44
|
+
assert all(len(c.text) <= cfg.max_chunk_chars for c in chunks)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_every_source_line_appears_in_some_chunk():
|
|
48
|
+
cfg = _cfg()
|
|
49
|
+
source = _multiline_source()
|
|
50
|
+
chunks = chunks_for_symbol(_symbol(source), cfg)
|
|
51
|
+
joined = [c.text for c in chunks]
|
|
52
|
+
for line in source.splitlines():
|
|
53
|
+
assert any(line in text for text in joined), f"line lost: {line!r}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_consecutive_windows_overlap_by_whole_lines():
|
|
57
|
+
# Overlap is whole-line granular: give it budget for at least one line.
|
|
58
|
+
cfg = _cfg(max_tokens=40, overlap_tokens=16)
|
|
59
|
+
chunks = chunks_for_symbol(_symbol(_multiline_source()), cfg)
|
|
60
|
+
assert len(chunks) > 1
|
|
61
|
+
for a, b in zip(chunks, chunks[1:]):
|
|
62
|
+
# The next window begins with whole lines that terminate the previous one:
|
|
63
|
+
# some non-empty line-suffix of `a` is a prefix of `b`, within budget.
|
|
64
|
+
a_lines = a.text.splitlines(keepends=True)
|
|
65
|
+
overlap = 0
|
|
66
|
+
for k in range(len(a_lines)):
|
|
67
|
+
suffix = "".join(a_lines[k:])
|
|
68
|
+
if b.text.startswith(suffix):
|
|
69
|
+
overlap = len(suffix)
|
|
70
|
+
break
|
|
71
|
+
assert overlap > 0
|
|
72
|
+
assert overlap <= cfg.chunk_overlap_chars
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_body_split_window_line_metadata():
|
|
76
|
+
cfg = _cfg()
|
|
77
|
+
sym = _symbol(_multiline_source(), start_line=10)
|
|
78
|
+
chunks = chunks_for_symbol(sym, cfg)
|
|
79
|
+
for c in chunks:
|
|
80
|
+
start = c.metadata["start_line"]
|
|
81
|
+
end = c.metadata["end_line"]
|
|
82
|
+
assert start >= 10
|
|
83
|
+
assert end >= start
|
|
84
|
+
# end_line reflects THIS window, not the symbol's overall end.
|
|
85
|
+
spanned = c.text.count("\n") + (0 if c.text.endswith("\n") else 1)
|
|
86
|
+
assert end == start + spanned - 1
|
|
87
|
+
assert chunks[0].metadata["start_line"] == 10
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_minified_line_is_not_truncated():
|
|
91
|
+
"""A single line 10x over budget must be fully preserved across chunks."""
|
|
92
|
+
cfg = _cfg()
|
|
93
|
+
long_line = "x=1;" * (cfg.max_chunk_chars * 10 // 4)
|
|
94
|
+
source = long_line + "\n"
|
|
95
|
+
chunks = chunks_for_symbol(_symbol(source), cfg)
|
|
96
|
+
|
|
97
|
+
assert len(chunks) > 1
|
|
98
|
+
assert all(len(c.text) <= cfg.max_chunk_chars for c in chunks)
|
|
99
|
+
assert all(c.metadata.get("is_line_split") for c in chunks)
|
|
100
|
+
|
|
101
|
+
# Consecutive pieces advance by (max - overlap); every char offset is covered
|
|
102
|
+
# and the full line (incl. trailing newline) is reconstructible.
|
|
103
|
+
full = long_line + "\n"
|
|
104
|
+
step = cfg.max_chunk_chars - cfg.chunk_overlap_chars
|
|
105
|
+
buf = [""] * len(full)
|
|
106
|
+
for idx, c in enumerate(chunks):
|
|
107
|
+
start = idx * step
|
|
108
|
+
for j, ch in enumerate(c.text):
|
|
109
|
+
buf[start + j] = ch
|
|
110
|
+
assert "".join(buf) == full
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_minified_line_mixed_with_normal_lines():
|
|
114
|
+
cfg = _cfg()
|
|
115
|
+
long_line = "y=2;" * (cfg.max_chunk_chars // 2)
|
|
116
|
+
source = "def big():\n" + f" a = {long_line}\n" + " return a\n"
|
|
117
|
+
sym = _symbol(source)
|
|
118
|
+
chunks = chunks_for_symbol(sym, cfg)
|
|
119
|
+
text_all = "".join(c.text for c in chunks)
|
|
120
|
+
assert "def big():" in text_all
|
|
121
|
+
assert "return a" in text_all
|
|
122
|
+
# The long middle line's tail must not be lost.
|
|
123
|
+
assert text_all.count("y=2;") >= source.count("y=2;")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_small_symbol_single_chunk_has_no_line_split_flag():
|
|
127
|
+
cfg = _cfg(max_tokens=512, overlap_tokens=64)
|
|
128
|
+
sym = _symbol("def small():\n return 1\n")
|
|
129
|
+
chunks = chunks_for_symbol(sym, cfg)
|
|
130
|
+
assert len(chunks) == 1
|
|
131
|
+
assert "is_line_split" not in chunks[0].metadata
|
|
132
|
+
assert "is_body_split" not in chunks[0].metadata
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Pins the ADR-044 stable, line-independent symbol-oid contract.
|
|
2
|
+
|
|
3
|
+
The correlated-CPG join key must survive edits that don't change *what a symbol is*:
|
|
4
|
+
a line move keeps the oid; a rename / signature change mints a new one; a body edit
|
|
5
|
+
bumps ``content_version`` but not the oid. Every cross-surface consumer (Victor, the
|
|
6
|
+
ProximaDB SDK, the AnvaiOps connector) derives identity from this one function, so if it
|
|
7
|
+
ever drifts this test fails loudly.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from victor_codegraph import parse, stable_symbol_oid, to_proxima_records
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _node(records, name):
|
|
16
|
+
for r in records:
|
|
17
|
+
if "graph_node" in r.get("labels", []) and r["props"].get("name") == name:
|
|
18
|
+
return r
|
|
19
|
+
raise AssertionError(f"no node record for {name!r}")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _records(content, *, repo="repo1", stable=True):
|
|
23
|
+
return to_proxima_records(
|
|
24
|
+
parse(content, file_path="m.py"), repo_graph_id=repo, stable_oid=stable
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _oid_of(content, name):
|
|
29
|
+
return _node(_records(content), name)["oid"]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_stable_oid_unchanged_when_symbol_moves_lines():
|
|
33
|
+
base = "def first():\n return second()\n\n\ndef second():\n return 1\n"
|
|
34
|
+
moved = "\n\n\n" + base # both symbols pushed down; identity must not churn
|
|
35
|
+
assert _oid_of(base, "second") == _oid_of(moved, "second"), "line move must not churn the oid"
|
|
36
|
+
assert _oid_of(base, "first") == _oid_of(moved, "first")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_stable_oid_changes_on_rename():
|
|
40
|
+
a = "def alpha():\n return 1\n"
|
|
41
|
+
b = "def beta():\n return 1\n"
|
|
42
|
+
assert _oid_of(a, "alpha") != _oid_of(b, "beta"), "rename must mint a new identity"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_stable_oid_changes_on_signature_change():
|
|
46
|
+
one = "def f(x):\n return x\n"
|
|
47
|
+
two = "def f(x, y):\n return x\n"
|
|
48
|
+
assert _oid_of(one, "f") != _oid_of(two, "f"), "signature change must mint a new identity"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_body_edit_bumps_content_version_not_oid():
|
|
52
|
+
a = _node(_records("def f(x):\n return x\n"), "f")
|
|
53
|
+
b = _node(_records("def f(x):\n return x + 1 # changed body\n"), "f")
|
|
54
|
+
assert a["oid"] == b["oid"], "body edit must not churn the oid"
|
|
55
|
+
assert (
|
|
56
|
+
a["props"]["content_version"] != b["props"]["content_version"]
|
|
57
|
+
), "body edit must bump content_version"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_dual_emit_and_default_on_gate(monkeypatch):
|
|
61
|
+
monkeypatch.delenv("VICTOR_CODEGRAPH_STABLE_OID", raising=False)
|
|
62
|
+
parsed = parse("def f(x):\n return x\n", file_path="m.py")
|
|
63
|
+
# P2 cutover — default ON: the record oid is the canonical stable id; BOTH ids are
|
|
64
|
+
# always emitted for mixed-read.
|
|
65
|
+
on = _node(to_proxima_records(parsed, repo_graph_id="r"), "f")
|
|
66
|
+
assert on["oid"] == on["props"]["stable_oid"]
|
|
67
|
+
assert on["props"]["stable_oid"] != on["props"]["legacy_oid"]
|
|
68
|
+
# Explicit opt-out → legacy line-coupled id (existing collections stay byte-identical).
|
|
69
|
+
off = _node(to_proxima_records(parsed, repo_graph_id="r", stable_oid=False), "f")
|
|
70
|
+
assert off["oid"] == off["props"]["legacy_oid"]
|
|
71
|
+
assert off["props"]["stable_oid"] == on["props"]["stable_oid"]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_stable_symbol_oid_is_pure_and_structural():
|
|
75
|
+
# The pure derivation depends only on structural coordinates — never on line/col.
|
|
76
|
+
k = stable_symbol_oid("repo1", "python", "m.py::f", "(x)")
|
|
77
|
+
assert k == stable_symbol_oid("repo1", "python", "m.py::f", "(x)") # deterministic
|
|
78
|
+
assert k != stable_symbol_oid("repo2", "python", "m.py::f", "(x)") # repo
|
|
79
|
+
assert k != stable_symbol_oid("repo1", "rust", "m.py::f", "(x)") # language
|
|
80
|
+
assert k != stable_symbol_oid("repo1", "python", "m.py::g", "(x)") # name (FQN)
|
|
81
|
+
assert k != stable_symbol_oid("repo1", "python", "m.py::f", "(x, y)") # overload
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""ADR-044 P2 cutover **parity ratchet**.
|
|
2
|
+
|
|
3
|
+
The canonical oid becomes the default only if it is a SAFE, non-lossy replacement for the
|
|
4
|
+
legacy one. This ratchet pins the invariants that gate the cutover, over a representative
|
|
5
|
+
multi-symbol corpus:
|
|
6
|
+
|
|
7
|
+
1. **No collisions** — distinct symbols get distinct canonical oids (else the cutover
|
|
8
|
+
would merge two symbols into one record and lose data).
|
|
9
|
+
2. **Completeness** — every symbol gets a non-empty canonical oid + a legacy alias +
|
|
10
|
+
a content_version.
|
|
11
|
+
3. **Line-shift stability** — no symbol's canonical oid changes when lines move
|
|
12
|
+
(the incremental-re-index property the whole ADR exists for).
|
|
13
|
+
4. **Determinism** — re-parsing yields identical oids.
|
|
14
|
+
|
|
15
|
+
If any of these regresses, the cutover is unsafe and CI fails.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from victor_codegraph import parse, to_proxima_records
|
|
21
|
+
|
|
22
|
+
_CORPUS = """\
|
|
23
|
+
import os
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Auth:
|
|
27
|
+
def login(self, user):
|
|
28
|
+
return self._check(user)
|
|
29
|
+
|
|
30
|
+
def _check(self, user):
|
|
31
|
+
return bool(user)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def helper(x):
|
|
35
|
+
return x + 1
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def helper2(x, y):
|
|
39
|
+
return x + y
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _nodes(records):
|
|
44
|
+
return [r for r in records if "graph_node" in r["labels"]]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _canonical_nodes(src):
|
|
48
|
+
return _nodes(
|
|
49
|
+
to_proxima_records(parse(src, file_path="auth.py"), repo_graph_id="r", stable_oid=True)
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_no_collisions_distinct_symbols_distinct_oids():
|
|
54
|
+
recs = _canonical_nodes(_CORPUS)
|
|
55
|
+
oids = [r["oid"] for r in recs]
|
|
56
|
+
assert len(recs) >= 5, "Auth, login, _check, helper, helper2 expected"
|
|
57
|
+
assert len(oids) == len(set(oids)), "canonical oids must be unique per symbol (no merge)"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_completeness_every_symbol_has_canonical_legacy_and_version():
|
|
61
|
+
for r in _canonical_nodes(_CORPUS):
|
|
62
|
+
assert r["props"]["stable_oid"], "missing canonical oid"
|
|
63
|
+
assert r["props"]["legacy_oid"], "missing legacy alias"
|
|
64
|
+
assert r["props"]["content_version"], "missing content_version"
|
|
65
|
+
assert r["oid"] == r["props"]["stable_oid"], "gated record oid must be the canonical one"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_line_shift_stable_for_all_symbols():
|
|
69
|
+
base = {r["props"]["name"]: r["oid"] for r in _canonical_nodes(_CORPUS)}
|
|
70
|
+
moved = {r["props"]["name"]: r["oid"] for r in _canonical_nodes("\n\n\n" + _CORPUS)}
|
|
71
|
+
assert base == moved, "no symbol's canonical oid may change on a line shift"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_deterministic_reparse():
|
|
75
|
+
a = {r["props"]["name"]: r["oid"] for r in _canonical_nodes(_CORPUS)}
|
|
76
|
+
b = {r["props"]["name"]: r["oid"] for r in _canonical_nodes(_CORPUS)}
|
|
77
|
+
assert a == b
|
|
@@ -11,7 +11,10 @@ import pytest
|
|
|
11
11
|
pytest.importorskip("tree_sitter_language_pack")
|
|
12
12
|
|
|
13
13
|
from victor_codegraph import CodeSymbolType, parse # noqa: E402
|
|
14
|
-
from victor_codegraph.treesitter_parser import
|
|
14
|
+
from victor_codegraph.treesitter_parser import (
|
|
15
|
+
GrammarUnavailable,
|
|
16
|
+
parse_treesitter,
|
|
17
|
+
) # noqa: E402
|
|
15
18
|
|
|
16
19
|
JS = """\
|
|
17
20
|
import { x } from "./x";
|