cortex-local 2.0.7__tar.gz → 2.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortex_local-2.0.7 → cortex_local-2.0.8}/PKG-INFO +1 -1
- cortex_local-2.0.8/cortex/core/__init__.py +0 -0
- cortex_local-2.0.8/cortex/core/cache.py +25 -0
- cortex_local-2.0.8/cortex/core/chunker.py +141 -0
- cortex_local-2.0.8/cortex/core/code_graph.py +286 -0
- cortex_local-2.0.8/cortex/core/global_graph.py +87 -0
- cortex_local-2.0.8/cortex/core/graph.py +6 -0
- cortex_local-2.0.8/cortex/core/notes_graph.py +94 -0
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/indexer.py +3 -12
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/mcp_server.py +3 -11
- {cortex_local-2.0.7 → cortex_local-2.0.8}/pyproject.toml +1 -1
- {cortex_local-2.0.7 → cortex_local-2.0.8}/README.md +0 -0
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/__init__.py +0 -0
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/cli.py +0 -0
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/config.py +0 -0
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/embedder.py +0 -0
- {cortex_local-2.0.7 → cortex_local-2.0.8}/cortex/state.py +0 -0
|
File without changes
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Mtime-based embedding cache — skip re-embedding unchanged files on full reindex."""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_cache(section: str, cache_file: Path) -> dict:
|
|
9
|
+
"""Load {rel_path: mtime} for given section ('notes' or 'code')."""
|
|
10
|
+
try:
|
|
11
|
+
if cache_file.exists():
|
|
12
|
+
return json.loads(cache_file.read_text()).get(section, {})
|
|
13
|
+
except Exception:
|
|
14
|
+
pass
|
|
15
|
+
return {}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def save_cache(section: str, data: dict, cache_file: Path) -> None:
|
|
19
|
+
"""Persist updated cache section without touching other sections."""
|
|
20
|
+
try:
|
|
21
|
+
existing = json.loads(cache_file.read_text()) if cache_file.exists() else {}
|
|
22
|
+
existing[section] = data
|
|
23
|
+
cache_file.write_text(json.dumps(existing))
|
|
24
|
+
except Exception as e:
|
|
25
|
+
print(f"warn: cache save failed: {e}", flush=True)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Source code chunking — tree-sitter semantic parsing with sliding-window fallback."""
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
CODE_EXTS = {".js", ".ts", ".tsx", ".jsx", ".svelte", ".py", ".java", ".go", ".rs", ".css", ".html", ".kt", ".kts", ".gd", ".yml", ".yaml"}
|
|
7
|
+
SKIP_DIRS = {"node_modules", ".git", "dist", "build", ".next", ".svelte-kit", "__pycache__", ".gradle", "target"}
|
|
8
|
+
LANG_MAP = {
|
|
9
|
+
".js": "javascript", ".ts": "typescript", ".tsx": "typescript",
|
|
10
|
+
".jsx": "javascript", ".svelte": "svelte", ".py": "python",
|
|
11
|
+
".java": "java", ".go": "go", ".rs": "rust", ".css": "css", ".html": "html",
|
|
12
|
+
".kt": "kotlin", ".kts": "kotlin", ".gd": "gdscript",
|
|
13
|
+
".yml": "yaml", ".yaml": "yaml",
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
CHUNK_LINES = 30
|
|
17
|
+
OVERLAP_LINES = 5
|
|
18
|
+
MAX_CHUNK_LINES = 80
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from tree_sitter import Language, Parser as _TSParser
|
|
22
|
+
import tree_sitter_python as _tspy
|
|
23
|
+
import tree_sitter_javascript as _tsjs
|
|
24
|
+
import tree_sitter_typescript as _tsts
|
|
25
|
+
import tree_sitter_kotlin as _tskotlin
|
|
26
|
+
|
|
27
|
+
_TS_LANGUAGES: dict = {
|
|
28
|
+
".py": Language(_tspy.language()),
|
|
29
|
+
".js": Language(_tsjs.language()),
|
|
30
|
+
".jsx": Language(_tsjs.language()),
|
|
31
|
+
".ts": Language(_tsts.language_typescript()),
|
|
32
|
+
".tsx": Language(_tsts.language_tsx()),
|
|
33
|
+
".kt": Language(_tskotlin.language()),
|
|
34
|
+
".kts": Language(_tskotlin.language()),
|
|
35
|
+
}
|
|
36
|
+
_TS_SEMANTIC: dict = {
|
|
37
|
+
".py": {"function_definition", "class_definition"},
|
|
38
|
+
".js": {"function_declaration", "class_declaration", "method_definition", "lexical_declaration"},
|
|
39
|
+
".jsx": {"function_declaration", "class_declaration", "method_definition", "lexical_declaration"},
|
|
40
|
+
".ts": {"function_declaration", "class_declaration", "method_definition", "interface_declaration", "lexical_declaration"},
|
|
41
|
+
".tsx": {"function_declaration", "class_declaration", "method_definition", "interface_declaration", "lexical_declaration"},
|
|
42
|
+
".kt": {"function_declaration", "class_declaration", "object_declaration"},
|
|
43
|
+
".kts": {"function_declaration", "class_declaration", "object_declaration"},
|
|
44
|
+
}
|
|
45
|
+
_TS_AVAILABLE = True
|
|
46
|
+
except Exception:
|
|
47
|
+
_TS_AVAILABLE = False
|
|
48
|
+
_TS_LANGUAGES = {}
|
|
49
|
+
_TS_SEMANTIC = {}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _sliding_window(lines: list, start: int, end: int, repo_name: str, rel: str, language: str, github_url_base: str = "") -> list:
|
|
53
|
+
step = CHUNK_LINES - OVERLAP_LINES
|
|
54
|
+
chunks = []
|
|
55
|
+
i = start
|
|
56
|
+
while i < end:
|
|
57
|
+
j = min(i + CHUNK_LINES, end)
|
|
58
|
+
body = "\n".join(lines[i:j]).strip()
|
|
59
|
+
if body:
|
|
60
|
+
url = f"{github_url_base}/{rel}#L{i+1}-L{j}" if github_url_base else ""
|
|
61
|
+
chunks.append({
|
|
62
|
+
"repo": repo_name, "file": rel, "language": language,
|
|
63
|
+
"start_line": i + 1, "end_line": j,
|
|
64
|
+
"text": f"# {repo_name}/{rel} (lines {i+1}-{j})\n\n{body}",
|
|
65
|
+
"github_url": url,
|
|
66
|
+
})
|
|
67
|
+
if j == end:
|
|
68
|
+
break
|
|
69
|
+
i += step
|
|
70
|
+
return chunks
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _has_function_value(node, depth: int = 0) -> bool:
|
|
74
|
+
if node.type in ("arrow_function", "function_expression"):
|
|
75
|
+
return True
|
|
76
|
+
if depth >= 3:
|
|
77
|
+
return False
|
|
78
|
+
return any(_has_function_value(c, depth + 1) for c in node.children)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _collect_semantic_nodes(node, target_types: set, depth: int = 0, max_depth: int = 5) -> list:
|
|
82
|
+
if node.type in target_types:
|
|
83
|
+
return [node]
|
|
84
|
+
if depth >= max_depth:
|
|
85
|
+
return []
|
|
86
|
+
result = []
|
|
87
|
+
for child in node.children:
|
|
88
|
+
result.extend(_collect_semantic_nodes(child, target_types, depth + 1, max_depth))
|
|
89
|
+
return result
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def chunk_file(path: Path, repo_name: str, base_dir: Path | None = None, github_url_base: str = "") -> list:
|
|
93
|
+
"""Chunk a source file into indexable segments.
|
|
94
|
+
|
|
95
|
+
base_dir: directory to compute relative paths from (defaults to path.parent).
|
|
96
|
+
github_url_base: e.g. "https://github.com/owner/repo/blob/master"
|
|
97
|
+
"""
|
|
98
|
+
try:
|
|
99
|
+
source = path.read_bytes()
|
|
100
|
+
lines = source.decode("utf-8", errors="replace").splitlines()
|
|
101
|
+
except Exception:
|
|
102
|
+
return []
|
|
103
|
+
if not lines:
|
|
104
|
+
return []
|
|
105
|
+
|
|
106
|
+
ext = path.suffix
|
|
107
|
+
base = base_dir if base_dir else path.parent
|
|
108
|
+
try:
|
|
109
|
+
rel = str(path.relative_to(base))
|
|
110
|
+
except ValueError:
|
|
111
|
+
rel = path.name
|
|
112
|
+
language = LANG_MAP.get(ext, ext.lstrip("."))
|
|
113
|
+
|
|
114
|
+
if _TS_AVAILABLE and ext in _TS_LANGUAGES:
|
|
115
|
+
try:
|
|
116
|
+
parser = _TSParser(_TS_LANGUAGES[ext])
|
|
117
|
+
tree = parser.parse(source)
|
|
118
|
+
nodes = _collect_semantic_nodes(tree.root_node, _TS_SEMANTIC[ext])
|
|
119
|
+
nodes = [n for n in nodes if n.type != "lexical_declaration" or _has_function_value(n)]
|
|
120
|
+
if nodes:
|
|
121
|
+
chunks = []
|
|
122
|
+
for node in nodes:
|
|
123
|
+
s = node.start_point[0]
|
|
124
|
+
e = node.end_point[0] + 1
|
|
125
|
+
if e - s > MAX_CHUNK_LINES:
|
|
126
|
+
chunks.extend(_sliding_window(lines, s, e, repo_name, rel, language, github_url_base))
|
|
127
|
+
else:
|
|
128
|
+
body = "\n".join(lines[s:e]).strip()
|
|
129
|
+
if body:
|
|
130
|
+
url = f"{github_url_base}/{rel}#L{s+1}-L{e}" if github_url_base else ""
|
|
131
|
+
chunks.append({
|
|
132
|
+
"repo": repo_name, "file": rel, "language": language,
|
|
133
|
+
"start_line": s + 1, "end_line": e,
|
|
134
|
+
"text": f"# {repo_name}/{rel} (lines {s+1}-{e})\n\n{body}",
|
|
135
|
+
"github_url": url,
|
|
136
|
+
})
|
|
137
|
+
return chunks
|
|
138
|
+
except Exception:
|
|
139
|
+
pass
|
|
140
|
+
|
|
141
|
+
return _sliding_window(lines, 0, len(lines), repo_name, rel, language, github_url_base)
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Code graph extraction — import/call/inheritance edges, centrality, Louvain communities."""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
import networkx as nx
|
|
10
|
+
_NX_AVAILABLE = True
|
|
11
|
+
except ImportError:
|
|
12
|
+
_NX_AVAILABLE = False
|
|
13
|
+
print("Warning: networkx not installed — graph features disabled")
|
|
14
|
+
|
|
15
|
+
_IMPORT_RE: dict = {
|
|
16
|
+
"py": re.compile(r'^(?:from|import)\s+([\w.]+)', re.MULTILINE),
|
|
17
|
+
"js": re.compile(r'''(?:import|from|require)\s*\(?['"](\.[^'"]+|\$lib/[^'"]+)['"]'''),
|
|
18
|
+
"ts": re.compile(r'''(?:import|from|require)\s*\(?['"](\.[^'"]+|\$lib/[^'"]+)['"]'''),
|
|
19
|
+
"kt": re.compile(r'^import\s+([\w.]+)', re.MULTILINE),
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
_NAMED_IMPORT_RE = re.compile(
|
|
23
|
+
r'''import\s*\{([^}]+)\}\s*from\s*['"](\.[^'"]+|\$lib/[^'"]+)['"]'''
|
|
24
|
+
)
|
|
25
|
+
_DEFAULT_IMPORT_RE = re.compile(
|
|
26
|
+
r'''import\s+(\w+)\s+from\s*['"](\.[^'"]+|\$lib/[^'"]+)['"]'''
|
|
27
|
+
)
|
|
28
|
+
_CALL_RE = re.compile(r'\b(\w+)\s*\(')
|
|
29
|
+
|
|
30
|
+
_TOP_LEVEL_DEF_RE: dict = {
|
|
31
|
+
"py": re.compile(r'^(?:class|def)\s+(\w+)', re.MULTILINE),
|
|
32
|
+
"ts": re.compile(r'(?:export\s+)?(?:class|function|const|interface|type)\s+(\w+)'),
|
|
33
|
+
"kt": re.compile(r'(?:class|fun|object)\s+(\w+)'),
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
_INHERIT_RE: dict = {
|
|
37
|
+
"py": re.compile(r'^class\s+\w+\s*\(([^)]+)\)', re.MULTILINE),
|
|
38
|
+
"ts": re.compile(
|
|
39
|
+
r'class\s+\w+(?:\s+extends\s+(\w+))?(?:\s+implements\s+([\w,\s]+))?'
|
|
40
|
+
),
|
|
41
|
+
"kt": re.compile(
|
|
42
|
+
r'(?:class|interface|object)\s+\w+[^:{]*:\s*([\w,\s()]+?)(?:\{|$)',
|
|
43
|
+
re.MULTILINE,
|
|
44
|
+
),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
_LANG_KEY: dict = {
|
|
48
|
+
".py": "py", ".js": "js", ".jsx": "js",
|
|
49
|
+
".ts": "ts", ".tsx": "ts", ".svelte": "js",
|
|
50
|
+
".kt": "kt", ".kts": "kt",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
_EDGE_PRIORITY: dict = {"import": 0, "call": 1, "inherits": 2}
|
|
54
|
+
|
|
55
|
+
_PATH_ALIASES: dict = {
|
|
56
|
+
"$lib/": "src/lib/",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
_PY_SKIP_BASES = frozenset({
|
|
60
|
+
"object", "ABC", "ABCMeta", "BaseModel", "Exception", "BaseException",
|
|
61
|
+
"ValueError", "TypeError", "RuntimeError", "str", "int", "float",
|
|
62
|
+
"list", "dict", "set", "tuple", "Enum", "IntEnum",
|
|
63
|
+
})
|
|
64
|
+
_TS_SKIP_BASES = frozenset({"Error", "EventEmitter", "Component", "React"})
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _extract_imports(path: Path, ext: str) -> list:
|
|
68
|
+
key = _LANG_KEY.get(ext)
|
|
69
|
+
if not key:
|
|
70
|
+
return []
|
|
71
|
+
try:
|
|
72
|
+
src = path.read_text(encoding="utf-8", errors="replace")
|
|
73
|
+
except Exception:
|
|
74
|
+
return []
|
|
75
|
+
return [m.group(1) for m in _IMPORT_RE[key].finditer(src)]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _resolve_import(source_file: str, imp: str, all_files: set, ext: str):
|
|
79
|
+
for alias, target in _PATH_ALIASES.items():
|
|
80
|
+
if imp.startswith(alias):
|
|
81
|
+
imp = target + imp[len(alias):]
|
|
82
|
+
break
|
|
83
|
+
|
|
84
|
+
if imp.startswith("."):
|
|
85
|
+
base = Path(source_file).parent
|
|
86
|
+
candidate = (base / imp).as_posix().lstrip("/")
|
|
87
|
+
for try_ext in (ext, ".ts", ".tsx", ".js", ".jsx", ".svelte"):
|
|
88
|
+
if candidate + try_ext in all_files:
|
|
89
|
+
return candidate + try_ext
|
|
90
|
+
if candidate in all_files:
|
|
91
|
+
return candidate
|
|
92
|
+
for idx in ("/index.ts", "/index.js"):
|
|
93
|
+
if candidate + idx in all_files:
|
|
94
|
+
return candidate + idx
|
|
95
|
+
return None
|
|
96
|
+
elif "/" in imp:
|
|
97
|
+
for try_ext in (ext, ".ts", ".tsx", ".js", ".jsx", ".svelte"):
|
|
98
|
+
if imp + try_ext in all_files:
|
|
99
|
+
return imp + try_ext
|
|
100
|
+
if imp in all_files:
|
|
101
|
+
return imp
|
|
102
|
+
for idx in ("/index.ts", "/index.js"):
|
|
103
|
+
if imp + idx in all_files:
|
|
104
|
+
return imp + idx
|
|
105
|
+
return None
|
|
106
|
+
else:
|
|
107
|
+
for f in all_files:
|
|
108
|
+
stem = f.replace(ext, "").replace("/", ".")
|
|
109
|
+
if stem == imp or stem.endswith("." + imp):
|
|
110
|
+
return f
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _build_symbol_map(repo_path: Path, all_files: set) -> dict:
|
|
115
|
+
symbol_to_file: dict = {}
|
|
116
|
+
for rel in all_files:
|
|
117
|
+
ext = Path(rel).suffix
|
|
118
|
+
key = _LANG_KEY.get(ext)
|
|
119
|
+
pat = _TOP_LEVEL_DEF_RE.get(key)
|
|
120
|
+
if not pat:
|
|
121
|
+
continue
|
|
122
|
+
try:
|
|
123
|
+
src = (repo_path / rel).read_text(encoding="utf-8", errors="replace")
|
|
124
|
+
except Exception:
|
|
125
|
+
continue
|
|
126
|
+
for m in pat.finditer(src):
|
|
127
|
+
name = m.group(1)
|
|
128
|
+
if name not in symbol_to_file:
|
|
129
|
+
symbol_to_file[name] = rel
|
|
130
|
+
return symbol_to_file
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _extract_named_imports_map(src: str) -> dict:
|
|
134
|
+
result: dict = {}
|
|
135
|
+
for m in _NAMED_IMPORT_RE.finditer(src):
|
|
136
|
+
raw_path = m.group(2)
|
|
137
|
+
for part in m.group(1).split(","):
|
|
138
|
+
name = part.strip().split(" as ")[0].strip()
|
|
139
|
+
if name:
|
|
140
|
+
result[name] = raw_path
|
|
141
|
+
for m in _DEFAULT_IMPORT_RE.finditer(src):
|
|
142
|
+
result[m.group(1)] = m.group(2)
|
|
143
|
+
return result
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _extract_call_targets(path: Path, ext: str, all_files: set, rel: str) -> list:
|
|
147
|
+
if ext not in (".js", ".jsx", ".ts", ".tsx", ".svelte"):
|
|
148
|
+
return []
|
|
149
|
+
try:
|
|
150
|
+
src = path.read_text(encoding="utf-8", errors="replace")
|
|
151
|
+
except Exception:
|
|
152
|
+
return []
|
|
153
|
+
named = _extract_named_imports_map(src)
|
|
154
|
+
if not named:
|
|
155
|
+
return []
|
|
156
|
+
called = {m.group(1) for m in _CALL_RE.finditer(src)}
|
|
157
|
+
targets: set = set()
|
|
158
|
+
for name, imp_path in named.items():
|
|
159
|
+
if name in called:
|
|
160
|
+
resolved = _resolve_import(rel, imp_path, all_files, ext)
|
|
161
|
+
if resolved and resolved != rel:
|
|
162
|
+
targets.add(resolved)
|
|
163
|
+
return list(targets)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _extract_inheritance_targets(
|
|
167
|
+
path: Path, ext: str, all_files: set, rel: str, symbol_to_file: dict
|
|
168
|
+
) -> list:
|
|
169
|
+
key = _LANG_KEY.get(ext)
|
|
170
|
+
if key not in _INHERIT_RE:
|
|
171
|
+
return []
|
|
172
|
+
try:
|
|
173
|
+
src = path.read_text(encoding="utf-8", errors="replace")
|
|
174
|
+
except Exception:
|
|
175
|
+
return []
|
|
176
|
+
targets: set = set()
|
|
177
|
+
skip = _PY_SKIP_BASES if key == "py" else _TS_SKIP_BASES
|
|
178
|
+
|
|
179
|
+
if key == "py":
|
|
180
|
+
for m in _INHERIT_RE["py"].finditer(src):
|
|
181
|
+
for base in m.group(1).split(","):
|
|
182
|
+
base = base.strip().split(".")[0]
|
|
183
|
+
if base and base not in skip:
|
|
184
|
+
t = symbol_to_file.get(base)
|
|
185
|
+
if t and t != rel:
|
|
186
|
+
targets.add(t)
|
|
187
|
+
elif key == "ts":
|
|
188
|
+
for m in _INHERIT_RE["ts"].finditer(src):
|
|
189
|
+
for grp in (m.group(1), m.group(2)):
|
|
190
|
+
if not grp:
|
|
191
|
+
continue
|
|
192
|
+
for base in grp.split(","):
|
|
193
|
+
base = base.strip().split("<")[0].strip()
|
|
194
|
+
if base and base not in skip:
|
|
195
|
+
t = symbol_to_file.get(base)
|
|
196
|
+
if t and t != rel:
|
|
197
|
+
targets.add(t)
|
|
198
|
+
elif key == "kt":
|
|
199
|
+
for m in _INHERIT_RE["kt"].finditer(src):
|
|
200
|
+
for base in m.group(1).split(","):
|
|
201
|
+
base = re.sub(r'\(.*?\)', '', base).strip()
|
|
202
|
+
if base and base not in skip:
|
|
203
|
+
t = symbol_to_file.get(base)
|
|
204
|
+
if t and t != rel:
|
|
205
|
+
targets.add(t)
|
|
206
|
+
|
|
207
|
+
return list(targets)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _add_edge(G, src: str, dst: str, edge_type: str) -> None:
|
|
211
|
+
if G.has_edge(src, dst):
|
|
212
|
+
existing = G[src][dst].get("type", "import")
|
|
213
|
+
if _EDGE_PRIORITY.get(edge_type, 0) > _EDGE_PRIORITY.get(existing, 0):
|
|
214
|
+
G[src][dst]["type"] = edge_type
|
|
215
|
+
else:
|
|
216
|
+
G.add_edge(src, dst, type=edge_type)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def build_code_graph(repo_path: Path, all_files: set):
|
|
220
|
+
if not _NX_AVAILABLE:
|
|
221
|
+
return None
|
|
222
|
+
G = nx.DiGraph()
|
|
223
|
+
G.add_nodes_from(all_files)
|
|
224
|
+
symbol_to_file = _build_symbol_map(repo_path, all_files)
|
|
225
|
+
for rel in all_files:
|
|
226
|
+
path = repo_path / rel
|
|
227
|
+
ext = Path(rel).suffix
|
|
228
|
+
for imp in _extract_imports(path, ext):
|
|
229
|
+
resolved = _resolve_import(rel, imp, all_files, ext)
|
|
230
|
+
if resolved and resolved != rel:
|
|
231
|
+
_add_edge(G, rel, resolved, "import")
|
|
232
|
+
for target in _extract_call_targets(path, ext, all_files, rel):
|
|
233
|
+
_add_edge(G, rel, target, "call")
|
|
234
|
+
for target in _extract_inheritance_targets(path, ext, all_files, rel, symbol_to_file):
|
|
235
|
+
_add_edge(G, rel, target, "inherits")
|
|
236
|
+
return G
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def compute_code_metadata(G) -> dict:
|
|
240
|
+
if not _NX_AVAILABLE or G is None or len(G) == 0:
|
|
241
|
+
return {}
|
|
242
|
+
in_c = nx.in_degree_centrality(G)
|
|
243
|
+
out_c = nx.out_degree_centrality(G)
|
|
244
|
+
centrality = {n: round((in_c.get(n, 0) + out_c.get(n, 0)) / 2, 4) for n in G.nodes}
|
|
245
|
+
undirected = G.to_undirected()
|
|
246
|
+
communities: dict = {}
|
|
247
|
+
try:
|
|
248
|
+
from networkx.algorithms.community import louvain_communities
|
|
249
|
+
for i, comm in enumerate(louvain_communities(undirected, seed=42)):
|
|
250
|
+
for node in comm:
|
|
251
|
+
communities[node] = i
|
|
252
|
+
except Exception:
|
|
253
|
+
for i, comp in enumerate(nx.connected_components(undirected)):
|
|
254
|
+
for node in comp:
|
|
255
|
+
communities[node] = i
|
|
256
|
+
return {
|
|
257
|
+
node: {
|
|
258
|
+
"centrality": centrality.get(node, 0.0),
|
|
259
|
+
"community_id": communities.get(node, 0),
|
|
260
|
+
"imports": [v for _, v in G.out_edges(node)],
|
|
261
|
+
"imported_by": [u for u, _ in G.in_edges(node)],
|
|
262
|
+
}
|
|
263
|
+
for node in G.nodes
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def persist_code_graph(G, metadata: dict, repo_name: str, data_dir: Path) -> None:
|
|
268
|
+
if not _NX_AVAILABLE or G is None:
|
|
269
|
+
return
|
|
270
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
271
|
+
edge_counts = {t: sum(1 for _, _, d in G.edges(data=True) if d.get("type") == t)
|
|
272
|
+
for t in ("import", "call", "inherits")}
|
|
273
|
+
data = {
|
|
274
|
+
"nodes": [{"id": n, **metadata.get(n, {})} for n in G.nodes],
|
|
275
|
+
"edges": [
|
|
276
|
+
{"source": u, "target": v, "type": d.get("type", "import")}
|
|
277
|
+
for u, v, d in G.edges(data=True)
|
|
278
|
+
],
|
|
279
|
+
}
|
|
280
|
+
out = data_dir / f"graph_{repo_name}.json"
|
|
281
|
+
out.write_text(json.dumps(data))
|
|
282
|
+
print(
|
|
283
|
+
f" graph: {len(data['nodes'])} nodes, {len(data['edges'])} edges "
|
|
284
|
+
f"(import:{edge_counts['import']} call:{edge_counts['call']} "
|
|
285
|
+
f"inherits:{edge_counts['inherits']}) -> {out.name}"
|
|
286
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Global cross-repo graph — scans root config files for inter-repo references."""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
import networkx as nx
|
|
10
|
+
_NX_AVAILABLE = True
|
|
11
|
+
except ImportError:
|
|
12
|
+
_NX_AVAILABLE = False
|
|
13
|
+
|
|
14
|
+
_CROSS_REPO_CONFIG = {
|
|
15
|
+
"package.json", "requirements.txt", "go.mod", "pyproject.toml",
|
|
16
|
+
"Cargo.toml", "docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml",
|
|
17
|
+
}
|
|
18
|
+
_CROSS_REPO_SKIP = {
|
|
19
|
+
"node_modules", ".git", "dist", "build", ".next", ".svelte-kit",
|
|
20
|
+
"__pycache__", ".gradle", "target",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _slug_variants(name: str) -> set:
|
|
25
|
+
return {name, name.replace("-", "_"), name.replace("_", "-")}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _find_cross_repo_refs(src_repo: str, repo_path: Path, other_repos: list) -> dict:
|
|
29
|
+
if not other_repos:
|
|
30
|
+
return {}
|
|
31
|
+
variant_to_repo: dict = {}
|
|
32
|
+
for r in other_repos:
|
|
33
|
+
for v in _slug_variants(r):
|
|
34
|
+
variant_to_repo[v.lower()] = r
|
|
35
|
+
refs: dict = {}
|
|
36
|
+
|
|
37
|
+
def _check(path: Path) -> None:
|
|
38
|
+
try:
|
|
39
|
+
text = path.read_text(encoding="utf-8", errors="replace").lower()
|
|
40
|
+
except Exception:
|
|
41
|
+
return
|
|
42
|
+
rel = str(path.relative_to(repo_path))
|
|
43
|
+
for variant, dst in variant_to_repo.items():
|
|
44
|
+
if variant in text:
|
|
45
|
+
refs.setdefault(dst, [])
|
|
46
|
+
if rel not in refs[dst]:
|
|
47
|
+
refs[dst].append(rel)
|
|
48
|
+
|
|
49
|
+
for fname in _CROSS_REPO_CONFIG:
|
|
50
|
+
p = repo_path / fname
|
|
51
|
+
if p.exists():
|
|
52
|
+
_check(p)
|
|
53
|
+
for p in repo_path.glob(".env*"):
|
|
54
|
+
if p.is_file():
|
|
55
|
+
_check(p)
|
|
56
|
+
return refs
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def build_global_graph(repo_paths: dict, data_dir: Path) -> None:
|
|
60
|
+
"""Build graph_global.json with cross-repo edges. repo_paths = {repo_name: Path}"""
|
|
61
|
+
if not _NX_AVAILABLE:
|
|
62
|
+
return
|
|
63
|
+
repo_names = list(repo_paths.keys())
|
|
64
|
+
G = nx.DiGraph()
|
|
65
|
+
G.add_nodes_from(repo_names)
|
|
66
|
+
for src_repo, repo_path in repo_paths.items():
|
|
67
|
+
if not repo_path.exists():
|
|
68
|
+
continue
|
|
69
|
+
other_repos = [r for r in repo_names if r != src_repo]
|
|
70
|
+
refs = _find_cross_repo_refs(src_repo, repo_path, other_repos)
|
|
71
|
+
for dst_repo, files in refs.items():
|
|
72
|
+
if G.has_edge(src_repo, dst_repo):
|
|
73
|
+
merged = list(set(G[src_repo][dst_repo].get("files", []) + files))
|
|
74
|
+
G[src_repo][dst_repo]["files"] = merged
|
|
75
|
+
else:
|
|
76
|
+
G.add_edge(src_repo, dst_repo, type="cross-repo", files=files)
|
|
77
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
data = {
|
|
79
|
+
"nodes": [{"id": n} for n in G.nodes],
|
|
80
|
+
"edges": [
|
|
81
|
+
{"source": u, "target": v, "type": "cross-repo", "files": d.get("files", [])}
|
|
82
|
+
for u, v, d in G.edges(data=True)
|
|
83
|
+
],
|
|
84
|
+
}
|
|
85
|
+
out = data_dir / "graph_global.json"
|
|
86
|
+
out.write_text(json.dumps(data))
|
|
87
|
+
print(f" global graph: {len(repo_names)} repos, {len(data['edges'])} cross-repo edges -> {out.name}")
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Graph module facade — re-exports all graph building and PPR utilities."""
|
|
3
|
+
|
|
4
|
+
from .code_graph import build_code_graph, compute_code_metadata, persist_code_graph # noqa: F401
|
|
5
|
+
from .notes_graph import build_notes_graph, persist_notes_graph, ppr_augment # noqa: F401
|
|
6
|
+
from .global_graph import build_global_graph # noqa: F401
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Notes graph — Obsidian wikilink edges and Personalized PageRank augmentation."""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
import networkx as nx
|
|
10
|
+
_NX_AVAILABLE = True
|
|
11
|
+
except ImportError:
|
|
12
|
+
_NX_AVAILABLE = False
|
|
13
|
+
print("Warning: networkx not installed — graph features disabled")
|
|
14
|
+
|
|
15
|
+
_WIKILINK_RE = re.compile(r'\[\[([^\[\]|#]+?)(?:\|[^\[\]]+)?\]\]')
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def build_notes_graph(notes_dir: Path):
|
|
19
|
+
if not _NX_AVAILABLE:
|
|
20
|
+
return None
|
|
21
|
+
G = nx.DiGraph()
|
|
22
|
+
md_files = [
|
|
23
|
+
p for p in notes_dir.rglob("*.md")
|
|
24
|
+
if ".obsidian" not in p.parts and ".git" not in p.parts
|
|
25
|
+
]
|
|
26
|
+
stem_to_rel: dict = {}
|
|
27
|
+
for path in md_files:
|
|
28
|
+
rel = str(path.relative_to(notes_dir))
|
|
29
|
+
G.add_node(rel)
|
|
30
|
+
stem_to_rel[path.stem.lower()] = rel
|
|
31
|
+
for path in md_files:
|
|
32
|
+
rel = str(path.relative_to(notes_dir))
|
|
33
|
+
try:
|
|
34
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
35
|
+
except Exception:
|
|
36
|
+
continue
|
|
37
|
+
for m in _WIKILINK_RE.finditer(text):
|
|
38
|
+
target_stem = m.group(1).lower().split("/")[-1].split(".")[0]
|
|
39
|
+
target_rel = stem_to_rel.get(target_stem)
|
|
40
|
+
if target_rel and target_rel != rel:
|
|
41
|
+
G.add_edge(rel, target_rel, type="wikilink")
|
|
42
|
+
return G
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def persist_notes_graph(G, data_dir: Path) -> None:
|
|
46
|
+
if not _NX_AVAILABLE or G is None:
|
|
47
|
+
return
|
|
48
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
data = {
|
|
50
|
+
"nodes": [{"id": n, "centrality": 0, "community_id": 0} for n in G.nodes],
|
|
51
|
+
"edges": [{"source": u, "target": v} for u, v in G.edges],
|
|
52
|
+
}
|
|
53
|
+
out = data_dir / "graph_notes.json"
|
|
54
|
+
out.write_text(json.dumps(data))
|
|
55
|
+
print(f" notes graph: {len(data['nodes'])} nodes, {len(data['edges'])} wikilink edges -> {out.name}")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def ppr_augment(matched_files: list, scores: list, graph_path: Path, threshold: float = 0.03, _return_reason: bool = False):
|
|
59
|
+
"""Personalized PageRank over wikilink graph seeded from vector-matched notes."""
|
|
60
|
+
def _ret(results, reason):
|
|
61
|
+
return (results, reason) if _return_reason else results
|
|
62
|
+
|
|
63
|
+
if not _NX_AVAILABLE or not graph_path.exists():
|
|
64
|
+
return _ret([], "nx_or_missing")
|
|
65
|
+
try:
|
|
66
|
+
data = json.loads(graph_path.read_text())
|
|
67
|
+
G = nx.DiGraph()
|
|
68
|
+
G.add_nodes_from(
|
|
69
|
+
n["id"] if isinstance(n, dict) else n
|
|
70
|
+
for n in data.get("nodes", [])
|
|
71
|
+
)
|
|
72
|
+
G.add_edges_from([(e["source"], e["target"]) for e in data.get("edges", [])])
|
|
73
|
+
except Exception:
|
|
74
|
+
return _ret([], "exception")
|
|
75
|
+
if len(G) < 2:
|
|
76
|
+
return _ret([], "exception")
|
|
77
|
+
total = sum(scores) or 1.0
|
|
78
|
+
personalization: dict = {n: 1e-6 for n in G.nodes}
|
|
79
|
+
for f, s in zip(matched_files, scores):
|
|
80
|
+
if f in personalization:
|
|
81
|
+
personalization[f] += s / total
|
|
82
|
+
norm = sum(personalization.values())
|
|
83
|
+
personalization = {k: v / norm for k, v in personalization.items()}
|
|
84
|
+
try:
|
|
85
|
+
ppr = nx.pagerank(G, alpha=0.85, personalization=personalization, max_iter=100)
|
|
86
|
+
except Exception:
|
|
87
|
+
return _ret([], "exception")
|
|
88
|
+
matched_set = set(matched_files)
|
|
89
|
+
results = [
|
|
90
|
+
{"file": node, "ppr_score": round(score, 4)}
|
|
91
|
+
for node, score in sorted(ppr.items(), key=lambda x: -x[1])
|
|
92
|
+
if node not in matched_set and score >= threshold
|
|
93
|
+
][:5]
|
|
94
|
+
return _ret(results, "ok" if results else "below_threshold")
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
import hashlib
|
|
4
4
|
import re
|
|
5
|
-
import sys
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
|
|
8
7
|
from qdrant_client import QdrantClient
|
|
@@ -13,17 +12,9 @@ from qdrant_client.models import (
|
|
|
13
12
|
|
|
14
13
|
from .config import QDRANT_PATH, DATA_DIR, CACHE_FILE, VECTOR_SIZE
|
|
15
14
|
from .embedder import embed, sparse_embed
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
import
|
|
19
|
-
from pathlib import Path as _Path
|
|
20
|
-
_core = _Path(__file__).parent.parent.parent / "core"
|
|
21
|
-
if str(_core) not in _sys.path:
|
|
22
|
-
_sys.path.insert(0, str(_core))
|
|
23
|
-
|
|
24
|
-
from chunker import chunk_file, CODE_EXTS, SKIP_DIRS # noqa: E402
|
|
25
|
-
from cache import load_cache, save_cache # noqa: E402
|
|
26
|
-
import graph as _graph # noqa: E402
|
|
15
|
+
from .core.chunker import chunk_file, CODE_EXTS, SKIP_DIRS
|
|
16
|
+
from .core.cache import load_cache, save_cache
|
|
17
|
+
from .core import graph as _graph
|
|
27
18
|
|
|
28
19
|
|
|
29
20
|
def _qdrant() -> QdrantClient:
|
|
@@ -51,17 +51,9 @@ def _run_reindex(path_str: str) -> None:
|
|
|
51
51
|
|
|
52
52
|
def _ppr_block(matched_files: list, matched_scores: list) -> str | None:
|
|
53
53
|
try:
|
|
54
|
-
from core.notes_graph import ppr_augment
|
|
55
|
-
except
|
|
56
|
-
|
|
57
|
-
import sys
|
|
58
|
-
from pathlib import Path as _P
|
|
59
|
-
_core = _P(__file__).parent.parent.parent / "core"
|
|
60
|
-
if str(_core) not in sys.path:
|
|
61
|
-
sys.path.insert(0, str(_core))
|
|
62
|
-
from notes_graph import ppr_augment
|
|
63
|
-
except Exception:
|
|
64
|
-
return None
|
|
54
|
+
from .core.notes_graph import ppr_augment
|
|
55
|
+
except Exception:
|
|
56
|
+
return None
|
|
65
57
|
graph_path = DATA_DIR / "graph_notes.json"
|
|
66
58
|
extras, reason = ppr_augment(matched_files, matched_scores, graph_path, _return_reason=True)
|
|
67
59
|
if extras:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|