sciogen 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sciogen/__init__.py +24 -0
- sciogen/api.py +151 -0
- sciogen/cli/__init__.py +1 -0
- sciogen/cli/banner.py +19 -0
- sciogen/cli/console.py +10 -0
- sciogen/cli/explore/__init__.py +57 -0
- sciogen/cli/explore/template.html +341 -0
- sciogen/cli/main.py +322 -0
- sciogen/config.py +82 -0
- sciogen/discovery.py +125 -0
- sciogen/embed/__init__.py +51 -0
- sciogen/embed/hashing.py +53 -0
- sciogen/embed/nomic.py +34 -0
- sciogen/index/__init__.py +5 -0
- sciogen/index/pipeline.py +363 -0
- sciogen/ir.py +182 -0
- sciogen/mcp/__init__.py +1 -0
- sciogen/mcp/server.py +94 -0
- sciogen/normalize/__init__.py +32 -0
- sciogen/normalize/base.py +121 -0
- sciogen/normalize/generic.py +18 -0
- sciogen/normalize/javascript.py +365 -0
- sciogen/normalize/python.py +358 -0
- sciogen/parsing/__init__.py +1 -0
- sciogen/parsing/languages.py +67 -0
- sciogen/parsing/parser.py +46 -0
- sciogen/query/__init__.py +27 -0
- sciogen/query/engine.py +326 -0
- sciogen/query/results.py +172 -0
- sciogen/resolve/__init__.py +5 -0
- sciogen/resolve/resolver.py +398 -0
- sciogen/stores/__init__.py +1 -0
- sciogen/stores/graph.py +316 -0
- sciogen/stores/meta.py +324 -0
- sciogen/stores/vectors.py +108 -0
- sciogen-0.1.0.dist-info/METADATA +217 -0
- sciogen-0.1.0.dist-info/RECORD +40 -0
- sciogen-0.1.0.dist-info/WHEEL +4 -0
- sciogen-0.1.0.dist-info/entry_points.txt +2 -0
- sciogen-0.1.0.dist-info/licenses/LICENSE +21 -0
sciogen/cli/main.py
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
"""sciogen CLI — Typer for command routing, Rich for all terminal output.
|
|
2
|
+
|
|
3
|
+
Only `sciogen index` shows the banner + progress flow; every other command
|
|
4
|
+
prints its output and nothing else. Heavier modules (stores, pipeline) import
|
|
5
|
+
inside command bodies so `sciogen --help` stays instant.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
import typer
|
|
14
|
+
|
|
15
|
+
from sciogen import __version__
|
|
16
|
+
from sciogen.cli.banner import banner_text
|
|
17
|
+
from sciogen.cli.console import console
|
|
18
|
+
|
|
19
|
+
app = typer.Typer(
|
|
20
|
+
name="sciogen",
|
|
21
|
+
help="Codebase intelligence layer — a queryable knowledge graph over your code.",
|
|
22
|
+
add_completion=False,
|
|
23
|
+
rich_markup_mode="rich",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _version_callback(value: bool) -> None:
|
|
28
|
+
if value:
|
|
29
|
+
console.print(f"sciogen {__version__}")
|
|
30
|
+
raise typer.Exit()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@app.callback(invoke_without_command=True)
|
|
34
|
+
def _main(
|
|
35
|
+
ctx: typer.Context,
|
|
36
|
+
version: Optional[bool] = typer.Option(
|
|
37
|
+
None, "--version", callback=_version_callback, is_eager=True, help="Show version and exit."
|
|
38
|
+
),
|
|
39
|
+
) -> None:
|
|
40
|
+
"""sciogen — index once, query forever.
|
|
41
|
+
|
|
42
|
+
Bare `sciogen` indexes the current directory: banner, pipeline progress,
|
|
43
|
+
then the completion summary. Agents read that output directly; querying
|
|
44
|
+
goes through the subcommands or the MCP server (`sciogen mcp`).
|
|
45
|
+
"""
|
|
46
|
+
if ctx.invoked_subcommand is None:
|
|
47
|
+
index(Path("."), rebuild=False)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _open_graph(path: Path):
|
|
51
|
+
from sciogen.api import SciogenGraph
|
|
52
|
+
|
|
53
|
+
return SciogenGraph(path)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _fail(message: str) -> None:
|
|
57
|
+
console.print(f" ⚠ {message}")
|
|
58
|
+
raise typer.Exit(code=1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class _RichReporter:
|
|
62
|
+
"""Adapts pipeline progress onto rich.progress.Progress: one row per step,
|
|
63
|
+
Braille spinner while running, replaced in place with ✓ when done."""
|
|
64
|
+
|
|
65
|
+
def __init__(self, progress) -> None:
|
|
66
|
+
self.progress = progress
|
|
67
|
+
self.tasks: dict[str, int] = {}
|
|
68
|
+
|
|
69
|
+
def start(self, step: str, text: str) -> None:
|
|
70
|
+
self.tasks[step] = self.progress.add_task(text, total=1)
|
|
71
|
+
|
|
72
|
+
def finish(self, step: str) -> None:
|
|
73
|
+
self.progress.update(self.tasks[step], completed=1)
|
|
74
|
+
|
|
75
|
+
def warn(self, message: str) -> None:
|
|
76
|
+
# print above the live display, keeping the ⚠ single-line error format
|
|
77
|
+
self.progress.console.print(f" ⚠ {message}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# ------------------------------------------------------------------- index
|
|
81
|
+
|
|
82
|
+
@app.command()
|
|
83
|
+
def index(
|
|
84
|
+
path: Path = typer.Argument(Path("."), help="Project directory to index."),
|
|
85
|
+
rebuild: bool = typer.Option(False, "--rebuild", help="Discard the existing index and rebuild from scratch."),
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Build or incrementally update the knowledge graph for a codebase."""
|
|
88
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn
|
|
89
|
+
|
|
90
|
+
console.print(banner_text(__version__))
|
|
91
|
+
graph = _open_graph(path)
|
|
92
|
+
try:
|
|
93
|
+
with Progress(
|
|
94
|
+
SpinnerColumn(spinner_name="dots", finished_text="✓"),
|
|
95
|
+
TextColumn("{task.description}"),
|
|
96
|
+
console=console,
|
|
97
|
+
) as progress:
|
|
98
|
+
stats = graph.index(reporter=_RichReporter(progress), rebuild=rebuild)
|
|
99
|
+
console.print(
|
|
100
|
+
f"✓ Done! {stats.nodes:,} nodes · {stats.edges:,} edges · indexed in {stats.seconds:.1f}s"
|
|
101
|
+
)
|
|
102
|
+
console.print(" Ready. Your codebase is now queryable.", style="dim")
|
|
103
|
+
except Exception:
|
|
104
|
+
console.print_exception()
|
|
105
|
+
raise typer.Exit(code=1)
|
|
106
|
+
finally:
|
|
107
|
+
graph.close()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# ----------------------------------------------------------------- queries
|
|
111
|
+
|
|
112
|
+
@app.command()
|
|
113
|
+
def search(
|
|
114
|
+
query: str = typer.Argument(..., help="Natural-language query."),
|
|
115
|
+
path: Path = typer.Option(Path("."), "--path", "-p", help="Indexed project directory."),
|
|
116
|
+
mode: str = typer.Option("semantic", "--mode", "-m", help="semantic | hybrid"),
|
|
117
|
+
granularity: Optional[str] = typer.Option(None, "--granularity", "-g", help="file | class | function | chunk"),
|
|
118
|
+
k: int = typer.Option(10, "--top", "-k", help="Number of results."),
|
|
119
|
+
) -> None:
|
|
120
|
+
"""Semantic or hybrid search over the indexed codebase."""
|
|
121
|
+
from rich.table import Table
|
|
122
|
+
|
|
123
|
+
graph = _open_graph(path)
|
|
124
|
+
try:
|
|
125
|
+
results = graph.search(query, mode=mode, granularity=granularity, k=k)
|
|
126
|
+
if not results.hits:
|
|
127
|
+
console.print(" No results.", style="dim")
|
|
128
|
+
return
|
|
129
|
+
table = Table(box=None, pad_edge=False, header_style="bold")
|
|
130
|
+
table.add_column("score", justify="right", style="cyan")
|
|
131
|
+
table.add_column("kind", style="dim")
|
|
132
|
+
table.add_column("symbol")
|
|
133
|
+
table.add_column("location", style="dim")
|
|
134
|
+
for hit in results.hits:
|
|
135
|
+
location = f"{hit.node.file}:{hit.node.line_start}" if hit.node.file else "—"
|
|
136
|
+
via = "" if hit.via == "semantic" else " [dim](via graph)[/dim]"
|
|
137
|
+
table.add_row(f"{hit.score:.2f}", hit.node.kind, f"{hit.node.name}{via}", location)
|
|
138
|
+
console.print(table)
|
|
139
|
+
except Exception as exc:
|
|
140
|
+
_handle_query_error(exc)
|
|
141
|
+
finally:
|
|
142
|
+
graph.close()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@app.command()
|
|
146
|
+
def callers(
|
|
147
|
+
symbol: str = typer.Argument(..., help="Symbol name, e.g. AuthService.login"),
|
|
148
|
+
path: Path = typer.Option(Path("."), "--path", "-p"),
|
|
149
|
+
depth: int = typer.Option(1, "--depth", "-d", help="Hops up the call chain."),
|
|
150
|
+
min_confidence: float = typer.Option(0.0, "--min-confidence", "-c"),
|
|
151
|
+
) -> None:
|
|
152
|
+
"""Who calls a symbol, up to N hops."""
|
|
153
|
+
_print_impact_table(path, symbol, depth, min_confidence, callers_only=True)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@app.command()
|
|
157
|
+
def impact(
|
|
158
|
+
symbol: str = typer.Argument(..., help="Symbol name, e.g. UserModel.find_by_email"),
|
|
159
|
+
path: Path = typer.Option(Path("."), "--path", "-p"),
|
|
160
|
+
depth: int = typer.Option(5, "--depth", "-d"),
|
|
161
|
+
min_confidence: float = typer.Option(0.0, "--min-confidence", "-c"),
|
|
162
|
+
) -> None:
|
|
163
|
+
"""What breaks if a symbol changes (callers, subclasses, tests, mutators)."""
|
|
164
|
+
_print_impact_table(path, symbol, depth, min_confidence, callers_only=False)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _print_impact_table(path: Path, symbol: str, depth: int, min_confidence: float, callers_only: bool) -> None:
|
|
168
|
+
from rich.table import Table
|
|
169
|
+
|
|
170
|
+
graph = _open_graph(path)
|
|
171
|
+
try:
|
|
172
|
+
if callers_only:
|
|
173
|
+
result = graph.get_callers(symbol, depth, min_confidence)
|
|
174
|
+
else:
|
|
175
|
+
result = graph.analyze_impact(symbol, depth, min_confidence)
|
|
176
|
+
target_names = ", ".join(t.name for t in result.targets)
|
|
177
|
+
if not result.impacted:
|
|
178
|
+
console.print(f" Nothing reaches [bold]{target_names}[/bold] within {depth} hops.", style="dim")
|
|
179
|
+
return
|
|
180
|
+
table = Table(box=None, pad_edge=False, header_style="bold",
|
|
181
|
+
title=f"{len(result.impacted)} symbols → {target_names}", title_justify="left")
|
|
182
|
+
table.add_column("hops", justify="right", style="cyan")
|
|
183
|
+
table.add_column("conf", justify="right")
|
|
184
|
+
table.add_column("via", style="dim")
|
|
185
|
+
table.add_column("symbol")
|
|
186
|
+
table.add_column("location", style="dim")
|
|
187
|
+
for item in result.impacted:
|
|
188
|
+
style = "" if item.confidence >= 0.8 else ("yellow" if item.confidence >= 0.5 else "red")
|
|
189
|
+
location = f"{item.node.file}:{item.node.line_start}" if item.node.file else "—"
|
|
190
|
+
table.add_row(
|
|
191
|
+
str(item.hops), f"[{style}]{item.confidence:.2f}[/{style}]" if style else f"{item.confidence:.2f}",
|
|
192
|
+
item.relation, item.node.name, location,
|
|
193
|
+
)
|
|
194
|
+
console.print(table)
|
|
195
|
+
except Exception as exc:
|
|
196
|
+
_handle_query_error(exc)
|
|
197
|
+
finally:
|
|
198
|
+
graph.close()
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@app.command()
|
|
202
|
+
def deps(
|
|
203
|
+
file: str = typer.Argument(..., help="Project-relative file path."),
|
|
204
|
+
path: Path = typer.Option(Path("."), "--path", "-p"),
|
|
205
|
+
) -> None:
|
|
206
|
+
"""All imports and transitive dependencies of a file."""
|
|
207
|
+
graph = _open_graph(path)
|
|
208
|
+
try:
|
|
209
|
+
info = graph.get_dependencies(file)
|
|
210
|
+
for title, items in (
|
|
211
|
+
("direct", info.direct_files),
|
|
212
|
+
("transitive", info.transitive_files),
|
|
213
|
+
("external", info.external_modules),
|
|
214
|
+
):
|
|
215
|
+
console.print(f"[bold]{title}[/bold] ({len(items)})")
|
|
216
|
+
for item in items:
|
|
217
|
+
console.print(f" {item}")
|
|
218
|
+
except Exception as exc:
|
|
219
|
+
_handle_query_error(exc)
|
|
220
|
+
finally:
|
|
221
|
+
graph.close()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@app.command()
|
|
225
|
+
def symbol(
|
|
226
|
+
name: str = typer.Argument(..., help="Symbol name to look up."),
|
|
227
|
+
path: Path = typer.Option(Path("."), "--path", "-p"),
|
|
228
|
+
file: Optional[str] = typer.Option(None, "--file", "-f", help="Narrow to a defining file."),
|
|
229
|
+
) -> None:
|
|
230
|
+
"""Typed node info for a named symbol."""
|
|
231
|
+
graph = _open_graph(path)
|
|
232
|
+
try:
|
|
233
|
+
info = graph.get_symbol(name, file)
|
|
234
|
+
n = info.node
|
|
235
|
+
console.print(f"[bold]{n.name}[/bold] [dim]{n.kind}[/dim]")
|
|
236
|
+
console.print(f" {n.file}:{n.line_start}-{n.line_end}" if n.file else " (no source location)")
|
|
237
|
+
if n.params:
|
|
238
|
+
console.print(f" params: {', '.join(n.params)}")
|
|
239
|
+
if n.return_type:
|
|
240
|
+
console.print(f" returns: {n.return_type}")
|
|
241
|
+
if n.extends:
|
|
242
|
+
console.print(f" extends: {', '.join(n.extends)}")
|
|
243
|
+
if n.complexity:
|
|
244
|
+
console.print(f" complexity: {n.complexity}")
|
|
245
|
+
if n.docstring:
|
|
246
|
+
console.print(f" [dim]{n.docstring}[/dim]")
|
|
247
|
+
if info.relations:
|
|
248
|
+
console.print(f" [bold]relations[/bold] ({len(info.relations)})")
|
|
249
|
+
for rel in info.relations[:30]:
|
|
250
|
+
arrow = "→" if rel.direction == "out" else "←"
|
|
251
|
+
console.print(f" {arrow} {rel.kind} {rel.other} [dim]{rel.confidence:.2f}[/dim]")
|
|
252
|
+
if len(info.relations) > 30:
|
|
253
|
+
console.print(f" [dim]… {len(info.relations) - 30} more[/dim]")
|
|
254
|
+
except Exception as exc:
|
|
255
|
+
_handle_query_error(exc)
|
|
256
|
+
finally:
|
|
257
|
+
graph.close()
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# ----------------------------------------------------------------- explore
|
|
261
|
+
|
|
262
|
+
@app.command()
|
|
263
|
+
def explore(
|
|
264
|
+
path: Path = typer.Argument(Path("."), help="Indexed project directory (or a subdirectory to scope the view)."),
|
|
265
|
+
) -> None:
|
|
266
|
+
"""Open an interactive visual view of the knowledge graph in the browser."""
|
|
267
|
+
import webbrowser
|
|
268
|
+
|
|
269
|
+
target = path.resolve()
|
|
270
|
+
root, prefix = target, ""
|
|
271
|
+
# Allow scoping to a subdirectory: find the enclosing indexed project.
|
|
272
|
+
while not (root / ".sciogen").exists() and root.parent != root:
|
|
273
|
+
root = root.parent
|
|
274
|
+
if (root / ".sciogen").exists() and root != target:
|
|
275
|
+
prefix = target.relative_to(root).as_posix() + "/"
|
|
276
|
+
elif not (root / ".sciogen").exists():
|
|
277
|
+
_fail(f"No sciogen index found at or above {target} — run `sciogen index` first.")
|
|
278
|
+
|
|
279
|
+
from sciogen.cli.explore import generate_html
|
|
280
|
+
|
|
281
|
+
graph = _open_graph(root)
|
|
282
|
+
try:
|
|
283
|
+
snapshot = graph.snapshot(prefix)
|
|
284
|
+
if not snapshot["nodes"]:
|
|
285
|
+
_fail("The index is empty — run `sciogen index` first.")
|
|
286
|
+
out = generate_html(snapshot, root.name, graph.config.data_dir / "explore.html")
|
|
287
|
+
console.print(f" Graph written to {out}")
|
|
288
|
+
console.print(" Snapshot view — re-run [bold]sciogen index[/bold] then "
|
|
289
|
+
"[bold]sciogen explore[/bold] after code changes.", style="dim")
|
|
290
|
+
webbrowser.open(out.resolve().as_uri())
|
|
291
|
+
except Exception as exc:
|
|
292
|
+
_handle_query_error(exc)
|
|
293
|
+
finally:
|
|
294
|
+
graph.close()
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
# --------------------------------------------------------------------- mcp
|
|
298
|
+
|
|
299
|
+
@app.command()
|
|
300
|
+
def mcp(
|
|
301
|
+
path: Path = typer.Argument(Path("."), help="Project directory the MCP server exposes."),
|
|
302
|
+
) -> None:
|
|
303
|
+
"""Run the MCP server (stdio) for agent integration."""
|
|
304
|
+
from sciogen.mcp.server import serve
|
|
305
|
+
|
|
306
|
+
serve(path)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# ------------------------------------------------------------------ errors
|
|
310
|
+
|
|
311
|
+
def _handle_query_error(exc: Exception) -> None:
|
|
312
|
+
from sciogen.query.engine import SymbolNotFound
|
|
313
|
+
from sciogen.stores.vectors import EmbedderMismatch
|
|
314
|
+
|
|
315
|
+
if isinstance(exc, (SymbolNotFound, EmbedderMismatch, FileNotFoundError)):
|
|
316
|
+
_fail(str(exc))
|
|
317
|
+
console.print_exception()
|
|
318
|
+
raise typer.Exit(code=1)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
if __name__ == "__main__":
|
|
322
|
+
app()
|
sciogen/config.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Runtime configuration.
|
|
2
|
+
|
|
3
|
+
All knobs have working defaults; nothing requires a config file. Environment
|
|
4
|
+
variables override defaults so CI and agents can tune behavior without code:
|
|
5
|
+
|
|
6
|
+
=========================== =================================================
|
|
7
|
+
``SCIOGEN_EMBEDDER`` ``nomic`` (default) or ``hash``. ``nomic`` falls
|
|
8
|
+
back to ``hash`` automatically when
|
|
9
|
+
sentence-transformers is not installed.
|
|
10
|
+
``SCIOGEN_WORKERS`` Parse worker processes. Default: ``os.cpu_count()``.
|
|
11
|
+
``0``/``1`` disables multiprocessing.
|
|
12
|
+
``SCIOGEN_DATA_DIR`` Index location, default ``<project>/.sciogen``.
|
|
13
|
+
=========================== =================================================
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
#: Directories never worth indexing. Discovery also honors .gitignore-style
|
|
23
|
+
#: hidden-dir skipping (any path segment starting with ".").
|
|
24
|
+
DEFAULT_IGNORED_DIRS = frozenset(
|
|
25
|
+
{
|
|
26
|
+
"node_modules", "__pycache__", "venv", ".venv", "env", "dist", "build",
|
|
27
|
+
"target", "vendor", "coverage", "htmlcov", ".sciogen", "site-packages",
|
|
28
|
+
".git", ".hg", ".svn", ".tox", ".mypy_cache", ".ruff_cache", ".pytest_cache",
|
|
29
|
+
}
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
#: Functions longer than this many lines are additionally split into
|
|
33
|
+
#: chunk-level embedding units (ChunkNode).
|
|
34
|
+
CHUNK_THRESHOLD_LINES = 40
|
|
35
|
+
CHUNK_SIZE_LINES = 30
|
|
36
|
+
|
|
37
|
+
SCHEMA_VERSION = 1
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(slots=True)
|
|
41
|
+
class Config:
|
|
42
|
+
"""Resolved configuration for one project index."""
|
|
43
|
+
|
|
44
|
+
project_root: Path
|
|
45
|
+
data_dir: Path
|
|
46
|
+
embedder: str = "nomic"
|
|
47
|
+
workers: int = 0 # 0 = auto (cpu_count), 1 = serial
|
|
48
|
+
min_files_for_parallel: int = 32 # below this, process-pool startup costs more than it saves
|
|
49
|
+
embed_batch_size: int = 64
|
|
50
|
+
ignored_dirs: frozenset[str] = field(default_factory=lambda: DEFAULT_IGNORED_DIRS)
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def for_project(cls, project_root: str | Path) -> "Config":
|
|
54
|
+
root = Path(project_root).resolve()
|
|
55
|
+
data_dir = Path(os.environ.get("SCIOGEN_DATA_DIR", root / ".sciogen"))
|
|
56
|
+
workers_env = os.environ.get("SCIOGEN_WORKERS", "")
|
|
57
|
+
return cls(
|
|
58
|
+
project_root=root,
|
|
59
|
+
data_dir=data_dir,
|
|
60
|
+
embedder=os.environ.get("SCIOGEN_EMBEDDER", "nomic"),
|
|
61
|
+
workers=int(workers_env) if workers_env.isdigit() else 0,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def graph_dir(self) -> Path:
|
|
66
|
+
return self.data_dir / "graph.kuzu"
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def vector_dir(self) -> Path:
|
|
70
|
+
return self.data_dir / "vectors"
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def meta_path(self) -> Path:
|
|
74
|
+
return self.data_dir / "meta.db"
|
|
75
|
+
|
|
76
|
+
def effective_workers(self, n_files: int) -> int:
|
|
77
|
+
"""Parallelism actually used for a batch of ``n_files``."""
|
|
78
|
+
if n_files < self.min_files_for_parallel:
|
|
79
|
+
return 1
|
|
80
|
+
if self.workers >= 1:
|
|
81
|
+
return self.workers
|
|
82
|
+
return max(1, min(os.cpu_count() or 1, 8))
|
sciogen/discovery.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""File discovery and two-tier change detection.
|
|
2
|
+
|
|
3
|
+
Discovery walks the project tree once, skipping ignored/hidden directories,
|
|
4
|
+
and returns every file whose extension maps to a supported language.
|
|
5
|
+
|
|
6
|
+
Change detection is deliberately two-tier — this is the hot path of
|
|
7
|
+
incremental re-indexing on large repos:
|
|
8
|
+
|
|
9
|
+
1. **Fast path (stat only):** if a file's ``(size, mtime_ns)`` pair matches
|
|
10
|
+
what SQLite recorded last time, the file is assumed unchanged and never
|
|
11
|
+
opened. A 100k-file repo re-index costs 100k ``stat`` calls, not 100k
|
|
12
|
+
reads + hashes.
|
|
13
|
+
2. **Slow path (content hash):** only files whose stat signature changed are
|
|
14
|
+
read and SHA256-hashed. If the hash still matches (e.g. ``touch``,
|
|
15
|
+
checkout that restored content), the stored stat signature is refreshed
|
|
16
|
+
but the file is *not* re-parsed.
|
|
17
|
+
|
|
18
|
+
SHA256 remains the source of truth — mtime alone is never trusted to declare
|
|
19
|
+
a file changed, and stat matching is never trusted across schema migrations
|
|
20
|
+
(the pipeline forces a full rebuild in that case).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
import os
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from sciogen.config import Config
|
|
31
|
+
from sciogen.parsing.languages import language_for_path
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(slots=True)
|
|
35
|
+
class DiscoveredFile:
|
|
36
|
+
path: str # project-relative, "/" separators
|
|
37
|
+
abs_path: str
|
|
38
|
+
language: str
|
|
39
|
+
size: int
|
|
40
|
+
mtime: float
|
|
41
|
+
mtime_ns: int
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(slots=True)
|
|
45
|
+
class ChangeSet:
|
|
46
|
+
"""Result of diffing the working tree against the stored index state."""
|
|
47
|
+
|
|
48
|
+
changed: list[DiscoveredFile] # new or content-changed -> re-parse
|
|
49
|
+
unchanged: list[DiscoveredFile] # skipped entirely
|
|
50
|
+
deleted: list[str] # relative paths present in the index but gone on disk
|
|
51
|
+
stat_refreshed: list[DiscoveredFile] # content identical, stat signature updated
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def discover(config: Config) -> list[DiscoveredFile]:
|
|
55
|
+
"""Walk the project and return all indexable source files."""
|
|
56
|
+
root = config.project_root
|
|
57
|
+
found: list[DiscoveredFile] = []
|
|
58
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
59
|
+
# Prune in place: os.walk never descends into removed entries.
|
|
60
|
+
dirnames[:] = [
|
|
61
|
+
d for d in dirnames if d not in config.ignored_dirs and not d.startswith(".")
|
|
62
|
+
]
|
|
63
|
+
for fname in filenames:
|
|
64
|
+
language = language_for_path(fname)
|
|
65
|
+
if language is None:
|
|
66
|
+
continue
|
|
67
|
+
abs_path = os.path.join(dirpath, fname)
|
|
68
|
+
try:
|
|
69
|
+
st = os.stat(abs_path)
|
|
70
|
+
except OSError:
|
|
71
|
+
continue
|
|
72
|
+
rel = os.path.relpath(abs_path, root).replace(os.sep, "/")
|
|
73
|
+
found.append(
|
|
74
|
+
DiscoveredFile(
|
|
75
|
+
path=rel,
|
|
76
|
+
abs_path=abs_path,
|
|
77
|
+
language=language,
|
|
78
|
+
size=st.st_size,
|
|
79
|
+
mtime=st.st_mtime,
|
|
80
|
+
mtime_ns=st.st_mtime_ns,
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
found.sort(key=lambda f: f.path)
|
|
84
|
+
return found
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def sha256_file(abs_path: str) -> str:
|
|
88
|
+
h = hashlib.sha256()
|
|
89
|
+
with open(abs_path, "rb") as fh:
|
|
90
|
+
for block in iter(lambda: fh.read(1 << 20), b""):
|
|
91
|
+
h.update(block)
|
|
92
|
+
return h.hexdigest()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def compute_changes(
|
|
96
|
+
files: list[DiscoveredFile],
|
|
97
|
+
stored: dict[str, tuple[str, int, int]],
|
|
98
|
+
) -> ChangeSet:
|
|
99
|
+
"""Diff discovered files against stored state.
|
|
100
|
+
|
|
101
|
+
``stored`` maps relative path -> ``(sha256, size, mtime_ns)`` as recorded
|
|
102
|
+
by the metadata store after the previous index run.
|
|
103
|
+
"""
|
|
104
|
+
changed: list[DiscoveredFile] = []
|
|
105
|
+
unchanged: list[DiscoveredFile] = []
|
|
106
|
+
stat_refreshed: list[DiscoveredFile] = []
|
|
107
|
+
seen: set[str] = set()
|
|
108
|
+
|
|
109
|
+
for f in files:
|
|
110
|
+
seen.add(f.path)
|
|
111
|
+
record = stored.get(f.path)
|
|
112
|
+
if record is not None:
|
|
113
|
+
old_sha, old_size, old_mtime_ns = record
|
|
114
|
+
if f.size == old_size and f.mtime_ns == old_mtime_ns:
|
|
115
|
+
unchanged.append(f) # fast path: stat signature identical
|
|
116
|
+
continue
|
|
117
|
+
if sha256_file(f.abs_path) == old_sha:
|
|
118
|
+
stat_refreshed.append(f) # touched but content identical
|
|
119
|
+
continue
|
|
120
|
+
changed.append(f)
|
|
121
|
+
|
|
122
|
+
deleted = sorted(path for path in stored if path not in seen)
|
|
123
|
+
return ChangeSet(
|
|
124
|
+
changed=changed, unchanged=unchanged, deleted=deleted, stat_refreshed=stat_refreshed
|
|
125
|
+
)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Embedders — pluggable text -> vector functions.
|
|
2
|
+
|
|
3
|
+
sciogen computes embeddings itself and hands raw vectors to ChromaDB (rather
|
|
4
|
+
than registering an embedding function with the collection). That keeps the
|
|
5
|
+
embedder swappable without ChromaDB persistence-config coupling, and lets the
|
|
6
|
+
pipeline batch + deduplicate embedding work.
|
|
7
|
+
|
|
8
|
+
Two implementations:
|
|
9
|
+
|
|
10
|
+
- :class:`~sciogen.embed.nomic.NomicEmbedder` — the documented default,
|
|
11
|
+
local nomic-embed-code via sentence-transformers (optional install:
|
|
12
|
+
``pip install sciogen[embeddings]``).
|
|
13
|
+
- :class:`~sciogen.embed.hashing.HashingEmbedder` — deterministic feature
|
|
14
|
+
hashing. Zero dependencies, instant, no model download. Used automatically
|
|
15
|
+
when sentence-transformers is unavailable, and always in tests. Retrieval
|
|
16
|
+
quality is keyword-level rather than semantic; fine for structural work,
|
|
17
|
+
not a substitute for real embeddings.
|
|
18
|
+
|
|
19
|
+
The embedder's identity is recorded in the vector store's metadata; mixing
|
|
20
|
+
vectors from different embedders in one index is refused at open time.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Protocol
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Embedder(Protocol):
|
|
29
|
+
"""Anything that maps a batch of texts to fixed-dimension vectors."""
|
|
30
|
+
|
|
31
|
+
#: Stable identity string persisted with the index (name@dim).
|
|
32
|
+
identity: str
|
|
33
|
+
dimension: int
|
|
34
|
+
|
|
35
|
+
def embed(self, texts: list[str]) -> list[list[float]]: ...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_embedder(preference: str = "nomic") -> "Embedder":
|
|
39
|
+
"""Return the configured embedder, falling back to hashing if the
|
|
40
|
+
semantic model stack is not installed."""
|
|
41
|
+
from sciogen.embed.hashing import HashingEmbedder
|
|
42
|
+
|
|
43
|
+
if preference == "hash":
|
|
44
|
+
return HashingEmbedder()
|
|
45
|
+
try:
|
|
46
|
+
import sentence_transformers # noqa: F401
|
|
47
|
+
except ImportError:
|
|
48
|
+
return HashingEmbedder()
|
|
49
|
+
from sciogen.embed.nomic import NomicEmbedder
|
|
50
|
+
|
|
51
|
+
return NomicEmbedder()
|
sciogen/embed/hashing.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Deterministic feature-hashing embedder (fallback / test embedder).
|
|
2
|
+
|
|
3
|
+
Hashes word unigrams and bigrams into a fixed-size vector, L2-normalized so
|
|
4
|
+
cosine similarity behaves. Shares vocabulary-free robustness with the classic
|
|
5
|
+
hashing trick: identical texts always produce identical vectors, similar
|
|
6
|
+
keyword sets produce nearby vectors. No model download, no native deps,
|
|
7
|
+
microseconds per text.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import math
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]+|\d+")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _tokens(text: str) -> list[str]:
|
|
20
|
+
words = _TOKEN_RE.findall(text.lower())
|
|
21
|
+
# split snake_case / camelCase-ish compounds into subtokens too
|
|
22
|
+
out: list[str] = []
|
|
23
|
+
for w in words:
|
|
24
|
+
out.append(w)
|
|
25
|
+
if "_" in w:
|
|
26
|
+
out.extend(p for p in w.split("_") if p)
|
|
27
|
+
return out
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class HashingEmbedder:
|
|
31
|
+
identity: str
|
|
32
|
+
dimension: int
|
|
33
|
+
|
|
34
|
+
def __init__(self, dimension: int = 256) -> None:
|
|
35
|
+
self.dimension = dimension
|
|
36
|
+
self.identity = f"hashing@{dimension}"
|
|
37
|
+
|
|
38
|
+
def embed(self, texts: list[str]) -> list[list[float]]:
|
|
39
|
+
return [self._one(t) for t in texts]
|
|
40
|
+
|
|
41
|
+
def _one(self, text: str) -> list[float]:
|
|
42
|
+
vec = [0.0] * self.dimension
|
|
43
|
+
toks = _tokens(text)
|
|
44
|
+
grams = toks + [f"{a} {b}" for a, b in zip(toks, toks[1:])]
|
|
45
|
+
for gram in grams:
|
|
46
|
+
digest = hashlib.blake2b(gram.encode("utf-8"), digest_size=8).digest()
|
|
47
|
+
bucket = int.from_bytes(digest[:4], "little") % self.dimension
|
|
48
|
+
sign = 1.0 if digest[4] & 1 else -1.0
|
|
49
|
+
vec[bucket] += sign
|
|
50
|
+
norm = math.sqrt(sum(v * v for v in vec))
|
|
51
|
+
if norm > 0:
|
|
52
|
+
vec = [v / norm for v in vec]
|
|
53
|
+
return vec
|
sciogen/embed/nomic.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""nomic-embed-code embedder (default when sentence-transformers is installed).
|
|
2
|
+
|
|
3
|
+
Loads lazily on first use — importing sciogen never pays the model-load cost.
|
|
4
|
+
The model runs fully locally; nothing leaves the machine.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
_MODEL_NAME = "nomic-ai/nomic-embed-code"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class NomicEmbedder:
|
|
13
|
+
identity: str
|
|
14
|
+
dimension: int
|
|
15
|
+
|
|
16
|
+
def __init__(self, model_name: str = _MODEL_NAME) -> None:
|
|
17
|
+
self._model_name = model_name
|
|
18
|
+
self._model = None
|
|
19
|
+
self.identity = f"{model_name}@pending"
|
|
20
|
+
self.dimension = 0
|
|
21
|
+
|
|
22
|
+
def _ensure_model(self):
|
|
23
|
+
if self._model is None:
|
|
24
|
+
from sentence_transformers import SentenceTransformer
|
|
25
|
+
|
|
26
|
+
self._model = SentenceTransformer(self._model_name, trust_remote_code=True)
|
|
27
|
+
self.dimension = int(self._model.get_sentence_embedding_dimension() or 0)
|
|
28
|
+
self.identity = f"{self._model_name}@{self.dimension}"
|
|
29
|
+
return self._model
|
|
30
|
+
|
|
31
|
+
def embed(self, texts: list[str]) -> list[list[float]]:
|
|
32
|
+
model = self._ensure_model()
|
|
33
|
+
vectors = model.encode(texts, normalize_embeddings=True, show_progress_bar=False)
|
|
34
|
+
return [v.tolist() for v in vectors]
|