sciogen 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sciogen/cli/main.py ADDED
@@ -0,0 +1,322 @@
1
+ """sciogen CLI — Typer for command routing, Rich for all terminal output.
2
+
3
+ Only `sciogen index` shows the banner + progress flow; every other command
4
+ prints its output and nothing else. Heavier modules (stores, pipeline) import
5
+ inside command bodies so `sciogen --help` stays instant.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from pathlib import Path
11
+ from typing import Optional
12
+
13
+ import typer
14
+
15
+ from sciogen import __version__
16
+ from sciogen.cli.banner import banner_text
17
+ from sciogen.cli.console import console
18
+
19
+ app = typer.Typer(
20
+ name="sciogen",
21
+ help="Codebase intelligence layer — a queryable knowledge graph over your code.",
22
+ add_completion=False,
23
+ rich_markup_mode="rich",
24
+ )
25
+
26
+
27
+ def _version_callback(value: bool) -> None:
28
+ if value:
29
+ console.print(f"sciogen {__version__}")
30
+ raise typer.Exit()
31
+
32
+
33
+ @app.callback(invoke_without_command=True)
34
+ def _main(
35
+ ctx: typer.Context,
36
+ version: Optional[bool] = typer.Option(
37
+ None, "--version", callback=_version_callback, is_eager=True, help="Show version and exit."
38
+ ),
39
+ ) -> None:
40
+ """sciogen — index once, query forever.
41
+
42
+ Bare `sciogen` indexes the current directory: banner, pipeline progress,
43
+ then the completion summary. Agents read that output directly; querying
44
+ goes through the subcommands or the MCP server (`sciogen mcp`).
45
+ """
46
+ if ctx.invoked_subcommand is None:
47
+ index(Path("."), rebuild=False)
48
+
49
+
50
+ def _open_graph(path: Path):
51
+ from sciogen.api import SciogenGraph
52
+
53
+ return SciogenGraph(path)
54
+
55
+
56
+ def _fail(message: str) -> None:
57
+ console.print(f" ⚠ {message}")
58
+ raise typer.Exit(code=1)
59
+
60
+
61
+ class _RichReporter:
62
+ """Adapts pipeline progress onto rich.progress.Progress: one row per step,
63
+ Braille spinner while running, replaced in place with ✓ when done."""
64
+
65
+ def __init__(self, progress) -> None:
66
+ self.progress = progress
67
+ self.tasks: dict[str, int] = {}
68
+
69
+ def start(self, step: str, text: str) -> None:
70
+ self.tasks[step] = self.progress.add_task(text, total=1)
71
+
72
+ def finish(self, step: str) -> None:
73
+ self.progress.update(self.tasks[step], completed=1)
74
+
75
+ def warn(self, message: str) -> None:
76
+ # print above the live display, keeping the ⚠ single-line error format
77
+ self.progress.console.print(f" ⚠ {message}")
78
+
79
+
80
+ # ------------------------------------------------------------------- index
81
+
82
+ @app.command()
83
+ def index(
84
+ path: Path = typer.Argument(Path("."), help="Project directory to index."),
85
+ rebuild: bool = typer.Option(False, "--rebuild", help="Discard the existing index and rebuild from scratch."),
86
+ ) -> None:
87
+ """Build or incrementally update the knowledge graph for a codebase."""
88
+ from rich.progress import Progress, SpinnerColumn, TextColumn
89
+
90
+ console.print(banner_text(__version__))
91
+ graph = _open_graph(path)
92
+ try:
93
+ with Progress(
94
+ SpinnerColumn(spinner_name="dots", finished_text="✓"),
95
+ TextColumn("{task.description}"),
96
+ console=console,
97
+ ) as progress:
98
+ stats = graph.index(reporter=_RichReporter(progress), rebuild=rebuild)
99
+ console.print(
100
+ f"✓ Done! {stats.nodes:,} nodes · {stats.edges:,} edges · indexed in {stats.seconds:.1f}s"
101
+ )
102
+ console.print(" Ready. Your codebase is now queryable.", style="dim")
103
+ except Exception:
104
+ console.print_exception()
105
+ raise typer.Exit(code=1)
106
+ finally:
107
+ graph.close()
108
+
109
+
110
+ # ----------------------------------------------------------------- queries
111
+
112
+ @app.command()
113
+ def search(
114
+ query: str = typer.Argument(..., help="Natural-language query."),
115
+ path: Path = typer.Option(Path("."), "--path", "-p", help="Indexed project directory."),
116
+ mode: str = typer.Option("semantic", "--mode", "-m", help="semantic | hybrid"),
117
+ granularity: Optional[str] = typer.Option(None, "--granularity", "-g", help="file | class | function | chunk"),
118
+ k: int = typer.Option(10, "--top", "-k", help="Number of results."),
119
+ ) -> None:
120
+ """Semantic or hybrid search over the indexed codebase."""
121
+ from rich.table import Table
122
+
123
+ graph = _open_graph(path)
124
+ try:
125
+ results = graph.search(query, mode=mode, granularity=granularity, k=k)
126
+ if not results.hits:
127
+ console.print(" No results.", style="dim")
128
+ return
129
+ table = Table(box=None, pad_edge=False, header_style="bold")
130
+ table.add_column("score", justify="right", style="cyan")
131
+ table.add_column("kind", style="dim")
132
+ table.add_column("symbol")
133
+ table.add_column("location", style="dim")
134
+ for hit in results.hits:
135
+ location = f"{hit.node.file}:{hit.node.line_start}" if hit.node.file else "—"
136
+ via = "" if hit.via == "semantic" else " [dim](via graph)[/dim]"
137
+ table.add_row(f"{hit.score:.2f}", hit.node.kind, f"{hit.node.name}{via}", location)
138
+ console.print(table)
139
+ except Exception as exc:
140
+ _handle_query_error(exc)
141
+ finally:
142
+ graph.close()
143
+
144
+
145
+ @app.command()
146
+ def callers(
147
+ symbol: str = typer.Argument(..., help="Symbol name, e.g. AuthService.login"),
148
+ path: Path = typer.Option(Path("."), "--path", "-p"),
149
+ depth: int = typer.Option(1, "--depth", "-d", help="Hops up the call chain."),
150
+ min_confidence: float = typer.Option(0.0, "--min-confidence", "-c"),
151
+ ) -> None:
152
+ """Who calls a symbol, up to N hops."""
153
+ _print_impact_table(path, symbol, depth, min_confidence, callers_only=True)
154
+
155
+
156
+ @app.command()
157
+ def impact(
158
+ symbol: str = typer.Argument(..., help="Symbol name, e.g. UserModel.find_by_email"),
159
+ path: Path = typer.Option(Path("."), "--path", "-p"),
160
+ depth: int = typer.Option(5, "--depth", "-d"),
161
+ min_confidence: float = typer.Option(0.0, "--min-confidence", "-c"),
162
+ ) -> None:
163
+ """What breaks if a symbol changes (callers, subclasses, tests, mutators)."""
164
+ _print_impact_table(path, symbol, depth, min_confidence, callers_only=False)
165
+
166
+
167
+ def _print_impact_table(path: Path, symbol: str, depth: int, min_confidence: float, callers_only: bool) -> None:
168
+ from rich.table import Table
169
+
170
+ graph = _open_graph(path)
171
+ try:
172
+ if callers_only:
173
+ result = graph.get_callers(symbol, depth, min_confidence)
174
+ else:
175
+ result = graph.analyze_impact(symbol, depth, min_confidence)
176
+ target_names = ", ".join(t.name for t in result.targets)
177
+ if not result.impacted:
178
+ console.print(f" Nothing reaches [bold]{target_names}[/bold] within {depth} hops.", style="dim")
179
+ return
180
+ table = Table(box=None, pad_edge=False, header_style="bold",
181
+ title=f"{len(result.impacted)} symbols → {target_names}", title_justify="left")
182
+ table.add_column("hops", justify="right", style="cyan")
183
+ table.add_column("conf", justify="right")
184
+ table.add_column("via", style="dim")
185
+ table.add_column("symbol")
186
+ table.add_column("location", style="dim")
187
+ for item in result.impacted:
188
+ style = "" if item.confidence >= 0.8 else ("yellow" if item.confidence >= 0.5 else "red")
189
+ location = f"{item.node.file}:{item.node.line_start}" if item.node.file else "—"
190
+ table.add_row(
191
+ str(item.hops), f"[{style}]{item.confidence:.2f}[/{style}]" if style else f"{item.confidence:.2f}",
192
+ item.relation, item.node.name, location,
193
+ )
194
+ console.print(table)
195
+ except Exception as exc:
196
+ _handle_query_error(exc)
197
+ finally:
198
+ graph.close()
199
+
200
+
201
+ @app.command()
202
+ def deps(
203
+ file: str = typer.Argument(..., help="Project-relative file path."),
204
+ path: Path = typer.Option(Path("."), "--path", "-p"),
205
+ ) -> None:
206
+ """All imports and transitive dependencies of a file."""
207
+ graph = _open_graph(path)
208
+ try:
209
+ info = graph.get_dependencies(file)
210
+ for title, items in (
211
+ ("direct", info.direct_files),
212
+ ("transitive", info.transitive_files),
213
+ ("external", info.external_modules),
214
+ ):
215
+ console.print(f"[bold]{title}[/bold] ({len(items)})")
216
+ for item in items:
217
+ console.print(f" {item}")
218
+ except Exception as exc:
219
+ _handle_query_error(exc)
220
+ finally:
221
+ graph.close()
222
+
223
+
224
+ @app.command()
225
+ def symbol(
226
+ name: str = typer.Argument(..., help="Symbol name to look up."),
227
+ path: Path = typer.Option(Path("."), "--path", "-p"),
228
+ file: Optional[str] = typer.Option(None, "--file", "-f", help="Narrow to a defining file."),
229
+ ) -> None:
230
+ """Typed node info for a named symbol."""
231
+ graph = _open_graph(path)
232
+ try:
233
+ info = graph.get_symbol(name, file)
234
+ n = info.node
235
+ console.print(f"[bold]{n.name}[/bold] [dim]{n.kind}[/dim]")
236
+ console.print(f" {n.file}:{n.line_start}-{n.line_end}" if n.file else " (no source location)")
237
+ if n.params:
238
+ console.print(f" params: {', '.join(n.params)}")
239
+ if n.return_type:
240
+ console.print(f" returns: {n.return_type}")
241
+ if n.extends:
242
+ console.print(f" extends: {', '.join(n.extends)}")
243
+ if n.complexity:
244
+ console.print(f" complexity: {n.complexity}")
245
+ if n.docstring:
246
+ console.print(f" [dim]{n.docstring}[/dim]")
247
+ if info.relations:
248
+ console.print(f" [bold]relations[/bold] ({len(info.relations)})")
249
+ for rel in info.relations[:30]:
250
+ arrow = "→" if rel.direction == "out" else "←"
251
+ console.print(f" {arrow} {rel.kind} {rel.other} [dim]{rel.confidence:.2f}[/dim]")
252
+ if len(info.relations) > 30:
253
+ console.print(f" [dim]… {len(info.relations) - 30} more[/dim]")
254
+ except Exception as exc:
255
+ _handle_query_error(exc)
256
+ finally:
257
+ graph.close()
258
+
259
+
260
+ # ----------------------------------------------------------------- explore
261
+
262
+ @app.command()
263
+ def explore(
264
+ path: Path = typer.Argument(Path("."), help="Indexed project directory (or a subdirectory to scope the view)."),
265
+ ) -> None:
266
+ """Open an interactive visual view of the knowledge graph in the browser."""
267
+ import webbrowser
268
+
269
+ target = path.resolve()
270
+ root, prefix = target, ""
271
+ # Allow scoping to a subdirectory: find the enclosing indexed project.
272
+ while not (root / ".sciogen").exists() and root.parent != root:
273
+ root = root.parent
274
+ if (root / ".sciogen").exists() and root != target:
275
+ prefix = target.relative_to(root).as_posix() + "/"
276
+ elif not (root / ".sciogen").exists():
277
+ _fail(f"No sciogen index found at or above {target} — run `sciogen index` first.")
278
+
279
+ from sciogen.cli.explore import generate_html
280
+
281
+ graph = _open_graph(root)
282
+ try:
283
+ snapshot = graph.snapshot(prefix)
284
+ if not snapshot["nodes"]:
285
+ _fail("The index is empty — run `sciogen index` first.")
286
+ out = generate_html(snapshot, root.name, graph.config.data_dir / "explore.html")
287
+ console.print(f" Graph written to {out}")
288
+ console.print(" Snapshot view — re-run [bold]sciogen index[/bold] then "
289
+ "[bold]sciogen explore[/bold] after code changes.", style="dim")
290
+ webbrowser.open(out.resolve().as_uri())
291
+ except Exception as exc:
292
+ _handle_query_error(exc)
293
+ finally:
294
+ graph.close()
295
+
296
+
297
+ # --------------------------------------------------------------------- mcp
298
+
299
+ @app.command()
300
+ def mcp(
301
+ path: Path = typer.Argument(Path("."), help="Project directory the MCP server exposes."),
302
+ ) -> None:
303
+ """Run the MCP server (stdio) for agent integration."""
304
+ from sciogen.mcp.server import serve
305
+
306
+ serve(path)
307
+
308
+
309
+ # ------------------------------------------------------------------ errors
310
+
311
+ def _handle_query_error(exc: Exception) -> None:
312
+ from sciogen.query.engine import SymbolNotFound
313
+ from sciogen.stores.vectors import EmbedderMismatch
314
+
315
+ if isinstance(exc, (SymbolNotFound, EmbedderMismatch, FileNotFoundError)):
316
+ _fail(str(exc))
317
+ console.print_exception()
318
+ raise typer.Exit(code=1)
319
+
320
+
321
+ if __name__ == "__main__":
322
+ app()
sciogen/config.py ADDED
@@ -0,0 +1,82 @@
1
+ """Runtime configuration.
2
+
3
+ All knobs have working defaults; nothing requires a config file. Environment
4
+ variables override defaults so CI and agents can tune behavior without code:
5
+
6
+ =========================== =================================================
7
+ ``SCIOGEN_EMBEDDER`` ``nomic`` (default) or ``hash``. ``nomic`` falls
8
+ back to ``hash`` automatically when
9
+ sentence-transformers is not installed.
10
+ ``SCIOGEN_WORKERS`` Parse worker processes. Default: ``os.cpu_count()``.
11
+ ``0``/``1`` disables multiprocessing.
12
+ ``SCIOGEN_DATA_DIR`` Index location, default ``<project>/.sciogen``.
13
+ =========================== =================================================
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import os
19
+ from dataclasses import dataclass, field
20
+ from pathlib import Path
21
+
22
+ #: Directories never worth indexing. Discovery also honors .gitignore-style
23
+ #: hidden-dir skipping (any path segment starting with ".").
24
+ DEFAULT_IGNORED_DIRS = frozenset(
25
+ {
26
+ "node_modules", "__pycache__", "venv", ".venv", "env", "dist", "build",
27
+ "target", "vendor", "coverage", "htmlcov", ".sciogen", "site-packages",
28
+ ".git", ".hg", ".svn", ".tox", ".mypy_cache", ".ruff_cache", ".pytest_cache",
29
+ }
30
+ )
31
+
32
+ #: Functions longer than this many lines are additionally split into
33
+ #: chunk-level embedding units (ChunkNode).
34
+ CHUNK_THRESHOLD_LINES = 40
35
+ CHUNK_SIZE_LINES = 30
36
+
37
+ SCHEMA_VERSION = 1
38
+
39
+
40
+ @dataclass(slots=True)
41
+ class Config:
42
+ """Resolved configuration for one project index."""
43
+
44
+ project_root: Path
45
+ data_dir: Path
46
+ embedder: str = "nomic"
47
+ workers: int = 0 # 0 = auto (cpu_count), 1 = serial
48
+ min_files_for_parallel: int = 32 # below this, process-pool startup costs more than it saves
49
+ embed_batch_size: int = 64
50
+ ignored_dirs: frozenset[str] = field(default_factory=lambda: DEFAULT_IGNORED_DIRS)
51
+
52
+ @classmethod
53
+ def for_project(cls, project_root: str | Path) -> "Config":
54
+ root = Path(project_root).resolve()
55
+ data_dir = Path(os.environ.get("SCIOGEN_DATA_DIR", root / ".sciogen"))
56
+ workers_env = os.environ.get("SCIOGEN_WORKERS", "")
57
+ return cls(
58
+ project_root=root,
59
+ data_dir=data_dir,
60
+ embedder=os.environ.get("SCIOGEN_EMBEDDER", "nomic"),
61
+ workers=int(workers_env) if workers_env.isdigit() else 0,
62
+ )
63
+
64
+ @property
65
+ def graph_dir(self) -> Path:
66
+ return self.data_dir / "graph.kuzu"
67
+
68
+ @property
69
+ def vector_dir(self) -> Path:
70
+ return self.data_dir / "vectors"
71
+
72
+ @property
73
+ def meta_path(self) -> Path:
74
+ return self.data_dir / "meta.db"
75
+
76
+ def effective_workers(self, n_files: int) -> int:
77
+ """Parallelism actually used for a batch of ``n_files``."""
78
+ if n_files < self.min_files_for_parallel:
79
+ return 1
80
+ if self.workers >= 1:
81
+ return self.workers
82
+ return max(1, min(os.cpu_count() or 1, 8))
sciogen/discovery.py ADDED
@@ -0,0 +1,125 @@
1
+ """File discovery and two-tier change detection.
2
+
3
+ Discovery walks the project tree once, skipping ignored/hidden directories,
4
+ and returns every file whose extension maps to a supported language.
5
+
6
+ Change detection is deliberately two-tier — this is the hot path of
7
+ incremental re-indexing on large repos:
8
+
9
+ 1. **Fast path (stat only):** if a file's ``(size, mtime_ns)`` pair matches
10
+ what SQLite recorded last time, the file is assumed unchanged and never
11
+ opened. A 100k-file repo re-index costs 100k ``stat`` calls, not 100k
12
+ reads + hashes.
13
+ 2. **Slow path (content hash):** only files whose stat signature changed are
14
+ read and SHA256-hashed. If the hash still matches (e.g. ``touch``,
15
+ checkout that restored content), the stored stat signature is refreshed
16
+ but the file is *not* re-parsed.
17
+
18
+ SHA256 remains the source of truth — mtime alone is never trusted to declare
19
+ a file changed, and stat matching is never trusted across schema migrations
20
+ (the pipeline forces a full rebuild in that case).
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import hashlib
26
+ import os
27
+ from dataclasses import dataclass
28
+ from pathlib import Path
29
+
30
+ from sciogen.config import Config
31
+ from sciogen.parsing.languages import language_for_path
32
+
33
+
34
+ @dataclass(slots=True)
35
+ class DiscoveredFile:
36
+ path: str # project-relative, "/" separators
37
+ abs_path: str
38
+ language: str
39
+ size: int
40
+ mtime: float
41
+ mtime_ns: int
42
+
43
+
44
+ @dataclass(slots=True)
45
+ class ChangeSet:
46
+ """Result of diffing the working tree against the stored index state."""
47
+
48
+ changed: list[DiscoveredFile] # new or content-changed -> re-parse
49
+ unchanged: list[DiscoveredFile] # skipped entirely
50
+ deleted: list[str] # relative paths present in the index but gone on disk
51
+ stat_refreshed: list[DiscoveredFile] # content identical, stat signature updated
52
+
53
+
54
+ def discover(config: Config) -> list[DiscoveredFile]:
55
+ """Walk the project and return all indexable source files."""
56
+ root = config.project_root
57
+ found: list[DiscoveredFile] = []
58
+ for dirpath, dirnames, filenames in os.walk(root):
59
+ # Prune in place: os.walk never descends into removed entries.
60
+ dirnames[:] = [
61
+ d for d in dirnames if d not in config.ignored_dirs and not d.startswith(".")
62
+ ]
63
+ for fname in filenames:
64
+ language = language_for_path(fname)
65
+ if language is None:
66
+ continue
67
+ abs_path = os.path.join(dirpath, fname)
68
+ try:
69
+ st = os.stat(abs_path)
70
+ except OSError:
71
+ continue
72
+ rel = os.path.relpath(abs_path, root).replace(os.sep, "/")
73
+ found.append(
74
+ DiscoveredFile(
75
+ path=rel,
76
+ abs_path=abs_path,
77
+ language=language,
78
+ size=st.st_size,
79
+ mtime=st.st_mtime,
80
+ mtime_ns=st.st_mtime_ns,
81
+ )
82
+ )
83
+ found.sort(key=lambda f: f.path)
84
+ return found
85
+
86
+
87
+ def sha256_file(abs_path: str) -> str:
88
+ h = hashlib.sha256()
89
+ with open(abs_path, "rb") as fh:
90
+ for block in iter(lambda: fh.read(1 << 20), b""):
91
+ h.update(block)
92
+ return h.hexdigest()
93
+
94
+
95
+ def compute_changes(
96
+ files: list[DiscoveredFile],
97
+ stored: dict[str, tuple[str, int, int]],
98
+ ) -> ChangeSet:
99
+ """Diff discovered files against stored state.
100
+
101
+ ``stored`` maps relative path -> ``(sha256, size, mtime_ns)`` as recorded
102
+ by the metadata store after the previous index run.
103
+ """
104
+ changed: list[DiscoveredFile] = []
105
+ unchanged: list[DiscoveredFile] = []
106
+ stat_refreshed: list[DiscoveredFile] = []
107
+ seen: set[str] = set()
108
+
109
+ for f in files:
110
+ seen.add(f.path)
111
+ record = stored.get(f.path)
112
+ if record is not None:
113
+ old_sha, old_size, old_mtime_ns = record
114
+ if f.size == old_size and f.mtime_ns == old_mtime_ns:
115
+ unchanged.append(f) # fast path: stat signature identical
116
+ continue
117
+ if sha256_file(f.abs_path) == old_sha:
118
+ stat_refreshed.append(f) # touched but content identical
119
+ continue
120
+ changed.append(f)
121
+
122
+ deleted = sorted(path for path in stored if path not in seen)
123
+ return ChangeSet(
124
+ changed=changed, unchanged=unchanged, deleted=deleted, stat_refreshed=stat_refreshed
125
+ )
@@ -0,0 +1,51 @@
1
+ """Embedders — pluggable text -> vector functions.
2
+
3
+ sciogen computes embeddings itself and hands raw vectors to ChromaDB (rather
4
+ than registering an embedding function with the collection). That keeps the
5
+ embedder swappable without ChromaDB persistence-config coupling, and lets the
6
+ pipeline batch + deduplicate embedding work.
7
+
8
+ Two implementations:
9
+
10
+ - :class:`~sciogen.embed.nomic.NomicEmbedder` — the documented default,
11
+ local nomic-embed-code via sentence-transformers (optional install:
12
+ ``pip install sciogen[embeddings]``).
13
+ - :class:`~sciogen.embed.hashing.HashingEmbedder` — deterministic feature
14
+ hashing. Zero dependencies, instant, no model download. Used automatically
15
+ when sentence-transformers is unavailable, and always in tests. Retrieval
16
+ quality is keyword-level rather than semantic; fine for structural work,
17
+ not a substitute for real embeddings.
18
+
19
+ The embedder's identity is recorded in the vector store's metadata; mixing
20
+ vectors from different embedders in one index is refused at open time.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ from typing import Protocol
26
+
27
+
28
+ class Embedder(Protocol):
29
+ """Anything that maps a batch of texts to fixed-dimension vectors."""
30
+
31
+ #: Stable identity string persisted with the index (name@dim).
32
+ identity: str
33
+ dimension: int
34
+
35
+ def embed(self, texts: list[str]) -> list[list[float]]: ...
36
+
37
+
38
+ def get_embedder(preference: str = "nomic") -> "Embedder":
39
+ """Return the configured embedder, falling back to hashing if the
40
+ semantic model stack is not installed."""
41
+ from sciogen.embed.hashing import HashingEmbedder
42
+
43
+ if preference == "hash":
44
+ return HashingEmbedder()
45
+ try:
46
+ import sentence_transformers # noqa: F401
47
+ except ImportError:
48
+ return HashingEmbedder()
49
+ from sciogen.embed.nomic import NomicEmbedder
50
+
51
+ return NomicEmbedder()
@@ -0,0 +1,53 @@
1
+ """Deterministic feature-hashing embedder (fallback / test embedder).
2
+
3
+ Hashes word unigrams and bigrams into a fixed-size vector, L2-normalized so
4
+ cosine similarity behaves. Shares vocabulary-free robustness with the classic
5
+ hashing trick: identical texts always produce identical vectors, similar
6
+ keyword sets produce nearby vectors. No model download, no native deps,
7
+ microseconds per text.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import math
14
+ import re
15
+
16
+ _TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]+|\d+")
17
+
18
+
19
+ def _tokens(text: str) -> list[str]:
20
+ words = _TOKEN_RE.findall(text.lower())
21
+ # split snake_case / camelCase-ish compounds into subtokens too
22
+ out: list[str] = []
23
+ for w in words:
24
+ out.append(w)
25
+ if "_" in w:
26
+ out.extend(p for p in w.split("_") if p)
27
+ return out
28
+
29
+
30
+ class HashingEmbedder:
31
+ identity: str
32
+ dimension: int
33
+
34
+ def __init__(self, dimension: int = 256) -> None:
35
+ self.dimension = dimension
36
+ self.identity = f"hashing@{dimension}"
37
+
38
+ def embed(self, texts: list[str]) -> list[list[float]]:
39
+ return [self._one(t) for t in texts]
40
+
41
+ def _one(self, text: str) -> list[float]:
42
+ vec = [0.0] * self.dimension
43
+ toks = _tokens(text)
44
+ grams = toks + [f"{a} {b}" for a, b in zip(toks, toks[1:])]
45
+ for gram in grams:
46
+ digest = hashlib.blake2b(gram.encode("utf-8"), digest_size=8).digest()
47
+ bucket = int.from_bytes(digest[:4], "little") % self.dimension
48
+ sign = 1.0 if digest[4] & 1 else -1.0
49
+ vec[bucket] += sign
50
+ norm = math.sqrt(sum(v * v for v in vec))
51
+ if norm > 0:
52
+ vec = [v / norm for v in vec]
53
+ return vec
sciogen/embed/nomic.py ADDED
@@ -0,0 +1,34 @@
1
+ """nomic-embed-code embedder (default when sentence-transformers is installed).
2
+
3
+ Loads lazily on first use — importing sciogen never pays the model-load cost.
4
+ The model runs fully locally; nothing leaves the machine.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ _MODEL_NAME = "nomic-ai/nomic-embed-code"
10
+
11
+
12
+ class NomicEmbedder:
13
+ identity: str
14
+ dimension: int
15
+
16
+ def __init__(self, model_name: str = _MODEL_NAME) -> None:
17
+ self._model_name = model_name
18
+ self._model = None
19
+ self.identity = f"{model_name}@pending"
20
+ self.dimension = 0
21
+
22
+ def _ensure_model(self):
23
+ if self._model is None:
24
+ from sentence_transformers import SentenceTransformer
25
+
26
+ self._model = SentenceTransformer(self._model_name, trust_remote_code=True)
27
+ self.dimension = int(self._model.get_sentence_embedding_dimension() or 0)
28
+ self.identity = f"{self._model_name}@{self.dimension}"
29
+ return self._model
30
+
31
+ def embed(self, texts: list[str]) -> list[list[float]]:
32
+ model = self._ensure_model()
33
+ vectors = model.encode(texts, normalize_embeddings=True, show_progress_bar=False)
34
+ return [v.tolist() for v in vectors]
@@ -0,0 +1,5 @@
1
+ """Indexing pipeline (phase 1). See :mod:`sciogen.index.pipeline`."""
2
+
3
+ from sciogen.index.pipeline import IndexStats, Pipeline, ProgressReporter
4
+
5
+ __all__ = ["IndexStats", "Pipeline", "ProgressReporter"]