groundly 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
groundly/__init__.py ADDED
File without changes
File without changes
@@ -0,0 +1,261 @@
1
+ """Groundly CLI — batch lifecycle verbs; the host agent is the interactive surface.
2
+
3
+ Command surface per docs/superpowers/specs/2026-07-16-p1-cli-surface-design.md.
4
+ Later phases add verbs: P2 import/export · P3 ask · P4 mcp/serve · P6 export-deck.
5
+ """
6
+
7
+ from importlib.metadata import version as _package_version
8
+ from pathlib import Path
9
+ from typing import Annotated, Optional
10
+
11
+ import typer
12
+ from rich.console import Console
13
+ from rich.markup import escape
14
+ from rich.table import Table
15
+
16
+ app = typer.Typer(
17
+ no_args_is_help=True,
18
+ add_completion=False,
19
+ help="Groundly — local course knowledge bases for AI agents.",
20
+ )
21
+ config_app = typer.Typer()
22
+ app.add_typer(config_app, name="config")
23
+
24
+ console = Console()
25
+
26
+
27
+ def _fail(message: str) -> None:
28
+ console.print(f"[red]error:[/red] {escape(message)}")
29
+ raise typer.Exit(code=1)
30
+
31
+
32
+ def _subject_dir_checked(subject: str) -> Path:
33
+ from groundly.core.paths import subject_dir
34
+
35
+ try:
36
+ sdir = subject_dir(subject)
37
+ except ValueError as exc:
38
+ _fail(str(exc))
39
+ if not (sdir / "manifest.json").exists():
40
+ _fail(f"subject '{subject}' is not initialized — run: groundly init {subject}")
41
+ return sdir
42
+
43
+
44
+ def _connect_checked(sdir: Path):
45
+ from groundly.core import store
46
+
47
+ try:
48
+ return store.connect(sdir / "store.db")
49
+ except RuntimeError as exc: # missing store.db / newer schema — named cause, no traceback
50
+ _fail(str(exc))
51
+
52
+
53
+ def _not_implemented(verb: str) -> None:
54
+ typer.echo(f"groundly {verb}: not implemented yet — arrives in a later phase")
55
+ raise typer.Exit(code=1)
56
+
57
+
58
+ def _print_version(value: bool) -> None:
59
+ if value:
60
+ typer.echo(_package_version("groundly"))
61
+ raise typer.Exit()
62
+
63
+
64
+ @app.callback()
65
+ def main(
66
+ version: Annotated[
67
+ bool,
68
+ typer.Option(
69
+ "--version", callback=_print_version, is_eager=True, help="Print version and exit."
70
+ ),
71
+ ] = False,
72
+ ) -> None:
73
+ """Groundly — local-first course knowledge bases for AI agents."""
74
+
75
+
76
+ @app.command()
77
+ def init(
78
+ subject: Annotated[str, typer.Argument(help="Subject name; becomes ~/.groundly/<SUBJECT>/.")],
79
+ ) -> None:
80
+ """Create a subject: manifest.json, materials/, store.db, progress.db."""
81
+ from groundly.core.subject import init_subject
82
+
83
+ try:
84
+ sdir, created = init_subject(subject)
85
+ except ValueError as exc:
86
+ _fail(str(exc))
87
+ if created:
88
+ console.print(f"initialized [bold]{subject}[/bold] at {sdir}")
89
+ else:
90
+ console.print(f"[bold]{subject}[/bold] already initialized at {sdir}")
91
+
92
+
93
+ @app.command()
94
+ def index(
95
+ subject: Annotated[str, typer.Argument(help="Subject to index into (must be initialized).")],
96
+ paths: Annotated[list[Path], typer.Argument(help="Files or directories to index.")],
97
+ ) -> None:
98
+ """Index course materials: hash-skip idempotent, per-file progress, resumable."""
99
+ from groundly.ingestion import pipeline
100
+
101
+ labels = {
102
+ pipeline.INDEXED: "[green]indexed[/green]",
103
+ pipeline.SKIPPED_DUPLICATE: "[dim]skipped (already indexed)[/dim]",
104
+ pipeline.SKIPPED_UNSUPPORTED: "[yellow]skipped[/yellow]",
105
+ pipeline.SKIPPED_FAILED: "[yellow]skipped[/yellow]",
106
+ pipeline.EXTRACTION_FAILED: "[red]failed[/red]",
107
+ pipeline.ERROR: "[red]error[/red]",
108
+ }
109
+
110
+ with console.status("indexing…") as status:
111
+
112
+ def on_event(path: Path, stage: str) -> None:
113
+ if stage in ("extracting", "embedding"):
114
+ status.update(f"{path.name}: {stage}…")
115
+
116
+ try:
117
+ results = pipeline.index_paths(subject, paths, on_event=on_event)
118
+ except (RuntimeError, ValueError) as exc:
119
+ _fail(str(exc))
120
+
121
+ for r in results:
122
+ # filenames and parser errors are document-influenced — never live markup
123
+ detail = f" — {escape(r.detail)}" if r.detail else ""
124
+ chunks = f" ({r.chunks} chunks)" if r.status == pipeline.INDEXED else ""
125
+ console.print(f" {escape(r.path.name)}: {labels[r.status]}{chunks}{detail}")
126
+
127
+ indexed = sum(r.status == pipeline.INDEXED for r in results)
128
+ failed = sum(r.status in (pipeline.EXTRACTION_FAILED, pipeline.ERROR) for r in results)
129
+ console.print(f"{indexed} indexed, {len(results) - indexed - failed} skipped, {failed} failed")
130
+ if failed:
131
+ raise typer.Exit(code=1)
132
+
133
+
134
+ @app.command(name="list")
135
+ def list_(
136
+ subject: Annotated[
137
+ Optional[str],
138
+ typer.Argument(help="Subject to inspect; omit to list all subjects."),
139
+ ] = None,
140
+ ) -> None:
141
+ """List subjects, or one subject's materials with status, pages, chunks."""
142
+ import sqlite3
143
+
144
+ from pydantic import ValidationError
145
+
146
+ from groundly.core import store
147
+ from groundly.core.manifest import Manifest
148
+ from groundly.core.paths import discover_subjects, subject_dir
149
+
150
+ if subject is None:
151
+ table = Table("subject", "materials", "chunks")
152
+ for name in discover_subjects():
153
+ try:
154
+ manifest = Manifest.load(subject_dir(name) / "manifest.json")
155
+ except ValidationError:
156
+ # one damaged subject must not take down the whole listing
157
+ console.print(f"[red]warning:[/red] {name}: manifest.json is corrupt — skipping")
158
+ continue
159
+ table.add_row(name, str(manifest.counts.materials), str(manifest.counts.chunks))
160
+ console.print(table)
161
+ return
162
+
163
+ sdir = _subject_dir_checked(subject)
164
+ conn = _connect_checked(sdir)
165
+ try:
166
+ table = Table("material", "status", "pages", "chunks", "detail")
167
+ for row in store.list_materials(conn):
168
+ table.add_row(
169
+ escape(row["filename"]),
170
+ row["status"],
171
+ str(row["pages"] or "—"),
172
+ str(row["chunk_count"]),
173
+ escape(row["error"] or ""),
174
+ )
175
+ console.print(table)
176
+ except sqlite3.OperationalError as exc:
177
+ _fail(f"store.db is corrupt or incomplete: {exc}")
178
+ finally:
179
+ conn.close()
180
+
181
+
182
+ @app.command()
183
+ def remove(
184
+ subject: Annotated[str, typer.Argument(help="Subject the material belongs to.")],
185
+ material: Annotated[
186
+ Optional[str],
187
+ typer.Argument(
188
+ help="Material filename (or sha256 prefix) as shown by `groundly list`; "
189
+ "omit to remove the whole subject."
190
+ ),
191
+ ] = None,
192
+ yes: Annotated[bool, typer.Option("--yes", "-y", help="Skip the confirmation prompt.")] = False,
193
+ ) -> None:
194
+ """Remove a material and all its indexed data, or a whole subject if no material given."""
195
+ import sqlite3
196
+
197
+ from groundly.core import store
198
+ from groundly.core.manifest import sync_counts
199
+
200
+ sdir = _subject_dir_checked(subject)
201
+
202
+ if material is None:
203
+ import shutil
204
+
205
+ if not yes:
206
+ typer.confirm(
207
+ f"remove subject {subject} and ALL its data (materials, index, progress, notes)?",
208
+ abort=True,
209
+ )
210
+ shutil.rmtree(sdir)
211
+ console.print(f"removed subject [bold]{subject}[/bold]")
212
+ return
213
+
214
+ conn = _connect_checked(sdir)
215
+ try:
216
+ matches = store.find_materials(conn, material)
217
+ if not matches:
218
+ _fail(f"no material {material!r} in {subject} — see: groundly list {subject}")
219
+ if len(matches) > 1:
220
+ candidates = ", ".join(f"{m['filename']} ({m['sha256'][:8]})" for m in matches)
221
+ _fail(f"{material!r} is ambiguous — candidates: {candidates}; use a sha256 prefix")
222
+ target = matches[0]
223
+ if not yes:
224
+ typer.confirm(
225
+ f"remove {target['filename']} and all its indexed data from {subject}?",
226
+ abort=True,
227
+ )
228
+ store.remove_material(conn, target["id"])
229
+ sync_counts(conn, sdir / "manifest.json")
230
+ if target["status"] == "indexed":
231
+ # failed rows never got a copy in materials/, and their original filename
232
+ # (no collision suffix) can shadow a different indexed material's file
233
+ stored = sdir / "materials" / target["filename"]
234
+ if stored.exists():
235
+ stored.unlink()
236
+ console.print(f"removed [bold]{escape(target['filename'])}[/bold] from {subject}")
237
+ if (sdir / "graph").exists():
238
+ console.print(
239
+ "[dim]note: the graph is now stale — it rebuilds on the next"
240
+ " corpus-hash-triggered index run[/dim]"
241
+ )
242
+ except sqlite3.OperationalError as exc:
243
+ _fail(f"store.db is corrupt or incomplete: {exc}")
244
+ finally:
245
+ conn.close()
246
+
247
+
248
+ @config_app.callback(invoke_without_command=True)
249
+ def config(ctx: typer.Context) -> None:
250
+ """Show the config file path and effective values per call class (keys masked)."""
251
+ if ctx.invoked_subcommand is None:
252
+ _not_implemented("config")
253
+
254
+
255
+ @config_app.command(name="set")
256
+ def config_set(
257
+ key: Annotated[str, typer.Argument(help="Dotted key, e.g. chat.model or chat.base_url.")],
258
+ value: Annotated[str, typer.Argument(help="Value to set.")],
259
+ ) -> None:
260
+ """Set a provider config value in ~/.groundly/config.toml."""
261
+ _not_implemented("config set")
File without changes
@@ -0,0 +1,76 @@
1
+ """manifest.json — the interchange contract (docs/architecture/data-model.md).
2
+
3
+ The embedding pin (model + hf_revision + dim + normalization) decides whether shared
4
+ vectors transfer as-is; changing it is a full re-index migration, never a tweak.
5
+ """
6
+
7
+ import sqlite3
8
+ from importlib.metadata import version as _package_version
9
+ from pathlib import Path
10
+
11
+ from pydantic import BaseModel
12
+
13
+ FORMAT_VERSION = 1
14
+ EMBEDDING_MODEL = "BAAI/bge-m3"
15
+ HF_REVISION = "5617a9f61b028005a4858fdac845db406aefb181" # pinned 2026-07-16 (P1 start)
16
+ EMBEDDING_DIM = 1024
17
+ CHUNK_MAX_TOKENS = 512
18
+ CHUNK_OVERLAP = 0 # HybridChunker is structure-aware (split+merge); it has no token overlap
19
+
20
+
21
+ class Embedding(BaseModel):
22
+ model: str = EMBEDDING_MODEL
23
+ hf_revision: str = HF_REVISION
24
+ dim: int = EMBEDDING_DIM
25
+ dtype: str = "float32"
26
+ normalized: bool = True
27
+
28
+
29
+ class Graphrag(BaseModel):
30
+ version: str | None = None
31
+ extraction_model: str | None = None
32
+
33
+
34
+ class Chunking(BaseModel):
35
+ strategy: str = "docling-hybrid"
36
+ max_tokens: int = CHUNK_MAX_TOKENS
37
+ overlap: int = CHUNK_OVERLAP
38
+
39
+
40
+ class Counts(BaseModel):
41
+ materials: int = 0
42
+ chunks: int = 0
43
+
44
+
45
+ class Manifest(BaseModel):
46
+ format_version: int = FORMAT_VERSION
47
+ subject: str
48
+ embedding: Embedding = Embedding()
49
+ graphrag: Graphrag = Graphrag()
50
+ chunking: Chunking = Chunking()
51
+ counts: Counts = Counts()
52
+ tool_version: str = ""
53
+
54
+ @classmethod
55
+ def new(cls, subject: str) -> "Manifest":
56
+ return cls(subject=subject, tool_version=_package_version("groundly"))
57
+
58
+ @classmethod
59
+ def load(cls, path: Path) -> "Manifest":
60
+ return cls.model_validate_json(path.read_text())
61
+
62
+ def save(self, path: Path) -> None:
63
+ # write-then-rename: a Ctrl-C or concurrent save never leaves torn JSON behind
64
+ tmp = path.with_suffix(".json.tmp")
65
+ tmp.write_text(self.model_dump_json(indent=2) + "\n")
66
+ tmp.replace(path)
67
+
68
+
69
+ def sync_counts(conn: sqlite3.Connection, manifest_path: Path) -> None:
70
+ """Keep manifest counts in sync after every mutation (UC-03 acceptance)."""
71
+ manifest = Manifest.load(manifest_path)
72
+ manifest.counts.materials = conn.execute(
73
+ "SELECT COUNT(*) FROM materials WHERE status = 'indexed'"
74
+ ).fetchone()[0]
75
+ manifest.counts.chunks = conn.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
76
+ manifest.save(manifest_path)
groundly/core/paths.py ADDED
@@ -0,0 +1,33 @@
1
+ """Filesystem layout: ~/.groundly/ (GROUNDLY_HOME overrides), one dir per subject.
2
+
3
+ Discovery scans */manifest.json — no registry database (docs/architecture/data-model.md).
4
+ """
5
+
6
+ import os
7
+ import re
8
+ from pathlib import Path
9
+
10
+ _SUBJECT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]*$")
11
+
12
+
13
+ def groundly_home() -> Path:
14
+ override = os.environ.get("GROUNDLY_HOME")
15
+ return Path(override) if override else Path.home() / ".groundly"
16
+
17
+
18
+ def validate_subject_name(name: str) -> None:
19
+ """Subject names become path components and MCP identifiers."""
20
+ if not _SUBJECT_RE.fullmatch(name): # fullmatch: `$` alone accepts a trailing newline
21
+ raise ValueError(
22
+ f"invalid subject name {name!r} — use letters, digits, '-' or '_' "
23
+ "(it becomes a directory name)"
24
+ )
25
+
26
+
27
+ def subject_dir(name: str) -> Path:
28
+ validate_subject_name(name)
29
+ return groundly_home() / name
30
+
31
+
32
+ def discover_subjects() -> list[str]:
33
+ return sorted(p.parent.name for p in groundly_home().glob("*/manifest.json"))
groundly/core/store.py ADDED
@@ -0,0 +1,143 @@
1
+ """store.db / progress.db access. Schema versioned via PRAGMA user_version — no
2
+ migration framework; refuse to open a newer schema than this tool understands.
3
+
4
+ Every connection gets WAL + busy_timeout: one-shot CLI runs and host-spawned MCP
5
+ processes share the same files (.claude/rules/architecture.md).
6
+ """
7
+
8
+ import sqlite3
9
+ from pathlib import Path
10
+
11
+ import sqlite_vec
12
+
13
+ from groundly.core.manifest import EMBEDDING_DIM
14
+
15
+ STORE_USER_VERSION = 1
16
+
17
+ _SCHEMA = f"""
18
+ CREATE TABLE materials (
19
+ id INTEGER PRIMARY KEY,
20
+ filename TEXT NOT NULL,
21
+ sha256 TEXT NOT NULL UNIQUE,
22
+ status TEXT NOT NULL CHECK (status IN ('indexed', 'extraction_failed')),
23
+ pages INTEGER,
24
+ error TEXT,
25
+ indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
26
+ );
27
+
28
+ CREATE TABLE chunks (
29
+ id INTEGER PRIMARY KEY,
30
+ material_id INTEGER NOT NULL REFERENCES materials(id) ON DELETE CASCADE,
31
+ page INTEGER,
32
+ heading_path TEXT,
33
+ text TEXT NOT NULL,
34
+ token_count INTEGER
35
+ );
36
+ CREATE INDEX idx_chunks_material ON chunks(material_id);
37
+
38
+ -- rowid = chunks.id; vec0 has no FK support, deletion is explicit in remove_material
39
+ CREATE VIRTUAL TABLE vectors USING vec0(embedding float[{EMBEDDING_DIM}]);
40
+
41
+ CREATE TABLE sparse_terms (
42
+ token_id INTEGER NOT NULL,
43
+ chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
44
+ weight REAL NOT NULL
45
+ );
46
+ CREATE INDEX idx_sparse_token ON sparse_terms(token_id);
47
+ CREATE INDEX idx_sparse_chunk ON sparse_terms(chunk_id);
48
+
49
+ CREATE VIRTUAL TABLE chunks_fts USING fts5(text, content='chunks', content_rowid='id');
50
+ CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
51
+ INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
52
+ END;
53
+ CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
54
+ INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
55
+ END;
56
+ CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN
57
+ INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
58
+ INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
59
+ END;
60
+ """
61
+
62
+
63
+ def connect(path: Path, create: bool = False) -> sqlite3.Connection:
64
+ if not create and not path.exists():
65
+ # sqlite3.connect would silently create an empty db — surface the real cause
66
+ raise RuntimeError(f"{path.name} is missing from {path.parent} — the subject is damaged")
67
+ conn = sqlite3.connect(path)
68
+ conn.row_factory = sqlite3.Row
69
+ conn.enable_load_extension(True)
70
+ sqlite_vec.load(conn)
71
+ conn.enable_load_extension(False)
72
+ conn.execute("PRAGMA journal_mode = WAL")
73
+ conn.execute("PRAGMA busy_timeout = 5000")
74
+ conn.execute("PRAGMA foreign_keys = ON")
75
+ version = conn.execute("PRAGMA user_version").fetchone()[0]
76
+ if version > STORE_USER_VERSION:
77
+ conn.close()
78
+ raise RuntimeError(
79
+ f"{path.name} has schema version {version}, newer than this groundly "
80
+ f"understands (max {STORE_USER_VERSION}) — upgrade groundly"
81
+ )
82
+ return conn
83
+
84
+
85
+ def create_store(path: Path) -> None:
86
+ conn = connect(path, create=True)
87
+ try:
88
+ conn.executescript(_SCHEMA)
89
+ conn.execute(f"PRAGMA user_version = {STORE_USER_VERSION}")
90
+ conn.commit()
91
+ finally:
92
+ conn.close()
93
+
94
+
95
+ def create_progress(path: Path) -> None:
96
+ # Tables arrive in P3 (traces) / P6 (quiz_events, notes); progress.db never
97
+ # travels, so its schema can grow locally without interchange impact.
98
+ conn = sqlite3.connect(path)
99
+ try:
100
+ conn.execute("PRAGMA journal_mode = WAL")
101
+ conn.execute("PRAGMA busy_timeout = 5000")
102
+ conn.execute("PRAGMA user_version = 1")
103
+ conn.commit()
104
+ finally:
105
+ conn.close()
106
+
107
+
108
+ def list_materials(conn: sqlite3.Connection) -> list[sqlite3.Row]:
109
+ return conn.execute(
110
+ """
111
+ SELECT m.id, m.filename, m.sha256, m.status, m.pages, m.error,
112
+ COUNT(c.id) AS chunk_count
113
+ FROM materials m LEFT JOIN chunks c ON c.material_id = m.id
114
+ GROUP BY m.id ORDER BY m.filename
115
+ """
116
+ ).fetchall()
117
+
118
+
119
+ def hash_status(conn: sqlite3.Connection) -> dict[str, str]:
120
+ """sha256 -> status, for hash-skip (indexed) and retry (extraction_failed)."""
121
+ return {r["sha256"]: r["status"] for r in conn.execute("SELECT sha256, status FROM materials")}
122
+
123
+
124
+ def find_materials(conn: sqlite3.Connection, ident: str) -> list[sqlite3.Row]:
125
+ """Match by exact filename or sha256 prefix (the disambiguator)."""
126
+ escaped = ident.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
127
+ return conn.execute(
128
+ "SELECT * FROM materials WHERE filename = ? OR sha256 LIKE ? ESCAPE '\\' ORDER BY filename",
129
+ (ident, escaped + "%"),
130
+ ).fetchall()
131
+
132
+
133
+ def remove_material(conn: sqlite3.Connection, material_id: int) -> None:
134
+ """One transaction. FTS syncs via the chunks_ad trigger; sparse_terms via FK
135
+ cascade; vectors (vec0, no FK) deleted explicitly by chunk rowid."""
136
+ with conn:
137
+ chunk_ids = [
138
+ r["id"]
139
+ for r in conn.execute("SELECT id FROM chunks WHERE material_id = ?", (material_id,))
140
+ ]
141
+ conn.executemany("DELETE FROM vectors WHERE rowid = ?", [(cid,) for cid in chunk_ids])
142
+ conn.execute("DELETE FROM chunks WHERE material_id = ?", (material_id,))
143
+ conn.execute("DELETE FROM materials WHERE id = ?", (material_id,))
@@ -0,0 +1,44 @@
1
+ """Subject lifecycle: create the on-disk layout that everything else assumes."""
2
+
3
+ from pathlib import Path
4
+
5
+ from groundly.core import store
6
+ from groundly.core.manifest import Manifest
7
+ from groundly.core.paths import subject_dir, groundly_home
8
+
9
+ _CONFIG_TEMPLATE = """\
10
+ # Groundly provider config — one OpenAI-compatible endpoint per call class.
11
+ # All classes are optional: indexing and search work with no provider at all.
12
+ #
13
+ # [providers.chat] # ask pipeline generation
14
+ # base_url = "http://localhost:1234/v1"
15
+ # model = "..."
16
+ # api_key = "..."
17
+ #
18
+ # [providers.generation] # exam/deck generation (thick path)
19
+ # [providers.extraction] # graphrag entity extraction
20
+ # [providers.router] # cheap query classifier
21
+ """
22
+
23
+
24
+ def init_subject(name: str) -> tuple[Path, bool]:
25
+ """Create ~/.groundly/<name>/ (manifest, materials/, store.db, progress.db).
26
+
27
+ Returns (subject_dir, created); created=False if already initialized (idempotent).
28
+ Also writes the top-level config.toml template on first ever init.
29
+ """
30
+ sdir = subject_dir(name)
31
+ manifest_path = sdir / "manifest.json"
32
+ if manifest_path.exists():
33
+ return sdir, False
34
+
35
+ sdir.mkdir(parents=True, exist_ok=True)
36
+ (sdir / "materials").mkdir(exist_ok=True)
37
+ store.create_store(sdir / "store.db")
38
+ store.create_progress(sdir / "progress.db")
39
+ Manifest.new(name).save(manifest_path)
40
+
41
+ config_path = groundly_home() / "config.toml"
42
+ if not config_path.exists():
43
+ config_path.write_text(_CONFIG_TEMPLATE)
44
+ return sdir, True
File without changes
@@ -0,0 +1,89 @@
1
+ """Parent side of extraction: spawn the worker, enforce a wall-clock timeout,
2
+ map failures to specific user-facing causes (conventions: never generic errors).
3
+
4
+ security.md §3 controls: argv exec (no shell), temp working directory, wall-clock
5
+ timeout, output size cap — the worker's stdout is discarded, stderr goes to a file
6
+ on disk (never an in-memory buffer) and only its tail is ever read."""
7
+
8
+ import json
9
+ import subprocess
10
+ import sys
11
+ import tempfile
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+
15
+ from groundly.ingestion import extract_worker
16
+
17
+ EXTRACT_TIMEOUT_SECONDS = 300
18
+ STDERR_TAIL_BYTES = 4096
19
+ MAX_EXTRACTION_JSON_BYTES = 200 * 1024 * 1024 # far beyond any real textbook
20
+
21
+
22
+ @dataclass
23
+ class ChunkData:
24
+ text: str
25
+ heading_path: str | None
26
+ page: int | None
27
+ token_count: int
28
+
29
+
30
+ @dataclass
31
+ class Extraction:
32
+ pages: int | None
33
+ chunks: list[ChunkData]
34
+
35
+
36
+ class ExtractionFailure(Exception):
37
+ """reason is user-facing and names the specific cause."""
38
+
39
+
40
+ class ModelUnavailable(Exception):
41
+ """The worker couldn't load its model (uncached + offline, HF rate-limit, missing
42
+ dep). Transient, not the document's fault — the pipeline retries, records no row."""
43
+
44
+
45
+ def _stderr_tail(path: Path) -> str:
46
+ with open(path, "rb") as f:
47
+ f.seek(max(0, path.stat().st_size - STDERR_TAIL_BYTES))
48
+ lines = [ln for ln in f.read().decode(errors="replace").strip().splitlines() if ln.strip()]
49
+ return lines[-1] if lines else ""
50
+
51
+
52
+ def extract(path: Path, timeout: float = EXTRACT_TIMEOUT_SECONDS) -> Extraction:
53
+ with tempfile.TemporaryDirectory() as tmp:
54
+ out_json = Path(tmp) / "extraction.json"
55
+ stderr_path = Path(tmp) / "stderr.log"
56
+ with open(stderr_path, "wb") as stderr_file:
57
+ try:
58
+ proc = subprocess.run(
59
+ [
60
+ sys.executable,
61
+ "-m",
62
+ "groundly.ingestion.extract_worker",
63
+ str(
64
+ path.resolve()
65
+ ), # worker runs with cwd=tmp; relative paths must survive
66
+ str(out_json),
67
+ ],
68
+ stdout=subprocess.DEVNULL,
69
+ stderr=stderr_file,
70
+ cwd=tmp,
71
+ timeout=timeout,
72
+ )
73
+ except subprocess.TimeoutExpired:
74
+ raise ExtractionFailure(f"extraction timed out after {int(timeout)}s") from None
75
+
76
+ if proc.returncode == extract_worker.EXIT_MODEL_UNAVAILABLE:
77
+ raise ModelUnavailable(_stderr_tail(stderr_path) or "extractor model unavailable")
78
+ if proc.returncode == extract_worker.EXIT_NO_TEXT:
79
+ if path.suffix.lower() == ".pdf":
80
+ raise ExtractionFailure("scanned PDF — not supported")
81
+ raise ExtractionFailure("no extractable text")
82
+ if proc.returncode != 0:
83
+ tail = _stderr_tail(stderr_path) or f"exit code {proc.returncode}"
84
+ raise ExtractionFailure(f"parser failed: {tail}")
85
+
86
+ if out_json.stat().st_size > MAX_EXTRACTION_JSON_BYTES:
87
+ raise ExtractionFailure("extraction output too large")
88
+ data = json.loads(out_json.read_text())
89
+ return Extraction(pages=data["pages"], chunks=[ChunkData(**c) for c in data["chunks"]])
@@ -0,0 +1,126 @@
1
+ """Extraction worker — runs as `python -m groundly.ingestion.extract_worker <in> <out.json>`.
2
+
3
+ Always a child process: a parser crash on a hostile/broken file kills this process,
4
+ not the indexing run (UC-01 A2, security.md §3). Digital documents only — no OCR;
5
+ an empty text layer exits with EXIT_NO_TEXT so the parent reports the specific cause.
6
+ A model that can't be loaded (uncached + offline, HF rate-limit, missing dep) exits
7
+ with EXIT_MODEL_UNAVAILABLE — an environment failure, retryable, never a bad document.
8
+
9
+ Output JSON: {"pages": N|null, "chunks": [{"text", "heading_path", "page", "token_count"}]}
10
+ """
11
+
12
+ import json
13
+ import sys
14
+ from pathlib import Path
15
+
16
+ EXIT_NO_TEXT = 3
17
+ EXIT_MODEL_UNAVAILABLE = 4
18
+
19
+ DOCLING_SUFFIXES = {".pdf", ".docx", ".pptx", ".md"}
20
+ # Everything else on the pipeline allowlist (txt + source code) is read as plain
21
+ # text and chunked by token windows — docling's converter does not accept it.
22
+
23
+
24
+ def _bge_m3_tokenizer():
25
+ from transformers import AutoTokenizer
26
+
27
+ from groundly.core.manifest import EMBEDDING_MODEL, HF_REVISION
28
+
29
+ return AutoTokenizer.from_pretrained(EMBEDDING_MODEL, revision=HF_REVISION)
30
+
31
+
32
+ def _model_step(fn):
33
+ """Model loading is the network/dependency-bound step. A failure (uncached + offline,
34
+ HF rate-limit, missing dep) is transient — the parent retries it — so it exits
35
+ distinctly, never collapsing into a terminal 'bad document'."""
36
+ try:
37
+ return fn()
38
+ except Exception as exc:
39
+ print(f"model unavailable: {exc}", file=sys.stderr)
40
+ sys.exit(EXIT_MODEL_UNAVAILABLE)
41
+
42
+
43
+ def _extract_docling(path: Path) -> dict:
44
+ from docling.chunking import HybridChunker
45
+ from docling.datamodel.base_models import InputFormat
46
+ from docling.document_converter import DocumentConverter
47
+ from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
48
+
49
+ from groundly.core.manifest import CHUNK_MAX_TOKENS
50
+
51
+ formats = {
52
+ ".pdf": InputFormat.PDF,
53
+ ".docx": InputFormat.DOCX,
54
+ ".pptx": InputFormat.PPTX,
55
+ ".md": InputFormat.MD,
56
+ }
57
+ converter = DocumentConverter()
58
+ # docling's layout models load here, before the document is touched — a fetch
59
+ # failure is the environment's fault, never this document's parse failure
60
+ _model_step(lambda: converter.initialize_pipeline(formats[path.suffix.lower()]))
61
+ doc = converter.convert(path).document
62
+ tokenizer = HuggingFaceTokenizer(
63
+ tokenizer=_model_step(_bge_m3_tokenizer), max_tokens=CHUNK_MAX_TOKENS
64
+ )
65
+ chunker = HybridChunker(tokenizer=tokenizer, merge_peers=True)
66
+
67
+ chunks = []
68
+ for chunk in chunker.chunk(doc):
69
+ text = chunker.contextualize(chunk) # heading path prepended — what gets embedded
70
+ headings = getattr(chunk.meta, "headings", None) or []
71
+ page = None
72
+ for item in getattr(chunk.meta, "doc_items", []) or []:
73
+ prov = getattr(item, "prov", None)
74
+ if prov:
75
+ page = prov[0].page_no
76
+ break
77
+ chunks.append(
78
+ {
79
+ "text": text,
80
+ "heading_path": " > ".join(headings) or None,
81
+ "page": page,
82
+ "token_count": tokenizer.count_tokens(text),
83
+ }
84
+ )
85
+
86
+ pages = len(doc.pages) if doc.pages else None
87
+ return {"pages": pages, "chunks": chunks}
88
+
89
+
90
+ def _extract_plain_text(path: Path) -> dict:
91
+ from groundly.core.manifest import CHUNK_MAX_TOKENS
92
+
93
+ text = path.read_text(errors="replace")
94
+ tokenizer = _model_step(_bge_m3_tokenizer)
95
+ ids = tokenizer.encode(text, add_special_tokens=False)
96
+ chunks = []
97
+ for start in range(0, len(ids), CHUNK_MAX_TOKENS):
98
+ window = ids[start : start + CHUNK_MAX_TOKENS]
99
+ chunks.append(
100
+ {
101
+ "text": tokenizer.decode(window),
102
+ "heading_path": None,
103
+ "page": None,
104
+ "token_count": len(window),
105
+ }
106
+ )
107
+ return {"pages": None, "chunks": chunks}
108
+
109
+
110
+ def main() -> None:
111
+ in_path, out_path = Path(sys.argv[1]), Path(sys.argv[2])
112
+
113
+ if in_path.suffix.lower() in DOCLING_SUFFIXES:
114
+ result = _extract_docling(in_path)
115
+ else:
116
+ result = _extract_plain_text(in_path)
117
+
118
+ if not any(c["text"].strip() for c in result["chunks"]):
119
+ # digital-documents-only rule (pivot #3): empty text layer, no OCR fallback
120
+ sys.exit(EXIT_NO_TEXT)
121
+
122
+ out_path.write_text(json.dumps(result))
123
+
124
+
125
+ if __name__ == "__main__":
126
+ main()
@@ -0,0 +1,210 @@
1
+ """The index pipeline (UC-01): hash-skip idempotent, per-file transactions, the run
2
+ continues past failures. Ingestion writes the stores; it never serves queries."""
3
+
4
+ import hashlib
5
+ import shutil
6
+ import sqlite3
7
+ from collections.abc import Callable
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+
11
+ import sqlite_vec
12
+
13
+ from groundly.core import store
14
+ from groundly.core.manifest import sync_counts
15
+ from groundly.core.paths import subject_dir
16
+ from groundly.ingestion.extract import ExtractionFailure, ModelUnavailable, extract
17
+ from groundly.llm.embeddings import BgeM3Embedder, Embedder
18
+
19
+ SUPPORTED_SUFFIXES = {
20
+ ".pdf",
21
+ ".docx",
22
+ ".pptx",
23
+ ".txt",
24
+ ".md",
25
+ ".py",
26
+ ".c",
27
+ ".cpp",
28
+ ".h",
29
+ ".hpp",
30
+ ".java",
31
+ ".js",
32
+ ".ts",
33
+ ".rs",
34
+ ".go",
35
+ }
36
+
37
+ # FileResult.status values
38
+ INDEXED = "indexed"
39
+ SKIPPED_DUPLICATE = "skipped_duplicate"
40
+ SKIPPED_UNSUPPORTED = "skipped_unsupported"
41
+ SKIPPED_FAILED = "skipped_failed" # failed on an earlier run; `remove` it to retry
42
+ EXTRACTION_FAILED = "extraction_failed" # terminal, recorded in store.db
43
+ ERROR = "error" # transient (e.g. embedder crash), no row recorded
44
+
45
+ OnEvent = Callable[[Path, str], None]
46
+
47
+
48
+ @dataclass
49
+ class FileResult:
50
+ path: Path
51
+ status: str
52
+ detail: str | None = None
53
+ chunks: int = 0
54
+
55
+
56
+ def _iter_files(paths: list[Path]) -> list[Path]:
57
+ files: list[Path] = []
58
+ for p in paths:
59
+ if p.is_dir():
60
+ files.extend(sorted(f for f in p.rglob("*") if f.is_file()))
61
+ else:
62
+ files.append(p)
63
+ return files
64
+
65
+
66
+ def _sha256(path: Path) -> str:
67
+ with open(path, "rb") as f:
68
+ return hashlib.file_digest(f, "sha256").hexdigest()
69
+
70
+
71
+ def _copy_to_materials(src: Path, sha256: str, materials_dir: Path) -> str:
72
+ """Original files are the citation targets; they ship in exports."""
73
+ dest = materials_dir / src.name
74
+ if dest.exists():
75
+ if dest.samefile(src): # orphan already in materials/ — or a hard link to it
76
+ return dest.name
77
+ if _sha256(dest) != sha256:
78
+ dest = materials_dir / f"{src.stem}-{sha256[:8]}{src.suffix}"
79
+ shutil.copy2(src, dest)
80
+ return dest.name
81
+
82
+
83
+ def index_paths(
84
+ subject: str,
85
+ paths: list[Path],
86
+ embedder: Embedder | None = None,
87
+ on_event: OnEvent | None = None,
88
+ ) -> list[FileResult]:
89
+ sdir = subject_dir(subject)
90
+ manifest_path = sdir / "manifest.json"
91
+ if not manifest_path.exists():
92
+ raise RuntimeError(f"subject '{subject}' is not initialized — run: groundly init {subject}")
93
+
94
+ emit = on_event or (lambda path, stage: None)
95
+ embedder = embedder or BgeM3Embedder()
96
+ results: list[FileResult] = []
97
+ conn = store.connect(sdir / "store.db")
98
+ try:
99
+ known = store.hash_status(conn)
100
+ for path in _iter_files(paths):
101
+ emit(path, "queued")
102
+ if path.is_symlink(): # a hostile symlink would index (and later export)
103
+ results.append(FileResult(path, SKIPPED_UNSUPPORTED, "symlink — not followed"))
104
+ emit(path, SKIPPED_UNSUPPORTED)
105
+ continue
106
+ if not path.exists():
107
+ results.append(FileResult(path, ERROR, "file not found"))
108
+ emit(path, ERROR)
109
+ continue
110
+ if path.suffix.lower() not in SUPPORTED_SUFFIXES:
111
+ results.append(
112
+ FileResult(path, SKIPPED_UNSUPPORTED, f"unsupported type {path.suffix!r}")
113
+ )
114
+ emit(path, SKIPPED_UNSUPPORTED)
115
+ continue
116
+
117
+ sha = _sha256(path)
118
+ if known.get(sha) == "indexed":
119
+ results.append(FileResult(path, SKIPPED_DUPLICATE, "already indexed"))
120
+ emit(path, SKIPPED_DUPLICATE)
121
+ continue
122
+ if known.get(sha) == "extraction_failed":
123
+ # terminal per UC-01: don't re-extract every run; `remove` it to retry
124
+ error = store.find_materials(conn, sha)[0]["error"]
125
+ results.append(
126
+ FileResult(
127
+ path, SKIPPED_FAILED, f"failed previously: {error} — remove to retry"
128
+ )
129
+ )
130
+ emit(path, SKIPPED_FAILED)
131
+ continue
132
+
133
+ result = _index_one(conn, path, sha, sdir, embedder, emit)
134
+ results.append(result)
135
+ if result.status in (INDEXED, EXTRACTION_FAILED):
136
+ known[sha] = result.status # a same-content sibling this run is a skip
137
+ sync_counts(conn, manifest_path)
138
+ finally:
139
+ conn.close()
140
+ return results
141
+
142
+
143
+ def _index_one(
144
+ conn, path: Path, sha: str, sdir: Path, embedder: Embedder, emit: OnEvent
145
+ ) -> FileResult:
146
+ emit(path, "extracting")
147
+ try:
148
+ extraction = extract(path)
149
+ except ModelUnavailable as exc: # transient (offline/uncached model): no row, next run retries
150
+ emit(path, ERROR)
151
+ return FileResult(path, ERROR, f"extractor unavailable: {exc}")
152
+ except ExtractionFailure as failure:
153
+ try:
154
+ with conn:
155
+ conn.execute(
156
+ "INSERT INTO materials (filename, sha256, status, error) "
157
+ "VALUES (?, ?, 'extraction_failed', ?)",
158
+ (path.name, sha, str(failure)),
159
+ )
160
+ except sqlite3.IntegrityError:
161
+ pass # sha256 UNIQUE lost a race: a concurrent run recorded this content first
162
+ emit(path, EXTRACTION_FAILED)
163
+ return FileResult(path, EXTRACTION_FAILED, str(failure))
164
+
165
+ emit(path, "embedding")
166
+ try:
167
+ dense, sparse = embedder.encode([c.text for c in extraction.chunks])
168
+ except Exception as exc: # transient (model load/OOM): no row, next run retries
169
+ emit(path, ERROR)
170
+ return FileResult(path, ERROR, f"embedding failed: {exc}")
171
+
172
+ try:
173
+ stored_name = _copy_to_materials(path, sha, sdir / "materials")
174
+ except OSError as exc: # transient (disk full, permissions): no row, next run retries
175
+ emit(path, ERROR)
176
+ return FileResult(path, ERROR, f"copy to materials failed: {exc}")
177
+ try:
178
+ return _write_indexed(conn, path, sha, stored_name, extraction, dense, sparse, emit)
179
+ except sqlite3.IntegrityError:
180
+ # sha256 UNIQUE lost a race: a concurrent run (CLI + MCP share the store)
181
+ # indexed the same content between our hash check and this write
182
+ emit(path, SKIPPED_DUPLICATE)
183
+ return FileResult(path, SKIPPED_DUPLICATE, "already indexed (concurrent run)")
184
+
185
+
186
+ def _write_indexed(
187
+ conn, path: Path, sha: str, stored_name: str, extraction, dense, sparse, emit: OnEvent
188
+ ) -> FileResult:
189
+ with conn: # one transaction per file: Ctrl-C loses at most the in-flight file
190
+ cur = conn.execute(
191
+ "INSERT INTO materials (filename, sha256, status, pages) VALUES (?, ?, 'indexed', ?)",
192
+ (stored_name, sha, extraction.pages),
193
+ )
194
+ material_id = cur.lastrowid
195
+ for chunk, vec, weights in zip(extraction.chunks, dense, sparse, strict=True):
196
+ cid = conn.execute(
197
+ "INSERT INTO chunks (material_id, page, heading_path, text, token_count) "
198
+ "VALUES (?, ?, ?, ?, ?)",
199
+ (material_id, chunk.page, chunk.heading_path, chunk.text, chunk.token_count),
200
+ ).lastrowid
201
+ conn.execute(
202
+ "INSERT INTO vectors (rowid, embedding) VALUES (?, ?)",
203
+ (cid, sqlite_vec.serialize_float32(vec)),
204
+ )
205
+ conn.executemany(
206
+ "INSERT INTO sparse_terms (token_id, chunk_id, weight) VALUES (?, ?, ?)",
207
+ [(token_id, cid, weight) for token_id, weight in weights.items()],
208
+ )
209
+ emit(path, INDEXED)
210
+ return FileResult(path, INDEXED, chunks=len(extraction.chunks))
File without changes
@@ -0,0 +1,62 @@
1
+ """bge-m3 embedding: dense (1024-d, normalized) + learned sparse from one forward pass.
2
+
3
+ Lives in llm/ because embedding clients are constructed only here (overview.md module
4
+ rules); it is local and key-free, so no cost metering applies. Lazy-loaded — never at
5
+ import/spawn time (.claude/rules/architecture.md). The model is
6
+ resolved at the pinned hf_revision via snapshot_download, which is the interchange
7
+ compatibility contract: same pin ⇒ shared vectors transfer as-is.
8
+ """
9
+
10
+ import sys
11
+ from typing import Protocol
12
+
13
+ from groundly.core.manifest import EMBEDDING_MODEL, HF_REVISION
14
+
15
+ SparseWeights = dict[int, float]
16
+
17
+
18
+ class Embedder(Protocol):
19
+ def encode(self, texts: list[str]) -> tuple[list[list[float]], list[SparseWeights]]: ...
20
+
21
+
22
+ class BgeM3Embedder:
23
+ def __init__(self) -> None:
24
+ self._model = None
25
+
26
+ def _load(self):
27
+ if self._model is None:
28
+ from pathlib import Path
29
+
30
+ from huggingface_hub import snapshot_download
31
+
32
+ try:
33
+ # a snapshot can exist with only tokenizer files (the extraction
34
+ # worker caches those) — "cached" means the weights are present
35
+ local = snapshot_download(
36
+ EMBEDDING_MODEL, revision=HF_REVISION, local_files_only=True
37
+ )
38
+ if not any(
39
+ (Path(local) / name).exists()
40
+ for name in ("model.safetensors", "pytorch_model.bin")
41
+ ):
42
+ raise FileNotFoundError("model weights not cached")
43
+ except Exception:
44
+ print(
45
+ f"downloading {EMBEDDING_MODEL} (one-time, ~2.3 GB) …",
46
+ file=sys.stderr,
47
+ )
48
+ local = snapshot_download(EMBEDDING_MODEL, revision=HF_REVISION)
49
+
50
+ from FlagEmbedding import BGEM3FlagModel
51
+
52
+ self._model = BGEM3FlagModel(local, use_fp16=False)
53
+ return self._model
54
+
55
+ def encode(self, texts: list[str]) -> tuple[list[list[float]], list[SparseWeights]]:
56
+ out = self._load().encode(texts, return_dense=True, return_sparse=True)
57
+ dense = [vec.tolist() for vec in out["dense_vecs"]]
58
+ sparse = [
59
+ {int(token_id): float(weight) for token_id, weight in weights.items()}
60
+ for weights in out["lexical_weights"]
61
+ ]
62
+ return dense, sparse
File without changes
File without changes
File without changes
@@ -0,0 +1,22 @@
1
+ :root {
2
+ --bg: #111318;
3
+ --surface: #1a1c20;
4
+ --primary: #c5c0ff;
5
+ --primary-contrast: #2a2179;
6
+ --button-primary: #2f277e;
7
+ --button-primary-hover: #413a91;
8
+ --secondary: #89ceff;
9
+ --secondary-contrast: #00344d;
10
+ --error: #ffb4ab;
11
+ --success: #6ee7b7;
12
+ --warning: #fcd34d;
13
+ --text: #e2e2e8;
14
+ --text-secondary: #c8c4d3;
15
+ --divider: #474551;
16
+
17
+ --font-body: "Inter", system-ui, sans-serif;
18
+ --font-heading: "Source Serif 4", Georgia, serif;
19
+ --font-mono: "JetBrains Mono", monospace;
20
+ --radius: 4px;
21
+ --radius-card: 8px;
22
+ }
@@ -0,0 +1,88 @@
1
+ Metadata-Version: 2.4
2
+ Name: groundly
3
+ Version: 0.3.0
4
+ Summary: Local-first course knowledge bases for AI agents: index your materials, study grounded, share the index.
5
+ License: MIT License
6
+
7
+ Copyright (c) 2026 Paul Hondola
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+ License-File: LICENSE
27
+ Requires-Python: <3.13,>=3.11
28
+ Requires-Dist: docling==2.113.0
29
+ Requires-Dist: fastapi>=0.111
30
+ Requires-Dist: fastmcp>=2.0
31
+ Requires-Dist: flagembedding==1.3.5
32
+ Requires-Dist: genanki>=0.13
33
+ Requires-Dist: graphrag==3.1.0
34
+ Requires-Dist: httpx>=0.27
35
+ Requires-Dist: llama-index==0.14.23
36
+ Requires-Dist: pydantic-settings>=2.3
37
+ Requires-Dist: pydantic>=2.7
38
+ Requires-Dist: rich>=13
39
+ Requires-Dist: sentence-transformers==5.6.0
40
+ Requires-Dist: sqlite-vec>=0.1
41
+ Requires-Dist: typer>=0.12
42
+ Requires-Dist: uvicorn>=0.30
43
+ Provides-Extra: dev
44
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
45
+ Requires-Dist: pytest>=8; extra == 'dev'
46
+ Requires-Dist: ruff>=0.5; extra == 'dev'
47
+ Provides-Extra: eval
48
+ Requires-Dist: ragas>=0.2; extra == 'eval'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # Groundly
52
+
53
+ **Local-first course knowledge bases for AI agents.** Index your course materials once; then any MCP-capable agent — Claude Code, Claude Desktop, Codex — can answer questions grounded in the actual course content with page-level citations, generate execution-verified tests and flashcards, quiz your weak areas, and track your mastery. Share the finished knowledge base with your coursemates as a single file.
54
+
55
+ Bachelor thesis project, Universitatea Politehnica Timișoara.
56
+
57
+ ```
58
+ $ groundly index <SUBJECT> ./slides/
59
+ ✓ 12 PDFs → 384 chunks → embedded (bge-m3, local) → indexed
60
+ $ # wire into Claude Code: { "command": "groundly", "args": ["mcp"] }
61
+ ```
62
+
63
+ Then, inside your agent: *"What did lecture 4 say about deadlock prevention?"* → answer cited to **lecture-04.pdf, page 12, "Deadlocks › Prevention"** — or *"not covered by the course materials"*, never a hallucination.
64
+
65
+ ## Why not NotebookLM / a Claude Project?
66
+
67
+ 1. **Verified generation** — every test question and flashcard passes a verifier: answerable from the cited sources, answer key checked, and code questions' reference solutions **actually executed** before acceptance.
68
+ 2. **Enforced citations** — structural, page-level, with honest refusal when the corpus doesn't cover it.
69
+ 3. **A portable index** — `groundly export <SUBJECT>` → one file; a coursemate imports it and their agent uses it directly. The expensive artifacts (knowledge graph, verified decks) are built once and shared.
70
+ 4. **Cross-host progress** — mastery per topic and study memory persist in local files, whatever agent you talk to. Your queries and results never leave your machine and are never part of an export.
71
+
72
+ ## Features
73
+
74
+ - **Grounded Q&A** — hybrid retrieval (bge-m3 dense + learned sparse + BM25, reranked; optional GraphRAG for synthesis questions), via MCP tools or `groundly ask`
75
+ - **Verified quizzes & coding challenges** — generated by Groundly (your API key) *or by your agent* (no key needed) — either way, nothing unverified is stored
76
+ - **Flashcards → Anki** — verified decks exported as `.apkg`
77
+ - **Mastery map & study memory** — per-topic mastery from quiz results; "continue studying" warm starts in any host
78
+ - **Import/export** — the whole knowledge base as one shareable file, with pinned-model compatibility
79
+
80
+ ## Install
81
+
82
+ ```
83
+ uv tool install groundly
84
+ ```
85
+
86
+ Local-first honestly stated: first run downloads the embedding models (~2.7GB total). Indexing and search need **no API key**; grounded answer generation and graph builds use your own OpenAI-compatible provider (cloud key, LM Studio, or Ollama).
87
+
88
+ **Thesis core:** four-arm RAG vs GraphRAG evaluation on real RO/EN course corpora, plus a measured comparison of enforced vs agent-mediated grounding.
@@ -0,0 +1,23 @@
1
+ groundly/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ groundly/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ groundly/cli/__init__.py,sha256=sbtFMmcGY_m4-yGRYZhNUFoBvVtZcfHSP4yVWDvVe2w,9173
4
+ groundly/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ groundly/core/manifest.py,sha256=u-wR8MNP1enbbFfmFegxY2-NUBpbegDDUPetHJf9vR4,2380
6
+ groundly/core/paths.py,sha256=H5RGprGcNNdi-a3cOUVyUFBpczYZjk97YsvzHD77YeE,1026
7
+ groundly/core/store.py,sha256=rJUtqkaIQteMoGuwFcToLDRB3C84X-Zqdp4M5deNmlg,5328
8
+ groundly/core/subject.py,sha256=ZwdH_Imwx6u3VVMxhLwol8k-BntB4hC59Da6OgAj328,1539
9
+ groundly/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ groundly/ingestion/extract.py,sha256=of6jCglIgeax8WhB7oWmRYaga-bxr-mBrJEslL7JbeI,3352
11
+ groundly/ingestion/extract_worker.py,sha256=-GDY_FYuYO7GTBVoIkXIhcpQHACi40NXkuM5j93vSO0,4516
12
+ groundly/ingestion/pipeline.py,sha256=lMfvtMgvwFajvFEZeAjD9vu3aNZrdFUYBHAKofbvV80,7809
13
+ groundly/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
+ groundly/llm/embeddings.py,sha256=uWT5zm9Jo80b9FEr7vlHidbewDpwSNEUghJ3D6WF7Jc,2375
15
+ groundly/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
16
+ groundly/retrieval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
17
+ groundly/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
18
+ groundly/web/static/theme.css,sha256=4e8rbyAy5RAkNURj48CVB30xppu1rZWC0-mi5Ldg9Ls,521
19
+ groundly-0.3.0.dist-info/METADATA,sha256=Q27eYKckOwuMt5ry0r0aTxC3g4oHKwZvcE8aAb5YdTI,4951
20
+ groundly-0.3.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
21
+ groundly-0.3.0.dist-info/entry_points.txt,sha256=blWT1cwUIOmqI_YrHRbAt_mJdYMm0fPKH9Wj5QJs59c,46
22
+ groundly-0.3.0.dist-info/licenses/LICENSE,sha256=lCwN0jTaLVFgdybko2FT6i7jjLrhHYwedxP3s6wQSj0,1069
23
+ groundly-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.31.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ groundly = groundly.cli:app
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Paul Hondola
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.