ihmt-memory 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ihmt/project_index.py ADDED
@@ -0,0 +1,440 @@
1
+ """Project index: IHMT as a token-saving cache of a source tree.
2
+
3
+ Long-term memory answers "what did we decide last month?". A coding session
4
+ asks a different question, over and over: "where is X, and what does it say?".
5
+ Answering it by reading whole files fills the context with text the model does
6
+ not need — and the same files get read again after every change or compaction.
7
+
8
+ A :class:`ProjectIndex` keeps a private IHMT store per project:
9
+
10
+ * **one leaf per symbol** (``code_chunk_mode="symbol"``), so a lookup returns a
11
+ single method with its ``file:line`` range instead of the whole file;
12
+ * **checksum sync** — every call rescans the tree cheaply (size + mtime), hashes
13
+ only what moved, re-ingests only files whose SHA-256 changed, deletes the
14
+ leaves of changed or removed files and rebuilds the branches. A lookup never
15
+ serves stale code;
16
+ * **a map for free** — :meth:`map` lists files and their symbols from the sync
17
+ manifest, without opening a single leaf.
18
+
19
+ The store is derived data: deleting it only costs a re-index. It never touches
20
+ the user's long-term memory.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import hashlib
26
+ import json
27
+ import re
28
+ import time
29
+ from dataclasses import dataclass, field
30
+ from datetime import datetime, timedelta, timezone
31
+ from pathlib import Path
32
+ from typing import Any, Dict, List, Optional, Tuple
33
+
34
+ from .api import IHMT
35
+ from .models import IHMTError
36
+ from .semantic_navigator import SCOPES, looks_like_code
37
+ from .textutils import estimate_tokens, short_hash, slugify
38
+
39
+ #: Text files worth indexing. Anything else (binaries, lock files, media) is skipped.
40
+ INDEXABLE_SUFFIXES = frozenset(
41
+ {
42
+ ".py", ".pyi", ".java", ".kt", ".kts", ".scala", ".groovy", ".js", ".mjs", ".cjs", ".jsx", ".ts",
43
+ ".tsx", ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".cs", ".go", ".rs", ".swift", ".php",
44
+ ".rb", ".sql", ".md", ".txt", ".rst", ".xml", ".yml", ".yaml", ".toml", ".ini", ".cfg",
45
+ ".properties", ".gradle", ".json", ".html", ".css", ".scss", ".sh", ".bash", ".zsh",
46
+ }
47
+ )
48
+ #: Directories never worth indexing, at any depth: VCS, dependencies, build output, IDE state.
49
+ SKIP_DIRECTORIES = frozenset(
50
+ {
51
+ ".git", ".hg", ".svn", "__pycache__", "node_modules", ".venv", "venv", "dist", "build",
52
+ "target", ".idea", ".vscode", ".gradle", ".mvn", ".next", ".nuxt", "coverage",
53
+ ".pytest_cache", ".mypy_cache", ".tox", "ihmt_memory", "ihmt_projects",
54
+ }
55
+ )
56
+ #: Build-output names that are also legitimate package names deeper down —
57
+ #: hexagonal code lives in ``adapter/out`` — so they are skipped only at the root.
58
+ SKIP_AT_ROOT = frozenset({"out", "bin", "obj", "env"})
59
+ #: Leaves are grouped into branches in timestamp order. Stamping each file with
60
+ #: its rank in path order makes every branch cover neighbouring files and
61
+ #: symbols — a package, a class — so branch summaries stay meaningful and the
62
+ #: descent can tell them apart. (Ingest-time stamps would mix files at random.)
63
+ _EPOCH = datetime(2000, 1, 1, tzinfo=timezone.utc)
64
+
65
+
66
+ def _path_order_timestamp(rank: int) -> str:
67
+ return (_EPOCH + timedelta(seconds=rank)).isoformat().replace("+00:00", "Z")
68
+
69
+
70
+ MAX_FILE_BYTES = 256 * 1024
71
+ MAX_FILES = 5000
72
+ MANIFEST = "sources.json"
73
+
74
+ #: Store shape tuned for code lookups: small leaves, one per symbol.
75
+ PROJECT_CONFIG: Dict[str, Any] = {
76
+ "branch_factor": 8,
77
+ "beam_width": 6, # a wider beam costs local file reads, not model tokens
78
+ "target_tokens": 150,
79
+ "max_tokens": 3000,
80
+ "min_tokens": 1,
81
+ "code_chunk_mode": "symbol",
82
+ }
83
+
84
+
85
+ @dataclass
86
+ class SyncReport:
87
+ """What a :meth:`ProjectIndex.sync` found and did."""
88
+
89
+ files: int = 0
90
+ added: int = 0
91
+ changed: int = 0
92
+ removed: int = 0
93
+ skipped: int = 0
94
+ truncated: bool = False
95
+ elapsed_ms: float = 0.0
96
+
97
+ @property
98
+ def touched(self) -> int:
99
+ return self.added + self.changed + self.removed
100
+
101
+ def summary(self) -> str:
102
+ parts = [f"{self.files} files indexed"]
103
+ if self.touched:
104
+ parts.append(f"{self.added} new, {self.changed} changed, {self.removed} removed")
105
+ else:
106
+ parts.append("no changes")
107
+ if self.truncated:
108
+ parts.append(f"stopped at {MAX_FILES} files")
109
+ return ", ".join(parts) + f" ({self.elapsed_ms:.0f} ms)"
110
+
111
+
112
+ @dataclass
113
+ class CodeHit:
114
+ """One symbol returned by :meth:`ProjectIndex.find`."""
115
+
116
+ source: str
117
+ title: str
118
+ start_line: int
119
+ end_line: int
120
+ content: str
121
+ context: str = ""
122
+
123
+ def render(self, max_chars: Optional[int] = None) -> str:
124
+ body = self.content
125
+ if max_chars is not None and len(body) > max_chars:
126
+ body = body[:max_chars] + "\n[... truncated]"
127
+ header = f"{self.source}:{self.start_line}-{self.end_line} {self.title}"
128
+ if self.context and self.context.strip() not in body:
129
+ header += f"\n(in: {self.context.strip().splitlines()[0]})"
130
+ return f"{header}\n{body.rstrip()}"
131
+
132
+
133
+ @dataclass
134
+ class FindResult:
135
+ """Outcome of a lookup: hits, or the candidates when it is ambiguous."""
136
+
137
+ status: str # "found" | "ambiguous" | "none"
138
+ hits: List[CodeHit] = field(default_factory=list)
139
+ candidates: List[str] = field(default_factory=list)
140
+ confidence: float = 0.0
141
+
142
+
143
+ class ProjectIndex:
144
+ """A per-project IHMT store kept in sync with the files on disk."""
145
+
146
+ def __init__(self, project_root: Path | str, store_dir: Path | str) -> None:
147
+ self.root = Path(project_root).expanduser().resolve()
148
+ if not self.root.is_dir():
149
+ raise IHMTError(f"not a directory: {self.root}")
150
+ self.store_dir = Path(store_dir).expanduser()
151
+ if not (self.store_dir / "ihmt_memory" / "root.json").exists():
152
+ IHMT.initialize(self.store_dir, **PROJECT_CONFIG)
153
+ self.memory = IHMT(self.store_dir, auto_consolidate=False)
154
+
155
+ @classmethod
156
+ def for_project(cls, project_root: Path | str, cache_home: Path | str) -> "ProjectIndex":
157
+ """Open (or create) the index of ``project_root`` under ``cache_home``."""
158
+ root = Path(project_root).expanduser().resolve()
159
+ name = f"{slugify(root.name, max_length=40) or 'project'}-{short_hash(str(root), 8)}"
160
+ return cls(root, Path(cache_home).expanduser() / name)
161
+
162
+ # ----------------------------------------------------------------- manifest
163
+ @property
164
+ def _manifest_path(self) -> Path:
165
+ return self.memory.config.state_dir / MANIFEST
166
+
167
+ def _load_manifest(self) -> Dict[str, Dict[str, Any]]:
168
+ try:
169
+ return json.loads(self._manifest_path.read_text(encoding="utf-8")).get("files", {})
170
+ except (FileNotFoundError, json.JSONDecodeError):
171
+ return {}
172
+
173
+ def _save_manifest(self, files: Dict[str, Dict[str, Any]]) -> None:
174
+ payload = {"project": str(self.root), "files": dict(sorted(files.items()))}
175
+ self.memory.store.write_json(self._manifest_path, payload)
176
+
177
+ # --------------------------------------------------------------------- scan
178
+ def scan(self) -> Tuple[Dict[str, Path], int, bool]:
179
+ """Indexable files under the root: ``({relative path: path}, skipped, truncated)``."""
180
+ found: Dict[str, Path] = {}
181
+ skipped = 0
182
+ for path in sorted(self.root.rglob("*")):
183
+ if not path.is_file():
184
+ continue
185
+ relative = path.relative_to(self.root)
186
+ directories = relative.parts[:-1]
187
+ if any(part in SKIP_DIRECTORIES or part.startswith(".") for part in directories):
188
+ continue
189
+ if directories and directories[0] in SKIP_AT_ROOT:
190
+ continue
191
+ if path.suffix.lower() not in INDEXABLE_SUFFIXES:
192
+ continue
193
+ try:
194
+ if path.stat().st_size > MAX_FILE_BYTES:
195
+ skipped += 1
196
+ continue
197
+ except OSError:
198
+ continue
199
+ if len(found) >= MAX_FILES:
200
+ return found, skipped, True
201
+ found[relative.as_posix()] = path
202
+ return found, skipped, False
203
+
204
+ # --------------------------------------------------------------------- sync
205
+ def sync(self) -> SyncReport:
206
+ """Bring the index in line with the files on disk. Cheap when nothing changed."""
207
+ started = time.perf_counter()
208
+ report = SyncReport()
209
+ known = self._load_manifest()
210
+ files, report.skipped, report.truncated = self.scan()
211
+ report.files = len(files)
212
+
213
+ to_ingest: Dict[str, str] = {}
214
+ manifest: Dict[str, Dict[str, Any]] = {}
215
+ for relative, path in files.items():
216
+ stat = path.stat()
217
+ previous = known.get(relative)
218
+ if previous and previous.get("size") == stat.st_size and previous.get("mtime_ns") == stat.st_mtime_ns:
219
+ manifest[relative] = previous # untouched: not even hashed
220
+ continue
221
+ data = path.read_bytes()
222
+ digest = hashlib.sha256(data).hexdigest()
223
+ if previous and previous.get("sha256") == digest:
224
+ manifest[relative] = {**previous, "size": stat.st_size, "mtime_ns": stat.st_mtime_ns}
225
+ continue
226
+ try:
227
+ text = data.decode("utf-8")
228
+ except UnicodeDecodeError:
229
+ report.skipped += 1
230
+ continue
231
+ to_ingest[relative] = text
232
+ manifest[relative] = {"sha256": digest, "size": stat.st_size, "mtime_ns": stat.st_mtime_ns}
233
+ if previous:
234
+ report.changed += 1
235
+ else:
236
+ report.added += 1
237
+
238
+ removed = [relative for relative in known if relative not in files]
239
+ report.removed = len(removed)
240
+
241
+ if report.touched:
242
+ store = self.memory.store
243
+ with store.batch():
244
+ for relative in removed + [r for r in to_ingest if r in known]:
245
+ for leaf in known[relative].get("leaves", []):
246
+ store.delete_leaf(leaf["id"])
247
+ rank = {relative: index for index, relative in enumerate(sorted(files))}
248
+ for relative, text in to_ingest.items():
249
+ ingest = self.memory.ingest_text(
250
+ text, source=relative, timestamp=_path_order_timestamp(rank[relative])
251
+ )
252
+ leaves = []
253
+ for leaf_id in ingest.leaf_ids:
254
+ leaf = self.memory.get_leaf(leaf_id)
255
+ title = leaf.title.split(" · ", 1)[-1]
256
+ leaves.append(
257
+ {"id": leaf_id, "title": title, "start": leaf.span.start_line, "end": leaf.span.end_line}
258
+ )
259
+ manifest[relative].update(tokens=estimate_tokens(text), leaves=leaves)
260
+ # Branches summarize their children, so rebuild them from the surviving leaves.
261
+ IHMT.initialize(self.store_dir, force=True)
262
+ self.memory = IHMT(self.store_dir, auto_consolidate=False)
263
+ self.memory.flush()
264
+
265
+ if report.touched or set(manifest) != set(known):
266
+ self._save_manifest(manifest)
267
+ report.elapsed_ms = (time.perf_counter() - started) * 1000.0
268
+ return report
269
+
270
+ # --------------------------------------------------------------------- read
271
+ def map(self, *, detail: str = "symbols", subpath: str = "", max_lines: int = 400) -> str:
272
+ """A compact map of the project, built from the manifest (no leaf is opened).
273
+
274
+ Args:
275
+ detail: ``"files"`` (one line per file) or ``"symbols"`` (plus the
276
+ symbols of each file with their line ranges).
277
+ subpath: Restrict the map to a directory, relative to the root.
278
+ max_lines: Cap on the number of lines returned.
279
+ """
280
+ if detail not in ("files", "symbols"):
281
+ raise ValueError('detail must be "files" or "symbols"')
282
+ manifest = self._load_manifest()
283
+ prefix = subpath.strip("/")
284
+ lines: List[str] = []
285
+ total_tokens = 0
286
+ current_dir = None
287
+ for relative, info in sorted(manifest.items()):
288
+ if prefix and not relative.startswith(prefix):
289
+ continue
290
+ total_tokens += int(info.get("tokens", 0))
291
+ directory, _, name = relative.rpartition("/")
292
+ if directory != current_dir:
293
+ lines.append(f"{directory or '.'}/")
294
+ current_dir = directory
295
+ line = f" {name} ~{info.get('tokens', 0)}"
296
+ if detail == "symbols" and looks_like_code(relative):
297
+ # Only source code has meaningful symbols; for HTML, Markdown or
298
+ # XML the "blocks" are markup fragments that cost tokens and
299
+ # orient no one.
300
+ stem = name.rsplit(".", 1)[0]
301
+ symbols = [
302
+ f"{self._short_symbol(leaf['title'], stem)} {leaf['start']}-{leaf['end']}"
303
+ for leaf in info.get("leaves", [])
304
+ if self._worth_listing(leaf.get("title", ""))
305
+ ]
306
+ if symbols:
307
+ line += ": " + ", ".join(symbols)
308
+ lines.append(line)
309
+ if len(lines) >= max_lines:
310
+ lines.append(f"[... map truncated at {max_lines} lines; pass subpath to zoom in]")
311
+ break
312
+ header = (
313
+ f"{self.root.name}: {len(manifest)} files, ~{total_tokens} tokens if read whole "
314
+ "(format: file ~tokens: symbol first-last line)"
315
+ )
316
+ return "\n".join([header] + lines)
317
+
318
+ @staticmethod
319
+ def _worth_listing(title: str) -> bool:
320
+ """Imports, module preambles and class headers add length, not orientation."""
321
+ if not title:
322
+ return False
323
+ lowered = title.lower()
324
+ return not (lowered.startswith("imports") or lowered.startswith("module level") or "(header)" in lowered)
325
+
326
+ @staticmethod
327
+ def _short_symbol(title: str, stem: str) -> str:
328
+ """``Pedido.confirmar`` in Pedido.java is just ``confirmar``; runs become ``a…b``."""
329
+ parts = [p.strip() for p in title.split("→")]
330
+ short = []
331
+ for part in parts:
332
+ for prefix in (f"{stem}.", f"class {stem}", f"record {stem}", f"interface {stem}", f"enum {stem}"):
333
+ if part.startswith(prefix) and part != prefix.strip("."):
334
+ part = part[len(prefix):].lstrip(". ") or part
335
+ break
336
+ short.append(part)
337
+ return "…".join(short) if len(short) > 1 else short[0]
338
+
339
+ def find(
340
+ self,
341
+ query: str,
342
+ *,
343
+ scope: str = "main",
344
+ subpath: str = "",
345
+ top_k: int = 1,
346
+ clue: Optional[str] = None,
347
+ ) -> FindResult:
348
+ """Return the symbol(s) that best answer ``query``."""
349
+ if scope not in SCOPES:
350
+ raise ValueError(f"scope must be one of {SCOPES}")
351
+ exact = self._find_exact_symbol(query, scope=scope, subpath=subpath.strip("/"))
352
+ if exact is not None:
353
+ return FindResult(status="found", hits=[exact], confidence=1.0)
354
+ response = self.memory.search(
355
+ query, top_k=max(3, top_k), clue=clue, scope=scope, path_prefix=subpath.strip("/") or None
356
+ )
357
+ if not response.results:
358
+ return FindResult(status="none")
359
+ sources = {self._source_of(r.leaf_id) for r in response.results[:3]}
360
+ if response.needs_clue and len(sources) > 1:
361
+ candidates = []
362
+ for result in response.results[:4]:
363
+ candidates.append(f"{self._source_of(result.leaf_id)} · {result.title.split(' · ', 1)[-1]}")
364
+ return FindResult(status="ambiguous", candidates=candidates, confidence=response.confidence)
365
+ hits = []
366
+ for result in response.results[:top_k]:
367
+ leaf = self.memory.get_leaf(result.leaf_id)
368
+ hits.append(
369
+ CodeHit(
370
+ source=leaf.source,
371
+ title=leaf.title.split(" · ", 1)[-1],
372
+ start_line=leaf.span.start_line,
373
+ end_line=leaf.span.end_line,
374
+ content=leaf.content,
375
+ context=str(leaf.extra.get("context", "")),
376
+ )
377
+ )
378
+ return FindResult(status="found", hits=hits, confidence=response.confidence)
379
+
380
+ #: Paths that hold legacy or compatibility copies of an API; a symbol found
381
+ #: there only wins when nothing else defines it.
382
+ _LEGACY_PARTS = ("v1/", "deprecated/", "compat/", "legacy/")
383
+
384
+ def _find_exact_symbol(self, query: str, *, scope: str, subpath: str) -> Optional[CodeHit]:
385
+ """Resolve a query that names a symbol (``BaseModel.model_dump``, ``field_validator``).
386
+
387
+ Looked up in the sync manifest — an in-memory scan, no tree descent — so
388
+ a lookup by name is never lost to lexical ranking. Among several
389
+ definitions (``@overload`` stubs, legacy copies, tests) the largest leaf
390
+ of the most relevant file wins: that is where the implementation lives.
391
+ """
392
+ from .semantic_navigator import looks_like_test
393
+
394
+ identifiers = [i for i in re.findall(r"[A-Za-z_][A-Za-z0-9_.]*[A-Za-z0-9_]", query) if len(i) >= 4]
395
+ if not identifiers:
396
+ return None
397
+ wanted_full = {i for i in identifiers if "." in i}
398
+ wanted_last = {i.rsplit(".", 1)[-1] for i in identifiers}
399
+ candidates = []
400
+ for relative, info in self._load_manifest().items():
401
+ if subpath and not relative.startswith(subpath):
402
+ continue
403
+ if scope != "all" and looks_like_test(relative) != (scope == "test"):
404
+ continue
405
+ for leaf in info.get("leaves", []):
406
+ for part in (p.strip() for p in leaf.get("title", "").split("→")):
407
+ name = re.sub(r"^(async\s+)?(def|function|class|method|record|interface|enum)\s+", "", part)
408
+ name = name.split(" (", 1)[0].strip()
409
+ if not name:
410
+ continue
411
+ full = name in wanted_full
412
+ last = name.rsplit(".", 1)[-1] in wanted_last
413
+ if full or last:
414
+ size = int(leaf.get("end", 0)) - int(leaf.get("start", 0))
415
+ legacy = any(marker in relative for marker in self._LEGACY_PARTS)
416
+ candidates.append(((full, not legacy, size), relative, leaf))
417
+ break
418
+ if not candidates:
419
+ return None
420
+ candidates.sort(key=lambda c: c[0], reverse=True)
421
+ _, _, leaf = candidates[0]
422
+ try:
423
+ stored = self.memory.get_leaf(leaf["id"])
424
+ except Exception: # noqa: BLE001 - a stale manifest entry falls back to search
425
+ return None
426
+ return CodeHit(
427
+ source=stored.source,
428
+ title=stored.title.split(" · ", 1)[-1],
429
+ start_line=stored.span.start_line,
430
+ end_line=stored.span.end_line,
431
+ content=stored.content,
432
+ context=str(stored.extra.get("context", "")),
433
+ )
434
+
435
+ def _source_of(self, leaf_id: str) -> str:
436
+ entry = self.memory.store.catalog.entries.get(leaf_id)
437
+ return entry.source if entry else ""
438
+
439
+
440
+ __all__ = ["ProjectIndex", "SyncReport", "CodeHit", "FindResult", "PROJECT_CONFIG"]