mintmory-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mintmory/cli/__init__.py +1 -0
- mintmory/cli/main.py +1889 -0
- mintmory/cli/py.typed +0 -0
- mintmory_cli-0.1.0.dist-info/METADATA +34 -0
- mintmory_cli-0.1.0.dist-info/RECORD +8 -0
- mintmory_cli-0.1.0.dist-info/WHEEL +4 -0
- mintmory_cli-0.1.0.dist-info/entry_points.txt +2 -0
- mintmory_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
mintmory/cli/main.py
ADDED
|
@@ -0,0 +1,1889 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MintMory CLI — direct stdio usage.
|
|
3
|
+
|
|
4
|
+
Commands:
|
|
5
|
+
mintmory add <content> --category fact
|
|
6
|
+
mintmory ingest <paths...> --category fact # bulk-ingest files/dirs (chunked)
|
|
7
|
+
mintmory search <query> [--around contradicts]
|
|
8
|
+
mintmory dream [--full]
|
|
9
|
+
mintmory stats
|
|
10
|
+
mintmory doctor # one-shot health check (DB, embedder, LLM tier)
|
|
11
|
+
mintmory serve [--port 8080] # start HTTP API
|
|
12
|
+
mintmory mcp # start MCP server (stdio)
|
|
13
|
+
mintmory note <content> [--about ...] [--when ISO] [--until ISO] [--category ...]
|
|
14
|
+
mintmory notes [--about ...] [--upcoming] [--overdue] [--limit N]
|
|
15
|
+
mintmory summary-jobs [--all/--needed] [--limit N] [--json]
|
|
16
|
+
mintmory summary-put CONCEPT [TEXT] [--file PATH]
|
|
17
|
+
mintmory image-jobs [--all/--needed] [--bytes/--no-bytes] [--limit N] [--json]
|
|
18
|
+
mintmory image-caption-put FILE_OR_ID [DESCRIPTION] [--file PATH]
|
|
19
|
+
mintmory vision-run [--limit N] [--budget MB] [--all/--needed]
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
from datetime import UTC, datetime
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import TYPE_CHECKING, cast
|
|
28
|
+
|
|
29
|
+
import numpy as np
|
|
30
|
+
import typer
|
|
31
|
+
from mintmory.core.types import ConceptLinkType, MemoryCategory, MemorySource
|
|
32
|
+
from numpy.typing import NDArray
|
|
33
|
+
from rich.console import Console
|
|
34
|
+
from rich.table import Table
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from mintmory.core.ports import StorageBackend
|
|
38
|
+
from mintmory.core.storage import SqliteBackend
|
|
39
|
+
|
|
40
|
+
app = typer.Typer(name="mintmory", help="MintMory — typed memory for LLM agents")
|
|
41
|
+
console = Console()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _get_store() -> StorageBackend:
|
|
45
|
+
from mintmory.core.config import load_settings
|
|
46
|
+
from mintmory.core.embedder import embedder_from_settings
|
|
47
|
+
from mintmory.core.storage_factory import build_storage
|
|
48
|
+
|
|
49
|
+
settings = load_settings()
|
|
50
|
+
store = build_storage(settings.storage, embedder=embedder_from_settings(settings.embed))
|
|
51
|
+
store.initialise()
|
|
52
|
+
return store
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _parse_category(category: str) -> MemoryCategory:
|
|
56
|
+
"""Coerce a raw string into a MemoryCategory or raise a clear typer error."""
|
|
57
|
+
try:
|
|
58
|
+
return MemoryCategory(category)
|
|
59
|
+
except ValueError as exc:
|
|
60
|
+
valid = ", ".join(c.value for c in MemoryCategory)
|
|
61
|
+
raise typer.BadParameter(f"invalid category {category!r}; choose one of: {valid}") from exc
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _parse_source(source: str) -> MemorySource:
|
|
65
|
+
"""Coerce a raw string into a MemorySource or raise a clear typer error."""
|
|
66
|
+
try:
|
|
67
|
+
return MemorySource(source)
|
|
68
|
+
except ValueError as exc:
|
|
69
|
+
valid = ", ".join(s.value for s in MemorySource)
|
|
70
|
+
raise typer.BadParameter(f"invalid source {source!r}; choose one of: {valid}") from exc
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _parse_link_type(link_type: str) -> ConceptLinkType:
|
|
74
|
+
"""Coerce a raw string into a ConceptLinkType or raise a clear typer error."""
|
|
75
|
+
try:
|
|
76
|
+
return ConceptLinkType(link_type)
|
|
77
|
+
except ValueError as exc:
|
|
78
|
+
valid = ", ".join(lt.value for lt in ConceptLinkType)
|
|
79
|
+
raise typer.BadParameter(
|
|
80
|
+
f"invalid link type {link_type!r}; choose one of: {valid}"
|
|
81
|
+
) from exc
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _parse_iso(value: str | None) -> datetime | None:
|
|
85
|
+
"""Parse an ISO-8601 date/datetime string or raise ``typer.BadParameter``.
|
|
86
|
+
|
|
87
|
+
Returns ``None`` when ``value`` is ``None``. The agent/caller is expected to
|
|
88
|
+
supply a valid ISO string; MintMory does no natural-language date parsing.
|
|
89
|
+
"""
|
|
90
|
+
if value is None:
|
|
91
|
+
return None
|
|
92
|
+
try:
|
|
93
|
+
return datetime.fromisoformat(value)
|
|
94
|
+
except ValueError as exc:
|
|
95
|
+
raise typer.BadParameter(
|
|
96
|
+
f"invalid ISO date/datetime {value!r}; expected e.g. '2026-07-01' or '2026-07-01T09:00'"
|
|
97
|
+
) from exc
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@app.command()
|
|
101
|
+
def add(
|
|
102
|
+
content: str = typer.Argument(..., help="Memory content"),
|
|
103
|
+
category: str = typer.Option("fact", help="Memory category"),
|
|
104
|
+
source: str = typer.Option("user", help="Memory source"),
|
|
105
|
+
verified: bool = typer.Option(False, "--verified/--unverified"),
|
|
106
|
+
) -> None:
|
|
107
|
+
"""Add a new memory."""
|
|
108
|
+
cat = _parse_category(category)
|
|
109
|
+
src = _parse_source(source)
|
|
110
|
+
store = _get_store()
|
|
111
|
+
record = store.add_memory(
|
|
112
|
+
content=content,
|
|
113
|
+
category=cat,
|
|
114
|
+
source=src,
|
|
115
|
+
verified=verified,
|
|
116
|
+
)
|
|
117
|
+
console.print(f"[green]Added memory[/green] [bold]{record.id}[/bold]")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@app.command()
|
|
121
|
+
def search(
|
|
122
|
+
query: str = typer.Argument(..., help="Search query"),
|
|
123
|
+
limit: int = typer.Option(10, help="Max results"),
|
|
124
|
+
around: str | None = typer.Option(None, help="Link type for graph traversal"),
|
|
125
|
+
category: str | None = typer.Option(None, help="Filter by category"),
|
|
126
|
+
) -> None:
|
|
127
|
+
"""Search memories. Use --around to traverse the concept graph."""
|
|
128
|
+
from mintmory.core.types import (
|
|
129
|
+
MemoryFilter,
|
|
130
|
+
SearchAroundSpec,
|
|
131
|
+
SearchRequest,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
mem_filter: MemoryFilter | None = None
|
|
135
|
+
if category is not None:
|
|
136
|
+
mem_filter = MemoryFilter(category=_parse_category(category))
|
|
137
|
+
|
|
138
|
+
search_around: SearchAroundSpec | None = None
|
|
139
|
+
if around is not None:
|
|
140
|
+
search_around = SearchAroundSpec(link_types=[_parse_link_type(around)])
|
|
141
|
+
|
|
142
|
+
store = _get_store()
|
|
143
|
+
response = store.search(
|
|
144
|
+
SearchRequest(
|
|
145
|
+
query=query,
|
|
146
|
+
limit=limit,
|
|
147
|
+
filter=mem_filter,
|
|
148
|
+
search_around=search_around,
|
|
149
|
+
)
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
table = Table(title=f"Search results for {query!r}")
|
|
153
|
+
table.add_column("id", style="cyan", no_wrap=True)
|
|
154
|
+
table.add_column("category", style="magenta")
|
|
155
|
+
table.add_column("content")
|
|
156
|
+
for mem in response.memories:
|
|
157
|
+
table.add_row(mem.id, mem.category.value, mem.content)
|
|
158
|
+
console.print(table)
|
|
159
|
+
console.print(f"[dim]{response.total_found} result(s)[/dim]")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@app.command()
|
|
163
|
+
def dream(
|
|
164
|
+
full: bool = typer.Option(False, "--full/--light", help="Full vs light dreaming"),
|
|
165
|
+
) -> None:
|
|
166
|
+
"""Run the dreaming consolidation process (uses the configured LLM tier for L3)."""
|
|
167
|
+
from mintmory.core.config import load_settings
|
|
168
|
+
from mintmory.core.llm import build_dreaming_engine
|
|
169
|
+
|
|
170
|
+
settings = load_settings()
|
|
171
|
+
store = _get_store()
|
|
172
|
+
engine = build_dreaming_engine(
|
|
173
|
+
store, settings.llm, link_settings=settings.link, summary_settings=settings.summary
|
|
174
|
+
)
|
|
175
|
+
report = engine.run_full() if full else engine.run_light()
|
|
176
|
+
|
|
177
|
+
table = Table(title=f"Dream report ({report.intensity.value})")
|
|
178
|
+
table.add_column("metric", style="cyan")
|
|
179
|
+
table.add_column("value", justify="right", style="green")
|
|
180
|
+
table.add_row("duration_ms", f"{report.duration_ms:.1f}")
|
|
181
|
+
table.add_row("new_links", str(report.new_links))
|
|
182
|
+
table.add_row("new_summaries", str(report.new_summaries))
|
|
183
|
+
table.add_row("contradictions_resolved", str(report.contradictions_resolved))
|
|
184
|
+
table.add_row("memories_archived", str(report.memories_archived))
|
|
185
|
+
table.add_row("memories_rehabilitated", str(report.memories_rehabilitated))
|
|
186
|
+
console.print(table)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@app.command()
|
|
190
|
+
def stats() -> None:
|
|
191
|
+
"""Show memory health statistics."""
|
|
192
|
+
store = _get_store()
|
|
193
|
+
s = store.get_stats()
|
|
194
|
+
|
|
195
|
+
table = Table(title="Memory statistics")
|
|
196
|
+
table.add_column("metric", style="cyan")
|
|
197
|
+
table.add_column("value", justify="right", style="green")
|
|
198
|
+
table.add_row("total_memories", str(s.total_memories))
|
|
199
|
+
table.add_row("active", str(s.active))
|
|
200
|
+
table.add_row("stale", str(s.stale))
|
|
201
|
+
table.add_row("archived", str(s.archived))
|
|
202
|
+
table.add_row("concept_links", str(s.concept_links))
|
|
203
|
+
table.add_row("memory_summaries", str(s.memory_summaries))
|
|
204
|
+
table.add_row("avg_usefulness_score", f"{s.avg_usefulness_score:.2f}")
|
|
205
|
+
table.add_row("avg_staleness_score", f"{s.avg_staleness_score:.2f}")
|
|
206
|
+
console.print(table)
|
|
207
|
+
if s.top_concepts:
|
|
208
|
+
concepts = ", ".join(f"{name} ({count})" for name, count in s.top_concepts)
|
|
209
|
+
console.print(f"[dim]top concepts: {concepts}[/dim]")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@app.command()
|
|
213
|
+
def serve(
|
|
214
|
+
port: int = typer.Option(8080, help="HTTP port"),
|
|
215
|
+
host: str = typer.Option("0.0.0.0", help="Bind host"),
|
|
216
|
+
) -> None:
|
|
217
|
+
"""Start the HTTP API server."""
|
|
218
|
+
import uvicorn
|
|
219
|
+
|
|
220
|
+
uvicorn.run("mintmory.api.app:app", host=host, port=port, reload=True)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _chunk_text(text: str, max_chars: int) -> list[str]:
|
|
224
|
+
"""Split text into <= max_chars chunks, preferring paragraph boundaries.
|
|
225
|
+
|
|
226
|
+
Hard-capped at MemoryRecord's 10_000-char limit; an oversized single paragraph
|
|
227
|
+
is split on character boundaries as a last resort.
|
|
228
|
+
"""
|
|
229
|
+
text = text.strip()
|
|
230
|
+
if not text:
|
|
231
|
+
return []
|
|
232
|
+
hard = max(1, min(max_chars, 10_000))
|
|
233
|
+
chunks: list[str] = []
|
|
234
|
+
buf = ""
|
|
235
|
+
for para in (p.strip() for p in text.split("\n\n") if p.strip()):
|
|
236
|
+
if len(para) > hard:
|
|
237
|
+
if buf:
|
|
238
|
+
chunks.append(buf)
|
|
239
|
+
buf = ""
|
|
240
|
+
chunks.extend(para[i : i + hard] for i in range(0, len(para), hard))
|
|
241
|
+
elif buf and len(buf) + 2 + len(para) > hard:
|
|
242
|
+
chunks.append(buf)
|
|
243
|
+
buf = para
|
|
244
|
+
else:
|
|
245
|
+
buf = f"{buf}\n\n{para}" if buf else para
|
|
246
|
+
if buf:
|
|
247
|
+
chunks.append(buf)
|
|
248
|
+
return chunks
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _resolve_files(paths: list[str], globs: list[str]) -> list[Path]:
|
|
252
|
+
"""Expand files + directories (recursive glob) into a deduplicated file list."""
|
|
253
|
+
out: list[Path] = []
|
|
254
|
+
seen: set[Path] = set()
|
|
255
|
+
for raw in paths:
|
|
256
|
+
p = Path(raw).expanduser()
|
|
257
|
+
if p.is_dir():
|
|
258
|
+
candidates = sorted({f for g in globs for f in p.rglob(g) if f.is_file()})
|
|
259
|
+
elif p.is_file():
|
|
260
|
+
candidates = [p]
|
|
261
|
+
else:
|
|
262
|
+
raise typer.BadParameter(f"path not found: {raw}")
|
|
263
|
+
for f in candidates:
|
|
264
|
+
rp = f.resolve()
|
|
265
|
+
if rp not in seen:
|
|
266
|
+
seen.add(rp)
|
|
267
|
+
out.append(f)
|
|
268
|
+
return out
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@app.command()
|
|
272
|
+
def ingest(
|
|
273
|
+
paths: list[str] = typer.Argument(..., help="Files or directories to ingest"),
|
|
274
|
+
category: str = typer.Option("fact", help="Category for ingested memories"),
|
|
275
|
+
source: str = typer.Option("document", help="Memory source"),
|
|
276
|
+
glob: str = typer.Option(
|
|
277
|
+
"*.md,*.txt,*.pdf,*.docx,*.pptx,*.xlsx,*.csv,*.html",
|
|
278
|
+
help="Comma-separated globs for directory paths",
|
|
279
|
+
),
|
|
280
|
+
chunk_chars: int = typer.Option(4000, help="Approx chars per chunk for large files"),
|
|
281
|
+
skip_duplicates: bool = typer.Option(
|
|
282
|
+
True, "--skip-duplicates/--allow-duplicates", help="Skip chunks whose exact content exists"
|
|
283
|
+
),
|
|
284
|
+
convert: bool = typer.Option(
|
|
285
|
+
True,
|
|
286
|
+
"--convert/--no-convert",
|
|
287
|
+
help=(
|
|
288
|
+
"Auto-convert PDF/DOCX/XLSX/PPTX/etc to markdown via markitdown "
|
|
289
|
+
"(needs the 'docs' extra)"
|
|
290
|
+
),
|
|
291
|
+
),
|
|
292
|
+
dream: bool = typer.Option(False, "--dream/--no-dream", help="Run a light dream after ingest"),
|
|
293
|
+
) -> None:
|
|
294
|
+
"""Bulk-ingest files or directories as memories (chunked + entity-extracted).
|
|
295
|
+
|
|
296
|
+
Idempotent by default: re-running skips chunks whose exact content already
|
|
297
|
+
exists (--allow-duplicates to force). Use this instead of hand-rolling an
|
|
298
|
+
add-per-file script.
|
|
299
|
+
"""
|
|
300
|
+
from mintmory.core.config import load_settings
|
|
301
|
+
from mintmory.core.conversion import ConversionError, extract_markdown
|
|
302
|
+
|
|
303
|
+
cat = _parse_category(category)
|
|
304
|
+
src = _parse_source(source)
|
|
305
|
+
globs = [g.strip() for g in glob.split(",") if g.strip()]
|
|
306
|
+
files = _resolve_files(paths, globs)
|
|
307
|
+
if not files:
|
|
308
|
+
console.print("[yellow]No matching files to ingest.[/yellow]")
|
|
309
|
+
raise typer.Exit(code=1)
|
|
310
|
+
|
|
311
|
+
settings = load_settings()
|
|
312
|
+
conv = settings.convert
|
|
313
|
+
store = _get_store()
|
|
314
|
+
added = skipped = converted = failed = 0
|
|
315
|
+
for f in files:
|
|
316
|
+
try:
|
|
317
|
+
result = extract_markdown(
|
|
318
|
+
f,
|
|
319
|
+
convert=convert and conv.enabled,
|
|
320
|
+
max_bytes=conv.max_bytes,
|
|
321
|
+
extra_text_suffixes=conv.extra_text_suffixes,
|
|
322
|
+
enable_plugins=conv.enable_plugins,
|
|
323
|
+
timeout_s=conv.timeout_s,
|
|
324
|
+
max_output_bytes=conv.max_output_bytes,
|
|
325
|
+
)
|
|
326
|
+
except ConversionError as exc:
|
|
327
|
+
console.print(f"[red]skip[/red] {f}: {exc}")
|
|
328
|
+
failed += 1
|
|
329
|
+
continue
|
|
330
|
+
chunks = _chunk_text(result.text, chunk_chars)
|
|
331
|
+
if not chunks:
|
|
332
|
+
console.print(f"[yellow]+0[/yellow] {f}: produced no extractable text")
|
|
333
|
+
failed += 1
|
|
334
|
+
continue
|
|
335
|
+
if result.method == "markitdown":
|
|
336
|
+
converted += 1
|
|
337
|
+
n_added = 0
|
|
338
|
+
for i, chunk in enumerate(chunks):
|
|
339
|
+
if skip_duplicates and store.memory_exists_with_content(chunk):
|
|
340
|
+
skipped += 1
|
|
341
|
+
continue
|
|
342
|
+
store.add_memory(
|
|
343
|
+
content=chunk,
|
|
344
|
+
category=cat,
|
|
345
|
+
source=src,
|
|
346
|
+
metadata={
|
|
347
|
+
"source_file": str(f),
|
|
348
|
+
"chunk": i,
|
|
349
|
+
"chunks": len(chunks),
|
|
350
|
+
"converter": result.method,
|
|
351
|
+
},
|
|
352
|
+
)
|
|
353
|
+
added += 1
|
|
354
|
+
n_added += 1
|
|
355
|
+
dup_note = " [dim](dups skipped)[/dim]" if n_added < len(chunks) else ""
|
|
356
|
+
console.print(f"[green]+{n_added}[/green] {f}{dup_note}")
|
|
357
|
+
console.print(
|
|
358
|
+
f"[bold green]Ingested {added} memory-chunk(s)[/bold green] from "
|
|
359
|
+
f"{len(files)} file(s) as [magenta]{cat.value}[/magenta]"
|
|
360
|
+
+ (f"; skipped {skipped} duplicate(s)" if skipped else "")
|
|
361
|
+
+ (f"; skipped {failed} file(s)" if failed else "")
|
|
362
|
+
+ (f" ({converted} via markitdown)" if converted else "")
|
|
363
|
+
+ "."
|
|
364
|
+
)
|
|
365
|
+
if dream:
|
|
366
|
+
from mintmory.core.llm import build_dreaming_engine
|
|
367
|
+
|
|
368
|
+
report = build_dreaming_engine(
|
|
369
|
+
store, settings.llm, link_settings=settings.link, summary_settings=settings.summary
|
|
370
|
+
).run_light()
|
|
371
|
+
console.print(
|
|
372
|
+
f"[dim]dream: +{report.new_links} links, +{report.new_summaries} summaries[/dim]"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
@app.command()
|
|
377
|
+
def doctor() -> None:
|
|
378
|
+
"""One-shot health check of the MintMory deployment (DB, embedder, LLM tier)."""
|
|
379
|
+
from mintmory.core.config import load_settings
|
|
380
|
+
|
|
381
|
+
settings = load_settings()
|
|
382
|
+
table = Table(title="MintMory doctor")
|
|
383
|
+
table.add_column("check", style="cyan")
|
|
384
|
+
table.add_column("status")
|
|
385
|
+
healthy = True
|
|
386
|
+
|
|
387
|
+
db_path = os.environ.get("MINTMORY_DB", str(Path.home() / ".mintmory" / "memories.db"))
|
|
388
|
+
try:
|
|
389
|
+
store = _get_store()
|
|
390
|
+
s = store.get_stats()
|
|
391
|
+
except Exception as exc: # noqa: BLE001 — doctor must report, not raise
|
|
392
|
+
table.add_row("database", f"[red]FAIL[/red] {db_path}: {exc}")
|
|
393
|
+
console.print(table)
|
|
394
|
+
raise typer.Exit(code=1) from exc
|
|
395
|
+
table.add_row(
|
|
396
|
+
"database",
|
|
397
|
+
f"[green]ok[/green] {db_path} "
|
|
398
|
+
f"({s.total_memories} mem, {s.concept_links} links, {s.memory_summaries} summaries)",
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
emb = store.embedder
|
|
402
|
+
table.add_row(
|
|
403
|
+
"embedder",
|
|
404
|
+
f"[green]{settings.embed.provider.value}[/green] (dim {emb.dimensions})"
|
|
405
|
+
if emb is not None
|
|
406
|
+
else "[yellow]none (FTS-only)[/yellow]",
|
|
407
|
+
)
|
|
408
|
+
table.add_row(
|
|
409
|
+
"vector search",
|
|
410
|
+
"[green]available[/green] (sqlite-vec)"
|
|
411
|
+
if store.vector_search_available()
|
|
412
|
+
else "[yellow]FTS-only[/yellow] (sqlite-vec not loaded)",
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
if settings.llm.enabled:
|
|
416
|
+
from mintmory.core.llm import LLMClient
|
|
417
|
+
|
|
418
|
+
reachable = LLMClient(settings.llm).ping()
|
|
419
|
+
healthy = healthy and reachable
|
|
420
|
+
verdict = "[green]reachable[/green]" if reachable else "[red]UNREACHABLE[/red]"
|
|
421
|
+
table.add_row(
|
|
422
|
+
"llm tier",
|
|
423
|
+
f"{settings.llm.provider.value} {settings.llm.model} @ "
|
|
424
|
+
f"{settings.llm.base_url} — {verdict}",
|
|
425
|
+
)
|
|
426
|
+
else:
|
|
427
|
+
table.add_row("llm tier", "[yellow]disabled[/yellow] (provider=none; L3 + resolution off)")
|
|
428
|
+
|
|
429
|
+
table.add_row(
|
|
430
|
+
"linking",
|
|
431
|
+
f"min_shared={settings.link.min_shared_entities} "
|
|
432
|
+
f"max_per_node={settings.link.max_per_node} stoplist={len(settings.link.stoplist)}",
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
from mintmory.core.conversion import CONVERTIBLE_SUFFIXES, markitdown_available
|
|
436
|
+
|
|
437
|
+
if markitdown_available():
|
|
438
|
+
table.add_row(
|
|
439
|
+
"conversion",
|
|
440
|
+
f"[green]markitdown available[/green] "
|
|
441
|
+
f"({len(CONVERTIBLE_SUFFIXES)} convertible formats)",
|
|
442
|
+
)
|
|
443
|
+
else:
|
|
444
|
+
table.add_row(
|
|
445
|
+
"conversion",
|
|
446
|
+
"[yellow]not installed — `uv sync --extra docs` for PDF/DOCX/XLSX[/yellow]",
|
|
447
|
+
)
|
|
448
|
+
console.print(table)
|
|
449
|
+
if not healthy:
|
|
450
|
+
raise typer.Exit(code=2)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
@app.command()
|
|
454
|
+
def index_tree(
|
|
455
|
+
roots: list[str] = typer.Argument(..., help="Root folder(s) to index recursively"),
|
|
456
|
+
collection: str = typer.Option(
|
|
457
|
+
"default", help="Collection tag stamped on every memory + manifest row"
|
|
458
|
+
),
|
|
459
|
+
include: str = typer.Option("*", help="Comma-separated include globs (filename or rel-path)"),
|
|
460
|
+
exclude: str = typer.Option("", help="Comma-separated exclude globs, e.g. 'Personal/**,*.tmp'"),
|
|
461
|
+
db: str | None = typer.Option(None, help="Target DB path (overrides MINTMORY_DB)"),
|
|
462
|
+
text_content: bool = typer.Option(
|
|
463
|
+
True,
|
|
464
|
+
"--text-content/--no-text-content",
|
|
465
|
+
help="Full-text small plain-text files (.txt/.md/.log/.rst) inline — cheap, no budget",
|
|
466
|
+
),
|
|
467
|
+
text_max_kb: int = typer.Option(
|
|
468
|
+
2048, help="Max size (KB) for inline text-content; larger text files stay metadata-only"
|
|
469
|
+
),
|
|
470
|
+
content: bool = typer.Option(
|
|
471
|
+
False,
|
|
472
|
+
"--content/--no-content",
|
|
473
|
+
help="Also download+markitdown-convert selected binary docs (pdf/docx/...) to full text",
|
|
474
|
+
),
|
|
475
|
+
content_types: str = typer.Option(
|
|
476
|
+
"pdf,docx,doc,xlsx,pptx,html,csv", help="Binary suffixes eligible for full-text extraction"
|
|
477
|
+
),
|
|
478
|
+
max_download_mb: float = typer.Option(
|
|
479
|
+
200.0, help="Download budget for the binary content pass (0 = unlimited)"
|
|
480
|
+
),
|
|
481
|
+
chunk_chars: int = typer.Option(4000, help="Approx chars per content chunk"),
|
|
482
|
+
prune: bool = typer.Option(
|
|
483
|
+
False, "--prune/--no-prune", help="Archive memories for files that disappeared"
|
|
484
|
+
),
|
|
485
|
+
force: bool = typer.Option(
|
|
486
|
+
False, "--force", help="Re-index every file even if unchanged (b: force everything)"
|
|
487
|
+
),
|
|
488
|
+
dream: bool = typer.Option(
|
|
489
|
+
False, "--dream/--no-dream", help="Run a light dream after indexing"
|
|
490
|
+
),
|
|
491
|
+
limit: int = typer.Option(0, help="Stop after N files (0 = all; for smoke tests)"),
|
|
492
|
+
vision: bool = typer.Option(
|
|
493
|
+
False,
|
|
494
|
+
"--vision/--no-vision",
|
|
495
|
+
help=(
|
|
496
|
+
"Describe image files: extract SVG text inline; queue raster images as "
|
|
497
|
+
"agent jobs (provider=agent) or run the configured vision provider "
|
|
498
|
+
"(llm/ocr, future). Records index_mode='vision'."
|
|
499
|
+
),
|
|
500
|
+
),
|
|
501
|
+
cochange: bool | None = typer.Option(
|
|
502
|
+
None,
|
|
503
|
+
"--cochange/--no-cochange",
|
|
504
|
+
help=(
|
|
505
|
+
"Cluster changed files into co-change sets (HDBSCAN over time+path+content). "
|
|
506
|
+
"Requires the 'cochange' extra (scikit-learn). "
|
|
507
|
+
"Defaults to settings.doc.cochange_enabled."
|
|
508
|
+
),
|
|
509
|
+
),
|
|
510
|
+
) -> None:
|
|
511
|
+
"""Recurrently index a directory tree.
|
|
512
|
+
|
|
513
|
+
Writes one metadata + folder-context memory per file (stat-only for the walk),
|
|
514
|
+
full-texts small plain-text files inline by default (cheap — .txt/.md/.log/.rst),
|
|
515
|
+
and optionally downloads+converts heavy binary docs with ``--content`` (bounded
|
|
516
|
+
by ``--max-download-mb``). Idempotent via a per-path manifest: re-runs skip
|
|
517
|
+
unchanged files and replace changed ones. Designed for cloud-backed
|
|
518
|
+
(online-only) libraries.
|
|
519
|
+
|
|
520
|
+
Pass ``--vision`` to also handle image files: SVG text is extracted inline
|
|
521
|
+
(pure-Python, no model); raster images (png/jpg/etc) are queued as agent jobs
|
|
522
|
+
for provider=agent, or described inline for future llm/ocr providers. Records
|
|
523
|
+
``index_mode='vision'`` in the manifest. Without ``--vision`` this command is
|
|
524
|
+
byte-for-byte unchanged.
|
|
525
|
+
"""
|
|
526
|
+
import hashlib
|
|
527
|
+
import json
|
|
528
|
+
|
|
529
|
+
from mintmory.core.cochange import ChangedDoc, CoChangeUnavailable, cluster_changesets
|
|
530
|
+
from mintmory.core.config import load_settings
|
|
531
|
+
from mintmory.core.conversion import TEXT_SUFFIXES, ConversionError, extract_markdown
|
|
532
|
+
from mintmory.core.tree_index import (
|
|
533
|
+
ARTIFACT_SUFFIXES,
|
|
534
|
+
IMAGE_SUFFIXES,
|
|
535
|
+
human_size,
|
|
536
|
+
iter_dir_groups,
|
|
537
|
+
render_file_record,
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
if db:
|
|
541
|
+
os.environ["MINTMORY_DB"] = db
|
|
542
|
+
inc = [g.strip() for g in include.split(",") if g.strip()] or ["*"]
|
|
543
|
+
exc = [g.strip() for g in exclude.split(",") if g.strip()]
|
|
544
|
+
ctypes = {f".{t.strip().lstrip('.').lower()}" for t in content_types.split(",") if t.strip()}
|
|
545
|
+
budget = int(max_download_mb * 1024 * 1024) if max_download_mb > 0 else None
|
|
546
|
+
text_max_bytes = text_max_kb * 1024 if text_max_kb > 0 else None
|
|
547
|
+
|
|
548
|
+
settings = load_settings()
|
|
549
|
+
conv = settings.convert
|
|
550
|
+
store = _get_store()
|
|
551
|
+
run_cochange = cochange if cochange is not None else settings.doc.cochange_enabled
|
|
552
|
+
doc_s = settings.doc
|
|
553
|
+
|
|
554
|
+
# MM-34 E: detect cold vs incremental BEFORE the walk
|
|
555
|
+
was_cold = len(store.manifest_paths(collection)) == 0
|
|
556
|
+
run_kind = "cold_full_index" if was_cold else "incremental"
|
|
557
|
+
|
|
558
|
+
# MM-34 B: build the co-change suffix exclusion set BEFORE the walk
|
|
559
|
+
_cochange_exclude: frozenset[str] = frozenset()
|
|
560
|
+
if doc_s.cochange_exclude_images:
|
|
561
|
+
_cochange_exclude |= IMAGE_SUFFIXES
|
|
562
|
+
if doc_s.cochange_exclude_artifacts:
|
|
563
|
+
_cochange_exclude |= ARTIFACT_SUFFIXES
|
|
564
|
+
_cochange_exclude |= doc_s.cochange_exclude_suffixes
|
|
565
|
+
|
|
566
|
+
# Vision: lazily import + validate the provider once before the walk, so
|
|
567
|
+
# a misconfigured llm/ocr provider fails fast with a clear message rather
|
|
568
|
+
# than a stack trace mid-walk. ``vision_mod`` is only ever referenced
|
|
569
|
+
# inside the ``if vision:`` / ``elif want_vision:`` branches, so mypy can
|
|
570
|
+
# see the import is guarded. We import it into a local name to satisfy
|
|
571
|
+
# type checkers that the name is always defined when used.
|
|
572
|
+
if vision:
|
|
573
|
+
from mintmory.core import vision as vision_mod
|
|
574
|
+
|
|
575
|
+
try:
|
|
576
|
+
captioner = vision_mod.captioner_from_settings(settings.vision)
|
|
577
|
+
except NotImplementedError as _nie:
|
|
578
|
+
console.print(f"[red]error[/red]: {_nie}")
|
|
579
|
+
raise typer.Exit(code=1) from _nie
|
|
580
|
+
else:
|
|
581
|
+
vision_mod = None # type: ignore[assignment] # never accessed when vision=False
|
|
582
|
+
captioner = None
|
|
583
|
+
|
|
584
|
+
scanned = added = updated = unchanged = converted = failed = pruned = 0
|
|
585
|
+
downloaded = 0
|
|
586
|
+
budget_hit = False
|
|
587
|
+
# Vision-specific counters (only relevant when --vision)
|
|
588
|
+
svg_described = 0
|
|
589
|
+
images_queued = 0
|
|
590
|
+
images_described = 0
|
|
591
|
+
vision_skipped = 0
|
|
592
|
+
seen: set[str] = set()
|
|
593
|
+
# Co-change: collect ChangedDoc for every added/updated file
|
|
594
|
+
changed_docs: list[ChangedDoc] = []
|
|
595
|
+
# Size cap for raster images (used when captioner is not None).
|
|
596
|
+
max_image_bytes: int | None = settings.vision.max_image_bytes if vision else None
|
|
597
|
+
|
|
598
|
+
for root in roots:
|
|
599
|
+
root_path = Path(root).expanduser()
|
|
600
|
+
if not root_path.is_dir():
|
|
601
|
+
raise typer.BadParameter(f"not a directory: {root}")
|
|
602
|
+
root_label = root_path.name
|
|
603
|
+
for group in iter_dir_groups(root_path, include=inc, exclude=exc):
|
|
604
|
+
for entry in group.entries:
|
|
605
|
+
if limit and scanned >= limit:
|
|
606
|
+
break
|
|
607
|
+
scanned += 1
|
|
608
|
+
path_str = str(entry.path)
|
|
609
|
+
seen.add(path_str)
|
|
610
|
+
# Plain-text files are cheap to read -> full-text them inline by
|
|
611
|
+
# default (no markitdown, no download budget). Heavy binary docs
|
|
612
|
+
# need --content and consume the download budget.
|
|
613
|
+
text_eligible = (
|
|
614
|
+
text_content
|
|
615
|
+
and entry.suffix in TEXT_SUFFIXES
|
|
616
|
+
and (text_max_bytes is None or entry.size <= text_max_bytes)
|
|
617
|
+
)
|
|
618
|
+
want_binary = content and entry.suffix in ctypes and not budget_hit
|
|
619
|
+
do_content = text_eligible or want_binary
|
|
620
|
+
|
|
621
|
+
# Vision: third content mode — separate from text/binary.
|
|
622
|
+
# Proprietary formats (xd/vsdx/dwg/psd/eps) are skip-and-flagged.
|
|
623
|
+
# ``vision_mod`` is not None when vision=True (guard above).
|
|
624
|
+
want_vision = (
|
|
625
|
+
vision
|
|
626
|
+
and vision_mod is not None
|
|
627
|
+
and entry.suffix in vision_mod.IMAGE_SUFFIXES
|
|
628
|
+
and entry.suffix not in vision_mod.PROPRIETARY_IMAGE_SUFFIXES
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
if (
|
|
632
|
+
vision
|
|
633
|
+
and vision_mod is not None
|
|
634
|
+
and entry.suffix in vision_mod.PROPRIETARY_IMAGE_SUFFIXES
|
|
635
|
+
):
|
|
636
|
+
vision_skipped += 1
|
|
637
|
+
|
|
638
|
+
# desired_mode: 'content' wins over 'vision' (an image that is
|
|
639
|
+
# also full-text converted keeps 'content'); 'vision' beats
|
|
640
|
+
# 'metadata' for image suffixes handled by --vision.
|
|
641
|
+
if do_content:
|
|
642
|
+
desired_mode = "content"
|
|
643
|
+
elif want_vision:
|
|
644
|
+
desired_mode = "vision"
|
|
645
|
+
else:
|
|
646
|
+
desired_mode = "metadata"
|
|
647
|
+
|
|
648
|
+
existing = store.manifest_get(path_str)
|
|
649
|
+
if existing is not None and not force:
|
|
650
|
+
same = (
|
|
651
|
+
existing["size"] == entry.size
|
|
652
|
+
and abs(existing["mtime"] - entry.mtime) < 1e-6
|
|
653
|
+
)
|
|
654
|
+
# 'content' and 'vision' are both richer than 'metadata';
|
|
655
|
+
# 'content' > 'vision' (if already content, don't downgrade).
|
|
656
|
+
existing_mode = existing["index_mode"]
|
|
657
|
+
covered = existing_mode in ("content", "vision") or desired_mode == "metadata"
|
|
658
|
+
if same and covered:
|
|
659
|
+
unchanged += 1
|
|
660
|
+
continue
|
|
661
|
+
|
|
662
|
+
new_ids: list[str] = []
|
|
663
|
+
file_mtime_dt = datetime.fromtimestamp(entry.mtime, tz=UTC).replace(tzinfo=None)
|
|
664
|
+
file_record_text = render_file_record(entry, group, root_label)
|
|
665
|
+
file_record = store.add_memory(
|
|
666
|
+
content=file_record_text,
|
|
667
|
+
category="context",
|
|
668
|
+
source="document",
|
|
669
|
+
valid_from=file_mtime_dt,
|
|
670
|
+
metadata={
|
|
671
|
+
"collection": collection,
|
|
672
|
+
"path": path_str,
|
|
673
|
+
"rel": entry.rel,
|
|
674
|
+
"ext": entry.suffix,
|
|
675
|
+
"size": entry.size,
|
|
676
|
+
"mtime": entry.mtime,
|
|
677
|
+
"online_only": entry.online_only,
|
|
678
|
+
"folder": str(Path(entry.rel).parent),
|
|
679
|
+
"index_mode": "metadata",
|
|
680
|
+
"modified_source": "fs_mtime",
|
|
681
|
+
"record_role": "file", # MM-34 D
|
|
682
|
+
},
|
|
683
|
+
)
|
|
684
|
+
new_ids.append(file_record.id)
|
|
685
|
+
mode = "metadata"
|
|
686
|
+
content_hash: str | None = None
|
|
687
|
+
|
|
688
|
+
if do_content:
|
|
689
|
+
try:
|
|
690
|
+
result = extract_markdown(
|
|
691
|
+
entry.path,
|
|
692
|
+
convert=True,
|
|
693
|
+
max_bytes=conv.max_bytes,
|
|
694
|
+
timeout_s=conv.timeout_s,
|
|
695
|
+
max_output_bytes=conv.max_output_bytes,
|
|
696
|
+
enable_plugins=conv.enable_plugins,
|
|
697
|
+
)
|
|
698
|
+
if want_binary:
|
|
699
|
+
downloaded += entry.size # only heavy docs count toward the budget
|
|
700
|
+
for i, chunk in enumerate(_chunk_text(result.text, chunk_chars)):
|
|
701
|
+
crec = store.add_memory(
|
|
702
|
+
content=chunk,
|
|
703
|
+
category="fact",
|
|
704
|
+
source="document",
|
|
705
|
+
valid_from=file_mtime_dt, # MM-34 D
|
|
706
|
+
metadata={
|
|
707
|
+
"collection": collection,
|
|
708
|
+
"source_file": path_str,
|
|
709
|
+
"rel": entry.rel,
|
|
710
|
+
"chunk": i,
|
|
711
|
+
"converter": result.method,
|
|
712
|
+
"modified_source": "fs_mtime", # MM-34 D
|
|
713
|
+
"record_role": "chunk", # MM-34 D
|
|
714
|
+
},
|
|
715
|
+
)
|
|
716
|
+
new_ids.append(crec.id)
|
|
717
|
+
if len(new_ids) > 1:
|
|
718
|
+
mode = "content"
|
|
719
|
+
content_hash = hashlib.blake2b(
|
|
720
|
+
result.text.encode("utf-8"), digest_size=16
|
|
721
|
+
).hexdigest()
|
|
722
|
+
converted += 1
|
|
723
|
+
if want_binary and budget is not None and downloaded >= budget:
|
|
724
|
+
budget_hit = True
|
|
725
|
+
except ConversionError as exc:
|
|
726
|
+
console.print(f"[yellow]content skip[/yellow] {entry.name}: {exc}")
|
|
727
|
+
failed += 1
|
|
728
|
+
elif want_vision:
|
|
729
|
+
# Vision branch: SVG → inline extraction; raster → queue for agent.
|
|
730
|
+
# ``want_vision`` is True only when vision_mod is not None (see above).
|
|
731
|
+
assert vision_mod is not None # noqa: S101
|
|
732
|
+
mode = "vision"
|
|
733
|
+
if entry.suffix in vision_mod.SVG_SUFFIXES:
|
|
734
|
+
# SVG: read bytes (counts against download budget for online-only),
|
|
735
|
+
# extract embedded text via pure stdlib, store description inline.
|
|
736
|
+
try:
|
|
737
|
+
svg_bytes: bytes = b""
|
|
738
|
+
if entry.online_only:
|
|
739
|
+
if budget is not None and downloaded >= budget:
|
|
740
|
+
# budget exhausted; skip this SVG's byte read
|
|
741
|
+
mode = "metadata"
|
|
742
|
+
console.print(
|
|
743
|
+
f"[yellow]vision skip[/yellow] {entry.name}: "
|
|
744
|
+
"download budget exhausted"
|
|
745
|
+
)
|
|
746
|
+
failed += 1
|
|
747
|
+
else:
|
|
748
|
+
svg_bytes = entry.path.read_bytes()
|
|
749
|
+
downloaded += len(svg_bytes)
|
|
750
|
+
if budget is not None and downloaded >= budget:
|
|
751
|
+
budget_hit = True
|
|
752
|
+
else:
|
|
753
|
+
svg_bytes = entry.path.read_bytes()
|
|
754
|
+
if mode == "vision":
|
|
755
|
+
svg_text = vision_mod.extract_svg_text(svg_bytes)
|
|
756
|
+
if svg_text:
|
|
757
|
+
desc = vision_mod.image_caption_put(
|
|
758
|
+
store, file_record.id, svg_text, settings=settings.vision
|
|
759
|
+
)
|
|
760
|
+
new_ids.append(desc.record.id)
|
|
761
|
+
content_hash = hashlib.blake2b(
|
|
762
|
+
svg_text.encode("utf-8"), digest_size=16
|
|
763
|
+
).hexdigest()
|
|
764
|
+
svg_described += 1
|
|
765
|
+
# Empty SVG text: mode stays 'vision' (manifest marks it
|
|
766
|
+
# as vision-attempted so a re-run doesn't re-attempt unless
|
|
767
|
+
# the file changes). No description created.
|
|
768
|
+
except OSError as exc:
|
|
769
|
+
console.print(
|
|
770
|
+
f"[yellow]vision skip[/yellow] {entry.name}: read error {exc}"
|
|
771
|
+
)
|
|
772
|
+
mode = "metadata"
|
|
773
|
+
failed += 1
|
|
774
|
+
else:
|
|
775
|
+
# Raster image: queue for agent OR caption inline.
|
|
776
|
+
# captioner is None → provider=agent (UNCHANGED MM-18 behaviour):
|
|
777
|
+
# count the job and move on; the agent loop will describe it later.
|
|
778
|
+
# captioner is not None → provider=llm (or future):
|
|
779
|
+
# caption inline under the SAME budget + manifest.
|
|
780
|
+
assert vision_mod is not None # noqa: S101 — want_vision guard
|
|
781
|
+
if captioner is None:
|
|
782
|
+
images_queued += 1
|
|
783
|
+
# mode already 'vision'; no bytes read, no description yet.
|
|
784
|
+
else:
|
|
785
|
+
# Inline captioning under the shared download budget.
|
|
786
|
+
try:
|
|
787
|
+
if max_image_bytes is not None and entry.size > max_image_bytes:
|
|
788
|
+
# Oversized: skip this image.
|
|
789
|
+
vision_skipped += 1
|
|
790
|
+
elif (
|
|
791
|
+
entry.online_only
|
|
792
|
+
and budget is not None
|
|
793
|
+
and downloaded >= budget
|
|
794
|
+
):
|
|
795
|
+
# Budget exhausted for online-only images.
|
|
796
|
+
vision_skipped += 1
|
|
797
|
+
else:
|
|
798
|
+
raw = entry.path.read_bytes()
|
|
799
|
+
if entry.online_only:
|
|
800
|
+
downloaded += len(raw)
|
|
801
|
+
if budget is not None and downloaded >= budget:
|
|
802
|
+
budget_hit = True
|
|
803
|
+
desc = captioner.describe(
|
|
804
|
+
vision_mod.ImageInput(
|
|
805
|
+
file_id=file_record.id,
|
|
806
|
+
path=path_str,
|
|
807
|
+
mime=vision_mod._mime_for(entry.suffix),
|
|
808
|
+
data=raw,
|
|
809
|
+
)
|
|
810
|
+
)
|
|
811
|
+
put = vision_mod.image_caption_put(
|
|
812
|
+
store,
|
|
813
|
+
file_record.id,
|
|
814
|
+
desc.record.content,
|
|
815
|
+
settings=settings.vision,
|
|
816
|
+
)
|
|
817
|
+
new_ids.append(put.record.id)
|
|
818
|
+
content_hash = hashlib.blake2b(
|
|
819
|
+
desc.record.content.encode("utf-8"), digest_size=16
|
|
820
|
+
).hexdigest()
|
|
821
|
+
images_described += 1
|
|
822
|
+
except vision_mod.VisionError as exc:
|
|
823
|
+
console.print(f"[yellow]vision skip[/yellow] {entry.name}: {exc}")
|
|
824
|
+
vision_skipped += 1
|
|
825
|
+
except OSError as exc:
|
|
826
|
+
console.print(
|
|
827
|
+
f"[yellow]vision skip[/yellow] {entry.name}: read error {exc}"
|
|
828
|
+
)
|
|
829
|
+
vision_skipped += 1
|
|
830
|
+
# mode stays 'vision' in all cases (manifest records the attempt).
|
|
831
|
+
|
|
832
|
+
if existing is not None:
|
|
833
|
+
for old_id in json.loads(existing["memory_ids"]):
|
|
834
|
+
store.archive_memory(old_id)
|
|
835
|
+
updated += 1
|
|
836
|
+
else:
|
|
837
|
+
added += 1
|
|
838
|
+
|
|
839
|
+
# Co-change: collect this doc for cluster_changesets (MM-34 B: exclude suffixes)
|
|
840
|
+
if run_cochange:
|
|
841
|
+
_suffix = Path(path_str).suffix.lower()
|
|
842
|
+
if _suffix not in _cochange_exclude:
|
|
843
|
+
try:
|
|
844
|
+
_emb: NDArray[np.float32] | None = (
|
|
845
|
+
store.embedder.embed(file_record_text) if store.embedder else None
|
|
846
|
+
)
|
|
847
|
+
except Exception:
|
|
848
|
+
_emb = None
|
|
849
|
+
changed_docs.append(
|
|
850
|
+
ChangedDoc(
|
|
851
|
+
memory_id=file_record.id,
|
|
852
|
+
doc_id=path_str,
|
|
853
|
+
rel=entry.rel,
|
|
854
|
+
mtime=entry.mtime,
|
|
855
|
+
embedding=_emb,
|
|
856
|
+
)
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
store.manifest_upsert(
|
|
860
|
+
path_str,
|
|
861
|
+
collection,
|
|
862
|
+
size=entry.size,
|
|
863
|
+
mtime=entry.mtime,
|
|
864
|
+
online_only=entry.online_only,
|
|
865
|
+
index_mode=mode,
|
|
866
|
+
memory_ids=new_ids,
|
|
867
|
+
content_hash=content_hash,
|
|
868
|
+
)
|
|
869
|
+
if limit and scanned >= limit:
|
|
870
|
+
break
|
|
871
|
+
if limit and scanned >= limit:
|
|
872
|
+
break
|
|
873
|
+
|
|
874
|
+
if prune:
|
|
875
|
+
for gone in store.manifest_paths(collection) - seen:
|
|
876
|
+
row = store.manifest_get(gone)
|
|
877
|
+
if row is not None:
|
|
878
|
+
for old_id in json.loads(row["memory_ids"]):
|
|
879
|
+
store.archive_memory(old_id)
|
|
880
|
+
store.manifest_delete(gone)
|
|
881
|
+
pruned += 1
|
|
882
|
+
|
|
883
|
+
# Co-change post-walk pass
|
|
884
|
+
n_changesets = 0
|
|
885
|
+
n_co_changed_files = 0
|
|
886
|
+
cochange_result_dropped_oversized = 0
|
|
887
|
+
cochange_result_dropped_singletons = 0
|
|
888
|
+
cochange_result_truncated = 0
|
|
889
|
+
if run_cochange and len(changed_docs) >= 2:
|
|
890
|
+
from mintmory.core.cochange import apply_changesets
|
|
891
|
+
|
|
892
|
+
try:
|
|
893
|
+
cochange_result = cluster_changesets(changed_docs, doc_s, run_kind=run_kind)
|
|
894
|
+
from mintmory.core.storage import SqliteBackend as _SqliteBackend
|
|
895
|
+
|
|
896
|
+
n_changesets = apply_changesets(cast(_SqliteBackend, store), cochange_result.changesets)
|
|
897
|
+
n_co_changed_files = sum(len(cs.member_ids) for cs in cochange_result.changesets)
|
|
898
|
+
cochange_result_dropped_oversized = cochange_result.dropped_oversized
|
|
899
|
+
cochange_result_dropped_singletons = cochange_result.dropped_singletons
|
|
900
|
+
cochange_result_truncated = cochange_result.truncated
|
|
901
|
+
if cochange_result.truncated > 0:
|
|
902
|
+
_max_p = doc_s.max_cochange_partition_size
|
|
903
|
+
console.print(
|
|
904
|
+
f"[yellow]co-change: {cochange_result.truncated} file(s) dropped"
|
|
905
|
+
f" from oversized blocks"
|
|
906
|
+
f" (max_cochange_partition_size={_max_p})[/yellow]"
|
|
907
|
+
)
|
|
908
|
+
except CoChangeUnavailable:
|
|
909
|
+
console.print(
|
|
910
|
+
"[dim]co-change skipped: install the 'cochange' extra "
|
|
911
|
+
"(pip install mintmory[cochange])[/dim]"
|
|
912
|
+
)
|
|
913
|
+
|
|
914
|
+
table = Table(title=f"index-tree [{collection}]")
|
|
915
|
+
table.add_column("metric", style="cyan")
|
|
916
|
+
table.add_column("value", justify="right")
|
|
917
|
+
base_rows: list[tuple[str, str]] = [
|
|
918
|
+
("scanned", str(scanned)),
|
|
919
|
+
("new", str(added)),
|
|
920
|
+
("updated", str(updated)),
|
|
921
|
+
("unchanged", str(unchanged)),
|
|
922
|
+
("full-text", str(converted)),
|
|
923
|
+
("content-failed", str(failed)),
|
|
924
|
+
("pruned", str(pruned)),
|
|
925
|
+
("downloaded", human_size(downloaded)),
|
|
926
|
+
]
|
|
927
|
+
for label, value in base_rows:
|
|
928
|
+
table.add_row(label, value)
|
|
929
|
+
if vision:
|
|
930
|
+
table.add_row("svg-described", str(svg_described))
|
|
931
|
+
if captioner is not None:
|
|
932
|
+
table.add_row("images-described", str(images_described))
|
|
933
|
+
else:
|
|
934
|
+
table.add_row("images-queued", str(images_queued))
|
|
935
|
+
if vision_skipped:
|
|
936
|
+
table.add_row("vision-skipped", str(vision_skipped))
|
|
937
|
+
if run_cochange:
|
|
938
|
+
table.add_row("changesets", str(n_changesets))
|
|
939
|
+
table.add_row("co_changed_files", str(n_co_changed_files))
|
|
940
|
+
table.add_row("cochange_kind", run_kind) # MM-34 E
|
|
941
|
+
if cochange_result_dropped_oversized > 0:
|
|
942
|
+
table.add_row("dropped_oversized", str(cochange_result_dropped_oversized))
|
|
943
|
+
if cochange_result_dropped_singletons > 0:
|
|
944
|
+
table.add_row("dropped_singletons", str(cochange_result_dropped_singletons))
|
|
945
|
+
if cochange_result_truncated > 0:
|
|
946
|
+
table.add_row("truncated_from_blocks", str(cochange_result_truncated))
|
|
947
|
+
if budget_hit:
|
|
948
|
+
table.add_row("budget", "[yellow]reached — remaining files metadata-only[/yellow]")
|
|
949
|
+
console.print(table)
|
|
950
|
+
|
|
951
|
+
if dream:
|
|
952
|
+
from mintmory.core.llm import build_dreaming_engine
|
|
953
|
+
|
|
954
|
+
report = build_dreaming_engine(
|
|
955
|
+
store, settings.llm, link_settings=settings.link, summary_settings=settings.summary
|
|
956
|
+
).run_light()
|
|
957
|
+
console.print(
|
|
958
|
+
f"[dim]dream: +{report.new_links} links, +{report.new_summaries} summaries[/dim]"
|
|
959
|
+
)
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
@app.command()
|
|
963
|
+
def note(
|
|
964
|
+
content: str = typer.Argument(..., help="The thing to remember"),
|
|
965
|
+
about: str | None = typer.Option(None, help="What this note is about (anchor)"),
|
|
966
|
+
when: str | None = typer.Option(None, help="ISO date this note is salient (e.g. 2026-07-01)"),
|
|
967
|
+
until: str | None = typer.Option(None, help="ISO deadline"),
|
|
968
|
+
category: str | None = typer.Option(None, help="Override category (default temporal/episodic)"),
|
|
969
|
+
) -> None:
|
|
970
|
+
"""Capture a user-authored note ('remember this about X')."""
|
|
971
|
+
from mintmory.core import notes as notes_mod
|
|
972
|
+
|
|
973
|
+
when_dt = _parse_iso(when)
|
|
974
|
+
until_dt = _parse_iso(until)
|
|
975
|
+
cat = _parse_category(category) if category is not None else None
|
|
976
|
+
store = _get_store()
|
|
977
|
+
result = notes_mod.create_note(
|
|
978
|
+
store,
|
|
979
|
+
content=content,
|
|
980
|
+
about=about,
|
|
981
|
+
when=when_dt,
|
|
982
|
+
until=until_dt,
|
|
983
|
+
category=cat,
|
|
984
|
+
)
|
|
985
|
+
console.print(f"[green]Added note[/green] [bold]{result.note.id}[/bold]")
|
|
986
|
+
if result.anchor_kind == "memory":
|
|
987
|
+
console.print(f" [dim]-> annotates [cyan]{result.anchor_memory_id}[/cyan][/dim]")
|
|
988
|
+
elif result.anchor_kind == "topic":
|
|
989
|
+
entities_str = ", ".join(result.anchor_entities) if result.anchor_entities else "(none)"
|
|
990
|
+
console.print(f" [dim]-> topic: {entities_str}[/dim]")
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
@app.command()
|
|
994
|
+
def notes(
|
|
995
|
+
about: str | None = typer.Option(None, help="Filter by subject/entity"),
|
|
996
|
+
upcoming: bool = typer.Option(False, "--upcoming", help="Future-dated notes, soonest first"),
|
|
997
|
+
overdue: bool = typer.Option(False, "--overdue", help="Past-due notes (valid_from < now)"),
|
|
998
|
+
limit: int = typer.Option(50, help="Max notes"),
|
|
999
|
+
) -> None:
|
|
1000
|
+
"""List user-authored notes. Use --upcoming / --overdue for time views."""
|
|
1001
|
+
from mintmory.core import notes as notes_mod
|
|
1002
|
+
|
|
1003
|
+
store = _get_store()
|
|
1004
|
+
try:
|
|
1005
|
+
records = notes_mod.notes_list(
|
|
1006
|
+
store,
|
|
1007
|
+
about=about,
|
|
1008
|
+
upcoming=upcoming,
|
|
1009
|
+
overdue=overdue,
|
|
1010
|
+
limit=limit,
|
|
1011
|
+
)
|
|
1012
|
+
except ValueError as exc:
|
|
1013
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1014
|
+
|
|
1015
|
+
table = Table(title="Notes")
|
|
1016
|
+
table.add_column("id", style="cyan", no_wrap=True)
|
|
1017
|
+
table.add_column("category", style="magenta")
|
|
1018
|
+
table.add_column("valid_from", style="yellow")
|
|
1019
|
+
table.add_column("content")
|
|
1020
|
+
for mem in records:
|
|
1021
|
+
valid_from_str = mem.valid_from.isoformat() if mem.valid_from is not None else ""
|
|
1022
|
+
table.add_row(mem.id, mem.category.value, valid_from_str, mem.content)
|
|
1023
|
+
console.print(table)
|
|
1024
|
+
console.print(f"[dim]{len(records)} note(s)[/dim]")
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
@app.command()
|
|
1028
|
+
def summary_jobs(
|
|
1029
|
+
include_all: bool = typer.Option(
|
|
1030
|
+
False, "--all/--needed", help="All qualifying concepts vs only those needing a (re)summary"
|
|
1031
|
+
),
|
|
1032
|
+
limit: int = typer.Option(0, help="Max jobs (0 = no cap)"),
|
|
1033
|
+
json_out: bool = typer.Option(False, "--json", help="Emit JSON instead of a table"),
|
|
1034
|
+
) -> None:
|
|
1035
|
+
"""List L3 concept-summary jobs for the agent to write (agent-supplied L3)."""
|
|
1036
|
+
from mintmory.core.config import load_settings
|
|
1037
|
+
from mintmory.core.llm import build_dreaming_engine
|
|
1038
|
+
|
|
1039
|
+
settings = load_settings()
|
|
1040
|
+
store = _get_store()
|
|
1041
|
+
engine = build_dreaming_engine(
|
|
1042
|
+
store, settings.llm, link_settings=settings.link, summary_settings=settings.summary
|
|
1043
|
+
)
|
|
1044
|
+
jobs = engine.collect_summary_jobs(include_all=include_all)
|
|
1045
|
+
if limit > 0:
|
|
1046
|
+
jobs = jobs[:limit]
|
|
1047
|
+
|
|
1048
|
+
if json_out:
|
|
1049
|
+
import json as _json
|
|
1050
|
+
|
|
1051
|
+
console.print_json(_json.dumps([j.model_dump(mode="json") for j in jobs]))
|
|
1052
|
+
return
|
|
1053
|
+
|
|
1054
|
+
table = Table(title="Summary jobs")
|
|
1055
|
+
table.add_column("concept", style="cyan", no_wrap=True)
|
|
1056
|
+
table.add_column("memories", justify="right", style="green")
|
|
1057
|
+
table.add_column("has_summary", style="magenta")
|
|
1058
|
+
for j in jobs:
|
|
1059
|
+
table.add_row(j.concept, str(j.memory_count), "yes" if j.current_summary else "no")
|
|
1060
|
+
console.print(table)
|
|
1061
|
+
console.print(f"[dim]{len(jobs)} job(s)[/dim]")
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
@app.command()
|
|
1065
|
+
def summary_put(
|
|
1066
|
+
concept: str = typer.Argument(..., help="Concept/entity name"),
|
|
1067
|
+
text: str | None = typer.Argument(None, help="Summary text (omit to read --file or stdin)"),
|
|
1068
|
+
file: Path | None = typer.Option(None, "--file", "-f", help="Read summary text from a file"),
|
|
1069
|
+
) -> None:
|
|
1070
|
+
"""Store an agent-supplied summary for a concept (text arg, --file, or stdin)."""
|
|
1071
|
+
import sys
|
|
1072
|
+
|
|
1073
|
+
from mintmory.core.config import load_settings
|
|
1074
|
+
from mintmory.core.llm import build_dreaming_engine
|
|
1075
|
+
|
|
1076
|
+
if text is not None:
|
|
1077
|
+
summary_text = text
|
|
1078
|
+
elif file is not None:
|
|
1079
|
+
summary_text = file.read_text()
|
|
1080
|
+
else:
|
|
1081
|
+
summary_text = sys.stdin.read()
|
|
1082
|
+
summary_text = summary_text.strip()
|
|
1083
|
+
if not summary_text:
|
|
1084
|
+
raise typer.BadParameter("empty summary text (provide TEXT, --file, or stdin)")
|
|
1085
|
+
|
|
1086
|
+
settings = load_settings()
|
|
1087
|
+
store = _get_store()
|
|
1088
|
+
engine = build_dreaming_engine(
|
|
1089
|
+
store, settings.llm, link_settings=settings.link, summary_settings=settings.summary
|
|
1090
|
+
)
|
|
1091
|
+
summary = engine.apply_summary(concept, summary_text)
|
|
1092
|
+
console.print(
|
|
1093
|
+
f"[green]Stored summary[/green] for [cyan]{summary.concept}[/cyan] "
|
|
1094
|
+
f"[dim]({summary.memory_count} memories)[/dim]"
|
|
1095
|
+
)
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
@app.command()
|
|
1099
|
+
def image_jobs(
|
|
1100
|
+
include_all: bool = typer.Option(
|
|
1101
|
+
False,
|
|
1102
|
+
"--all/--needed",
|
|
1103
|
+
help="All raster images vs only those needing a description",
|
|
1104
|
+
),
|
|
1105
|
+
include_bytes: bool = typer.Option(
|
|
1106
|
+
False,
|
|
1107
|
+
"--bytes/--no-bytes",
|
|
1108
|
+
help="Force-embed base64 for local files too (use when agent runs on a different host)",
|
|
1109
|
+
),
|
|
1110
|
+
limit: int = typer.Option(0, help="Max jobs (0 = no cap)"),
|
|
1111
|
+
json_out: bool = typer.Option(False, "--json", help="Emit JSON instead of a rich table"),
|
|
1112
|
+
) -> None:
|
|
1113
|
+
"""List image-description jobs for the agent (agent-supplied vision).
|
|
1114
|
+
|
|
1115
|
+
Returns raster image file-records from the index that need an agent-supplied
|
|
1116
|
+
description (or all raster images with ``--all``). Each job carries the
|
|
1117
|
+
image path, rel, mime, size, online_only flag, and optionally an inline
|
|
1118
|
+
base64 payload (``--bytes`` or when the image is online-only). SVG and
|
|
1119
|
+
proprietary image formats are never returned as agent jobs.
|
|
1120
|
+
|
|
1121
|
+
After describing an image, store the result with ``image-caption-put``.
|
|
1122
|
+
"""
|
|
1123
|
+
import json as _json
|
|
1124
|
+
|
|
1125
|
+
from mintmory.core import vision as vision_mod
|
|
1126
|
+
from mintmory.core.config import load_settings
|
|
1127
|
+
|
|
1128
|
+
settings = load_settings()
|
|
1129
|
+
store = _get_store()
|
|
1130
|
+
jobs = vision_mod.image_jobs(
|
|
1131
|
+
store,
|
|
1132
|
+
include_all=include_all,
|
|
1133
|
+
include_bytes=include_bytes,
|
|
1134
|
+
limit=limit,
|
|
1135
|
+
settings=settings.vision,
|
|
1136
|
+
)
|
|
1137
|
+
|
|
1138
|
+
if json_out:
|
|
1139
|
+
console.print_json(_json.dumps([j.model_dump(mode="json") for j in jobs]))
|
|
1140
|
+
return
|
|
1141
|
+
|
|
1142
|
+
table = Table(title="Image jobs")
|
|
1143
|
+
table.add_column("file_id", style="cyan", no_wrap=True)
|
|
1144
|
+
table.add_column("rel", style="white")
|
|
1145
|
+
table.add_column("mime", style="magenta")
|
|
1146
|
+
table.add_column("online_only", style="yellow")
|
|
1147
|
+
table.add_column("has_desc", style="green")
|
|
1148
|
+
table.add_column("bytes", style="blue")
|
|
1149
|
+
for j in jobs:
|
|
1150
|
+
table.add_row(
|
|
1151
|
+
j.file_id,
|
|
1152
|
+
j.rel,
|
|
1153
|
+
j.mime,
|
|
1154
|
+
"yes" if j.online_only else "no",
|
|
1155
|
+
"yes" if j.current_description else "no",
|
|
1156
|
+
"yes" if j.image_b64 else ("oversized" if j.oversized else "no"),
|
|
1157
|
+
)
|
|
1158
|
+
console.print(table)
|
|
1159
|
+
console.print(f"[dim]{len(jobs)} image job(s)[/dim]")
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
@app.command()
|
|
1163
|
+
def image_caption_put(
|
|
1164
|
+
file_id_or_path: str = typer.Argument(..., help="Image file-record id or path"),
|
|
1165
|
+
text: str | None = typer.Argument(None, help="Description (omit to read from --file or stdin)"),
|
|
1166
|
+
file: Path | None = typer.Option(None, "--file", "-f", help="Read description from a file"),
|
|
1167
|
+
) -> None:
|
|
1168
|
+
"""Store an agent-supplied image description (text arg, --file, or stdin).
|
|
1169
|
+
|
|
1170
|
+
Persists the description as a context memory ANNOTATES-linked to the image
|
|
1171
|
+
file-record. Idempotent: re-putting replaces the prior description, and the
|
|
1172
|
+
image then drops out of the default ``image-jobs`` work-list. No vision
|
|
1173
|
+
backend is required.
|
|
1174
|
+
|
|
1175
|
+
``FILE_OR_ID`` may be the ``file_id`` from ``image-jobs`` (preferred) or
|
|
1176
|
+
the image's absolute path. Description text may be given inline, read from
|
|
1177
|
+
``--file``, or piped via stdin.
|
|
1178
|
+
"""
|
|
1179
|
+
import sys
|
|
1180
|
+
|
|
1181
|
+
from mintmory.core import vision as vision_mod
|
|
1182
|
+
from mintmory.core.config import load_settings
|
|
1183
|
+
|
|
1184
|
+
if text is not None:
|
|
1185
|
+
description = text
|
|
1186
|
+
elif file is not None:
|
|
1187
|
+
description = file.read_text()
|
|
1188
|
+
else:
|
|
1189
|
+
description = sys.stdin.read()
|
|
1190
|
+
|
|
1191
|
+
description = description.strip()
|
|
1192
|
+
if not description:
|
|
1193
|
+
raise typer.BadParameter(
|
|
1194
|
+
"empty description (provide TEXT argument, --file, or pipe via stdin)"
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
settings = load_settings()
|
|
1198
|
+
store = _get_store()
|
|
1199
|
+
try:
|
|
1200
|
+
result = vision_mod.image_caption_put(
|
|
1201
|
+
store, file_id_or_path, description, settings=settings.vision
|
|
1202
|
+
)
|
|
1203
|
+
except KeyError as exc:
|
|
1204
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1205
|
+
|
|
1206
|
+
console.print(
|
|
1207
|
+
f"[green]Stored description[/green] [bold]{result.record.id}[/bold] "
|
|
1208
|
+
f"for [cyan]{result.source_image}[/cyan]"
|
|
1209
|
+
)
|
|
1210
|
+
if result.replaced_description_id is not None:
|
|
1211
|
+
console.print(
|
|
1212
|
+
f" [dim]-> replaced (archived) [yellow]{result.replaced_description_id}[/yellow][/dim]"
|
|
1213
|
+
)
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
@app.command()
|
|
1217
|
+
def vision_run(
|
|
1218
|
+
limit: int = typer.Option(0, help="Max images to caption (0 = no cap)"),
|
|
1219
|
+
budget_mb: float = typer.Option(
|
|
1220
|
+
0.0, "--budget", help="Download budget MB for online-only images (0 = settings default)"
|
|
1221
|
+
),
|
|
1222
|
+
include_all: bool = typer.Option(
|
|
1223
|
+
False, "--all/--needed", help="Re-caption all raster images vs only pending"
|
|
1224
|
+
),
|
|
1225
|
+
) -> None:
|
|
1226
|
+
"""Caption already-indexed pending images with the configured vision provider.
|
|
1227
|
+
|
|
1228
|
+
Requires MINTMORY_VISION_PROVIDER=llm (a reachable OpenAI-compatible vision
|
|
1229
|
+
model). With provider=agent this is a no-op (use the image-jobs/image-caption-put
|
|
1230
|
+
agent loop instead). Does NOT re-walk the tree — it processes the pending
|
|
1231
|
+
image_jobs in place. Per-image failures are skipped and counted; one bad image
|
|
1232
|
+
never aborts the run.
|
|
1233
|
+
"""
|
|
1234
|
+
from mintmory.core import vision as vision_mod
|
|
1235
|
+
from mintmory.core.config import load_settings
|
|
1236
|
+
|
|
1237
|
+
settings = load_settings()
|
|
1238
|
+
try:
|
|
1239
|
+
captioner = vision_mod.captioner_from_settings(settings.vision)
|
|
1240
|
+
except NotImplementedError as exc:
|
|
1241
|
+
console.print(f"[red]error[/red]: {exc}")
|
|
1242
|
+
raise typer.Exit(code=1) from exc
|
|
1243
|
+
|
|
1244
|
+
if captioner is None:
|
|
1245
|
+
console.print(
|
|
1246
|
+
"[dim]provider=agent: nothing to run (use image-jobs/image-caption-put)[/dim]"
|
|
1247
|
+
)
|
|
1248
|
+
return
|
|
1249
|
+
|
|
1250
|
+
store = _get_store()
|
|
1251
|
+
budget: int | None = int(budget_mb * 1024 * 1024) if budget_mb > 0 else None
|
|
1252
|
+
report = vision_mod.caption_pending_images(
|
|
1253
|
+
store,
|
|
1254
|
+
captioner=captioner,
|
|
1255
|
+
limit=limit,
|
|
1256
|
+
budget=budget,
|
|
1257
|
+
include_all=include_all,
|
|
1258
|
+
settings=settings.vision,
|
|
1259
|
+
)
|
|
1260
|
+
|
|
1261
|
+
table = Table(title="vision-run")
|
|
1262
|
+
table.add_column("metric", style="cyan")
|
|
1263
|
+
table.add_column("value", justify="right", style="green")
|
|
1264
|
+
table.add_row("described", str(report.described))
|
|
1265
|
+
table.add_row("skipped", str(report.skipped))
|
|
1266
|
+
table.add_row("failed", str(report.failed))
|
|
1267
|
+
table.add_row("budget_hit", "yes" if report.budget_hit else "no")
|
|
1268
|
+
console.print(table)
|
|
1269
|
+
|
|
1270
|
+
if report.items:
|
|
1271
|
+
items_table = Table(title="items", show_header=True)
|
|
1272
|
+
items_table.add_column("file_id", style="cyan", no_wrap=True)
|
|
1273
|
+
items_table.add_column("rel")
|
|
1274
|
+
items_table.add_column("status", style="magenta")
|
|
1275
|
+
items_table.add_column("note", style="dim")
|
|
1276
|
+
for item in report.items:
|
|
1277
|
+
items_table.add_row(item.file_id, item.rel, item.status, item.note)
|
|
1278
|
+
console.print(items_table)
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
@app.command()
|
|
1282
|
+
def mcp_serve(
|
|
1283
|
+
transport: str = typer.Option("stdio", help="stdio or sse"),
|
|
1284
|
+
port: int = typer.Option(8081, help="Port for SSE transport"),
|
|
1285
|
+
) -> None:
|
|
1286
|
+
"""Start the MCP server."""
|
|
1287
|
+
os.environ.setdefault("MINTMORY_TRANSPORT", transport)
|
|
1288
|
+
from mintmory.mcp.server import main as mcp_main
|
|
1289
|
+
|
|
1290
|
+
mcp_main()
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
# ---------------------------------------------------------------------------
|
|
1294
|
+
# mintmory history — agent-history sub-commands
|
|
1295
|
+
# ---------------------------------------------------------------------------
|
|
1296
|
+
|
|
1297
|
+
history_app = typer.Typer(name="history", help="Agent-history index commands")
|
|
1298
|
+
app.add_typer(history_app)
|
|
1299
|
+
|
|
1300
|
+
|
|
1301
|
+
def _get_history_store(db_path: str) -> SqliteBackend:
|
|
1302
|
+
"""Open the dedicated history DB (not the working store)."""
|
|
1303
|
+
from mintmory.core.history.ingest import _assert_not_working_db
|
|
1304
|
+
from mintmory.core.storage import SqliteBackend
|
|
1305
|
+
|
|
1306
|
+
_assert_not_working_db(db_path)
|
|
1307
|
+
expanded = os.path.expanduser(db_path)
|
|
1308
|
+
os.makedirs(os.path.dirname(os.path.abspath(expanded)), exist_ok=True)
|
|
1309
|
+
store = SqliteBackend(expanded)
|
|
1310
|
+
store.initialise()
|
|
1311
|
+
return store
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def _parse_since(since: str) -> datetime:
|
|
1315
|
+
"""Parse --since like '75d', '8w', '3m' into an absolute UTC datetime."""
|
|
1316
|
+
import re
|
|
1317
|
+
|
|
1318
|
+
m = re.fullmatch(r"(\d+)([dwm])", since.strip())
|
|
1319
|
+
if not m:
|
|
1320
|
+
raise typer.BadParameter(f"invalid --since {since!r}; expected e.g. '60d', '8w', '3m'")
|
|
1321
|
+
n = int(m.group(1))
|
|
1322
|
+
unit = m.group(2)
|
|
1323
|
+
|
|
1324
|
+
now = datetime.now(UTC).replace(tzinfo=None)
|
|
1325
|
+
if unit == "d":
|
|
1326
|
+
from datetime import timedelta
|
|
1327
|
+
|
|
1328
|
+
return now - timedelta(days=n)
|
|
1329
|
+
elif unit == "w":
|
|
1330
|
+
from datetime import timedelta
|
|
1331
|
+
|
|
1332
|
+
return now - timedelta(weeks=n)
|
|
1333
|
+
else: # 'm'
|
|
1334
|
+
# Approximate: 1 month ~ 30 days
|
|
1335
|
+
from datetime import timedelta
|
|
1336
|
+
|
|
1337
|
+
return now - timedelta(days=n * 30)
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
@history_app.command("backfill")
|
|
1341
|
+
def history_backfill(
|
|
1342
|
+
source: list[str] = typer.Option(
|
|
1343
|
+
[], "--source", help="Adapter(s) to run: claude_code, codex, kiro, hermes (repeatable)"
|
|
1344
|
+
),
|
|
1345
|
+
db: str = typer.Option("", "--db", help="History DB path (default: agent-history.db)"),
|
|
1346
|
+
limit: int = typer.Option(0, "--limit", help="Cap sessions per source (0 = unlimited)"),
|
|
1347
|
+
max_llm_calls: int = typer.Option(
|
|
1348
|
+
0, "--max-llm-calls", help="Max LLM distiller calls (0=unlimited)"
|
|
1349
|
+
),
|
|
1350
|
+
llm_only_since: str | None = typer.Option(
|
|
1351
|
+
None, "--llm-only-since", help="Only use LLM for sessions on/after DATE (ISO)"
|
|
1352
|
+
),
|
|
1353
|
+
allow_cloud_llm: bool = typer.Option(
|
|
1354
|
+
False,
|
|
1355
|
+
"--allow-cloud-llm/--no-allow-cloud-llm",
|
|
1356
|
+
help="Allow non-localhost LLM endpoint for distillation",
|
|
1357
|
+
),
|
|
1358
|
+
) -> None:
|
|
1359
|
+
"""Backfill all sessions from agent adapters into the history DB."""
|
|
1360
|
+
from mintmory.core.config import load_settings
|
|
1361
|
+
from mintmory.core.history.ingest import (
|
|
1362
|
+
DEFAULT_HISTORY_DB,
|
|
1363
|
+
HermesGuardError,
|
|
1364
|
+
LLMBudget,
|
|
1365
|
+
backfill,
|
|
1366
|
+
)
|
|
1367
|
+
from mintmory.core.llm import build_history_distiller
|
|
1368
|
+
|
|
1369
|
+
db_path = db if db else DEFAULT_HISTORY_DB
|
|
1370
|
+
settings = load_settings()
|
|
1371
|
+
|
|
1372
|
+
seg_settings = (
|
|
1373
|
+
settings.seg.model_copy(update={"allow_cloud_llm": True})
|
|
1374
|
+
if allow_cloud_llm
|
|
1375
|
+
else settings.seg
|
|
1376
|
+
)
|
|
1377
|
+
|
|
1378
|
+
distiller = build_history_distiller(settings.llm, seg_settings)
|
|
1379
|
+
budget = LLMBudget(max_calls=max_llm_calls) if max_llm_calls > 0 else None
|
|
1380
|
+
|
|
1381
|
+
llm_since_dt: datetime | None = None
|
|
1382
|
+
if llm_only_since:
|
|
1383
|
+
try:
|
|
1384
|
+
llm_since_dt = datetime.fromisoformat(llm_only_since)
|
|
1385
|
+
except ValueError as exc:
|
|
1386
|
+
raise typer.BadParameter(
|
|
1387
|
+
f"invalid --llm-only-since {llm_only_since!r}; expected ISO date"
|
|
1388
|
+
) from exc
|
|
1389
|
+
|
|
1390
|
+
try:
|
|
1391
|
+
report = backfill(
|
|
1392
|
+
db_path=db_path,
|
|
1393
|
+
sources=list(source) if source else None,
|
|
1394
|
+
limit=limit,
|
|
1395
|
+
seg_settings=seg_settings,
|
|
1396
|
+
distiller=distiller,
|
|
1397
|
+
budget=budget,
|
|
1398
|
+
llm_only_since=llm_since_dt,
|
|
1399
|
+
max_concurrency=settings.llm.max_concurrency,
|
|
1400
|
+
)
|
|
1401
|
+
except HermesGuardError as exc:
|
|
1402
|
+
console.print(f"[red]error[/red]: {exc}")
|
|
1403
|
+
raise typer.Exit(code=1) from exc
|
|
1404
|
+
|
|
1405
|
+
table = Table(title="history backfill")
|
|
1406
|
+
table.add_column("metric", style="cyan")
|
|
1407
|
+
table.add_column("value", justify="right", style="green")
|
|
1408
|
+
table.add_row("scanned", str(report.scanned))
|
|
1409
|
+
table.add_row("written", str(report.written))
|
|
1410
|
+
table.add_row("updated", str(report.updated))
|
|
1411
|
+
table.add_row("skipped", str(report.skipped))
|
|
1412
|
+
table.add_row("segments_written", str(report.segments_written))
|
|
1413
|
+
table.add_row("llm_calls", str(report.llm_calls))
|
|
1414
|
+
table.add_row("llm_cache_hits", str(report.llm_cache_hits))
|
|
1415
|
+
table.add_row("llm_fallbacks", str(report.llm_fallbacks))
|
|
1416
|
+
for src_name, count in report.by_source.items():
|
|
1417
|
+
table.add_row(f" {src_name}", str(count))
|
|
1418
|
+
console.print(table)
|
|
1419
|
+
|
|
1420
|
+
|
|
1421
|
+
@history_app.command("sync")
|
|
1422
|
+
def history_sync(
|
|
1423
|
+
source: list[str] = typer.Option(
|
|
1424
|
+
[], "--source", help="Adapter(s): claude_code, codex, kiro, hermes (repeatable)"
|
|
1425
|
+
),
|
|
1426
|
+
db: str = typer.Option("", "--db", help="History DB path"),
|
|
1427
|
+
max_llm_calls: int = typer.Option(
|
|
1428
|
+
0, "--max-llm-calls", help="Max LLM distiller calls (0=unlimited)"
|
|
1429
|
+
),
|
|
1430
|
+
llm_only_since: str | None = typer.Option(
|
|
1431
|
+
None, "--llm-only-since", help="Only use LLM for sessions on/after DATE (ISO)"
|
|
1432
|
+
),
|
|
1433
|
+
allow_cloud_llm: bool = typer.Option(
|
|
1434
|
+
False,
|
|
1435
|
+
"--allow-cloud-llm/--no-allow-cloud-llm",
|
|
1436
|
+
help="Allow non-localhost LLM endpoint for distillation",
|
|
1437
|
+
),
|
|
1438
|
+
) -> None:
|
|
1439
|
+
"""Sync new/changed sessions only (skip unchanged files per manifest)."""
|
|
1440
|
+
from mintmory.core.config import load_settings
|
|
1441
|
+
from mintmory.core.history.ingest import (
|
|
1442
|
+
DEFAULT_HISTORY_DB,
|
|
1443
|
+
HermesGuardError,
|
|
1444
|
+
LLMBudget,
|
|
1445
|
+
sync,
|
|
1446
|
+
)
|
|
1447
|
+
from mintmory.core.llm import build_history_distiller
|
|
1448
|
+
|
|
1449
|
+
db_path = db if db else DEFAULT_HISTORY_DB
|
|
1450
|
+
settings = load_settings()
|
|
1451
|
+
|
|
1452
|
+
seg_settings = (
|
|
1453
|
+
settings.seg.model_copy(update={"allow_cloud_llm": True})
|
|
1454
|
+
if allow_cloud_llm
|
|
1455
|
+
else settings.seg
|
|
1456
|
+
)
|
|
1457
|
+
|
|
1458
|
+
distiller = build_history_distiller(settings.llm, seg_settings)
|
|
1459
|
+
budget = LLMBudget(max_calls=max_llm_calls) if max_llm_calls > 0 else None
|
|
1460
|
+
|
|
1461
|
+
llm_since_dt: datetime | None = None
|
|
1462
|
+
if llm_only_since:
|
|
1463
|
+
try:
|
|
1464
|
+
llm_since_dt = datetime.fromisoformat(llm_only_since)
|
|
1465
|
+
except ValueError as exc:
|
|
1466
|
+
raise typer.BadParameter(
|
|
1467
|
+
f"invalid --llm-only-since {llm_only_since!r}; expected ISO date"
|
|
1468
|
+
) from exc
|
|
1469
|
+
|
|
1470
|
+
try:
|
|
1471
|
+
report = sync(
|
|
1472
|
+
db_path=db_path,
|
|
1473
|
+
sources=list(source) if source else None,
|
|
1474
|
+
seg_settings=seg_settings,
|
|
1475
|
+
distiller=distiller,
|
|
1476
|
+
budget=budget,
|
|
1477
|
+
llm_only_since=llm_since_dt,
|
|
1478
|
+
max_concurrency=settings.llm.max_concurrency,
|
|
1479
|
+
)
|
|
1480
|
+
except HermesGuardError as exc:
|
|
1481
|
+
console.print(f"[red]error[/red]: {exc}")
|
|
1482
|
+
raise typer.Exit(code=1) from exc
|
|
1483
|
+
|
|
1484
|
+
table = Table(title="history sync")
|
|
1485
|
+
table.add_column("metric", style="cyan")
|
|
1486
|
+
table.add_column("value", justify="right", style="green")
|
|
1487
|
+
table.add_row("scanned", str(report.scanned))
|
|
1488
|
+
table.add_row("written", str(report.written))
|
|
1489
|
+
table.add_row("skipped", str(report.skipped))
|
|
1490
|
+
table.add_row("segments_written", str(report.segments_written))
|
|
1491
|
+
table.add_row("llm_calls", str(report.llm_calls))
|
|
1492
|
+
table.add_row("llm_cache_hits", str(report.llm_cache_hits))
|
|
1493
|
+
table.add_row("llm_fallbacks", str(report.llm_fallbacks))
|
|
1494
|
+
for src_name, count in report.by_source.items():
|
|
1495
|
+
table.add_row(f" {src_name}", str(count))
|
|
1496
|
+
console.print(table)
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
@history_app.command("timeline")
|
|
1500
|
+
def history_timeline(
|
|
1501
|
+
since: str | None = typer.Option(
|
|
1502
|
+
None, "--since", help="Window like '75d', '8w', '3m' (mutual excl. with --from/--to)"
|
|
1503
|
+
),
|
|
1504
|
+
from_date: str | None = typer.Option(None, "--from", help="ISO start date/datetime"),
|
|
1505
|
+
to_date: str | None = typer.Option(None, "--to", help="ISO end date/datetime"),
|
|
1506
|
+
repo: str | None = typer.Option(None, "--repo", help="Filter by repo name"),
|
|
1507
|
+
kind: str | None = typer.Option(None, "--kind", help="Filter by kind (fix/feature/...)"),
|
|
1508
|
+
limit: int = typer.Option(50, "--limit", help="Max rows"),
|
|
1509
|
+
db: str = typer.Option("", "--db", help="History DB path"),
|
|
1510
|
+
group_by_session: bool = typer.Option(
|
|
1511
|
+
False,
|
|
1512
|
+
"--group-by-session/--no-group-by-session",
|
|
1513
|
+
help="Group multi-segment sessions together",
|
|
1514
|
+
),
|
|
1515
|
+
) -> None:
|
|
1516
|
+
"""Print a dated changelog of sessions within the time window."""
|
|
1517
|
+
from mintmory.core.history.ingest import DEFAULT_HISTORY_DB, HermesGuardError
|
|
1518
|
+
from mintmory.core.history.query import timeline as _timeline
|
|
1519
|
+
|
|
1520
|
+
db_path = db if db else DEFAULT_HISTORY_DB
|
|
1521
|
+
try:
|
|
1522
|
+
rows = _timeline(
|
|
1523
|
+
db_path,
|
|
1524
|
+
since=since,
|
|
1525
|
+
from_iso=from_date,
|
|
1526
|
+
to_iso=to_date,
|
|
1527
|
+
repo=repo,
|
|
1528
|
+
kind=kind,
|
|
1529
|
+
limit=limit,
|
|
1530
|
+
group_by_session=group_by_session,
|
|
1531
|
+
)
|
|
1532
|
+
except HermesGuardError as exc:
|
|
1533
|
+
console.print(f"[red]error[/red]: {exc}")
|
|
1534
|
+
raise typer.Exit(code=1) from exc
|
|
1535
|
+
except ValueError as exc:
|
|
1536
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1537
|
+
|
|
1538
|
+
table = Table(title="history timeline")
|
|
1539
|
+
table.add_column("date", style="yellow", no_wrap=True)
|
|
1540
|
+
table.add_column("repo", style="cyan")
|
|
1541
|
+
table.add_column("kind", style="magenta")
|
|
1542
|
+
table.add_column("title")
|
|
1543
|
+
table.add_column("summary")
|
|
1544
|
+
for row in rows:
|
|
1545
|
+
seg_count = row.get("segment_count", 1)
|
|
1546
|
+
seg_idx = row.get("segment_index", 0)
|
|
1547
|
+
marker = f"[{seg_idx + 1}/{seg_count}] " if seg_count > 1 else ""
|
|
1548
|
+
title_cell = f"{marker}{row.get('title', '')}"
|
|
1549
|
+
table.add_row(
|
|
1550
|
+
row["date"],
|
|
1551
|
+
row["repo"],
|
|
1552
|
+
row["kind"],
|
|
1553
|
+
title_cell,
|
|
1554
|
+
row["summary"][:100],
|
|
1555
|
+
)
|
|
1556
|
+
console.print(table)
|
|
1557
|
+
console.print(f"[dim]{len(rows)} segment(s)[/dim]")
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
@history_app.command("search")
|
|
1561
|
+
def history_search(
|
|
1562
|
+
query: str = typer.Argument(..., help="Search query"),
|
|
1563
|
+
repo: str | None = typer.Option(None, "--repo", help="Filter by repo name"),
|
|
1564
|
+
since: str | None = typer.Option(None, "--since", help="Only sessions within window"),
|
|
1565
|
+
limit: int = typer.Option(10, "--limit", help="Max results"),
|
|
1566
|
+
db: str = typer.Option("", "--db", help="History DB path"),
|
|
1567
|
+
) -> None:
|
|
1568
|
+
"""Hybrid search over session summaries in the history DB."""
|
|
1569
|
+
from mintmory.core.history.ingest import DEFAULT_HISTORY_DB, HermesGuardError
|
|
1570
|
+
from mintmory.core.history.query import search as _search
|
|
1571
|
+
|
|
1572
|
+
db_path = db if db else DEFAULT_HISTORY_DB
|
|
1573
|
+
try:
|
|
1574
|
+
results = _search(db_path, query, repo=repo, since=since, limit=limit)
|
|
1575
|
+
except HermesGuardError as exc:
|
|
1576
|
+
console.print(f"[red]error[/red]: {exc}")
|
|
1577
|
+
raise typer.Exit(code=1) from exc
|
|
1578
|
+
except ValueError as exc:
|
|
1579
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1580
|
+
|
|
1581
|
+
table = Table(title=f"history search: {query!r}")
|
|
1582
|
+
table.add_column("date", style="yellow", no_wrap=True)
|
|
1583
|
+
table.add_column("repo", style="cyan")
|
|
1584
|
+
table.add_column("kind", style="magenta")
|
|
1585
|
+
table.add_column("summary")
|
|
1586
|
+
for row in results:
|
|
1587
|
+
table.add_row(
|
|
1588
|
+
row["date"],
|
|
1589
|
+
row["repo"],
|
|
1590
|
+
row["kind"],
|
|
1591
|
+
row["summary"][:120],
|
|
1592
|
+
)
|
|
1593
|
+
console.print(table)
|
|
1594
|
+
console.print(f"[dim]{len(results)} result(s)[/dim]")
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
@history_app.command("scrub")
|
|
1598
|
+
def history_scrub(
|
|
1599
|
+
db: str = typer.Option("", "--db", help="History DB path"),
|
|
1600
|
+
) -> None:
|
|
1601
|
+
"""Scan stored summaries for residual secrets; exit non-zero if any found."""
|
|
1602
|
+
|
|
1603
|
+
from mintmory.core.history.ingest import DEFAULT_HISTORY_DB, HermesGuardError
|
|
1604
|
+
from mintmory.core.history.redact import scan
|
|
1605
|
+
|
|
1606
|
+
db_path = db if db else DEFAULT_HISTORY_DB
|
|
1607
|
+
try:
|
|
1608
|
+
store = _get_history_store(db_path)
|
|
1609
|
+
except HermesGuardError as exc:
|
|
1610
|
+
console.print(f"[red]error[/red]: {exc}")
|
|
1611
|
+
raise typer.Exit(code=1) from exc
|
|
1612
|
+
|
|
1613
|
+
conn = store.connect()
|
|
1614
|
+
rows = conn.execute(
|
|
1615
|
+
"SELECT id, content FROM memories "
|
|
1616
|
+
"WHERE json_extract(metadata, '$.record_type') = 'session_summary' "
|
|
1617
|
+
" AND is_archived = 0"
|
|
1618
|
+
).fetchall()
|
|
1619
|
+
|
|
1620
|
+
total_hits: dict[str, int] = {}
|
|
1621
|
+
flagged = 0
|
|
1622
|
+
for row in rows:
|
|
1623
|
+
hits = scan(row["content"] or "")
|
|
1624
|
+
if hits:
|
|
1625
|
+
flagged += 1
|
|
1626
|
+
for label, count in hits.items():
|
|
1627
|
+
total_hits[label] = total_hits.get(label, 0) + count
|
|
1628
|
+
|
|
1629
|
+
# (summary_text/title live in `content`, already scanned above — the envelope
|
|
1630
|
+
# metadata carries no free-text secret-bearing fields, so no second pass.)
|
|
1631
|
+
|
|
1632
|
+
table = Table(title="history scrub")
|
|
1633
|
+
table.add_column("pattern", style="cyan")
|
|
1634
|
+
table.add_column("occurrences", justify="right", style="red")
|
|
1635
|
+
for label, count in total_hits.items():
|
|
1636
|
+
table.add_row(label, str(count))
|
|
1637
|
+
console.print(table)
|
|
1638
|
+
|
|
1639
|
+
if total_hits:
|
|
1640
|
+
console.print(
|
|
1641
|
+
f"[red]FAIL[/red] {flagged} record(s) contain residual secrets "
|
|
1642
|
+
f"({sum(total_hits.values())} total hit(s))"
|
|
1643
|
+
)
|
|
1644
|
+
raise typer.Exit(code=1)
|
|
1645
|
+
else:
|
|
1646
|
+
console.print("[green]ok[/green] no residual secrets found")
|
|
1647
|
+
|
|
1648
|
+
|
|
1649
|
+
# ---------------------------------------------------------------------------
|
|
1650
|
+
# mintmory docs — document index query sub-commands (MM-33)
|
|
1651
|
+
# ---------------------------------------------------------------------------
|
|
1652
|
+
|
|
1653
|
+
docs_app = typer.Typer(name="docs", help="Document index query commands")
|
|
1654
|
+
app.add_typer(docs_app)
|
|
1655
|
+
|
|
1656
|
+
|
|
1657
|
+
@docs_app.command("timeline")
|
|
1658
|
+
def docs_timeline(
|
|
1659
|
+
since: str | None = typer.Option(
|
|
1660
|
+
None, "--since", help="Window like '30d', '4w', '2m' (mutually exclusive with --from/--to)"
|
|
1661
|
+
),
|
|
1662
|
+
from_iso: str | None = typer.Option(None, "--from", help="ISO start date (e.g. 2026-01-01)"),
|
|
1663
|
+
to_iso: str | None = typer.Option(None, "--to", help="ISO end date (e.g. 2026-06-30)"),
|
|
1664
|
+
collection: str | None = typer.Option(None, help="Filter by collection name"),
|
|
1665
|
+
limit: int = typer.Option(50, help="Max rows"),
|
|
1666
|
+
db: str | None = typer.Option(None, help="Target DB path (overrides MINTMORY_DB)"),
|
|
1667
|
+
) -> None:
|
|
1668
|
+
"""List indexed documents newest-first (by file mtime / valid_from).
|
|
1669
|
+
|
|
1670
|
+
Shows documents indexed by index-tree, ordered by file modified-time
|
|
1671
|
+
(recorded as valid_from). Filters source='document' only.
|
|
1672
|
+
"""
|
|
1673
|
+
from mintmory.core.cochange import documents_timeline
|
|
1674
|
+
from mintmory.core.storage import SqliteBackend as _SqliteBackend
|
|
1675
|
+
|
|
1676
|
+
if db:
|
|
1677
|
+
os.environ["MINTMORY_DB"] = db
|
|
1678
|
+
|
|
1679
|
+
store = _get_store()
|
|
1680
|
+
try:
|
|
1681
|
+
rows = documents_timeline(
|
|
1682
|
+
cast(_SqliteBackend, store),
|
|
1683
|
+
since=since,
|
|
1684
|
+
from_iso=from_iso,
|
|
1685
|
+
to_iso=to_iso,
|
|
1686
|
+
collection=collection,
|
|
1687
|
+
limit=limit,
|
|
1688
|
+
)
|
|
1689
|
+
except ValueError as exc:
|
|
1690
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1691
|
+
|
|
1692
|
+
table = Table(title="Documents timeline (newest first)")
|
|
1693
|
+
table.add_column("date", style="cyan", no_wrap=True)
|
|
1694
|
+
table.add_column("collection", style="magenta")
|
|
1695
|
+
table.add_column("path", style="white")
|
|
1696
|
+
table.add_column("title", style="dim")
|
|
1697
|
+
for row in rows:
|
|
1698
|
+
table.add_row(row["date"], row["collection"], row["path"], row["title"])
|
|
1699
|
+
console.print(table)
|
|
1700
|
+
console.print(f"[dim]{len(rows)} document(s)[/dim]")
|
|
1701
|
+
|
|
1702
|
+
|
|
1703
|
+
@docs_app.command("changed-with")
|
|
1704
|
+
def docs_changed_with(
|
|
1705
|
+
path: str = typer.Argument(..., help="Absolute path of the document to look up"),
|
|
1706
|
+
db: str | None = typer.Option(None, help="Target DB path (overrides MINTMORY_DB)"),
|
|
1707
|
+
) -> None:
|
|
1708
|
+
"""Show documents observed to co-change with the given path.
|
|
1709
|
+
|
|
1710
|
+
Observed co-change (time + folder + content) — not a version-controlled commit.
|
|
1711
|
+
Co-change is inferred from HDBSCAN clustering of the index-tree run that
|
|
1712
|
+
processed these files; it is NOT an atomic git change-set.
|
|
1713
|
+
"""
|
|
1714
|
+
from mintmory.core.cochange import changed_with
|
|
1715
|
+
from mintmory.core.storage import SqliteBackend as _SqliteBackend
|
|
1716
|
+
|
|
1717
|
+
if db:
|
|
1718
|
+
os.environ["MINTMORY_DB"] = db
|
|
1719
|
+
|
|
1720
|
+
store = _get_store()
|
|
1721
|
+
peers = changed_with(cast(_SqliteBackend, store), path)
|
|
1722
|
+
|
|
1723
|
+
console.print(
|
|
1724
|
+
"[dim]observed co-change (time+folder+content) — not a version-controlled commit[/dim]"
|
|
1725
|
+
)
|
|
1726
|
+
if not peers:
|
|
1727
|
+
console.print(f"[dim]no co-change peers found for {path!r}[/dim]")
|
|
1728
|
+
return
|
|
1729
|
+
|
|
1730
|
+
table = Table(title=f"Co-changed with: {path}")
|
|
1731
|
+
table.add_column("peer path", style="white")
|
|
1732
|
+
table.add_column("strength", justify="right", style="green")
|
|
1733
|
+
table.add_column("observed at", style="cyan")
|
|
1734
|
+
table.add_column("kind", style="magenta")
|
|
1735
|
+
for peer in peers:
|
|
1736
|
+
peer_kind: str = peer.get("kind", "")
|
|
1737
|
+
if peer_kind == "cold_full_index":
|
|
1738
|
+
kind_label = "cold_full_index (co-location-dominated)"
|
|
1739
|
+
else:
|
|
1740
|
+
kind_label = peer_kind
|
|
1741
|
+
table.add_row(
|
|
1742
|
+
peer["path"],
|
|
1743
|
+
f"{peer['strength']:.3f}",
|
|
1744
|
+
peer["observed_at"],
|
|
1745
|
+
kind_label,
|
|
1746
|
+
)
|
|
1747
|
+
console.print(table)
|
|
1748
|
+
console.print(f"[dim]{len(peers)} peer(s)[/dim]")
|
|
1749
|
+
# Print full paths on separate lines so long paths are never truncated by Rich.
|
|
1750
|
+
for peer in peers:
|
|
1751
|
+
console.print(f" [dim]path:[/dim] {peer['path']}")
|
|
1752
|
+
|
|
1753
|
+
|
|
1754
|
+
@app.command("migrate-storage")
|
|
1755
|
+
def migrate_storage(
|
|
1756
|
+
source: str = typer.Option(
|
|
1757
|
+
...,
|
|
1758
|
+
"--from",
|
|
1759
|
+
help="Source database URI: sqlite:/path/to/memories.db",
|
|
1760
|
+
),
|
|
1761
|
+
target: str | None = typer.Option(
|
|
1762
|
+
None,
|
|
1763
|
+
"--to",
|
|
1764
|
+
help=("Target database URI: postgres:<DSN>. Defaults to MINTMORY_STORAGE_DSN env var."),
|
|
1765
|
+
),
|
|
1766
|
+
dry_run: bool = typer.Option(
|
|
1767
|
+
False,
|
|
1768
|
+
"--dry-run",
|
|
1769
|
+
help="Validate and count rows without writing to the target.",
|
|
1770
|
+
),
|
|
1771
|
+
force: bool = typer.Option(
|
|
1772
|
+
False,
|
|
1773
|
+
"--force",
|
|
1774
|
+
help="Truncate the target before writing (allows idempotent re-runs).",
|
|
1775
|
+
),
|
|
1776
|
+
batch_size: int = typer.Option(
|
|
1777
|
+
500,
|
|
1778
|
+
"--batch-size",
|
|
1779
|
+
help="Number of rows per INSERT batch.",
|
|
1780
|
+
),
|
|
1781
|
+
source_db: str | None = typer.Option(
|
|
1782
|
+
None,
|
|
1783
|
+
"--source-db",
|
|
1784
|
+
help="Alias for --from (SQLite path, no prefix required).",
|
|
1785
|
+
hidden=True,
|
|
1786
|
+
),
|
|
1787
|
+
target_dsn: str | None = typer.Option(
|
|
1788
|
+
None,
|
|
1789
|
+
"--target-dsn",
|
|
1790
|
+
help="Alias for --to (DSN string, no prefix required).",
|
|
1791
|
+
hidden=True,
|
|
1792
|
+
),
|
|
1793
|
+
) -> None:
|
|
1794
|
+
"""Migrate a SQLite MintMory database to PostgreSQL (one-shot, verified).
|
|
1795
|
+
|
|
1796
|
+
The source is always a SQLite file (v1 scope). The target must be a
|
|
1797
|
+
PostgreSQL database reachable via a psycopg DSN.
|
|
1798
|
+
|
|
1799
|
+
Example::
|
|
1800
|
+
|
|
1801
|
+
mintmory migrate-storage \\
|
|
1802
|
+
--from sqlite:/home/user/.mintmory/memories.db \\
|
|
1803
|
+
--to postgres:postgresql://user:pass@host:5432/mintmory
|
|
1804
|
+
|
|
1805
|
+
Run with --dry-run first to validate counts without writing.
|
|
1806
|
+
Use --force to overwrite an already-populated target.
|
|
1807
|
+
"""
|
|
1808
|
+
from mintmory.core.migrate import migrate_sqlite_to_postgres # noqa: PLC0415
|
|
1809
|
+
|
|
1810
|
+
# Resolve source path.
|
|
1811
|
+
# Strip the 'sqlite:' convenience prefix only when it is NOT a real URI
|
|
1812
|
+
# scheme (i.e. not followed by '//'). This preserves the standard
|
|
1813
|
+
# 'sqlite:///abs/path' URI form if a caller passes one.
|
|
1814
|
+
raw_src = source_db or source
|
|
1815
|
+
if raw_src.startswith("sqlite:") and not raw_src.startswith("sqlite://"):
|
|
1816
|
+
src_path = raw_src[len("sqlite:") :]
|
|
1817
|
+
else:
|
|
1818
|
+
src_path = raw_src
|
|
1819
|
+
|
|
1820
|
+
# Resolve target DSN.
|
|
1821
|
+
# Strip the 'postgres:' convenience prefix only when it is NOT a real URI
|
|
1822
|
+
# scheme (i.e. not followed by '//'). Both 'postgres://' and
|
|
1823
|
+
# 'postgresql://' are valid libpq URI schemes and must be passed through
|
|
1824
|
+
# unchanged.
|
|
1825
|
+
raw_target = target_dsn or target or os.environ.get("MINTMORY_STORAGE_DSN", "")
|
|
1826
|
+
if raw_target.startswith("postgres:") and not raw_target.startswith("postgres://"):
|
|
1827
|
+
dst_dsn = raw_target[len("postgres:") :]
|
|
1828
|
+
else:
|
|
1829
|
+
dst_dsn = raw_target
|
|
1830
|
+
|
|
1831
|
+
if not dst_dsn:
|
|
1832
|
+
raise typer.BadParameter(
|
|
1833
|
+
"Target DSN is required. Pass --to postgres:<DSN> or set MINTMORY_STORAGE_DSN."
|
|
1834
|
+
)
|
|
1835
|
+
|
|
1836
|
+
# Pre-flight: verify PG reachability with a short timeout BEFORE building
|
|
1837
|
+
# the connection pool. This converts the 30-second PoolTimeout hang + 130-line
|
|
1838
|
+
# traceback into a clear one-line error message with exit code 1.
|
|
1839
|
+
if not dry_run:
|
|
1840
|
+
try:
|
|
1841
|
+
import psycopg as _psycopg # noqa: PLC0415
|
|
1842
|
+
|
|
1843
|
+
_tc = _psycopg.connect(dst_dsn, connect_timeout=5)
|
|
1844
|
+
_tc.close()
|
|
1845
|
+
except Exception as conn_exc:
|
|
1846
|
+
console.print(f"\n[red]Error:[/red] could not connect to target Postgres: {conn_exc}")
|
|
1847
|
+
raise typer.Exit(code=1) from conn_exc
|
|
1848
|
+
|
|
1849
|
+
def _progress(table: str, written: int, total: int) -> None:
|
|
1850
|
+
console.print(f" [dim]{table:<20} {written:>6}/{total:<6}[/dim]", end="\r")
|
|
1851
|
+
|
|
1852
|
+
console.print(
|
|
1853
|
+
f"[bold]migrate-storage[/bold] {'[DRY-RUN] ' if dry_run else ''}{src_path!r} → Postgres"
|
|
1854
|
+
)
|
|
1855
|
+
console.print(f"[dim]batch_size={batch_size} force={force}[/dim]")
|
|
1856
|
+
|
|
1857
|
+
try:
|
|
1858
|
+
report = migrate_sqlite_to_postgres(
|
|
1859
|
+
src_path,
|
|
1860
|
+
dst_dsn,
|
|
1861
|
+
dry_run=dry_run,
|
|
1862
|
+
batch_size=batch_size,
|
|
1863
|
+
force=force,
|
|
1864
|
+
on_progress=_progress,
|
|
1865
|
+
)
|
|
1866
|
+
except (FileNotFoundError, ValueError, ImportError) as exc:
|
|
1867
|
+
console.print(f"\n[red]Error:[/red] {exc}")
|
|
1868
|
+
raise typer.Exit(code=1) from exc
|
|
1869
|
+
except Exception as exc:
|
|
1870
|
+
# Catch psycopg_pool.PoolTimeout and other unexpected connectivity errors
|
|
1871
|
+
# that weren't caught by the pre-flight check (e.g. transient failures).
|
|
1872
|
+
console.print(f"\n[red]Error:[/red] {type(exc).__name__}: {exc}")
|
|
1873
|
+
raise typer.Exit(code=1) from exc
|
|
1874
|
+
|
|
1875
|
+
console.print() # newline after progress line
|
|
1876
|
+
for line in report.summary_lines():
|
|
1877
|
+
if "FAIL" in line and not line.startswith(" "):
|
|
1878
|
+
console.print(f"[red]{line}[/red]")
|
|
1879
|
+
elif "ALL PASS" in line:
|
|
1880
|
+
console.print(f"[green]{line}[/green]")
|
|
1881
|
+
else:
|
|
1882
|
+
console.print(line)
|
|
1883
|
+
|
|
1884
|
+
if not report.all_ok and not dry_run:
|
|
1885
|
+
raise typer.Exit(code=1)
|
|
1886
|
+
|
|
1887
|
+
|
|
1888
|
+
if __name__ == "__main__":
|
|
1889
|
+
app()
|