memor-cli 0.6.0__tar.gz → 0.6.1__tar.gz

{memor_cli-0.6.0/memor_cli.egg-info → memor_cli-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memor-cli
-Version: 0.6.0
+Version: 0.6.1
 Summary: Measured memory for coding agents. Fire and forget — no API keys needed.
 Author-email: Nimit Bhandari <nimitbhandari17@gmail.com>
 License-Expression: MIT

memor_cli-0.6.1/memor/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.1"

{memor_cli-0.6.0 → memor_cli-0.6.1}/memor/cli.py

@@ -72,6 +72,7 @@ MAINTENANCE
   memor reingest --project <name>  Re-ingest only one project
   memor distill --project <name>   Run distillation manually
   memor forget-stale               Deactivate memories not recalled in 30 days
+  memor compact                    Rebuild vector index, reclaim space
   memor scan                       Audit DB for leaked secrets
   memor scan --purge               Redact secrets in place
   memor setup-model                Download/retry the embedding model (~60MB)
@@ -391,6 +392,36 @@ def forget_stale(days: int = typer.Option(30, help="Deactivate memories not reca
     typer.echo(f"Deactivated {count} stale memories.")
 
 
+@app.command("compact")
+def compact(db: str = typer.Option(str(Path.home() / ".memor" / "memor.db")),
+            fake: bool = False,
+            confirm: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation")):
+    """Rebuild the vector index to reclaim wasted space."""
+    db_path = _db_path(db)
+    if not Path(db_path).exists():
+        typer.echo("No database found.")
+        raise typer.Exit(1)
+    dim = _get_dim(db_path)
+    embedder = _embedder(fake)
+    if embedder.dim != dim:
+        typer.echo(f"Warning: embedder dim={embedder.dim} != DB dim={dim}. Using DB dim.")
+        from memor.embed.fake import FakeEmbedder
+        embedder = FakeEmbedder(dim=dim)
+    store = SqliteStore(db_path, dim=dim)
+    chunk_count = store.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_chunks").fetchone()["c"]
+    active_count = store.db.execute("SELECT COUNT(*) as c FROM artifacts WHERE active=1").fetchone()["c"]
+    db_size_mb = round(Path(db_path).stat().st_size / 1_048_576, 1)
+    typer.echo(f"Current: {chunk_count} chunks, {active_count} active vectors, {db_size_mb} MB")
+    if not confirm:
+        typer.confirm("Rebuild vector index?", abort=True)
+    typer.echo("Compacting...")
+    result = store.rebuild_vec_index(embedder)
+    db_size_after = round(Path(db_path).stat().st_size / 1_048_576, 1)
+    typer.echo(f"Done: {result['vectors_reindexed']} vectors reindexed in {result['duration_ms']}ms")
+    typer.echo(f"Chunks: {result['before_chunks']} -> {result['after_chunks']} (chunk_size={result['chunk_size']})")
+    typer.echo(f"DB size: {db_size_mb} MB -> {db_size_after} MB")
+
+
 @app.command("scan")
 def scan(db: str = typer.Option(str(Path.home() / ".memor" / "memor.db")),
          purge: bool = typer.Option(False, "--purge", help="Redact secrets in place"),

{memor_cli-0.6.0 → memor_cli-0.6.1}/memor/daemon.py

@@ -186,6 +186,29 @@ def compact_memories(store: SqliteStore, embedder) -> int:
     return deactivated
 
 
+def should_compact(store: SqliteStore) -> bool:
+    try:
+        chunk_count = store.db.execute(
+            "SELECT COUNT(*) as c FROM vec_artifacts_chunks").fetchone()["c"]
+        active_count = store.db.execute(
+            "SELECT COUNT(*) as c FROM artifacts WHERE active=1").fetchone()["c"]
+    except Exception as e:
+        print(f"  should_compact error: {e}")
+        return False
+    if chunk_count == 0 or active_count == 0:
+        return False
+    from memor.store.sqlite_store import _choose_chunk_size
+    chunk_size = _choose_chunk_size(active_count)
+    ideal_chunks = max(1, active_count // chunk_size + 1)
+    return chunk_count > ideal_chunks * 2
+
+
+def auto_compact(store: SqliteStore, embedder) -> dict | None:
+    if not should_compact(store):
+        return None
+    return store.rebuild_vec_index(embedder, vacuum=False)
+
+
 def run_poll_cycle(
     state: dict[str, float],
     store: SqliteStore,
@@ -286,6 +309,16 @@ def run_poll_cycle(
         except Exception:
             pass
 
+    # Auto-compact vec index if bloated
+    if new_ingested:
+        try:
+            result = auto_compact(store, embedder)
+            if result:
+                print(f"  vec compact: {result['before_chunks']} -> {result['after_chunks']} chunks "
+                      f"({result['vectors_reindexed']} vectors, {result['duration_ms']}ms)")
+        except Exception:
+            pass
+
     return state, distilled
 
 

{memor_cli-0.6.0 → memor_cli-0.6.1}/memor/dashboard/server.py

@@ -14,9 +14,13 @@ def create_app(db_path: str | None = None) -> FastAPI:
 
     app = FastAPI(title="Memor Dashboard")
     _db_path = db_path
+    _cached_store: SqliteStore | None = None
 
     def _store() -> SqliteStore:
-        return SqliteStore(_db_path, dim=_get_dim(_db_path))
+        nonlocal _cached_store
+        if _cached_store is None:
+            _cached_store = SqliteStore(_db_path, dim=_get_dim(_db_path))
+        return _cached_store
 
     @app.get("/", response_class=HTMLResponse)
     def index():
@@ -66,11 +70,16 @@ def create_app(db_path: str | None = None) -> FastAPI:
             ORDER BY artifacts DESC
         """).fetchall()
 
+        active_rows = [
+            r for r in artifact_rows
+            if (r["memories"] or 0) > 0 or r["project"] in recall_projects
+        ]
+
         if not recall_stats:
-            return [dict(r) for r in artifact_rows]
+            return [dict(r) for r in active_rows]
 
         result = list(recall_stats)
-        for r in artifact_rows:
+        for r in active_rows:
             if r["project"] not in recall_projects:
                 result.append(dict(r))
         return result

{memor_cli-0.6.0 → memor_cli-0.6.1}/memor/store/sqlite_store.py

@@ -7,6 +7,15 @@ from memor.types import Artifact, Scope, SessionUsage
 def _serialize(v: list[float]) -> bytes:
     return struct.pack("%sf" % len(v), *v)
 
+def _choose_chunk_size(active_count: int) -> int:
+    if active_count < 1000:
+        return 64
+    if active_count < 10000:
+        return 256
+    if active_count < 100000:
+        return 512
+    return 1024
+
 class SqliteStore:
     def __init__(self, path: str, dim: int):
         self.dim = dim
@@ -126,12 +135,15 @@ class SqliteStore:
     def add_artifacts(self, artifacts: list[Artifact], vectors: list[list[float]]) -> None:
         cur = self.db.cursor()
         for a, v in zip(artifacts, vectors):
+            old = cur.execute("SELECT rowid FROM artifacts WHERE id=?", (a.id,)).fetchone()
+            if old:
+                cur.execute("DELETE FROM vec_artifacts WHERE rowid=?", (old[0],))
             cur.execute(
               "INSERT OR REPLACE INTO artifacts(id,kind,project,source,text,token_count,created_at,meta,active,superseded_by)"
               " VALUES(?,?,?,?,?,?,?,?,1,NULL)",
               (a.id, a.kind, a.project, a.source, a.text, a.token_count, a.created_at, json.dumps(a.meta)))
             rowid = cur.execute("SELECT rowid FROM artifacts WHERE id=?", (a.id,)).fetchone()[0]
-            cur.execute("INSERT OR REPLACE INTO vec_artifacts(rowid, embedding) VALUES(?,?)",
+            cur.execute("INSERT INTO vec_artifacts(rowid, embedding) VALUES(?,?)",
                         (rowid, _serialize(v)))
             cur.execute("DELETE FROM fts_artifacts WHERE id=?", (a.id,))
             cur.execute("INSERT INTO fts_artifacts(id, text) VALUES(?,?)", (a.id, a.text))
@@ -182,6 +194,50 @@ class SqliteStore:
         self.db.commit()
         return n
 
+    def rebuild_vec_index(self, embedder, vacuum: bool = True) -> dict:
+        import time as _time
+        start = _time.time()
+        chunk_count_before = self.db.execute(
+            "SELECT COUNT(*) as c FROM vec_artifacts_chunks").fetchone()["c"]
+
+        rows = self.db.execute(
+            "SELECT id, text, rowid FROM artifacts WHERE active=1"
+        ).fetchall()
+        texts = [r["text"] for r in rows]
+        rowids = [r["rowid"] for r in rows]
+
+        if texts:
+            vectors = embedder.embed(texts)
+        else:
+            vectors = []
+
+        self.db.execute("DROP TABLE IF EXISTS vec_artifacts")
+        chunk_size = _choose_chunk_size(len(rows))
+        self.db.execute(
+            f"CREATE VIRTUAL TABLE vec_artifacts USING vec0("
+            f"embedding float[{self.dim}], chunk_size={chunk_size})")
+
+        cur = self.db.cursor()
+        for rowid, vec in zip(rowids, vectors):
+            cur.execute("INSERT INTO vec_artifacts(rowid, embedding) VALUES(?,?)",
+                        (rowid, _serialize(vec)))
+        self.db.commit()
+
+        if vacuum:
+            self.db.execute("VACUUM")
+
+        chunk_count_after = self.db.execute(
+            "SELECT COUNT(*) as c FROM vec_artifacts_chunks").fetchone()["c"]
+        elapsed = round((_time.time() - start) * 1000, 1)
+
+        return {
+            "before_chunks": chunk_count_before,
+            "after_chunks": chunk_count_after,
+            "vectors_reindexed": len(rows),
+            "chunk_size": chunk_size,
+            "duration_ms": elapsed,
+        }
+
     def search_lexical(self, query: str, scope: Scope, k: int) -> list[tuple[Artifact, float]]:
         """Lexical BM25 search over artifact text via FTS5. Returns
         (artifact, bm25_score) ordered best-first (lower bm25 = better match).
@@ -225,7 +281,15 @@ class SqliteStore:
         """, (*ids, *types, hops, *types)).fetchall()
         return [self._row_to_artifact(r) for r in rows]
 
+    def _deactivate_artifact(self, artifact_id: str) -> None:
+        row = self.db.execute(
+            "SELECT rowid FROM artifacts WHERE id=?", (artifact_id,)).fetchone()
+        if row:
+            self.db.execute("DELETE FROM vec_artifacts WHERE rowid=?", (row[0],))
+            self.db.execute("DELETE FROM fts_artifacts WHERE id=?", (artifact_id,))
+
     def deactivate(self, artifact_id: str, superseded_by: str) -> None:
+        self._deactivate_artifact(artifact_id)
         self.db.execute("UPDATE artifacts SET active=0, superseded_by=? WHERE id=?",
                         (superseded_by, artifact_id))
         self.add_edge(superseded_by, artifact_id, "supersedes")
@@ -405,6 +469,7 @@ class SqliteStore:
         for r in rows:
             new_score = round(r["quality_score"] * factor, 4)
             if new_score < deactivate_floor:
+                self._deactivate_artifact(r["artifact_id"])
                 self.db.execute("UPDATE artifacts SET active=0 WHERE id=?",
                                 (r["artifact_id"],))
             self.db.execute(
@@ -417,6 +482,7 @@ class SqliteStore:
     def deactivate_stale(self, days: int = 30) -> int:
         ids = self.get_stale_memories(days)
         for aid in ids:
+            self._deactivate_artifact(aid)
             self.db.execute("UPDATE artifacts SET active=0 WHERE id=?", (aid,))
         self.db.commit()
         return len(ids)

{memor_cli-0.6.0 → memor_cli-0.6.1/memor_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memor-cli
-Version: 0.6.0
+Version: 0.6.1
 Summary: Measured memory for coding agents. Fire and forget — no API keys needed.
 Author-email: Nimit Bhandari <nimitbhandari17@gmail.com>
 License-Expression: MIT

{memor_cli-0.6.0 → memor_cli-0.6.1}/memor_cli.egg-info/SOURCES.txt

@@ -98,4 +98,5 @@ tests/test_store.py
 tests/test_supersession.py
 tests/test_tokencount.py
 tests/test_turn_metrics.py
-tests/test_types.py
+tests/test_types.py
+tests/test_vec_compaction.py

{memor_cli-0.6.0 → memor_cli-0.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "memor-cli"
-version = "0.6.0"
+version = "0.6.1"
 description = "Measured memory for coding agents. Fire and forget — no API keys needed."
 readme = "README.md"
 license = "MIT"

memor_cli-0.6.1/tests/test_vec_compaction.py

@@ -0,0 +1,202 @@
+"""Tests for vec storage bloat fix: deactivation cleanup, rebuild, auto-compact."""
+from memor.store.sqlite_store import SqliteStore, _serialize
+from memor.embed.fake import FakeEmbedder
+from memor.types import Artifact
+
+
+def _make_store(tmp_path, n=5):
+    """Create a store with n artifacts and return (store, embedder, artifacts)."""
+    db_path = str(tmp_path / "test.db")
+    e = FakeEmbedder(dim=16)
+    s = SqliteStore(db_path, dim=16)
+    arts = []
+    for i in range(n):
+        a = Artifact(
+            id=f"art-{i}", kind="memory", project="proj", source="distill",
+            text=f"memory about topic {i}", token_count=5,
+            created_at=100.0 + i, meta={"mem_type": "decision", "session_id": "s1"})
+        arts.append(a)
+    vecs = e.embed([a.text for a in arts])
+    s.add_artifacts(arts, vecs)
+    return s, e, arts
+
+
+def test_deactivate_cleans_vec_and_fts(tmp_path):
+    s, e, arts = _make_store(tmp_path)
+    vec_count_before = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    fts_count_before = s.db.execute("SELECT COUNT(*) as c FROM fts_artifacts").fetchone()["c"]
+    assert vec_count_before == 5
+    assert fts_count_before == 5
+
+    s.deactivate("art-2", superseded_by="art-3")
+
+    vec_count_after = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    fts_count_after = s.db.execute("SELECT COUNT(*) as c FROM fts_artifacts").fetchone()["c"]
+    assert vec_count_after == 4
+    assert fts_count_after == 4
+
+    row = s.db.execute("SELECT active FROM artifacts WHERE id='art-2'").fetchone()
+    assert row["active"] == 0
+
+
+def test_deactivate_stale_cleans_vec(tmp_path):
+    s, e, arts = _make_store(tmp_path)
+    vec_before = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert vec_before == 5
+
+    count = s.deactivate_stale(days=0)
+    assert count > 0
+
+    vec_after = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert vec_after < vec_before
+
+
+def test_decay_quality_cleans_vec(tmp_path):
+    s, e, arts = _make_store(tmp_path)
+    for a in arts[:2]:
+        s.db.execute(
+            "INSERT INTO memory_quality(artifact_id, recall_count, use_count, quality_score, last_recalled) "
+            "VALUES(?, 10, 0, 0.02, ?)", (a.id, 1.0))
+    s.db.commit()
+
+    vec_before = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert vec_before == 5
+
+    decayed = s.decay_quality(stale_days=0, factor=0.5, deactivate_floor=0.03)
+    assert decayed >= 2
+
+    vec_after = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert vec_after < vec_before
+
+
+def test_add_artifacts_re_add_no_extra_chunks(tmp_path):
+    """Re-adding the same artifact should not leak extra vec0 chunk slots."""
+    s, e, arts = _make_store(tmp_path, n=3)
+    rowids_before = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert rowids_before == 3
+
+    vecs = e.embed([a.text for a in arts])
+    s.add_artifacts(arts, vecs)
+
+    rowids_after = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert rowids_after == 3
+
+
+def test_rebuild_vec_index(tmp_path):
+    s, e, arts = _make_store(tmp_path, n=10)
+    for a in arts[:5]:
+        s.deactivate(a.id, superseded_by=arts[5].id)
+
+    active_before = s.db.execute(
+        "SELECT COUNT(*) as c FROM artifacts WHERE active=1").fetchone()["c"]
+    assert active_before == 5
+
+    result = s.rebuild_vec_index(e)
+    assert result["vectors_reindexed"] == 5
+
+    vec_count = s.db.execute("SELECT COUNT(*) as c FROM vec_artifacts_rowids").fetchone()["c"]
+    assert vec_count == 5
+
+    from memor.types import Scope
+    hits = s.search(e.embed(["topic 7"])[0], Scope(project="proj"), k=3)
+    assert len(hits) > 0
+
+
+def test_rebuild_chunk_size_selection(tmp_path):
+    from memor.store.sqlite_store import _choose_chunk_size
+    assert _choose_chunk_size(500) == 64
+    assert _choose_chunk_size(1000) == 256
+    assert _choose_chunk_size(5000) == 256
+    assert _choose_chunk_size(10000) == 512
+    assert _choose_chunk_size(50000) == 512
+    assert _choose_chunk_size(100000) == 1024
+
+
+def test_compact_cli_command(tmp_path):
+    from typer.testing import CliRunner
+    from memor.cli import app
+
+    db_path = str(tmp_path / "test.db")
+    e = FakeEmbedder(dim=16)
+    s = SqliteStore(db_path, dim=16)
+    arts = [Artifact(id=f"a{i}", kind="memory", project="p", source="distill",
+                     text=f"text {i}", token_count=3, created_at=100.0 + i,
+                     meta={"mem_type": "decision"}) for i in range(3)]
+    s.add_artifacts(arts, e.embed([a.text for a in arts]))
+
+    runner = CliRunner()
+    result = runner.invoke(app, ["compact", "--db", db_path, "--fake", "--yes"])
+    assert result.exit_code == 0
+    assert "reindexed" in result.output.lower()
+
+
+def test_auto_compact_skips_when_healthy(tmp_path):
+    from memor.daemon import should_compact
+    db_path = str(tmp_path / "test.db")
+    e = FakeEmbedder(dim=16)
+    s = SqliteStore(db_path, dim=16)
+    arts = [Artifact(id=f"a{i}", kind="memory", project="p", source="distill",
+                     text=f"text {i}", token_count=3, created_at=100.0 + i,
+                     meta={"mem_type": "decision"}) for i in range(3)]
+    s.add_artifacts(arts, e.embed([a.text for a in arts]))
+
+    assert should_compact(s) is False
+
+
+def test_auto_compact_returns_none_when_healthy(tmp_path):
+    from memor.daemon import auto_compact
+    db_path = str(tmp_path / "test.db")
+    e = FakeEmbedder(dim=16)
+    s = SqliteStore(db_path, dim=16)
+    arts = [Artifact(id=f"a{i}", kind="memory", project="p", source="distill",
+                     text=f"text {i}", token_count=3, created_at=100.0 + i,
+                     meta={"mem_type": "decision"}) for i in range(3)]
+    s.add_artifacts(arts, e.embed([a.text for a in arts]))
+
+    assert auto_compact(s, e) is None
+
+
+def test_dashboard_reuses_store(tmp_path):
+    db_path = str(tmp_path / "test.db")
+    e = FakeEmbedder(dim=16)
+    s = SqliteStore(db_path, dim=16)
+    a = Artifact(id="a1", kind="memory", project="p", source="distill",
+                 text="test", token_count=1, created_at=100.0, meta={})
+    s.add_artifacts([a], e.embed(["test"]))
+
+    from memor.dashboard.server import create_app
+    from starlette.testclient import TestClient
+
+    app = create_app(db_path)
+    client = TestClient(app)
+
+    r1 = client.get("/api/health")
+    r2 = client.get("/api/health")
+    assert r1.status_code == 200
+    assert r2.status_code == 200
+
+
+def test_dashboard_projects_hides_inactive(tmp_path):
+    """Projects with only raw chunks and no recalls/memories should be hidden."""
+    db_path = str(tmp_path / "test.db")
+    e = FakeEmbedder(dim=16)
+    s = SqliteStore(db_path, dim=16)
+
+    active_art = Artifact(id="m1", kind="memory", project="active-proj", source="distill",
+                          text="a distilled memory", token_count=5, created_at=100.0, meta={})
+    inactive_art = Artifact(id="c1", kind="session_chunk", project="dormant-proj", source="ingest",
+                            text="raw chunk only", token_count=10, created_at=100.0, meta={})
+    s.add_artifacts([active_art, inactive_art], e.embed(["a distilled memory", "raw chunk only"]))
+
+    from memor.dashboard.server import create_app
+    from starlette.testclient import TestClient
+
+    app = create_app(db_path)
+    client = TestClient(app)
+
+    resp = client.get("/api/projects")
+    assert resp.status_code == 200
+    projects = resp.json()
+    project_names = [p["project"] for p in projects]
+    assert "active-proj" in project_names
+    assert "dormant-proj" not in project_names

memor_cli-0.6.0/memor/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.6.0"