embedsync 0.4.0__tar.gz → 0.5.0__tar.gz

{embedsync-0.4.0 → embedsync-0.5.0}/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.5.0] - 2026-09-14
+
+### Added
+- `--full-reindex` on `run` / `plan`: force re-embed all current docs (ignore content-hash equality); removed docs still DELETE
+
 ## [0.4.0] - 2026-09-14
 
 ### Added

{embedsync-0.4.0 → embedsync-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: embedsync
-Version: 0.4.0
+Version: 0.5.0
 Summary: Incremental synchronization between source documents and vector indexes
 Project-URL: Homepage, https://github.com/yashshah9/embedsync
 Project-URL: Repository, https://github.com/yashshah9/embedsync
@@ -29,7 +29,7 @@ Incremental synchronization between **source documents** and **vector indexes**
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![CI](https://github.com/yashshah9/embedsync/actions/workflows/ci.yml/badge.svg)](https://github.com/yashshah9/embedsync/actions/workflows/ci.yml)
 
-> **Status:** v0.4 — hash + Ollama embedders, paragraph chunks, JSONL destination, chunk-level re-embed.
+> **Status:** v0.5 — hash + Ollama embedders, paragraph chunks, JSONL destination, chunk-level re-embed, `--full-reindex`.
 
 ## 60-second try
 
@@ -84,6 +84,7 @@ embedsync health
 embedsync plan examples/docs --state-db /tmp/embedsync-demo.db
 embedsync run examples/docs --dry-run --state-db /tmp/embedsync-demo.db
 embedsync run examples/docs --embedder hash --destination memory --state-db /tmp/embedsync-demo.db
+embedsync run examples/docs --full-reindex --embedder hash --destination jsonl:/tmp/index.jsonl
 embedsync run examples/docs --embedder hash --destination jsonl:/tmp/index.jsonl
 # Requires a running Ollama with an embedding model:
 embedsync run examples/docs --embedder ollama --destination jsonl:/tmp/index.jsonl

{embedsync-0.4.0 → embedsync-0.5.0}/README.md

@@ -6,7 +6,7 @@ Incremental synchronization between **source documents** and **vector indexes**
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![CI](https://github.com/yashshah9/embedsync/actions/workflows/ci.yml/badge.svg)](https://github.com/yashshah9/embedsync/actions/workflows/ci.yml)
 
-> **Status:** v0.4 — hash + Ollama embedders, paragraph chunks, JSONL destination, chunk-level re-embed.
+> **Status:** v0.5 — hash + Ollama embedders, paragraph chunks, JSONL destination, chunk-level re-embed, `--full-reindex`.
 
 ## 60-second try
 
@@ -61,6 +61,7 @@ embedsync health
 embedsync plan examples/docs --state-db /tmp/embedsync-demo.db
 embedsync run examples/docs --dry-run --state-db /tmp/embedsync-demo.db
 embedsync run examples/docs --embedder hash --destination memory --state-db /tmp/embedsync-demo.db
+embedsync run examples/docs --full-reindex --embedder hash --destination jsonl:/tmp/index.jsonl
 embedsync run examples/docs --embedder hash --destination jsonl:/tmp/index.jsonl
 # Requires a running Ollama with an embedding model:
 embedsync run examples/docs --embedder ollama --destination jsonl:/tmp/index.jsonl

{embedsync-0.4.0 → embedsync-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "embedsync"
-version = "0.4.0"
+version = "0.5.0"
 description = "Incremental synchronization between source documents and vector indexes"
 readme = "README.md"
 license = "MIT"

{embedsync-0.4.0 → embedsync-0.5.0}/src/embedsync/__init__.py

@@ -1,3 +1,3 @@
 """Incremental vector index synchronization."""
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"

{embedsync-0.4.0 → embedsync-0.5.0}/src/embedsync/cli.py

@@ -46,11 +46,12 @@ def _destination(dest_spec: str) -> MemoryDestination | JsonlDestination:
 @main.command("plan")
 @click.argument("source_dir", type=click.Path(exists=True, file_okay=False, path_type=Path))
 @click.option("--state-db", default=None, help="SQLite state database path")
-def plan_cmd(source_dir: Path, state_db: str | None) -> None:
+@click.option("--full-reindex", is_flag=True, help="Treat all current docs as updates")
+def plan_cmd(source_dir: Path, state_db: str | None, full_reindex: bool) -> None:
     store, path = _store(state_db)
     try:
         source = LocalFileSource(source_dir)
-        sync_plan = plan_sync(source, store)
+        sync_plan = plan_sync(source, store, full_reindex=full_reindex)
     except ValueError as exc:
         console.print(f"[red]Error:[/red] {exc}")
         raise SystemExit(2) from exc
@@ -76,12 +77,14 @@ def plan_cmd(source_dir: Path, state_db: str | None) -> None:
 @click.option("--state-db", default=None)
 @click.option("--embedder", default="hash")
 @click.option("--destination", "dest_spec", default="memory", help="memory | jsonl:/path")
+@click.option("--full-reindex", is_flag=True, help="Force re-embed all current docs")
 def run_cmd(
     source_dir: Path,
     dry_run: bool,
     state_db: str | None,
     embedder: str,
     dest_spec: str,
+    full_reindex: bool,
 ) -> None:
     store, path = _store(state_db)
     try:
@@ -92,6 +95,7 @@ def run_cmd(
             destination=dest,
             dry_run=dry_run,
             embedder=resolve_embedder(embedder),
+            full_reindex=full_reindex,
         )
     except (ValueError, click.UsageError) as exc:
         store.close()

{embedsync-0.4.0 → embedsync-0.5.0}/src/embedsync/sync/engine.py

@@ -5,7 +5,12 @@ from dataclasses import dataclass, field
 import structlog
 
 from embedsync.chunking import chunk_document
-from embedsync.destinations.memory import Destination, DestinationReport, MemoryDestination, SyncAction
+from embedsync.destinations.memory import (
+    Destination,
+    DestinationReport,
+    MemoryDestination,
+    SyncAction,
+)
 from embedsync.embedders import Embedder, HashEmbedder
 from embedsync.sources.local import LocalFileSource
 from embedsync.state.store import DocumentState, StateStore, content_hash
@@ -24,7 +29,11 @@ class SyncPlan:
         return len(self.adds) + len(self.updates) + len(self.deletes)
 
 
-def plan_sync(source: LocalFileSource, store: StateStore) -> SyncPlan:
+def plan_sync(
+    source: LocalFileSource,
+    store: StateStore,
+    full_reindex: bool = False,
+) -> SyncPlan:
     """Compute add/update/delete plan without touching the destination."""
     plan = SyncPlan()
     current_ids: set[str] = set()
@@ -43,7 +52,7 @@ def plan_sync(source: LocalFileSource, store: StateStore) -> SyncPlan:
         )
         if existing is None:
             plan.adds.append(action)
-        elif existing.content_hash != digest:
+        elif full_reindex or existing.content_hash != digest:
             action.action = "update"
             plan.updates.append(action)
 
@@ -60,19 +69,27 @@ def execute_sync(
     destination: Destination | None = None,
     dry_run: bool = False,
     embedder: Embedder | None = None,
+    full_reindex: bool = False,
 ) -> DestinationReport:
     dest: Destination = destination or MemoryDestination()
     encoder = embedder or HashEmbedder()
-    plan = plan_sync(source, store)
+    plan = plan_sync(source, store, full_reindex=full_reindex)
     report = DestinationReport()
     docs = {d.doc_id: d for d in source.list_documents()}
 
     for action in plan.adds + plan.updates:
         old = store.chunks_for(action.doc_id)
         new_hashes = {chunk.chunk_id: content_hash(chunk.content) for chunk in action.chunks}
-        changed = [chunk for chunk in action.chunks if old.get(chunk.chunk_id) != new_hashes[chunk.chunk_id]]
-        removed = [chunk_id for chunk_id in old if chunk_id not in new_hashes]
-        write_chunks = action.chunks if action.action == "add" else changed
+        if full_reindex and action.action == "update":
+            # Force DELETE old chunks + ADD fresh embeddings, ignore content-hash equality.
+            write_chunks = action.chunks
+            removed = list(old.keys())
+        else:
+            changed = [
+                chunk for chunk in action.chunks if old.get(chunk.chunk_id) != new_hashes[chunk.chunk_id]
+            ]
+            removed = [chunk_id for chunk_id in old if chunk_id not in new_hashes]
+            write_chunks = action.chunks if action.action == "add" else changed
         texts = [c.content for c in write_chunks]
         vectors = encoder.embed(texts) if texts else []
         dest_action = SyncAction(

{embedsync-0.4.0 → embedsync-0.5.0}/tests/test_embedders.py

@@ -1,8 +1,8 @@
 """Tests for embedders."""
 
 import json
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 from embedsync.embedders import HashEmbedder, OllamaEmbedder, resolve_embedder
 
@@ -23,7 +23,7 @@ def test_resolve_ollama_model_name() -> None:
 
 def test_ollama_embedder_against_mock_server() -> None:
     class Handler(BaseHTTPRequestHandler):
-        def do_POST(self) -> None:  # noqa: N802
+        def do_POST(self) -> None:
             length = int(self.headers.get("Content-Length", "0"))
             body = json.loads(self.rfile.read(length))
             assert body["model"] == "nomic-embed-text"
@@ -34,7 +34,7 @@ def test_ollama_embedder_against_mock_server() -> None:
             self.end_headers()
             self.wfile.write(payload)
 
-        def log_message(self, format: str, *args: object) -> None:  # noqa: A003
+        def log_message(self, format: str, *args: object) -> None:
             return
 
     httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)

{embedsync-0.4.0 → embedsync-0.5.0}/tests/test_engine.py

@@ -30,6 +30,19 @@ def test_run_indexes_documents(tmp_path: Path) -> None:
     store.close()
 
 
+def test_full_reindex_rewrites_unchanged_content(tmp_path: Path) -> None:
+    store = StateStore(str(tmp_path / "state.db"))
+    source = LocalFileSource(DOCS)
+    dest = MemoryDestination()
+    first = execute_sync(source, store, dest)
+    assert first.embeddings_written > 0
+    second = execute_sync(source, store, dest)
+    assert second.embeddings_written == 0
+    third = execute_sync(source, store, dest, full_reindex=True)
+    assert third.embeddings_written == first.embeddings_written
+    store.close()
+
+
 def test_update_reembeds_changed_chunks_only(tmp_path: Path) -> None:
     docs = tmp_path / "docs"
     docs.mkdir()

embedsync-0.5.0/tests/test_scenarios_v05.py

@@ -0,0 +1,196 @@
+"""Scenario tests for embedsync 0.5 behavior."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+import pytest
+from click.testing import CliRunner
+
+from embedsync.cli import _destination, main
+from embedsync.destinations.jsonl import JsonlDestination
+from embedsync.destinations.memory import MemoryDestination
+from embedsync.embedders import HashEmbedder, OllamaEmbedder, resolve_embedder
+from embedsync.sources.local import LocalFileSource
+from embedsync.state.store import StateStore
+from embedsync.sync.engine import execute_sync, plan_sync
+
+DOCS = Path(__file__).parent.parent / "examples" / "docs"
+
+
+def _docs(tmp_path: Path, files: dict[str, str]) -> Path:
+    root = tmp_path / "docs"
+    root.mkdir()
+    for name, text in files.items():
+        path = root / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    return root
+
+
+def test_full_reindex_forces_rewrite_when_unchanged(tmp_path: Path) -> None:
+    store = StateStore(str(tmp_path / "state.db"))
+    source = LocalFileSource(DOCS)
+    dest = MemoryDestination()
+    first = execute_sync(source, store, dest)
+    assert first.embeddings_written > 0
+    assert execute_sync(source, store, dest).embeddings_written == 0
+    third = execute_sync(source, store, dest, full_reindex=True)
+    assert third.embeddings_written == first.embeddings_written
+    store.close()
+
+
+def test_plan_adds_new_documents(tmp_path: Path) -> None:
+    docs = _docs(tmp_path, {"a.md": "alpha content here"})
+    store = StateStore(str(tmp_path / "state.db"))
+    plan = plan_sync(LocalFileSource(docs), store)
+    assert len(plan.adds) == 1
+    assert plan.adds[0].doc_id == "a.md"
+    assert plan.updates == []
+    assert plan.deletes == []
+    store.close()
+
+
+def test_plan_updates_changed_documents(tmp_path: Path) -> None:
+    docs = _docs(tmp_path, {"a.md": "original text"})
+    store = StateStore(str(tmp_path / "state.db"))
+    execute_sync(LocalFileSource(docs), store, MemoryDestination())
+    (docs / "a.md").write_text("changed text", encoding="utf-8")
+    plan = plan_sync(LocalFileSource(docs), store)
+    assert plan.adds == []
+    assert len(plan.updates) == 1
+    assert plan.updates[0].doc_id == "a.md"
+    store.close()
+
+
+def test_plan_deletes_removed_documents(tmp_path: Path) -> None:
+    docs = _docs(tmp_path, {"keep.md": "keep", "gone.md": "gone"})
+    store = StateStore(str(tmp_path / "state.db"))
+    execute_sync(LocalFileSource(docs), store, MemoryDestination())
+    (docs / "gone.md").unlink()
+    plan = plan_sync(LocalFileSource(docs), store)
+    assert len(plan.deletes) == 1
+    assert plan.deletes[0].doc_id == "gone.md"
+    store.close()
+
+
+def test_plan_full_reindex_marks_existing_as_updates(tmp_path: Path) -> None:
+    docs = _docs(tmp_path, {"a.md": "stable"})
+    store = StateStore(str(tmp_path / "state.db"))
+    execute_sync(LocalFileSource(docs), store, MemoryDestination())
+    plan = plan_sync(LocalFileSource(docs), store, full_reindex=True)
+    assert plan.adds == []
+    assert len(plan.updates) == 1
+    store.close()
+
+
+def test_idempotent_run_writes_zero_on_second_pass(tmp_path: Path) -> None:
+    store = StateStore(str(tmp_path / "state.db"))
+    source = LocalFileSource(DOCS)
+    dest = MemoryDestination()
+    first = execute_sync(source, store, dest)
+    second = execute_sync(source, store, dest)
+    assert first.embeddings_written > 0
+    assert second.embeddings_written == 0
+    assert second.actions == []
+    store.close()
+
+
+def test_hash_embedder_dimension() -> None:
+    emb = HashEmbedder(dimension=16)
+    vectors = emb.embed(["hello", "world"])
+    assert emb.dimension == 16
+    assert len(vectors) == 2
+    assert all(len(v) == 16 for v in vectors)
+
+
+def test_hash_embedder_stability() -> None:
+    emb = HashEmbedder(dimension=8)
+    a = emb.embed(["stable text"])[0]
+    b = emb.embed(["stable text"])[0]
+    c = emb.embed(["different"])[0]
+    assert a == b
+    assert a != c
+
+
+def test_resolve_ollama_default_model() -> None:
+    emb = resolve_embedder("ollama")
+    assert isinstance(emb, OllamaEmbedder)
+    assert emb.model == "nomic-embed-text"
+
+
+def test_resolve_ollama_colon_model() -> None:
+    emb = resolve_embedder("ollama:mxbai-embed-large")
+    assert isinstance(emb, OllamaEmbedder)
+    assert emb.model == "mxbai-embed-large"
+
+
+def test_bad_destination_raises() -> None:
+    with pytest.raises(click.UsageError, match="memory.*jsonl"):
+        _destination("pgvector://localhost")
+
+
+def test_empty_source_dir_cli_error(tmp_path: Path) -> None:
+    empty = tmp_path / "empty"
+    empty.mkdir()
+    runner = CliRunner()
+    # Empty dir is valid for click Path, but plan reports zero actions (no docs).
+    result = runner.invoke(
+        main,
+        ["plan", str(empty), "--state-db", str(tmp_path / "state.db")],
+    )
+    assert result.exit_code == 0
+    assert "Total: 0 action(s)" in result.output
+
+
+def test_missing_source_dir_cli_error(tmp_path: Path) -> None:
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        ["plan", str(tmp_path / "does-not-exist"), "--state-db", str(tmp_path / "state.db")],
+    )
+    assert result.exit_code != 0
+
+
+def test_bad_destination_cli_error(tmp_path: Path) -> None:
+    docs = _docs(tmp_path, {"a.md": "hi"})
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "run",
+            str(docs),
+            "--state-db",
+            str(tmp_path / "state.db"),
+            "--destination",
+            "bogus",
+        ],
+    )
+    assert result.exit_code != 0
+
+
+def test_jsonl_destination_write_roundtrip(tmp_path: Path) -> None:
+    docs = _docs(tmp_path, {"note.md": "jsonl roundtrip content"})
+    out = tmp_path / "out" / "embeddings.jsonl"
+    store = StateStore(str(tmp_path / "state.db"))
+    dest = JsonlDestination(out)
+    report = execute_sync(LocalFileSource(docs), store, dest)
+    assert report.embeddings_written > 0
+    assert out.exists()
+    rows = [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines() if line.strip()]
+    assert len(rows) == report.embeddings_written
+    assert all("embedding" in r and "chunk_id" in r and r["doc_id"] == "note.md" for r in rows)
+    store.close()
+
+
+def test_dry_run_writes_zero_embeddings(tmp_path: Path) -> None:
+    store = StateStore(str(tmp_path / "state.db"))
+    dest = MemoryDestination()
+    report = execute_sync(LocalFileSource(DOCS), store, dest, dry_run=True)
+    assert report.embeddings_written == 0
+    assert dest.indexed == {}
+    assert dest.vectors == {}
+    assert store.all_ids() == set()
+    store.close()