embedsync 0.4.0__py3-none-any.whl

embedsync/__init__.py

@@ -0,0 +1,3 @@
+"""Incremental vector index synchronization."""
+
+__version__ = "0.4.0"

embedsync/chunking.py

@@ -0,0 +1,39 @@
+"""Stable-ish chunking for documents."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+
+@dataclass
+class Chunk:
+    chunk_id: str
+    doc_id: str
+    content: str
+    index: int
+
+
+def chunk_document(doc_id: str, content: str, size: int = 500) -> list[Chunk]:
+    """Split on paragraph boundaries, then pad to size.
+
+    ponytail: hash of (doc_id, index, first 64 chars) — upgrade to simhash merge in M4.
+    """
+    paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()]
+    if not paragraphs:
+        paragraphs = [content]
+    pieces: list[str] = []
+    buf = ""
+    for para in paragraphs:
+        if buf and len(buf) + len(para) > size:
+            pieces.append(buf)
+            buf = para
+        else:
+            buf = f"{buf}\n\n{para}".strip() if buf else para
+    if buf:
+        pieces.append(buf)
+    chunks: list[Chunk] = []
+    for index, text in enumerate(pieces):
+        digest = hashlib.sha256(f"{doc_id}:{index}:{text[:64]}".encode()).hexdigest()[:16]
+        chunks.append(Chunk(chunk_id=digest, doc_id=doc_id, content=text, index=index))
+    return chunks

embedsync/cli.py

@@ -0,0 +1,111 @@
+"""CLI for embedsync."""
+
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from embedsync import __version__
+from embedsync.config import Settings
+from embedsync.destinations.jsonl import JsonlDestination
+from embedsync.destinations.memory import MemoryDestination
+from embedsync.embedders import resolve_embedder
+from embedsync.sources.local import LocalFileSource
+from embedsync.state.store import StateStore
+from embedsync.sync.engine import execute_sync, plan_sync
+
+console = Console()
+
+
+@click.group()
+@click.version_option(__version__)
+def main() -> None:
+    """Incremental sync between documents and vector indexes."""
+
+
+@main.command("health")
+def health() -> None:
+    console.print(f"[green]embedsync {__version__} OK[/green]")
+
+
+def _store(state_db: str | None) -> tuple[StateStore, str]:
+    settings = Settings()
+    path = state_db or settings.state_db
+    return StateStore(path), path
+
+
+def _destination(dest_spec: str) -> MemoryDestination | JsonlDestination:
+    if dest_spec == "memory":
+        return MemoryDestination()
+    if dest_spec.startswith("jsonl:"):
+        return JsonlDestination(Path(dest_spec.split(":", 1)[1]))
+    raise click.UsageError("destination must be 'memory' or 'jsonl:/path'")
+
+
+@main.command("plan")
+@click.argument("source_dir", type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.option("--state-db", default=None, help="SQLite state database path")
+def plan_cmd(source_dir: Path, state_db: str | None) -> None:
+    store, path = _store(state_db)
+    try:
+        source = LocalFileSource(source_dir)
+        sync_plan = plan_sync(source, store)
+    except ValueError as exc:
+        console.print(f"[red]Error:[/red] {exc}")
+        raise SystemExit(2) from exc
+
+    table = Table(title="Sync Plan (dry)")
+    table.add_column("Action")
+    table.add_column("Doc ID")
+    table.add_column("Chunks")
+    for action in sync_plan.adds:
+        table.add_row("[green]ADD[/green]", action.doc_id, str(action.chunk_count))
+    for action in sync_plan.updates:
+        table.add_row("[yellow]UPDATE[/yellow]", action.doc_id, str(action.chunk_count))
+    for action in sync_plan.deletes:
+        table.add_row("[red]DELETE[/red]", action.doc_id, "0")
+    console.print(table)
+    console.print(f"Total: {sync_plan.total} action(s)  state={path}")
+    store.close()
+
+
+@main.command("run")
+@click.argument("source_dir", type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.option("--dry-run", is_flag=True)
+@click.option("--state-db", default=None)
+@click.option("--embedder", default="hash")
+@click.option("--destination", "dest_spec", default="memory", help="memory | jsonl:/path")
+def run_cmd(
+    source_dir: Path,
+    dry_run: bool,
+    state_db: str | None,
+    embedder: str,
+    dest_spec: str,
+) -> None:
+    store, path = _store(state_db)
+    try:
+        dest = _destination(dest_spec)
+        report = execute_sync(
+            source=LocalFileSource(source_dir),
+            store=store,
+            destination=dest,
+            dry_run=dry_run,
+            embedder=resolve_embedder(embedder),
+        )
+    except (ValueError, click.UsageError) as exc:
+        store.close()
+        if isinstance(exc, click.UsageError):
+            raise
+        console.print(f"[red]Error:[/red] {exc}")
+        raise SystemExit(2) from exc
+    suffix = " (dry-run)" if dry_run else ""
+    console.print(
+        f"Applied {len(report.actions)} action(s), "
+        f"wrote {report.embeddings_written} embedding(s){suffix}  state={path}"
+    )
+    store.close()
+
+
+if __name__ == "__main__":
+    main()

embedsync/config.py

@@ -0,0 +1,11 @@
+"""Configuration."""
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="EMBEDSYNC_", env_file=".env", extra="ignore")
+
+    state_db: str = ".embedsync/state.db"
+    log_level: str = "INFO"
+    dry_run: bool = False

embedsync/destinations/jsonl.py

@@ -0,0 +1,55 @@
+"""JSONL destination — local stand-in until pgvector is wired."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from embedsync.destinations.memory import SyncAction
+
+
+class JsonlDestination:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        if not self.path.exists():
+            self.path.write_text("", encoding="utf-8")
+
+    def apply(self, action: SyncAction, embeddings: list[list[float]], dry_run: bool = False) -> None:
+        if dry_run:
+            return
+        rows = self._load()
+        if action.action == "delete":
+            rows = [r for r in rows if r.get("doc_id") != action.doc_id]
+        else:
+            if action.action == "add":
+                rows = [r for r in rows if r.get("doc_id") != action.doc_id]
+            else:
+                drop = set(action.removed_chunk_ids) | {c.chunk_id for c in action.chunks}
+                rows = [
+                    r
+                    for r in rows
+                    if not (r.get("doc_id") == action.doc_id and r.get("chunk_id") in drop)
+                ]
+            for chunk, vector in zip(action.chunks, embeddings, strict=False):
+                rows.append(
+                    {
+                        "chunk_id": chunk.chunk_id,
+                        "doc_id": action.doc_id,
+                        "content": chunk.content,
+                        "embedding": vector,
+                    }
+                )
+        self.path.write_text(
+            "\n".join(json.dumps(r) for r in rows) + ("\n" if rows else ""),
+            encoding="utf-8",
+        )
+
+    def _load(self) -> list[dict[object, object]]:
+        if not self.path.exists() or self.path.stat().st_size == 0:
+            return []
+        out: list[dict[object, object]] = []
+        for line in self.path.read_text(encoding="utf-8").splitlines():
+            if line.strip():
+                out.append(json.loads(line))
+        return out

embedsync/destinations/memory.py

@@ -0,0 +1,52 @@
+"""Destination protocol and in-memory implementation."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Protocol
+
+from embedsync.chunking import Chunk
+
+
+@dataclass
+class SyncAction:
+    action: str  # add | update | delete
+    doc_id: str
+    chunk_count: int = 0
+    chunks: list[Chunk] = field(default_factory=list)
+    removed_chunk_ids: list[str] = field(default_factory=list)
+
+
+@dataclass
+class DestinationReport:
+    actions: list[SyncAction] = field(default_factory=list)
+    embeddings_written: int = 0
+
+
+class Destination(Protocol):
+    def apply(self, action: SyncAction, embeddings: list[list[float]], dry_run: bool = False) -> None:
+        ...
+
+
+class MemoryDestination:
+    """In-memory destination for dry-run and testing."""
+
+    def __init__(self) -> None:
+        self.indexed: dict[str, int] = {}
+        self.vectors: dict[str, dict[str, list[float]]] = {}
+
+    def apply(self, action: SyncAction, embeddings: list[list[float]], dry_run: bool = False) -> None:
+        if dry_run:
+            return
+        if action.action == "delete":
+            self.indexed.pop(action.doc_id, None)
+            self.vectors.pop(action.doc_id, None)
+            return
+        bucket = self.vectors.setdefault(action.doc_id, {})
+        if action.action == "add":
+            bucket.clear()
+        for chunk_id in action.removed_chunk_ids:
+            bucket.pop(chunk_id, None)
+        for chunk, vector in zip(action.chunks, embeddings, strict=False):
+            bucket[chunk.chunk_id] = vector
+        self.indexed[action.doc_id] = action.chunk_count

embedsync/embedders.py

@@ -0,0 +1,88 @@
+"""Pluggable embedding functions."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import urllib.error
+import urllib.request
+from typing import Protocol
+
+
+class Embedder(Protocol):
+    dimension: int
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        ...
+
+
+class HashEmbedder:
+    """Deterministic embedding for tests and offline dry-runs (not semantic)."""
+
+    def __init__(self, dimension: int = 8) -> None:
+        self.dimension = dimension
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for text in texts:
+            digest = hashlib.sha256(text.encode()).digest()
+            raw = [(digest[i % len(digest)] / 255.0) * 2 - 1 for i in range(self.dimension)]
+            norm = math.sqrt(sum(x * x for x in raw)) or 1.0
+            vectors.append([x / norm for x in raw])
+        return vectors
+
+
+class OllamaEmbedder:
+    """Call a local Ollama /api/embeddings endpoint (stdlib urllib)."""
+
+    def __init__(
+        self,
+        model: str = "nomic-embed-text",
+        host: str | None = None,
+        dimension: int | None = None,
+        timeout: float = 60.0,
+    ) -> None:
+        self.model = model
+        self.host = (host or os.environ.get("OLLAMA_HOST") or "http://127.0.0.1:11434").rstrip("/")
+        self.dimension = dimension or 0
+        self.timeout = timeout
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for text in texts:
+            payload = json.dumps({"model": self.model, "prompt": text}).encode()
+            req = urllib.request.Request(
+                f"{self.host}/api/embeddings",
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    body = json.loads(resp.read().decode())
+            except urllib.error.URLError as exc:
+                raise RuntimeError(f"Ollama embed failed: {exc}") from exc
+            vector = body.get("embedding")
+            if not isinstance(vector, list) or not vector:
+                raise RuntimeError("Ollama response missing embedding vector")
+            if self.dimension and len(vector) != self.dimension:
+                raise RuntimeError(
+                    f"Ollama returned dim {len(vector)}, expected {self.dimension}"
+                )
+            if not self.dimension:
+                self.dimension = len(vector)
+            vectors.append([float(x) for x in vector])
+        return vectors
+
+
+def resolve_embedder(name: str) -> Embedder:
+    if name in {"hash", "test"}:
+        return HashEmbedder()
+    if name == "ollama" or name.startswith("ollama:"):
+        model = name.split(":", 1)[1] if ":" in name else "nomic-embed-text"
+        return OllamaEmbedder(model=model)
+    raise ValueError(
+        f"Unknown embedder '{name}'. Use hash, ollama, or ollama:<model>."
+    )

embedsync/sources/local.py

@@ -0,0 +1,34 @@
+"""Source document providers."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class SourceDocument:
+    doc_id: str
+    content: str
+    metadata: dict[str, str]
+
+
+class LocalFileSource:
+    """Read documents from a local directory."""
+
+    def __init__(self, directory: Path, glob: str = "**/*.md") -> None:
+        self.directory = directory
+        self.glob = glob
+
+    def list_documents(self) -> list[SourceDocument]:
+        docs: list[SourceDocument] = []
+        for path in self.directory.glob(self.glob):
+            if not path.is_file():
+                continue
+            rel = str(path.relative_to(self.directory))
+            docs.append(
+                SourceDocument(
+                    doc_id=rel,
+                    content=path.read_text(encoding="utf-8"),
+                    metadata={"path": rel},
+                )
+            )
+        return docs

embedsync/state/store.py

@@ -0,0 +1,93 @@
+"""Sync state persistence."""
+
+import hashlib
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class DocumentState:
+    doc_id: str
+    content_hash: str
+    chunk_count: int = 0
+
+
+class StateStore:
+    """SQLite-backed document state tracking."""
+
+    def __init__(self, db_path: str) -> None:
+        self.path = Path(db_path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(str(self.path))
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS documents (
+                doc_id TEXT PRIMARY KEY,
+                content_hash TEXT NOT NULL,
+                chunk_count INTEGER DEFAULT 0
+            )
+            """
+        )
+        self._conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS chunks (
+                chunk_id TEXT PRIMARY KEY,
+                doc_id TEXT NOT NULL,
+                content_hash TEXT NOT NULL
+            )
+            """
+        )
+        self._conn.commit()
+
+    def get(self, doc_id: str) -> DocumentState | None:
+        row = self._conn.execute(
+            "SELECT doc_id, content_hash, chunk_count FROM documents WHERE doc_id = ?",
+            (doc_id,),
+        ).fetchone()
+        if row is None:
+            return None
+        return DocumentState(doc_id=row[0], content_hash=row[1], chunk_count=row[2])
+
+    def upsert(self, state: DocumentState) -> None:
+        self._conn.execute(
+            """
+            INSERT INTO documents (doc_id, content_hash, chunk_count)
+            VALUES (?, ?, ?)
+            ON CONFLICT(doc_id) DO UPDATE SET content_hash=excluded.content_hash,
+                chunk_count=excluded.chunk_count
+            """,
+            (state.doc_id, state.content_hash, state.chunk_count),
+        )
+        self._conn.commit()
+
+    def delete(self, doc_id: str) -> None:
+        self._conn.execute("DELETE FROM chunks WHERE doc_id = ?", (doc_id,))
+        self._conn.execute("DELETE FROM documents WHERE doc_id = ?", (doc_id,))
+        self._conn.commit()
+
+    def chunks_for(self, doc_id: str) -> dict[str, str]:
+        rows = self._conn.execute(
+            "SELECT chunk_id, content_hash FROM chunks WHERE doc_id = ?",
+            (doc_id,),
+        ).fetchall()
+        return {r[0]: r[1] for r in rows}
+
+    def replace_chunks(self, doc_id: str, chunks: list[tuple[str, str]]) -> None:
+        self._conn.execute("DELETE FROM chunks WHERE doc_id = ?", (doc_id,))
+        self._conn.executemany(
+            "INSERT INTO chunks (chunk_id, doc_id, content_hash) VALUES (?, ?, ?)",
+            [(chunk_id, doc_id, digest) for chunk_id, digest in chunks],
+        )
+        self._conn.commit()
+
+    def all_ids(self) -> set[str]:
+        rows = self._conn.execute("SELECT doc_id FROM documents").fetchall()
+        return {r[0] for r in rows}
+
+    def close(self) -> None:
+        self._conn.close()
+
+
+def content_hash(text: str) -> str:
+    return hashlib.sha256(text.encode()).hexdigest()

embedsync/sync/engine.py

@@ -0,0 +1,107 @@
+"""Core sync engine — diff source against state, plan actions, embed deltas."""
+
+from dataclasses import dataclass, field
+
+import structlog
+
+from embedsync.chunking import chunk_document
+from embedsync.destinations.memory import Destination, DestinationReport, MemoryDestination, SyncAction
+from embedsync.embedders import Embedder, HashEmbedder
+from embedsync.sources.local import LocalFileSource
+from embedsync.state.store import DocumentState, StateStore, content_hash
+
+log = structlog.get_logger()
+
+
+@dataclass
+class SyncPlan:
+    adds: list[SyncAction] = field(default_factory=list)
+    updates: list[SyncAction] = field(default_factory=list)
+    deletes: list[SyncAction] = field(default_factory=list)
+
+    @property
+    def total(self) -> int:
+        return len(self.adds) + len(self.updates) + len(self.deletes)
+
+
+def plan_sync(source: LocalFileSource, store: StateStore) -> SyncPlan:
+    """Compute add/update/delete plan without touching the destination."""
+    plan = SyncPlan()
+    current_ids: set[str] = set()
+    docs = {doc.doc_id: doc for doc in source.list_documents()}
+
+    for doc_id, doc in docs.items():
+        current_ids.add(doc_id)
+        digest = content_hash(doc.content)
+        existing = store.get(doc_id)
+        chunks = chunk_document(doc_id, doc.content)
+        action = SyncAction(
+            "add" if existing is None else "update",
+            doc_id,
+            chunk_count=len(chunks),
+            chunks=chunks,
+        )
+        if existing is None:
+            plan.adds.append(action)
+        elif existing.content_hash != digest:
+            action.action = "update"
+            plan.updates.append(action)
+
+    for stale_id in store.all_ids() - current_ids:
+        plan.deletes.append(SyncAction("delete", stale_id))
+
+    log.info("sync_planned", adds=len(plan.adds), updates=len(plan.updates), deletes=len(plan.deletes))
+    return plan
+
+
+def execute_sync(
+    source: LocalFileSource,
+    store: StateStore,
+    destination: Destination | None = None,
+    dry_run: bool = False,
+    embedder: Embedder | None = None,
+) -> DestinationReport:
+    dest: Destination = destination or MemoryDestination()
+    encoder = embedder or HashEmbedder()
+    plan = plan_sync(source, store)
+    report = DestinationReport()
+    docs = {d.doc_id: d for d in source.list_documents()}
+
+    for action in plan.adds + plan.updates:
+        old = store.chunks_for(action.doc_id)
+        new_hashes = {chunk.chunk_id: content_hash(chunk.content) for chunk in action.chunks}
+        changed = [chunk for chunk in action.chunks if old.get(chunk.chunk_id) != new_hashes[chunk.chunk_id]]
+        removed = [chunk_id for chunk_id in old if chunk_id not in new_hashes]
+        write_chunks = action.chunks if action.action == "add" else changed
+        texts = [c.content for c in write_chunks]
+        vectors = encoder.embed(texts) if texts else []
+        dest_action = SyncAction(
+            action.action,
+            action.doc_id,
+            chunk_count=action.chunk_count,
+            chunks=write_chunks,
+            removed_chunk_ids=removed,
+        )
+        report.actions.append(action)
+        report.embeddings_written += 0 if dry_run else len(vectors)
+        dest.apply(dest_action, vectors, dry_run=dry_run)
+        if not dry_run:
+            store.upsert(
+                DocumentState(
+                    doc_id=action.doc_id,
+                    content_hash=content_hash(docs[action.doc_id].content),
+                    chunk_count=action.chunk_count,
+                )
+            )
+            store.replace_chunks(
+                action.doc_id,
+                [(chunk.chunk_id, content_hash(chunk.content)) for chunk in action.chunks],
+            )
+
+    for action in plan.deletes:
+        report.actions.append(action)
+        dest.apply(action, [], dry_run=dry_run)
+        if not dry_run:
+            store.delete(action.doc_id)
+
+    return report

embedsync-0.4.0.dist-info/METADATA

@@ -0,0 +1,128 @@
+Metadata-Version: 2.5
+Name: embedsync
+Version: 0.4.0
+Summary: Incremental synchronization between source documents and vector indexes
+Project-URL: Homepage, https://github.com/yashshah9/embedsync
+Project-URL: Repository, https://github.com/yashshah9/embedsync
+Project-URL: Issues, https://github.com/yashshah9/embedsync/issues
+Author-email: Yash Shah <yash376351@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: click>=8.1
+Requires-Dist: pydantic-settings>=2.2
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.7
+Requires-Dist: structlog>=24.1
+Provides-Extra: dev
+Requires-Dist: mypy>=1.9; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# embedsync
+
+Incremental synchronization between **source documents** and **vector indexes** — detect changes, re-embed only deltas, and delete stale chunks.
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![CI](https://github.com/yashshah9/embedsync/actions/workflows/ci.yml/badge.svg)](https://github.com/yashshah9/embedsync/actions/workflows/ci.yml)
+
+> **Status:** v0.4 — hash + Ollama embedders, paragraph chunks, JSONL destination, chunk-level re-embed.
+
+## 60-second try
+
+```bash
+docker compose run --rm plan  # plan sync for examples/docs
+docker compose run --rm test  # pytest
+```
+
+## Why this vs alternatives
+
+| Approach | Strength | Gap |
+|----------|----------|-----|
+| **embedsync** | Content-hash deltas + pluggable embedders | Destinations still local (JSONL/memory) |
+| Full re-embed pipelines | Simple mentally | Expensive; misses deletes |
+| Framework ingestion (e.g. LlamaIndex) | Rich connectors | Change detection is DIY |
+| One-off sync scripts | Fits one repo | No shared plan/state model |
+
+## Problem
+
+RAG indexes rot when documents change. Full re-embeds are expensive and miss deletes. Every team rebuilds change detection from scratch.
+
+## Key features (v0.4)
+
+- Content-hash change detection per document
+- Sync plan: add / update / delete actions
+- Hash embedder for offline/CI (`--embedder hash`)
+- Ollama embedder (`--embedder ollama` or `ollama:nomic-embed-text`)
+- JSONL or in-memory destination
+- Unchanged docs/chunks skip re-embedding on the next run
+
+## Architecture
+
+```
+embedsync run ./docs
+    ├── LocalFileSource
+    ├── StateStore (SQLite)
+    ├── plan_sync() → diff
+    └── Destination (Memory → pgvector next)
+```
+
+## Installation
+
+```bash
+pip install embedsync
+pip install -e ".[dev]"
+```
+
+## Usage
+
+```bash
+embedsync health
+embedsync plan examples/docs --state-db /tmp/embedsync-demo.db
+embedsync run examples/docs --dry-run --state-db /tmp/embedsync-demo.db
+embedsync run examples/docs --embedder hash --destination memory --state-db /tmp/embedsync-demo.db
+embedsync run examples/docs --embedder hash --destination jsonl:/tmp/index.jsonl
+# Requires a running Ollama with an embedding model:
+embedsync run examples/docs --embedder ollama --destination jsonl:/tmp/index.jsonl
+embedsync run examples/docs --embedder ollama:nomic-embed-text --destination memory
+```
+
+## Docker
+
+```bash
+docker compose run --rm test
+docker compose run --rm plan
+```
+
+## Configuration
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `EMBEDSYNC_STATE_DB` | `.embedsync/state.db` | State database path |
+| `EMBEDSYNC_LOG_LEVEL` | `INFO` | Log level |
+
+Ollama uses `OLLAMA_HOST` when set (otherwise the embedder default host).
+
+## Roadmap
+
+- [x] Pluggable embedder protocol + hash backend
+- [x] JSONL destination (local stand-in)
+- [x] Chunk-level stable IDs across edits
+- [x] Ollama embedder (`--embedder ollama`)
+- [ ] pgvector and Qdrant destinations
+- [ ] Notion and sitemap sources
+
+## License
+
+MIT
+
+## Known limitations (v0.4)
+
+- Hash embeddings are not semantic — use `--embedder ollama` for local semantic vectors
+- JSONL is not a vector DB
+- Local markdown files only
+- Re-runs reuse `.embedsync/state.db`; pass `--state-db` for an isolated plan
+- Ollama must already be running and have the embedding model pulled

embedsync-0.4.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+embedsync/__init__.py,sha256=AcwPcc5nsyPa74u6KuiX9tLOr_Yy-06ic520bUQBq_Y,71
+embedsync/chunking.py,sha256=VCWQdiSs0okcXuL83xSuAph8Wx9IAQXbDgE_vR5LmYw,1135
+embedsync/cli.py,sha256=mvh3ys2Lno5fWiS_EnHwPF7Rr8rOXdnfoL3AmWG9hWI,3632
+embedsync/config.py,sha256=kBEusbxpQlWANqaCTAeMfxVgJh7iP4NHf6wtAJ5Nm_A,310
+embedsync/embedders.py,sha256=i2uHI7kI9osd4cEb4MboMgkBnL-_Rp39gpyFR6Qao8o,3052
+embedsync/destinations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+embedsync/destinations/jsonl.py,sha256=Wwy_Ns7kFymbCef881_W7oqykew_todSvfEn8jFY72Q,2018
+embedsync/destinations/memory.py,sha256=2lHqG_ji_81_4WrE3G1f8_VW8kOuvvTYHmBTpivSA1E,1614
+embedsync/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+embedsync/sources/local.py,sha256=yR-zhVNmNbzB7OazUvYy7q2tr4murRD9Js1sZbBl3Bw,907
+embedsync/state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+embedsync/state/store.py,sha256=sF7ROxKTZPZ4cpT7Y6q8ThBvKf4a-oPq-hlSRHLToDc,3001
+embedsync/sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+embedsync/sync/engine.py,sha256=IeE-cg-2VGrPlawi4yo5BSW8Q-jUgGj1j-kIDqyrIK8,3914
+embedsync-0.4.0.dist-info/METADATA,sha256=xR87KH0TXqvjfFSxkuJwjXIkxe1UGVJeyx5bVZ2mAK0,4255
+embedsync-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+embedsync-0.4.0.dist-info/entry_points.txt,sha256=LqgRBazDjPccjWy_1xNKcvTBZGHQ9nHTR7cgjBQWdhM,49
+embedsync-0.4.0.dist-info/licenses/LICENSE,sha256=RHgdJwl3QH1HrtSvNLAzpPLh5tRz047xaVmZDiFBMXU,1079
+embedsync-0.4.0.dist-info/RECORD,,

embedsync-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

embedsync-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+embedsync = embedsync.cli:main

embedsync-0.4.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 embedsync contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.