nerva-mneme 0.1.0__tar.gz

nerva_mneme-0.1.0/.env.example

@@ -0,0 +1,15 @@
+DATABASE_URL=postgresql://postgres@localhost:5432/mneme
+
+# Path to documents for digest. Local path or URL.
+DATA_PATH=https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json
+
+# API key for cloud providers. Leave empty for local (Ollama, vLLM).
+API_KEY=
+
+# Override defaults (Ollama localhost with bge-m3 / llama3).
+# Works with any /v1/-compatible API (Ollama, vLLM, etc).
+# EMBEDDER_URL=http://localhost:11434
+# EMBEDDER_MODEL=bge-m3
+# EMBEDDING_DIM=1024
+# INFERENCE_URL=http://localhost:11434
+# INFERENCE_MODEL=llama3:8b-instruct-q4_K_M

nerva_mneme-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+
+# Env
+.env
+.env.*
+!.env.example
+
+# Editors
+.idea/
+.vscode/
+
+# Logs
+*.log
+
+# Cache
+.cache/
+
+# Project-local notes
+CLAUDE.md
+.design/
+.research/
+.eval/
+todo.md
+tests/

nerva_mneme-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pavel Dolgov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nerva_mneme-0.1.0/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: nerva-mneme
+Version: 0.1.0
+Summary: Local-first RAG with built-in eval. Hybrid search, parameter sweep, one command.
+Project-URL: Repository, https://github.com/nerva-project/mneme
+Author-email: Pavel Dolgov <mikepromogratus@proton.me>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: embeddings,eval,local-first,pgvector,rag,search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: asyncpg>=0.29
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: numpy>=2.0
+Requires-Dist: pgvector>=0.3
+Requires-Dist: python-dotenv>=1.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Mneme
+
+Local RAG pipeline with built-in evaluation. Postgres + pgvector for hybrid search, any /v1/-compatible LLM backend for embeddings and inference.
+
+## Setup
+
+Requires Python 3.12+, [uv](https://docs.astral.sh/uv/), Postgres with the `pgvector` extension.
+
+Copy `.env.example` to `.env` and fill in your values, then install:
+
+```bash
+cp .env.example .env
+uv sync
+```
+
+## Usage
+
+```bash
+uv run mneme digest              # parse DATA_PATH source into cache
+uv run mneme ingest <file.jsonl>
+uv run mneme ask "query"
+uv run mneme sweep <fast|medium|thorough> --limit 30
+```
+
+## Library
+
+```python
+from mneme import Mneme, Config
+
+cfg = Config(database_url="postgresql://...", api_key="sk-...")
+
+async with Mneme(cfg) as m:
+    await m.ingest("./corpus")
+    answer = await m.ask("What is X?")
+
+rows = await Mneme.sweep(cfg, "medium", limit=30)
+```
+
+## Input format
+
+JSONL, one document per line:
+
+```json
+{"content": "...", "source": "optional", "created_at": "2026-04-01T12:00:00Z", "metadata": {}}
+```
+
+Only `content` is required. `source` falls back to the file stem, `created_at` to the current time, `metadata` to `{}`.

nerva_mneme-0.1.0/README.md

@@ -0,0 +1,47 @@
+# Mneme
+
+Local RAG pipeline with built-in evaluation. Postgres + pgvector for hybrid search, any /v1/-compatible LLM backend for embeddings and inference.
+
+## Setup
+
+Requires Python 3.12+, [uv](https://docs.astral.sh/uv/), Postgres with the `pgvector` extension.
+
+Copy `.env.example` to `.env` and fill in your values, then install:
+
+```bash
+cp .env.example .env
+uv sync
+```
+
+## Usage
+
+```bash
+uv run mneme digest              # parse DATA_PATH source into cache
+uv run mneme ingest <file.jsonl>
+uv run mneme ask "query"
+uv run mneme sweep <fast|medium|thorough> --limit 30
+```
+
+## Library
+
+```python
+from mneme import Mneme, Config
+
+cfg = Config(database_url="postgresql://...", api_key="sk-...")
+
+async with Mneme(cfg) as m:
+    await m.ingest("./corpus")
+    answer = await m.ask("What is X?")
+
+rows = await Mneme.sweep(cfg, "medium", limit=30)
+```
+
+## Input format
+
+JSONL, one document per line:
+
+```json
+{"content": "...", "source": "optional", "created_at": "2026-04-01T12:00:00Z", "metadata": {}}
+```
+
+Only `content` is required. `source` falls back to the file stem, `created_at` to the current time, `metadata` to `{}`.

nerva_mneme-0.1.0/mneme/__init__.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import hashlib
+
+import httpx
+
+from .chunker import chunk
+from .config import Config
+from .db import Db
+from .loader import load_docs
+from .models import chat, embed
+from .types import Chunk, SearchHit
+
+
+class Mneme:
+    """RAG engine with built-in eval. Lifecycle: Mneme(cfg) → open() → work → close().
+    Or use `async with Mneme(cfg) as m:` for automatic cleanup."""
+
+    def __init__(self, cfg: Config) -> None:
+        self.cfg = cfg.resolved()
+
+    async def open(self) -> None:
+        self.db = Db(self.cfg.database_url, self.cfg.embedding_dim)
+        await self.db.open()
+        await self.db.init_schema()
+        self.http = httpx.AsyncClient(timeout=60.0)
+
+    async def close(self) -> None:
+        await self.db.close()
+        await self.http.aclose()
+
+    async def __aenter__(self) -> Mneme:
+        await self.open()
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        await self.close()
+
+    async def reset(self) -> None:
+        await self.db.truncate()
+
+    async def ingest(self, source_path: str) -> None:
+        docs = load_docs(source_path)
+
+        class Piece:
+            def __init__(self, doc, idx, raw):
+                self.doc = doc
+                self.idx = idx
+                self.raw = raw
+
+        pieces = []
+        for doc in docs:
+            for idx, raw in enumerate(chunk(doc.content, self.cfg.chunk_size, self.cfg.overlap)):
+                pieces.append(Piece(doc, idx, raw))
+
+        texts = [p.raw.overlapped() for p in pieces]
+        vectors = await embed(self.cfg, self.http, texts)
+        print(f"embedded {len(texts)} chunks in one call")
+
+        chunks = [
+            Chunk(
+                id=hashlib.md5(f"{p.doc.source}:{p.idx}:{p.raw.clean}".encode()).hexdigest(),
+                source=p.doc.source,
+                chunk_index=p.idx,
+                content=p.raw.clean,
+                embedding=vectors[i],
+                metadata=p.doc.metadata,
+                created_at=p.doc.created_at,
+            )
+            for i, p in enumerate(pieces)
+        ]
+        await self.db.insert(chunks)
+        print(f"ingest done: {len(chunks)} chunks from {len(docs)} docs")
+
+    async def ask(self, query: str) -> str:
+        vectors = await embed(self.cfg, self.http, [query])
+        hits = await self.db.search(self.cfg, vectors[0], query)
+
+        if hits:
+            return await self._answer_with_context(query, hits)
+        else:
+            return await self._answer_without_context(query)
+
+    async def _answer_with_context(self, query: str, hits: list[SearchHit]) -> str:
+        prompt = "You are a personal knowledge assistant. You answer questions based ONLY on the provided context. If the context doesn't contain enough information, say so honestly. Answer in the same language as the question. Be concise and direct."
+
+        parts: list[str] = []
+        for i, h in enumerate(hits):
+            c = h.chunk
+            date = c.created_at.date().isoformat()
+            parts.append(f"[{i + 1}] ({date}, {c.source}, sim={h.similarity:.3f})\n{c.content}")
+        context = "\n\n".join(parts)
+
+        return await chat(self.cfg, self.http, prompt, f"Context:\n{context}\n\nQuestion: {query}")
+
+    async def _answer_without_context(self, query: str) -> str:
+        prompt = "You are a knowledge assistant. Answer the question directly based on your general knowledge. Answer in the same language as the question. Be concise and direct."
+        return await chat(self.cfg, self.http, prompt, query)
+
+
+from .digest import digest as _digest  # noqa: E402
+from .sweep import run_sweep, SweepRow, EvalMetrics, EvalResult  # noqa: E402
+
+Mneme.digest = staticmethod(_digest)
+Mneme.sweep = staticmethod(run_sweep)
+
+__all__ = [
+    "Mneme",
+    "Config",
+    "SweepRow",
+    "EvalMetrics",
+    "EvalResult",
+]

nerva_mneme-0.1.0/mneme/cache.py

@@ -0,0 +1,50 @@
+"""File-based JSONL cache for expensive operations."""
+import hashlib
+import json
+from pathlib import Path
+
+CACHE_DIR = Path(".cache")
+
+
+def corpus_hash(source_path: str) -> str:
+    """Deterministic hash of source file(s) content."""
+    h = hashlib.md5()
+    p = Path(source_path)
+    if p.is_file():
+        h.update(p.read_bytes())
+    elif p.is_dir():
+        for f in sorted(p.rglob("*.jsonl")):
+            h.update(f.read_bytes())
+    return h.hexdigest()[:12]
+
+
+class Cache:
+    """Generic JSONL cache. Doesn't know what it stores — just dicts."""
+
+    def __init__(self, **params: object) -> None:
+        raw = json.dumps(params, sort_keys=True, default=str)
+        key = hashlib.md5(raw.encode()).hexdigest()[:16]
+        self._path = CACHE_DIR / f"{key}.jsonl"
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def exists(self) -> bool:
+        return self._path.exists()
+
+    def load(self) -> list[dict] | None:
+        if not self._path.exists():
+            return None
+        try:
+            return [json.loads(line) for line in self._path.read_text().splitlines()]
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            self._path.unlink(missing_ok=True)
+            print(f"cache corrupt, deleted: {self._path}")
+            return None
+
+    def save(self, rows: list[dict]) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self._path, "w") as f:
+            for row in rows:
+                f.write(json.dumps(row) + "\n")

nerva_mneme-0.1.0/mneme/chunker.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass
+
+# Strongest separators first. Empty string is the final fallback (per-character).
+SEPARATORS = ["\n\n", "\n", ". ", ", ", " ", ""]
+
+
+@dataclass(frozen=True)
+class ChunkData:
+    """A chunk with its overlap context. The embedding is computed from
+    `overlapped()` so the vector 'knows' about its neighbors, but only
+    `clean` is stored in the database — no text duplication between rows."""
+    clean: str
+    head: str
+    tail: str
+
+    def overlapped(self) -> str:
+        return self.head + self.clean + self.tail
+
+
+def chunk(text: str, chunk_size: int, overlap: float) -> list[ChunkData]:
+    """Recursive character text splitter: split by separator hierarchy,
+    then greedily merge adjacent pieces up to chunk_size."""
+    if not text:
+        return []
+
+    clean = _merge(_separate(text.strip(), chunk_size), chunk_size)
+    overlap_chars = int(chunk_size * overlap)
+    result: list[ChunkData] = []
+
+    for i, piece in enumerate(clean):
+        head = clean[i - 1][-overlap_chars:] if (overlap_chars > 0 and i > 0) else ""
+        tail = clean[i + 1][:overlap_chars] if (overlap_chars > 0 and i < len(clean) - 1) else ""
+        result.append(ChunkData(clean=piece, head=head, tail=tail))
+
+    return result
+
+
+def _separate(text: str, chunk_size: int, depth: int = 0) -> list[str]:
+    if depth >= len(SEPARATORS):
+        return [text]
+
+    sep = SEPARATORS[depth]
+    parts = text.split(sep)
+    result: list[str] = []
+
+    for i, part in enumerate(parts):
+        if len(part) < chunk_size:
+            # Reattach the separator as a suffix so _merge can reconstruct
+            # the original text without losing whitespace or newlines.
+            suffix = sep if i < len(parts) - 1 else ""
+            result.append(part + suffix)
+        else:
+            result.extend(_separate(part, chunk_size, depth + 1))
+
+    return result
+
+
+def _merge(splits: list[str], chunk_size: int) -> list[str]:
+    raw: list[str] = []
+    for s in splits:
+        if not s:
+            continue
+
+        if raw and len(raw[-1]) + len(s) < chunk_size:
+            raw[-1] += s
+        else:
+            raw.append(s)
+
+    return raw

nerva_mneme-0.1.0/mneme/cli.py

@@ -0,0 +1,66 @@
+import asyncio
+
+import click
+from dotenv import load_dotenv
+
+from . import Mneme
+from .config import Config
+
+
+def _mneme() -> Mneme:
+    return Mneme(Config.from_env())
+
+
+@click.group(help="Mneme — RAG with built-in eval")
+def app() -> None:
+    pass
+
+
+@app.command(help="Digest raw source into cached JSONL.")
+@click.argument("source", default="")
+def digest(source: str) -> None:
+    cfg = Config.from_env()
+    data_path = source or cfg.data_path
+    Mneme.digest(data_path)
+
+
+@app.command(help="Ingest documents from a JSONL file or directory.")
+@click.argument("source")
+def ingest(source: str) -> None:
+    async def run() -> None:
+        async with _mneme() as m:
+            await m.ingest(source)
+    asyncio.run(run())
+
+
+@app.command(help="Ask a question against the ingested corpus.")
+@click.argument("query")
+def ask(query: str) -> None:
+    async def run() -> None:
+        async with _mneme() as m:
+            answer = await m.ask(query)
+            print(f"\n{answer}\n")
+    asyncio.run(run())
+
+
+@app.command(help="Run an eval sweep across preset configurations.")
+@click.argument("level")
+@click.option("--limit", "-l", default=30, type=int, help="Number of sample chunks for eval")
+def sweep(level: str, limit: int) -> None:
+    async def run() -> None:
+        cfg = Config.from_env()
+        await Mneme.sweep(cfg, level, limit)
+    asyncio.run(run())
+
+
+def main() -> None:
+    load_dotenv()
+    try:
+        app()
+    except (RuntimeError, ValueError) as exc:
+        print(f"error: {exc}")
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

nerva_mneme-0.1.0/mneme/config.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, replace
+
+
+@dataclass(frozen=True)
+class Config:
+    database_url: str = ""
+    data_path: str = ""
+    api_key: str = ""
+    embedder_url: str = ""
+    embedder_model: str = ""
+    embedding_dim: int = 0
+    inference_url: str = ""
+    inference_model: str = ""
+    chunk_size: int = 600
+    overlap: float = 0.0
+    alpha: float = 0.7
+    k: int = 5
+
+    def resolved(self) -> Config:
+        """Fills empty fields from defaults, validates, returns new Config."""
+        if not self.database_url:
+            raise ValueError("config: database_url is required")
+
+        cfg = replace(
+            self,
+            embedder_url=self.embedder_url or DEFAULTS.embedder_url,
+            embedder_model=self.embedder_model or DEFAULTS.embedder_model,
+            embedding_dim=self.embedding_dim or DEFAULTS.embedding_dim,
+            inference_url=self.inference_url or DEFAULTS.inference_url,
+            inference_model=self.inference_model or DEFAULTS.inference_model,
+        )
+
+        if cfg.chunk_size < 100 or cfg.chunk_size > 10000:
+            raise ValueError(f"config: chunk_size must be 100..10000, got {cfg.chunk_size}")
+        if cfg.overlap < 0 or cfg.overlap > 0.5:
+            raise ValueError(f"config: overlap must be 0..0.5, got {cfg.overlap}")
+        if cfg.alpha < 0 or cfg.alpha > 1:
+            raise ValueError(f"config: alpha must be 0..1, got {cfg.alpha}")
+        if cfg.k < 1 or cfg.k > 20:
+            raise ValueError(f"config: k must be 1..20, got {cfg.k}")
+
+        return cfg
+
+    @staticmethod
+    def from_env() -> Config:
+        return Config(
+            database_url=os.environ.get("DATABASE_URL", ""),
+            data_path=os.environ.get("DATA_PATH", ""),
+            api_key=os.environ.get("API_KEY", ""),
+            embedder_url=os.environ.get("EMBEDDER_URL", ""),
+            embedder_model=os.environ.get("EMBEDDER_MODEL", ""),
+            embedding_dim=int(os.environ.get("EMBEDDING_DIM", "0")),
+            inference_url=os.environ.get("INFERENCE_URL", ""),
+            inference_model=os.environ.get("INFERENCE_MODEL", ""),
+        )
+
+
+DEFAULTS = Config(
+    embedder_url="http://localhost:11434",
+    embedder_model="bge-m3",
+    embedding_dim=1024,
+    inference_url="http://localhost:11434",
+    inference_model="llama3:8b-instruct-q4_K_M",
+)

nerva_mneme-0.1.0/mneme/corpus.py

@@ -0,0 +1,34 @@
+"""Built-in SQuAD parser. Used by digest when DATA_PATH is a URL."""
+import json
+from urllib.error import URLError
+from urllib.request import urlopen
+
+SQUAD_LIMIT = 200
+
+
+def download_squad(url: str) -> list[dict]:
+    print(f"downloading from {url}...")
+    try:
+        raw = json.loads(urlopen(url).read())
+    except (URLError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"failed to download {url}: {exc}") from exc
+
+    docs = []
+    for article in raw["data"]:
+        title = article["title"]
+        for para in article["paragraphs"]:
+            context = para["context"].strip()
+            if len(context) < 50:
+                continue
+            docs.append({
+                "content": context,
+                "source": title,
+                "metadata": {"dataset": "squad-v2"},
+            })
+            if len(docs) >= SQUAD_LIMIT:
+                break
+        if len(docs) >= SQUAD_LIMIT:
+            break
+
+    print(f"downloaded {len(docs)} paragraphs")
+    return docs