era-memory 0.1.0__py3-none-any.whl

era_memory/__init__.py

@@ -0,0 +1,49 @@
+"""era-memory: portable, tier-selected memory system (ports & adapters).
+
+Top-level convenience exports. The composition root (``build_memory`` /
+``build_memory_async``) and the public data models are importable directly:
+
+    from era_memory import build_memory, MemoryRecord, SearchRequest
+
+Backend-specific adapters (SQLite, Postgres, OpenAI-compatible embedder, …) live under
+``era_memory.adapters.*`` and are imported lazily so the base package keeps zero
+third-party dependencies — import them explicitly only when you use that backend.
+"""
+
+from .config import Settings
+from .memory import Memory
+from .models import (
+    ExtractedMemory,
+    ExtractionResult,
+    MemoryRecord,
+    MemoryType,
+    SearchFilters,
+    SearchRequest,
+    SearchResponse,
+    SearchResult,
+    SearchStrategy,
+    SessionPayload,
+    VectorRecord,
+)
+from .wiring import build_memory, build_memory_async
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Settings",
+    "Memory",
+    "build_memory",
+    "build_memory_async",
+    "MemoryRecord",
+    "MemoryType",
+    "SearchFilters",
+    "SearchRequest",
+    "SearchResponse",
+    "SearchResult",
+    "SearchStrategy",
+    "SessionPayload",
+    "ExtractedMemory",
+    "ExtractionResult",
+    "VectorRecord",
+    "__version__",
+]

era_memory/__main__.py

@@ -0,0 +1,152 @@
+"""
+``era-memory`` command-line entry point.
+
+Subcommands:
+  setup    Interactively download a local embedding model from Hugging Face and cache it,
+           so era-memory does semantic search offline with no endpoint.
+  status   Show the current embedder situation (endpoint / cached models / fastembed).
+
+This is the *only* place that initiates a download by default (plus the explicit
+``build_memory(embedder="auto")``). A bare import or ``build_memory()`` never downloads.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+
+def _print_status() -> None:
+    from .adapters.fastembed import (
+        SUPPORTED_MODELS,
+        default_cache_dir,
+        fastembed_available,
+        is_cached,
+    )
+
+    url = os.environ.get("MEMORY_EMBEDDING_URL")
+    print("era-memory embedder status")
+    print("-" * 40)
+    print(f"Endpoint (MEMORY_EMBEDDING_URL): {url or '(not set)'}")
+    print(f"fastembed installed:            {fastembed_available()}")
+    print(f"Model cache dir:                {default_cache_dir()}")
+    print("Local models:")
+    any_cached = False
+    for spec in SUPPORTED_MODELS.values():
+        cached = is_cached(spec)
+        any_cached = any_cached or cached
+        mark = "✓ cached" if cached else "— not downloaded"
+        print(f"  {spec.key:<12} {spec.fastembed_name:<40} dim={spec.dimensions:<5} {mark}")
+    if not url and not any_cached:
+        print("\nNo real embedder configured → search would use the non-semantic dev embedder.")
+        print("Run `era-memory setup` to download one.")
+
+
+def _cmd_setup(args: argparse.Namespace) -> int:
+    from .adapters.fastembed import (
+        DEFAULT_MODEL_KEY,
+        SUPPORTED_MODELS,
+        default_cache_dir,
+        download_model,
+        fastembed_available,
+        first_cached_model,
+        is_cached,
+    )
+
+    if os.environ.get("MEMORY_EMBEDDING_URL"):
+        print("MEMORY_EMBEDDING_URL is set — era-memory will use that endpoint. Nothing to do.")
+        return 0
+
+    if not fastembed_available():
+        print(
+            "The local embedder needs the [localembed] extra.\n"
+            "Install it with:\n\n    pip install 'era-memory[localembed]'\n",
+            file=sys.stderr,
+        )
+        return 1
+
+    already = first_cached_model()
+    if already is not None and not args.force:
+        print(f"A local model is already cached: {already.fastembed_name} (dim {already.dimensions}).")
+        print("Use it with build_memory(...) directly, or pass --force to add another.")
+        return 0
+
+    # Resolve which model to fetch.
+    model_key = args.model
+    if model_key is None:
+        if not sys.stdin.isatty() or args.yes:
+            model_key = DEFAULT_MODEL_KEY  # non-interactive default
+        else:
+            model_key = _prompt_model_choice()
+            if model_key is None:
+                print("Aborted.")
+                return 1
+    if model_key not in SUPPORTED_MODELS:
+        print(f"Unknown model '{model_key}'. Choices: {', '.join(SUPPORTED_MODELS)}", file=sys.stderr)
+        return 2
+
+    spec = SUPPORTED_MODELS[model_key]
+    if is_cached(spec) and not args.force:
+        print(f"{spec.fastembed_name} is already cached (dim {spec.dimensions}).")
+        return 0
+
+    if sys.stdin.isatty() and not args.yes:
+        ans = input(
+            f"Download {spec.fastembed_name} (~{spec.approx_size_mb} MB, {spec.license}, "
+            f"dim {spec.dimensions}) from Hugging Face? [Y/n] "
+        ).strip().lower()
+        if ans not in ("", "y", "yes"):
+            print("Aborted.")
+            return 1
+
+    print(f"Downloading {spec.fastembed_name} → {default_cache_dir()} …")
+    download_model(spec)
+    print(
+        f"Done. Cached '{spec.key}' (dim {spec.dimensions}).\n"
+        "era-memory will now use it automatically — e.g. build_memory(tier=0, db_path='memory.db')."
+    )
+    return 0
+
+
+def _prompt_model_choice() -> "str | None":
+    from .adapters.fastembed import DEFAULT_MODEL_KEY, SUPPORTED_MODELS
+
+    keys = list(SUPPORTED_MODELS)
+    print("Choose a local embedding model to download:")
+    for i, key in enumerate(keys, 1):
+        spec = SUPPORTED_MODELS[key]
+        default = " (default)" if key == DEFAULT_MODEL_KEY else ""
+        print(
+            f"  {i}) {key}{default}: {spec.fastembed_name} — dim {spec.dimensions}, "
+            f"~{spec.approx_size_mb} MB, {spec.license}"
+        )
+    raw = input(f"Enter 1-{len(keys)} [default {keys.index(DEFAULT_MODEL_KEY) + 1}]: ").strip()
+    if raw == "":
+        return DEFAULT_MODEL_KEY
+    if raw.isdigit() and 1 <= int(raw) <= len(keys):
+        return keys[int(raw) - 1]
+    if raw in SUPPORTED_MODELS:
+        return raw
+    return None
+
+
+def main(argv: "list[str] | None" = None) -> int:
+    parser = argparse.ArgumentParser(prog="era-memory", description="era-memory utilities")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_setup = sub.add_parser("setup", help="download & cache a local embedding model")
+    p_setup.add_argument("--model", choices=["bge-small", "mxbai-large"], help="skip the prompt")
+    p_setup.add_argument("--yes", "-y", action="store_true", help="assume yes (non-interactive)")
+    p_setup.add_argument("--force", action="store_true", help="download even if one is cached")
+    p_setup.set_defaults(func=_cmd_setup)
+
+    p_status = sub.add_parser("status", help="show embedder configuration")
+    p_status.set_defaults(func=lambda _a: (_print_status() or 0))
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

era_memory/adapters/__init__.py

@@ -0,0 +1 @@
+"""Adapters: concrete backends behind the ports. ``memory`` is the in-process reference set."""

era_memory/adapters/auth.py

@@ -0,0 +1,24 @@
+"""Static bearer-token Auth adapter (Tier 1). Drop-in for the Oathkeeper-JWT Auth port."""
+
+from __future__ import annotations
+
+import hmac
+from typing import Optional
+
+from ..ports import Auth
+
+
+class BearerAuth(Auth):
+    def __init__(self, token: str, *, user_header: str = "X-User-Id") -> None:
+        self._token = token
+        self._user_header = user_header
+
+    async def authenticate(self, headers: dict[str, str]) -> Optional[str]:
+        # HTTP header names are case-insensitive (Starlette lower-cases them).
+        lower = {k.lower(): v for k, v in headers.items()}
+        auth = lower.get("authorization", "")
+        if not auth.startswith("Bearer "):
+            return None
+        if not hmac.compare_digest(auth[7:], self._token):
+            return None
+        return lower.get(self._user_header.lower())

era_memory/adapters/fastembed/__init__.py

@@ -0,0 +1,168 @@
+"""
+Offline local embedder backed by ``fastembed`` (ONNX, CPU-only).
+
+Downloads a small embedding model straight from Hugging Face on demand, caches it under a
+known directory, and serves embeddings in-process — no embedding endpoint, GPU, or API key.
+This is the "real" embedder for the offline/laptop tier; the OpenAI-compatible embedder
+remains the path for hosted endpoints.
+
+Needs the ``[localembed]`` extra (``fastembed``). The download is **never** triggered by
+import or by a bare ``build_memory()`` — only by ``era-memory setup`` / ``build_memory(
+embedder="auto")`` (explicit opt-in). See ``era_memory.embedders`` for the resolver and
+``docs/adr/0001`` for the dimension contract.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from ...core.logic.matryoshka import truncate_and_normalize
+from ...ports import Embedder
+
+
+@dataclass(frozen=True)
+class ModelSpec:
+    """A downloadable local model: short key, its HF/fastembed name, dim, and footprint."""
+
+    key: str
+    fastembed_name: str
+    dimensions: int
+    approx_size_mb: int
+    license: str
+
+
+# The curated set offered by `era-memory setup`. Both are fastembed-supported and
+# redistributable; keep this list small and verified rather than exposing all of fastembed.
+SUPPORTED_MODELS: dict[str, ModelSpec] = {
+    "bge-small": ModelSpec(
+        key="bge-small",
+        fastembed_name="BAAI/bge-small-en-v1.5",
+        dimensions=384,
+        approx_size_mb=90,
+        license="MIT",
+    ),
+    "mxbai-large": ModelSpec(
+        key="mxbai-large",
+        fastembed_name="mixedbread-ai/mxbai-embed-large-v1",
+        dimensions=1024,
+        approx_size_mb=640,
+        license="Apache-2.0",
+    ),
+}
+
+DEFAULT_MODEL_KEY = "bge-small"
+
+
+def default_cache_dir() -> Path:
+    """Where models + readiness sentinels live. Overridable via ERA_MEMORY_MODEL_DIR."""
+    env = os.environ.get("ERA_MEMORY_MODEL_DIR")
+    base = Path(env).expanduser() if env else Path.home() / ".cache" / "era-memory" / "models"
+    return base
+
+
+def _sentinel(cache_dir: Path, spec: ModelSpec) -> Path:
+    return cache_dir / f"{spec.key}.ready"
+
+
+def is_cached(spec: ModelSpec, cache_dir: Optional[Path] = None) -> bool:
+    """True iff this model was already downloaded by us (readiness sentinel present).
+
+    Cheap and side-effect-free — the resolver uses it to decide *without* hitting the network.
+    """
+    cache_dir = cache_dir or default_cache_dir()
+    return _sentinel(cache_dir, spec).exists()
+
+
+def first_cached_model(cache_dir: Optional[Path] = None) -> Optional[ModelSpec]:
+    cache_dir = cache_dir or default_cache_dir()
+    for spec in SUPPORTED_MODELS.values():
+        if is_cached(spec, cache_dir):
+            return spec
+    return None
+
+
+def fastembed_available() -> bool:
+    try:
+        import fastembed  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+def _require_fastembed():
+    try:
+        from fastembed import TextEmbedding
+    except ImportError as e:
+        raise RuntimeError(
+            "The local embedder needs the [localembed] extra: "
+            "pip install 'era-memory[localembed]'"
+        ) from e
+    return TextEmbedding
+
+
+def download_model(spec: ModelSpec, cache_dir: Optional[Path] = None) -> Path:
+    """Fetch ``spec`` from Hugging Face into the cache and write a readiness sentinel.
+
+    Materializes the model by running one embed so the ONNX weights are fully fetched, then
+    records ``<key>.ready`` so :func:`is_cached` can detect it offline afterwards.
+    """
+    cache_dir = cache_dir or default_cache_dir()
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    TextEmbedding = _require_fastembed()
+    model = TextEmbedding(model_name=spec.fastembed_name, cache_dir=str(cache_dir))
+    # Force a real forward pass so the download is complete before we mark it ready.
+    list(model.embed(["warmup"]))
+    _sentinel(cache_dir, spec).write_text(
+        json.dumps({"model": spec.fastembed_name, "dim": spec.dimensions})
+    )
+    return cache_dir
+
+
+class FastEmbedEmbedder(Embedder):
+    """Serves a cached fastembed model. Construction is cheap; the model loads on first embed."""
+
+    def __init__(
+        self,
+        spec: ModelSpec,
+        *,
+        cache_dir: Optional[Path] = None,
+        dimensions: Optional[int] = None,
+    ) -> None:
+        self._spec = spec
+        self._cache_dir = cache_dir or default_cache_dir()
+        if dimensions is not None and dimensions > spec.dimensions:
+            raise ValueError(
+                f"requested dim {dimensions} exceeds {spec.fastembed_name}'s native "
+                f"{spec.dimensions} (Matryoshka truncates down, never up)"
+            )
+        self._dim = dimensions or spec.dimensions
+        self._model = None
+
+    @property
+    def dimensions(self) -> int:
+        return self._dim
+
+    @property
+    def model_id(self) -> str:
+        return self._spec.fastembed_name
+
+    def _get_model(self):
+        if self._model is None:
+            TextEmbedding = _require_fastembed()
+            self._model = TextEmbedding(
+                model_name=self._spec.fastembed_name, cache_dir=str(self._cache_dir)
+            )
+        return self._model
+
+    def _embed_sync(self, texts: list[str]) -> list[list[float]]:
+        model = self._get_model()
+        return [truncate_and_normalize(vec.tolist(), self._dim) for vec in model.embed(texts)]
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        # fastembed is synchronous + CPU-bound; keep the event loop responsive.
+        return await asyncio.to_thread(self._embed_sync, texts)

era_memory/adapters/memory/__init__.py

@@ -0,0 +1,307 @@
+"""
+In-memory reference adapters for all nine ports.
+
+Purpose: lock the port contracts (via the conformance suite) before any infrastructure,
+and provide a zero-dependency tier for tests and quick embedding-in-an-app. The RecordStore
+and VectorStore here are SEPARATE stores, so M0 exercises the split-store (Tier-2-shaped)
+dual-write path; the collapsed single-store path arrives with the SQLite adapter (M1).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+import re
+from contextlib import asynccontextmanager
+from typing import Optional
+
+from ...core.logic.dedup import cosine_similarity
+from ...core.logic.entropy import entropy_score
+from ...core.logic.matryoshka import l2_normalize
+from ...errors import VectorStoreWriteError
+from ...models import (
+    ExtractedMemory,
+    ExtractionResult,
+    MemoryRecord,
+    MemoryType,
+    SearchFilters,
+    SessionPayload,
+    VectorRecord,
+)
+from ...ports import (
+    Auth,
+    BlobStore,
+    Embedder,
+    Extractor,
+    KMS,
+    Queue,
+    RecordStore,
+    Telemetry,
+    UnitOfWork,
+    VectorStore,
+)
+from ...ports.queue import Consumer
+
+_TOKEN = re.compile(r"\w+")
+
+
+def _matches(rec: MemoryRecord, f: SearchFilters) -> bool:
+    if f.memory_type is not None and rec.memory_type != f.memory_type:
+        return False
+    if f.experience_id is not None and (rec.experience_id or "") != f.experience_id:
+        return False
+    if f.created_after is not None and rec.created_at < f.created_after:
+        return False
+    if f.created_before is not None and rec.created_at > f.created_before:
+        return False
+    return True
+
+
+# --------------------------------------------------------------------------- RecordStore
+
+
+class _MemUoW(UnitOfWork):
+    pass
+
+
+class InMemoryRecordStore(RecordStore):
+    def __init__(self) -> None:
+        self._by_user: dict[str, dict[str, MemoryRecord]] = {}
+        self._hash_index: dict[tuple[str, str], str] = {}
+
+    @asynccontextmanager
+    async def unit_of_work(self):
+        snap_users = {u: dict(d) for u, d in self._by_user.items()}
+        snap_hash = dict(self._hash_index)
+        try:
+            yield _MemUoW()
+        except BaseException:
+            self._by_user = snap_users
+            self._hash_index = snap_hash
+            raise
+
+    async def insert_memory(self, uow, record):
+        users = self._by_user.setdefault(record.user_id, {})
+        if record.content_hash:
+            key = (record.user_id, record.content_hash)
+            existing = self._hash_index.get(key)
+            if existing is not None:
+                return users[existing], False
+            users[record.id] = record
+            self._hash_index[key] = record.id
+            return record, True
+        users[record.id] = record
+        return record, True
+
+    async def fetch_by_ids(self, user_id, ids):
+        users = self._by_user.get(user_id, {})
+        out = [users[i] for i in ids if i in users and users[i].status == "active"]
+        return out
+
+    async def lexical_search(self, user_id, query, filters, limit):
+        users = self._by_user.get(user_id, {})
+        terms = set(_TOKEN.findall(query.lower()))
+        scored: list[tuple[str, float]] = []
+        for rec in users.values():
+            if rec.status != "active" or not _matches(rec, filters):
+                continue
+            content_terms = _TOKEN.findall(rec.content.lower())
+            if not content_terms:
+                continue
+            overlap = sum(1 for t in content_terms if t in terms)
+            if overlap > 0:
+                scored.append((rec.id, float(overlap)))
+        scored.sort(key=lambda x: x[1], reverse=True)
+        return scored[:limit]
+
+    async def soft_delete(self, user_id, memory_id):
+        rec = self._by_user.get(user_id, {}).get(memory_id)
+        if rec is None or rec.status != "active":
+            return False
+        rec.status = "archived"
+        return True
+
+
+# --------------------------------------------------------------------------- VectorStore
+
+
+class InMemoryVectorStore(VectorStore):
+    def __init__(self) -> None:
+        self._vecs: dict[str, VectorRecord] = {}
+
+    async def insert(self, records):
+        for r in records:
+            if not r.embedding:
+                raise VectorStoreWriteError(f"empty embedding for {r.id}")
+            self._vecs[r.id] = r
+        return len(records)
+
+    async def search(self, user_id, embedding, limit, filters):
+        scored: list[tuple[str, float]] = []
+        for r in self._vecs.values():
+            if r.user_id != user_id:
+                continue
+            if filters.memory_type is not None and r.memory_type != filters.memory_type:
+                continue
+            if filters.experience_id is not None and r.experience_id != filters.experience_id:
+                continue
+            sim = cosine_similarity(embedding, r.embedding)
+            scored.append((r.id, sim))
+        scored.sort(key=lambda x: x[1], reverse=True)
+        return scored[:limit]
+
+    async def delete(self, ids):
+        n = 0
+        for i in ids:
+            if i in self._vecs:
+                del self._vecs[i]
+                n += 1
+        return n
+
+
+# ----------------------------------------------------------------------------- Embedder
+
+
+class InMemoryEmbedder(Embedder):
+    """Deterministic hashed bag-of-words embedding. Shared words -> higher cosine."""
+
+    def __init__(self, dim: int = 64, model_id: str = "in-memory-hash") -> None:
+        self._dim = dim
+        self._model = model_id
+
+    @property
+    def dimensions(self) -> int:
+        return self._dim
+
+    @property
+    def model_id(self) -> str:
+        return self._model
+
+    async def embed(self, texts):
+        return [self._embed_one(t) for t in texts]
+
+    def _embed_one(self, text: str) -> list[float]:
+        vec = [0.0] * self._dim
+        for tok in _TOKEN.findall(text.lower()):
+            h = int(hashlib.md5(tok.encode("utf-8")).hexdigest(), 16)
+            vec[h % self._dim] += 1.0
+        return l2_normalize(vec)
+
+
+# -------------------------------------------------------------------------------- Queue
+
+
+class InProcessQueue(Queue):
+    """Runs the consumer synchronously on submit — no Redis, no durability."""
+
+    def __init__(self, consumer: Consumer) -> None:
+        self._consumer = consumer
+
+    async def submit(self, payload: SessionPayload) -> None:
+        await self._consumer(payload)
+
+
+# ---------------------------------------------------------------------------- Extractor
+
+
+class HeuristicExtractor(Extractor):
+    """Offline, no-LLM: each substantive line of the conversation becomes a candidate."""
+
+    def __init__(self, min_words: int = 3) -> None:
+        self._min_words = min_words
+
+    async def extract(self, payload):
+        memories: list[ExtractedMemory] = []
+        for line in payload.conversation.splitlines():
+            line = line.strip()
+            if len(line.split()) < self._min_words:
+                continue
+            memories.append(
+                ExtractedMemory(
+                    content=line,
+                    memory_type=MemoryType.EPISODE,
+                    importance_score=min(1.0, max(0.1, entropy_score(line))),
+                )
+            )
+        return ExtractionResult(memories=memories)
+
+
+# ---------------------------------------------------------------------------- BlobStore
+
+
+class InMemoryBlobStore(BlobStore):
+    def __init__(self) -> None:
+        self._blobs: dict[str, bytes] = {}
+
+    async def put(self, key, data):
+        self._blobs[key] = data
+
+    async def get(self, key):
+        return self._blobs.get(key)
+
+
+# ---------------------------------------------------------------------------------- KMS
+
+
+class LocalKMS(KMS):
+    """
+    Dependency-free local/dev KMS: HMAC-SHA256 keystream + HMAC tag (authenticated wrap).
+    Tier 0's real local KMS (M1) can swap to AESGCM via the [encryption] extra.
+    """
+
+    def __init__(self, master_key: Optional[bytes] = None) -> None:
+        self._mk = master_key or os.urandom(32)
+
+    @property
+    def provider_name(self) -> str:
+        return "local"
+
+    async def generate_dek(self) -> bytes:
+        return os.urandom(32)
+
+    def _keystream(self, nonce: bytes, length: int) -> bytes:
+        out = b""
+        counter = 0
+        while len(out) < length:
+            out += hmac.new(
+                self._mk, nonce + counter.to_bytes(4, "big"), hashlib.sha256
+            ).digest()
+            counter += 1
+        return out[:length]
+
+    async def wrap_dek(self, dek, aad):
+        nonce = os.urandom(16)
+        ks = self._keystream(nonce, len(dek))
+        ct = bytes(a ^ b for a, b in zip(dek, ks))
+        tag = hmac.new(self._mk, nonce + ct + aad.encode("utf-8"), hashlib.sha256).digest()
+        return nonce + tag + ct
+
+    async def unwrap_dek(self, wrapped, aad):
+        nonce, tag, ct = wrapped[:16], wrapped[16:48], wrapped[48:]
+        expected = hmac.new(self._mk, nonce + ct + aad.encode("utf-8"), hashlib.sha256).digest()
+        if not hmac.compare_digest(tag, expected):
+            raise ValueError("KMS unwrap authentication failure")
+        ks = self._keystream(nonce, len(ct))
+        return bytes(a ^ b for a, b in zip(ct, ks))
+
+
+# --------------------------------------------------------------------------------- Auth
+
+
+class NoopAuth(Auth):
+    """Single-user / trusted-caller: returns X-User-Id header or a default."""
+
+    def __init__(self, default_user: str = "local") -> None:
+        self._default = default_user
+
+    async def authenticate(self, headers):
+        return headers.get("X-User-Id", self._default)
+
+
+# ---------------------------------------------------------------------------- Telemetry
+
+
+class NoopTelemetry(Telemetry):
+    def event(self, name, **fields):
+        return None