coremem 0.1.0__tar.gz

coremem-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+.venv/
+.pytest_cache/
+*.db
+vectors/
+
+# Benchmark results
+results/

coremem-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Eddy Vinck
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

coremem-0.1.0/PKG-INFO

@@ -0,0 +1,170 @@
+Metadata-Version: 2.4
+Name: coremem
+Version: 0.1.0
+Summary: Zero-LLM memory retrieval for AI agents — semantic search and deterministic heuristics
+Author: Eddy Vinck
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai-agents,llm,memory,retrieval,semantic-search
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Requires-Dist: chromadb>=0.5.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: sentence-transformers>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Provides-Extra: hybrid
+Requires-Dist: hybriddb>=0.2.0; extra == 'hybrid'
+Description-Content-Type: text/markdown
+
+# CoreMem
+
+> **Zero-LLM memory retrieval for AI agents.** CoreMem gives agents instant access to conversation history — semantic search plus deterministic retrieval heuristics, all without a single API call. Scores **98.0% R@5 on LongMemEval (500 questions)** in the Executive Assistant retrieval stack — no LLM, no tuning, no cloud.
+
+> **Embedded. Local. Open source.** No external APIs, no vector DB services, no internet connection required. Runs entirely on-device with ChromaDB or HybridDB + sentence-transformers. Ships as a single Python package with zero infrastructure dependencies.
+
+**Dual-backend architecture.** Drop-in backends (ChromaDB baseline, HybridDB enhanced) with the same API. Ranking pipeline: backend retrieval → deterministic heuristics → recency-aware rescoring → session-aware retrieval.
+
+```python
+from coremem import MemoryCore
+from coremem.backends.chroma import ChromaBackend
+
+core = MemoryCore(backend=ChromaBackend(path="./memory"))
+
+# Ingest conversation turns
+core.ingest("user", "I visited the Museum of Modern Art today")
+core.ingest("assistant", "That sounds wonderful! How was it?")
+core.ingest("user", "I went to an Ancient Civilizations exhibition at the Natural History Museum")
+
+# Search with deterministic heuristic reranking
+results = core.search("When did I visit art museums?")
+
+for r in results:
+    print(f"[{r.memory.ts}] [{r.memory.role}] {r.memory.content}")
+```
+
+## Why CoreMem?
+
+Every AI agent needs memory. But cloud-based vector search is expensive, slow, and doesn't work offline. Pure embedding similarity misses keyword matches and temporal context. LLM-based memory systems cost tokens per query.
+
+CoreMem solves all three:
+
+| Component | What it does |
+|-----------|-------------|
+| **Semantic search** | Embedding similarity via ChromaDB or HybridDB |
+| **Deterministic heuristics** | Keyword overlap, temporal recency, person-name boost, quoted-phrase matching |
+| **Session deduplication** | One result per conversation, with full context retrieval |
+
+## LongMemEval Results (500 questions, no LLM, no tuning)
+
+| Metric | Score |
+|--------|-------|
+| R@5 | **98.0%** |
+| R@10 | **98.4%** |
+| MRR | 0.944 |
+| P@5 | 0.592 |
+| F1@5 | 0.684 |
+| Selectivity | 11.5% haystack scanned |
+| Rank distribution | #1: 91.8%, #2-3: 5.0%, #4-5: 1.2%, #6-10: 0.4%, >10: 1.6% |
+
+Outperforms MemPalace raw (96.6%) and matches their hybrid v4 held-out (98.4%) — with zero tuning, zero dev-set peeking.
+
+## Installation
+
+```bash
+pip install coremem
+```
+
+With HybridDB backend for enhanced FTS5 + vector hybrid search:
+
+```bash
+pip install coremem[hybrid]
+```
+
+## Core Concepts
+
+### Backends
+
+```python
+# ChromaDB baseline — pure vector search
+from coremem.backends.chroma import ChromaBackend
+core = MemoryCore(backend=ChromaBackend(path="./data"))
+
+# HybridDB enhanced — FTS5 + vector hybrid search
+from coremem.backends.hybrid import HybridBackend
+core = MemoryCore(backend=HybridBackend(path="./data"))
+```
+
+### Ingestion
+
+```python
+# Simple ingestion
+core.ingest("user", "I built a Spitfire model kit", session_id="conv_001")
+
+# Batch ingestion
+from coremem import ingest_batch
+ingest_batch(core, [
+    ("user", "What's the weather today?"),
+    ("assistant", "Sunny with a high of 72°F"),
+], session_id="conv_001")
+```
+
+### Search
+
+```python
+# Basic search
+results = core.search("How many model kits?", limit=10)
+
+# Limit results
+results = core.search("model building projects", limit=5)
+```
+
+### Heuristics
+
+Deterministic, zero-LLM scoring boosts applied to every result:
+
+| Heuristic | What it catches |
+|-----------|----------------|
+| `keyword_overlap` | Exact word matches between query and content |
+| `temporal_boost` | Queries with "latest", "current", "recently" |
+| `recency_decay` | Unconditional exponential decay (30-day half-life) |
+| `person_name_boost` | Proper name mentions in content |
+| `quoted_phrase_boost` | Exact phrase matches in quotes |
+
+```python
+from coremem import SearchHeuristics
+
+# Apply all heuristics to a single result
+score = SearchHeuristics.apply_all(
+    query="latest project",
+    content="Just finished the Q3 project report",
+    score=0.75,
+    ts="2026-05-28T10:00:00Z",
+)
+```
+
+### Wake-Up Context
+
+Give the agent instant situational awareness:
+
+```python
+context = core.wake_up(user_id="alice")
+# Returns a compact string with L0 identity and L1 recent context.
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+## Author
+
+Eddy Vinck
+
+CoreMem is the retrieval engine behind the [Executive Assistant](https://github.com/open-assistants-lab) agent system. Pairs with [HybridDB](https://github.com/open-assistants-lab) for storage and ConnectKit for real-time sync.

coremem-0.1.0/README.md

@@ -0,0 +1,144 @@
+# CoreMem
+
+> **Zero-LLM memory retrieval for AI agents.** CoreMem gives agents instant access to conversation history — semantic search plus deterministic retrieval heuristics, all without a single API call. Scores **98.0% R@5 on LongMemEval (500 questions)** in the Executive Assistant retrieval stack — no LLM, no tuning, no cloud.
+
+> **Embedded. Local. Open source.** No external APIs, no vector DB services, no internet connection required. Runs entirely on-device with ChromaDB or HybridDB + sentence-transformers. Ships as a single Python package with zero infrastructure dependencies.
+
+**Dual-backend architecture.** Drop-in backends (ChromaDB baseline, HybridDB enhanced) with the same API. Ranking pipeline: backend retrieval → deterministic heuristics → recency-aware rescoring → session-aware retrieval.
+
+```python
+from coremem import MemoryCore
+from coremem.backends.chroma import ChromaBackend
+
+core = MemoryCore(backend=ChromaBackend(path="./memory"))
+
+# Ingest conversation turns
+core.ingest("user", "I visited the Museum of Modern Art today")
+core.ingest("assistant", "That sounds wonderful! How was it?")
+core.ingest("user", "I went to an Ancient Civilizations exhibition at the Natural History Museum")
+
+# Search with deterministic heuristic reranking
+results = core.search("When did I visit art museums?")
+
+for r in results:
+    print(f"[{r.memory.ts}] [{r.memory.role}] {r.memory.content}")
+```
+
+## Why CoreMem?
+
+Every AI agent needs memory. But cloud-based vector search is expensive, slow, and doesn't work offline. Pure embedding similarity misses keyword matches and temporal context. LLM-based memory systems cost tokens per query.
+
+CoreMem solves all three:
+
+| Component | What it does |
+|-----------|-------------|
+| **Semantic search** | Embedding similarity via ChromaDB or HybridDB |
+| **Deterministic heuristics** | Keyword overlap, temporal recency, person-name boost, quoted-phrase matching |
+| **Session deduplication** | One result per conversation, with full context retrieval |
+
+## LongMemEval Results (500 questions, no LLM, no tuning)
+
+| Metric | Score |
+|--------|-------|
+| R@5 | **98.0%** |
+| R@10 | **98.4%** |
+| MRR | 0.944 |
+| P@5 | 0.592 |
+| F1@5 | 0.684 |
+| Selectivity | 11.5% haystack scanned |
+| Rank distribution | #1: 91.8%, #2-3: 5.0%, #4-5: 1.2%, #6-10: 0.4%, >10: 1.6% |
+
+Outperforms MemPalace raw (96.6%) and matches their hybrid v4 held-out (98.4%) — with zero tuning, zero dev-set peeking.
+
+## Installation
+
+```bash
+pip install coremem
+```
+
+With HybridDB backend for enhanced FTS5 + vector hybrid search:
+
+```bash
+pip install coremem[hybrid]
+```
+
+## Core Concepts
+
+### Backends
+
+```python
+# ChromaDB baseline — pure vector search
+from coremem.backends.chroma import ChromaBackend
+core = MemoryCore(backend=ChromaBackend(path="./data"))
+
+# HybridDB enhanced — FTS5 + vector hybrid search
+from coremem.backends.hybrid import HybridBackend
+core = MemoryCore(backend=HybridBackend(path="./data"))
+```
+
+### Ingestion
+
+```python
+# Simple ingestion
+core.ingest("user", "I built a Spitfire model kit", session_id="conv_001")
+
+# Batch ingestion
+from coremem import ingest_batch
+ingest_batch(core, [
+    ("user", "What's the weather today?"),
+    ("assistant", "Sunny with a high of 72°F"),
+], session_id="conv_001")
+```
+
+### Search
+
+```python
+# Basic search
+results = core.search("How many model kits?", limit=10)
+
+# Limit results
+results = core.search("model building projects", limit=5)
+```
+
+### Heuristics
+
+Deterministic, zero-LLM scoring boosts applied to every result:
+
+| Heuristic | What it catches |
+|-----------|----------------|
+| `keyword_overlap` | Exact word matches between query and content |
+| `temporal_boost` | Queries with "latest", "current", "recently" |
+| `recency_decay` | Unconditional exponential decay (30-day half-life) |
+| `person_name_boost` | Proper name mentions in content |
+| `quoted_phrase_boost` | Exact phrase matches in quotes |
+
+```python
+from coremem import SearchHeuristics
+
+# Apply all heuristics to a single result
+score = SearchHeuristics.apply_all(
+    query="latest project",
+    content="Just finished the Q3 project report",
+    score=0.75,
+    ts="2026-05-28T10:00:00Z",
+)
+```
+
+### Wake-Up Context
+
+Give the agent instant situational awareness:
+
+```python
+context = core.wake_up(user_id="alice")
+# Returns a compact string with L0 identity and L1 recent context.
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+## Author
+
+Eddy Vinck
+
+CoreMem is the retrieval engine behind the [Executive Assistant](https://github.com/open-assistants-lab) agent system. Pairs with [HybridDB](https://github.com/open-assistants-lab) for storage and ConnectKit for real-time sync.

coremem-0.1.0/benchmarks/longmemeval/adapter.py

@@ -0,0 +1,104 @@
+"""LongMemEval benchmark adapter for coremem.
+
+Direct-injection mode — injects haystack sessions into coremem,
+runs search, measures Recall@K. No LLM. No HTTP. No agent loop.
+Pure retrieval benchmarking.
+
+Usage:
+    python -m coremem.benchmarks.longmemeval.eval --backend chroma --limit 5
+    python -m coremem.benchmarks.longmemeval.eval --backend hybrid --limit 10
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from coremem.core import MemoryCore
+
+
+class LongMemEvalAdapter:
+    """Inject LongMemEval sessions and measure retrieval recall."""
+
+    def __init__(self, core: MemoryCore):
+        self._core = core
+
+    def inject_sessions(
+        self,
+        haystack_sessions: list[dict],
+        verbose: bool = False,
+    ) -> dict[str, list[str]]:
+        """Inject all haystack sessions into coremem.
+
+        Each session is a list of message dicts. Each message gets tagged
+        with its session_id for later dedup during search.
+
+        Args:
+            haystack_sessions: List of sessions, each a list of {"role", "content"} dicts.
+            verbose: Print injection progress.
+
+        Returns:
+            Dict mapping session_id → list of ingested memory IDs.
+        """
+        session_memory_ids: dict[str, list[str]] = {}
+        for si, session in enumerate(haystack_sessions):
+            sid = f"session_{si:04d}"
+            ids = self._core.ingest_many(session, session_id=sid)
+            session_memory_ids[sid] = ids
+            if verbose:
+                print(f"  Session {sid}: {len(ids)}/{len(session)} messages ingested", flush=True)
+        return session_memory_ids
+
+    def search(self, query: str, limit: int = 10) -> list[dict]:
+        """Search and return session_ids from results."""
+        results = self._core.search(query, limit=limit)
+        return [
+            {
+                "session_id": r.memory.session_id,
+                "content": r.memory.content[:200],
+                "score": r.score,
+                "source": r.source,
+            }
+            for r in results
+        ]
+
+    def recall_at_k(self, query: str, answer_session_ids: list[str], k: int = 5) -> tuple[bool, int]:
+        """Check if any answer session appears in top-K results.
+
+        Returns (is_hit, count_of_answer_sessions_found).
+        """
+        results = self._core.search(query, limit=k)
+        found_sessions = {r.memory.session_id for r in results}
+        matches = found_sessions & set(answer_session_ids)
+        return len(matches) > 0, len(matches)
+
+
+def load_longmemeval_questions(
+    data_path: str | Path,
+    question_types: list[str] | None = None,
+    limit: int | None = None,
+) -> list[dict[str, Any]]:
+    """Load LongMemEval questions from JSON.
+
+    Args:
+        data_path: Path to LongMemEval JSON data file.
+        question_types: Optional filter by question_type field.
+        limit: Optional max number of questions to load.
+
+    Returns:
+        List of question dicts with keys: question_id, question, question_type,
+        answer, answer_session_id, haystack_sessions.
+    """
+    with open(data_path) as f:
+        data = json.load(f)
+
+    questions = data if isinstance(data, list) else data.get("questions", [])
+
+    if question_types:
+        questions = [q for q in questions if q.get("question_type") in question_types]
+
+    if limit:
+        questions = questions[:limit]
+
+    return questions

coremem-0.1.0/benchmarks/longmemeval/eval.py

@@ -0,0 +1,230 @@
+"""LongMemEval retrieval benchmark runner for coremem.
+
+Measures Recall@K without any LLM involvement. Two backends supported:
+  --backend chroma  → ChromaBackend (baseline, target 95%+)
+  --backend hybrid  → HybridBackend (enhanced, requires hybriddb)
+
+Dataset format (LongMemEval):
+  Each question has: question_id, question_type, question, answer_session_ids,
+  haystack_session_ids, haystack_sessions.
+  haystack_session_ids[i] maps to haystack_sessions[i].
+
+Injection: sessions are injected in batch, tagged as session_{i:04d}.
+
+Recall check: answer_session_ids[aid] → find aid in haystack_session_ids → get
+index → our injected id session_{index:04d} → check if in top-K results.
+
+Usage:
+    uv run python -m coremem.benchmarks.longmemeval.eval \
+        --data /tmp/lme_cache/.../longmemeval_s_cleaned.json \
+        --backend chroma \
+        --limit 20 \
+        --k 5
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Any
+
+from coremem.core import MemoryCore
+
+
+def _load_questions(data_path: str, question_types: list[str] | None = None, limit: int | None = None) -> list[dict]:
+    with open(data_path) as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        data = list(data.values())
+
+    if question_types:
+        data = [q for q in data if q.get("question_type") in question_types]
+
+    if limit:
+        data = data[:limit]
+
+    return data
+
+
+def _setup_backend(backend: str, path: str):
+    if backend == "chroma":
+        from coremem.backends.chroma import ChromaBackend
+
+        return ChromaBackend(path=path)
+    elif backend == "hybrid":
+        from coremem.backends.hybrid import HybridBackend
+
+        return HybridBackend(path=path)
+    else:
+        raise ValueError(f"Unknown backend: {backend}. Use 'chroma' or 'hybrid'.")
+
+
+def _map_answer_sids(
+    haystack_session_ids: list[str],
+    answer_session_ids: list[str] | str,
+) -> list[str]:
+    if isinstance(answer_session_ids, str):
+        answer_session_ids = [answer_session_ids]
+
+    id_to_index = {hid: idx for idx, hid in enumerate(haystack_session_ids)}
+    result = []
+    for aid in answer_session_ids:
+        idx = id_to_index.get(aid)
+        if idx is not None:
+            result.append(f"session_{idx:04d}")
+    return result
+
+
+def _inject_sessions_batch(core: MemoryCore, haystack_sessions: list) -> float:
+    """Inject all sessions using batch insert. Returns time taken."""
+    from coremem.types import Memory
+
+    t0 = time.time()
+    batch: list[Memory] = []
+    for si, session_messages in enumerate(haystack_sessions):
+        sid = f"session_{si:04d}"
+        for msg in session_messages:
+            batch.append(Memory(
+                id="",
+                content=msg.get("content", ""),
+                role=msg.get("role", "user"),
+                session_id=sid,
+            ))
+    if batch:
+        core.backend.ingest_batch(batch)
+    return time.time() - t0
+
+
+def run_retrieval_benchmark(
+    data_path: str | Path,
+    backend: str = "chroma",
+    question_types: list[str] | None = None,
+    limit: int | None = None,
+    k: int = 5,
+    verbose: bool = True,
+    memory_base: str = "/tmp/coremem_bench",
+) -> dict[str, Any]:
+    questions = _load_questions(str(data_path), question_types=question_types, limit=limit)
+
+    if not questions:
+        raise ValueError(f"No questions found in {data_path}")
+
+    results: list[dict] = []
+    type_scores: dict[str, list[bool]] = {}
+
+    if verbose:
+        print(f"Backend: {backend}")
+        print(f"Questions: {len(questions)}")
+        print(f"Recall: R@{k}")
+        print("-" * 60)
+
+    start_time = time.time()
+
+    for qi, q in enumerate(questions):
+        q_id = q.get("question_id", f"q_{qi}")
+        q_text = q.get("question", "")
+        q_type = q.get("question_type", "unknown")
+        haystack_ids = q.get("haystack_session_ids", [])
+        haystack = q.get("haystack_sessions", [])
+
+        answer_sids = _map_answer_sids(haystack_ids, q.get("answer_session_ids", []))
+        if not answer_sids:
+            if verbose:
+                print(f"  [{qi+1}/{len(questions)}] {q_id}: SKIP (no answer IDs mapped)", flush=True)
+            continue
+
+        mem_path = f"{memory_base}_{q_id}_{os.getpid()}"
+        be = _setup_backend(backend, mem_path)
+        core = MemoryCore(backend=be)
+
+        try:
+            inject_time = _inject_sessions_batch(core, haystack)
+
+            t0 = time.time()
+            search_results = core.search(q_text, limit=k)
+            found = {r.memory.session_id for r in search_results}
+            hits = found & set(answer_sids)
+            is_hit = len(hits) > 0
+            search_time = time.time() - t0
+
+            results.append({
+                "question_id": q_id,
+                "question_type": q_type,
+                "recall": is_hit,
+                "sessions_injected": len(haystack),
+                "inject_time_s": round(inject_time, 3),
+                "search_time_s": round(search_time, 4),
+                "matches": sorted(hits),
+            })
+
+            if q_type not in type_scores:
+                type_scores[q_type] = []
+            type_scores[q_type].append(is_hit)
+
+            if verbose:
+                status = f"HIT ({len(hits)} of {len(answer_sids)})" if is_hit else "MISS"
+                print(
+                    f"  [{qi+1}/{len(questions)}] {q_id} ({q_type}): {status} "
+                    f"| inject={inject_time:.1f}s search={search_time:.4f}s",
+                    flush=True,
+                )
+
+        finally:
+            shutil.rmtree(mem_path, ignore_errors=True)
+
+    elapsed = time.time() - start_time
+    total_hits = sum(r["recall"] for r in results)
+    total = len(results) or 1
+    overall = total_hits / total
+
+    if verbose:
+        print("-" * 60)
+        print(f"Overall R@{k}: {overall:.1%} ({total_hits}/{total})")
+        for qt, scores in sorted(type_scores.items()):
+            s = sum(scores)
+            print(f"  {qt}: {s}/{len(scores)} = {s/len(scores):.1%}")
+        print(f"Time: {elapsed:.1f}s")
+
+    return {
+        "backend": backend,
+        "k": k,
+        "total": total,
+        "hits": total_hits,
+        "recall": overall,
+        "by_type": {
+            qt: {"hits": sum(s), "total": len(s), "recall": sum(s) / len(s) if s else 0}
+            for qt, s in type_scores.items()
+        },
+        "results": results,
+        "elapsed_s": round(elapsed, 1),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="coremem LongMemEval retrieval benchmark")
+    parser.add_argument("--data", type=str, required=True,
+                        help="Path to longmemeval_s_cleaned.json")
+    parser.add_argument("--backend", type=str, default="chroma",
+                        choices=["chroma", "hybrid"])
+    parser.add_argument("--question-types", type=str, nargs="*", default=None)
+    parser.add_argument("--limit", type=int, default=None)
+    parser.add_argument("--k", type=int, default=5)
+    args = parser.parse_args()
+
+    run_retrieval_benchmark(
+        data_path=args.data,
+        backend=args.backend,
+        question_types=args.question_types,
+        limit=args.limit,
+        k=args.k,
+        verbose=True,
+    )
+
+
+if __name__ == "__main__":
+    main()