engrava 0.3.0__py3-none-any.whl

engrava/__init__.py

@@ -0,0 +1,213 @@
+"""engrava — standalone thought-graph persistence engine."""
+
+from engrava.config import (
+    ConfigError,
+    DreamingConfig,
+    DreamingGates,
+    EmbeddingConfig,
+    EngravaConfig,
+    JournalConfig,
+    MetricsConfig,
+    SearchConfig,
+    ServiceConfig,
+    ServicesConfig,
+    TTLConfig,
+    load_config,
+    resolve_embedding_provider,
+    resolve_hooks,
+    resolve_manifests,
+)
+from engrava.domain.enums import (
+    ActionStatus,
+    ActionType,
+    EdgeType,
+    KnowledgeSource,
+    LifecycleStatus,
+    Priority,
+    ThoughtType,
+    ThoughtVisibility,
+    VerificationStatus,
+)
+from engrava.domain.exceptions import (
+    EmbeddingModelMismatchError,
+    EngravaError,
+    ExtensionMigrationError,
+    InvalidTransitionError,
+    ReadOnlyViolationError,
+    StaleDataError,
+    ThoughtNotFoundError,
+)
+from engrava.domain.manifest import ExtensionManifest
+from engrava.domain.models.action import ActionRecord
+from engrava.domain.models.edge import EdgeRecord
+from engrava.domain.models.embedding import EmbeddingRecord
+from engrava.domain.models.journal import JournalEntry, JournalIntegrityResult
+from engrava.domain.models.metrics import (
+    EdgeCounts,
+    EngravaMetrics,
+    LatencyHistogram,
+    StorageFootprint,
+    ThoughtCounts,
+)
+from engrava.domain.models.mutation_type import MutationType
+from engrava.domain.models.search import HybridSearchResult
+from engrava.domain.models.thought import ThoughtRecord
+from engrava.domain.models.thought import ThoughtRecord as CoreThoughtRecord
+from engrava.domain.models.ttl import CleanupResult, CleanupStrategy
+from engrava.domain.protocols.embedding_provider import EmbeddingProviderProtocol
+from engrava.domain.protocols.engrava_core import EngravaCoreProtocol
+from engrava.domain.protocols.hooks import (
+    DefaultEngravaHooks,
+    EngravaHooksProtocol,
+    MindQLExtension,
+    ScoringContext,
+)
+from engrava.embeddings.callback import CallbackProvider
+from engrava.embeddings.huggingface import HuggingFaceProvider
+from engrava.embeddings.ollama import OllamaProvider
+from engrava.embeddings.openai_compatible import OpenAICompatibleProvider
+from engrava.embeddings.sentence_transformer import SentenceTransformerProvider
+from engrava.extensions.discovery import discover_manifests
+from engrava.extensions.dreaming import ConsolidationResult, DreamingExtension
+from engrava.extensions.dreaming_signals import (
+    ConfidenceSignal,
+    ConfirmationSignal,
+    DreamingContext,
+    DreamingSignalProtocol,
+    FrequencySignal,
+    RecencySignal,
+    StalenessSignal,
+)
+from engrava.extensions.vector_sqlite_vec import SqliteVecSearchBackend
+from engrava.infrastructure.read_only_store import ReadOnlyEngrava
+from engrava.infrastructure.service_manager import EngravaManager
+from engrava.infrastructure.sqlite.engrava_core import SqliteEngravaCore
+from engrava.infrastructure.sqlite.journal_writer import JournalWriter
+from engrava.metadata import percept, thought, utterance
+from engrava.mindql.executor import MindQLExecutor, MindQLResult
+from engrava.mindql.parser import MindQLCommand, MindQLParseError, MindQLQuery, parse
+
+__all__ = [
+    "ActionRecord",
+    "ActionStatus",
+    "ActionType",
+    "CallbackProvider",
+    "CleanupResult",
+    "CleanupStrategy",
+    "ConfidenceSignal",
+    "ConfigError",
+    "ConfirmationSignal",
+    "ConsolidationResult",
+    "CoreThoughtRecord",
+    "DefaultEngravaHooks",
+    "DefaultMindStoreHooks",
+    "DreamingConfig",
+    "DreamingContext",
+    "DreamingExtension",
+    "DreamingGates",
+    "DreamingSignalProtocol",
+    "EdgeCounts",
+    "EdgeRecord",
+    "EdgeType",
+    "EmbeddingConfig",
+    "EmbeddingModelMismatchError",
+    "EmbeddingProviderProtocol",
+    "EmbeddingRecord",
+    "EngravaConfig",
+    "EngravaCoreProtocol",
+    "EngravaError",
+    "EngravaHooksProtocol",
+    "EngravaManager",
+    "EngravaMetrics",
+    "ExtensionManifest",
+    "ExtensionMigrationError",
+    "FrequencySignal",
+    "HuggingFaceProvider",
+    "HybridSearchResult",
+    "InvalidTransitionError",
+    "JournalConfig",
+    "JournalEntry",
+    "JournalIntegrityResult",
+    "JournalWriter",
+    "KnowledgeSource",
+    "LatencyHistogram",
+    "LifecycleStatus",
+    "MetricsConfig",
+    "MindQLCommand",
+    "MindQLExecutor",
+    "MindQLExtension",
+    "MindQLParseError",
+    "MindQLQuery",
+    "MindQLResult",
+    "MindStoreConfig",
+    "MindStoreCoreProtocol",
+    "MindStoreError",
+    "MindStoreHooksProtocol",
+    "MindStoreManager",
+    "MutationType",
+    "OllamaProvider",
+    "OpenAICompatibleProvider",
+    "Priority",
+    "ReadOnlyEngrava",
+    "ReadOnlyMindStore",
+    "ReadOnlyViolationError",
+    "RecencySignal",
+    "ScoringContext",
+    "SearchConfig",
+    "SentenceTransformerProvider",
+    "ServiceConfig",
+    "ServicesConfig",
+    "SqliteEngravaCore",
+    "SqliteMindStoreCore",
+    "SqliteVecSearchBackend",
+    "StaleDataError",
+    "StalenessSignal",
+    "StorageFootprint",
+    "TTLConfig",
+    "ThoughtCounts",
+    "ThoughtNotFoundError",
+    "ThoughtRecord",
+    "ThoughtType",
+    "ThoughtVisibility",
+    "VerificationStatus",
+    "discover_manifests",
+    "load_config",
+    "parse",
+    "percept",
+    "resolve_embedding_provider",
+    "resolve_hooks",
+    "resolve_manifests",
+    "thought",
+    "utterance",
+]
+
+
+# ------------------------------------------------------------------
+# Backward-compatibility aliases — deprecated, remove in v0.4
+# ------------------------------------------------------------------
+import warnings as _warnings
+
+
+def __getattr__(name: str) -> object:
+    """Lazy deprecation aliases for renamed symbols."""
+    _aliases: dict[str, object] = {
+        "SqliteMindStoreCore": SqliteEngravaCore,
+        "MindStoreManager": EngravaManager,
+        "MindStoreConfig": EngravaConfig,
+        "MindStoreError": EngravaError,
+        "MindStoreCoreProtocol": EngravaCoreProtocol,
+        "MindStoreHooksProtocol": EngravaHooksProtocol,
+        "DefaultMindStoreHooks": DefaultEngravaHooks,
+        "ReadOnlyMindStore": ReadOnlyEngrava,
+    }
+    if name in _aliases:
+        target = _aliases[name]
+        target_name = getattr(target, "__name__", str(target))
+        _warnings.warn(
+            f"{name} is deprecated, use {target_name} instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return target
+    msg = f"module 'engrava' has no attribute {name!r}"
+    raise AttributeError(msg)

engrava/benchmarks/__init__.py

@@ -0,0 +1,6 @@
+"""Benchmark suites for engrava.
+
+This commit adds only the synthetic scenario library; the generator,
+evaluator and CLI runner ship in subsequent commits on the same
+feature branch.
+"""

engrava/benchmarks/longmemeval/README.md

@@ -0,0 +1,179 @@
+# LongMemEval — Free harness
+
+A reproducible engrava-side runner for the public **LongMemEval**
+memory-evaluation benchmark. Use it to measure end-to-end retrieval
+quality against the published dataset on any laptop, without API keys
+in the default configuration.
+
+## Attribution
+
+LongMemEval was introduced by:
+
+> Di Wu, Hongwei Wang, Wenhao Yu, Yuwei Zhang, Kai-Wei Chang, Dong Yu.
+> *LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive
+> Memory.*  ICLR 2025. arXiv:2410.10813.
+
+The original dataset, evaluation protocol, and corpus are the work of
+the LongMemEval authors. This package only wraps the dataset for
+engrava-side ingestion + scoring.
+
+## License + redistribution
+
+- The upstream LongMemEval code repository is **MIT-licensed**
+  (`github.com/xiaowu0162/LongMemEval`).
+- The dataset is distributed via the upstream HuggingFace mirror under
+  a permissive license.
+- This package **does NOT bundle the dataset.** Engrava ships only the
+  loader, runner, evaluator, and a small hand-authored test fixture
+  (in `tests/benchmarks/fixtures/`) that mirrors the upstream schema
+  but contains no upstream content.
+- The loader downloads the requested variant at runtime to
+  `~/.engrava/benchmarks/longmemeval/`. Cached files are reused on
+  subsequent invocations.
+
+If you need to operate offline, manually place the upstream JSON files
+in that cache directory; the loader will skip the download.
+
+## Download instructions
+
+The loader handles this automatically on first invocation. If you
+prefer to fetch the files manually:
+
+```bash
+mkdir -p ~/.engrava/benchmarks/longmemeval/
+cd ~/.engrava/benchmarks/longmemeval/
+
+# Oracle variant (smallest — recommended for smoke runs):
+curl -L -o longmemeval_oracle.json \
+    https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_oracle.json
+
+# S variant:
+curl -L -o longmemeval_s_cleaned.json \
+    https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_s_cleaned.json
+
+# M variant (largest):
+curl -L -o longmemeval_m_cleaned.json \
+    https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_m_cleaned.json
+```
+
+## Running the harness
+
+The harness ships with engrava's embeddings extras. Install with:
+
+```bash
+pip install 'engrava[embeddings-local]'
+```
+
+Default invocation (oracle variant, substring mode, dreaming OFF):
+
+```bash
+python -m engrava.benchmarks.longmemeval
+```
+
+Cosine-mode against the oracle variant with dreaming ON:
+
+```bash
+python -m engrava.benchmarks.longmemeval --eval-mode=cosine --dreaming
+```
+
+Subset filter (one question type only):
+
+```bash
+python -m engrava.benchmarks.longmemeval --subset=single-session-recall
+```
+
+## Evaluation modes
+
+| Mode | Deterministic | LLM required | When to use |
+|---|---|---|---|
+| `substring` (default) | yes | no | Reference for the Free-side quality gate. |
+| `cosine` | yes (given fixed provider) | no | Catches paraphrased matches the substring check misses. |
+| `llm` | depends on judge | yes (user-supplied) | Closest to the canonical LongMemEval scoring; opt-in. |
+
+The CLI exposes `substring` and `cosine`. The `llm` mode is
+deliberately **not** wired through the CLI — argparse rejects
+`--eval-mode=llm`. To use the LLM judge, drive `run_longmemeval`
+directly from Python and pass an `llm_judge` argument that implements
+the `engrava.benchmarks.longmemeval.evaluate.LLMJudgeClient` protocol.
+SDK wiring varies per provider and engrava does not pin one — see the
+snippet below.
+
+### Storage semantics
+
+By default every question runs in its own `:memory:` SQLite database
+(LongMemEval forbids cross-question state). Passing `db_path=<dir>` to
+`run_longmemeval` switches to on-disk storage: the runner creates one
+DB file per `question_id` inside that directory, so on-disk runs also
+keep the haystacks fully isolated.
+
+## Programmatic usage
+
+Default (substring mode, in-memory store per question):
+
+```python
+import asyncio
+
+from engrava.benchmarks.longmemeval import load_dataset
+from engrava.benchmarks.longmemeval.runner import run_longmemeval
+from engrava.benchmarks.synthetic.evaluate import (
+    resolve_embedding_provider_or_exit,
+)
+
+
+async def main() -> None:
+    questions = load_dataset(variant="oracle")
+    provider = resolve_embedding_provider_or_exit()
+    result = await run_longmemeval(
+        questions,
+        dreaming_enabled=False,
+        embedding_provider=provider,
+        eval_mode="substring",
+    )
+    print(f"aggregate score: {result.aggregate_score:.4f}")
+
+
+asyncio.run(main())
+```
+
+LLM-judge mode (caller supplies any client matching `LLMJudgeClient`):
+
+```python
+import asyncio
+
+from engrava.benchmarks.longmemeval import load_dataset
+from engrava.benchmarks.longmemeval.runner import run_longmemeval
+from engrava.benchmarks.synthetic.evaluate import (
+    resolve_embedding_provider_or_exit,
+)
+
+
+class MyJudge:
+    """Thin wrapper around any LLM SDK (Anthropic, OpenAI, local)."""
+
+    async def judge(self, *, system: str, user: str) -> str:
+        # Call your SDK here; return the model's reply.
+        ...
+
+
+async def main() -> None:
+    questions = load_dataset(variant="oracle")
+    provider = resolve_embedding_provider_or_exit()
+    result = await run_longmemeval(
+        questions,
+        dreaming_enabled=False,
+        embedding_provider=provider,
+        eval_mode="llm",
+        llm_judge=MyJudge(),
+    )
+    print(f"aggregate score: {result.aggregate_score:.4f}")
+
+
+asyncio.run(main())
+```
+
+## Determinism
+
+Substring mode is byte-deterministic across runs given the same dataset
+and the same engrava configuration. Cosine mode is deterministic given
+a fixed embedding provider. LLM mode is non-deterministic by design
+(provider sampling), so its results carry an informational role.

engrava/benchmarks/longmemeval/__init__.py

@@ -0,0 +1,30 @@
+"""LongMemEval Free harness — public benchmark runner for engrava.
+
+Uses the publicly distributed LongMemEval dataset (Wu et al., ICLR 2025,
+arXiv:2410.10813). The dataset is **not bundled** with engrava; users
+download it from the upstream HuggingFace distribution at runtime via
+``load_dataset``.
+
+The CLI exposes two deterministic evaluation modes (``substring`` and
+``cosine``); both run without LLM API keys. LLM-judge mode is opt-in
+and reachable only by driving ``run_longmemeval`` from Python with a
+caller-supplied ``LLMJudgeClient`` — see
+``benchmarks/longmemeval/README.md`` for a code snippet, attribution,
+license, and download instructions.
+"""
+
+from __future__ import annotations
+
+from engrava.benchmarks.longmemeval.dataset_loader import (
+    LongMemEvalQuestion,
+    LongMemEvalSession,
+    LongMemEvalTurn,
+    load_dataset,
+)
+
+__all__ = [
+    "LongMemEvalQuestion",
+    "LongMemEvalSession",
+    "LongMemEvalTurn",
+    "load_dataset",
+]

engrava/benchmarks/longmemeval/__main__.py

@@ -0,0 +1,10 @@
+"""``python -m engrava.benchmarks.longmemeval`` entry point."""
+
+from __future__ import annotations
+
+import sys
+
+from engrava.benchmarks.longmemeval.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

engrava/benchmarks/longmemeval/cli.py

@@ -0,0 +1,178 @@
+"""Command-line front-end for the LongMemEval Free harness.
+
+Invoke as ``python -m engrava.benchmarks.longmemeval``. The default
+configuration runs the ``oracle`` variant in substring-evaluation mode
+without any LLM dependency. Flags expose the cosine and LLM modes, the
+variant choice, dreaming OFF/ON arms, and an optional subset filter on
+``question_type``.
+
+Exit codes:
+
+* ``0`` — at least one question was evaluated, no fatal error.
+* ``1`` — a runtime error fired during the run (download, parsing,
+  unsupported configuration).
+* ``2`` — the embeddings extras are missing (re-uses the synthetic
+  benchmark's clean-exit convention).
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import TYPE_CHECKING
+
+from engrava.benchmarks.longmemeval.dataset_loader import (
+    DatasetDownloadError,
+    LongMemEvalQuestion,
+    load_dataset,
+)
+from engrava.benchmarks.longmemeval.runner import (
+    DEFAULT_TOP_K,
+    EvalMode,
+    LongMemEvalResults,
+    run_longmemeval_sync,
+)
+from engrava.benchmarks.synthetic.evaluate import (
+    resolve_embedding_provider_or_exit,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+__all__ = ["main"]
+
+
+_VALID_VARIANTS = ("oracle", "s", "m")
+# CLI offers the two deterministic Free-side modes only. The LLM-judge
+# mode requires a caller-supplied client implementing
+# :class:`engrava.benchmarks.longmemeval.evaluate.LLMJudgeClient` and is
+# accessible by driving :func:`run_longmemeval` from Python — see the
+# README for a code snippet.
+_CLI_MODES: tuple[EvalMode, ...] = ("substring", "cosine")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Drive the LongMemEval harness from command-line arguments.
+
+    Args:
+        argv: Argument vector; defaults to :data:`sys.argv` when ``None``.
+
+    Returns:
+        Process exit code (see module docstring).
+
+    """
+    args = _parse_args(argv)
+    try:
+        questions = load_dataset(variant=args.variant)
+    except DatasetDownloadError as exc:
+        sys.stderr.write(f"LongMemEval: {exc}\n")
+        return 1
+
+    if args.subset:
+        questions = [q for q in questions if q.question_type == args.subset]
+    if args.limit is not None:
+        questions = questions[: args.limit]
+    if not questions:
+        sys.stderr.write(
+            "LongMemEval: no questions to evaluate after applying subset/limit filters.\n",
+        )
+        return 1
+
+    provider = resolve_embedding_provider_or_exit()
+    try:
+        results = run_longmemeval_sync(
+            questions,
+            dreaming_enabled=args.dreaming,
+            embedding_provider=provider,
+            eval_mode=args.eval_mode,
+            retrieval_top_k=args.top_k,
+        )
+    except DatasetDownloadError as exc:
+        sys.stderr.write(f"LongMemEval: {exc}\n")
+        return 1
+
+    _emit_report(results, questions=questions)
+    return 0
+
+
+def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="python -m engrava.benchmarks.longmemeval",
+        description=(
+            "Run the LongMemEval public benchmark against engrava. The "
+            "default invocation uses the oracle variant in substring "
+            "mode (no LLM required, deterministic)."
+        ),
+    )
+    parser.add_argument(
+        "--variant",
+        choices=_VALID_VARIANTS,
+        default="oracle",
+        help="Dataset variant to load (default: %(default)s).",
+    )
+    parser.add_argument(
+        "--eval-mode",
+        choices=_CLI_MODES,
+        default="substring",
+        help=(
+            "Evaluation mode (default: %(default)s). The LLM-judge mode "
+            "is reachable only from Python; see README.md."
+        ),
+    )
+    parser.add_argument(
+        "--subset",
+        default=None,
+        help=(
+            "Filter questions by question_type (e.g. single-session-recall). "
+            "Omit to evaluate every question in the loaded variant."
+        ),
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="Cap the number of questions evaluated (default: no cap).",
+    )
+    parser.add_argument(
+        "--top-k",
+        type=int,
+        default=DEFAULT_TOP_K,
+        help="Retrieval top-K passed to search_hybrid (default: %(default)s).",
+    )
+    parser.add_argument(
+        "--dreaming",
+        dest="dreaming",
+        action="store_true",
+        help="Run one dreaming consolidation cycle per question.",
+    )
+    parser.add_argument(
+        "--no-dreaming",
+        dest="dreaming",
+        action="store_false",
+        help="Disable dreaming for this run (default).",
+    )
+    parser.set_defaults(dreaming=False)
+    return parser.parse_args(argv)
+
+
+def _emit_report(
+    results: LongMemEvalResults,
+    *,
+    questions: list[LongMemEvalQuestion],
+) -> None:
+    """Print a human-readable summary of the run."""
+    sys.stdout.write("LongMemEval Free harness\n")
+    sys.stdout.write("=" * 60 + "\n")
+    sys.stdout.write(f"variant questions evaluated: {results.total_questions}\n")
+    sys.stdout.write(f"dreaming enabled: {results.dreaming_enabled}\n")
+    sys.stdout.write(f"top_k: {results.top_k}\n")
+    sys.stdout.write(f"eval_mode: {results.eval_mode}\n")
+    sys.stdout.write("-" * 60 + "\n")
+    sys.stdout.write(f"aggregate score : {results.aggregate_score:.4f}\n")
+    sys.stdout.write(f"raw signal mean : {results.aggregate_raw_signal:.4f}\n")
+    if results.per_type:
+        sys.stdout.write("\nper question_type:\n")
+        for qtype, score in sorted(results.per_type.items()):
+            count = sum(1 for q in questions if q.question_type == qtype)
+            sys.stdout.write(f"  {qtype:<32} ({count:>3}): {score:.4f}\n")
+    sys.stdout.write("=" * 60 + "\n")