memtrust-cli 0.3.1__tar.gz → 0.3.2__tar.gz

memtrust_cli-0.3.2/.gitignore

@@ -0,0 +1,34 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+.env
+.env.*
+!.env.example
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+*.log
+.DS_Store
+results/*/
+!results/.gitkeep
+node_modules/
+.idea/
+.vscode/
+.claude/
+uv.lock
+memtrust-report-*.json
+
+# Locally-downloaded uv binaries staged into the npm platform packages
+# before `npm publish` -- real binaries, ~50-70MB each across 6 platforms,
+# never meant to be committed or swept into the Python sdist. See
+# npm/platforms/*/README.md for how each one is populated at publish time.
+npm/platforms/*/bin/

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: memtrust-cli
-Version: 0.3.1
-Summary: Independent, reproducible benchmark harness for agent-memory backends
+Version: 0.3.2
+Summary: Independent, reproducible benchmark harness for agent-memory backends (MemPalace, Mem0, Zep/Graphiti, OpenViking): runs LongMemEval, LoCoMo, contradiction-detection, and a dozen other evals against all four and publishes the raw logs, not vendor-curated numbers.
 Project-URL: Homepage, https://github.com/RudrenduPaul/memtrust
 Project-URL: Issues, https://github.com/RudrenduPaul/memtrust/issues
 Project-URL: Author - Rudrendu Paul, https://github.com/RudrenduPaul
@@ -9,8 +9,8 @@ Project-URL: Author - Sourav Nandy, https://github.com/Sourav-nandy-ai
 Author: Rudrendu Paul, Sourav Nandy
 License: Apache-2.0
 License-File: LICENSE
-Keywords: agent,benchmark,eval,evaluation,llm,memory
-Classifier: Development Status :: 3 - Alpha
+Keywords: agent-memory,ai-agents,benchmark,contradiction-detection,graphiti,llm-eval,llm-memory,locomo,longmemeval,mem0,memory-backend,mempalace,reproducible-benchmark,zep
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/npm/memtrust-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memtrust-cli",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "npx-runnable wrapper for the memtrust agent-memory benchmark harness. Bootstraps a verified copy of Astral's uv to fetch and run memtrust from PyPI -- no Python toolchain to provision by hand.",
   "bin": { "memtrust-cli": "bin/memtrust.js", "memtrust": "bin/memtrust.js" },
   "files": ["bin/", "LICENSE", "README.md"],

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
 
 [project]
 name = "memtrust-cli"
-version = "0.3.1"
-description = "Independent, reproducible benchmark harness for agent-memory backends"
+version = "0.3.2"
+description = "Independent, reproducible benchmark harness for agent-memory backends (MemPalace, Mem0, Zep/Graphiti, OpenViking): runs LongMemEval, LoCoMo, contradiction-detection, and a dozen other evals against all four and publishes the raw logs, not vendor-curated numbers."
 readme = "README.md"
 license = { text = "Apache-2.0" }
 requires-python = ">=3.11"
@@ -13,9 +13,13 @@ authors = [
     { name = "Rudrendu Paul" },
     { name = "Sourav Nandy" },
 ]
-keywords = ["llm", "agent", "memory", "benchmark", "eval", "evaluation"]
+keywords = [
+    "agent-memory", "benchmark", "mem0", "mempalace", "zep", "graphiti",
+    "longmemeval", "locomo", "contradiction-detection", "llm-eval",
+    "memory-backend", "reproducible-benchmark", "ai-agents", "llm-memory",
+]
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Python :: 3",
@@ -108,6 +112,18 @@ memtrust = "memtrust.cli:main"
 [tool.hatch.build.targets.wheel]
 packages = ["src/memtrust"]
 
+[tool.hatch.build.targets.sdist]
+# Without an explicit exclude, hatchling's sdist sweeps in every file in
+# the project directory that .gitignore doesn't exclude -- including
+# npm/platforms/*/bin/'s locally-downloaded uv binaries (~50-70MB each
+# across 6 platforms) whenever a contributor has them staged locally for
+# an npm publish, ballooning the Python sdist to 100MB+ for a package
+# that ships none of that. Excluded explicitly here (belt-and-suspenders
+# on top of the .gitignore entry) so the sdist stays reproducible
+# regardless of what a given contributor's working tree happens to have
+# staged locally.
+exclude = ["npm/platforms/*/bin"]
+
 [tool.ruff]
 line-length = 100
 target-version = "py311"

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/src/memtrust/__init__.py

@@ -3,7 +3,7 @@
 from importlib.metadata import PackageNotFoundError, version
 
 try:
-    __version__ = version("memtrust")
+    __version__ = version("memtrust-cli")
 except PackageNotFoundError:
     # Not installed at all (e.g. importing directly from a raw source
     # checkout with no `pip install -e .` run) -- there is no installed

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/src/memtrust/adapters/mem0_direct_adapter.py

@@ -307,6 +307,50 @@ disagree. `query()` also accepts an optional `threshold` keyword argument,
 forwarded straight
 through to the real `Memory.search()`.
 
+## Default LLM extraction is broken out of the box (mem0ai==2.0.12)
+
+`Memory.from_config()` builds an LLM client for the `"llm"` config key
+whenever the caller doesn't set one -- and until this adapter started
+setting one explicitly, that meant relying on the installed package's own
+defaults, which are broken for the median case: an operator who has only
+set `OPENAI_API_KEY`, exactly the minimum viable setup the `Configuration`
+section above documents.
+
+Confirmed by reading the installed `mem0ai==2.0.12` source directly:
+
+  * `mem0/llms/openai.py::OpenAILLM.__init__` falls back to
+    `self.config.model = "gpt-5-mini"` whenever no model is configured.
+  * `mem0/llms/base.py::LLMBase._is_reasoning_model` decides whether to
+    send `temperature`/`top_p`/`max_tokens` at all, via a hardcoded set:
+    `{"o1", "o1-preview", "o3-mini", "o3", "gpt-5", "gpt-5o", "gpt-5o-mini",
+    "gpt-5o-micro"}`, plus an `o1-`/`o3-` prefix match. `"gpt-5-mini"` --
+    the actual default model string set two files away -- is not in that
+    set and matches neither prefix (`"gpt-5o-mini"` is a different string
+    from `"gpt-5-mini"`).
+  * `BaseLlmConfig.temperature` defaults to `0.1`. Because the
+    reasoning-model check above never fires for the real default model,
+    `generate_response()` sends `temperature=0.1` on every call.
+  * `gpt-5-mini` is a real OpenAI reasoning-tier model that only accepts
+    the API's default temperature (`1`) and rejects any other value with
+    a `400 Unsupported value: 'temperature' does not support 0.1 with
+    this model` error.
+
+Net effect: a fresh install with nothing but `OPENAI_API_KEY` set fails
+every single LLM-based fact-extraction call. `contradiction`,
+`compression`, and `extraction_quality` all still complete -- mem0
+catches and logs the extraction failure rather than raising -- but every
+scoreable case comes back empty, which this project's evals report as
+`N/A (no scoreable cases)` rather than a real result.
+
+This adapter works around it the way `BaseLlmConfig`'s own docstring
+says a caller should for a model its name heuristic can't classify: pass
+`is_reasoning_model=True` explicitly in the `"llm"` config's `config`
+dict, which `LLMBase._is_reasoning_model` checks *before* falling back to
+the name heuristic. This doesn't change which model mem0 calls -- still
+whatever mem0 itself defaults to, currently `gpt-5-mini` -- it only fixes
+which request parameters get sent for it. No upstream mem0ai issue filed
+for this specific mismatch as of this writing.
+
 ## Custom fact-extraction prompt passthrough
 
 `Mem0DirectAdapter.__init__` accepts an optional `custom_instructions`
@@ -687,6 +731,15 @@ class Mem0DirectAdapter(MemoryBackendAdapter):
                 "provider": resolved_vector_store_provider,
                 "config": built_vector_store_config,
             },
+            # Works around the installed package's own default-model /
+            # reasoning-model-detection mismatch -- see this module's
+            # "Default LLM extraction is broken out of the box" section
+            # above for the exact mem0ai==2.0.12 source citations. Setting
+            # `is_reasoning_model` explicitly does not pin a model; mem0
+            # still chooses its own default (`gpt-5-mini` as of this
+            # writing) -- this only fixes which request parameters get
+            # sent to it.
+            "llm": {"provider": "openai", "config": {"is_reasoning_model": True}},
             # `MemoryConfig.custom_instructions` (mem0/configs/base.py) is
             # the real, current top-level key -- mem0 renamed the field
             # from `custom_fact_extraction_prompt` to `custom_instructions`

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/src/memtrust/cli.py

@@ -29,6 +29,7 @@ from rich.table import Table
 from memtrust import __version__
 from memtrust.adapters import ADAPTER_REGISTRY
 from memtrust.adapters.base import BackendNotConfiguredError, MemoryBackendAdapter
+from memtrust.adapters.mempalace_adapter import MemPalaceAdapter
 from memtrust.evals.compression import CompressionEvalResult, run_compression_eval
 from memtrust.evals.contradiction import ContradictionEvalResult, run_contradiction_eval
 from memtrust.evals.crash_recovery import CrashRecoveryEvalResult, run_crash_recovery_eval
@@ -55,6 +56,10 @@ from memtrust.evals.scale_stress import (
     run_scale_stress_eval,
 )
 from memtrust.evals.stats_accuracy import StatsAccuracyEvalResult, run_stats_accuracy_eval
+from memtrust.evals.temporal_kg_boundary import (
+    TemporalKGBoundaryEvalResult,
+    run_temporal_kg_boundary_eval,
+)
 from memtrust.receipt import (
     PUBLIC_KEY_ENV_VAR,
     ReceiptError,
@@ -96,6 +101,7 @@ ALL_EVALS = [
     "stats_accuracy",
     "orphan_cleanup",
     "result_consistency",
+    "temporal_kg_boundary",
 ]
 
 
@@ -869,6 +875,36 @@ def run(
             else:
                 console.print("    N/A (no scoreable cases)")
 
+        if "temporal_kg_boundary" in eval_names:
+            console.print(f"  Running Temporal-KG Boundary against {backend_name}...")
+            if isinstance(adapter, MemPalaceAdapter):
+                tkgb_result = run_temporal_kg_boundary_eval(adapter)
+                backend_report["evals"]["temporal_kg_boundary"] = _serialize_eval_result(
+                    tkgb_result
+                )
+                dcr = tkgb_result.double_count_rate
+                if dcr is not None:
+                    console.print(
+                        f"    double-count: {dcr:.1%}  clean: {tkgb_result.clean_rate:.1%}"
+                        f"  self-report-agreement: {tkgb_result.self_report_agreement_rate:.1%}"
+                    )
+                else:
+                    console.print("    N/A (no scoreable cases)")
+            else:
+                # Only MemPalaceAdapter wires the kg_add/kg_invalidate/kg_query
+                # primitives this eval calls -- see temporal_kg_boundary.py's
+                # module docstring. Reported the same way resource_sync_safety
+                # reports its own backend-specific skip, above.
+                console.print(
+                    "    N/A (only applies to the mempalace backend, not "
+                    f"{backend_name})"
+                )
+                backend_report["evals"]["temporal_kg_boundary"] = {
+                    "status": "not_applicable",
+                    "reason": f"temporal_kg_boundary only applies to the mempalace backend, "
+                    f"not {backend_name}",
+                }
+
         report["results"][backend_name] = backend_report
         close = getattr(adapter, "close", None)
         if callable(close):

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/tests/test_cli.py

@@ -296,6 +296,52 @@ def test_run_resource_sync_safety_skips_cleanly_for_unsupported_backend(
     assert rss["n_files"] == 0
 
 
+def test_temporal_kg_boundary_registered_in_eval_list() -> None:
+    assert "temporal_kg_boundary" in ALL_EVALS
+
+
+def test_run_temporal_kg_boundary_skips_cleanly_with_no_credentials(tmp_path: Path) -> None:
+    """No backend is configured at all (the autouse fixture above clears
+    every credential env var), so this never reaches the eval-dispatch
+    code path -- exercises the CLI's "unknown eval name" contract in
+    reverse: temporal_kg_boundary must be a recognized eval name that
+    still exits 0 and reports every backend SKIPPED, same as `--eval all`
+    does above."""
+    runner = CliRunner()
+    out_path = tmp_path / "report.json"
+    result = runner.invoke(
+        main,
+        ["run", "--backends", "all", "--eval", "temporal_kg_boundary", "--output", str(out_path)],
+    )
+    assert result.exit_code == 0, result.output
+    assert "SKIPPED" in result.output
+    data = json.loads(out_path.read_text())
+    for backend_result in data["results"].values():
+        assert backend_result["status"] == "skipped"
+
+
+def test_run_temporal_kg_boundary_not_applicable_for_non_mempalace_backend(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """mem0 has no kg_add/kg_invalidate/kg_query primitives (only
+    MemPalaceAdapter wires them -- see temporal_kg_boundary.py's module
+    docstring), so this exercises the CLI's backend-specific
+    not-applicable path, mirroring
+    test_run_resource_sync_safety_skips_cleanly_for_unsupported_backend
+    above."""
+    monkeypatch.setenv("MEM0_API_KEY", "test-key")
+    runner = CliRunner()
+    out_path = tmp_path / "report.json"
+    result = runner.invoke(
+        main,
+        ["run", "--backends", "mem0", "--eval", "temporal_kg_boundary", "--output", str(out_path)],
+    )
+    assert result.exit_code == 0, result.output
+    data = json.loads(out_path.read_text())
+    tkgb = data["results"]["mem0"]["evals"]["temporal_kg_boundary"]
+    assert tkgb["status"] == "not_applicable"
+
+
 def test_run_against_configured_backend_full_flow(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch, httpx_mock: HTTPXMock
 ) -> None:

{memtrust_cli-0.3.1 → memtrust_cli-0.3.2}/tests/test_mem0_direct_adapter.py

@@ -1141,6 +1141,54 @@ def test_real_memory_config_accepts_and_stores_custom_instructions(
     assert config.custom_instructions == custom_prompt
 
 
+# ---------------------------------------------------------------------------
+# Real-package regression: the installed mem0ai==2.0.12's own default LLM
+# model ("gpt-5-mini", set in mem0/llms/openai.py) is not recognized by its
+# own reasoning-model name heuristic (mem0/llms/base.py's `reasoning_models`
+# set contains "gpt-5o-mini", a different string), so a fresh install with
+# only OPENAI_API_KEY set sends `temperature=0.1` to a model that only
+# accepts the API default and 400s on every call. See module docstring's
+# "Default LLM extraction is broken out of the box" section for the full
+# finding this pins.
+# ---------------------------------------------------------------------------
+
+
+def test_config_dict_sets_is_reasoning_model_to_work_around_mem0s_broken_default(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("MEM0_DIRECT_EMBEDDER_PROVIDER", "fastembed")
+    monkeypatch.setenv("MEM0_DIRECT_VECTOR_STORE_URL", "redis://localhost:6379")
+
+    adapter = Mem0DirectAdapter()
+
+    assert adapter._config_dict["llm"] == {  # type: ignore[comparison-overlap]
+        "provider": "openai",
+        "config": {"is_reasoning_model": True},
+    }
+
+
+def test_real_openai_llm_omits_temperature_when_is_reasoning_model_is_set(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Exercises the actual installed `mem0.llms.openai.OpenAILLM` and
+    `mem0.configs.llms.openai.OpenAIConfig` classes directly with exactly
+    the `"llm"` config shape `Mem0DirectAdapter._config_dict` produces --
+    proving the workaround genuinely stops mem0's real `_get_supported_params`
+    from including `temperature` for its own default model, not just that
+    the config dict looks right in isolation.
+    """
+    monkeypatch.setenv("OPENAI_API_KEY", "sk-test-not-a-real-key")
+
+    from mem0.llms.openai import OpenAILLM
+
+    llm = OpenAILLM(config={"is_reasoning_model": True})
+
+    assert llm.config.model == "gpt-5-mini"
+    params = llm._get_supported_params(messages=[{"role": "user", "content": "hi"}])
+    assert "temperature" not in params
+    assert "max_tokens" not in params
+
+
 # ---------------------------------------------------------------------------
 # Real-package regression: mem0ai/mem0#4297 (embedding-dimension mismatch).
 # Exercises the actual installed mem0.vector_stores.qdrant.Qdrant and

memtrust_cli-0.3.1/.gitignore

@@ -1,28 +0,0 @@
-__pycache__/
-*.py[cod]
-*.egg-info/
-.eggs/
-build/
-dist/
-.venv/
-venv/
-.env
-.env.*
-!.env.example
-.pytest_cache/
-.mypy_cache/
-.ruff_cache/
-.coverage
-.coverage.*
-htmlcov/
-coverage.xml
-*.log
-.DS_Store
-results/*/
-!results/.gitkeep
-node_modules/
-.idea/
-.vscode/
-.claude/
-uv.lock
-memtrust-report-*.json