mindforge-memory 6.4.6__tar.gz → 6.4.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. {mindforge_memory-6.4.6/mindforge_memory.egg-info → mindforge_memory-6.4.9}/PKG-INFO +17 -12
  2. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/README.md +16 -11
  3. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/indexer.py +166 -13
  4. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/query.py +31 -17
  5. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/storage.py +10 -0
  6. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/version.py +2 -2
  7. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9/mindforge_memory.egg-info}/PKG-INFO +17 -12
  8. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge_memory.egg-info/SOURCES.txt +2 -0
  9. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v568_fixes.py +1 -1
  10. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v575_release_consistency.py +2 -2
  11. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v612_fixes.py +2 -2
  12. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v643_fixes.py +1 -1
  13. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v645_fixes.py +1 -1
  14. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v646_fixes.py +1 -1
  15. mindforge_memory-6.4.9/tests/test_v648_fixes.py +191 -0
  16. mindforge_memory-6.4.9/tests/test_v649_fixes.py +126 -0
  17. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/LICENSE +0 -0
  18. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/MindForge.py +0 -0
  19. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/__init__.py +0 -0
  20. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/adapters/__init__.py +0 -0
  21. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/adapters/claude_adapter.py +0 -0
  22. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/adapters/crewai_memory.py +0 -0
  23. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/adapters/generic_api.py +0 -0
  24. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/adapters/langgraph_store.py +0 -0
  25. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/adapters/openclaw_adapter.py +0 -0
  26. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/api/__init__.py +0 -0
  27. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/api/server.py +0 -0
  28. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/cli/__init__.py +0 -0
  29. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/cli/main.py +0 -0
  30. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/__init__.py +0 -0
  31. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/embedding.py +0 -0
  32. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/encryption.py +0 -0
  33. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/mindforge.py +0 -0
  34. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/paths.py +0 -0
  35. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/core/types.py +0 -0
  36. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/daemon/__init__.py +0 -0
  37. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/daemon/server.py +0 -0
  38. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/mcp/__init__.py +0 -0
  39. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/mcp/server.py +0 -0
  40. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/__init__.py +0 -0
  41. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/categorizer.py +0 -0
  42. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/conflict_detector.py +0 -0
  43. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/connectors.py +0 -0
  44. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/context_window.py +0 -0
  45. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/event_bus.py +0 -0
  46. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/evolution.py +0 -0
  47. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/experience.py +0 -0
  48. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/federated.py +0 -0
  49. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/federated_acl.py +0 -0
  50. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/follow_up.py +0 -0
  51. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/hybrid_search.py +0 -0
  52. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/importer.py +0 -0
  53. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/integrator.py +0 -0
  54. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/intent_router.py +0 -0
  55. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/knowledge_graph.py +0 -0
  56. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/memory_compressor.py +0 -0
  57. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/memory_decay.py +0 -0
  58. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/memory_passport.py +0 -0
  59. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/multi_agent.py +0 -0
  60. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/multimodal.py +0 -0
  61. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/outcome_learning.py +0 -0
  62. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/personality.py +0 -0
  63. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/principled_forgetting.py +0 -0
  64. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/privacy.py +0 -0
  65. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/recall.py +0 -0
  66. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/session_focus.py +0 -0
  67. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/share_conflict.py +0 -0
  68. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/skill_extractor.py +0 -0
  69. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/skill_pack.py +0 -0
  70. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/timeline_viz.py +0 -0
  71. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge/modules/v6.py +0 -0
  72. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge_memory.egg-info/dependency_links.txt +0 -0
  73. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge_memory.egg-info/entry_points.txt +0 -0
  74. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge_memory.egg-info/requires.txt +0 -0
  75. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/mindforge_memory.egg-info/top_level.txt +0 -0
  76. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/pyproject.toml +0 -0
  77. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/setup.cfg +0 -0
  78. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/setup.py +0 -0
  79. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_core.py +0 -0
  80. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_security_hardening.py +0 -0
  81. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_trash_hard_delete.py +0 -0
  82. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v546_features.py +0 -0
  83. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v547_fixes.py +0 -0
  84. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v548_features.py +0 -0
  85. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v550_features.py +0 -0
  86. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v552_features.py +0 -0
  87. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v554_features.py +0 -0
  88. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v556_features.py +0 -0
  89. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v557_banner_insight_wal.py +0 -0
  90. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v557_security_fixes.py +0 -0
  91. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v557_webhook_sig_export.py +0 -0
  92. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v558_audit_fixes.py +0 -0
  93. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v558_features.py +0 -0
  94. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v560_decay_gc.py +0 -0
  95. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v560_kdf_versioning.py +0 -0
  96. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v560_module_coverage.py +0 -0
  97. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v561_security.py +0 -0
  98. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v562_p2_fixes.py +0 -0
  99. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v563_entry_coverage.py +0 -0
  100. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v563_perf_fixes.py +0 -0
  101. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v565_security.py +0 -0
  102. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v566_encrypted_search.py +0 -0
  103. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v567_security_fixes.py +0 -0
  104. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v569_fts_sync.py +0 -0
  105. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v570_coverage_expansion.py +0 -0
  106. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v570_drama_and_utils.py +0 -0
  107. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v570_encrypted_fallback.py +0 -0
  108. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v571_fixes.py +0 -0
  109. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v572_security.py +0 -0
  110. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v573_encrypted_export.py +0 -0
  111. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v573_security.py +0 -0
  112. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v576_bitemporal.py +0 -0
  113. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v576_gdpr.py +0 -0
  114. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v577_agent_governance.py +0 -0
  115. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v578_embedding.py +0 -0
  116. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v579_graph.py +0 -0
  117. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v580_experience.py +0 -0
  118. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v5812_bugfixes.py +0 -0
  119. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v583_security.py +0 -0
  120. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v584_adapters.py +0 -0
  121. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v585_benchmark.py +0 -0
  122. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v586_federated_mcp.py +0 -0
  123. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v590_features.py +0 -0
  124. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v592_features.py +0 -0
  125. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v593_features.py +0 -0
  126. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v595_features.py +0 -0
  127. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v600_forgetting_followup.py +0 -0
  128. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v600_import_timeline.py +0 -0
  129. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v600_multi_agent_formal.py +0 -0
  130. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v600_outcome_passport.py +0 -0
  131. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v600_skillpack_evolution.py +0 -0
  132. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v611_fixes.py +0 -0
  133. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v612_cascade_fixes.py +0 -0
  134. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v613_integrity.py +0 -0
  135. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v620_growth_governance.py +0 -0
  136. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v630_search_explain.py +0 -0
  137. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v631_scrypt.py +0 -0
  138. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v632_explain_interfaces.py +0 -0
  139. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v635_features.py +0 -0
  140. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v635_integration.py +0 -0
  141. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v636_fixes.py +0 -0
  142. {mindforge_memory-6.4.6 → mindforge_memory-6.4.9}/tests/test_v6_ports.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mindforge-memory
3
- Version: 6.4.6
3
+ Version: 6.4.9
4
4
  Summary: AI Agent 终身记忆系统 - 四层记忆架构 · 知识图谱 · 多模态支持
5
5
  License: MIT
6
6
  Project-URL: Homepage, https://github.com/opok-ops/MindForge
@@ -36,11 +36,11 @@ Dynamic: license-file
36
36
 
37
37
  MindForge gives AI agents a persistent, structured long-term memory: a four-tier lifecycle model, knowledge graphs, hybrid retrieval, end-to-end encryption, and MCP / CLI / REST interfaces. All data stays on your machine.
38
38
 
39
- > **The story** — one developer, seven months, from v1.0 (March 2026) to v6.4.5: no team, no funding, 1,243 passing tests. [Read the journey →](https://opok-ops.github.io/MindForge/#developer)
39
+ > **The story** — one developer, seven months, from v1.0 (March 2026) to v6.4.9: no team, no funding, 1,257 passing tests. [Read the journey →](https://opok-ops.github.io/MindForge/#developer)
40
40
 
41
41
  [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
42
42
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
43
- [![Version](https://img.shields.io/badge/version-6.4.6-green.svg)](https://github.com/opok-ops/MindForge/releases)
43
+ [![Version](https://img.shields.io/badge/version-6.4.9-green.svg)](https://github.com/opok-ops/MindForge/releases)
44
44
  [![Stars](https://img.shields.io/github/stars/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/stargazers)
45
45
  [![Forks](https://img.shields.io/github/forks/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/forks)
46
46
  [![Open Issues](https://img.shields.io/github/issues/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/issues)
@@ -95,22 +95,27 @@ MindForge stats
95
95
  ```
96
96
 
97
97
 
98
- ## Scale & Quality Benchmarks (v6.4.6, measured)
98
+ ## Scale & Quality Benchmarks (v6.4.9, measured)
99
99
 
100
- | Scale | add throughput | search p50 (rare token) | search p50 (realistic mixed) | db size |
101
- |---|---|---|---|---|
102
- | 200 entries (baseline) | — | 1.8 ms | — | <1 MB |
103
- | 10,000 | 1,080/s | ~9.5 ms | ~16 ms | 10 MB |
104
- | 100,000 | ~558/s | 7.4 ms | 143 ms | 98 MB |
100
+ | Scale | add throughput | search p50 (rare token) | search p50 (realistic mixed) | search p50 (pathological all-common) | db size |
101
+ |---|---|---|---|---|---|
102
+ | 200 entries (baseline) | — | 1.8 ms | — | — | <1 MB |
103
+ | 10,000 | 1,080/s | ~9.5 ms | ~16 ms | — | 10 MB |
104
+ | 100,000 (~10M chars) | ~546/s | 6.8 ms | 3.0 ms | 0.4 ms | 98 MB |
105
105
 
106
+ - **Ten-million-character library, millisecond-class across all query types (v6.4.9)**: pathological 218→**0.4ms** (−99.8%), mixed 143→**3.0ms** (−98%), rare 6.8ms, combined 17.8ms (100k full-pipeline, clean environment). The last bottleneck — FTS5 high-frequency-phrase bm25 scoring — is now pruned query-side via trigram-df pre-check (df upper bound = min trigram df, pruned at 5% corpus size, symmetric to the TF-IDF max_df design; df served from an in-process cache — fts5vocab is a streaming virtual table, per-term probes scan it fully).
107
+ - **Index-blindness fix (v6.4.8)**: TF-IDF vocabulary was locked to the first 5 documents since v6.4.6 — incremental adds indexed as empty vectors (TF-IDF effectively half-dead; previous numbers carried by FTS5/fuzzy). Fixed with incremental vocabulary growth; all pipeline numbers above are now *true full-pipeline* measurements.
106
108
  - Retrieval quality (fixed-seed, `benchmarks/quality_bench.py`, 2k entries): needle Recall@1 = 1.0, MRR = 1.0, topic Recall@1 = 1.0, noise false-recall = 0.
107
- - Known characteristic: all-common-word queries on synthetic template data fall back to O(N) scan (documented, not a defect); realistic queries hit the pruned inverted index.
108
109
  - Full LLM-judged benchmarks (LoCoMo / LongMemEval) — interface ready, not yet run (requires external LLM API key).
109
110
  - TF-IDF degraded mode process memory at 100k: ~700 MB. Index LRU cap: 100,000 docs (MINDFORGE_INDEX_MAX_DOCS).
110
111
 
111
112
  ## What's New
112
113
 
113
- v6.1.3 – v6.4.5 shipped in ten days (2026-11-01 → 11-10; two same-day releases), guarded by **1,244 tests** (1,243 passing, 1 environment-skipped).
114
+ v6.1.3 – v6.4.9 shipped in ten days (2026-11-01 → 11-11; four releases in the last two days), guarded by **1,258 tests** (1,257 passing, 1 environment-skipped).
115
+
116
+ - **v6.4.9 — ten-million-char library at millisecond latency: FTS5 high-frequency-phrase scoring pruned** (2026-11-11). The last honest bottleneck from v6.4.8's ledger. Root cause (30k per-path timing): under the trigram tokenizer, corpus-wide phrases (template data's "陈述性记忆") match every row — bm25 full-match scoring cost 50.5ms/query, the dominant remainder of 100k mixed latency. Fix, query-side df pre-pruning symmetric to TF-IDF max_df: a phrase's df upper bound = min df of its trigrams (phrase matches ⊆ every trigram's row set); prune at 5% of corpus size; keep the lowest-ub phrase if all pruned. df is served from an in-process cache keyed by `max(rowid)` (add/update invalidate; hard-delete leaves df slightly stale = prunes more conservatively) — **two real pitfalls found en route: fts5vocab is a *streaming virtual table* — per-term probes AND `IN` batches trigger full scans (combined regressed 17→45ms before the cache)**. Plus: FTS5 path skipped on pathological fallback (no discriminating signal). Clean-environment 100k re-bench: pathological 122→**0.4ms** (−99.7%), mixed 134.9→**3.0ms** (−98%), all query types millisecond-class at ~10M chars. AI-flavored comment cleanup (essay-style docstrings → engineer-speak, info preserved). 5 new guards.
117
+
118
+ - **v6.4.8 — pathological query path & index-blindness fix: O(N) fallback eliminated, TF-IDF actually indexing** (2026-11-11). Two deeper findings beyond the v6.4.6 stress test: (1) the all-common-word fallback traversed *all* inverted chains at full length (O(N×dims), 209ms p50 at 100k), and the 0.5×N max_df threshold still let half-corpus chains into the active set; (2) **the TF-IDF vocabulary had been locked to the first 5 documents** — every incremental add after that indexed as an empty vector (the TF-IDF path was effectively half-dead since v6.4.6; published numbers were carried by FTS5/fuzzy). Fix, six cuts: max_df tightened 0.5×N→**0.05×N**; fallback semantics → **shortest-chain-first** (≤3 dims, per-chain cap 5,000, islice); fuzzy candidate budget (SQL cap 2,000 with exact-substring-first ordering); fuzzy gating (runs only on zero TF-IDF hits — the true typo-correction scenario; encrypted mode unaffected); `incremental_fit()` vocabulary growth for adds after fit (old vectors not recomputed — new tokens contribute zero on old docs mathematically); empty-query fallback-signal reset. 30k repro: pathological steady-state 3825→**4.0ms** (−99.9%), mixed pipeline 1607→1.2ms, fuzzy single-path −93%. 100k re-bench: pathological −44%, all numbers now true full-pipeline. Remaining bottleneck honestly documented: FTS5 high-frequency-phrase scoring (v6.5). New observable `_last_traversed` + deterministic budget contracts. 9 new guards.
114
119
 
115
120
  - **v6.4.6 — scale validation & retrieval benchmarks** (2026-11-10). First 100,000-entry stress test: add throughput ~600/s (linear), ~98MB db, memory peak ~700MB (TF-IDF degraded mode), search p50 7.4ms (rare token, −92% vs pre-fix) / 143ms (realistic mixed); pathological all-common query 421→218ms (−48%). Fixed a real bottleneck found by the stress test: Chinese long-query p50 was 421ms — profile showed low-IDF common words degenerating the inverted index into full-corpus scans. Fix: two-tier pruning (relative-eps + standard IR max_df stop-wording for df > 0.5×N) + heapq.nlargest; pathological all-common-word queries (synthetic-template data only) honestly documented as known O(N) characteristic, realistic queries verified unaffected (8k A/B, 30k side-by-side: 43ms mixed vs 86ms pathological). Shipped reproducible benchmarks (`benchmarks/scale_bench.py`, `benchmarks/quality_bench.py`): needle Recall@1 = 1.0 / MRR = 1.0, zero noise false-recall. Full LLM-judged benchmarks (LoCoMo/LongMemEval) deferred — needs external LLM API key. 7 new guards.
116
121
  - **v6.4.5 — audit-driven fixes** (2026-11-10; v6.4.4 skipped by maintainer decision). Two inverse doc-code inconsistencies (v6.4.3 fixed "claimed but not done"; this one fixes "done but not tracked"): `connectors.py` outgoing HTTP User-Agent was hardcoded `MindForge-Connector/5.8.9` — lying about the version to every target site since the 5.8.9 era (`event_bus.py` fixed the same disease in v5.6.8; this one slipped through). Now dynamic from the single version truth. Website "core modules" count was stale (17 in prose, 15 in stat card, 31 measured) — both fixed to the measured 31; MCP tool count 87 verified exact, untouched. Audit archive (not a defect): WAL auto-checkpoint verified healthy under daemon-like load — 4 rounds × 400 mixed ops, WAL converged at ~4.1MB (default 1000-page threshold), TRUNCATE resets to zero. 5 new guards in `tests/test_v645_fixes.py`.
@@ -183,7 +188,7 @@ v6.1.3 – v6.4.5 shipped in ten days (2026-11-01 → 11-10; two same-day releas
183
188
 
184
189
  ```
185
190
  ┌──────────────────────────────────────────────────────┐
186
- │ MindForge v6.4.6 │
191
+ │ MindForge v6.4.9 │
187
192
  │ Cognitive Layer Personality · KnowledgeGraph │
188
193
  │ MemoryEvolution · FederatedMemory │
189
194
  ├──────────────────────────────────────────────────────┤
@@ -6,11 +6,11 @@
6
6
 
7
7
  MindForge gives AI agents a persistent, structured long-term memory: a four-tier lifecycle model, knowledge graphs, hybrid retrieval, end-to-end encryption, and MCP / CLI / REST interfaces. All data stays on your machine.
8
8
 
9
- > **The story** — one developer, seven months, from v1.0 (March 2026) to v6.4.5: no team, no funding, 1,243 passing tests. [Read the journey →](https://opok-ops.github.io/MindForge/#developer)
9
+ > **The story** — one developer, seven months, from v1.0 (March 2026) to v6.4.9: no team, no funding, 1,257 passing tests. [Read the journey →](https://opok-ops.github.io/MindForge/#developer)
10
10
 
11
11
  [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
12
12
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
13
- [![Version](https://img.shields.io/badge/version-6.4.6-green.svg)](https://github.com/opok-ops/MindForge/releases)
13
+ [![Version](https://img.shields.io/badge/version-6.4.9-green.svg)](https://github.com/opok-ops/MindForge/releases)
14
14
  [![Stars](https://img.shields.io/github/stars/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/stargazers)
15
15
  [![Forks](https://img.shields.io/github/forks/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/forks)
16
16
  [![Open Issues](https://img.shields.io/github/issues/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/issues)
@@ -65,22 +65,27 @@ MindForge stats
65
65
  ```
66
66
 
67
67
 
68
- ## Scale & Quality Benchmarks (v6.4.6, measured)
68
+ ## Scale & Quality Benchmarks (v6.4.9, measured)
69
69
 
70
- | Scale | add throughput | search p50 (rare token) | search p50 (realistic mixed) | db size |
71
- |---|---|---|---|---|
72
- | 200 entries (baseline) | — | 1.8 ms | — | <1 MB |
73
- | 10,000 | 1,080/s | ~9.5 ms | ~16 ms | 10 MB |
74
- | 100,000 | ~558/s | 7.4 ms | 143 ms | 98 MB |
70
+ | Scale | add throughput | search p50 (rare token) | search p50 (realistic mixed) | search p50 (pathological all-common) | db size |
71
+ |---|---|---|---|---|---|
72
+ | 200 entries (baseline) | — | 1.8 ms | — | — | <1 MB |
73
+ | 10,000 | 1,080/s | ~9.5 ms | ~16 ms | — | 10 MB |
74
+ | 100,000 (~10M chars) | ~546/s | 6.8 ms | 3.0 ms | 0.4 ms | 98 MB |
75
75
 
76
+ - **Ten-million-character library, millisecond-class across all query types (v6.4.9)**: pathological 218→**0.4ms** (−99.8%), mixed 143→**3.0ms** (−98%), rare 6.8ms, combined 17.8ms (100k full-pipeline, clean environment). The last bottleneck — FTS5 high-frequency-phrase bm25 scoring — is now pruned query-side via trigram-df pre-check (df upper bound = min trigram df, pruned at 5% corpus size, symmetric to the TF-IDF max_df design; df served from an in-process cache — fts5vocab is a streaming virtual table, per-term probes scan it fully).
77
+ - **Index-blindness fix (v6.4.8)**: TF-IDF vocabulary was locked to the first 5 documents since v6.4.6 — incremental adds indexed as empty vectors (TF-IDF effectively half-dead; previous numbers carried by FTS5/fuzzy). Fixed with incremental vocabulary growth; all pipeline numbers above are now *true full-pipeline* measurements.
76
78
  - Retrieval quality (fixed-seed, `benchmarks/quality_bench.py`, 2k entries): needle Recall@1 = 1.0, MRR = 1.0, topic Recall@1 = 1.0, noise false-recall = 0.
77
- - Known characteristic: all-common-word queries on synthetic template data fall back to O(N) scan (documented, not a defect); realistic queries hit the pruned inverted index.
78
79
  - Full LLM-judged benchmarks (LoCoMo / LongMemEval) — interface ready, not yet run (requires external LLM API key).
79
80
  - TF-IDF degraded mode process memory at 100k: ~700 MB. Index LRU cap: 100,000 docs (MINDFORGE_INDEX_MAX_DOCS).
80
81
 
81
82
  ## What's New
82
83
 
83
- v6.1.3 – v6.4.5 shipped in ten days (2026-11-01 → 11-10; two same-day releases), guarded by **1,244 tests** (1,243 passing, 1 environment-skipped).
84
+ v6.1.3 – v6.4.9 shipped in ten days (2026-11-01 → 11-11; four releases in the last two days), guarded by **1,258 tests** (1,257 passing, 1 environment-skipped).
85
+
86
+ - **v6.4.9 — ten-million-char library at millisecond latency: FTS5 high-frequency-phrase scoring pruned** (2026-11-11). The last honest bottleneck from v6.4.8's ledger. Root cause (30k per-path timing): under the trigram tokenizer, corpus-wide phrases (template data's "陈述性记忆") match every row — bm25 full-match scoring cost 50.5ms/query, the dominant remainder of 100k mixed latency. Fix, query-side df pre-pruning symmetric to TF-IDF max_df: a phrase's df upper bound = min df of its trigrams (phrase matches ⊆ every trigram's row set); prune at 5% of corpus size; keep the lowest-ub phrase if all pruned. df is served from an in-process cache keyed by `max(rowid)` (add/update invalidate; hard-delete leaves df slightly stale = prunes more conservatively) — **two real pitfalls found en route: fts5vocab is a *streaming virtual table* — per-term probes AND `IN` batches trigger full scans (combined regressed 17→45ms before the cache)**. Plus: FTS5 path skipped on pathological fallback (no discriminating signal). Clean-environment 100k re-bench: pathological 122→**0.4ms** (−99.7%), mixed 134.9→**3.0ms** (−98%), all query types millisecond-class at ~10M chars. AI-flavored comment cleanup (essay-style docstrings → engineer-speak, info preserved). 5 new guards.
87
+
88
+ - **v6.4.8 — pathological query path & index-blindness fix: O(N) fallback eliminated, TF-IDF actually indexing** (2026-11-11). Two deeper findings beyond the v6.4.6 stress test: (1) the all-common-word fallback traversed *all* inverted chains at full length (O(N×dims), 209ms p50 at 100k), and the 0.5×N max_df threshold still let half-corpus chains into the active set; (2) **the TF-IDF vocabulary had been locked to the first 5 documents** — every incremental add after that indexed as an empty vector (the TF-IDF path was effectively half-dead since v6.4.6; published numbers were carried by FTS5/fuzzy). Fix, six cuts: max_df tightened 0.5×N→**0.05×N**; fallback semantics → **shortest-chain-first** (≤3 dims, per-chain cap 5,000, islice); fuzzy candidate budget (SQL cap 2,000 with exact-substring-first ordering); fuzzy gating (runs only on zero TF-IDF hits — the true typo-correction scenario; encrypted mode unaffected); `incremental_fit()` vocabulary growth for adds after fit (old vectors not recomputed — new tokens contribute zero on old docs mathematically); empty-query fallback-signal reset. 30k repro: pathological steady-state 3825→**4.0ms** (−99.9%), mixed pipeline 1607→1.2ms, fuzzy single-path −93%. 100k re-bench: pathological −44%, all numbers now true full-pipeline. Remaining bottleneck honestly documented: FTS5 high-frequency-phrase scoring (v6.5). New observable `_last_traversed` + deterministic budget contracts. 9 new guards.
84
89
 
85
90
  - **v6.4.6 — scale validation & retrieval benchmarks** (2026-11-10). First 100,000-entry stress test: add throughput ~600/s (linear), ~98MB db, memory peak ~700MB (TF-IDF degraded mode), search p50 7.4ms (rare token, −92% vs pre-fix) / 143ms (realistic mixed); pathological all-common query 421→218ms (−48%). Fixed a real bottleneck found by the stress test: Chinese long-query p50 was 421ms — profile showed low-IDF common words degenerating the inverted index into full-corpus scans. Fix: two-tier pruning (relative-eps + standard IR max_df stop-wording for df > 0.5×N) + heapq.nlargest; pathological all-common-word queries (synthetic-template data only) honestly documented as known O(N) characteristic, realistic queries verified unaffected (8k A/B, 30k side-by-side: 43ms mixed vs 86ms pathological). Shipped reproducible benchmarks (`benchmarks/scale_bench.py`, `benchmarks/quality_bench.py`): needle Recall@1 = 1.0 / MRR = 1.0, zero noise false-recall. Full LLM-judged benchmarks (LoCoMo/LongMemEval) deferred — needs external LLM API key. 7 new guards.
86
91
  - **v6.4.5 — audit-driven fixes** (2026-11-10; v6.4.4 skipped by maintainer decision). Two inverse doc-code inconsistencies (v6.4.3 fixed "claimed but not done"; this one fixes "done but not tracked"): `connectors.py` outgoing HTTP User-Agent was hardcoded `MindForge-Connector/5.8.9` — lying about the version to every target site since the 5.8.9 era (`event_bus.py` fixed the same disease in v5.6.8; this one slipped through). Now dynamic from the single version truth. Website "core modules" count was stale (17 in prose, 15 in stat card, 31 measured) — both fixed to the measured 31; MCP tool count 87 verified exact, untouched. Audit archive (not a defect): WAL auto-checkpoint verified healthy under daemon-like load — 4 rounds × 400 mixed ops, WAL converged at ~4.1MB (default 1000-page threshold), TRUNCATE resets to zero. 5 new guards in `tests/test_v645_fixes.py`.
@@ -153,7 +158,7 @@ v6.1.3 – v6.4.5 shipped in ten days (2026-11-01 → 11-10; two same-day releas
153
158
 
154
159
  ```
155
160
  ┌──────────────────────────────────────────────────────┐
156
- │ MindForge v6.4.6 │
161
+ │ MindForge v6.4.9 │
157
162
  │ Cognitive Layer Personality · KnowledgeGraph │
158
163
  │ MemoryEvolution · FederatedMemory │
159
164
  ├──────────────────────────────────────────────────────┤
@@ -9,6 +9,7 @@ import os
9
9
  import re
10
10
  import sqlite3
11
11
  from collections import Counter, defaultdict
12
+ from itertools import islice
12
13
  from typing import List, Dict, Tuple, Optional
13
14
  from dataclasses import dataclass
14
15
 
@@ -28,6 +29,7 @@ class TFIDFVectorizer:
28
29
  def __init__(self):
29
30
  self.vocab: Dict[str, int] = {}
30
31
  self.idf: Dict[str, float] = {}
32
+ self._df: Dict[str, int] = {} # v6.4.8: 增量 fit 的精确 df 延续
31
33
  self.doc_count: int = 0
32
34
 
33
35
  def _tokenize(self, text: str) -> List[str]:
@@ -59,6 +61,33 @@ class TFIDFVectorizer:
59
61
 
60
62
  for word, idx in self.vocab.items():
61
63
  self.idf[word] = math.log((self.doc_count + 1) / (df[word] + 1)) + 1
64
+ # v6.4.8: df 表留存,供 incremental_fit 延续精确计数
65
+ self._df = dict(df)
66
+
67
+ def incremental_fit(self, text: str) -> None:
68
+ """v6.4.8 新增:单文档增量词表扩展。
69
+
70
+ 修复「fit 后新增文档失索引」:新 token 按当前 doc_count+1 估
71
+ IDF 补进词表;已知 token 的 df 计数 +1(IDF 漂移不回算旧向量,
72
+ 见 index_memory 注释)。维度号从当前词表长度起追加,与既有
73
+ 向量维度兼容(旧文档在新词上恒为零分量)。
74
+ """
75
+ if not self._df:
76
+ # 未 fit 过(不应发生——调用方保证 fitted),防御性直通
77
+ return
78
+ self.doc_count += 1
79
+ next_idx = len(self.vocab)
80
+ for token in set(self._tokenize(text)):
81
+ if token in self.vocab:
82
+ self._df[token] = self._df.get(token, 0) + 1
83
+ self.idf[token] = math.log(
84
+ (self.doc_count + 1) / (self._df[token] + 1)) + 1
85
+ else:
86
+ self._df[token] = 1
87
+ self.vocab[token] = next_idx
88
+ next_idx += 1
89
+ self.idf[token] = math.log(
90
+ (self.doc_count + 1) / 2) + 1
62
91
 
63
92
  def transform(self, text: str) -> Dict[int, float]:
64
93
  tokens = self._tokenize(text)
@@ -108,6 +137,10 @@ class VectorIndex:
108
137
  self.metadata: Dict[str, Dict] = {}
109
138
  self._norms: Dict[str, float] = {} # 预计算 L2 范数缓存
110
139
  self._postings: Dict[int, Dict[str, float]] = {} # 倒排链
140
+ # v6.4.8: 上次 search 实际遍历的倒排链条目数(性能契约可观测)
141
+ self._last_traversed = 0
142
+ # v6.4.8: 上次 search 是否走了病态回退(全库词查询信号)
143
+ self.last_fallback = False
111
144
 
112
145
  @staticmethod
113
146
  def _as_sparse(vector) -> Dict[int, float]:
@@ -150,31 +183,70 @@ class VectorIndex:
150
183
  q = self._as_sparse(query_vector)
151
184
  query_norm = math.sqrt(sum(v * v for v in q.values()))
152
185
  if query_norm == 0:
186
+ # v6.4.8: 空查询(无 token/全 vocab miss)≠ 全库词回退。
187
+ # 显式复位 last_fallback,防止继承上一查询的 True 导致
188
+ # query.py 误跳过 fuzzy 拼写兜底(空查询正是 fuzzy 的
189
+ # 设计场景:用户词打错 → 全 miss → fuzzy 纠错召回)。
190
+ self.last_fallback = False
191
+ self._last_traversed = 0
153
192
  return [(doc_id, 0.0) for doc_id in list(self.vectors.keys())[:top_k]]
154
193
 
155
194
  # v6.4.6 perf: 低区分度查询维剪枝。100k 规模压测发现中文长查询
156
195
  # p50 421ms,profile 定位到常见词(如「条目/陈述性/记忆」)的倒排
157
- # 链全量遍历。两层剪枝:eps 相对阈值 + max_df 停用词(df > 0.5×N
158
- # 的维是平滑 IDF 下 qval≈tf×1 的全库词——遍历整条链却无区分度,
159
- # 所有文档点积同加近似常数,top-K 排序不变;30k profile 实证
160
- # eps 剪不掉全库词,必须按链长剪)。
196
+ # 链全量遍历。两层剪枝:eps 相对阈值 + max_df 停用词。
197
+ # v6.4.8 perf: max_df 从 0.5×N 收紧到 0.05×N。0.5 时半库词链
198
+ # (数十万级遍历)仍进 active,100k 实测 mixed_realistic p50
199
+ # 134ms。常见词的点积贡献近似常数(30k profile 实证 eps 剪不掉
200
+ # 全库词,v6.4.6 注释已论证 top-K 排序不变),链长超库容 5% 的
201
+ # 维一律剪——100k 库下 active 链长上界 5k,遍历毫秒级。
161
202
  _EPS_REL = 0.02
162
- _half = 0.5 * len(self.vectors)
203
+ _max_df_len = 0.05 * len(self.vectors)
163
204
  active = [(idx, qval) for idx, qval in q.items()
164
205
  if qval > _EPS_REL * query_norm
165
- and len(self._postings.get(idx, ())) <= _half]
206
+ and len(self._postings.get(idx, ())) <= _max_df_len]
207
+ self._last_traversed = 0
208
+ # v6.4.8: 全库词回退信号(供 query.py 跳过 fuzzy/FTS5 兜底)
209
+ self.last_fallback = False
166
210
  if not active:
167
- # 全部维度被剪(极端退化查询):退回原语义,保证不空召回
168
- active = list(q.items())
211
+ self.last_fallback = True
212
+ # v6.4.8: 回退不再遍历全部倒排链(旧 O(N×dims) 是 100k
213
+ # 全库词查询 209ms 的根因)。全库词点积贡献是常数,top-K
214
+ # 只由相对区分度决定:取最短 _FALLBACK_DIMS 维(区分度最高
215
+ # 的信号),单链截断 _TRAVERSE_CAP 项(平分数下截断不改
216
+ # top-K 语义)。
217
+ _FALLBACK_DIMS = 3
218
+ _TRAVERSE_CAP = 5000
219
+ ranked = sorted(
220
+ q.items(),
221
+ key=lambda kv: len(self._postings.get(kv[0], ())))
222
+ active = ranked[:_FALLBACK_DIMS]
223
+ # islice 免建截断副本(dict(list(x)[:cap]) 的 15k 次 hash
224
+ # 插入在实测里占 ~3ms,30k 病态复现 14→11ms)
225
+ dots: Dict[str, float] = {}
226
+ for idx, qval in active:
227
+ chain = self._postings.get(idx, {})
228
+ pairs = (chain.items() if len(chain) <= _TRAVERSE_CAP
229
+ else islice(chain.items(), _TRAVERSE_CAP))
230
+ for doc_id, dval in pairs:
231
+ dots[doc_id] = dots.get(doc_id, 0.0) + qval * dval
232
+ self._last_traversed += 1
233
+ return self._topk_from_dots(dots, query_norm, top_k)
169
234
 
170
235
  # 倒排链累加点积:只访问与查询共享非零维度的候选文档
236
+ # (active 链长上界 _max_df_len,遍历量与库容解耦)
171
237
  dots: Dict[str, float] = {}
172
238
  for idx, qval in active:
173
239
  for doc_id, dval in self._postings.get(idx, {}).items():
174
240
  dots[doc_id] = dots.get(doc_id, 0.0) + qval * dval
241
+ self._last_traversed += 1
175
242
 
176
243
  # v6.4.6 perf: heapq.nlargest 替代全量 sort——候选集大时
177
244
  # O(n·log k) 优于 O(n·log n),且免建全量列表排序的中间开销。
245
+ return self._topk_from_dots(dots, query_norm, top_k)
246
+
247
+ def _topk_from_dots(self, dots: Dict[str, float], query_norm: float,
248
+ top_k: int) -> List[Tuple[str, float]]:
249
+ """点积累积结果归一化 + top-K 选取(v6.4.8 抽出,回退/正常路径共用)"""
178
250
  return heapq.nlargest(
179
251
  top_k,
180
252
  ((doc_id, dot / (query_norm * self._norms[doc_id]))
@@ -212,6 +284,11 @@ class IndexEngine:
212
284
  self.db_path = db_path
213
285
  self.vectorizer = TFIDFVectorizer()
214
286
  self.vector_index = VectorIndex()
287
+ # v6.4.8: 病态回退信号门面透传(见 search())
288
+ self.last_fallback = False
289
+ # v6.4.9: FTS5 trigram df 进程内缓存(见 _prune_fts_query)
290
+ self._fts_vocab_cache: Optional[Dict[str, int]] = None
291
+ self._fts_vocab_rowid: int = -1
215
292
  self._fitted = False
216
293
  self._hydrated = False
217
294
  self._doc_texts: Dict[str, str] = {}
@@ -269,16 +346,20 @@ class IndexEngine:
269
346
  """索引记忆"""
270
347
  self._doc_texts[doc_id] = text
271
348
 
349
+ # v5.6.5 P1 #5: 渐进 fit 触发点(LRU 守卫契约:add 即索引)。
350
+ # v6.4.8 fix: fit 后的增量 add 先 incremental_fit 扩词表再
351
+ # transform——旧逻辑词表锁死在前 5 条文档,此后增量 add 的
352
+ # transform 全 vocab miss(空向量入索引,TF-IDF 路半瘫痪)。
272
353
  if not self._fitted and len(self._doc_texts) >= 5:
273
354
  self._fit_vectorizer()
355
+ elif self._fitted:
356
+ self.vectorizer.incremental_fit(text)
274
357
 
275
358
  if self._fitted:
276
- # v5.6.3 性能修复:直接存稀疏向量({dim: weight}),不再展开为
277
- # 词表长度的稠密 list
278
359
  vector_dict = self.vectorizer.transform(text)
279
360
  self.vector_index.add(doc_id, vector_dict, metadata)
280
361
 
281
- # v5.6.5 P1 #5:写入后做 LRU 淘汰(覆盖写不增加条目数,故在 add 之后)
362
+ # v5.6.5 P1 #5: 写入后 LRU 淘汰
282
363
  self._evict_if_needed()
283
364
 
284
365
  def remove_memory(self, doc_id: str):
@@ -292,11 +373,17 @@ class IndexEngine:
292
373
  self._fit_vectorizer()
293
374
 
294
375
  if not self._fitted:
376
+ # v6.4.8: 降级路径同样透传病态回退信号(保持门面契约一致)
377
+ self.last_fallback = False
295
378
  return self._keyword_search(query, top_k)
296
379
 
297
380
  # v5.6.3 性能修复:查询向量同样保持稀疏,走倒排链检索
298
381
  query_vec_dict = self.vectorizer.transform(query)
299
- return self.vector_index.search(query_vec_dict, top_k)
382
+ results = self.vector_index.search(query_vec_dict, top_k)
383
+ # v6.4.8: 把 VectorIndex 的病态回退信号透传到门面层
384
+ # (query.py 据此跳过 fuzzy 兜底路)
385
+ self.last_fallback = self.vector_index.last_fallback
386
+ return results
300
387
 
301
388
  def _keyword_search(self, query: str, top_k: int) -> List[Tuple[str, float]]:
302
389
  """关键词搜索(降级方案)"""
@@ -352,9 +439,16 @@ class IndexEngine:
352
439
 
353
440
  def fts_search(self, conn: sqlite3.Connection, query: str,
354
441
  top_k: int = 10) -> List[Tuple[str, float]]:
355
- """FTS5 全文搜索(v5.4.7 修复:特殊字符转义)"""
442
+ """FTS5 全文搜索(v5.4.7 修复:特殊字符转义)
443
+
444
+ v6.4.9: 多词查询先经 _prune_fts_query 剔除高频短语——
445
+ trigram 索引下全库短语的 bm25 评分是 100k 延迟的大头。
446
+ """
356
447
  try:
357
448
  escaped = self._escape_fts5_query(query)
449
+ if not escaped:
450
+ return []
451
+ escaped = self._prune_fts_query(conn, query, escaped)
358
452
  if not escaped:
359
453
  return []
360
454
  rows = conn.execute("""
@@ -380,3 +474,62 @@ class IndexEngine:
380
474
  return results
381
475
  except sqlite3.OperationalError:
382
476
  return []
477
+
478
+ # v6.4.9: 高频短语剪枝比例(与 TF-IDF max_df 0.05 对齐)
479
+ _FTS_MAX_DF_RATIO = 0.05
480
+
481
+ def _prune_fts_query(self, conn: sqlite3.Connection, query: str,
482
+ escaped: str) -> str:
483
+ """剔除 MATCH 里必然无区分度的高频短语。
484
+
485
+ 短语 df 上界 = min(该词全部 trigram 的 df)(短语匹配行 ⊆
486
+ 每个 trigram 行集的交集)。上界超库容 5% 的词剔除;全被剔除
487
+ 时留上界最小的那个,防空 MATCH。
488
+ df 来自 fts5vocab 的进程内缓存(首查全表读一次,此后 O(1)
489
+ 点查;以 memories.max_rowid 为失效标记,覆盖 add/update;
490
+ 硬删导致的 df 缓存偏大只会让剪枝偏保守,5% 阈值级近似)。
491
+ """
492
+ tokens = [t for t in query.split() if t]
493
+ if len(tokens) <= 1:
494
+ return escaped
495
+ try:
496
+ row = conn.execute(
497
+ "SELECT max(rowid) FROM memories").fetchone()
498
+ total = (row[0] or 0) if row else 0
499
+ if total < 200:
500
+ return escaped # 小库 df 语义不稳
501
+ if (self._fts_vocab_cache is None
502
+ or total != self._fts_vocab_rowid):
503
+ # v6.4.9: vocab 是流式虚拟表,逐 term 点查/IN 批查
504
+ # 都触发全表扫(combined 100k 45ms 回退的根因)——
505
+ # 全表读一次进 dict,之后纯内存点查。
506
+ conn.execute(
507
+ "CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts_v "
508
+ "USING fts5vocab('memory_fts', 'row')")
509
+ self._fts_vocab_cache = dict(
510
+ conn.execute("SELECT term, doc FROM memory_fts_v"))
511
+ self._fts_vocab_rowid = total
512
+ df_map = self._fts_vocab_cache
513
+ budget = self._FTS_MAX_DF_RATIO * total
514
+ kept = []
515
+ best, best_ub = None, None
516
+ for tok in tokens:
517
+ if len(tok) < 3:
518
+ ub = 0 # trigram 查不了短词,保留
519
+ else:
520
+ ub = min(df_map.get(tok[i:i + 3], 0)
521
+ for i in range(len(tok) - 2))
522
+ if ub <= budget:
523
+ kept.append(tok)
524
+ if best_ub is None or ub < best_ub:
525
+ best, best_ub = tok, ub
526
+ if not kept:
527
+ kept = [best] if best else []
528
+ return " OR ".join(
529
+ '"' + t.replace('"', '""') + '"' for t in kept)
530
+ if len(kept) == len(tokens):
531
+ return escaped
532
+ return " OR ".join(
533
+ '"' + t.replace('"', '""') + '"' for t in kept)
534
+ except sqlite3.Error:
535
+ return escaped
@@ -257,11 +257,19 @@ class QueryEngine:
257
257
  _bd(doc_id)["winner"] = "tfidf"
258
258
  _bd(doc_id)["final"] = s
259
259
 
260
- # 路 2:Fuzzy(TF-IDF 召回不足时补充;v5.7.1:加密模式下总是跑,
261
- # 供加密兜底复用结果,避免二次全表扫描)
262
- positive = sum(1 for _, s in tfidf_results if s >= min_relevance)
260
+ # 路 2:Fuzzy(拼写纠错兜底;v5.7.1:加密模式下总是跑供加密
261
+ # 兜底复用,避免二次全表扫描)
262
+ # v6.4.8: 触发收紧为「TF-IDF 零有效命中」。fuzzy 的价值在
263
+ # 「查询词全不在库」(打错字场景),此时 TF-IDF 必空。有命中
264
+ # (含低分)时 fuzzy 只补凑数条目还白扫候选集(30k 实测
265
+ # 140ms/查询)。全库词回退(tfidf_fallback)同理跳过。
266
+ # 注意按非零分判定:空查询走零分 top-K 兼容分支,
267
+ # tfidf_results 恒非空,不看分会死锁拼写兜底。
268
+ tfidf_fallback = getattr(self.index, "last_fallback", False)
269
+ tfidf_effective = any(s > 0.0 for _, s in tfidf_results)
263
270
  fuzzy_hits = []
264
- if self.storage.encrypted or positive < max_results:
271
+ if (self.storage.encrypted
272
+ or (not tfidf_effective and not tfidf_fallback)):
265
273
  fuzzy_hits = self.storage.fuzzy_search(
266
274
  query, limit=max_results * 2, threshold=0.1)
267
275
  for item in fuzzy_hits:
@@ -276,23 +284,29 @@ class QueryEngine:
276
284
  _bd(entry.id)["final"] = s
277
285
 
278
286
  # 路 3:FTS5 全文检索
287
+ # v6.4.9: 全库词回退时跳过(FTS5 对高频短语的 bm25 全匹配行
288
+ # 评分是病态查询的主要剩余成本,且无区分度信号)。正常查询
289
+ # 走 fts_search 的 df 剪枝。
279
290
  fts5_used = False
280
- # v5.6.2 安全修复:初始化默认值,防止异常路径下 UnboundLocalError
291
+ # v5.6.2: 防异常路径下 UnboundLocalError
281
292
  strategy_used = "tfidf+fuzzy"
282
293
  try:
283
- conn = self.storage._get_conn()
284
- fts_results = self.index.fts_search(conn, query, top_k=max_results * 3)
285
- for doc_id, s in fts_results:
286
- if explain:
287
- _bd(doc_id)["fts5"] = max(s, _bd(doc_id)["fts5"])
288
- if s >= score_map.get(doc_id, 0.0):
289
- score_map[doc_id] = s
294
+ if not tfidf_fallback:
295
+ conn = self.storage._get_conn()
296
+ fts_results = self.index.fts_search(conn, query, top_k=max_results * 3)
297
+ for doc_id, s in fts_results:
290
298
  if explain:
291
- _bd(doc_id)["winner"] = "fts5"
292
- _bd(doc_id)["final"] = s
293
- if fts_results:
294
- fts5_used = True
295
- strategy_used = "tfidf+fuzzy+fts5"
299
+ _bd(doc_id)["fts5"] = max(s, _bd(doc_id)["fts5"])
300
+ if s >= score_map.get(doc_id, 0.0):
301
+ score_map[doc_id] = s
302
+ if explain:
303
+ _bd(doc_id)["winner"] = "fts5"
304
+ _bd(doc_id)["final"] = s
305
+ if fts_results:
306
+ fts5_used = True
307
+ strategy_used = "tfidf+fuzzy+fts5"
308
+ else:
309
+ strategy_used = "tfidf+fuzzy"
296
310
  else:
297
311
  strategy_used = "tfidf+fuzzy"
298
312
  except Exception:
@@ -9127,6 +9127,16 @@ class StorageEngine:
9127
9127
  params_or.append(f"%{_ch}%")
9128
9128
  cand_query = base_query + " AND (" + " OR ".join(conds) + ")"
9129
9129
 
9130
+ # v6.4.8: 候选集预算。短条目库 + 16 字级中文查询会让候选
9131
+ # ≈ 全库(30k 实测 1642ms);超预算只发生在无区分度查询,
9132
+ # 平分数下截断不改 top-K。排序:精确子串命中(LIKE,最强
9133
+ # fuzzy 信号)排最前,短文本兜底路按 rowid 截断。
9134
+ _FUZZY_CAND_CAP = 2000
9135
+ cand_query += (
9136
+ " ORDER BY (CASE WHEN content LIKE ? THEN 0 ELSE 1 END), rowid"
9137
+ f" LIMIT {_FUZZY_CAND_CAP}")
9138
+ params_or.append(f"%{query_lower}%")
9139
+
9130
9140
  def _entry_iter():
9131
9141
  for _r in conn.execute(cand_query, params + params_or):
9132
9142
  yield self._row_to_entry(_r)
@@ -20,10 +20,10 @@ core/mindforge.py、cli/main.py、mcp/server.py),每处都带一份硬编码
20
20
  from typing import Tuple
21
21
 
22
22
  #: 当前发布版本(唯一真值)
23
- __version__: str = "6.4.6"
23
+ __version__: str = "6.4.9"
24
24
 
25
25
  #: 结构化版本元组,便于比较(如 `VERSION_INFO >= (6, 1, 0)`)
26
- VERSION_INFO: Tuple[int, int, int] = (6, 4, 6)
26
+ VERSION_INFO: Tuple[int, int, int] = (6, 4, 9)
27
27
 
28
28
  #: 下一个大版本的目标版本号(供 v6 端口预留引用)
29
29
  NEXT_MAJOR_TARGET: str = "7.0.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mindforge-memory
3
- Version: 6.4.6
3
+ Version: 6.4.9
4
4
  Summary: AI Agent 终身记忆系统 - 四层记忆架构 · 知识图谱 · 多模态支持
5
5
  License: MIT
6
6
  Project-URL: Homepage, https://github.com/opok-ops/MindForge
@@ -36,11 +36,11 @@ Dynamic: license-file
36
36
 
37
37
  MindForge gives AI agents a persistent, structured long-term memory: a four-tier lifecycle model, knowledge graphs, hybrid retrieval, end-to-end encryption, and MCP / CLI / REST interfaces. All data stays on your machine.
38
38
 
39
- > **The story** — one developer, seven months, from v1.0 (March 2026) to v6.4.5: no team, no funding, 1,243 passing tests. [Read the journey →](https://opok-ops.github.io/MindForge/#developer)
39
+ > **The story** — one developer, seven months, from v1.0 (March 2026) to v6.4.9: no team, no funding, 1,257 passing tests. [Read the journey →](https://opok-ops.github.io/MindForge/#developer)
40
40
 
41
41
  [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
42
42
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
43
- [![Version](https://img.shields.io/badge/version-6.4.6-green.svg)](https://github.com/opok-ops/MindForge/releases)
43
+ [![Version](https://img.shields.io/badge/version-6.4.9-green.svg)](https://github.com/opok-ops/MindForge/releases)
44
44
  [![Stars](https://img.shields.io/github/stars/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/stargazers)
45
45
  [![Forks](https://img.shields.io/github/forks/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/forks)
46
46
  [![Open Issues](https://img.shields.io/github/issues/opok-ops/MindForge.svg)](https://github.com/opok-ops/MindForge/issues)
@@ -95,22 +95,27 @@ MindForge stats
95
95
  ```
96
96
 
97
97
 
98
- ## Scale & Quality Benchmarks (v6.4.6, measured)
98
+ ## Scale & Quality Benchmarks (v6.4.9, measured)
99
99
 
100
- | Scale | add throughput | search p50 (rare token) | search p50 (realistic mixed) | db size |
101
- |---|---|---|---|---|
102
- | 200 entries (baseline) | — | 1.8 ms | — | <1 MB |
103
- | 10,000 | 1,080/s | ~9.5 ms | ~16 ms | 10 MB |
104
- | 100,000 | ~558/s | 7.4 ms | 143 ms | 98 MB |
100
+ | Scale | add throughput | search p50 (rare token) | search p50 (realistic mixed) | search p50 (pathological all-common) | db size |
101
+ |---|---|---|---|---|---|
102
+ | 200 entries (baseline) | — | 1.8 ms | — | — | <1 MB |
103
+ | 10,000 | 1,080/s | ~9.5 ms | ~16 ms | — | 10 MB |
104
+ | 100,000 (~10M chars) | ~546/s | 6.8 ms | 3.0 ms | 0.4 ms | 98 MB |
105
105
 
106
+ - **Ten-million-character library, millisecond-class across all query types (v6.4.9)**: pathological 218→**0.4ms** (−99.8%), mixed 143→**3.0ms** (−98%), rare 6.8ms, combined 17.8ms (100k full-pipeline, clean environment). The last bottleneck — FTS5 high-frequency-phrase bm25 scoring — is now pruned query-side via trigram-df pre-check (df upper bound = min trigram df, pruned at 5% corpus size, symmetric to the TF-IDF max_df design; df served from an in-process cache — fts5vocab is a streaming virtual table, per-term probes scan it fully).
107
+ - **Index-blindness fix (v6.4.8)**: TF-IDF vocabulary was locked to the first 5 documents since v6.4.6 — incremental adds indexed as empty vectors (TF-IDF effectively half-dead; previous numbers carried by FTS5/fuzzy). Fixed with incremental vocabulary growth; all pipeline numbers above are now *true full-pipeline* measurements.
106
108
  - Retrieval quality (fixed-seed, `benchmarks/quality_bench.py`, 2k entries): needle Recall@1 = 1.0, MRR = 1.0, topic Recall@1 = 1.0, noise false-recall = 0.
107
- - Known characteristic: all-common-word queries on synthetic template data fall back to O(N) scan (documented, not a defect); realistic queries hit the pruned inverted index.
108
109
  - Full LLM-judged benchmarks (LoCoMo / LongMemEval) — interface ready, not yet run (requires external LLM API key).
109
110
  - TF-IDF degraded mode process memory at 100k: ~700 MB. Index LRU cap: 100,000 docs (MINDFORGE_INDEX_MAX_DOCS).
110
111
 
111
112
  ## What's New
112
113
 
113
- v6.1.3 – v6.4.5 shipped in ten days (2026-11-01 → 11-10; two same-day releases), guarded by **1,244 tests** (1,243 passing, 1 environment-skipped).
114
+ v6.1.3 – v6.4.9 shipped in ten days (2026-11-01 → 11-11; four releases in the last two days), guarded by **1,258 tests** (1,257 passing, 1 environment-skipped).
115
+
116
+ - **v6.4.9 — ten-million-char library at millisecond latency: FTS5 high-frequency-phrase scoring pruned** (2026-11-11). The last honest bottleneck from v6.4.8's ledger. Root cause (30k per-path timing): under the trigram tokenizer, corpus-wide phrases (template data's "陈述性记忆") match every row — bm25 full-match scoring cost 50.5ms/query, the dominant remainder of 100k mixed latency. Fix, query-side df pre-pruning symmetric to TF-IDF max_df: a phrase's df upper bound = min df of its trigrams (phrase matches ⊆ every trigram's row set); prune at 5% of corpus size; keep the lowest-ub phrase if all pruned. df is served from an in-process cache keyed by `max(rowid)` (add/update invalidate; hard-delete leaves df slightly stale = prunes more conservatively) — **two real pitfalls found en route: fts5vocab is a *streaming virtual table* — per-term probes AND `IN` batches trigger full scans (combined regressed 17→45ms before the cache)**. Plus: FTS5 path skipped on pathological fallback (no discriminating signal). Clean-environment 100k re-bench: pathological 122→**0.4ms** (−99.7%), mixed 134.9→**3.0ms** (−98%), all query types millisecond-class at ~10M chars. AI-flavored comment cleanup (essay-style docstrings → engineer-speak, info preserved). 5 new guards.
117
+
118
+ - **v6.4.8 — pathological query path & index-blindness fix: O(N) fallback eliminated, TF-IDF actually indexing** (2026-11-11). Two deeper findings beyond the v6.4.6 stress test: (1) the all-common-word fallback traversed *all* inverted chains at full length (O(N×dims), 209ms p50 at 100k), and the 0.5×N max_df threshold still let half-corpus chains into the active set; (2) **the TF-IDF vocabulary had been locked to the first 5 documents** — every incremental add after that indexed as an empty vector (the TF-IDF path was effectively half-dead since v6.4.6; published numbers were carried by FTS5/fuzzy). Fix, six cuts: max_df tightened 0.5×N→**0.05×N**; fallback semantics → **shortest-chain-first** (≤3 dims, per-chain cap 5,000, islice); fuzzy candidate budget (SQL cap 2,000 with exact-substring-first ordering); fuzzy gating (runs only on zero TF-IDF hits — the true typo-correction scenario; encrypted mode unaffected); `incremental_fit()` vocabulary growth for adds after fit (old vectors not recomputed — new tokens contribute zero on old docs mathematically); empty-query fallback-signal reset. 30k repro: pathological steady-state 3825→**4.0ms** (−99.9%), mixed pipeline 1607→1.2ms, fuzzy single-path −93%. 100k re-bench: pathological −44%, all numbers now true full-pipeline. Remaining bottleneck honestly documented: FTS5 high-frequency-phrase scoring (v6.5). New observable `_last_traversed` + deterministic budget contracts. 9 new guards.
114
119
 
115
120
  - **v6.4.6 — scale validation & retrieval benchmarks** (2026-11-10). First 100,000-entry stress test: add throughput ~600/s (linear), ~98MB db, memory peak ~700MB (TF-IDF degraded mode), search p50 7.4ms (rare token, −92% vs pre-fix) / 143ms (realistic mixed); pathological all-common query 421→218ms (−48%). Fixed a real bottleneck found by the stress test: Chinese long-query p50 was 421ms — profile showed low-IDF common words degenerating the inverted index into full-corpus scans. Fix: two-tier pruning (relative-eps + standard IR max_df stop-wording for df > 0.5×N) + heapq.nlargest; pathological all-common-word queries (synthetic-template data only) honestly documented as known O(N) characteristic, realistic queries verified unaffected (8k A/B, 30k side-by-side: 43ms mixed vs 86ms pathological). Shipped reproducible benchmarks (`benchmarks/scale_bench.py`, `benchmarks/quality_bench.py`): needle Recall@1 = 1.0 / MRR = 1.0, zero noise false-recall. Full LLM-judged benchmarks (LoCoMo/LongMemEval) deferred — needs external LLM API key. 7 new guards.
116
121
  - **v6.4.5 — audit-driven fixes** (2026-11-10; v6.4.4 skipped by maintainer decision). Two inverse doc-code inconsistencies (v6.4.3 fixed "claimed but not done"; this one fixes "done but not tracked"): `connectors.py` outgoing HTTP User-Agent was hardcoded `MindForge-Connector/5.8.9` — lying about the version to every target site since the 5.8.9 era (`event_bus.py` fixed the same disease in v5.6.8; this one slipped through). Now dynamic from the single version truth. Website "core modules" count was stale (17 in prose, 15 in stat card, 31 measured) — both fixed to the measured 31; MCP tool count 87 verified exact, untouched. Audit archive (not a defect): WAL auto-checkpoint verified healthy under daemon-like load — 4 rounds × 400 mixed ops, WAL converged at ~4.1MB (default 1000-page threshold), TRUNCATE resets to zero. 5 new guards in `tests/test_v645_fixes.py`.
@@ -183,7 +188,7 @@ v6.1.3 – v6.4.5 shipped in ten days (2026-11-01 → 11-10; two same-day releas
183
188
 
184
189
  ```
185
190
  ┌──────────────────────────────────────────────────────┐
186
- │ MindForge v6.4.6 │
191
+ │ MindForge v6.4.9 │
187
192
  │ Cognitive Layer Personality · KnowledgeGraph │
188
193
  │ MemoryEvolution · FederatedMemory │
189
194
  ├──────────────────────────────────────────────────────┤
@@ -135,4 +135,6 @@ tests/test_v636_fixes.py
135
135
  tests/test_v643_fixes.py
136
136
  tests/test_v645_fixes.py
137
137
  tests/test_v646_fixes.py
138
+ tests/test_v648_fixes.py
139
+ tests/test_v649_fixes.py
138
140
  tests/test_v6_ports.py
@@ -39,7 +39,7 @@ import mindforge.core.version as core_version
39
39
 
40
40
 
41
41
  _REPO = Path(__file__).resolve().parent.parent
42
- _EXPECTED_VERSION = "6.4.6"
42
+ _EXPECTED_VERSION = "6.4.9"
43
43
 
44
44
  # 本轮修复涉及的、必须与 mindforge/core/version.py 保持一致的源码文件
45
45
  _VERSION_TRUTH_FILES = (
@@ -40,8 +40,8 @@ class TestVersionTruth(unittest.TestCase):
40
40
  def test_core_version_matches_release(self):
41
41
  # 版本推进时必须一并更新此处的钉值(与 test_v568_fixes 的
42
42
  # _EXPECTED_VERSION 同一约定:让「忘记 bump 版本」在测试里显形)。
43
- self.assertEqual(core_version.__version__, "6.4.6")
44
- self.assertEqual(core_version.VERSION_INFO, (6, 4, 6))
43
+ self.assertEqual(core_version.__version__, "6.4.9")
44
+ self.assertEqual(core_version.VERSION_INFO, (6, 4, 9))
45
45
 
46
46
  def test_pyproject_matches_core_version(self):
47
47
  # v5.8.13 起 pyproject 不再写死静态版本: