memfg 6.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. memfg-6.0.0/.gitignore +21 -0
  2. memfg-6.0.0/LICENSE +21 -0
  3. memfg-6.0.0/Makefile +47 -0
  4. memfg-6.0.0/PKG-INFO +322 -0
  5. memfg-6.0.0/README.md +281 -0
  6. memfg-6.0.0/benchmarks/locomo_benchmark.py +364 -0
  7. memfg-6.0.0/benchmarks/longmemeval_benchmark.py +1290 -0
  8. memfg-6.0.0/benchmarks/stress_sessions.py +207 -0
  9. memfg-6.0.0/benchmarks/synthetic_test.py +42 -0
  10. memfg-6.0.0/docs/API.md +23 -0
  11. memfg-6.0.0/docs/ARCHITECTURE.md +201 -0
  12. memfg-6.0.0/docs/BENCHMARKS.md +223 -0
  13. memfg-6.0.0/docs/RLM.md +45 -0
  14. memfg-6.0.0/docs/SECOND_BRAIN_ROADMAP.md +617 -0
  15. memfg-6.0.0/memoryforge/__init__.py +31 -0
  16. memfg-6.0.0/memoryforge/_core.py +391 -0
  17. memfg-6.0.0/memoryforge/agents/__init__.py +27 -0
  18. memfg-6.0.0/memoryforge/agents/codex_sync.py +132 -0
  19. memfg-6.0.0/memoryforge/agents/operators.py +285 -0
  20. memfg-6.0.0/memoryforge/agents/runners.py +513 -0
  21. memfg-6.0.0/memoryforge/agents/subagents.py +36 -0
  22. memfg-6.0.0/memoryforge/api/__init__.py +6 -0
  23. memfg-6.0.0/memoryforge/api/active_recall.py +214 -0
  24. memfg-6.0.0/memoryforge/api/app.py +110 -0
  25. memfg-6.0.0/memoryforge/api/context_assembly.py +144 -0
  26. memfg-6.0.0/memoryforge/api/context_builder.py +659 -0
  27. memfg-6.0.0/memoryforge/api/context_bundle.py +48 -0
  28. memfg-6.0.0/memoryforge/api/lcm_facade.py +208 -0
  29. memfg-6.0.0/memoryforge/api/lifecycle.py +433 -0
  30. memfg-6.0.0/memoryforge/api/memory_io.py +267 -0
  31. memfg-6.0.0/memoryforge/api/rlm_facade.py +378 -0
  32. memfg-6.0.0/memoryforge/benchmark/__init__.py +16 -0
  33. memfg-6.0.0/memoryforge/benchmark/adapter.py +102 -0
  34. memfg-6.0.0/memoryforge/benchmark/locomo.py +174 -0
  35. memfg-6.0.0/memoryforge/benchmark/longmemeval.py +97 -0
  36. memfg-6.0.0/memoryforge/benchmark/secondbrain.py +673 -0
  37. memfg-6.0.0/memoryforge/cli/__init__.py +5 -0
  38. memfg-6.0.0/memoryforge/cli/commands.py +366 -0
  39. memfg-6.0.0/memoryforge/cli/main.py +14 -0
  40. memfg-6.0.0/memoryforge/cli/parser.py +293 -0
  41. memfg-6.0.0/memoryforge/db/__init__.py +5 -0
  42. memfg-6.0.0/memoryforge/db/schema.py +284 -0
  43. memfg-6.0.0/memoryforge/init/__init__.py +6 -0
  44. memfg-6.0.0/memoryforge/init/bootstrap.py +141 -0
  45. memfg-6.0.0/memoryforge/init/codex.py +341 -0
  46. memfg-6.0.0/memoryforge/init/hooks.py +268 -0
  47. memfg-6.0.0/memoryforge/lcm/__init__.py +52 -0
  48. memfg-6.0.0/memoryforge/lcm/compaction/__init__.py +22 -0
  49. memfg-6.0.0/memoryforge/lcm/compaction/compactor.py +479 -0
  50. memfg-6.0.0/memoryforge/lcm/compaction/engine.py +595 -0
  51. memfg-6.0.0/memoryforge/lcm/compaction/file_ids.py +233 -0
  52. memfg-6.0.0/memoryforge/lcm/compaction/pruner.py +78 -0
  53. memfg-6.0.0/memoryforge/lcm/compaction/subagent_provider.py +60 -0
  54. memfg-6.0.0/memoryforge/lcm/context/__init__.py +5 -0
  55. memfg-6.0.0/memoryforge/lcm/context/builder.py +352 -0
  56. memfg-6.0.0/memoryforge/lcm/conversation/__init__.py +9 -0
  57. memfg-6.0.0/memoryforge/lcm/conversation/store.py +242 -0
  58. memfg-6.0.0/memoryforge/lcm/events/__init__.py +5 -0
  59. memfg-6.0.0/memoryforge/lcm/events/bus.py +101 -0
  60. memfg-6.0.0/memoryforge/lcm/refs/__init__.py +5 -0
  61. memfg-6.0.0/memoryforge/lcm/refs/hash_ref.py +29 -0
  62. memfg-6.0.0/memoryforge/lcm/store/__init__.py +5 -0
  63. memfg-6.0.0/memoryforge/lcm/store/immutable.py +454 -0
  64. memfg-6.0.0/memoryforge/lcm/summary/__init__.py +5 -0
  65. memfg-6.0.0/memoryforge/lcm/summary/dag.py +359 -0
  66. memfg-6.0.0/memoryforge/lcm/tokens/__init__.py +5 -0
  67. memfg-6.0.0/memoryforge/lcm/tokens/estimator.py +96 -0
  68. memfg-6.0.0/memoryforge/mcp/__init__.py +5 -0
  69. memfg-6.0.0/memoryforge/mcp/server.py +417 -0
  70. memfg-6.0.0/memoryforge/mcp/tools.py +349 -0
  71. memfg-6.0.0/memoryforge/memory/__init__.py +22 -0
  72. memfg-6.0.0/memoryforge/memory/longterm/__init__.py +26 -0
  73. memfg-6.0.0/memoryforge/memory/longterm/indexing.py +621 -0
  74. memfg-6.0.0/memoryforge/memory/longterm/models.py +99 -0
  75. memfg-6.0.0/memoryforge/memory/longterm/retrieval.py +483 -0
  76. memfg-6.0.0/memoryforge/memory/longterm/store.py +46 -0
  77. memfg-6.0.0/memoryforge/memory/longterm/utils.py +43 -0
  78. memfg-6.0.0/memoryforge/py.typed +1 -0
  79. memfg-6.0.0/memoryforge/rlm/__init__.py +11 -0
  80. memfg-6.0.0/memoryforge/rlm/chunking.py +581 -0
  81. memfg-6.0.0/memoryforge/rlm/common.py +61 -0
  82. memfg-6.0.0/memoryforge/rlm/engine.py +36 -0
  83. memfg-6.0.0/memoryforge/rlm/records.py +314 -0
  84. memfg-6.0.0/memoryforge/rlm/runner.py +766 -0
  85. memfg-6.0.0/memoryforge/rlm/schema.py +56 -0
  86. memfg-6.0.0/memoryforge/rlm/search.py +200 -0
  87. memfg-6.0.0/memoryforge/rlm/storage.py +99 -0
  88. memfg-6.0.0/memoryforge/rlm/types.py +21 -0
  89. memfg-6.0.0/memoryforge/runtime.py +213 -0
  90. memfg-6.0.0/memoryforge/search/__init__.py +18 -0
  91. memfg-6.0.0/memoryforge/search/fts.py +56 -0
  92. memfg-6.0.0/memoryforge/search/router.py +28 -0
  93. memfg-6.0.0/memoryforge/search/vector.py +551 -0
  94. memfg-6.0.0/pyproject.toml +96 -0
  95. memfg-6.0.0/tests/__init__.py +1 -0
  96. memfg-6.0.0/tests/conftest.py +54 -0
  97. memfg-6.0.0/tests/test_benchmark_metrics.py +524 -0
  98. memfg-6.0.0/tests/test_chunking.py +30 -0
  99. memfg-6.0.0/tests/test_content_store.py +15 -0
  100. memfg-6.0.0/tests/test_file_ids.py +212 -0
  101. memfg-6.0.0/tests/test_hindsight_benchmarks.py +155 -0
  102. memfg-6.0.0/tests/test_imports.py +8 -0
  103. memfg-6.0.0/tests/test_init.py +227 -0
  104. memfg-6.0.0/tests/test_integration.py +18 -0
  105. memfg-6.0.0/tests/test_lcm.py +153 -0
  106. memfg-6.0.0/tests/test_lcm_runtime.py +767 -0
  107. memfg-6.0.0/tests/test_long_term_memory.py +784 -0
  108. memfg-6.0.0/tests/test_mcp_tools.py +214 -0
  109. memfg-6.0.0/tests/test_memory.py +563 -0
  110. memfg-6.0.0/tests/test_package_layout.py +39 -0
  111. memfg-6.0.0/tests/test_prompt_topology.py +93 -0
  112. memfg-6.0.0/tests/test_real_subagents.py +129 -0
  113. memfg-6.0.0/tests/test_rlm_pipeline.py +510 -0
  114. memfg-6.0.0/tests/test_runtime_integration.py +113 -0
  115. memfg-6.0.0/tests/test_secondbrain_benchmark.py +93 -0
  116. memfg-6.0.0/tests/test_subagents.py +304 -0
  117. memfg-6.0.0/tests/test_unified_query.py +16 -0
  118. memfg-6.0.0/tests/test_vector.py +294 -0
memfg-6.0.0/.gitignore ADDED
@@ -0,0 +1,21 @@
1
+ *.db
2
+ *.db-shm
3
+ *.db-wal
4
+ *.sqlite
5
+ *.sqlite3
6
+ *.pyc
7
+ __pycache__/
8
+ .pytest_cache/
9
+ .mypy_cache/
10
+ .ruff_cache/
11
+ .tmp/
12
+ .venv/
13
+ dist/
14
+ build/
15
+ *.egg-info/
16
+ htmlcov/
17
+ .coverage
18
+ .codebase-memory/
19
+ .memoryforge/
20
+ benchmarks/results/
21
+ .pypirc
memfg-6.0.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 MemoryForge Contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
memfg-6.0.0/Makefile ADDED
@@ -0,0 +1,47 @@
1
+ .PHONY: help fmt check lint typecheck test test-python test-real-subagents build develop benchmark clean
2
+
3
+ PYTHON ?= python3
4
+ PYTHONPATH ?= .
5
+ COVERAGE_MIN ?= 77
6
+ REAL_SUBAGENT_MODEL ?= gpt-5.4
7
+
8
+ help:
9
+ @echo "MemoryForge development commands"
10
+ @echo " make fmt Format Python code"
11
+ @echo " make check Run lint + typecheck + tests + coverage gate"
12
+ @echo " make test Run all required tests"
13
+ @echo " make test-real-subagents Run real Codex CLI sub-agent tests"
14
+ @echo " make build Build pure-Python wheel"
15
+ @echo " make develop Install editable package"
16
+ @echo " make benchmark Run benchmark smoke suite"
17
+ @echo " make clean Remove generated artifacts"
18
+
19
+ fmt:
20
+ uv run --with ruff ruff format memoryforge tests
21
+
22
+ check:
23
+ $(MAKE) lint
24
+ $(MAKE) typecheck
25
+ $(MAKE) test-python
26
+
27
+ test:
28
+ $(MAKE) check
29
+
30
+ test-python:
31
+ env PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=$(PYTHONPATH) uv run --with pytest --with pytest-cov pytest \
32
+ --cov=memoryforge --cov-report=term-missing --cov-fail-under=$(COVERAGE_MIN)
33
+
34
+ test-real-subagents:
35
+ env PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=$(PYTHONPATH) MEMORYFORGE_REAL_SUBAGENT=1 MEMORYFORGE_REAL_PROJECT_ROOT=$$(pwd) MEMORYFORGE_SUBAGENT_RUNNER=codex MEMORYFORGE_MODEL=$(REAL_SUBAGENT_MODEL) uv run --with pytest pytest tests/test_real_subagents.py -vv
36
+
37
+ build:
38
+ uv build
39
+
40
+ develop:
41
+ uv pip install -e .
42
+
43
+ benchmark:
44
+ PYTHONPATH=$(PYTHONPATH) $(PYTHON) benchmarks/synthetic_test.py
45
+
46
+ clean:
47
+ rm -rf dist build .pytest_cache **/__pycache__ *.egg-info memoryforge.egg-info
memfg-6.0.0/PKG-INFO ADDED
@@ -0,0 +1,322 @@
1
+ Metadata-Version: 2.4
2
+ Name: memfg
3
+ Version: 6.0.0
4
+ Summary: Lossless Context Management for Recursive Language Model
5
+ Author: MemoryForge Contributors
6
+ License: MIT
7
+ License-File: LICENSE
8
+ Classifier: Programming Language :: Python :: 3.10
9
+ Classifier: Programming Language :: Python :: 3.11
10
+ Classifier: Programming Language :: Python :: 3.12
11
+ Classifier: Programming Language :: Python :: Implementation :: CPython
12
+ Requires-Python: >=3.10
13
+ Requires-Dist: anyio>=4.0
14
+ Requires-Dist: mcp>=1.0.0
15
+ Requires-Dist: tomli>=2.0; python_version < '3.11'
16
+ Provides-Extra: all
17
+ Requires-Dist: fastembed>=0.5.0; extra == 'all'
18
+ Requires-Dist: matplotlib>=3.7; extra == 'all'
19
+ Requires-Dist: mypy>=1.10; extra == 'all'
20
+ Requires-Dist: pandas>=2.0; extra == 'all'
21
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'all'
22
+ Requires-Dist: pytest-cov>=5.0; extra == 'all'
23
+ Requires-Dist: pytest>=8.0; extra == 'all'
24
+ Requires-Dist: ruff>=0.9; extra == 'all'
25
+ Requires-Dist: tqdm>=4.0; extra == 'all'
26
+ Requires-Dist: twine>=5.0; extra == 'all'
27
+ Provides-Extra: benchmark
28
+ Requires-Dist: matplotlib>=3.7; extra == 'benchmark'
29
+ Requires-Dist: pandas>=2.0; extra == 'benchmark'
30
+ Requires-Dist: tqdm>=4.0; extra == 'benchmark'
31
+ Provides-Extra: dev
32
+ Requires-Dist: mypy>=1.10; extra == 'dev'
33
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
34
+ Requires-Dist: pytest-cov>=5.0; extra == 'dev'
35
+ Requires-Dist: pytest>=8.0; extra == 'dev'
36
+ Requires-Dist: ruff>=0.9; extra == 'dev'
37
+ Requires-Dist: twine>=5.0; extra == 'dev'
38
+ Provides-Extra: embeddings
39
+ Requires-Dist: fastembed>=0.5.0; extra == 'embeddings'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # MemoryForge
43
+
44
+ MemoryForge is a local-first memory layer for Codex CLI workflows.
45
+
46
+ It stores durable project evidence in SQLite, keeps live context bounded, and
47
+ returns source-backed context bundles to the model the user is already running.
48
+ MemoryForge is not a separate answering agent and it is not a codebase AST graph
49
+ engine. The host model answers; MemoryForge supplies memory, references, and
50
+ provenance.
51
+
52
+ ```text
53
+ User -> Codex CLI -> MemoryForge MCP -> SQLite memory.db
54
+ |
55
+ +-> bounded CoreContextBundle
56
+ +-> optional RLM/LCM worker runs
57
+ ```
58
+
59
+ ## What It Is For
60
+
61
+ MemoryForge focuses on long-form project evidence that is common in real coding
62
+ workflows but too large or noisy to paste into every prompt:
63
+
64
+ - design notes and architecture documents
65
+ - requirements and implementation plans
66
+ - setup guides and decision records
67
+ - benchmark descriptions and experiment logs
68
+ - large Markdown files used while "vibe coding" or building a project over many sessions
69
+
70
+ These sources are ingested through RLM, stored as durable LTM evidence, and
71
+ recalled as bounded context when Codex needs them.
72
+
73
+ MemoryForge intentionally avoids a large AST/code graph schema in the core
74
+ release. Source files can still be stored as RLM/LTM evidence, but dedicated
75
+ code indexing is future work rather than part of the current SQLite schema.
76
+
77
+ ## Memory Layers
78
+
79
+ | Layer | Role | Model worker use |
80
+ | --- | --- | --- |
81
+ | RLM | Raw Large Memory. Chunks large files/prompts and indexes them into durable memory. | Optional, for batch analysis. |
82
+ | LTM | Long-Term Memory. Recalls durable evidence across sessions and sources. | No model call. |
83
+ | LCM | Lossless Context Management Keeps the active session view bounded with summaries and refs. | Optional, for compaction. |
84
+
85
+ Important boundary: LCM compacts MemoryForge's SQLite-backed active context. It
86
+ does not directly erase Codex's own context window. Codex manages its live
87
+ thread and can compact it with `/compact`; MemoryForge hooks and MCP tools then
88
+ help preserve and rehydrate the evidence needed after compaction.
89
+
90
+ ## Install
91
+
92
+ From PyPI in a project that uses `uv`:
93
+
94
+ ```bash
95
+ uv add memfg
96
+ ```
97
+
98
+ With optional local embeddings:
99
+
100
+ ```bash
101
+ uv add "memfg[embeddings]"
102
+ ```
103
+
104
+ For local development from this repository:
105
+
106
+ ```bash
107
+ uv sync --extra dev --extra benchmark
108
+ ```
109
+
110
+ ## Initialize A Codex Project
111
+
112
+ Run this at the project root:
113
+
114
+ ```bash
115
+ uv run memoryforge init . --agent-id default --force
116
+ ```
117
+
118
+ This creates:
119
+
120
+ ```text
121
+ .memoryforge/memory.db
122
+ .memoryforge/config.json
123
+ .memoryforge/hooks/memoryforge-hook.sh
124
+ .codex/config.toml
125
+ .codex/hooks.json
126
+ ```
127
+
128
+ The Codex config registers the MemoryForge MCP server:
129
+
130
+ ```toml
131
+ [mcp_servers.memoryforge]
132
+ command = "uv"
133
+ args = ["run", "memoryforge-mcp"]
134
+
135
+ [mcp_servers.memoryforge.env]
136
+ MEMORYFORGE_DB = "/absolute/path/.memoryforge/memory.db"
137
+ ```
138
+
139
+ The hook file records prompt and compaction lifecycle events. Project-local
140
+ Codex hooks only run after the project `.codex/` layer is trusted by Codex.
141
+ Inspect and trust them in the CLI with `/hooks` if Codex asks.
142
+
143
+ ## Prompt Submit, Cancel, And Retract
144
+
145
+ MemoryForge uses a two-phase hook flow for user prompts:
146
+
147
+ - `UserPromptSubmit` writes the latest prompt for the session to
148
+ `.memoryforge/pending/`.
149
+ - `Stop` commits that pending prompt into SQLite, then RLM/LTM/LCM processing
150
+ can run.
151
+ - `SessionStart` removes stale pending prompts after the local TTL.
152
+
153
+ This keeps interrupted turns out of durable memory when the Codex turn is
154
+ cancelled before `Stop` runs. MemoryForge does not infer cancellation from
155
+ free-form hook payload fields such as status or reason strings; that is too
156
+ fragile across Codex versions and locales. Explicit retract integrations should
157
+ call the internal `discard-pending` hook event for the matching session before a
158
+ later `Stop` can commit it.
159
+
160
+ ## Basic Usage
161
+
162
+ Ingest a long Markdown file or project document:
163
+
164
+ ```bash
165
+ uv run memoryforge --db .memoryforge/memory.db ingest-file docs/notes.md \
166
+ --agent-id default
167
+ ```
168
+
169
+ Load a large source through RLM:
170
+
171
+ ```bash
172
+ uv run memoryforge --db .memoryforge/memory.db rlm-load docs/design.md \
173
+ --agent-id default \
174
+ --name design-notes
175
+ ```
176
+
177
+ Recall durable evidence:
178
+
179
+ ```bash
180
+ uv run memoryforge --db .memoryforge/memory.db recall-memory \
181
+ --agent-id default \
182
+ --query "why did we choose sqlite"
183
+ ```
184
+
185
+ Build a runtime context bundle for the active Codex project:
186
+
187
+ ```bash
188
+ uv run memoryforge --db .memoryforge/memory.db runtime-context \
189
+ --agent-id default \
190
+ --session-id session-1 \
191
+ --query "what context should Codex use now" \
192
+ --project-root .
193
+ ```
194
+
195
+ Run LCM compaction over MemoryForge's stored active context:
196
+
197
+ ```bash
198
+ uv run memoryforge --db .memoryforge/memory.db lcm-compact \
199
+ --agent-id default \
200
+ --session-id session-1 \
201
+ --project-root . \
202
+ --force
203
+ ```
204
+
205
+ Run the MCP server directly:
206
+
207
+ ```bash
208
+ uv run memoryforge-mcp
209
+ ```
210
+
211
+ ## Optional Vector Recall
212
+
213
+ MemoryForge works without embeddings by using lexical/FTS retrieval. To enable
214
+ semantic recall, install the embeddings extra and choose FastEmbed:
215
+
216
+ ```bash
217
+ uv add "memfg[embeddings]"
218
+
219
+ export MEMORYFORGE_VECTOR_BACKEND=fastembed
220
+ export MEMORYFORGE_VECTOR_MODEL=BAAI/bge-small-en-v1.5
221
+ export MEMORYFORGE_REQUIRE_VECTOR_MODEL=1
222
+ ```
223
+
224
+ The project intentionally keeps one vector cache table, `vec_index`, and avoids
225
+ SQLite extension backends such as `sqlite-vec` in the core release. This keeps
226
+ the package easier to install, test, and publish.
227
+
228
+ ## CLI Surface
229
+
230
+ Public commands:
231
+
232
+ - Project/runtime: `init`, `mcp-server`, `runtime-context`
233
+ - Conversation memory: `store-session`, `search`, `recall-memory`, `active-recall`, `long-term-source`
234
+ - Contradictions: `record-contradiction`, `find-contradictions`
235
+ - LCM: `lcm-context`, `lcm-compact`, `lcm-maintain`
236
+ - RLM/source loading: `ingest-file`, `rlm-load`, `rlm-search`, `rlm-chunk-get`, `dispatch`, `context-get`, `rlm-record`, `aggregate`, `rlm-run`
237
+ - Diagnostics: `chunk`, `benchmark`
238
+
239
+ `memoryforge hook` is an internal Codex hook endpoint created by `memoryforge init`.
240
+ RLM/LCM sub-agents are internal MemoryForge workers. For real worker runs,
241
+ MemoryForge uses Codex CLI through `codex exec` when configured. Development-time
242
+ Codex host subagents are separate review/triage helpers and are not the
243
+ MemoryForge runtime worker API.
244
+
245
+ ## Benchmarks
246
+
247
+ The current benchmark focus is long-memory behavior, not static code indexing:
248
+
249
+ - LoCoMo
250
+ - LongMemEval
251
+ - deterministic multi-session stress benchmark
252
+ - synthetic smoke benchmark
253
+
254
+ Example smoke check:
255
+
256
+ ```bash
257
+ uv run python benchmarks/synthetic_test.py
258
+ ```
259
+
260
+ Stress check for many real SQLite sessions:
261
+
262
+ ```bash
263
+ uv run python benchmarks/stress_sessions.py \
264
+ --sessions 100 \
265
+ --turns-per-session 12 \
266
+ --output benchmarks/results/stress_sessions_100x12.json
267
+ ```
268
+
269
+ Real LoCoMo and LongMemEval runs require their datasets and model credentials.
270
+ See [docs/BENCHMARKS.md](docs/BENCHMARKS.md) for run modes and result fields.
271
+
272
+ ## Development
273
+
274
+ Run the normal quality gate on Unix-like shells:
275
+
276
+ ```bash
277
+ make check
278
+ ```
279
+
280
+ Equivalent commands:
281
+
282
+ ```bash
283
+ uv run ruff check memoryforge tests benchmarks
284
+ PYTHONDONTWRITEBYTECODE=1 uv run mypy memoryforge
285
+ PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. uv run pytest --ignore=tests/test_real_subagents.py --cov=memoryforge --cov-report=term-missing --cov-fail-under=77
286
+ MEMORYFORGE_REAL_SUBAGENT=1 MEMORYFORGE_REAL_PROJECT_ROOT="$PWD" MEMORYFORGE_SUBAGENT_RUNNER=codex MEMORYFORGE_MODEL=gpt-5.4 uv run pytest tests/test_real_subagents.py -vv
287
+ uv build
288
+ uv run twine check dist/*
289
+ ```
290
+
291
+ On Windows PowerShell, keep pytest temp/cache paths in writable directories:
292
+
293
+ ```powershell
294
+ $env:TMP='C:\tmp'; $env:TEMP='C:\tmp'
295
+ Remove-Item Env:\MEMORYFORGE_SUBAGENT_RUNNER -ErrorAction SilentlyContinue
296
+ Remove-Item Env:\MEMORYFORGE_MODEL -ErrorAction SilentlyContinue
297
+ uv run pytest --ignore=tests/test_real_subagents.py --basetemp=C:\tmp\memoryforge-pytest-basetemp -o cache_dir=.tmp\pytest-cache
298
+
299
+ $env:MEMORYFORGE_REAL_SUBAGENT='1'; $env:MEMORYFORGE_REAL_PROJECT_ROOT=(Get-Location).Path
300
+ $env:MEMORYFORGE_SUBAGENT_RUNNER='codex'; $env:MEMORYFORGE_MODEL='gpt-5.4'
301
+ uv run pytest tests/test_real_subagents.py -vv --basetemp=C:\tmp\memoryforge-pytest-basetemp-real -o cache_dir=.tmp\pytest-cache-real
302
+ ```
303
+
304
+ The real Codex sub-agent smoke tests are local-only. Run them on a machine with the Codex CLI installed and authenticated if you want to verify `runner="codex"`; they are not part of CI/CD. Mock runners are only for targeted unit tests that verify MemoryForge's own control flow.
305
+
306
+ ## Release Notes For Maintainers
307
+
308
+ Before pushing or publishing:
309
+
310
+ 1. Keep generated data out of the release: `.venv/`, caches, `.coverage`, `.memoryforge/`, `.codebase-memory/`, `dist/`, and `benchmarks/results/`.
311
+ 2. Run the full gate on Python 3.10, 3.11, and 3.12 through CI.
312
+ 3. Build the wheel and sdist with `uv build`.
313
+ 4. Check distributions with `twine check`.
314
+ 5. Prefer the `Publish` GitHub workflow with PyPI trusted publishing. Direct maintainer uploads may use `twine upload` with `TWINE_USERNAME=__token__` and `TWINE_PASSWORD` supplied from the shell environment, never from a committed config file.
315
+
316
+ ## Documentation
317
+
318
+ - [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)
319
+ - [docs/BENCHMARKS.md](docs/BENCHMARKS.md)
320
+ - [docs/API.md](docs/API.md)
321
+ - [docs/RLM.md](docs/RLM.md)
322
+ - [docs/SECOND_BRAIN_ROADMAP.md](docs/SECOND_BRAIN_ROADMAP.md)
memfg-6.0.0/README.md ADDED
@@ -0,0 +1,281 @@
1
+ # MemoryForge
2
+
3
+ MemoryForge is a local-first memory layer for Codex CLI workflows.
4
+
5
+ It stores durable project evidence in SQLite, keeps live context bounded, and
6
+ returns source-backed context bundles to the model the user is already running.
7
+ MemoryForge is not a separate answering agent and it is not a codebase AST graph
8
+ engine. The host model answers; MemoryForge supplies memory, references, and
9
+ provenance.
10
+
11
+ ```text
12
+ User -> Codex CLI -> MemoryForge MCP -> SQLite memory.db
13
+ |
14
+ +-> bounded CoreContextBundle
15
+ +-> optional RLM/LCM worker runs
16
+ ```
17
+
18
+ ## What It Is For
19
+
20
+ MemoryForge focuses on long-form project evidence that is common in real coding
21
+ workflows but too large or noisy to paste into every prompt:
22
+
23
+ - design notes and architecture documents
24
+ - requirements and implementation plans
25
+ - setup guides and decision records
26
+ - benchmark descriptions and experiment logs
27
+ - large Markdown files used while "vibe coding" or building a project over many sessions
28
+
29
+ These sources are ingested through RLM, stored as durable LTM evidence, and
30
+ recalled as bounded context when Codex needs them.
31
+
32
+ MemoryForge intentionally avoids a large AST/code graph schema in the core
33
+ release. Source files can still be stored as RLM/LTM evidence, but dedicated
34
+ code indexing is future work rather than part of the current SQLite schema.
35
+
36
+ ## Memory Layers
37
+
38
+ | Layer | Role | Model worker use |
39
+ | --- | --- | --- |
40
+ | RLM | Raw Large Memory. Chunks large files/prompts and indexes them into durable memory. | Optional, for batch analysis. |
41
+ | LTM | Long-Term Memory. Recalls durable evidence across sessions and sources. | No model call. |
42
+ | LCM | Lossless Context Management Keeps the active session view bounded with summaries and refs. | Optional, for compaction. |
43
+
44
+ Important boundary: LCM compacts MemoryForge's SQLite-backed active context. It
45
+ does not directly erase Codex's own context window. Codex manages its live
46
+ thread and can compact it with `/compact`; MemoryForge hooks and MCP tools then
47
+ help preserve and rehydrate the evidence needed after compaction.
48
+
49
+ ## Install
50
+
51
+ From PyPI in a project that uses `uv`:
52
+
53
+ ```bash
54
+ uv add memfg
55
+ ```
56
+
57
+ With optional local embeddings:
58
+
59
+ ```bash
60
+ uv add "memfg[embeddings]"
61
+ ```
62
+
63
+ For local development from this repository:
64
+
65
+ ```bash
66
+ uv sync --extra dev --extra benchmark
67
+ ```
68
+
69
+ ## Initialize A Codex Project
70
+
71
+ Run this at the project root:
72
+
73
+ ```bash
74
+ uv run memoryforge init . --agent-id default --force
75
+ ```
76
+
77
+ This creates:
78
+
79
+ ```text
80
+ .memoryforge/memory.db
81
+ .memoryforge/config.json
82
+ .memoryforge/hooks/memoryforge-hook.sh
83
+ .codex/config.toml
84
+ .codex/hooks.json
85
+ ```
86
+
87
+ The Codex config registers the MemoryForge MCP server:
88
+
89
+ ```toml
90
+ [mcp_servers.memoryforge]
91
+ command = "uv"
92
+ args = ["run", "memoryforge-mcp"]
93
+
94
+ [mcp_servers.memoryforge.env]
95
+ MEMORYFORGE_DB = "/absolute/path/.memoryforge/memory.db"
96
+ ```
97
+
98
+ The hook file records prompt and compaction lifecycle events. Project-local
99
+ Codex hooks only run after the project `.codex/` layer is trusted by Codex.
100
+ Inspect and trust them in the CLI with `/hooks` if Codex asks.
101
+
102
+ ## Prompt Submit, Cancel, And Retract
103
+
104
+ MemoryForge uses a two-phase hook flow for user prompts:
105
+
106
+ - `UserPromptSubmit` writes the latest prompt for the session to
107
+ `.memoryforge/pending/`.
108
+ - `Stop` commits that pending prompt into SQLite, then RLM/LTM/LCM processing
109
+ can run.
110
+ - `SessionStart` removes stale pending prompts after the local TTL.
111
+
112
+ This keeps interrupted turns out of durable memory when the Codex turn is
113
+ cancelled before `Stop` runs. MemoryForge does not infer cancellation from
114
+ free-form hook payload fields such as status or reason strings; that is too
115
+ fragile across Codex versions and locales. Explicit retract integrations should
116
+ call the internal `discard-pending` hook event for the matching session before a
117
+ later `Stop` can commit it.
118
+
119
+ ## Basic Usage
120
+
121
+ Ingest a long Markdown file or project document:
122
+
123
+ ```bash
124
+ uv run memoryforge --db .memoryforge/memory.db ingest-file docs/notes.md \
125
+ --agent-id default
126
+ ```
127
+
128
+ Load a large source through RLM:
129
+
130
+ ```bash
131
+ uv run memoryforge --db .memoryforge/memory.db rlm-load docs/design.md \
132
+ --agent-id default \
133
+ --name design-notes
134
+ ```
135
+
136
+ Recall durable evidence:
137
+
138
+ ```bash
139
+ uv run memoryforge --db .memoryforge/memory.db recall-memory \
140
+ --agent-id default \
141
+ --query "why did we choose sqlite"
142
+ ```
143
+
144
+ Build a runtime context bundle for the active Codex project:
145
+
146
+ ```bash
147
+ uv run memoryforge --db .memoryforge/memory.db runtime-context \
148
+ --agent-id default \
149
+ --session-id session-1 \
150
+ --query "what context should Codex use now" \
151
+ --project-root .
152
+ ```
153
+
154
+ Run LCM compaction over MemoryForge's stored active context:
155
+
156
+ ```bash
157
+ uv run memoryforge --db .memoryforge/memory.db lcm-compact \
158
+ --agent-id default \
159
+ --session-id session-1 \
160
+ --project-root . \
161
+ --force
162
+ ```
163
+
164
+ Run the MCP server directly:
165
+
166
+ ```bash
167
+ uv run memoryforge-mcp
168
+ ```
169
+
170
+ ## Optional Vector Recall
171
+
172
+ MemoryForge works without embeddings by using lexical/FTS retrieval. To enable
173
+ semantic recall, install the embeddings extra and choose FastEmbed:
174
+
175
+ ```bash
176
+ uv add "memfg[embeddings]"
177
+
178
+ export MEMORYFORGE_VECTOR_BACKEND=fastembed
179
+ export MEMORYFORGE_VECTOR_MODEL=BAAI/bge-small-en-v1.5
180
+ export MEMORYFORGE_REQUIRE_VECTOR_MODEL=1
181
+ ```
182
+
183
+ The project intentionally keeps one vector cache table, `vec_index`, and avoids
184
+ SQLite extension backends such as `sqlite-vec` in the core release. This keeps
185
+ the package easier to install, test, and publish.
186
+
187
+ ## CLI Surface
188
+
189
+ Public commands:
190
+
191
+ - Project/runtime: `init`, `mcp-server`, `runtime-context`
192
+ - Conversation memory: `store-session`, `search`, `recall-memory`, `active-recall`, `long-term-source`
193
+ - Contradictions: `record-contradiction`, `find-contradictions`
194
+ - LCM: `lcm-context`, `lcm-compact`, `lcm-maintain`
195
+ - RLM/source loading: `ingest-file`, `rlm-load`, `rlm-search`, `rlm-chunk-get`, `dispatch`, `context-get`, `rlm-record`, `aggregate`, `rlm-run`
196
+ - Diagnostics: `chunk`, `benchmark`
197
+
198
+ `memoryforge hook` is an internal Codex hook endpoint created by `memoryforge init`.
199
+ RLM/LCM sub-agents are internal MemoryForge workers. For real worker runs,
200
+ MemoryForge uses Codex CLI through `codex exec` when configured. Development-time
201
+ Codex host subagents are separate review/triage helpers and are not the
202
+ MemoryForge runtime worker API.
203
+
204
+ ## Benchmarks
205
+
206
+ The current benchmark focus is long-memory behavior, not static code indexing:
207
+
208
+ - LoCoMo
209
+ - LongMemEval
210
+ - deterministic multi-session stress benchmark
211
+ - synthetic smoke benchmark
212
+
213
+ Example smoke check:
214
+
215
+ ```bash
216
+ uv run python benchmarks/synthetic_test.py
217
+ ```
218
+
219
+ Stress check for many real SQLite sessions:
220
+
221
+ ```bash
222
+ uv run python benchmarks/stress_sessions.py \
223
+ --sessions 100 \
224
+ --turns-per-session 12 \
225
+ --output benchmarks/results/stress_sessions_100x12.json
226
+ ```
227
+
228
+ Real LoCoMo and LongMemEval runs require their datasets and model credentials.
229
+ See [docs/BENCHMARKS.md](docs/BENCHMARKS.md) for run modes and result fields.
230
+
231
+ ## Development
232
+
233
+ Run the normal quality gate on Unix-like shells:
234
+
235
+ ```bash
236
+ make check
237
+ ```
238
+
239
+ Equivalent commands:
240
+
241
+ ```bash
242
+ uv run ruff check memoryforge tests benchmarks
243
+ PYTHONDONTWRITEBYTECODE=1 uv run mypy memoryforge
244
+ PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. uv run pytest --ignore=tests/test_real_subagents.py --cov=memoryforge --cov-report=term-missing --cov-fail-under=77
245
+ MEMORYFORGE_REAL_SUBAGENT=1 MEMORYFORGE_REAL_PROJECT_ROOT="$PWD" MEMORYFORGE_SUBAGENT_RUNNER=codex MEMORYFORGE_MODEL=gpt-5.4 uv run pytest tests/test_real_subagents.py -vv
246
+ uv build
247
+ uv run twine check dist/*
248
+ ```
249
+
250
+ On Windows PowerShell, keep pytest temp/cache paths in writable directories:
251
+
252
+ ```powershell
253
+ $env:TMP='C:\tmp'; $env:TEMP='C:\tmp'
254
+ Remove-Item Env:\MEMORYFORGE_SUBAGENT_RUNNER -ErrorAction SilentlyContinue
255
+ Remove-Item Env:\MEMORYFORGE_MODEL -ErrorAction SilentlyContinue
256
+ uv run pytest --ignore=tests/test_real_subagents.py --basetemp=C:\tmp\memoryforge-pytest-basetemp -o cache_dir=.tmp\pytest-cache
257
+
258
+ $env:MEMORYFORGE_REAL_SUBAGENT='1'; $env:MEMORYFORGE_REAL_PROJECT_ROOT=(Get-Location).Path
259
+ $env:MEMORYFORGE_SUBAGENT_RUNNER='codex'; $env:MEMORYFORGE_MODEL='gpt-5.4'
260
+ uv run pytest tests/test_real_subagents.py -vv --basetemp=C:\tmp\memoryforge-pytest-basetemp-real -o cache_dir=.tmp\pytest-cache-real
261
+ ```
262
+
263
+ The real Codex sub-agent smoke tests are local-only. Run them on a machine with the Codex CLI installed and authenticated if you want to verify `runner="codex"`; they are not part of CI/CD. Mock runners are only for targeted unit tests that verify MemoryForge's own control flow.
264
+
265
+ ## Release Notes For Maintainers
266
+
267
+ Before pushing or publishing:
268
+
269
+ 1. Keep generated data out of the release: `.venv/`, caches, `.coverage`, `.memoryforge/`, `.codebase-memory/`, `dist/`, and `benchmarks/results/`.
270
+ 2. Run the full gate on Python 3.10, 3.11, and 3.12 through CI.
271
+ 3. Build the wheel and sdist with `uv build`.
272
+ 4. Check distributions with `twine check`.
273
+ 5. Prefer the `Publish` GitHub workflow with PyPI trusted publishing. Direct maintainer uploads may use `twine upload` with `TWINE_USERNAME=__token__` and `TWINE_PASSWORD` supplied from the shell environment, never from a committed config file.
274
+
275
+ ## Documentation
276
+
277
+ - [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)
278
+ - [docs/BENCHMARKS.md](docs/BENCHMARKS.md)
279
+ - [docs/API.md](docs/API.md)
280
+ - [docs/RLM.md](docs/RLM.md)
281
+ - [docs/SECOND_BRAIN_ROADMAP.md](docs/SECOND_BRAIN_ROADMAP.md)