memstack-skill-loader 4.13.0__tar.gz → 4.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {memstack_skill_loader-4.13.0/src/memstack_skill_loader.egg-info → memstack_skill_loader-4.14.0}/PKG-INFO +1 -1
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/README.md +13 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/pyproject.toml +1 -1
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/__init__.py +1 -1
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/dashboard.py +8 -2
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/diary_ingest.py +22 -2
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/memory_db.py +97 -15
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/memory_engine.py +179 -15
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/pro_compressor.py +23 -2
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/session_context.py +34 -10
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0/src/memstack_skill_loader.egg-info}/PKG-INFO +1 -1
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader.egg-info/SOURCES.txt +1 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_diary_ingest.py +69 -1
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_memory_scoring.py +127 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_pro_compressor.py +158 -0
- memstack_skill_loader-4.14.0/tests/test_project_dir_canonical.py +229 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_search_memory.py +6 -2
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_session_context.py +31 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/MANIFEST.in +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/setup.cfg +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/__main__.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/agent_runner.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/categories.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/compression.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/config.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/dashboard.html +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/goal.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/goal_engine.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/hooks/__init__.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/hooks/deny_git.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/hooks/deny_manager_write.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/indexer.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/license.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/__init__.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/_diag.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/body_parser.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/compressor.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/forwarder.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/server.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/proxy/stats_tracker.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/search.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/server.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/skill_config.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/skill_descriptions.json +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/stats.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/tfidf_search.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/version_check.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader.egg-info/dependency_links.txt +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader.egg-info/entry_points.txt +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader.egg-info/requires.txt +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader.egg-info/top_level.txt +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_advisor_strategy.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_body_parser_metric.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_builder_scope.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_code_read_savings.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_failed_approaches.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_license_environment.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_license_grace.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_memory_engine.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_memory_inspector.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_memory_tools.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_pro_skills_update.py +0 -0
- {memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/tests/test_skill_drift.py +0 -0
|
@@ -91,6 +91,9 @@ MCP server that vector-indexes all 130 MemStack™ skills so Claude Code can cal
|
|
|
91
91
|
- **No Anthropic API key required:** skill search and indexing run locally — the only outbound network call is license validation, which contacts the MemStack license server on session start ([Data and privacy](#data-and-privacy))
|
|
92
92
|
- **Pro skills auto-detected:** set your license key and they appear automatically
|
|
93
93
|
- **Auto-reindex on start:** skills stay current without manual rebuilds
|
|
94
|
+
- **Memory Engine:** an append-only, versioned fact store with query-time recall scoring — facts decay with age and decay more slowly the more they are recalled. Facts flow in from the diary's `## FACTS` block and are exposed to Claude through five `mem_*` tools (see [Available Tools](#available-tools))
|
|
95
|
+
- **Living memory at session start:** the SessionStart hook injects a compact, token-budgeted digest of the project's top recall-scored facts into context. It fails open — a missing, empty, or unreachable store never blocks startup
|
|
96
|
+
- **8-page localhost dashboard:** Overview, Skills Manager, Burn Report, Memory Browser (project-aware diary reader), Memory Inspector (read-only Memory Engine fact views), Agent Monitor, Referrals, and Settings
|
|
94
97
|
|
|
95
98
|
### Data and privacy
|
|
96
99
|
|
|
@@ -148,6 +151,16 @@ Rebuild the vector index after adding or modifying skills.
|
|
|
148
151
|
reindex_skills()
|
|
149
152
|
```
|
|
150
153
|
|
|
154
|
+
### Memory Engine
|
|
155
|
+
|
|
156
|
+
The `mem_*` tools back a persistent, append-only fact store shared across sessions (see [How It Works](#how-it-works)).
|
|
157
|
+
|
|
158
|
+
- **`mem_store`** — Store a new immutable fact (subject, claim, provenance). Returns the fact id and its provenance tag.
|
|
159
|
+
- **`mem_correct`** — Correct a fact by superseding it with a new one. Corrections extend only from the live tip of a chain; correcting a superseded fact is refused and names the current tip.
|
|
160
|
+
- **`mem_recall`** — Recall live facts for a project, each rendered with an inline provenance tag `[method · source · recalled Nx]`. Each returned fact is logged as a recall. Optional full-text query or exact-subject filter.
|
|
161
|
+
- **`mem_inspect`** — Inspect a single fact and its full supersession chain (root → tip) with statuses, provenance, entities, and recall count.
|
|
162
|
+
- **`mem_stats`** — Fact-store statistics: totals by status (live/superseded/retracted), distinct subjects, and recall count. Optionally scoped to a project.
|
|
163
|
+
|
|
151
164
|
## Configuration
|
|
152
165
|
|
|
153
166
|
The `config.json` file controls where skills are loaded from:
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "memstack-skill-loader"
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.14.0"
|
|
8
8
|
description = "MCP server that vector-indexes MemStack Pro skills for on-demand loading"
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
dependencies = [
|
{memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/dashboard.py
RENAMED
|
@@ -2127,13 +2127,19 @@ class _Handler(BaseHTTPRequestHandler):
|
|
|
2127
2127
|
if route == "/api/memory-inspector/facts":
|
|
2128
2128
|
project = qs.get("project", ["__all__"])[0]
|
|
2129
2129
|
projects = store.list_projects()
|
|
2130
|
+
# Raised from a hard-coded 500, which truncated by insertion order
|
|
2131
|
+
# (list_facts is id DESC) *before* the score sort below ever ran, so
|
|
2132
|
+
# the highest-scoring facts could be missing from the response
|
|
2133
|
+
# entirely. _MAX_ROWS is the store's runaway guard. Pagination is
|
|
2134
|
+
# deliberately still absent — tracked as separate work.
|
|
2135
|
+
cap = memory_engine._MAX_ROWS
|
|
2130
2136
|
if project and project != "__all__":
|
|
2131
|
-
rows = store.list_facts(project=project, status="live", limit=
|
|
2137
|
+
rows = store.list_facts(project=project, status="live", limit=cap)
|
|
2132
2138
|
else:
|
|
2133
2139
|
rows = []
|
|
2134
2140
|
for p in projects:
|
|
2135
2141
|
rows.extend(
|
|
2136
|
-
store.list_facts(project=p["project"], status="live", limit=
|
|
2142
|
+
store.list_facts(project=p["project"], status="live", limit=cap)
|
|
2137
2143
|
)
|
|
2138
2144
|
facts = [self._inspector_fact(f) for f in rows]
|
|
2139
2145
|
facts.sort(key=lambda x: x["score"], reverse=True)
|
|
@@ -7,7 +7,8 @@ module by path, exactly like the devlog-webhook:
|
|
|
7
7
|
python -m memstack_skill_loader.diary_ingest "<markdown-backup-path>"
|
|
8
8
|
|
|
9
9
|
Each fact line is stored through :class:`MemoryStore` with ``source_type='diary'``
|
|
10
|
-
and ``source_ref`` set to the diary file path
|
|
10
|
+
and ``source_ref`` set to the diary file path, canonicalized to an absolute
|
|
11
|
+
forward-slash path so the same file re-ingested from any cwd/arg form dedupes.
|
|
11
12
|
|
|
12
13
|
FAIL-OPEN CONTRACT (do not "improve" away)
|
|
13
14
|
------------------------------------------
|
|
@@ -141,6 +142,25 @@ def _project_from_path(path: str) -> str:
|
|
|
141
142
|
return slug or "default"
|
|
142
143
|
|
|
143
144
|
|
|
145
|
+
def _canonical_source_ref(path: str) -> str:
|
|
146
|
+
"""Canonicalize a diary path into a stable dedupe key.
|
|
147
|
+
|
|
148
|
+
The Memory Engine dedupe key is ``(source_ref, subject, claim)`` compared as
|
|
149
|
+
raw strings. A diary saved once via a relative path and once via an absolute
|
|
150
|
+
path therefore produced two different ``source_ref`` strings and re-stored
|
|
151
|
+
every fact as a duplicate live row. Resolving to a real absolute path
|
|
152
|
+
(collapsing ``..`` segments and following symlinks) and emitting it with
|
|
153
|
+
forward slashes makes every path form of the same file collapse to one key.
|
|
154
|
+
|
|
155
|
+
Fail-open: this runs in the never-break-the-diary-save path, so if resolution
|
|
156
|
+
fails for any reason, fall back to the raw string rather than raise.
|
|
157
|
+
"""
|
|
158
|
+
try:
|
|
159
|
+
return Path(path).resolve().as_posix()
|
|
160
|
+
except OSError:
|
|
161
|
+
return str(path)
|
|
162
|
+
|
|
163
|
+
|
|
144
164
|
def _warn(message: str) -> None:
|
|
145
165
|
print(f"{_STDERR_PREFIX} {message}", file=sys.stderr)
|
|
146
166
|
|
|
@@ -169,7 +189,7 @@ def ingest_diary_file(path: str, store: MemoryStore | None = None) -> dict:
|
|
|
169
189
|
if store is None:
|
|
170
190
|
store = MemoryStore()
|
|
171
191
|
project = _project_from_path(path)
|
|
172
|
-
source_ref =
|
|
192
|
+
source_ref = _canonical_source_ref(path)
|
|
173
193
|
|
|
174
194
|
ingested = 0
|
|
175
195
|
duplicates = 0
|
{memstack_skill_loader-4.13.0 → memstack_skill_loader-4.14.0}/src/memstack_skill_loader/memory_db.py
RENAMED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
|
+
import re
|
|
6
7
|
import sqlite3
|
|
7
8
|
import sys
|
|
8
9
|
from pathlib import Path, PureWindowsPath
|
|
@@ -10,6 +11,44 @@ from pathlib import Path, PureWindowsPath
|
|
|
10
11
|
DB_PATH = Path.home() / ".memstack" / "memory.db"
|
|
11
12
|
SCHEMA_VERSION = 1
|
|
12
13
|
|
|
14
|
+
# ── project_dir canonicalization ─────────────────────────────────────────────
|
|
15
|
+
# project_dir was stored raw from os.getcwd(), which on Windows returns whichever
|
|
16
|
+
# drive-letter case the process happened to be launched with. Two spellings of
|
|
17
|
+
# ONE directory ("C:\Projects\X" and "c:\Projects\X") therefore landed in the
|
|
18
|
+
# store as distinct projects, and resolve_project_dir_by_name() — which refuses
|
|
19
|
+
# to guess between multiple matches — began reporting the project as unknown.
|
|
20
|
+
#
|
|
21
|
+
# Storage form upper-cases the drive letter and normalizes separators, but keeps
|
|
22
|
+
# the display case of the rest ("AlgoStack" must not become "algostack").
|
|
23
|
+
# Comparison folds case across the whole path, because the stored paths are
|
|
24
|
+
# Windows paths whether or not the reader is running on Windows — which is also
|
|
25
|
+
# why PureWindowsPath is used rather than os.path (os.path.normcase is a no-op
|
|
26
|
+
# on POSIX and would not fold these at all).
|
|
27
|
+
#
|
|
28
|
+
# Both are no-ops on anything that is not a drive-letter path, so POSIX paths
|
|
29
|
+
# are never mangled.
|
|
30
|
+
|
|
31
|
+
_WIN_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def canonical_project_dir(project_dir):
|
|
35
|
+
"""Storage form of a project_dir: upper-case drive, normalized separators."""
|
|
36
|
+
if not isinstance(project_dir, str) or not project_dir:
|
|
37
|
+
return project_dir
|
|
38
|
+
if not _WIN_DRIVE_RE.match(project_dir):
|
|
39
|
+
return project_dir
|
|
40
|
+
text = str(PureWindowsPath(project_dir))
|
|
41
|
+
return text[0].upper() + text[1:]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _project_dir_key(project_dir):
|
|
45
|
+
"""Comparison key: two spellings of the same directory share a key."""
|
|
46
|
+
if not isinstance(project_dir, str) or not project_dir:
|
|
47
|
+
return ""
|
|
48
|
+
if not _WIN_DRIVE_RE.match(project_dir):
|
|
49
|
+
return project_dir
|
|
50
|
+
return str(PureWindowsPath(project_dir)).lower()
|
|
51
|
+
|
|
13
52
|
|
|
14
53
|
def _resolve_db_path() -> Path:
|
|
15
54
|
"""Resolve the memory DB path at call time.
|
|
@@ -210,6 +249,7 @@ def insert_event(
|
|
|
210
249
|
tokens_used: int = 0,
|
|
211
250
|
) -> int | None:
|
|
212
251
|
try:
|
|
252
|
+
project_dir = canonical_project_dir(project_dir)
|
|
213
253
|
conn = _get_conn()
|
|
214
254
|
try:
|
|
215
255
|
details_json = json.dumps(details) if details is not None else None
|
|
@@ -242,6 +282,7 @@ def insert_summary(
|
|
|
242
282
|
tokens_total: int = 0,
|
|
243
283
|
) -> int | None:
|
|
244
284
|
try:
|
|
285
|
+
project_dir = canonical_project_dir(project_dir)
|
|
245
286
|
conn = _get_conn()
|
|
246
287
|
try:
|
|
247
288
|
cur = conn.execute(
|
|
@@ -286,7 +327,7 @@ def insert_lesson(
|
|
|
286
327
|
try:
|
|
287
328
|
conn = _get_conn()
|
|
288
329
|
try:
|
|
289
|
-
pdir = None if global_ else project_dir
|
|
330
|
+
pdir = None if global_ else canonical_project_dir(project_dir)
|
|
290
331
|
tags_json = json.dumps(stack_tags) if stack_tags else None
|
|
291
332
|
if created_at:
|
|
292
333
|
# Caller-supplied timestamp: bind it (never string-format it in).
|
|
@@ -338,6 +379,7 @@ def insert_fact(
|
|
|
338
379
|
source: str | None = None,
|
|
339
380
|
) -> int | None:
|
|
340
381
|
try:
|
|
382
|
+
project_dir = canonical_project_dir(project_dir)
|
|
341
383
|
conn = _get_conn()
|
|
342
384
|
try:
|
|
343
385
|
# Delete old FTS entry if updating
|
|
@@ -386,6 +428,7 @@ def record_failed_approach(
|
|
|
386
428
|
session_id: str | None = None,
|
|
387
429
|
) -> int | None:
|
|
388
430
|
try:
|
|
431
|
+
project_dir = canonical_project_dir(project_dir)
|
|
389
432
|
conn = _get_conn()
|
|
390
433
|
try:
|
|
391
434
|
cur = conn.execute(
|
|
@@ -435,6 +478,7 @@ def query_failed_approaches(
|
|
|
435
478
|
) -> list[dict]:
|
|
436
479
|
if top_k > 50:
|
|
437
480
|
top_k = 50
|
|
481
|
+
project_dir = canonical_project_dir(project_dir)
|
|
438
482
|
match = _fts_or_query(task_text)
|
|
439
483
|
if not match:
|
|
440
484
|
return []
|
|
@@ -472,6 +516,7 @@ def search_memory(
|
|
|
472
516
|
) -> list[dict]:
|
|
473
517
|
if limit > 500:
|
|
474
518
|
limit = 500
|
|
519
|
+
project_dir = canonical_project_dir(project_dir)
|
|
475
520
|
match = _fts_or_query(query)
|
|
476
521
|
if not match:
|
|
477
522
|
return []
|
|
@@ -509,6 +554,7 @@ def search_memory(
|
|
|
509
554
|
def get_recent_events(project_dir: str, limit: int = 20) -> list[dict]:
|
|
510
555
|
if limit > 500:
|
|
511
556
|
limit = 500
|
|
557
|
+
project_dir = canonical_project_dir(project_dir)
|
|
512
558
|
try:
|
|
513
559
|
conn = _get_conn()
|
|
514
560
|
try:
|
|
@@ -537,6 +583,7 @@ def get_recent_events(project_dir: str, limit: int = 20) -> list[dict]:
|
|
|
537
583
|
def get_session_summaries(project_dir: str, limit: int = 50) -> list[dict]:
|
|
538
584
|
if limit > 500:
|
|
539
585
|
limit = 500
|
|
586
|
+
project_dir = canonical_project_dir(project_dir)
|
|
540
587
|
try:
|
|
541
588
|
conn = _get_conn()
|
|
542
589
|
try:
|
|
@@ -582,6 +629,7 @@ def get_lessons(
|
|
|
582
629
|
project_dir: str | None = None, category: str | None = None
|
|
583
630
|
) -> list[dict]:
|
|
584
631
|
try:
|
|
632
|
+
project_dir = canonical_project_dir(project_dir)
|
|
585
633
|
conn = _get_conn()
|
|
586
634
|
try:
|
|
587
635
|
sql = "SELECT * FROM procedural_memory WHERE 1=1"
|
|
@@ -613,6 +661,7 @@ def get_lessons(
|
|
|
613
661
|
|
|
614
662
|
def get_facts(project_dir: str) -> list[dict]:
|
|
615
663
|
try:
|
|
664
|
+
project_dir = canonical_project_dir(project_dir)
|
|
616
665
|
conn = _get_conn()
|
|
617
666
|
try:
|
|
618
667
|
rows = conn.execute(
|
|
@@ -629,6 +678,7 @@ def get_facts(project_dir: str) -> list[dict]:
|
|
|
629
678
|
|
|
630
679
|
def get_memory_stats(project_dir: str | None = None) -> dict:
|
|
631
680
|
try:
|
|
681
|
+
project_dir = canonical_project_dir(project_dir)
|
|
632
682
|
conn = _get_conn()
|
|
633
683
|
try:
|
|
634
684
|
result: dict = {}
|
|
@@ -695,33 +745,41 @@ def get_recent_project_dirs(limit: int = 10) -> list[str]:
|
|
|
695
745
|
return []
|
|
696
746
|
|
|
697
747
|
|
|
698
|
-
def
|
|
699
|
-
"""
|
|
748
|
+
def find_project_dirs_by_name(name: str) -> list[str]:
|
|
749
|
+
"""Every DISTINCT directory the store knows whose basename matches ``name``.
|
|
750
|
+
|
|
751
|
+
Returns canonical paths, sorted. Spellings of the same directory collapse to
|
|
752
|
+
one entry, so the length is a count of real directories:
|
|
753
|
+
|
|
754
|
+
0 -> the store has never seen this project
|
|
755
|
+
1 -> unambiguous; ``resolve_project_dir_by_name`` returns it
|
|
756
|
+
2+ -> a genuine basename collision across different directories
|
|
757
|
+
|
|
758
|
+
``resolve_project_dir_by_name`` cannot distinguish 0 from 2+ (both are
|
|
759
|
+
``None``), which left callers reporting a colliding project as "unknown".
|
|
760
|
+
Callers that need to tell those apart — the diary insight bridge, which
|
|
761
|
+
auto-registers on 0 but must never guess on 2+ — use this instead.
|
|
700
762
|
|
|
701
763
|
The store keeps ``project_dir`` as absolute paths (e.g.
|
|
702
764
|
``C:\\Projects\\adminstack``); callers such as the Diary only know the
|
|
703
765
|
project *name* (``adminstack``). This matches ``name`` case-insensitively
|
|
704
766
|
against the basename of every distinct ``project_dir`` the store already
|
|
705
767
|
knows, unioned across the five tables that carry the column. It assumes no
|
|
706
|
-
on-disk layout and never fabricates a path
|
|
707
|
-
returns None, so the function works on any machine.
|
|
768
|
+
on-disk layout and never fabricates a path, so it works on any machine.
|
|
708
769
|
|
|
709
770
|
Basenames are split with ``PureWindowsPath``, not ``os.path.basename``: the
|
|
710
771
|
stored paths are Windows backslash paths, and ``os.path.basename`` does not
|
|
711
772
|
split on ``\\`` when this DB is read on a POSIX host. ``PureWindowsPath``
|
|
712
773
|
treats both ``\\`` and ``/`` as separators regardless of the running OS.
|
|
713
|
-
|
|
714
|
-
If a basename matches more than one distinct path (a collision), returns
|
|
715
|
-
None rather than guessing which project was meant.
|
|
716
774
|
"""
|
|
717
775
|
try:
|
|
718
776
|
if name is None:
|
|
719
|
-
return
|
|
777
|
+
return []
|
|
720
778
|
if not isinstance(name, str):
|
|
721
779
|
name = str(name)
|
|
722
780
|
key = name.strip().lower()
|
|
723
781
|
if not key:
|
|
724
|
-
return
|
|
782
|
+
return []
|
|
725
783
|
|
|
726
784
|
conn = _get_conn()
|
|
727
785
|
try:
|
|
@@ -745,15 +803,39 @@ def resolve_project_dir_by_name(name: str) -> str | None:
|
|
|
745
803
|
if PureWindowsPath(pdir).name.lower() == key:
|
|
746
804
|
matches.add(pdir)
|
|
747
805
|
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
806
|
+
# Collapse spellings that denote the SAME directory (drive-letter case,
|
|
807
|
+
# separator style) before counting — otherwise one stray "c:\..." row
|
|
808
|
+
# turns a perfectly well-known project into an ambiguous one. Distinct
|
|
809
|
+
# directories that merely share a basename remain separate entries.
|
|
810
|
+
groups: dict = {}
|
|
811
|
+
for match in matches:
|
|
812
|
+
groups.setdefault(_project_dir_key(match), []).append(match)
|
|
813
|
+
|
|
814
|
+
# Deterministic representative per group: sorted() puts the upper-case
|
|
815
|
+
# drive first, and canonicalizing makes the result independent of which
|
|
816
|
+
# spellings happen to be in the store.
|
|
817
|
+
return sorted(
|
|
818
|
+
canonical_project_dir(sorted(spellings)[0])
|
|
819
|
+
for spellings in groups.values()
|
|
820
|
+
)
|
|
751
821
|
except Exception as exc:
|
|
752
822
|
print(
|
|
753
|
-
f"[memstack-memory]
|
|
823
|
+
f"[memstack-memory] find_project_dirs_by_name failed: {exc}",
|
|
754
824
|
file=sys.stderr,
|
|
755
825
|
)
|
|
756
|
-
return
|
|
826
|
+
return []
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def resolve_project_dir_by_name(name: str) -> str | None:
|
|
830
|
+
"""The single directory named ``name``, or None if unknown OR ambiguous.
|
|
831
|
+
|
|
832
|
+
Kept as the narrow, safe default for callers that only need a path and have
|
|
833
|
+
no sensible behavior for a collision. Callers that must distinguish "never
|
|
834
|
+
seen" from "shares a basename with another project" — and act differently —
|
|
835
|
+
should use :func:`find_project_dirs_by_name` and inspect the length.
|
|
836
|
+
"""
|
|
837
|
+
matches = find_project_dirs_by_name(name)
|
|
838
|
+
return matches[0] if len(matches) == 1 else None
|
|
757
839
|
|
|
758
840
|
|
|
759
841
|
def _human_size(nbytes: int) -> str:
|
|
@@ -78,7 +78,18 @@ RECENT_WINDOW = 7.0 # days: a recall this recent floors the score
|
|
|
78
78
|
RECENT_FLOOR = 0.5 # floor applied to recently-recalled facts
|
|
79
79
|
SCORE_ARCHIVE_THRESHOLD = 0.05 # below this a fact is archived out of default recall
|
|
80
80
|
|
|
81
|
+
#: Hard ceiling on rows any single read returns. This is a runaway guard, NOT a
|
|
82
|
+
#: ranking window: recall() orders by score *in SQL* before this applies, so the
|
|
83
|
+
#: rows it trims are genuinely the lowest-scoring ones. It replaced a pair of
|
|
84
|
+
#: hard-coded 500s in list_facts/search_live that silently truncated by insertion
|
|
85
|
+
#: order (id DESC) before scoring ever ran — see the module tests.
|
|
86
|
+
_MAX_ROWS = 10_000
|
|
87
|
+
|
|
81
88
|
_TS_FMT = "%Y-%m-%d %H:%M:%S"
|
|
89
|
+
#: ``now`` is bound into SQL with sub-second precision so the SQL and Python
|
|
90
|
+
#: scorers evaluate at the *identical* instant (julianday parses fractional
|
|
91
|
+
#: seconds). Truncating to whole seconds would let the two drift by up to ~3e-7.
|
|
92
|
+
_TS_FMT_MICRO = "%Y-%m-%d %H:%M:%S.%f"
|
|
82
93
|
|
|
83
94
|
|
|
84
95
|
def _parse_ts(ts: str) -> datetime:
|
|
@@ -112,6 +123,68 @@ def recall_score(fact: dict, now: datetime | None = None) -> float:
|
|
|
112
123
|
return score
|
|
113
124
|
|
|
114
125
|
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
# SQL port of recall_score (the fast recall path)
|
|
128
|
+
#
|
|
129
|
+
# recall() ranks in SQL so the top-k can be chosen without materializing every
|
|
130
|
+
# live fact as a Python dict. This is an EXACT port, not an approximation: the
|
|
131
|
+
# only divergence is float noise between julianday's day arithmetic and
|
|
132
|
+
# datetime's second arithmetic (~1e-9 on real data). `test_sql_python_score_
|
|
133
|
+
# agreement` pins the two together across the whole store so they cannot drift.
|
|
134
|
+
#
|
|
135
|
+
# Every tunable is a BOUND PARAMETER re-read from module level on each call —
|
|
136
|
+
# never interpolated into the SQL text — so the documented monkeypatch contract
|
|
137
|
+
# above still holds for the SQL path.
|
|
138
|
+
#
|
|
139
|
+
# Parameter order (the `?`s sit in the SELECT list, so they bind before any
|
|
140
|
+
# WHERE parameters):
|
|
141
|
+
# now, RECENT_WINDOW, RECENT_FLOOR,
|
|
142
|
+
# now, HALF_LIFE_BASE, HALF_LIFE_PER_RECALL, <- floored branch
|
|
143
|
+
# now, HALF_LIFE_BASE, HALF_LIFE_PER_RECALL <- unfloored branch
|
|
144
|
+
# ---------------------------------------------------------------------------
|
|
145
|
+
_SQL_SCORE = """CASE
|
|
146
|
+
WHEN f.last_recalled_at IS NOT NULL
|
|
147
|
+
AND julianday(?) - julianday(f.last_recalled_at) <= ?
|
|
148
|
+
THEN max(?, f.confidence * exp(
|
|
149
|
+
-max(0.0, julianday(?) - julianday(f.created_at))
|
|
150
|
+
/ (? + ? * f.recall_count)))
|
|
151
|
+
ELSE f.confidence * exp(
|
|
152
|
+
-max(0.0, julianday(?) - julianday(f.created_at))
|
|
153
|
+
/ (? + ? * f.recall_count))
|
|
154
|
+
END"""
|
|
155
|
+
|
|
156
|
+
#: Probe result for SQLite's math functions, cached per process. `exp()` needs a
|
|
157
|
+
#: build with SQLITE_ENABLE_MATH_FUNCTIONS (SQLite >= 3.35); `requires-python =
|
|
158
|
+
#: ">=3.10"` allows stdlib builds without it. None = not yet probed. Tests force
|
|
159
|
+
#: the Python path by setting this to False.
|
|
160
|
+
_SQL_MATH_AVAILABLE: bool | None = None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _sql_score_params(now_ts: str) -> list[Any]:
|
|
164
|
+
"""Bind list for :data:`_SQL_SCORE`, reading every constant at call time."""
|
|
165
|
+
return [
|
|
166
|
+
now_ts, RECENT_WINDOW, RECENT_FLOOR,
|
|
167
|
+
now_ts, HALF_LIFE_BASE, HALF_LIFE_PER_RECALL,
|
|
168
|
+
now_ts, HALF_LIFE_BASE, HALF_LIFE_PER_RECALL,
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _sql_math_available(conn: sqlite3.Connection) -> bool:
|
|
173
|
+
"""True when this SQLite build exposes ``exp()``, cached for the process.
|
|
174
|
+
|
|
175
|
+
Correctness never depends on this probe — only speed. A build without math
|
|
176
|
+
functions falls back to the Python full scan, which returns identical rows.
|
|
177
|
+
"""
|
|
178
|
+
global _SQL_MATH_AVAILABLE
|
|
179
|
+
if _SQL_MATH_AVAILABLE is None:
|
|
180
|
+
try:
|
|
181
|
+
conn.execute("SELECT exp(1.0)").fetchone()
|
|
182
|
+
_SQL_MATH_AVAILABLE = True
|
|
183
|
+
except sqlite3.Error:
|
|
184
|
+
_SQL_MATH_AVAILABLE = False
|
|
185
|
+
return _SQL_MATH_AVAILABLE
|
|
186
|
+
|
|
187
|
+
|
|
115
188
|
def provenance_tag(fact: dict) -> str:
|
|
116
189
|
"""Inline provenance tag for a fact, e.g.
|
|
117
190
|
``[verified · diary memory/sessions/2026-07-23-demo.md · recalled 3x]``.
|
|
@@ -629,9 +702,14 @@ class MemoryStore:
|
|
|
629
702
|
limit: int = 20,
|
|
630
703
|
) -> list[dict]:
|
|
631
704
|
"""List facts for a project (newest first), filtered by status
|
|
632
|
-
(default ``live``) and optionally by exact subject.
|
|
633
|
-
|
|
634
|
-
|
|
705
|
+
(default ``live``) and optionally by exact subject.
|
|
706
|
+
|
|
707
|
+
Ordered by insertion (``id DESC``), so a ``limit`` below the project's
|
|
708
|
+
live count truncates by *recency of insertion*, not by relevance. Use
|
|
709
|
+
:meth:`recall` when you want the highest-scoring facts.
|
|
710
|
+
"""
|
|
711
|
+
if limit > _MAX_ROWS:
|
|
712
|
+
limit = _MAX_ROWS
|
|
635
713
|
conn = self._connect()
|
|
636
714
|
try:
|
|
637
715
|
sql = "SELECT * FROM memory_facts WHERE project=? AND status=?"
|
|
@@ -645,6 +723,22 @@ class MemoryStore:
|
|
|
645
723
|
finally:
|
|
646
724
|
conn.close()
|
|
647
725
|
|
|
726
|
+
def count_live(self, project: str) -> int:
|
|
727
|
+
"""Number of live facts for ``project``.
|
|
728
|
+
|
|
729
|
+
The SessionStart block uses this to report how many live facts it did
|
|
730
|
+
NOT inject. Without it the footer could only count what fell out of the
|
|
731
|
+
candidate window, which understates the true remainder.
|
|
732
|
+
"""
|
|
733
|
+
conn = self._connect()
|
|
734
|
+
try:
|
|
735
|
+
return conn.execute(
|
|
736
|
+
"SELECT COUNT(*) FROM memory_facts WHERE project=? AND status='live'",
|
|
737
|
+
(project,),
|
|
738
|
+
).fetchone()[0]
|
|
739
|
+
finally:
|
|
740
|
+
conn.close()
|
|
741
|
+
|
|
648
742
|
def list_projects(self) -> list[dict]:
|
|
649
743
|
"""Distinct projects that hold at least one live fact, each with its live
|
|
650
744
|
count, ordered by count desc then name. Read-only; backs the dashboard
|
|
@@ -725,8 +819,8 @@ class MemoryStore:
|
|
|
725
819
|
match = _fts_or_query(query)
|
|
726
820
|
if not match:
|
|
727
821
|
return []
|
|
728
|
-
if limit >
|
|
729
|
-
limit =
|
|
822
|
+
if limit > _MAX_ROWS:
|
|
823
|
+
limit = _MAX_ROWS
|
|
730
824
|
conn = self._connect()
|
|
731
825
|
try:
|
|
732
826
|
sql = (
|
|
@@ -745,11 +839,6 @@ class MemoryStore:
|
|
|
745
839
|
finally:
|
|
746
840
|
conn.close()
|
|
747
841
|
|
|
748
|
-
# Broad candidate cap for scored recall: we fetch up to this many live facts
|
|
749
|
-
# to rank, then return the top `limit`. Far above any realistic per-project
|
|
750
|
-
# live-fact count for a personal memory store.
|
|
751
|
-
_RECALL_CANDIDATE_CAP = 500
|
|
752
|
-
|
|
753
842
|
def recall(
|
|
754
843
|
self,
|
|
755
844
|
project: str,
|
|
@@ -767,14 +856,87 @@ class MemoryStore:
|
|
|
767
856
|
below ``SCORE_ARCHIVE_THRESHOLD`` are archived out of the default result
|
|
768
857
|
and only surface when ``deep=True``. The archive threshold and all
|
|
769
858
|
scoring constants are read from module level at call time.
|
|
859
|
+
|
|
860
|
+
Ranking happens in SQL (:data:`_SQL_SCORE`) so the whole live set is
|
|
861
|
+
ordered before ``limit`` applies. On a SQLite build without math
|
|
862
|
+
functions this falls back to scoring every candidate in Python, which
|
|
863
|
+
returns identical rows for more wall-clock. Either way the result is the
|
|
864
|
+
true top-``limit`` by score, never a slice of the most recently inserted.
|
|
770
865
|
"""
|
|
771
|
-
if
|
|
772
|
-
|
|
773
|
-
|
|
866
|
+
if now is None:
|
|
867
|
+
now = datetime.now(timezone.utc)
|
|
868
|
+
if limit > _MAX_ROWS:
|
|
869
|
+
limit = _MAX_ROWS
|
|
870
|
+
|
|
871
|
+
conn = self._connect()
|
|
872
|
+
try:
|
|
873
|
+
if _sql_math_available(conn):
|
|
874
|
+
return self._recall_sql(conn, project, query, subject, deep, limit, now)
|
|
875
|
+
finally:
|
|
876
|
+
conn.close()
|
|
877
|
+
return self._recall_python(project, query, subject, deep, limit, now)
|
|
878
|
+
|
|
879
|
+
def _recall_sql(
|
|
880
|
+
self,
|
|
881
|
+
conn: sqlite3.Connection,
|
|
882
|
+
project: str,
|
|
883
|
+
query: str | None,
|
|
884
|
+
subject: str | None,
|
|
885
|
+
deep: bool,
|
|
886
|
+
limit: int,
|
|
887
|
+
now: datetime,
|
|
888
|
+
) -> list[dict]:
|
|
889
|
+
"""Score-ordered recall evaluated inside SQLite (the fast path)."""
|
|
890
|
+
now_ts = now.astimezone(timezone.utc).strftime(_TS_FMT_MICRO)
|
|
891
|
+
params: list[Any] = _sql_score_params(now_ts)
|
|
892
|
+
|
|
893
|
+
if query is not None:
|
|
894
|
+
match = _fts_or_query(query)
|
|
895
|
+
if not match:
|
|
896
|
+
return []
|
|
897
|
+
source = (
|
|
898
|
+
"memory_facts_fts fts JOIN memory_facts f ON f.id = fts.rowid "
|
|
899
|
+
"WHERE memory_facts_fts MATCH ? AND f.status='live' AND f.project=?"
|
|
774
900
|
)
|
|
901
|
+
params += [match, project]
|
|
902
|
+
else:
|
|
903
|
+
source = "memory_facts f WHERE f.project=? AND f.status='live'"
|
|
904
|
+
params.append(project)
|
|
905
|
+
if subject is not None:
|
|
906
|
+
source += " AND f.subject=?"
|
|
907
|
+
params.append(subject)
|
|
908
|
+
|
|
909
|
+
# The archive threshold filters the *computed* score, so it has to sit
|
|
910
|
+
# outside the subquery that produces it.
|
|
911
|
+
outer = ""
|
|
912
|
+
if not deep:
|
|
913
|
+
outer = "WHERE score >= ? "
|
|
914
|
+
params.append(SCORE_ARCHIVE_THRESHOLD)
|
|
915
|
+
params.append(limit)
|
|
916
|
+
|
|
917
|
+
# `id DESC` tie-breaks equal scores, matching the Python path's stable
|
|
918
|
+
# sort over an id-DESC candidate list, so both paths agree exactly.
|
|
919
|
+
sql = (
|
|
920
|
+
f"SELECT * FROM (SELECT f.*, {_SQL_SCORE} AS score FROM {source}) "
|
|
921
|
+
f"{outer}ORDER BY score DESC, id DESC LIMIT ?"
|
|
922
|
+
)
|
|
923
|
+
return [dict(r) for r in conn.execute(sql, params).fetchall()]
|
|
924
|
+
|
|
925
|
+
def _recall_python(
|
|
926
|
+
self,
|
|
927
|
+
project: str,
|
|
928
|
+
query: str | None,
|
|
929
|
+
subject: str | None,
|
|
930
|
+
deep: bool,
|
|
931
|
+
limit: int,
|
|
932
|
+
now: datetime,
|
|
933
|
+
) -> list[dict]:
|
|
934
|
+
"""Score-ordered recall computed in Python (fallback; see :meth:`recall`)."""
|
|
935
|
+
if query is not None:
|
|
936
|
+
candidates = self.search_live(query, project=project, limit=_MAX_ROWS)
|
|
775
937
|
else:
|
|
776
938
|
candidates = self.list_facts(
|
|
777
|
-
project=project, subject=subject, limit=
|
|
939
|
+
project=project, subject=subject, limit=_MAX_ROWS
|
|
778
940
|
)
|
|
779
941
|
|
|
780
942
|
scored = []
|
|
@@ -786,7 +948,9 @@ class MemoryStore:
|
|
|
786
948
|
if not deep:
|
|
787
949
|
scored = [f for f in scored if f["score"] >= SCORE_ARCHIVE_THRESHOLD]
|
|
788
950
|
|
|
789
|
-
|
|
951
|
+
# Explicit id tie-break so this path orders equal scores exactly as the
|
|
952
|
+
# SQL path does (which cannot rely on a stable sort).
|
|
953
|
+
scored.sort(key=lambda f: (-f["score"], -f["id"]))
|
|
790
954
|
return scored[:limit]
|
|
791
955
|
|
|
792
956
|
# -- internal ---------------------------------------------------------
|
|
@@ -191,7 +191,15 @@ def _truncate_python_indexed(lines: list[str]) -> list[tuple]:
|
|
|
191
191
|
line = lines[i]
|
|
192
192
|
recs.append(("keep", i))
|
|
193
193
|
i += 1
|
|
194
|
-
#
|
|
194
|
+
# Optional docstring: the summary line is always kept, and every
|
|
195
|
+
# remaining line is ACCOUNTED FOR — never dropped without a marker.
|
|
196
|
+
# Eliding the continuation lines silently made a multi-line docstring
|
|
197
|
+
# read back as a complete one-liner, so callers built edit anchors
|
|
198
|
+
# against text that does not exist on disk (function bodies never had
|
|
199
|
+
# this problem: they always leave a labelled stub behind). Interior
|
|
200
|
+
# lines therefore get the same stub treatment as a body, and the
|
|
201
|
+
# closing-quote line is kept so the docstring still reads as spanning
|
|
202
|
+
# multiple lines.
|
|
195
203
|
if i < n:
|
|
196
204
|
ns = lines[i].lstrip()
|
|
197
205
|
if ns.startswith(('"""', "'''")):
|
|
@@ -199,10 +207,23 @@ def _truncate_python_indexed(lines: list[str]) -> list[tuple]:
|
|
|
199
207
|
q = '"""' if '"""' in ns else "'''"
|
|
200
208
|
# single-line docstring has closing quotes on same line after the open
|
|
201
209
|
if q not in ns[3:]:
|
|
202
|
-
# multi-line:
|
|
210
|
+
# multi-line: advance to the closing quote, stubbing the
|
|
211
|
+
# interior. Same >=2-non-blank gate as the body below — a
|
|
212
|
+
# single interior line is kept verbatim, which is lossless
|
|
213
|
+
# and no dearer than the labelled stub replacing it.
|
|
214
|
+
doc_start = i + 1
|
|
203
215
|
i += 1
|
|
204
216
|
while i < n and q not in lines[i]:
|
|
205
217
|
i += 1
|
|
218
|
+
# lines[i] is the closing-quote line, or i == n if the
|
|
219
|
+
# docstring is unterminated (truncated/malformed input).
|
|
220
|
+
doc_nonblank = [j for j in range(doc_start, i) if lines[j].strip()]
|
|
221
|
+
if len(doc_nonblank) >= 2:
|
|
222
|
+
recs.append(("stub", indent, doc_start, i - 1))
|
|
223
|
+
elif len(doc_nonblank) == 1:
|
|
224
|
+
recs.append(("keep", doc_nonblank[0]))
|
|
225
|
+
if i < n:
|
|
226
|
+
recs.append(("keep", i))
|
|
206
227
|
i += 1
|
|
207
228
|
else:
|
|
208
229
|
# single-line docstring: step past it so the body range
|