superlocalmemory 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +50 -0
- package/CHANGELOG.md +8 -0
- package/README.md +31 -20
- package/api_server.py +5 -0
- package/bin/aider-smart +2 -2
- package/bin/slm +18 -18
- package/bin/slm.bat +3 -3
- package/configs/continue-skills.yaml +4 -4
- package/docs/ARCHITECTURE.md +3 -3
- package/docs/CLI-COMMANDS-REFERENCE.md +18 -18
- package/docs/FRAMEWORK-INTEGRATIONS.md +4 -4
- package/docs/SECURITY-QUICK-REFERENCE.md +214 -0
- package/docs/UNIVERSAL-INTEGRATION.md +15 -15
- package/install.ps1 +11 -11
- package/install.sh +4 -4
- package/mcp_server.py +4 -4
- package/package.json +5 -3
- package/requirements-core.txt +16 -18
- package/requirements-learning.txt +8 -8
- package/requirements.txt +9 -7
- package/scripts/prepack.js +33 -0
- package/scripts/verify-v27.ps1 +301 -0
- package/src/agent_registry.py +32 -28
- package/src/auto_backup.py +12 -6
- package/src/cache_manager.py +2 -2
- package/src/compression/__init__.py +25 -0
- package/src/compression/cli.py +150 -0
- package/src/compression/cold_storage.py +217 -0
- package/src/compression/config.py +72 -0
- package/src/compression/orchestrator.py +133 -0
- package/src/compression/tier2_compressor.py +228 -0
- package/src/compression/tier3_compressor.py +153 -0
- package/src/compression/tier_classifier.py +148 -0
- package/src/db_connection_manager.py +5 -5
- package/src/event_bus.py +24 -22
- package/src/graph/graph_core.py +3 -3
- package/src/hnsw_index.py +3 -3
- package/src/learning/__init__.py +5 -4
- package/src/learning/adaptive_ranker.py +14 -265
- package/src/learning/bootstrap/__init__.py +69 -0
- package/src/learning/bootstrap/constants.py +93 -0
- package/src/learning/bootstrap/db_queries.py +316 -0
- package/src/learning/bootstrap/sampling.py +82 -0
- package/src/learning/bootstrap/text_utils.py +71 -0
- package/src/learning/cross_project_aggregator.py +58 -57
- package/src/learning/db/__init__.py +40 -0
- package/src/learning/db/constants.py +44 -0
- package/src/learning/db/schema.py +279 -0
- package/src/learning/learning_db.py +15 -234
- package/src/learning/ranking/__init__.py +33 -0
- package/src/learning/ranking/constants.py +84 -0
- package/src/learning/ranking/helpers.py +278 -0
- package/src/learning/source_quality_scorer.py +66 -65
- package/src/learning/synthetic_bootstrap.py +28 -310
- package/src/memory/__init__.py +36 -0
- package/src/memory/cli.py +205 -0
- package/src/memory/constants.py +39 -0
- package/src/memory/helpers.py +28 -0
- package/src/memory/schema.py +166 -0
- package/src/memory-profiles.py +94 -86
- package/src/memory-reset.py +187 -185
- package/src/memory_compression.py +2 -2
- package/src/memory_store_v2.py +40 -355
- package/src/migrate_v1_to_v2.py +11 -10
- package/src/patterns/analyzers.py +104 -100
- package/src/patterns/learner.py +17 -13
- package/src/patterns/scoring.py +25 -21
- package/src/patterns/store.py +40 -38
- package/src/patterns/terminology.py +53 -51
- package/src/provenance_tracker.py +2 -2
- package/src/qualixar_attribution.py +139 -0
- package/src/qualixar_watermark.py +78 -0
- package/src/search/engine.py +16 -14
- package/src/search/index_loader.py +13 -11
- package/src/setup_validator.py +162 -160
- package/src/subscription_manager.py +20 -18
- package/src/tree/builder.py +66 -64
- package/src/tree/nodes.py +103 -97
- package/src/tree/queries.py +142 -137
- package/src/tree/schema.py +46 -42
- package/src/webhook_dispatcher.py +3 -3
- package/ui_server.py +7 -4
- /package/bin/{superlocalmemoryv2:learning → superlocalmemoryv2-learning} +0 -0
- /package/bin/{superlocalmemoryv2:list → superlocalmemoryv2-list} +0 -0
- /package/bin/{superlocalmemoryv2:patterns → superlocalmemoryv2-patterns} +0 -0
- /package/bin/{superlocalmemoryv2:profile → superlocalmemoryv2-profile} +0 -0
- /package/bin/{superlocalmemoryv2:recall → superlocalmemoryv2-recall} +0 -0
- /package/bin/{superlocalmemoryv2:remember → superlocalmemoryv2-remember} +0 -0
- /package/bin/{superlocalmemoryv2:reset → superlocalmemoryv2-reset} +0 -0
- /package/bin/{superlocalmemoryv2:status → superlocalmemoryv2-status} +0 -0
|
@@ -53,7 +53,7 @@ class SubscriptionManager:
|
|
|
53
53
|
return cls._instances[key]
|
|
54
54
|
|
|
55
55
|
@classmethod
|
|
56
|
-
def reset_instance(cls, db_path: Optional[Path] = None):
|
|
56
|
+
def reset_instance(cls, db_path: Optional[Path] = None) -> None:
|
|
57
57
|
"""Remove singleton. Used for testing."""
|
|
58
58
|
with cls._instances_lock:
|
|
59
59
|
if db_path is None:
|
|
@@ -103,22 +103,24 @@ class SubscriptionManager:
|
|
|
103
103
|
except ImportError:
|
|
104
104
|
import sqlite3
|
|
105
105
|
conn = sqlite3.connect(str(self.db_path))
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
106
|
+
try:
|
|
107
|
+
conn.execute('''
|
|
108
|
+
CREATE TABLE IF NOT EXISTS subscriptions (
|
|
109
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
110
|
+
subscriber_id TEXT NOT NULL UNIQUE,
|
|
111
|
+
channel TEXT NOT NULL,
|
|
112
|
+
filter TEXT NOT NULL DEFAULT '{}',
|
|
113
|
+
webhook_url TEXT,
|
|
114
|
+
durable INTEGER DEFAULT 1,
|
|
115
|
+
last_event_id INTEGER DEFAULT 0,
|
|
116
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
117
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
118
|
+
)
|
|
119
|
+
''')
|
|
120
|
+
conn.execute('CREATE INDEX IF NOT EXISTS idx_subs_channel ON subscriptions(channel)')
|
|
121
|
+
conn.commit()
|
|
122
|
+
finally:
|
|
123
|
+
conn.close()
|
|
122
124
|
|
|
123
125
|
# =========================================================================
|
|
124
126
|
# Subscribe / Unsubscribe
|
|
@@ -244,7 +246,7 @@ class SubscriptionManager:
|
|
|
244
246
|
|
|
245
247
|
return removed
|
|
246
248
|
|
|
247
|
-
def update_last_event_id(self, subscriber_id: str, event_id: int):
|
|
249
|
+
def update_last_event_id(self, subscriber_id: str, event_id: int) -> None:
|
|
248
250
|
"""Update the last event ID received by a durable subscriber (for replay)."""
|
|
249
251
|
try:
|
|
250
252
|
from db_connection_manager import DbConnectionManager
|
package/src/tree/builder.py
CHANGED
|
@@ -22,70 +22,72 @@ class TreeBuilderMixin:
|
|
|
22
22
|
5. Update aggregated counts
|
|
23
23
|
"""
|
|
24
24
|
conn = sqlite3.connect(self.db_path)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
25
|
+
try:
|
|
26
|
+
cursor = conn.cursor()
|
|
27
|
+
|
|
28
|
+
# Clear existing tree (keep root)
|
|
29
|
+
cursor.execute('DELETE FROM memory_tree WHERE node_type != ?', ('root',))
|
|
30
|
+
|
|
31
|
+
# Step 1: Create project nodes
|
|
32
|
+
cursor.execute('''
|
|
33
|
+
SELECT DISTINCT project_path, project_name
|
|
34
|
+
FROM memories
|
|
35
|
+
WHERE project_path IS NOT NULL
|
|
36
|
+
ORDER BY project_path
|
|
37
|
+
''')
|
|
38
|
+
projects = cursor.fetchall()
|
|
39
|
+
|
|
40
|
+
project_map = {} # project_path -> node_id
|
|
41
|
+
|
|
42
|
+
for project_path, project_name in projects:
|
|
43
|
+
name = project_name or project_path.split('/')[-1]
|
|
44
|
+
node_id = self.add_node('project', name, self.root_id, description=project_path)
|
|
45
|
+
project_map[project_path] = node_id
|
|
46
|
+
|
|
47
|
+
# Step 2: Create category nodes within projects
|
|
48
|
+
cursor.execute('''
|
|
49
|
+
SELECT DISTINCT project_path, category
|
|
50
|
+
FROM memories
|
|
51
|
+
WHERE project_path IS NOT NULL AND category IS NOT NULL
|
|
52
|
+
ORDER BY project_path, category
|
|
53
|
+
''')
|
|
54
|
+
categories = cursor.fetchall()
|
|
55
|
+
|
|
56
|
+
category_map = {} # (project_path, category) -> node_id
|
|
57
|
+
|
|
58
|
+
for project_path, category in categories:
|
|
59
|
+
parent_id = project_map.get(project_path)
|
|
60
|
+
if parent_id:
|
|
61
|
+
node_id = self.add_node('category', category, parent_id)
|
|
62
|
+
category_map[(project_path, category)] = node_id
|
|
63
|
+
|
|
64
|
+
# Step 3: Link memories as leaf nodes
|
|
65
|
+
cursor.execute('''
|
|
66
|
+
SELECT id, content, summary, project_path, category, importance, created_at
|
|
67
|
+
FROM memories
|
|
68
|
+
ORDER BY created_at DESC
|
|
69
|
+
''')
|
|
70
|
+
memories = cursor.fetchall()
|
|
71
|
+
|
|
72
|
+
for mem_id, content, summary, project_path, category, importance, created_at in memories:
|
|
73
|
+
# Determine parent node
|
|
74
|
+
if project_path and category and (project_path, category) in category_map:
|
|
75
|
+
parent_id = category_map[(project_path, category)]
|
|
76
|
+
elif project_path and project_path in project_map:
|
|
77
|
+
parent_id = project_map[project_path]
|
|
78
|
+
else:
|
|
79
|
+
parent_id = self.root_id
|
|
80
|
+
|
|
81
|
+
# Create memory node
|
|
82
|
+
name = summary or content[:60].replace('\n', ' ')
|
|
83
|
+
self.add_node('memory', name, parent_id, memory_id=mem_id, description=content[:200])
|
|
84
|
+
|
|
85
|
+
# Step 4: Update aggregated counts
|
|
86
|
+
self._update_all_counts()
|
|
87
|
+
|
|
88
|
+
conn.commit()
|
|
89
|
+
finally:
|
|
90
|
+
conn.close()
|
|
89
91
|
|
|
90
92
|
|
|
91
93
|
def run_cli():
|
package/src/tree/nodes.py
CHANGED
|
@@ -35,46 +35,48 @@ class TreeNodesMixin:
|
|
|
35
35
|
New node ID
|
|
36
36
|
"""
|
|
37
37
|
conn = sqlite3.connect(self.db_path)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
38
|
+
try:
|
|
39
|
+
cursor = conn.cursor()
|
|
40
|
+
|
|
41
|
+
# Get parent path and depth
|
|
42
|
+
cursor.execute('SELECT tree_path, depth FROM memory_tree WHERE id = ?', (parent_id,))
|
|
43
|
+
result = cursor.fetchone()
|
|
44
|
+
|
|
45
|
+
if not result:
|
|
46
|
+
raise ValueError(f"Parent node {parent_id} not found")
|
|
47
|
+
|
|
48
|
+
parent_path, parent_depth = result
|
|
49
|
+
|
|
50
|
+
# Calculate new node position
|
|
51
|
+
depth = parent_depth + 1
|
|
52
|
+
|
|
53
|
+
cursor.execute('''
|
|
54
|
+
INSERT INTO memory_tree (
|
|
55
|
+
node_type, name, description,
|
|
56
|
+
parent_id, tree_path, depth,
|
|
57
|
+
memory_id, last_updated
|
|
58
|
+
)
|
|
59
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
60
|
+
''', (
|
|
61
|
+
node_type,
|
|
62
|
+
name,
|
|
63
|
+
description,
|
|
64
|
+
parent_id,
|
|
65
|
+
'', # Placeholder, updated below
|
|
66
|
+
depth,
|
|
67
|
+
memory_id,
|
|
68
|
+
datetime.now().isoformat()
|
|
69
|
+
))
|
|
70
|
+
|
|
71
|
+
node_id = cursor.lastrowid
|
|
72
|
+
|
|
73
|
+
# Update tree_path with actual node_id
|
|
74
|
+
tree_path = f"{parent_path}.{node_id}"
|
|
75
|
+
cursor.execute('UPDATE memory_tree SET tree_path = ? WHERE id = ?', (tree_path, node_id))
|
|
76
|
+
|
|
77
|
+
conn.commit()
|
|
78
|
+
finally:
|
|
79
|
+
conn.close()
|
|
78
80
|
|
|
79
81
|
return node_id
|
|
80
82
|
|
|
@@ -92,25 +94,26 @@ class TreeNodesMixin:
|
|
|
92
94
|
raise ValueError("Cannot delete root node")
|
|
93
95
|
|
|
94
96
|
conn = sqlite3.connect(self.db_path)
|
|
95
|
-
|
|
97
|
+
try:
|
|
98
|
+
cursor = conn.cursor()
|
|
96
99
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
+
# Get tree_path
|
|
101
|
+
cursor.execute('SELECT tree_path, parent_id FROM memory_tree WHERE id = ?', (node_id,))
|
|
102
|
+
result = cursor.fetchone()
|
|
100
103
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
return False
|
|
104
|
+
if not result:
|
|
105
|
+
return False
|
|
104
106
|
|
|
105
|
-
|
|
107
|
+
tree_path, parent_id = result
|
|
106
108
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
109
|
+
# Delete node and all descendants (CASCADE handles children)
|
|
110
|
+
cursor.execute('DELETE FROM memory_tree WHERE id = ? OR tree_path LIKE ?',
|
|
111
|
+
(node_id, f"{tree_path}.%"))
|
|
110
112
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
113
|
+
deleted = cursor.rowcount > 0
|
|
114
|
+
conn.commit()
|
|
115
|
+
finally:
|
|
116
|
+
conn.close()
|
|
114
117
|
|
|
115
118
|
# Update parent counts
|
|
116
119
|
if deleted and parent_id:
|
|
@@ -127,56 +130,59 @@ class TreeNodesMixin:
|
|
|
127
130
|
node_id: Node ID to update
|
|
128
131
|
"""
|
|
129
132
|
conn = sqlite3.connect(self.db_path)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
133
|
+
try:
|
|
134
|
+
cursor = conn.cursor()
|
|
135
|
+
|
|
136
|
+
# Get all descendant memory nodes
|
|
137
|
+
cursor.execute('SELECT tree_path FROM memory_tree WHERE id = ?', (node_id,))
|
|
138
|
+
result = cursor.fetchone()
|
|
139
|
+
|
|
140
|
+
if not result:
|
|
141
|
+
return
|
|
142
|
+
|
|
143
|
+
tree_path = result[0]
|
|
144
|
+
|
|
145
|
+
# Count memories in subtree
|
|
146
|
+
cursor.execute('''
|
|
147
|
+
SELECT COUNT(*), COALESCE(SUM(LENGTH(m.content)), 0)
|
|
148
|
+
FROM memory_tree t
|
|
149
|
+
LEFT JOIN memories m ON t.memory_id = m.id
|
|
150
|
+
WHERE t.tree_path LIKE ? AND t.memory_id IS NOT NULL
|
|
151
|
+
''', (f"{tree_path}%",))
|
|
152
|
+
|
|
153
|
+
memory_count, total_size = cursor.fetchone()
|
|
154
|
+
|
|
155
|
+
# Update node
|
|
156
|
+
cursor.execute('''
|
|
157
|
+
UPDATE memory_tree
|
|
158
|
+
SET memory_count = ?, total_size = ?, last_updated = ?
|
|
159
|
+
WHERE id = ?
|
|
160
|
+
''', (memory_count, total_size, datetime.now().isoformat(), node_id))
|
|
161
|
+
|
|
162
|
+
# Update all ancestors
|
|
163
|
+
path_ids = [int(x) for x in tree_path.split('.')]
|
|
164
|
+
for ancestor_id in path_ids[:-1]: # Exclude current node
|
|
165
|
+
self.update_counts(ancestor_id)
|
|
166
|
+
|
|
167
|
+
conn.commit()
|
|
168
|
+
finally:
|
|
137
169
|
conn.close()
|
|
138
|
-
return
|
|
139
|
-
|
|
140
|
-
tree_path = result[0]
|
|
141
|
-
|
|
142
|
-
# Count memories in subtree
|
|
143
|
-
cursor.execute('''
|
|
144
|
-
SELECT COUNT(*), COALESCE(SUM(LENGTH(m.content)), 0)
|
|
145
|
-
FROM memory_tree t
|
|
146
|
-
LEFT JOIN memories m ON t.memory_id = m.id
|
|
147
|
-
WHERE t.tree_path LIKE ? AND t.memory_id IS NOT NULL
|
|
148
|
-
''', (f"{tree_path}%",))
|
|
149
|
-
|
|
150
|
-
memory_count, total_size = cursor.fetchone()
|
|
151
|
-
|
|
152
|
-
# Update node
|
|
153
|
-
cursor.execute('''
|
|
154
|
-
UPDATE memory_tree
|
|
155
|
-
SET memory_count = ?, total_size = ?, last_updated = ?
|
|
156
|
-
WHERE id = ?
|
|
157
|
-
''', (memory_count, total_size, datetime.now().isoformat(), node_id))
|
|
158
|
-
|
|
159
|
-
# Update all ancestors
|
|
160
|
-
path_ids = [int(x) for x in tree_path.split('.')]
|
|
161
|
-
for ancestor_id in path_ids[:-1]: # Exclude current node
|
|
162
|
-
self.update_counts(ancestor_id)
|
|
163
|
-
|
|
164
|
-
conn.commit()
|
|
165
|
-
conn.close()
|
|
166
170
|
|
|
167
171
|
def _update_all_counts(self):
|
|
168
172
|
"""Update counts for all nodes (used after build_tree)."""
|
|
169
173
|
conn = sqlite3.connect(self.db_path)
|
|
170
|
-
|
|
174
|
+
try:
|
|
175
|
+
cursor = conn.cursor()
|
|
171
176
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
+
# Get all nodes in reverse depth order (leaves first)
|
|
178
|
+
cursor.execute('''
|
|
179
|
+
SELECT id FROM memory_tree
|
|
180
|
+
ORDER BY depth DESC
|
|
181
|
+
''')
|
|
177
182
|
|
|
178
|
-
|
|
179
|
-
|
|
183
|
+
node_ids = [row[0] for row in cursor.fetchall()]
|
|
184
|
+
finally:
|
|
185
|
+
conn.close()
|
|
180
186
|
|
|
181
187
|
# Update each node (will cascade to parents)
|
|
182
188
|
processed = set()
|