superlocalmemory 2.8.2 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +7 -5
  2. package/api_server.py +5 -0
  3. package/bin/slm.bat +3 -3
  4. package/docs/SECURITY-QUICK-REFERENCE.md +214 -0
  5. package/install.ps1 +11 -11
  6. package/mcp_server.py +3 -3
  7. package/package.json +2 -2
  8. package/requirements-core.txt +16 -18
  9. package/requirements-learning.txt +8 -8
  10. package/requirements.txt +9 -7
  11. package/scripts/prepack.js +33 -0
  12. package/scripts/verify-v27.ps1 +301 -0
  13. package/src/agent_registry.py +32 -28
  14. package/src/auto_backup.py +12 -6
  15. package/src/cache_manager.py +2 -2
  16. package/src/compression/__init__.py +25 -0
  17. package/src/compression/cli.py +150 -0
  18. package/src/compression/cold_storage.py +217 -0
  19. package/src/compression/config.py +72 -0
  20. package/src/compression/orchestrator.py +133 -0
  21. package/src/compression/tier2_compressor.py +228 -0
  22. package/src/compression/tier3_compressor.py +153 -0
  23. package/src/compression/tier_classifier.py +148 -0
  24. package/src/db_connection_manager.py +5 -5
  25. package/src/event_bus.py +24 -22
  26. package/src/hnsw_index.py +3 -3
  27. package/src/learning/__init__.py +5 -4
  28. package/src/learning/adaptive_ranker.py +14 -265
  29. package/src/learning/bootstrap/__init__.py +69 -0
  30. package/src/learning/bootstrap/constants.py +93 -0
  31. package/src/learning/bootstrap/db_queries.py +316 -0
  32. package/src/learning/bootstrap/sampling.py +82 -0
  33. package/src/learning/bootstrap/text_utils.py +71 -0
  34. package/src/learning/cross_project_aggregator.py +58 -57
  35. package/src/learning/db/__init__.py +40 -0
  36. package/src/learning/db/constants.py +44 -0
  37. package/src/learning/db/schema.py +279 -0
  38. package/src/learning/learning_db.py +15 -234
  39. package/src/learning/ranking/__init__.py +33 -0
  40. package/src/learning/ranking/constants.py +84 -0
  41. package/src/learning/ranking/helpers.py +278 -0
  42. package/src/learning/source_quality_scorer.py +66 -65
  43. package/src/learning/synthetic_bootstrap.py +28 -310
  44. package/src/memory/__init__.py +36 -0
  45. package/src/memory/cli.py +205 -0
  46. package/src/memory/constants.py +39 -0
  47. package/src/memory/helpers.py +28 -0
  48. package/src/memory/schema.py +166 -0
  49. package/src/memory-profiles.py +94 -86
  50. package/src/memory-reset.py +187 -185
  51. package/src/memory_compression.py +2 -2
  52. package/src/memory_store_v2.py +34 -354
  53. package/src/migrate_v1_to_v2.py +11 -10
  54. package/src/patterns/analyzers.py +104 -100
  55. package/src/patterns/learner.py +17 -13
  56. package/src/patterns/scoring.py +25 -21
  57. package/src/patterns/store.py +40 -38
  58. package/src/patterns/terminology.py +53 -51
  59. package/src/provenance_tracker.py +2 -2
  60. package/src/qualixar_attribution.py +1 -1
  61. package/src/search/engine.py +16 -14
  62. package/src/search/index_loader.py +13 -11
  63. package/src/setup_validator.py +160 -158
  64. package/src/subscription_manager.py +20 -18
  65. package/src/tree/builder.py +66 -64
  66. package/src/tree/nodes.py +103 -97
  67. package/src/tree/queries.py +142 -137
  68. package/src/tree/schema.py +46 -42
  69. package/src/webhook_dispatcher.py +3 -3
  70. package/ui_server.py +7 -4
@@ -46,7 +46,7 @@ class MemoryReset:
46
46
  print("⚠ No database found to backup")
47
47
  return None
48
48
 
49
- def soft_reset(self):
49
+ def soft_reset(self) -> None:
50
50
  """Clear all memories but keep V2 schema structure."""
51
51
  print("\n" + "="*60)
52
52
  print("SOFT RESET - Clear Memories, Keep Schema")
@@ -94,7 +94,7 @@ class MemoryReset:
94
94
  print(" Schema preserved, all memories cleared")
95
95
  print(" You can now add new memories to clean system")
96
96
 
97
- def hard_reset(self):
97
+ def hard_reset(self) -> None:
98
98
  """Delete database completely and reinitialize with V2 schema."""
99
99
  print("\n" + "="*60)
100
100
  print("HARD RESET - Delete Everything & Reinitialize")
@@ -116,7 +116,7 @@ class MemoryReset:
116
116
  print(" Fresh V2 database created")
117
117
  print(" Ready for new memories")
118
118
 
119
- def layer_reset(self, layers: list):
119
+ def layer_reset(self, layers: list) -> None:
120
120
  """Reset specific layers only."""
121
121
  print("\n" + "="*60)
122
122
  print(f"LAYER RESET - Clearing Layers: {', '.join(layers)}")
@@ -170,188 +170,190 @@ class MemoryReset:
170
170
  def _initialize_v2_schema(self):
171
171
  """Initialize fresh V2 database schema."""
172
172
  conn = sqlite3.connect(self.db_path)
173
- cursor = conn.cursor()
174
-
175
- # Layer 1: Memories table
176
- cursor.execute('''
177
- CREATE TABLE IF NOT EXISTS memories (
178
- id INTEGER PRIMARY KEY AUTOINCREMENT,
179
- content TEXT NOT NULL,
180
- summary TEXT,
181
- project_path TEXT,
182
- project_name TEXT,
183
- tags TEXT,
184
- category TEXT,
185
- memory_type TEXT DEFAULT 'session',
186
- importance INTEGER DEFAULT 5,
187
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
188
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
189
- last_accessed TIMESTAMP,
190
- access_count INTEGER DEFAULT 0,
191
- content_hash TEXT UNIQUE,
192
- parent_id INTEGER,
193
- tree_path TEXT,
194
- depth INTEGER DEFAULT 0,
195
- cluster_id INTEGER,
196
- tier INTEGER DEFAULT 1,
197
- FOREIGN KEY (parent_id) REFERENCES memories(id) ON DELETE CASCADE
198
- )
199
- ''')
200
- print(" ✓ Created memories table")
201
-
202
- # FTS5 index
203
- cursor.execute('''
204
- CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts
205
- USING fts5(content, summary, tags, content='memories', content_rowid='id')
206
- ''')
207
- print(" ✓ Created FTS index")
208
-
209
- # Layer 2: Tree
210
- cursor.execute('''
211
- CREATE TABLE IF NOT EXISTS memory_tree (
212
- id INTEGER PRIMARY KEY AUTOINCREMENT,
213
- node_type TEXT NOT NULL,
214
- name TEXT NOT NULL,
215
- description TEXT,
216
- parent_id INTEGER,
217
- tree_path TEXT NOT NULL,
218
- depth INTEGER DEFAULT 0,
219
- memory_count INTEGER DEFAULT 0,
220
- total_size INTEGER DEFAULT 0,
221
- last_updated TIMESTAMP,
222
- memory_id INTEGER,
223
- FOREIGN KEY (parent_id) REFERENCES memory_tree(id) ON DELETE CASCADE,
224
- FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
225
- )
226
- ''')
227
- print(" ✓ Created memory_tree table")
228
-
229
- # Layer 3: Graph
230
- cursor.execute('''
231
- CREATE TABLE IF NOT EXISTS graph_nodes (
232
- id INTEGER PRIMARY KEY AUTOINCREMENT,
233
- memory_id INTEGER UNIQUE NOT NULL,
234
- entities TEXT,
235
- embedding_vector TEXT,
236
- FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
237
- )
238
- ''')
239
-
240
- cursor.execute('''
241
- CREATE TABLE IF NOT EXISTS graph_edges (
242
- id INTEGER PRIMARY KEY AUTOINCREMENT,
243
- source_memory_id INTEGER NOT NULL,
244
- target_memory_id INTEGER NOT NULL,
245
- relationship_type TEXT,
246
- weight REAL DEFAULT 1.0,
247
- shared_entities TEXT,
248
- similarity_score REAL,
249
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
250
- FOREIGN KEY (source_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
251
- FOREIGN KEY (target_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
252
- UNIQUE(source_memory_id, target_memory_id)
253
- )
254
- ''')
255
-
256
- cursor.execute('''
257
- CREATE TABLE IF NOT EXISTS graph_clusters (
258
- id INTEGER PRIMARY KEY AUTOINCREMENT,
259
- name TEXT NOT NULL,
260
- description TEXT,
261
- member_count INTEGER DEFAULT 0,
262
- avg_importance REAL,
263
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
264
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
265
- )
266
- ''')
267
- print(" ✓ Created graph tables")
268
-
269
- # Layer 4: Patterns
270
- cursor.execute('''
271
- CREATE TABLE IF NOT EXISTS identity_patterns (
272
- id INTEGER PRIMARY KEY AUTOINCREMENT,
273
- pattern_type TEXT NOT NULL,
274
- key TEXT NOT NULL,
275
- value TEXT NOT NULL,
276
- confidence REAL DEFAULT 0.5,
277
- evidence_count INTEGER DEFAULT 1,
278
- memory_ids TEXT,
279
- category TEXT,
280
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
281
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
282
- UNIQUE(pattern_type, key, category)
283
- )
284
- ''')
285
-
286
- cursor.execute('''
287
- CREATE TABLE IF NOT EXISTS pattern_examples (
288
- id INTEGER PRIMARY KEY AUTOINCREMENT,
289
- pattern_id INTEGER NOT NULL,
290
- memory_id INTEGER NOT NULL,
291
- example_text TEXT,
292
- FOREIGN KEY (pattern_id) REFERENCES identity_patterns(id) ON DELETE CASCADE,
293
- FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
294
- )
295
- ''')
296
- print(" ✓ Created pattern tables")
297
-
298
- # Archive table
299
- cursor.execute('''
300
- CREATE TABLE IF NOT EXISTS memory_archive (
301
- id INTEGER PRIMARY KEY AUTOINCREMENT,
302
- memory_id INTEGER UNIQUE NOT NULL,
303
- full_content TEXT NOT NULL,
304
- archived_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
305
- FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
306
- )
307
- ''')
308
- print(" ✓ Created archive table")
309
-
310
- # Sessions table
311
- cursor.execute('''
312
- CREATE TABLE IF NOT EXISTS sessions (
313
- id INTEGER PRIMARY KEY AUTOINCREMENT,
314
- session_id TEXT UNIQUE,
315
- project_path TEXT,
316
- started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
317
- ended_at TIMESTAMP,
318
- summary TEXT
319
- )
320
- ''')
321
- print(" ✓ Created sessions table")
322
-
323
- # System metadata
324
- cursor.execute('''
325
- CREATE TABLE IF NOT EXISTS system_metadata (
326
- key TEXT PRIMARY KEY,
327
- value TEXT,
328
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
329
- )
330
- ''')
331
-
332
- cursor.execute('''
333
- INSERT OR REPLACE INTO system_metadata (key, value)
334
- VALUES ('schema_version', '2.0.0')
335
- ''')
336
- print(" ✓ Created metadata table")
337
-
338
- # Create indexes
339
- indexes = [
340
- 'CREATE INDEX IF NOT EXISTS idx_project ON memories(project_path)',
341
- 'CREATE INDEX IF NOT EXISTS idx_category ON memories(category)',
342
- 'CREATE INDEX IF NOT EXISTS idx_cluster ON memories(cluster_id)',
343
- 'CREATE INDEX IF NOT EXISTS idx_tree_path ON memories(tree_path)',
344
- 'CREATE INDEX IF NOT EXISTS idx_tier ON memories(tier)',
345
- 'CREATE INDEX IF NOT EXISTS idx_graph_source ON graph_edges(source_memory_id)',
346
- 'CREATE INDEX IF NOT EXISTS idx_graph_target ON graph_edges(target_memory_id)',
347
- ]
348
-
349
- for idx_sql in indexes:
350
- cursor.execute(idx_sql)
351
- print(" ✓ Created indexes")
352
-
353
- conn.commit()
354
- conn.close()
173
+ try:
174
+ cursor = conn.cursor()
175
+
176
+ # Layer 1: Memories table
177
+ cursor.execute('''
178
+ CREATE TABLE IF NOT EXISTS memories (
179
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
180
+ content TEXT NOT NULL,
181
+ summary TEXT,
182
+ project_path TEXT,
183
+ project_name TEXT,
184
+ tags TEXT,
185
+ category TEXT,
186
+ memory_type TEXT DEFAULT 'session',
187
+ importance INTEGER DEFAULT 5,
188
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
189
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
190
+ last_accessed TIMESTAMP,
191
+ access_count INTEGER DEFAULT 0,
192
+ content_hash TEXT UNIQUE,
193
+ parent_id INTEGER,
194
+ tree_path TEXT,
195
+ depth INTEGER DEFAULT 0,
196
+ cluster_id INTEGER,
197
+ tier INTEGER DEFAULT 1,
198
+ FOREIGN KEY (parent_id) REFERENCES memories(id) ON DELETE CASCADE
199
+ )
200
+ ''')
201
+ print(" ✓ Created memories table")
202
+
203
+ # FTS5 index
204
+ cursor.execute('''
205
+ CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts
206
+ USING fts5(content, summary, tags, content='memories', content_rowid='id')
207
+ ''')
208
+ print(" ✓ Created FTS index")
209
+
210
+ # Layer 2: Tree
211
+ cursor.execute('''
212
+ CREATE TABLE IF NOT EXISTS memory_tree (
213
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
214
+ node_type TEXT NOT NULL,
215
+ name TEXT NOT NULL,
216
+ description TEXT,
217
+ parent_id INTEGER,
218
+ tree_path TEXT NOT NULL,
219
+ depth INTEGER DEFAULT 0,
220
+ memory_count INTEGER DEFAULT 0,
221
+ total_size INTEGER DEFAULT 0,
222
+ last_updated TIMESTAMP,
223
+ memory_id INTEGER,
224
+ FOREIGN KEY (parent_id) REFERENCES memory_tree(id) ON DELETE CASCADE,
225
+ FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
226
+ )
227
+ ''')
228
+ print(" ✓ Created memory_tree table")
229
+
230
+ # Layer 3: Graph
231
+ cursor.execute('''
232
+ CREATE TABLE IF NOT EXISTS graph_nodes (
233
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
234
+ memory_id INTEGER UNIQUE NOT NULL,
235
+ entities TEXT,
236
+ embedding_vector TEXT,
237
+ FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
238
+ )
239
+ ''')
240
+
241
+ cursor.execute('''
242
+ CREATE TABLE IF NOT EXISTS graph_edges (
243
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
244
+ source_memory_id INTEGER NOT NULL,
245
+ target_memory_id INTEGER NOT NULL,
246
+ relationship_type TEXT,
247
+ weight REAL DEFAULT 1.0,
248
+ shared_entities TEXT,
249
+ similarity_score REAL,
250
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
251
+ FOREIGN KEY (source_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
252
+ FOREIGN KEY (target_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
253
+ UNIQUE(source_memory_id, target_memory_id)
254
+ )
255
+ ''')
256
+
257
+ cursor.execute('''
258
+ CREATE TABLE IF NOT EXISTS graph_clusters (
259
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
260
+ name TEXT NOT NULL,
261
+ description TEXT,
262
+ member_count INTEGER DEFAULT 0,
263
+ avg_importance REAL,
264
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
265
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
266
+ )
267
+ ''')
268
+ print(" ✓ Created graph tables")
269
+
270
+ # Layer 4: Patterns
271
+ cursor.execute('''
272
+ CREATE TABLE IF NOT EXISTS identity_patterns (
273
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
274
+ pattern_type TEXT NOT NULL,
275
+ key TEXT NOT NULL,
276
+ value TEXT NOT NULL,
277
+ confidence REAL DEFAULT 0.5,
278
+ evidence_count INTEGER DEFAULT 1,
279
+ memory_ids TEXT,
280
+ category TEXT,
281
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
282
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
283
+ UNIQUE(pattern_type, key, category)
284
+ )
285
+ ''')
286
+
287
+ cursor.execute('''
288
+ CREATE TABLE IF NOT EXISTS pattern_examples (
289
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
290
+ pattern_id INTEGER NOT NULL,
291
+ memory_id INTEGER NOT NULL,
292
+ example_text TEXT,
293
+ FOREIGN KEY (pattern_id) REFERENCES identity_patterns(id) ON DELETE CASCADE,
294
+ FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
295
+ )
296
+ ''')
297
+ print(" ✓ Created pattern tables")
298
+
299
+ # Archive table
300
+ cursor.execute('''
301
+ CREATE TABLE IF NOT EXISTS memory_archive (
302
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
303
+ memory_id INTEGER UNIQUE NOT NULL,
304
+ full_content TEXT NOT NULL,
305
+ archived_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
306
+ FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
307
+ )
308
+ ''')
309
+ print(" ✓ Created archive table")
310
+
311
+ # Sessions table
312
+ cursor.execute('''
313
+ CREATE TABLE IF NOT EXISTS sessions (
314
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
315
+ session_id TEXT UNIQUE,
316
+ project_path TEXT,
317
+ started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
318
+ ended_at TIMESTAMP,
319
+ summary TEXT
320
+ )
321
+ ''')
322
+ print(" ✓ Created sessions table")
323
+
324
+ # System metadata
325
+ cursor.execute('''
326
+ CREATE TABLE IF NOT EXISTS system_metadata (
327
+ key TEXT PRIMARY KEY,
328
+ value TEXT,
329
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
330
+ )
331
+ ''')
332
+
333
+ cursor.execute('''
334
+ INSERT OR REPLACE INTO system_metadata (key, value)
335
+ VALUES ('schema_version', '2.0.0')
336
+ ''')
337
+ print(" ✓ Created metadata table")
338
+
339
+ # Create indexes
340
+ indexes = [
341
+ 'CREATE INDEX IF NOT EXISTS idx_project ON memories(project_path)',
342
+ 'CREATE INDEX IF NOT EXISTS idx_category ON memories(category)',
343
+ 'CREATE INDEX IF NOT EXISTS idx_cluster ON memories(cluster_id)',
344
+ 'CREATE INDEX IF NOT EXISTS idx_tree_path ON memories(tree_path)',
345
+ 'CREATE INDEX IF NOT EXISTS idx_tier ON memories(tier)',
346
+ 'CREATE INDEX IF NOT EXISTS idx_graph_source ON graph_edges(source_memory_id)',
347
+ 'CREATE INDEX IF NOT EXISTS idx_graph_target ON graph_edges(target_memory_id)',
348
+ ]
349
+
350
+ for idx_sql in indexes:
351
+ cursor.execute(idx_sql)
352
+ print(" ✓ Created indexes")
353
+
354
+ conn.commit()
355
+ finally:
356
+ conn.close()
355
357
 
356
358
  def show_stats(self):
357
359
  """Show current database statistics."""
@@ -45,7 +45,7 @@ class CompressionConfig:
45
45
  return json.load(f)
46
46
  return {}
47
47
 
48
- def save(self):
48
+ def save(self) -> None:
49
49
  """Save configuration back to config.json."""
50
50
  with open(CONFIG_PATH, 'w') as f:
51
51
  json.dump(self.config, f, indent=2)
@@ -74,7 +74,7 @@ class CompressionConfig:
74
74
  def preserve_recently_accessed(self) -> bool:
75
75
  return self.compression_settings.get('preserve_recently_accessed', True)
76
76
 
77
- def initialize_defaults(self):
77
+ def initialize_defaults(self) -> None:
78
78
  """Initialize compression settings in config if not present."""
79
79
  if 'compression' not in self.config:
80
80
  self.config['compression'] = {