superlocalmemory 3.7.7 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +85 -0
- package/README.md +199 -29
- package/package.json +4 -2
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/agents/slm-governance-advisor.md +80 -0
- package/plugin-src/agents/slm-loop-runner.md +71 -0
- package/plugin-src/agents/slm-memory-advisor.md +10 -5
- package/plugin-src/agents/slm-optimize-advisor.md +9 -3
- package/plugin-src/commands/slm-loop.md +31 -0
- package/plugin-src/hooks/hooks.json +79 -0
- package/plugin-src/manifest.json +7 -2
- package/plugin-src/requirements.txt +1 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/rules/CLAUDE.md.fragment +8 -8
- package/plugin-src/scripts/slm-launch +46 -7
- package/plugin-src/settings.json +9 -0
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +1 -2
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +96 -12
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/loop_cmd.py +192 -0
- package/src/superlocalmemory/cli/main.py +39 -4
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +5 -43
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +17 -2
- package/src/superlocalmemory/core/embeddings.py +12 -1
- package/src/superlocalmemory/core/engine.py +17 -1
- package/src/superlocalmemory/core/engine_ingestion.py +29 -0
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +2 -0
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +35 -11
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +114 -1
- package/src/superlocalmemory/infra/auth_middleware.py +28 -0
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +243 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +21 -49
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +50 -5
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +168 -26
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +21 -4
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +34 -10
- package/src/superlocalmemory/server/routes/behavioral.py +13 -12
- package/src/superlocalmemory/server/routes/brain.py +21 -5
- package/src/superlocalmemory/server/routes/chat.py +72 -16
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +436 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +9 -4
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +135 -17
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +14 -14
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +221 -49
- package/src/superlocalmemory/server/routes/mesh.py +95 -15
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +124 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +13 -6
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/v3_api.py +194 -81
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +30 -5
- package/src/superlocalmemory/server/unified_daemon.py +431 -75
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +53 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +87 -0
- package/src/superlocalmemory/storage/schema_v32.py +0 -9
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +306 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +47 -31
- package/src/superlocalmemory/ui/js/dashboard.js +314 -63
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +780 -0
- package/src/superlocalmemory/ui/js/od-brain.js +779 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +593 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +887 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +539 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1053 -0
- package/src/superlocalmemory/ui/js/od-shell.js +593 -0
- package/src/superlocalmemory/ui/js/od-skills.js +573 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +2 -2
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 — PII redaction on ingest (C4)
|
|
4
|
+
|
|
5
|
+
"""Opt-in PII redaction for ingested memory content.
|
|
6
|
+
|
|
7
|
+
For team / company deployments an operator may need memory to never persist
|
|
8
|
+
personal identifiers (email, phone, national ID, payment card, IP). This
|
|
9
|
+
module provides a pure, well-bounded scrubber that replaces detected PII with
|
|
10
|
+
``[PII:TYPE]`` markers. It is complementary to ``security_primitives.
|
|
11
|
+
redact_secrets`` (which handles API keys / tokens and always runs).
|
|
12
|
+
|
|
13
|
+
Design goals:
|
|
14
|
+
* **Low false-positive rate.** Card numbers are Luhn-validated; SSNs use the
|
|
15
|
+
canonical grouping; phone matching requires a plausible separator shape.
|
|
16
|
+
* **Deterministic + pure.** No I/O, no config — the caller decides when to run
|
|
17
|
+
it (gated by SLM_PII_REDACTION / config), so it is trivially testable.
|
|
18
|
+
* **Order matters.** Emails are redacted before phone/number sweeps so an
|
|
19
|
+
email's local part is never mistaken for a number.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
# Order-sensitive: earlier patterns win over later ones on overlapping spans.
|
|
27
|
+
_EMAIL = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")
|
|
28
|
+
_SSN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
|
|
29
|
+
_IPV4 = re.compile(
|
|
30
|
+
r"\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b"
|
|
31
|
+
)
|
|
32
|
+
# Phone: conservative to avoid eating ISO dates (4-2-2) / version strings.
|
|
33
|
+
# Only unambiguous shapes match:
|
|
34
|
+
# * international +CC then grouped digits
|
|
35
|
+
# * parenthesized (415) 555-0132
|
|
36
|
+
# * strict US 415-555-0132 / 415.555.0132 (3-3-4, dot/dash only — a
|
|
37
|
+
# space separator is excluded so "2026-07-22 12" style runs never match).
|
|
38
|
+
_PHONE = re.compile(
|
|
39
|
+
r"(?<!\w)(?:"
|
|
40
|
+
r"\+\d{1,3}[\s.\-]?\d{1,4}[\s.\-]?\d{2,4}[\s.\-]?\d{2,4}"
|
|
41
|
+
r"|\(\d{3}\)[\s.\-]?\d{3}[\s.\-]?\d{4}"
|
|
42
|
+
r"|\d{3}[.\-]\d{3}[.\-]\d{4}"
|
|
43
|
+
r")(?!\w)"
|
|
44
|
+
)
|
|
45
|
+
# Candidate card: 13–19 digits, optionally grouped by space/dash. Luhn-checked.
|
|
46
|
+
_CARD_CANDIDATE = re.compile(r"(?<!\w)(?:\d[ -]?){13,19}(?!\w)")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _luhn_ok(digits: str) -> bool:
|
|
50
|
+
"""Return True if ``digits`` (0-9 only) passes the Luhn checksum."""
|
|
51
|
+
if not (13 <= len(digits) <= 19):
|
|
52
|
+
return False
|
|
53
|
+
total = 0
|
|
54
|
+
parity = len(digits) % 2
|
|
55
|
+
for i, ch in enumerate(digits):
|
|
56
|
+
d = ord(ch) - 48
|
|
57
|
+
if i % 2 == parity:
|
|
58
|
+
d *= 2
|
|
59
|
+
if d > 9:
|
|
60
|
+
d -= 9
|
|
61
|
+
total += d
|
|
62
|
+
return total % 10 == 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _redact_cards(text: str, counter: list[int]) -> str:
|
|
66
|
+
def _sub(m: re.Match[str]) -> str:
|
|
67
|
+
raw = m.group(0)
|
|
68
|
+
digits = re.sub(r"\D", "", raw)
|
|
69
|
+
if _luhn_ok(digits):
|
|
70
|
+
counter[0] += 1
|
|
71
|
+
return "[PII:CARD]"
|
|
72
|
+
return raw
|
|
73
|
+
return _CARD_CANDIDATE.sub(_sub, text)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def redact_pii(text: str) -> tuple[str, int]:
|
|
77
|
+
"""Return ``(redacted_text, num_redactions)``.
|
|
78
|
+
|
|
79
|
+
Never raises; a non-string or empty input is returned unchanged with 0.
|
|
80
|
+
"""
|
|
81
|
+
if not isinstance(text, str) or not text:
|
|
82
|
+
return text, 0
|
|
83
|
+
|
|
84
|
+
counter = [0]
|
|
85
|
+
|
|
86
|
+
def _count_sub(pattern: re.Pattern[str], label: str, s: str) -> str:
|
|
87
|
+
def _sub(_m: re.Match[str]) -> str:
|
|
88
|
+
counter[0] += 1
|
|
89
|
+
return label
|
|
90
|
+
return pattern.sub(_sub, s)
|
|
91
|
+
|
|
92
|
+
out = text
|
|
93
|
+
# Email first (protects local parts from the number sweeps).
|
|
94
|
+
out = _count_sub(_EMAIL, "[PII:EMAIL]", out)
|
|
95
|
+
# Payment cards before generic phone/number matching (Luhn-gated).
|
|
96
|
+
out = _redact_cards(out, counter)
|
|
97
|
+
out = _count_sub(_SSN, "[PII:SSN]", out)
|
|
98
|
+
out = _count_sub(_IPV4, "[PII:IP]", out)
|
|
99
|
+
out = _count_sub(_PHONE, "[PII:PHONE]", out)
|
|
100
|
+
return out, counter[0]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def redact_pii_text(text: str) -> str:
|
|
104
|
+
"""Convenience wrapper returning only the redacted string."""
|
|
105
|
+
return redact_pii(text)[0]
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Progressive abstraction (Wave Q3) — the top persona tier + drill-down.
|
|
6
|
+
|
|
7
|
+
Completes the abstraction hierarchy on the ONE principled backbone (rather
|
|
8
|
+
than a fourth divergent clustering):
|
|
9
|
+
|
|
10
|
+
atoms (atomic_facts)
|
|
11
|
+
-> entity communities (Wave Q backbone)
|
|
12
|
+
-> community summaries (Wave Q2)
|
|
13
|
+
-> persona roll-up (this module)
|
|
14
|
+
|
|
15
|
+
The persona is one bounded roll-up per profile that consumes the top community
|
|
16
|
+
summaries. It is recall-GATED (never auto-injected into the hot recall path —
|
|
17
|
+
avoids the V3.4.40 summary-pollution regression) and SIZE-bounded. Drill-down
|
|
18
|
+
(``get_sources``) walks the hierarchy back down to the source atoms, matching
|
|
19
|
+
the market bar for summary->source provenance (Zep-style).
|
|
20
|
+
|
|
21
|
+
Runs in the background consolidation lane after community summaries.
|
|
22
|
+
Fail-open throughout; recompute replaces a profile's row.
|
|
23
|
+
|
|
24
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
25
|
+
License: AGPL-3.0-or-later
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import logging
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from superlocalmemory.core.community_summary import CommunitySummaryBuilder
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
_PERSONA_MAX_CHARS = 2048
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ProgressiveAbstraction:
|
|
42
|
+
"""Build + persist the persona tier; provide hierarchy drill-down."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
db: Any,
|
|
47
|
+
summarizer: Any = None,
|
|
48
|
+
max_communities_in_persona: int = 8,
|
|
49
|
+
persona_max_chars: int = _PERSONA_MAX_CHARS,
|
|
50
|
+
max_keywords: int = 12,
|
|
51
|
+
) -> None:
|
|
52
|
+
self._db = db
|
|
53
|
+
self._summarizer = summarizer
|
|
54
|
+
self._max_communities = max(1, int(max_communities_in_persona))
|
|
55
|
+
self._persona_max_chars = max(256, int(persona_max_chars))
|
|
56
|
+
self._max_keywords = max(1, int(max_keywords))
|
|
57
|
+
|
|
58
|
+
# ------------------------------------------------------------------
|
|
59
|
+
# Build
|
|
60
|
+
# ------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
def compute_and_store(self, profile_id: str) -> dict[str, Any]:
|
|
63
|
+
summaries = CommunitySummaryBuilder(self._db).get_summaries(profile_id)
|
|
64
|
+
try:
|
|
65
|
+
self._db.execute(
|
|
66
|
+
"DELETE FROM persona_summary WHERE profile_id = ?",
|
|
67
|
+
(profile_id,),
|
|
68
|
+
)
|
|
69
|
+
except Exception as exc:
|
|
70
|
+
logger.debug("persona_summary clear failed: %s", exc)
|
|
71
|
+
if not summaries:
|
|
72
|
+
return {"built": False, "communities_in_persona": 0}
|
|
73
|
+
|
|
74
|
+
top = summaries[: self._max_communities]
|
|
75
|
+
summary = self._persona_summary(top)
|
|
76
|
+
keywords = self._merge_keywords(top)
|
|
77
|
+
community_ids = [int(s["community_id"]) for s in top]
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
self._db.execute(
|
|
81
|
+
"INSERT OR REPLACE INTO persona_summary "
|
|
82
|
+
"(profile_id, summary, keywords, community_ids_json, computed_at) "
|
|
83
|
+
"VALUES (?, ?, ?, ?, datetime('now'))",
|
|
84
|
+
(profile_id, summary, keywords, json.dumps(community_ids)),
|
|
85
|
+
)
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
logger.debug("persona_summary write failed: %s", exc)
|
|
88
|
+
return {"built": False, "communities_in_persona": 0}
|
|
89
|
+
|
|
90
|
+
return {"built": True, "communities_in_persona": len(top)}
|
|
91
|
+
|
|
92
|
+
# ------------------------------------------------------------------
|
|
93
|
+
# Read API
|
|
94
|
+
# ------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
def get_persona(self, profile_id: str) -> dict | None:
|
|
97
|
+
try:
|
|
98
|
+
rows = self._db.execute(
|
|
99
|
+
"SELECT * FROM persona_summary WHERE profile_id = ?",
|
|
100
|
+
(profile_id,),
|
|
101
|
+
)
|
|
102
|
+
except Exception as exc:
|
|
103
|
+
logger.debug("get_persona failed: %s", exc)
|
|
104
|
+
return None
|
|
105
|
+
if not rows:
|
|
106
|
+
return None
|
|
107
|
+
d = dict(rows[0])
|
|
108
|
+
try:
|
|
109
|
+
community_ids = json.loads(d.get("community_ids_json") or "[]")
|
|
110
|
+
except (ValueError, TypeError):
|
|
111
|
+
community_ids = []
|
|
112
|
+
return {
|
|
113
|
+
"profile_id": d.get("profile_id", profile_id),
|
|
114
|
+
"summary": d.get("summary", ""),
|
|
115
|
+
"keywords": d.get("keywords", ""),
|
|
116
|
+
"community_ids": community_ids,
|
|
117
|
+
"computed_at": d.get("computed_at", ""),
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
def get_sources(self, profile_id: str, node_id: Any) -> dict:
|
|
121
|
+
"""Drill-down: a tier node -> its child communities + source atoms.
|
|
122
|
+
|
|
123
|
+
node_id == "persona" -> the persona's member communities + their facts.
|
|
124
|
+
node_id == <community> -> that community's member facts.
|
|
125
|
+
Unknown node -> empty (never raises).
|
|
126
|
+
"""
|
|
127
|
+
result: dict[str, Any] = {
|
|
128
|
+
"node_id": node_id, "node_type": "unknown",
|
|
129
|
+
"communities": [], "fact_ids": [],
|
|
130
|
+
}
|
|
131
|
+
try:
|
|
132
|
+
if isinstance(node_id, str) and node_id.lower() == "persona":
|
|
133
|
+
persona = self.get_persona(profile_id)
|
|
134
|
+
cids = persona["community_ids"] if persona else []
|
|
135
|
+
fact_ids: list[str] = []
|
|
136
|
+
seen: set[str] = set()
|
|
137
|
+
for cid in cids:
|
|
138
|
+
for fid in self._community_fact_ids(profile_id, cid):
|
|
139
|
+
if fid not in seen:
|
|
140
|
+
seen.add(fid)
|
|
141
|
+
fact_ids.append(fid)
|
|
142
|
+
result.update(
|
|
143
|
+
node_type="persona", communities=list(cids), fact_ids=fact_ids,
|
|
144
|
+
)
|
|
145
|
+
return result
|
|
146
|
+
|
|
147
|
+
# Otherwise treat node_id as a community id.
|
|
148
|
+
cid = int(node_id)
|
|
149
|
+
fids = self._community_fact_ids(profile_id, cid)
|
|
150
|
+
result.update(
|
|
151
|
+
node_type="community", communities=[cid], fact_ids=fids,
|
|
152
|
+
)
|
|
153
|
+
return result
|
|
154
|
+
except (ValueError, TypeError):
|
|
155
|
+
return result
|
|
156
|
+
except Exception as exc:
|
|
157
|
+
logger.debug("get_sources failed: %s", exc)
|
|
158
|
+
return result
|
|
159
|
+
|
|
160
|
+
# ------------------------------------------------------------------
|
|
161
|
+
# Internal
|
|
162
|
+
# ------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
def _community_fact_ids(self, profile_id: str, community_id: Any) -> list[str]:
|
|
165
|
+
try:
|
|
166
|
+
rows = self._db.execute(
|
|
167
|
+
"SELECT fact_ids_json FROM community_summaries "
|
|
168
|
+
"WHERE profile_id = ? AND community_id = ?",
|
|
169
|
+
(profile_id, int(community_id)),
|
|
170
|
+
)
|
|
171
|
+
except Exception as exc:
|
|
172
|
+
logger.debug("_community_fact_ids failed: %s", exc)
|
|
173
|
+
return []
|
|
174
|
+
if not rows:
|
|
175
|
+
return []
|
|
176
|
+
try:
|
|
177
|
+
return [str(f) for f in json.loads(dict(rows[0]).get("fact_ids_json") or "[]")]
|
|
178
|
+
except (ValueError, TypeError):
|
|
179
|
+
return []
|
|
180
|
+
|
|
181
|
+
def _persona_summary(self, top: list[dict]) -> str:
|
|
182
|
+
if self._summarizer is not None:
|
|
183
|
+
try:
|
|
184
|
+
text = self._summarizer.summarize_cluster(
|
|
185
|
+
[{"content": s.get("summary", "")} for s in top],
|
|
186
|
+
)
|
|
187
|
+
if text and text.strip():
|
|
188
|
+
return text.strip()[: self._persona_max_chars]
|
|
189
|
+
except Exception as exc:
|
|
190
|
+
logger.debug("persona summarizer failed (fail-open): %s", exc)
|
|
191
|
+
# Mode A keyword-dense fallback: stitch the top community summaries.
|
|
192
|
+
heads = [s.get("summary", "").strip() for s in top if s.get("summary")]
|
|
193
|
+
base = " ".join(heads) if heads else "No persona yet."
|
|
194
|
+
return base[: self._persona_max_chars]
|
|
195
|
+
|
|
196
|
+
def _merge_keywords(self, top: list[dict]) -> str:
|
|
197
|
+
seen: set[str] = set()
|
|
198
|
+
merged: list[str] = []
|
|
199
|
+
for s in top:
|
|
200
|
+
for kw in (s.get("keywords", "") or "").split(","):
|
|
201
|
+
k = kw.strip()
|
|
202
|
+
low = k.lower()
|
|
203
|
+
if k and low not in seen:
|
|
204
|
+
seen.add(low)
|
|
205
|
+
merged.append(k)
|
|
206
|
+
if len(merged) >= self._max_keywords:
|
|
207
|
+
return ", ".join(merged)
|
|
208
|
+
return ", ".join(merged)
|
|
@@ -623,6 +623,7 @@ def run_recall(
|
|
|
623
623
|
fast: bool = False,
|
|
624
624
|
include_global: bool = False,
|
|
625
625
|
include_shared: bool = False,
|
|
626
|
+
window: str | tuple[str, str] | None = None,
|
|
626
627
|
) -> RecallResponse:
|
|
627
628
|
"""Recall relevant facts for a query.
|
|
628
629
|
|
|
@@ -663,6 +664,7 @@ def run_recall(
|
|
|
663
664
|
extra_disabled_channels=extra_disabled,
|
|
664
665
|
include_global=include_global,
|
|
665
666
|
include_shared=include_shared,
|
|
667
|
+
window=window,
|
|
666
668
|
)
|
|
667
669
|
_mark("retrieval(chan+rerank)")
|
|
668
670
|
|
|
@@ -62,6 +62,7 @@ def _get_engine():
|
|
|
62
62
|
def _handle_recall(
|
|
63
63
|
query: str, limit: int, session_id: str = "", fast: bool = False,
|
|
64
64
|
include_global: bool | None = None, include_shared: bool | None = None,
|
|
65
|
+
window: str | None = None,
|
|
65
66
|
) -> dict:
|
|
66
67
|
engine = _get_engine()
|
|
67
68
|
# v3.6.15 multi-scope: None flags let engine.recall resolve the configured
|
|
@@ -70,11 +71,19 @@ def _handle_recall(
|
|
|
70
71
|
response = engine.recall(
|
|
71
72
|
query, limit=limit, session_id=session_id or None, fast=bool(fast),
|
|
72
73
|
include_global=include_global, include_shared=include_shared,
|
|
74
|
+
window=window or None,
|
|
73
75
|
)
|
|
74
76
|
|
|
75
|
-
# Batch-fetch original memory text for all results
|
|
77
|
+
# Batch-fetch original memory text for all results. Retrieval already
|
|
78
|
+
# enforced scope, so resolve content for everything it returned (own +
|
|
79
|
+
# global + shared-with-me); another tenant's PRIVATE content still can't
|
|
80
|
+
# resolve. Using the raw (possibly-None) recall flags here would drop
|
|
81
|
+
# content for legitimately-recalled global/shared memories.
|
|
76
82
|
memory_ids = list({r.fact.memory_id for r in response.results[:limit] if r.fact.memory_id})
|
|
77
|
-
memory_map = engine._db.get_memory_content_batch(
|
|
83
|
+
memory_map = engine._db.get_memory_content_batch(
|
|
84
|
+
memory_ids, engine.profile_id,
|
|
85
|
+
include_global=True, include_shared=True,
|
|
86
|
+
) if memory_ids else {}
|
|
78
87
|
|
|
79
88
|
# v3.6.6: same shared chokepoint as the daemon HTTP route + CLI fallback,
|
|
80
89
|
# so the MCP WorkerPool subprocess path returns identical budgeted output.
|
|
@@ -172,11 +181,15 @@ def _handle_store(content: str, metadata: dict) -> dict:
|
|
|
172
181
|
def _handle_get_memory_facts(memory_id: str) -> dict:
|
|
173
182
|
engine = _get_engine()
|
|
174
183
|
pid = engine.profile_id
|
|
175
|
-
# Get original memory content
|
|
176
|
-
mem_map = engine._db.get_memory_content_batch(
|
|
184
|
+
# Get original memory content (C4: tenant-scoped; global/shared resolvable)
|
|
185
|
+
mem_map = engine._db.get_memory_content_batch(
|
|
186
|
+
[memory_id], pid, include_global=True, include_shared=True,
|
|
187
|
+
)
|
|
177
188
|
original = mem_map.get(memory_id, "")
|
|
178
|
-
# Get child facts
|
|
179
|
-
facts
|
|
189
|
+
# Get child facts — same scope as the content fetch so a shared/global
|
|
190
|
+
# memory's facts are not silently empty.
|
|
191
|
+
facts = engine._db.get_facts_by_memory_id(
|
|
192
|
+
memory_id, pid, include_global=True, include_shared=True)
|
|
180
193
|
fact_list = []
|
|
181
194
|
for f in facts:
|
|
182
195
|
fact_list.append({
|
|
@@ -308,6 +321,7 @@ def _worker_main() -> None:
|
|
|
308
321
|
req.get("session_id", ""), bool(req.get("fast", False)),
|
|
309
322
|
include_global=req.get("include_global"),
|
|
310
323
|
include_shared=req.get("include_shared"),
|
|
324
|
+
window=req.get("window"),
|
|
311
325
|
)
|
|
312
326
|
_respond(result)
|
|
313
327
|
elif cmd == "store":
|
|
@@ -116,6 +116,13 @@ class ScaleEngineManager:
|
|
|
116
116
|
# This command reads persisted state, not the live daemon. Never
|
|
117
117
|
# turn a last-known backend row into a present-tense routing claim.
|
|
118
118
|
"active": {"cozo": False, "lance": False},
|
|
119
|
+
# LOW-2 (3.7.9): after promote the daemon must restart before the
|
|
120
|
+
# backends actually serve; surface that explicitly so `status` isn't
|
|
121
|
+
# mistaken for "promotion failed".
|
|
122
|
+
"daemon_must_restart_to_activate": (
|
|
123
|
+
state == "promoted"
|
|
124
|
+
and not any(v == "active" for v in runtime.values())
|
|
125
|
+
),
|
|
119
126
|
"last_daemon_observation": runtime,
|
|
120
127
|
"paths_present": paths_present,
|
|
121
128
|
"retrieval_routing": (
|
|
@@ -319,6 +326,8 @@ class ScaleEngineManager:
|
|
|
319
326
|
raise ScaleEngineError(
|
|
320
327
|
f"projection parity failed: canonical={canonical}, observed={observed}"
|
|
321
328
|
)
|
|
329
|
+
with self._readonly_connection() as conn:
|
|
330
|
+
self._verify_content_sample(conn, cozo, canonical)
|
|
322
331
|
manifest.update({"state": "verified", "verified_at": _utc_now(), "observed": observed})
|
|
323
332
|
self._write_manifest(stage_dir, manifest)
|
|
324
333
|
self.config.scale_engine_state = "verified"
|
|
@@ -510,7 +519,31 @@ class ScaleEngineManager:
|
|
|
510
519
|
).fetchone()[0]
|
|
511
520
|
edges = count_logical_edges(conn, self.profile_id)
|
|
512
521
|
vectors = count_canonical_vectors(conn, self.profile_id)
|
|
513
|
-
|
|
522
|
+
fact_entity = self._count_fact_entity_links(conn)
|
|
523
|
+
return {
|
|
524
|
+
"entities": int(nodes),
|
|
525
|
+
"edges": int(edges),
|
|
526
|
+
"vectors": int(vectors),
|
|
527
|
+
"fact_entity": int(fact_entity),
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
def _count_fact_entity_links(self, conn: sqlite3.Connection) -> int:
|
|
531
|
+
"""Count fact->entity bridge rows exactly as ``bulk_import_from_sqlite``
|
|
532
|
+
projects them: dedup entity IDs per fact, drop empties. This bridge is
|
|
533
|
+
what lets Cozo map a query seed into the fact graph; if its import
|
|
534
|
+
silently fails, count parity on entities/edges/vectors still passes but
|
|
535
|
+
entity recall returns empty — so it must be verified explicitly."""
|
|
536
|
+
total = 0
|
|
537
|
+
for (raw,) in conn.execute(
|
|
538
|
+
"SELECT canonical_entities_json FROM atomic_facts WHERE profile_id=?",
|
|
539
|
+
(self.profile_id,),
|
|
540
|
+
):
|
|
541
|
+
try:
|
|
542
|
+
entity_ids = json.loads(raw or "[]")
|
|
543
|
+
except (TypeError, ValueError, json.JSONDecodeError):
|
|
544
|
+
continue
|
|
545
|
+
total += sum(1 for eid in dict.fromkeys(entity_ids) if eid)
|
|
546
|
+
return total
|
|
514
547
|
|
|
515
548
|
def _observed_counts(self, cozo: Any, lance: Any) -> dict[str, int]:
|
|
516
549
|
graph = cozo.health_check()
|
|
@@ -521,8 +554,34 @@ class ScaleEngineManager:
|
|
|
521
554
|
"entities": int(graph["entities"]),
|
|
522
555
|
"edges": int(graph["edges"]),
|
|
523
556
|
"vectors": int(vector["vectors"]),
|
|
557
|
+
"fact_entity": int(graph.get("fact_entity", 0)),
|
|
524
558
|
}
|
|
525
559
|
|
|
560
|
+
def _verify_content_sample(
|
|
561
|
+
self, conn: sqlite3.Connection, cozo: Any, canonical: dict[str, int]
|
|
562
|
+
) -> None:
|
|
563
|
+
"""Content parity: count equality does not prove the import reproduced
|
|
564
|
+
the correct rows. Compare the projected entity-ID set against canonical
|
|
565
|
+
SQLite (bounded), catching a projection that has the right entity count
|
|
566
|
+
but the wrong identities."""
|
|
567
|
+
getter = getattr(cozo, "entity_ids", None)
|
|
568
|
+
n = int(canonical.get("entities", 0))
|
|
569
|
+
if not callable(getter) or n == 0:
|
|
570
|
+
return # backend cannot sample, or nothing to compare
|
|
571
|
+
observed_ids = set(getter(limit=n))
|
|
572
|
+
rows = conn.execute(
|
|
573
|
+
"SELECT entity_id FROM canonical_entities WHERE profile_id=?",
|
|
574
|
+
(self.profile_id,),
|
|
575
|
+
).fetchall()
|
|
576
|
+
expected_ids = {r[0] for r in rows}
|
|
577
|
+
if expected_ids != observed_ids:
|
|
578
|
+
missing = sorted(expected_ids - observed_ids)[:5]
|
|
579
|
+
extra = sorted(observed_ids - expected_ids)[:5]
|
|
580
|
+
raise ScaleEngineError(
|
|
581
|
+
"projection content parity failed: entity IDs diverged "
|
|
582
|
+
f"(missing sample={missing}, unexpected sample={extra})"
|
|
583
|
+
)
|
|
584
|
+
|
|
526
585
|
def _projection_fingerprint(
|
|
527
586
|
self, conn: sqlite3.Connection, counts: dict[str, int]
|
|
528
587
|
) -> str:
|
|
@@ -399,6 +399,35 @@ def redact_secrets(text: str, *, entropy_threshold: float = 4.5,
|
|
|
399
399
|
# ---------------------------------------------------------------------------
|
|
400
400
|
|
|
401
401
|
|
|
402
|
+
def harden_db_perms(db_path: str | Path) -> None:
|
|
403
|
+
"""Restrict a database file (and its WAL/SHM sidecars) to owner-only 0600
|
|
404
|
+
and its parent directory to 0700 (C4 encryption-at-rest defense-in-depth).
|
|
405
|
+
|
|
406
|
+
Best-effort and POSIX-only: on Windows or if the file does not exist yet
|
|
407
|
+
this is a silent no-op. The daemon's data dir is already 0700, but the DB
|
|
408
|
+
files themselves shipped 0644 (world-readable); on a shared host that is a
|
|
409
|
+
real exposure even with full-disk encryption at rest.
|
|
410
|
+
"""
|
|
411
|
+
if _is_windows():
|
|
412
|
+
return
|
|
413
|
+
try:
|
|
414
|
+
p = Path(db_path)
|
|
415
|
+
parent = p.parent
|
|
416
|
+
try:
|
|
417
|
+
os.chmod(parent, 0o700)
|
|
418
|
+
except OSError:
|
|
419
|
+
pass
|
|
420
|
+
for suffix in ("", "-wal", "-shm"):
|
|
421
|
+
f = Path(str(p) + suffix)
|
|
422
|
+
if f.exists():
|
|
423
|
+
try:
|
|
424
|
+
os.chmod(f, 0o600)
|
|
425
|
+
except OSError:
|
|
426
|
+
pass
|
|
427
|
+
except Exception: # pragma: no cover — never block DB open on a chmod
|
|
428
|
+
pass
|
|
429
|
+
|
|
430
|
+
|
|
402
431
|
def _install_token_path() -> Path: # pragma: no cover — monkeypatched in tests
|
|
403
432
|
"""Default install-token location — override in tests via monkeypatch."""
|
|
404
433
|
from superlocalmemory.infra.data_root import state_path
|
|
@@ -598,8 +627,17 @@ def run_subprocess_safe(
|
|
|
598
627
|
- Restricted environment by default — only a minimal set of safe keys.
|
|
599
628
|
- Callers may pass an explicit ``env`` to add specific variables.
|
|
600
629
|
|
|
601
|
-
This is the
|
|
602
|
-
|
|
630
|
+
This is the PREFERRED wrapper for any subprocess whose ``argv`` includes
|
|
631
|
+
dynamic or externally-influenced values — routing them through here keeps
|
|
632
|
+
``shell=False``, a mandatory timeout, and a restricted environment.
|
|
633
|
+
|
|
634
|
+
It is not the *only* ``subprocess.run`` call site: a small set of vetted
|
|
635
|
+
callers invoke ``subprocess.run`` directly where they need inherited stdio
|
|
636
|
+
for live progress, a long-lived managed process, or process-group control
|
|
637
|
+
(e.g. model downloads in ``cli/setup_wizard.py``, ``infra/process_reaper``,
|
|
638
|
+
``cli/service_installer``). Those pass only fixed/argv-quoted values — never
|
|
639
|
+
a shell string. Do not read this wrapper as a guarantee that no other
|
|
640
|
+
subprocess call exists; audit new call sites individually. (L-01, 3.7.9)
|
|
603
641
|
"""
|
|
604
642
|
if not isinstance(argv, list):
|
|
605
643
|
raise TypeError("argv must be list[str], shell=False only")
|
|
@@ -208,21 +208,33 @@ def run_store(
|
|
|
208
208
|
if not pre_authorized:
|
|
209
209
|
hooks.run_pre("store", hook_ctx)
|
|
210
210
|
|
|
211
|
-
|
|
211
|
+
# Admission gates apply to FRESH submissions only. A materialization pass
|
|
212
|
+
# re-runs this pipeline for content whose queryable projection was already
|
|
213
|
+
# committed at submit (``queryable_fact_ids`` is set). Re-applying admission
|
|
214
|
+
# here — the entropy near-duplicate gate especially — would discard that
|
|
215
|
+
# already-committed fact (return []) and wedge the operation in an endless
|
|
216
|
+
# materialize retry loop ("materialization produced no final facts"). The
|
|
217
|
+
# near-duplicate verdict is expected at materialize: the submitted
|
|
218
|
+
# projection itself is in the gate's window. Admission was already decided
|
|
219
|
+
# at submit (store_fast enforces the same gates), so skip it here.
|
|
220
|
+
is_materialization = bool(queryable_fact_ids)
|
|
221
|
+
|
|
222
|
+
if entropy_gate and not is_materialization and not entropy_gate.should_pass(content):
|
|
212
223
|
return []
|
|
213
224
|
|
|
214
225
|
# v3.5.0: store-side quality gate (H3). Reject prompt-template leakage,
|
|
215
226
|
# empty placeholders, and other non-memory content BEFORE it enters the
|
|
216
227
|
# DB. Uses the shared is_low_quality from core/injection so both store
|
|
217
228
|
# AND injection filter by identical rules. Saves DB IO + recall pollution.
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
229
|
+
if not is_materialization:
|
|
230
|
+
try:
|
|
231
|
+
from superlocalmemory.core.injection import is_low_quality
|
|
232
|
+
if is_low_quality(content):
|
|
233
|
+
logger.debug("Store rejected (low-quality content): %s...",
|
|
234
|
+
content[:80].replace("\n", " "))
|
|
235
|
+
return []
|
|
236
|
+
except Exception:
|
|
237
|
+
pass # Best-effort gate; store succeeds if import fails
|
|
226
238
|
|
|
227
239
|
from superlocalmemory.encoding.temporal_parser import TemporalParser
|
|
228
240
|
parser = temporal_parser or TemporalParser()
|
|
@@ -583,14 +595,26 @@ def run_store(
|
|
|
583
595
|
fact.fact_id, exc,
|
|
584
596
|
)
|
|
585
597
|
|
|
598
|
+
# Phase 4b: fact-augmented key expansion (T3b). Index entity aliases /
|
|
599
|
+
# canonical names as BM25 alt-keys so paraphrased queries match. Mode A
|
|
600
|
+
# (entity graph) is zero-LLM and safe on the hot store path; LLM
|
|
601
|
+
# paraphrase enrichment (Mode B/C) is left to background consolidation.
|
|
602
|
+
try:
|
|
603
|
+
from superlocalmemory.core.key_expander import KeyExpander
|
|
604
|
+
_alt_keys = KeyExpander(db).expand(fact, profile_id, mode="a")
|
|
605
|
+
if _alt_keys:
|
|
606
|
+
db.upsert_fact_expansion(fact.fact_id, _alt_keys)
|
|
607
|
+
except Exception as exc:
|
|
608
|
+
logger.debug("Key expansion skipped for %s: %s", fact.fact_id, exc)
|
|
609
|
+
|
|
586
610
|
if observation_builder:
|
|
587
611
|
for eid in fact.canonical_entities:
|
|
588
612
|
observation_builder.update_profile(eid, fact, profile_id)
|
|
589
613
|
|
|
590
|
-
# Increment fact_count for each linked canonical entity
|
|
614
|
+
# Increment fact_count for each linked canonical entity (scoped to profile).
|
|
591
615
|
for eid in fact.canonical_entities:
|
|
592
616
|
try:
|
|
593
|
-
db.increment_entity_fact_count(eid)
|
|
617
|
+
db.increment_entity_fact_count(eid, profile_id)
|
|
594
618
|
except Exception:
|
|
595
619
|
pass # Non-critical — entity may have been deleted
|
|
596
620
|
if scene_builder:
|
|
@@ -70,6 +70,7 @@ class WorkerPool:
|
|
|
70
70
|
fast: bool = False,
|
|
71
71
|
include_global: bool | None = None,
|
|
72
72
|
include_shared: bool | None = None,
|
|
73
|
+
window: str | None = None,
|
|
73
74
|
) -> dict:
|
|
74
75
|
"""Run recall in worker subprocess. Returns result dict.
|
|
75
76
|
|
|
@@ -91,6 +92,8 @@ class WorkerPool:
|
|
|
91
92
|
msg["include_global"] = bool(include_global)
|
|
92
93
|
if include_shared is not None:
|
|
93
94
|
msg["include_shared"] = bool(include_shared)
|
|
95
|
+
if window:
|
|
96
|
+
msg["window"] = window
|
|
94
97
|
return self._send(msg)
|
|
95
98
|
|
|
96
99
|
def store(self, content: str, metadata: dict | None = None) -> dict:
|
|
@@ -291,15 +294,27 @@ class WorkerPool:
|
|
|
291
294
|
self._idle_timer.cancel()
|
|
292
295
|
self._idle_timer = None
|
|
293
296
|
if self._proc is not None:
|
|
294
|
-
|
|
297
|
+
proc = self._proc
|
|
298
|
+
pid = proc.pid
|
|
295
299
|
try:
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
300
|
+
proc.stdin.write('{"cmd":"quit"}\n')
|
|
301
|
+
proc.stdin.flush()
|
|
302
|
+
proc.wait(timeout=3)
|
|
299
303
|
except Exception:
|
|
300
304
|
try:
|
|
301
|
-
|
|
302
|
-
|
|
305
|
+
proc.kill()
|
|
306
|
+
proc.wait(timeout=2)
|
|
307
|
+
except Exception:
|
|
308
|
+
pass
|
|
309
|
+
# L-CONC-1: deterministically close the pipe fds rather than leaving
|
|
310
|
+
# them to GC. This releases the OS handles immediately and unblocks
|
|
311
|
+
# any orphaned _readline_with_timeout reader thread (its readline
|
|
312
|
+
# returns/raises on the closed pipe), so repeated request timeouts
|
|
313
|
+
# cannot accumulate reader threads or file descriptors. Cross-platform.
|
|
314
|
+
for stream in (proc.stdin, proc.stdout, proc.stderr):
|
|
315
|
+
try:
|
|
316
|
+
if stream is not None:
|
|
317
|
+
stream.close()
|
|
303
318
|
except Exception:
|
|
304
319
|
pass
|
|
305
320
|
self._proc = None
|