superlocalmemory 3.8.13 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +4 -4
- package/CHANGELOG.md +113 -121
- package/README.md +65 -63
- package/docs/pi-dev-integration.md +1 -1
- package/package.json +6 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +11 -4
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +125 -11
- package/src/superlocalmemory/cli/daemon.py +5 -1
- package/src/superlocalmemory/cli/main.py +35 -2
- package/src/superlocalmemory/cli/ops_cmd.py +281 -0
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/compliance/audit.py +65 -0
- package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
- package/src/superlocalmemory/compliance/gdpr.py +416 -20
- package/src/superlocalmemory/compliance/retention.py +74 -22
- package/src/superlocalmemory/compliance/scheduler.py +78 -9
- package/src/superlocalmemory/core/actor_context.py +166 -0
- package/src/superlocalmemory/core/admission.py +549 -0
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
- package/src/superlocalmemory/core/config.py +202 -24
- package/src/superlocalmemory/core/consolidation_engine.py +13 -13
- package/src/superlocalmemory/core/context_cache.py +28 -0
- package/src/superlocalmemory/core/embeddings.py +64 -2
- package/src/superlocalmemory/core/engine.py +7 -2
- package/src/superlocalmemory/core/engine_ingestion.py +65 -3
- package/src/superlocalmemory/core/engine_wiring.py +36 -9
- package/src/superlocalmemory/core/ingest_policy.py +38 -0
- package/src/superlocalmemory/core/maintenance.py +255 -0
- package/src/superlocalmemory/core/modes.py +40 -13
- package/src/superlocalmemory/core/mutations.py +437 -44
- package/src/superlocalmemory/core/operation_policy.py +92 -0
- package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
- package/src/superlocalmemory/core/operation_request.py +127 -0
- package/src/superlocalmemory/core/ops_remediation.py +542 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -0
- package/src/superlocalmemory/core/remember_runtime.py +202 -4
- package/src/superlocalmemory/core/remote_mode.py +20 -5
- package/src/superlocalmemory/core/store_pipeline.py +150 -0
- package/src/superlocalmemory/core/topic_signature.py +19 -4
- package/src/superlocalmemory/core/transactions/__init__.py +78 -0
- package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
- package/src/superlocalmemory/core/transactions/erasure.py +825 -0
- package/src/superlocalmemory/core/transactions/manifest.py +255 -0
- package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
- package/src/superlocalmemory/core/transactions/obligations.py +272 -0
- package/src/superlocalmemory/core/transactions/owners.py +114 -0
- package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
- package/src/superlocalmemory/core/transactions/service.py +330 -0
- package/src/superlocalmemory/core/worker_pool.py +33 -5
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
- package/src/superlocalmemory/encoding/emotional.py +75 -14
- package/src/superlocalmemory/encoding/scene_builder.py +115 -13
- package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
- package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
- package/src/superlocalmemory/evolution/evolution_store.py +244 -4
- package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
- package/src/superlocalmemory/evolution/model_selection.py +18 -3
- package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
- package/src/superlocalmemory/evolution/skill_activator.py +270 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
- package/src/superlocalmemory/evolution/types.py +30 -8
- package/src/superlocalmemory/graph/cozo_backend.py +17 -9
- package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
- package/src/superlocalmemory/hooks/auto_recall.py +64 -30
- package/src/superlocalmemory/hooks/codex_assets.py +14 -1
- package/src/superlocalmemory/infra/backup.py +434 -7
- package/src/superlocalmemory/infra/process_reaper.py +18 -0
- package/src/superlocalmemory/infra/self_heal.py +401 -0
- package/src/superlocalmemory/learning/feedback.py +52 -9
- package/src/superlocalmemory/loops/engine.py +10 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/http_transport.py +30 -331
- package/src/superlocalmemory/mcp/profiles.py +5 -0
- package/src/superlocalmemory/mcp/resources.py +8 -0
- package/src/superlocalmemory/mcp/server.py +51 -4
- package/src/superlocalmemory/mcp/shared.py +19 -0
- package/src/superlocalmemory/mcp/tools_active.py +25 -4
- package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
- package/src/superlocalmemory/mcp/tools_context.py +50 -8
- package/src/superlocalmemory/mcp/tools_core.py +69 -21
- package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
- package/src/superlocalmemory/mcp/tools_learning.py +21 -10
- package/src/superlocalmemory/mcp/tools_loops.py +29 -18
- package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
- package/src/superlocalmemory/mcp/tools_ops.py +115 -0
- package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
- package/src/superlocalmemory/mcp/tools_v28.py +10 -3
- package/src/superlocalmemory/mcp/tools_v3.py +34 -14
- package/src/superlocalmemory/mcp/tools_v33.py +18 -33
- package/src/superlocalmemory/mesh/broker.py +124 -46
- package/src/superlocalmemory/mesh/broker_security.py +470 -0
- package/src/superlocalmemory/mesh/discovery.py +365 -0
- package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
- package/src/superlocalmemory/mesh/node_identity.py +97 -0
- package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
- package/src/superlocalmemory/mesh/remote_sync.py +511 -28
- package/src/superlocalmemory/mesh/state_sync.py +286 -0
- package/src/superlocalmemory/optimize/config/store.py +45 -0
- package/src/superlocalmemory/parameterization/cross_project.py +12 -0
- package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
- package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
- package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
- package/src/superlocalmemory/retrieval/ann_index.py +5 -0
- package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
- package/src/superlocalmemory/retrieval/engine.py +19 -4
- package/src/superlocalmemory/retrieval/fusion.py +4 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
- package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
- package/src/superlocalmemory/retrieval/reranker.py +32 -1
- package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
- package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
- package/src/superlocalmemory/retrieval/vector_store.py +214 -8
- package/src/superlocalmemory/server/api.py +5 -5
- package/src/superlocalmemory/server/egress_policy.py +258 -0
- package/src/superlocalmemory/server/rbac_enforce.py +32 -0
- package/src/superlocalmemory/server/route_mutations.py +20 -0
- package/src/superlocalmemory/server/routes/compliance.py +153 -7
- package/src/superlocalmemory/server/routes/data_io.py +43 -2
- package/src/superlocalmemory/server/routes/events.py +15 -0
- package/src/superlocalmemory/server/routes/memories.py +56 -3
- package/src/superlocalmemory/server/routes/mesh.py +82 -1
- package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
- package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
- package/src/superlocalmemory/server/routes/v3_api.py +50 -27
- package/src/superlocalmemory/server/routes/ws.py +86 -0
- package/src/superlocalmemory/server/ui.py +6 -6
- package/src/superlocalmemory/server/unified_daemon.py +942 -119
- package/src/superlocalmemory/storage/_migration_internals.py +568 -0
- package/src/superlocalmemory/storage/_schema_version.py +110 -0
- package/src/superlocalmemory/storage/database.py +329 -24
- package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
- package/src/superlocalmemory/storage/erasure_fence.py +45 -0
- package/src/superlocalmemory/storage/generation_fence.py +63 -0
- package/src/superlocalmemory/storage/migration_runner.py +140 -417
- package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
- package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
- package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
- package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
- package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
- package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
- package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
- package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
- package/src/superlocalmemory/storage/schema.py +67 -0
- package/src/superlocalmemory/storage/write_coordinator.py +125 -0
- package/src/superlocalmemory/trust/scorer.py +28 -4
- package/src/superlocalmemory/ui/index.html +14 -3
- package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
- package/src/superlocalmemory/ui/js/brain.js +6 -4
- package/src/superlocalmemory/ui/js/compliance.js +66 -12
- package/src/superlocalmemory/ui/js/dashboard.js +13 -3
- package/src/superlocalmemory/ui/js/feedback.js +8 -2
- package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
- package/src/superlocalmemory/ui/js/modal.js +272 -5
- package/src/superlocalmemory/ui/js/od-backup.js +9 -2
- package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
- package/src/superlocalmemory/ui/js/od-operations.js +154 -23
- package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
- package/src/superlocalmemory/ui/js/od-team.js +9 -2
- package/src/superlocalmemory/ui/js/optimize.js +13 -16
- package/src/superlocalmemory/ui/js/profiles.js +7 -3
- package/src/superlocalmemory/ui/js/settings.js +7 -1
- package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
- package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
- package/src/superlocalmemory/cli/post_install.py +0 -114
- package/src/superlocalmemory/core/clock_monitor.py +0 -45
- package/src/superlocalmemory/core/db_pool.py +0 -80
- package/src/superlocalmemory/core/error_catalog.py +0 -113
- package/src/superlocalmemory/core/loop_watchdog.py +0 -56
- package/src/superlocalmemory/core/priority_queue.py +0 -61
- package/src/superlocalmemory/core/pruning_engine.py +0 -216
- package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
- package/src/superlocalmemory/core/slmignore.py +0 -125
- package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
- package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
- package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
- package/src/superlocalmemory/storage/access_control.py +0 -182
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""SSRF-safe egress validation for outbound provider requests.
|
|
2
|
+
|
|
3
|
+
Pre-admission, stateless validation for outbound HTTP fetches (embedding test,
|
|
4
|
+
provider test, local model probe). It enforces three protections:
|
|
5
|
+
|
|
6
|
+
* Cloud-metadata endpoints are blocked unconditionally and after hostname
|
|
7
|
+
normalization (trailing dot / case / IDNA), so ``metadata.google.internal.``
|
|
8
|
+
cannot slip past — including on the loopback path.
|
|
9
|
+
* The full DNS answer set is validated (all A/AAAA records), not just the first
|
|
10
|
+
resolved address, which defeats mixed public/private answers (DNS rebinding).
|
|
11
|
+
* DNS resolution failure fails closed (deny) for untrusted callers rather than
|
|
12
|
+
deferring the decision to the HTTP client.
|
|
13
|
+
|
|
14
|
+
Trusted callers (the local dashboard on loopback, or an allowlisted LAN
|
|
15
|
+
dashboard) keep the latitude to probe local/LAN model endpoints; metadata
|
|
16
|
+
endpoints are denied for everyone.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import ipaddress
|
|
21
|
+
import socket
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from enum import Enum
|
|
24
|
+
from urllib.parse import urlparse
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class EgressVerdict(str, Enum):
|
|
28
|
+
ALLOW = "allow"
|
|
29
|
+
DENY_SCHEME = "deny_scheme"
|
|
30
|
+
DENY_CREDENTIALS = "deny_credentials"
|
|
31
|
+
DENY_FRAGMENT = "deny_fragment"
|
|
32
|
+
DENY_HOST = "deny_host"
|
|
33
|
+
DENY_METADATA = "deny_metadata"
|
|
34
|
+
DENY_PRIVATE = "deny_private"
|
|
35
|
+
DENY_DNS_FAILURE = "deny_dns_failure"
|
|
36
|
+
DENY_MIXED_DNS = "deny_mixed_dns"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# Cloud metadata endpoints — always blocked regardless of caller trust.
|
|
40
|
+
METADATA_HOSTS: frozenset[str] = frozenset(
|
|
41
|
+
{
|
|
42
|
+
"169.254.169.254",
|
|
43
|
+
"metadata.google.internal",
|
|
44
|
+
"metadata",
|
|
45
|
+
"metadata.azure.internal",
|
|
46
|
+
"169.254.170.2", # AWS ECS task metadata
|
|
47
|
+
"fd00:ec2::254", # AWS IMDS over IPv6
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"})
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True, slots=True)
|
|
55
|
+
class EgressPolicy:
|
|
56
|
+
"""Configurable egress constraints (safe defaults)."""
|
|
57
|
+
|
|
58
|
+
allowed_schemes: frozenset[str] = _ALLOWED_SCHEMES
|
|
59
|
+
allow_private_for_local_actor: bool = True
|
|
60
|
+
allow_private_for_lan_actor: bool = True
|
|
61
|
+
reject_credentials: bool = True
|
|
62
|
+
reject_fragment: bool = True
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True, slots=True)
|
|
66
|
+
class EgressActor:
|
|
67
|
+
"""Minimal trust descriptor for the caller.
|
|
68
|
+
|
|
69
|
+
``is_local`` — request originated from the loopback dashboard.
|
|
70
|
+
``is_lan`` — request originated from an allowlisted LAN dashboard
|
|
71
|
+
(remote mode ON *and* client IP in the allowlist).
|
|
72
|
+
A caller with neither flag is an untrusted/remote caller.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
is_local: bool = False
|
|
76
|
+
is_lan: bool = False
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def is_trusted(self) -> bool:
|
|
80
|
+
return self.is_local or self.is_lan
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True, slots=True)
|
|
84
|
+
class ResolvedTarget:
|
|
85
|
+
addresses: tuple[str, ...] = ()
|
|
86
|
+
has_private: bool = False
|
|
87
|
+
has_public: bool = False
|
|
88
|
+
error: str = ""
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def is_mixed(self) -> bool:
|
|
92
|
+
return self.has_private and self.has_public
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass(frozen=True, slots=True)
|
|
96
|
+
class EgressResult:
|
|
97
|
+
verdict: EgressVerdict
|
|
98
|
+
resolved_ip: str = ""
|
|
99
|
+
error: str = ""
|
|
100
|
+
hostname: str = ""
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def allowed(self) -> bool:
|
|
104
|
+
return self.verdict is EgressVerdict.ALLOW
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _normalize_host(hostname: str) -> str:
|
|
108
|
+
"""Lowercase, strip a trailing dot, IDNA-encode non-ASCII hosts.
|
|
109
|
+
|
|
110
|
+
IDNA encoding is only attempted for hosts containing non-ASCII characters,
|
|
111
|
+
so IP literals and ordinary ASCII hostnames are left untouched (the ``idna``
|
|
112
|
+
codec rejects some all-ASCII inputs).
|
|
113
|
+
"""
|
|
114
|
+
host = hostname.strip().lower().rstrip(".")
|
|
115
|
+
if host and any(ord(ch) > 127 for ch in host):
|
|
116
|
+
try:
|
|
117
|
+
host = host.encode("idna").decode("ascii")
|
|
118
|
+
except (UnicodeError, UnicodeDecodeError):
|
|
119
|
+
pass # keep the raw host — classification below still applies
|
|
120
|
+
return host
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _is_dangerous_ip(
|
|
124
|
+
addr: ipaddress.IPv4Address | ipaddress.IPv6Address,
|
|
125
|
+
) -> bool:
|
|
126
|
+
return (
|
|
127
|
+
addr.is_private
|
|
128
|
+
or addr.is_loopback
|
|
129
|
+
or addr.is_link_local
|
|
130
|
+
or addr.is_reserved
|
|
131
|
+
or addr.is_multicast
|
|
132
|
+
or addr.is_unspecified
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _coerce_ip(
|
|
137
|
+
ip_str: str,
|
|
138
|
+
) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
|
139
|
+
try:
|
|
140
|
+
addr = ipaddress.ip_address(ip_str)
|
|
141
|
+
except ValueError:
|
|
142
|
+
return None
|
|
143
|
+
# Unwrap IPv4-mapped IPv6 (``::ffff:127.0.0.1``) before classification.
|
|
144
|
+
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
|
|
145
|
+
return addr.ipv4_mapped
|
|
146
|
+
return addr
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def resolve_and_validate_dns(
|
|
150
|
+
hostname: str, policy: EgressPolicy = EgressPolicy()
|
|
151
|
+
) -> ResolvedTarget:
|
|
152
|
+
"""Resolve every A/AAAA record and classify each address.
|
|
153
|
+
|
|
154
|
+
Fails closed: any resolution error, or an empty answer set, returns a
|
|
155
|
+
``ResolvedTarget`` carrying a non-empty ``error``.
|
|
156
|
+
"""
|
|
157
|
+
try:
|
|
158
|
+
infos = socket.getaddrinfo(
|
|
159
|
+
hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM
|
|
160
|
+
)
|
|
161
|
+
except (socket.gaierror, OSError, UnicodeError) as exc:
|
|
162
|
+
return ResolvedTarget(error=f"dns resolution failed: {exc}")
|
|
163
|
+
|
|
164
|
+
addresses: list[str] = []
|
|
165
|
+
has_private = False
|
|
166
|
+
has_public = False
|
|
167
|
+
for info in infos:
|
|
168
|
+
sockaddr = info[4]
|
|
169
|
+
addr = _coerce_ip(sockaddr[0])
|
|
170
|
+
if addr is None:
|
|
171
|
+
continue
|
|
172
|
+
addresses.append(str(addr))
|
|
173
|
+
if _is_dangerous_ip(addr):
|
|
174
|
+
has_private = True
|
|
175
|
+
else:
|
|
176
|
+
has_public = True
|
|
177
|
+
|
|
178
|
+
if not addresses:
|
|
179
|
+
return ResolvedTarget(error="no addresses resolved")
|
|
180
|
+
return ResolvedTarget(
|
|
181
|
+
addresses=tuple(addresses),
|
|
182
|
+
has_private=has_private,
|
|
183
|
+
has_public=has_public,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def validate_egress_url(
|
|
188
|
+
url: str,
|
|
189
|
+
actor: EgressActor = EgressActor(),
|
|
190
|
+
policy: EgressPolicy = EgressPolicy(),
|
|
191
|
+
) -> EgressResult:
|
|
192
|
+
"""Full SSRF-safe validation of an outbound URL for a given caller."""
|
|
193
|
+
parsed = urlparse(url)
|
|
194
|
+
|
|
195
|
+
if parsed.scheme not in policy.allowed_schemes:
|
|
196
|
+
return EgressResult(
|
|
197
|
+
verdict=EgressVerdict.DENY_SCHEME, error=f"scheme {parsed.scheme!r}"
|
|
198
|
+
)
|
|
199
|
+
if policy.reject_credentials and (parsed.username or parsed.password):
|
|
200
|
+
return EgressResult(verdict=EgressVerdict.DENY_CREDENTIALS)
|
|
201
|
+
if policy.reject_fragment and parsed.fragment:
|
|
202
|
+
return EgressResult(verdict=EgressVerdict.DENY_FRAGMENT)
|
|
203
|
+
|
|
204
|
+
hostname = _normalize_host(parsed.hostname or "")
|
|
205
|
+
if not hostname:
|
|
206
|
+
return EgressResult(verdict=EgressVerdict.DENY_HOST, error="empty host")
|
|
207
|
+
|
|
208
|
+
# Metadata block — unconditional, before any trust short-circuit or DNS
|
|
209
|
+
# (defeats the trailing-dot / loopback-path bypass and DNS rebinding).
|
|
210
|
+
if hostname in METADATA_HOSTS:
|
|
211
|
+
return EgressResult(verdict=EgressVerdict.DENY_METADATA, hostname=hostname)
|
|
212
|
+
literal = _coerce_ip(hostname)
|
|
213
|
+
if literal is not None and str(literal) in METADATA_HOSTS:
|
|
214
|
+
return EgressResult(verdict=EgressVerdict.DENY_METADATA, hostname=hostname)
|
|
215
|
+
|
|
216
|
+
trusted = (
|
|
217
|
+
actor.is_local and policy.allow_private_for_local_actor
|
|
218
|
+
) or (actor.is_lan and policy.allow_private_for_lan_actor)
|
|
219
|
+
|
|
220
|
+
# Literal IP target: classify directly, no DNS needed.
|
|
221
|
+
if literal is not None:
|
|
222
|
+
if _is_dangerous_ip(literal) and not trusted:
|
|
223
|
+
return EgressResult(
|
|
224
|
+
verdict=EgressVerdict.DENY_PRIVATE,
|
|
225
|
+
resolved_ip=str(literal),
|
|
226
|
+
hostname=hostname,
|
|
227
|
+
)
|
|
228
|
+
return EgressResult(
|
|
229
|
+
verdict=EgressVerdict.ALLOW, resolved_ip=str(literal), hostname=hostname
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
# Trusted callers may probe local/LAN endpoints by name without a forced
|
|
233
|
+
# DNS lookup — preserving pre-V4 loopback/LAN dashboard behaviour. Metadata
|
|
234
|
+
# was already denied above for everyone.
|
|
235
|
+
if trusted:
|
|
236
|
+
return EgressResult(verdict=EgressVerdict.ALLOW, hostname=hostname)
|
|
237
|
+
|
|
238
|
+
# Untrusted caller: resolve the full answer set and fail closed on error.
|
|
239
|
+
resolved = resolve_and_validate_dns(hostname, policy)
|
|
240
|
+
if resolved.error:
|
|
241
|
+
return EgressResult(
|
|
242
|
+
verdict=EgressVerdict.DENY_DNS_FAILURE,
|
|
243
|
+
error=resolved.error,
|
|
244
|
+
hostname=hostname,
|
|
245
|
+
)
|
|
246
|
+
if resolved.has_private:
|
|
247
|
+
verdict = (
|
|
248
|
+
EgressVerdict.DENY_MIXED_DNS
|
|
249
|
+
if resolved.is_mixed
|
|
250
|
+
else EgressVerdict.DENY_PRIVATE
|
|
251
|
+
)
|
|
252
|
+
return EgressResult(verdict=verdict, hostname=hostname)
|
|
253
|
+
|
|
254
|
+
return EgressResult(
|
|
255
|
+
verdict=EgressVerdict.ALLOW,
|
|
256
|
+
resolved_ip=resolved.addresses[0],
|
|
257
|
+
hostname=hostname,
|
|
258
|
+
)
|
|
@@ -114,6 +114,38 @@ def require_manage(request: Request, *, profile: str | None = None) -> dict:
|
|
|
114
114
|
return require_permission(request, Permission.MANAGE, profile=profile)
|
|
115
115
|
|
|
116
116
|
|
|
117
|
+
def resolve_actor_roles(request: Request, *, profile: str | None = None):
|
|
118
|
+
"""Resolve the caller to concrete ActorContext roles (server-derived).
|
|
119
|
+
|
|
120
|
+
The machine operator (owner) is root. A logged-in user is mapped from their
|
|
121
|
+
persisted RBAC role on ``profile``. This must be called only after
|
|
122
|
+
``require_permission`` has already authorized the operation, so the returned
|
|
123
|
+
role always includes the permission the caller was admitted with.
|
|
124
|
+
"""
|
|
125
|
+
from superlocalmemory.core.actor_context import ActorRole
|
|
126
|
+
|
|
127
|
+
principal = resolve_principal(request)
|
|
128
|
+
if principal.get("kind") == "owner":
|
|
129
|
+
return frozenset({ActorRole.OWNER})
|
|
130
|
+
rbac = get_rbac_engine(request.app.state)
|
|
131
|
+
role = None
|
|
132
|
+
if rbac is not None:
|
|
133
|
+
try:
|
|
134
|
+
role = rbac.get_role(principal["user_id"], profile or _active_profile())
|
|
135
|
+
except Exception:
|
|
136
|
+
# The caller already passed require_permission for this operation, so
|
|
137
|
+
# a transient role lookup must not surface as a 500. Fall back to the
|
|
138
|
+
# least-privileged write-capable role rather than deny an authorized
|
|
139
|
+
# write.
|
|
140
|
+
return frozenset({ActorRole.MEMBER})
|
|
141
|
+
mapped = {
|
|
142
|
+
Role.ADMIN: ActorRole.ADMIN,
|
|
143
|
+
Role.MEMBER: ActorRole.MEMBER,
|
|
144
|
+
Role.VIEWER: ActorRole.VIEWER,
|
|
145
|
+
}.get(role)
|
|
146
|
+
return frozenset({mapped}) if mapped is not None else frozenset({ActorRole.ANONYMOUS})
|
|
147
|
+
|
|
148
|
+
|
|
117
149
|
def principal_info(request: Request) -> dict:
|
|
118
150
|
"""Rich identity for /whoami: principal + role + effective permissions on
|
|
119
151
|
the active profile. Never raises — used by the dashboard to render UI."""
|
|
@@ -74,6 +74,26 @@ def authorize_route_mutation(
|
|
|
74
74
|
if content_preview:
|
|
75
75
|
context["content_preview"] = content_preview[:100]
|
|
76
76
|
|
|
77
|
+
# Phase-1/E2: run admission registry before trust-hook pre-handler so
|
|
78
|
+
# enterprise policy can deny the mutation even if the trust hook allows it.
|
|
79
|
+
from superlocalmemory.core.admission import (
|
|
80
|
+
AdmissionDenied,
|
|
81
|
+
OperationKind,
|
|
82
|
+
_resolve_deployment,
|
|
83
|
+
admit,
|
|
84
|
+
resolve_actor,
|
|
85
|
+
)
|
|
86
|
+
from superlocalmemory.core.actor_context import Transport
|
|
87
|
+
deployment = _resolve_deployment()
|
|
88
|
+
tier = "enterprise" if deployment.is_enterprise else "personal"
|
|
89
|
+
mode = "company" if deployment.is_enterprise else "local"
|
|
90
|
+
http_actor = resolve_actor(Transport.HTTP, tier=tier, mode=mode)
|
|
91
|
+
kind = OperationKind.FORGET if operation == "delete" else OperationKind.CORRECT
|
|
92
|
+
try:
|
|
93
|
+
admit(kind, http_actor, mode=mode)
|
|
94
|
+
except AdmissionDenied as exc:
|
|
95
|
+
raise HTTPException(403, detail="Write authorization rejected") from exc
|
|
96
|
+
|
|
77
97
|
try:
|
|
78
98
|
engine._hooks.run_pre(operation, context)
|
|
79
99
|
except Exception as exc:
|
|
@@ -8,17 +8,17 @@ Routes: /api/compliance/status, /api/compliance/audit,
|
|
|
8
8
|
/api/compliance/retention-policy
|
|
9
9
|
Uses V3 compliance modules: ABACEngine, AuditChain, RetentionEngine.
|
|
10
10
|
"""
|
|
11
|
-
import json
|
|
12
11
|
import logging
|
|
13
12
|
from typing import Optional
|
|
14
13
|
|
|
15
14
|
from fastapi import APIRouter, Query, Request
|
|
16
15
|
from fastapi.responses import JSONResponse
|
|
17
16
|
|
|
18
|
-
from .helpers import get_active_profile, get_engine_lazy, MEMORY_DIR, DB_PATH
|
|
19
17
|
from superlocalmemory.server.route_mutations import authorize_route_mutation
|
|
20
18
|
from superlocalmemory.storage.memory_write import memory_write
|
|
21
19
|
|
|
20
|
+
from .helpers import DB_PATH, MEMORY_DIR, get_active_profile, get_engine_lazy
|
|
21
|
+
|
|
22
22
|
logger = logging.getLogger("superlocalmemory.routes.compliance")
|
|
23
23
|
router = APIRouter()
|
|
24
24
|
|
|
@@ -31,10 +31,10 @@ AUDIT_DB = MEMORY_DIR / "audit_chain.db"
|
|
|
31
31
|
# Feature detection
|
|
32
32
|
COMPLIANCE_AVAILABLE = False
|
|
33
33
|
try:
|
|
34
|
-
from superlocalmemory.compliance.audit import AuditChain
|
|
35
|
-
from superlocalmemory.compliance.retention import RetentionEngine
|
|
36
34
|
from superlocalmemory.compliance.abac import ABACEngine
|
|
35
|
+
from superlocalmemory.compliance.audit import AuditChain
|
|
37
36
|
from superlocalmemory.compliance.gdpr import GDPRCompliance
|
|
37
|
+
from superlocalmemory.compliance.retention import RetentionEngine
|
|
38
38
|
COMPLIANCE_AVAILABLE = True
|
|
39
39
|
except ImportError:
|
|
40
40
|
logger.info("V3 compliance engine not available")
|
|
@@ -239,8 +239,8 @@ async def gdpr_export(request: Request):
|
|
|
239
239
|
# remote uncredentialed fails closed) AND require MANAGE — in company mode
|
|
240
240
|
# a session cookie flows on navigation, so a non-admin user is still denied;
|
|
241
241
|
# the machine owner keeps MANAGE.
|
|
242
|
-
from superlocalmemory.server.write_identity import require_http_mutation_actor
|
|
243
242
|
from superlocalmemory.server.rbac_enforce import require_manage
|
|
243
|
+
from superlocalmemory.server.write_identity import require_http_mutation_actor
|
|
244
244
|
require_http_mutation_actor(request, getattr(request.app.state, "daemon_descriptor", None),
|
|
245
245
|
actor_kind="gdpr-export")
|
|
246
246
|
require_manage(request)
|
|
@@ -299,9 +299,155 @@ async def gdpr_erase(request: Request, data: dict = {}):
|
|
|
299
299
|
source_agent_id="http-gdpr-erase",
|
|
300
300
|
profile_id=profile,
|
|
301
301
|
)
|
|
302
|
-
result = GDPRCompliance(engine._db).forget_profile(profile)
|
|
302
|
+
result = GDPRCompliance(engine._db, engine=engine).forget_profile(profile)
|
|
303
303
|
authorization.complete()
|
|
304
|
-
|
|
304
|
+
result = result or {}
|
|
305
|
+
failure_markers = (
|
|
306
|
+
"vector_store_failures",
|
|
307
|
+
"audit_completion_failed",
|
|
308
|
+
"audit_request_failed",
|
|
309
|
+
"receipt_persist_failed",
|
|
310
|
+
)
|
|
311
|
+
success = not any(result.get(marker) for marker in failure_markers)
|
|
312
|
+
return {"success": success, "active_profile": profile, **result}
|
|
305
313
|
except Exception:
|
|
306
314
|
logger.exception("gdpr_erase error")
|
|
307
315
|
return {"success": False, "error": "Internal server error"}
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
# ── GDPR Art. 17 — Entity-level Erasure ──────────────────────────────────────
|
|
319
|
+
|
|
320
|
+
@router.post("/api/compliance/gdpr/erase-entity")
|
|
321
|
+
async def gdpr_erase_entity(request: Request, data: dict = {}):
|
|
322
|
+
"""Erase all facts mentioning a named entity for the active profile.
|
|
323
|
+
|
|
324
|
+
IRREVERSIBLE. Body must contain an ``entity_name`` and a ``confirm``
|
|
325
|
+
field that exactly matches ``entity_name`` — same confirm-guard pattern
|
|
326
|
+
as profile erasure. Mutation-authorized; requires MANAGE permission.
|
|
327
|
+
"""
|
|
328
|
+
if not COMPLIANCE_AVAILABLE:
|
|
329
|
+
return {"success": False, "error": "Compliance engine not available"}
|
|
330
|
+
try:
|
|
331
|
+
engine = get_engine_lazy(request.app.state)
|
|
332
|
+
if engine is None:
|
|
333
|
+
return {"success": False, "error": "Engine not initialized"}
|
|
334
|
+
profile = get_active_profile()
|
|
335
|
+
entity_name = ((data or {}).get("entity_name") or "").strip()
|
|
336
|
+
confirm = (data or {}).get("confirm", "")
|
|
337
|
+
if not entity_name:
|
|
338
|
+
return {"success": False, "error": "entity_name is required"}
|
|
339
|
+
if confirm != entity_name:
|
|
340
|
+
return {
|
|
341
|
+
"success": False,
|
|
342
|
+
"error": (
|
|
343
|
+
"Confirmation required: send {\"confirm\": \"" + entity_name +
|
|
344
|
+
"\"} to erase this entity. This is irreversible."
|
|
345
|
+
),
|
|
346
|
+
}
|
|
347
|
+
from superlocalmemory.server.rbac_enforce import require_manage
|
|
348
|
+
from superlocalmemory.server.write_identity import require_http_mutation_actor
|
|
349
|
+
require_http_mutation_actor(
|
|
350
|
+
request,
|
|
351
|
+
getattr(request.app.state, "daemon_descriptor", None),
|
|
352
|
+
actor_kind="gdpr-erase-entity",
|
|
353
|
+
)
|
|
354
|
+
require_manage(request, profile=profile)
|
|
355
|
+
authorization = authorize_route_mutation(
|
|
356
|
+
request,
|
|
357
|
+
operation="delete",
|
|
358
|
+
source_agent_id="http-gdpr-erase-entity",
|
|
359
|
+
profile_id=profile,
|
|
360
|
+
)
|
|
361
|
+
result = GDPRCompliance(engine._db, engine=engine).forget_entity(entity_name, profile)
|
|
362
|
+
authorization.complete()
|
|
363
|
+
result = result or {}
|
|
364
|
+
failure_markers = (
|
|
365
|
+
"vector_store_failures",
|
|
366
|
+
"audit_completion_failed",
|
|
367
|
+
"audit_request_failed",
|
|
368
|
+
"receipt_persist_failed",
|
|
369
|
+
)
|
|
370
|
+
success = not any(result.get(marker) for marker in failure_markers)
|
|
371
|
+
return {
|
|
372
|
+
"success": success, "active_profile": profile,
|
|
373
|
+
"entity_name": entity_name, **result,
|
|
374
|
+
}
|
|
375
|
+
except Exception:
|
|
376
|
+
logger.exception("gdpr_erase_entity error")
|
|
377
|
+
return {"success": False, "error": "Internal server error"}
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# ── Erasure Receipts — list + cryptographic verify ───────────────────────────
|
|
381
|
+
|
|
382
|
+
@router.get("/api/compliance/receipts")
|
|
383
|
+
async def list_erasure_receipts(
|
|
384
|
+
request: Request,
|
|
385
|
+
limit: int = Query(default=50, ge=1, le=200),
|
|
386
|
+
):
|
|
387
|
+
"""List recent erasure receipts for the active profile.
|
|
388
|
+
|
|
389
|
+
Returns summary rows from ``erasure_receipts`` (migration M035):
|
|
390
|
+
erasure_id, subject_type, subject_id, state, all_erased, fact_count,
|
|
391
|
+
requested_at, completed_at. Read-only; loopback-gated.
|
|
392
|
+
"""
|
|
393
|
+
if not COMPLIANCE_AVAILABLE:
|
|
394
|
+
return {"available": False, "error": "Compliance engine not available"}
|
|
395
|
+
from superlocalmemory.server.write_identity import require_http_mutation_actor
|
|
396
|
+
require_http_mutation_actor(
|
|
397
|
+
request,
|
|
398
|
+
getattr(request.app.state, "daemon_descriptor", None),
|
|
399
|
+
actor_kind="erasure-receipts-read",
|
|
400
|
+
)
|
|
401
|
+
try:
|
|
402
|
+
engine = get_engine_lazy(request.app.state)
|
|
403
|
+
if engine is None:
|
|
404
|
+
return {"available": False, "error": "Engine not initialized"}
|
|
405
|
+
profile = get_active_profile()
|
|
406
|
+
try:
|
|
407
|
+
rows = engine._db.execute(
|
|
408
|
+
"SELECT erasure_id, subject_type, subject_id, state, all_erased, "
|
|
409
|
+
"fact_count, requested_at, completed_at "
|
|
410
|
+
"FROM erasure_receipts WHERE profile_id = ? "
|
|
411
|
+
"ORDER BY requested_at DESC LIMIT ?",
|
|
412
|
+
(profile, limit),
|
|
413
|
+
)
|
|
414
|
+
receipts = [dict(r) for r in rows]
|
|
415
|
+
except Exception:
|
|
416
|
+
receipts = []
|
|
417
|
+
return {
|
|
418
|
+
"available": True, "active_profile": profile,
|
|
419
|
+
"receipts": receipts, "total": len(receipts),
|
|
420
|
+
}
|
|
421
|
+
except Exception:
|
|
422
|
+
logger.exception("list_erasure_receipts error")
|
|
423
|
+
return {"available": False, "error": "Internal server error"}
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@router.get("/api/compliance/receipts/{erasure_id}/verify")
|
|
427
|
+
async def verify_erasure_receipt(request: Request, erasure_id: str):
|
|
428
|
+
"""Cryptographically verify an erasure receipt's audit hash.
|
|
429
|
+
|
|
430
|
+
Returns ``{verified: bool}`` — True means the stored hash matches a
|
|
431
|
+
recompute from the receipt fields (tamper-evident). Read-only;
|
|
432
|
+
loopback-gated.
|
|
433
|
+
"""
|
|
434
|
+
if not COMPLIANCE_AVAILABLE:
|
|
435
|
+
return {"available": False, "error": "Compliance engine not available"}
|
|
436
|
+
from superlocalmemory.server.write_identity import require_http_mutation_actor
|
|
437
|
+
require_http_mutation_actor(
|
|
438
|
+
request,
|
|
439
|
+
getattr(request.app.state, "daemon_descriptor", None),
|
|
440
|
+
actor_kind="erasure-receipt-verify",
|
|
441
|
+
)
|
|
442
|
+
try:
|
|
443
|
+
engine = get_engine_lazy(request.app.state)
|
|
444
|
+
if engine is None:
|
|
445
|
+
return {"available": False, "error": "Engine not initialized"}
|
|
446
|
+
profile = get_active_profile()
|
|
447
|
+
from superlocalmemory.core.transactions.erasure import verify_receipt
|
|
448
|
+
with engine._db.raw_connection() as conn:
|
|
449
|
+
verified = verify_receipt(conn, erasure_id, profile_id=profile)
|
|
450
|
+
return {"available": True, "erasure_id": erasure_id, "verified": verified}
|
|
451
|
+
except Exception:
|
|
452
|
+
logger.exception("verify_erasure_receipt error")
|
|
453
|
+
return {"available": False, "error": "Internal server error"}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
Routes: /api/export, /api/import
|
|
8
8
|
"""
|
|
9
|
+
import asyncio
|
|
9
10
|
import io
|
|
10
11
|
import gzip
|
|
11
12
|
import hashlib
|
|
@@ -27,6 +28,13 @@ from .helpers import (
|
|
|
27
28
|
|
|
28
29
|
logger = logging.getLogger("superlocalmemory.routes.data_io")
|
|
29
30
|
|
|
31
|
+
# Hard cap on the total decompressed byte count for gzip imports.
|
|
32
|
+
# Bounding the compressed upload size alone does not prevent a decompression
|
|
33
|
+
# bomb: a few kilobytes of input can expand to gigabytes. This cap is checked
|
|
34
|
+
# incrementally during streaming decompression so the full expanded content is
|
|
35
|
+
# never materialized before the guard fires.
|
|
36
|
+
_MAX_DECOMPRESSED_BYTES: int = 200 * 1024 * 1024 # 200 MB
|
|
37
|
+
|
|
30
38
|
|
|
31
39
|
def _internal_error(detail: str = "Internal server error") -> HTTPException:
|
|
32
40
|
"""SEC-H-02: log full traceback server-side; return a generic message to the client."""
|
|
@@ -158,7 +166,27 @@ async def import_memories(request: Request, file: UploadFile = File(...)):
|
|
|
158
166
|
raise HTTPException(status_code=413,
|
|
159
167
|
detail="Import file exceeds the 50 MB limit")
|
|
160
168
|
if file.filename and file.filename.endswith('.gz'):
|
|
161
|
-
|
|
169
|
+
# Stream-decompress with an incremental byte counter so the full
|
|
170
|
+
# expanded payload is never allocated before the guard fires.
|
|
171
|
+
_chunk_size = 65_536
|
|
172
|
+
chunks: list[bytes] = []
|
|
173
|
+
total_decompressed = 0
|
|
174
|
+
with gzip.GzipFile(fileobj=io.BytesIO(content)) as _gz:
|
|
175
|
+
while True:
|
|
176
|
+
chunk = _gz.read(_chunk_size)
|
|
177
|
+
if not chunk:
|
|
178
|
+
break
|
|
179
|
+
total_decompressed += len(chunk)
|
|
180
|
+
if total_decompressed > _MAX_DECOMPRESSED_BYTES:
|
|
181
|
+
raise HTTPException(
|
|
182
|
+
status_code=413,
|
|
183
|
+
detail=(
|
|
184
|
+
f"Decompressed content exceeds the "
|
|
185
|
+
f"{_MAX_DECOMPRESSED_BYTES // (1024 * 1024)} MB limit"
|
|
186
|
+
),
|
|
187
|
+
)
|
|
188
|
+
chunks.append(chunk)
|
|
189
|
+
content = b"".join(chunks)
|
|
162
190
|
|
|
163
191
|
try:
|
|
164
192
|
data = json.loads(content)
|
|
@@ -204,12 +232,25 @@ async def import_memories(request: Request, file: UploadFile = File(...)):
|
|
|
204
232
|
if not memory_content:
|
|
205
233
|
errors.append(f"Memory {idx}: missing 'content' field")
|
|
206
234
|
continue
|
|
235
|
+
# Imported content is untrusted: scrub secrets before it reaches
|
|
236
|
+
# any durable or queryable store, exactly as the canonical
|
|
237
|
+
# ingest path does.
|
|
238
|
+
from superlocalmemory.core.ingest_policy import scrub_secrets_for_ingest
|
|
239
|
+
_scrub = scrub_secrets_for_ingest(memory_content)
|
|
240
|
+
if _scrub.redacted:
|
|
241
|
+
memory_content = _scrub.content
|
|
207
242
|
|
|
208
243
|
metadata = {
|
|
209
244
|
"project_name": memory.get('project_name'),
|
|
210
245
|
"category": memory.get('category'),
|
|
211
246
|
"tags": memory.get('tags', ''),
|
|
212
247
|
}
|
|
248
|
+
for _field in (
|
|
249
|
+
"fact_type", "confidence", "importance", "entities",
|
|
250
|
+
"canonical_entities", "referenced_date", "pinned",
|
|
251
|
+
):
|
|
252
|
+
if _field in memory:
|
|
253
|
+
metadata[_field] = memory[_field]
|
|
213
254
|
receipt, created = command.submit_with_status(IngestionRequest(
|
|
214
255
|
content=memory_content,
|
|
215
256
|
profile_id=engine._profile_id,
|
|
@@ -224,7 +265,7 @@ async def import_memories(request: Request, file: UploadFile = File(...)):
|
|
|
224
265
|
speaker=memory.get('speaker') or "",
|
|
225
266
|
role=memory.get('role') or "user",
|
|
226
267
|
))
|
|
227
|
-
completed = command.materialize
|
|
268
|
+
completed = await asyncio.to_thread(command.materialize, receipt.operation_id)
|
|
228
269
|
if completed.state is not IngestionState.COMPLETE:
|
|
229
270
|
raise RuntimeError(
|
|
230
271
|
completed.last_error or "canonical import failed"
|
|
@@ -34,6 +34,11 @@ except ImportError:
|
|
|
34
34
|
_sse_queues: Set = set()
|
|
35
35
|
_sse_queues_lock = threading.Lock()
|
|
36
36
|
|
|
37
|
+
# Maximum concurrent SSE connections. Connections beyond this limit are
|
|
38
|
+
# rejected immediately with an SSE error frame so the server cannot be
|
|
39
|
+
# exhausted by many idle clients.
|
|
40
|
+
_MAX_SSE_CONNECTIONS = 64
|
|
41
|
+
|
|
37
42
|
|
|
38
43
|
def _event_to_sse_bridge(event: dict):
|
|
39
44
|
"""EventBus listener that pushes events to all SSE client queues."""
|
|
@@ -78,6 +83,16 @@ async def event_stream(
|
|
|
78
83
|
|
|
79
84
|
client_queue = _queue.Queue(maxsize=100)
|
|
80
85
|
with _sse_queues_lock:
|
|
86
|
+
if len(_sse_queues) >= _MAX_SSE_CONNECTIONS:
|
|
87
|
+
return StreamingResponse(
|
|
88
|
+
iter(['data: {"error": "SSE connection limit reached"}\n\n']),
|
|
89
|
+
media_type="text/event-stream",
|
|
90
|
+
headers={
|
|
91
|
+
"Cache-Control": "no-cache",
|
|
92
|
+
"Connection": "close",
|
|
93
|
+
"X-Accel-Buffering": "no",
|
|
94
|
+
},
|
|
95
|
+
)
|
|
81
96
|
_sse_queues.add(client_queue)
|
|
82
97
|
|
|
83
98
|
async def generate():
|