remdb 0.3.180__py3-none-any.whl → 0.3.258__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rem/agentic/README.md +36 -2
- rem/agentic/__init__.py +10 -1
- rem/agentic/context.py +185 -1
- rem/agentic/context_builder.py +56 -35
- rem/agentic/mcp/tool_wrapper.py +2 -2
- rem/agentic/providers/pydantic_ai.py +303 -111
- rem/agentic/schema.py +2 -2
- rem/api/main.py +1 -1
- rem/api/mcp_router/resources.py +223 -0
- rem/api/mcp_router/server.py +4 -0
- rem/api/mcp_router/tools.py +608 -166
- rem/api/routers/admin.py +30 -4
- rem/api/routers/auth.py +219 -20
- rem/api/routers/chat/child_streaming.py +393 -0
- rem/api/routers/chat/completions.py +77 -40
- rem/api/routers/chat/sse_events.py +7 -3
- rem/api/routers/chat/streaming.py +381 -291
- rem/api/routers/chat/streaming_utils.py +325 -0
- rem/api/routers/common.py +18 -0
- rem/api/routers/dev.py +7 -1
- rem/api/routers/feedback.py +11 -3
- rem/api/routers/messages.py +176 -38
- rem/api/routers/models.py +9 -1
- rem/api/routers/query.py +17 -15
- rem/api/routers/shared_sessions.py +16 -0
- rem/auth/jwt.py +19 -4
- rem/auth/middleware.py +42 -28
- rem/cli/README.md +62 -0
- rem/cli/commands/ask.py +205 -114
- rem/cli/commands/db.py +55 -31
- rem/cli/commands/experiments.py +1 -1
- rem/cli/commands/process.py +179 -43
- rem/cli/commands/query.py +109 -0
- rem/cli/commands/session.py +117 -0
- rem/cli/main.py +2 -0
- rem/models/core/experiment.py +1 -1
- rem/models/entities/ontology.py +18 -20
- rem/models/entities/session.py +1 -0
- rem/schemas/agents/core/agent-builder.yaml +1 -1
- rem/schemas/agents/rem.yaml +1 -1
- rem/schemas/agents/test_orchestrator.yaml +42 -0
- rem/schemas/agents/test_structured_output.yaml +52 -0
- rem/services/content/providers.py +151 -49
- rem/services/content/service.py +18 -5
- rem/services/embeddings/worker.py +26 -12
- rem/services/postgres/__init__.py +28 -3
- rem/services/postgres/diff_service.py +57 -5
- rem/services/postgres/programmable_diff_service.py +635 -0
- rem/services/postgres/pydantic_to_sqlalchemy.py +2 -2
- rem/services/postgres/register_type.py +11 -10
- rem/services/postgres/repository.py +39 -28
- rem/services/postgres/schema_generator.py +5 -5
- rem/services/postgres/sql_builder.py +6 -5
- rem/services/rem/README.md +4 -3
- rem/services/rem/parser.py +7 -10
- rem/services/rem/service.py +47 -0
- rem/services/session/__init__.py +8 -1
- rem/services/session/compression.py +47 -5
- rem/services/session/pydantic_messages.py +310 -0
- rem/services/session/reload.py +2 -1
- rem/settings.py +92 -7
- rem/sql/migrations/001_install.sql +125 -7
- rem/sql/migrations/002_install_models.sql +159 -149
- rem/sql/migrations/004_cache_system.sql +10 -276
- rem/sql/migrations/migrate_session_id_to_uuid.sql +45 -0
- rem/utils/schema_loader.py +180 -120
- {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/METADATA +7 -6
- {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/RECORD +70 -61
- {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/WHEEL +0 -0
- {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""Convert stored session messages to pydantic-ai native message format.
|
|
2
|
+
|
|
3
|
+
This module enables proper conversation history replay by converting our simplified
|
|
4
|
+
storage format into pydantic-ai's native ModelRequest/ModelResponse types.
|
|
5
|
+
|
|
6
|
+
Key insight: When we store tool results, we only store the result (ToolReturnPart).
|
|
7
|
+
But LLM APIs require matching ToolCallPart for each ToolReturnPart. So we synthesize
|
|
8
|
+
the ToolCallPart from stored metadata (tool_name, tool_call_id) and arguments.
|
|
9
|
+
|
|
10
|
+
Tool arguments can come from two places:
|
|
11
|
+
- Parent tool calls (ask_agent): tool_arguments stored in metadata (content = result)
|
|
12
|
+
- Child tool calls (register_metadata): arguments parsed from content (content = args as JSON)
|
|
13
|
+
|
|
14
|
+
Storage format (our simplified format):
|
|
15
|
+
{"role": "user", "content": "..."}
|
|
16
|
+
{"role": "assistant", "content": "..."}
|
|
17
|
+
{"role": "tool", "content": "{...}", "tool_name": "...", "tool_call_id": "...", "tool_arguments": {...}} # optional
|
|
18
|
+
|
|
19
|
+
Pydantic-ai format (what the LLM expects):
|
|
20
|
+
ModelRequest(parts=[UserPromptPart(content="...")])
|
|
21
|
+
ModelResponse(parts=[TextPart(content="..."), ToolCallPart(...)]) # Call
|
|
22
|
+
ModelRequest(parts=[ToolReturnPart(...)]) # Result
|
|
23
|
+
|
|
24
|
+
Example usage:
|
|
25
|
+
from rem.services.session.pydantic_messages import session_to_pydantic_messages
|
|
26
|
+
|
|
27
|
+
# Load session history
|
|
28
|
+
session_history = await store.load_session_messages(session_id)
|
|
29
|
+
|
|
30
|
+
# Convert to pydantic-ai format
|
|
31
|
+
message_history = session_to_pydantic_messages(session_history)
|
|
32
|
+
|
|
33
|
+
# Use with agent.run()
|
|
34
|
+
result = await agent.run(user_prompt, message_history=message_history)
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
import json
|
|
38
|
+
import re
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
from loguru import logger
|
|
42
|
+
from pydantic_ai.messages import (
|
|
43
|
+
ModelMessage,
|
|
44
|
+
ModelRequest,
|
|
45
|
+
ModelResponse,
|
|
46
|
+
SystemPromptPart,
|
|
47
|
+
TextPart,
|
|
48
|
+
ToolCallPart,
|
|
49
|
+
ToolReturnPart,
|
|
50
|
+
UserPromptPart,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _sanitize_tool_name(tool_name: str) -> str:
|
|
55
|
+
"""Sanitize tool name for OpenAI API compatibility.
|
|
56
|
+
|
|
57
|
+
OpenAI requires tool names to match pattern: ^[a-zA-Z0-9_-]+$
|
|
58
|
+
This replaces invalid characters (like colons) with underscores.
|
|
59
|
+
"""
|
|
60
|
+
return re.sub(r'[^a-zA-Z0-9_-]', '_', tool_name)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def session_to_pydantic_messages(
|
|
64
|
+
session_history: list[dict[str, Any]],
|
|
65
|
+
system_prompt: str | None = None,
|
|
66
|
+
) -> list[ModelMessage]:
|
|
67
|
+
"""Convert stored session messages to pydantic-ai ModelMessage format.
|
|
68
|
+
|
|
69
|
+
Handles the conversion of our simplified storage format to pydantic-ai's
|
|
70
|
+
native message types, including synthesizing ToolCallPart for tool results.
|
|
71
|
+
|
|
72
|
+
IMPORTANT: pydantic-ai only auto-adds system prompts when message_history is empty.
|
|
73
|
+
When passing message_history to agent.run(), you MUST include the system prompt
|
|
74
|
+
via the system_prompt parameter here.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
session_history: List of message dicts from SessionMessageStore.load_session_messages()
|
|
78
|
+
Each dict has: role, content, and optionally tool_name, tool_call_id, tool_arguments
|
|
79
|
+
system_prompt: The agent's system prompt (from schema description). This is REQUIRED
|
|
80
|
+
for proper agent behavior on subsequent turns, as pydantic-ai won't add it
|
|
81
|
+
automatically when message_history is provided.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
List of ModelMessage (ModelRequest | ModelResponse) ready for agent.run(message_history=...)
|
|
85
|
+
|
|
86
|
+
Note:
|
|
87
|
+
- System prompts ARE included as SystemPromptPart when system_prompt is provided
|
|
88
|
+
- Tool results require synthesized ToolCallPart to satisfy LLM API requirements
|
|
89
|
+
- The first message in session_history should be "user" role (from context builder)
|
|
90
|
+
"""
|
|
91
|
+
messages: list[ModelMessage] = []
|
|
92
|
+
|
|
93
|
+
# CRITICAL: Prepend agent's system prompt if provided
|
|
94
|
+
# This ensures the agent's instructions are present on every turn
|
|
95
|
+
# pydantic-ai only auto-adds system prompts when message_history is empty
|
|
96
|
+
if system_prompt:
|
|
97
|
+
messages.append(ModelRequest(parts=[SystemPromptPart(content=system_prompt)]))
|
|
98
|
+
logger.debug(f"Prepended agent system prompt ({len(system_prompt)} chars) to message history")
|
|
99
|
+
|
|
100
|
+
# Track pending tool results to batch them with assistant responses
|
|
101
|
+
# When we see a tool message, we need to:
|
|
102
|
+
# 1. Add a ModelResponse with ToolCallPart (synthesized)
|
|
103
|
+
# 2. Add a ModelRequest with ToolReturnPart (actual result)
|
|
104
|
+
|
|
105
|
+
i = 0
|
|
106
|
+
while i < len(session_history):
|
|
107
|
+
msg = session_history[i]
|
|
108
|
+
role = msg.get("role", "")
|
|
109
|
+
content = msg.get("content") or ""
|
|
110
|
+
|
|
111
|
+
if role == "user":
|
|
112
|
+
# User messages become ModelRequest with UserPromptPart
|
|
113
|
+
messages.append(ModelRequest(parts=[UserPromptPart(content=content)]))
|
|
114
|
+
|
|
115
|
+
elif role == "assistant":
|
|
116
|
+
# Assistant text becomes ModelResponse with TextPart
|
|
117
|
+
# Check if there are following tool messages that should be grouped
|
|
118
|
+
tool_calls = []
|
|
119
|
+
tool_returns = []
|
|
120
|
+
|
|
121
|
+
# Look ahead for tool messages that follow this assistant message
|
|
122
|
+
j = i + 1
|
|
123
|
+
while j < len(session_history) and session_history[j].get("role") == "tool":
|
|
124
|
+
tool_msg = session_history[j]
|
|
125
|
+
tool_name = tool_msg.get("tool_name", "unknown_tool")
|
|
126
|
+
tool_call_id = tool_msg.get("tool_call_id", f"call_{j}")
|
|
127
|
+
tool_content = tool_msg.get("content") or "{}"
|
|
128
|
+
|
|
129
|
+
# tool_arguments: prefer explicit field, fallback to parsing content
|
|
130
|
+
tool_arguments = tool_msg.get("tool_arguments")
|
|
131
|
+
if tool_arguments is None and isinstance(tool_content, str) and tool_content:
|
|
132
|
+
try:
|
|
133
|
+
tool_arguments = json.loads(tool_content)
|
|
134
|
+
except json.JSONDecodeError:
|
|
135
|
+
tool_arguments = {}
|
|
136
|
+
|
|
137
|
+
# Parse tool content if it's a JSON string
|
|
138
|
+
if isinstance(tool_content, str):
|
|
139
|
+
try:
|
|
140
|
+
tool_result = json.loads(tool_content)
|
|
141
|
+
except json.JSONDecodeError:
|
|
142
|
+
tool_result = {"raw": tool_content}
|
|
143
|
+
else:
|
|
144
|
+
tool_result = tool_content
|
|
145
|
+
|
|
146
|
+
# Sanitize tool name for OpenAI API compatibility
|
|
147
|
+
safe_tool_name = _sanitize_tool_name(tool_name)
|
|
148
|
+
|
|
149
|
+
# Synthesize ToolCallPart (what the model "called")
|
|
150
|
+
tool_calls.append(ToolCallPart(
|
|
151
|
+
tool_name=safe_tool_name,
|
|
152
|
+
args=tool_arguments if tool_arguments else {},
|
|
153
|
+
tool_call_id=tool_call_id,
|
|
154
|
+
))
|
|
155
|
+
|
|
156
|
+
# Create ToolReturnPart (the actual result)
|
|
157
|
+
tool_returns.append(ToolReturnPart(
|
|
158
|
+
tool_name=safe_tool_name,
|
|
159
|
+
content=tool_result,
|
|
160
|
+
tool_call_id=tool_call_id,
|
|
161
|
+
))
|
|
162
|
+
|
|
163
|
+
j += 1
|
|
164
|
+
|
|
165
|
+
# Build the assistant's ModelResponse
|
|
166
|
+
response_parts = []
|
|
167
|
+
|
|
168
|
+
# Add tool calls first (if any)
|
|
169
|
+
response_parts.extend(tool_calls)
|
|
170
|
+
|
|
171
|
+
# Add text content (if any)
|
|
172
|
+
if content:
|
|
173
|
+
response_parts.append(TextPart(content=content))
|
|
174
|
+
|
|
175
|
+
# Only add ModelResponse if we have parts
|
|
176
|
+
if response_parts:
|
|
177
|
+
messages.append(ModelResponse(
|
|
178
|
+
parts=response_parts,
|
|
179
|
+
model_name="recovered", # We don't store model name
|
|
180
|
+
))
|
|
181
|
+
|
|
182
|
+
# Add tool returns as ModelRequest (required by LLM API)
|
|
183
|
+
if tool_returns:
|
|
184
|
+
messages.append(ModelRequest(parts=tool_returns))
|
|
185
|
+
|
|
186
|
+
# Skip the tool messages we just processed
|
|
187
|
+
i = j - 1
|
|
188
|
+
|
|
189
|
+
elif role == "tool":
|
|
190
|
+
# Orphan tool message (no preceding assistant) - synthesize both parts
|
|
191
|
+
tool_name = msg.get("tool_name", "unknown_tool")
|
|
192
|
+
tool_call_id = msg.get("tool_call_id", f"call_{i}")
|
|
193
|
+
tool_content = msg.get("content") or "{}"
|
|
194
|
+
|
|
195
|
+
# tool_arguments: prefer explicit field, fallback to parsing content
|
|
196
|
+
tool_arguments = msg.get("tool_arguments")
|
|
197
|
+
if tool_arguments is None and isinstance(tool_content, str) and tool_content:
|
|
198
|
+
try:
|
|
199
|
+
tool_arguments = json.loads(tool_content)
|
|
200
|
+
except json.JSONDecodeError:
|
|
201
|
+
tool_arguments = {}
|
|
202
|
+
|
|
203
|
+
# Parse tool content
|
|
204
|
+
if isinstance(tool_content, str):
|
|
205
|
+
try:
|
|
206
|
+
tool_result = json.loads(tool_content)
|
|
207
|
+
except json.JSONDecodeError:
|
|
208
|
+
tool_result = {"raw": tool_content}
|
|
209
|
+
else:
|
|
210
|
+
tool_result = tool_content
|
|
211
|
+
|
|
212
|
+
# Sanitize tool name for OpenAI API compatibility
|
|
213
|
+
safe_tool_name = _sanitize_tool_name(tool_name)
|
|
214
|
+
|
|
215
|
+
# Synthesize the tool call (ModelResponse with ToolCallPart)
|
|
216
|
+
messages.append(ModelResponse(
|
|
217
|
+
parts=[ToolCallPart(
|
|
218
|
+
tool_name=safe_tool_name,
|
|
219
|
+
args=tool_arguments if tool_arguments else {},
|
|
220
|
+
tool_call_id=tool_call_id,
|
|
221
|
+
)],
|
|
222
|
+
model_name="recovered",
|
|
223
|
+
))
|
|
224
|
+
|
|
225
|
+
# Add the tool return (ModelRequest with ToolReturnPart)
|
|
226
|
+
messages.append(ModelRequest(
|
|
227
|
+
parts=[ToolReturnPart(
|
|
228
|
+
tool_name=safe_tool_name,
|
|
229
|
+
content=tool_result,
|
|
230
|
+
tool_call_id=tool_call_id,
|
|
231
|
+
)]
|
|
232
|
+
))
|
|
233
|
+
|
|
234
|
+
elif role == "system":
|
|
235
|
+
# Skip system messages - pydantic-ai handles these via Agent.system_prompt
|
|
236
|
+
logger.debug("Skipping system message in session history (handled by Agent)")
|
|
237
|
+
|
|
238
|
+
else:
|
|
239
|
+
logger.warning(f"Unknown message role in session history: {role}")
|
|
240
|
+
|
|
241
|
+
i += 1
|
|
242
|
+
|
|
243
|
+
logger.debug(f"Converted {len(session_history)} stored messages to {len(messages)} pydantic-ai messages")
|
|
244
|
+
return messages
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def audit_session_history(
|
|
248
|
+
session_id: str,
|
|
249
|
+
agent_name: str,
|
|
250
|
+
prompt: str,
|
|
251
|
+
raw_session_history: list[dict[str, Any]],
|
|
252
|
+
pydantic_messages_count: int,
|
|
253
|
+
) -> None:
|
|
254
|
+
"""
|
|
255
|
+
Dump session history to a YAML file for debugging.
|
|
256
|
+
|
|
257
|
+
Only runs when DEBUG__AUDIT_SESSION=true. Writes to DEBUG__AUDIT_DIR (default /tmp).
|
|
258
|
+
Appends to the same file for a session, so all agent invocations are in one place.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
session_id: The session identifier
|
|
262
|
+
agent_name: Name of the agent being invoked
|
|
263
|
+
prompt: The prompt being sent to the agent
|
|
264
|
+
raw_session_history: The raw session messages from the database
|
|
265
|
+
pydantic_messages_count: Count of converted pydantic-ai messages
|
|
266
|
+
"""
|
|
267
|
+
from ...settings import settings
|
|
268
|
+
|
|
269
|
+
if not settings.debug.audit_session:
|
|
270
|
+
return
|
|
271
|
+
|
|
272
|
+
try:
|
|
273
|
+
import yaml
|
|
274
|
+
from pathlib import Path
|
|
275
|
+
from ...utils.date_utils import utc_now, to_iso
|
|
276
|
+
|
|
277
|
+
audit_dir = Path(settings.debug.audit_dir)
|
|
278
|
+
audit_dir.mkdir(parents=True, exist_ok=True)
|
|
279
|
+
audit_file = audit_dir / f"{session_id}.yaml"
|
|
280
|
+
|
|
281
|
+
# Create entry for this agent invocation
|
|
282
|
+
entry = {
|
|
283
|
+
"timestamp": to_iso(utc_now()),
|
|
284
|
+
"agent_name": agent_name,
|
|
285
|
+
"prompt": prompt,
|
|
286
|
+
"raw_history_count": len(raw_session_history),
|
|
287
|
+
"pydantic_messages_count": pydantic_messages_count,
|
|
288
|
+
"raw_session_history": raw_session_history,
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
# Load existing data or create new
|
|
292
|
+
existing_data: dict[str, Any] = {"session_id": session_id, "invocations": []}
|
|
293
|
+
if audit_file.exists():
|
|
294
|
+
with open(audit_file) as f:
|
|
295
|
+
loaded = yaml.safe_load(f)
|
|
296
|
+
if loaded:
|
|
297
|
+
# Ensure session_id is always present (backfill if missing)
|
|
298
|
+
existing_data = {
|
|
299
|
+
"session_id": loaded.get("session_id", session_id),
|
|
300
|
+
"invocations": loaded.get("invocations", []),
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
# Append this invocation
|
|
304
|
+
existing_data["invocations"].append(entry)
|
|
305
|
+
|
|
306
|
+
with open(audit_file, "w") as f:
|
|
307
|
+
yaml.dump(existing_data, f, default_flow_style=False, allow_unicode=True)
|
|
308
|
+
logger.info(f"DEBUG: Session audit updated: {audit_file}")
|
|
309
|
+
except Exception as e:
|
|
310
|
+
logger.warning(f"DEBUG: Failed to dump session audit: {e}")
|
rem/services/session/reload.py
CHANGED
|
@@ -12,7 +12,8 @@ Design Pattern:
|
|
|
12
12
|
|
|
13
13
|
Message Types on Reload:
|
|
14
14
|
- user: Returned as-is
|
|
15
|
-
- tool: Returned
|
|
15
|
+
- tool: Returned with metadata (tool_call_id, tool_name). tool_arguments may be in
|
|
16
|
+
metadata (parent calls) or parsed from content (child calls) by pydantic_messages.py
|
|
16
17
|
- assistant: Compressed on load if long (>400 chars), with REM LOOKUP for recovery
|
|
17
18
|
"""
|
|
18
19
|
|
rem/settings.py
CHANGED
|
@@ -424,6 +424,49 @@ class AuthSettings(BaseSettings):
|
|
|
424
424
|
google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
|
|
425
425
|
microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)
|
|
426
426
|
|
|
427
|
+
# Pre-approved login codes (bypass email verification)
|
|
428
|
+
# Format: comma-separated codes with prefix A=admin, B=normal user
|
|
429
|
+
# Example: "A12345,A67890,B11111,B22222"
|
|
430
|
+
preapproved_codes: str = Field(
|
|
431
|
+
default="",
|
|
432
|
+
description=(
|
|
433
|
+
"Comma-separated list of pre-approved login codes. "
|
|
434
|
+
"Prefix A = admin user, B = normal user. "
|
|
435
|
+
"Example: 'A12345,A67890,B11111'. "
|
|
436
|
+
"Users can login with these codes without email verification."
|
|
437
|
+
),
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
def check_preapproved_code(self, code: str) -> dict | None:
|
|
441
|
+
"""
|
|
442
|
+
Check if a code is in the pre-approved list.
|
|
443
|
+
|
|
444
|
+
Args:
|
|
445
|
+
code: The code to check (including prefix)
|
|
446
|
+
|
|
447
|
+
Returns:
|
|
448
|
+
Dict with 'role' key if valid, None if not found.
|
|
449
|
+
- A prefix -> role='admin'
|
|
450
|
+
- B prefix -> role='user'
|
|
451
|
+
"""
|
|
452
|
+
if not self.preapproved_codes:
|
|
453
|
+
return None
|
|
454
|
+
|
|
455
|
+
codes = [c.strip().upper() for c in self.preapproved_codes.split(",") if c.strip()]
|
|
456
|
+
code_upper = code.strip().upper()
|
|
457
|
+
|
|
458
|
+
if code_upper not in codes:
|
|
459
|
+
return None
|
|
460
|
+
|
|
461
|
+
# Parse prefix to determine role
|
|
462
|
+
if code_upper.startswith("A"):
|
|
463
|
+
return {"role": "admin", "code": code_upper}
|
|
464
|
+
elif code_upper.startswith("B"):
|
|
465
|
+
return {"role": "user", "code": code_upper}
|
|
466
|
+
else:
|
|
467
|
+
# Unknown prefix, treat as user
|
|
468
|
+
return {"role": "user", "code": code_upper}
|
|
469
|
+
|
|
427
470
|
@field_validator("session_secret", mode="before")
|
|
428
471
|
@classmethod
|
|
429
472
|
def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
|
|
@@ -722,7 +765,7 @@ class DataLakeSettings(BaseSettings):
|
|
|
722
765
|
│ └── cpt/ # CPT codes
|
|
723
766
|
└── calibration/ # Agent calibration
|
|
724
767
|
├── experiments/ # Experiment configs + results
|
|
725
|
-
│ └── {agent}/{task}/ # e.g.,
|
|
768
|
+
│ └── {agent}/{task}/ # e.g., rem/risk-assessment
|
|
726
769
|
└── datasets/ # Shared evaluation datasets
|
|
727
770
|
|
|
728
771
|
Experiment Storage:
|
|
@@ -1598,7 +1641,7 @@ class EmailSettings(BaseSettings):
|
|
|
1598
1641
|
"Existing users can always login regardless of domain. "
|
|
1599
1642
|
"New users must have an email from a trusted domain. "
|
|
1600
1643
|
"Empty string means all domains are allowed. "
|
|
1601
|
-
"Example: '
|
|
1644
|
+
"Example: 'mycompany.com,example.com'"
|
|
1602
1645
|
),
|
|
1603
1646
|
)
|
|
1604
1647
|
|
|
@@ -1651,6 +1694,33 @@ class EmailSettings(BaseSettings):
|
|
|
1651
1694
|
return kwargs
|
|
1652
1695
|
|
|
1653
1696
|
|
|
1697
|
+
class DebugSettings(BaseSettings):
|
|
1698
|
+
"""
|
|
1699
|
+
Debug settings for development and troubleshooting.
|
|
1700
|
+
|
|
1701
|
+
Environment variables:
|
|
1702
|
+
DEBUG__AUDIT_SESSION - Dump session history to /tmp/{session_id}.yaml
|
|
1703
|
+
DEBUG__AUDIT_DIR - Directory for session audit files (default: /tmp)
|
|
1704
|
+
"""
|
|
1705
|
+
|
|
1706
|
+
model_config = SettingsConfigDict(
|
|
1707
|
+
env_prefix="DEBUG__",
|
|
1708
|
+
env_file=".env",
|
|
1709
|
+
env_file_encoding="utf-8",
|
|
1710
|
+
extra="ignore",
|
|
1711
|
+
)
|
|
1712
|
+
|
|
1713
|
+
audit_session: bool = Field(
|
|
1714
|
+
default=False,
|
|
1715
|
+
description="When true, dump full session history to audit files for debugging",
|
|
1716
|
+
)
|
|
1717
|
+
|
|
1718
|
+
audit_dir: str = Field(
|
|
1719
|
+
default="/tmp",
|
|
1720
|
+
description="Directory for session audit files",
|
|
1721
|
+
)
|
|
1722
|
+
|
|
1723
|
+
|
|
1654
1724
|
class TestSettings(BaseSettings):
|
|
1655
1725
|
"""
|
|
1656
1726
|
Test environment settings.
|
|
@@ -1767,16 +1837,31 @@ class Settings(BaseSettings):
|
|
|
1767
1837
|
schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
|
|
1768
1838
|
email: EmailSettings = Field(default_factory=EmailSettings)
|
|
1769
1839
|
test: TestSettings = Field(default_factory=TestSettings)
|
|
1840
|
+
debug: DebugSettings = Field(default_factory=DebugSettings)
|
|
1770
1841
|
|
|
1771
1842
|
|
|
1772
|
-
# Auto-load .env file from current directory
|
|
1773
|
-
# This happens BEFORE config file loading, so .env takes precedence
|
|
1843
|
+
# Auto-load .env file from current directory or parent directories
|
|
1844
|
+
# This happens BEFORE config file loading, so .env takes precedence over shell env vars
|
|
1774
1845
|
from pathlib import Path
|
|
1775
1846
|
from dotenv import load_dotenv
|
|
1776
1847
|
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1848
|
+
|
|
1849
|
+
def _find_dotenv() -> Path | None:
|
|
1850
|
+
"""Search for .env in current dir and up to 3 parent directories."""
|
|
1851
|
+
current = Path.cwd()
|
|
1852
|
+
for _ in range(4): # Current + 3 parents
|
|
1853
|
+
env_path = current / ".env"
|
|
1854
|
+
if env_path.exists():
|
|
1855
|
+
return env_path
|
|
1856
|
+
if current.parent == current: # Reached root
|
|
1857
|
+
break
|
|
1858
|
+
current = current.parent
|
|
1859
|
+
return None
|
|
1860
|
+
|
|
1861
|
+
|
|
1862
|
+
_dotenv_path = _find_dotenv()
|
|
1863
|
+
if _dotenv_path:
|
|
1864
|
+
load_dotenv(_dotenv_path, override=True) # .env takes precedence over shell env vars
|
|
1780
1865
|
logger.debug(f"Loaded environment from {_dotenv_path.resolve()}")
|
|
1781
1866
|
|
|
1782
1867
|
# Load configuration from ~/.rem/config.yaml before initializing settings
|
|
@@ -121,18 +121,18 @@ CREATE UNLOGGED TABLE IF NOT EXISTS kv_store (
|
|
|
121
121
|
entity_key VARCHAR(255) NOT NULL,
|
|
122
122
|
entity_type VARCHAR(100) NOT NULL,
|
|
123
123
|
entity_id UUID NOT NULL,
|
|
124
|
-
tenant_id VARCHAR(100)
|
|
124
|
+
tenant_id VARCHAR(100), -- NULL = public/shared data
|
|
125
125
|
user_id VARCHAR(100),
|
|
126
126
|
content_summary TEXT,
|
|
127
127
|
metadata JSONB DEFAULT '{}',
|
|
128
128
|
graph_edges JSONB DEFAULT '[]'::jsonb, -- Cached edges for fast graph traversal
|
|
129
129
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
130
|
-
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
131
|
-
|
|
132
|
-
-- Composite primary key: entity_key unique per tenant
|
|
133
|
-
PRIMARY KEY (tenant_id, entity_key)
|
|
130
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
134
131
|
);
|
|
135
132
|
|
|
133
|
+
-- Unique constraint on (tenant_id, entity_key) using COALESCE to handle NULL tenant_id
|
|
134
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_kv_store_tenant_key ON kv_store (COALESCE(tenant_id, ''), entity_key);
|
|
135
|
+
|
|
136
136
|
-- Index for user-scoped lookups (when user_id IS NOT NULL)
|
|
137
137
|
CREATE INDEX IF NOT EXISTS idx_kv_store_user ON kv_store (tenant_id, user_id)
|
|
138
138
|
WHERE user_id IS NOT NULL;
|
|
@@ -173,7 +173,7 @@ COMMENT ON COLUMN kv_store.entity_id IS
|
|
|
173
173
|
'UUID from primary table for reverse lookup';
|
|
174
174
|
|
|
175
175
|
COMMENT ON COLUMN kv_store.tenant_id IS
|
|
176
|
-
'Tenant identifier for multi-tenancy isolation';
|
|
176
|
+
'Tenant identifier for multi-tenancy isolation. NULL = public/shared data visible to all.';
|
|
177
177
|
|
|
178
178
|
COMMENT ON COLUMN kv_store.user_id IS
|
|
179
179
|
'Optional user scoping. NULL = system-level entity, visible to all users in tenant';
|
|
@@ -271,8 +271,12 @@ BEGIN
|
|
|
271
271
|
AND kv.entity_key = normalize_key(p_entity_key)
|
|
272
272
|
LIMIT 1;
|
|
273
273
|
|
|
274
|
-
-- If not found,
|
|
274
|
+
-- If not found, check if cache is empty and maybe trigger rebuild
|
|
275
275
|
IF entity_table IS NULL THEN
|
|
276
|
+
-- SELF-HEALING: Check if this is because cache is empty
|
|
277
|
+
IF rem_kv_store_empty(effective_user_id) THEN
|
|
278
|
+
PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_lookup');
|
|
279
|
+
END IF;
|
|
276
280
|
RETURN;
|
|
277
281
|
END IF;
|
|
278
282
|
|
|
@@ -357,6 +361,7 @@ DECLARE
|
|
|
357
361
|
entities_by_table JSONB := '{}'::jsonb;
|
|
358
362
|
table_keys JSONB;
|
|
359
363
|
effective_user_id VARCHAR(100);
|
|
364
|
+
v_found_any BOOLEAN := FALSE;
|
|
360
365
|
BEGIN
|
|
361
366
|
effective_user_id := COALESCE(p_user_id, p_tenant_id);
|
|
362
367
|
|
|
@@ -373,6 +378,7 @@ BEGIN
|
|
|
373
378
|
ORDER BY sim_score DESC
|
|
374
379
|
LIMIT p_limit
|
|
375
380
|
LOOP
|
|
381
|
+
v_found_any := TRUE;
|
|
376
382
|
-- Build JSONB mapping {table: [keys]}
|
|
377
383
|
IF entities_by_table ? kv_matches.entity_type THEN
|
|
378
384
|
table_keys := entities_by_table->kv_matches.entity_type;
|
|
@@ -390,6 +396,11 @@ BEGIN
|
|
|
390
396
|
END IF;
|
|
391
397
|
END LOOP;
|
|
392
398
|
|
|
399
|
+
-- SELF-HEALING: If no matches and cache is empty, trigger rebuild
|
|
400
|
+
IF NOT v_found_any AND rem_kv_store_empty(effective_user_id) THEN
|
|
401
|
+
PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_fuzzy');
|
|
402
|
+
END IF;
|
|
403
|
+
|
|
393
404
|
-- Fetch full records using rem_fetch (which now supports NULL user_id)
|
|
394
405
|
RETURN QUERY
|
|
395
406
|
SELECT
|
|
@@ -436,9 +447,25 @@ DECLARE
|
|
|
436
447
|
entities_by_table JSONB := '{}'::jsonb;
|
|
437
448
|
table_keys JSONB;
|
|
438
449
|
effective_user_id VARCHAR(100);
|
|
450
|
+
v_found_start BOOLEAN := FALSE;
|
|
439
451
|
BEGIN
|
|
440
452
|
effective_user_id := COALESCE(p_user_id, p_tenant_id);
|
|
441
453
|
|
|
454
|
+
-- Check if start entity exists in kv_store
|
|
455
|
+
SELECT TRUE INTO v_found_start
|
|
456
|
+
FROM kv_store kv
|
|
457
|
+
WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
|
|
458
|
+
AND kv.entity_key = normalize_key(p_entity_key)
|
|
459
|
+
LIMIT 1;
|
|
460
|
+
|
|
461
|
+
-- SELF-HEALING: If start not found and cache is empty, trigger rebuild
|
|
462
|
+
IF NOT COALESCE(v_found_start, FALSE) THEN
|
|
463
|
+
IF rem_kv_store_empty(effective_user_id) THEN
|
|
464
|
+
PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_traverse');
|
|
465
|
+
END IF;
|
|
466
|
+
RETURN;
|
|
467
|
+
END IF;
|
|
468
|
+
|
|
442
469
|
FOR graph_keys IN
|
|
443
470
|
WITH RECURSIVE graph_traversal AS (
|
|
444
471
|
-- Base case: Find starting entity (user-owned OR public)
|
|
@@ -789,6 +816,97 @@ $$ LANGUAGE plpgsql STABLE;
|
|
|
789
816
|
COMMENT ON FUNCTION fn_get_shared_messages IS
|
|
790
817
|
'Get messages from sessions shared by a specific user with the recipient.';
|
|
791
818
|
|
|
819
|
+
-- ============================================================================
|
|
820
|
+
-- SESSIONS WITH USER INFO
|
|
821
|
+
-- ============================================================================
|
|
822
|
+
-- Function to list sessions with user details (name, email) for admin views
|
|
823
|
+
|
|
824
|
+
-- List sessions with user info, CTE pagination
|
|
825
|
+
-- Note: messages.session_id stores the session UUID (sessions.id)
|
|
826
|
+
CREATE OR REPLACE FUNCTION fn_list_sessions_with_user(
|
|
827
|
+
p_user_id VARCHAR(256) DEFAULT NULL, -- Filter by user_id (NULL = all users, admin only)
|
|
828
|
+
p_user_name VARCHAR(256) DEFAULT NULL, -- Filter by user name (partial match, admin only)
|
|
829
|
+
p_user_email VARCHAR(256) DEFAULT NULL, -- Filter by user email (partial match, admin only)
|
|
830
|
+
p_mode VARCHAR(50) DEFAULT NULL, -- Filter by session mode
|
|
831
|
+
p_page INTEGER DEFAULT 1,
|
|
832
|
+
p_page_size INTEGER DEFAULT 50
|
|
833
|
+
)
|
|
834
|
+
RETURNS TABLE(
|
|
835
|
+
id UUID,
|
|
836
|
+
name VARCHAR(256),
|
|
837
|
+
mode TEXT,
|
|
838
|
+
description TEXT,
|
|
839
|
+
user_id VARCHAR(256),
|
|
840
|
+
user_name VARCHAR(256),
|
|
841
|
+
user_email VARCHAR(256),
|
|
842
|
+
message_count INTEGER,
|
|
843
|
+
total_tokens INTEGER,
|
|
844
|
+
created_at TIMESTAMP,
|
|
845
|
+
updated_at TIMESTAMP,
|
|
846
|
+
metadata JSONB,
|
|
847
|
+
total_count BIGINT
|
|
848
|
+
) AS $$
|
|
849
|
+
BEGIN
|
|
850
|
+
RETURN QUERY
|
|
851
|
+
WITH session_msg_counts AS (
|
|
852
|
+
-- Count messages per session (joining on session UUID)
|
|
853
|
+
SELECT
|
|
854
|
+
m.session_id,
|
|
855
|
+
COUNT(*)::INTEGER as actual_message_count
|
|
856
|
+
FROM messages m
|
|
857
|
+
GROUP BY m.session_id
|
|
858
|
+
),
|
|
859
|
+
filtered_sessions AS (
|
|
860
|
+
SELECT
|
|
861
|
+
s.id,
|
|
862
|
+
s.name,
|
|
863
|
+
s.mode,
|
|
864
|
+
s.description,
|
|
865
|
+
s.user_id,
|
|
866
|
+
COALESCE(u.name, s.user_id)::VARCHAR(256) AS user_name,
|
|
867
|
+
u.email::VARCHAR(256) AS user_email,
|
|
868
|
+
COALESCE(mc.actual_message_count, 0) AS message_count,
|
|
869
|
+
s.total_tokens,
|
|
870
|
+
s.created_at,
|
|
871
|
+
s.updated_at,
|
|
872
|
+
s.metadata
|
|
873
|
+
FROM sessions s
|
|
874
|
+
LEFT JOIN users u ON u.id::text = s.user_id
|
|
875
|
+
LEFT JOIN session_msg_counts mc ON mc.session_id = s.id::text
|
|
876
|
+
WHERE s.deleted_at IS NULL
|
|
877
|
+
AND (p_user_id IS NULL OR s.user_id = p_user_id)
|
|
878
|
+
AND (p_user_name IS NULL OR u.name ILIKE '%' || p_user_name || '%')
|
|
879
|
+
AND (p_user_email IS NULL OR u.email ILIKE '%' || p_user_email || '%')
|
|
880
|
+
AND (p_mode IS NULL OR s.mode = p_mode)
|
|
881
|
+
),
|
|
882
|
+
counted AS (
|
|
883
|
+
SELECT *, COUNT(*) OVER () AS total_count
|
|
884
|
+
FROM filtered_sessions
|
|
885
|
+
)
|
|
886
|
+
SELECT
|
|
887
|
+
c.id,
|
|
888
|
+
c.name,
|
|
889
|
+
c.mode,
|
|
890
|
+
c.description,
|
|
891
|
+
c.user_id,
|
|
892
|
+
c.user_name,
|
|
893
|
+
c.user_email,
|
|
894
|
+
c.message_count,
|
|
895
|
+
c.total_tokens,
|
|
896
|
+
c.created_at,
|
|
897
|
+
c.updated_at,
|
|
898
|
+
c.metadata,
|
|
899
|
+
c.total_count
|
|
900
|
+
FROM counted c
|
|
901
|
+
ORDER BY c.created_at DESC
|
|
902
|
+
LIMIT p_page_size
|
|
903
|
+
OFFSET (p_page - 1) * p_page_size;
|
|
904
|
+
END;
|
|
905
|
+
$$ LANGUAGE plpgsql STABLE;
|
|
906
|
+
|
|
907
|
+
COMMENT ON FUNCTION fn_list_sessions_with_user IS
|
|
908
|
+
'List sessions with user details and computed message counts. Joins messages on session name.';
|
|
909
|
+
|
|
792
910
|
-- ============================================================================
|
|
793
911
|
-- RECORD INSTALLATION
|
|
794
912
|
-- ============================================================================
|