remdb 0.3.180__py3-none-any.whl → 0.3.258__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. rem/agentic/README.md +36 -2
  2. rem/agentic/__init__.py +10 -1
  3. rem/agentic/context.py +185 -1
  4. rem/agentic/context_builder.py +56 -35
  5. rem/agentic/mcp/tool_wrapper.py +2 -2
  6. rem/agentic/providers/pydantic_ai.py +303 -111
  7. rem/agentic/schema.py +2 -2
  8. rem/api/main.py +1 -1
  9. rem/api/mcp_router/resources.py +223 -0
  10. rem/api/mcp_router/server.py +4 -0
  11. rem/api/mcp_router/tools.py +608 -166
  12. rem/api/routers/admin.py +30 -4
  13. rem/api/routers/auth.py +219 -20
  14. rem/api/routers/chat/child_streaming.py +393 -0
  15. rem/api/routers/chat/completions.py +77 -40
  16. rem/api/routers/chat/sse_events.py +7 -3
  17. rem/api/routers/chat/streaming.py +381 -291
  18. rem/api/routers/chat/streaming_utils.py +325 -0
  19. rem/api/routers/common.py +18 -0
  20. rem/api/routers/dev.py +7 -1
  21. rem/api/routers/feedback.py +11 -3
  22. rem/api/routers/messages.py +176 -38
  23. rem/api/routers/models.py +9 -1
  24. rem/api/routers/query.py +17 -15
  25. rem/api/routers/shared_sessions.py +16 -0
  26. rem/auth/jwt.py +19 -4
  27. rem/auth/middleware.py +42 -28
  28. rem/cli/README.md +62 -0
  29. rem/cli/commands/ask.py +205 -114
  30. rem/cli/commands/db.py +55 -31
  31. rem/cli/commands/experiments.py +1 -1
  32. rem/cli/commands/process.py +179 -43
  33. rem/cli/commands/query.py +109 -0
  34. rem/cli/commands/session.py +117 -0
  35. rem/cli/main.py +2 -0
  36. rem/models/core/experiment.py +1 -1
  37. rem/models/entities/ontology.py +18 -20
  38. rem/models/entities/session.py +1 -0
  39. rem/schemas/agents/core/agent-builder.yaml +1 -1
  40. rem/schemas/agents/rem.yaml +1 -1
  41. rem/schemas/agents/test_orchestrator.yaml +42 -0
  42. rem/schemas/agents/test_structured_output.yaml +52 -0
  43. rem/services/content/providers.py +151 -49
  44. rem/services/content/service.py +18 -5
  45. rem/services/embeddings/worker.py +26 -12
  46. rem/services/postgres/__init__.py +28 -3
  47. rem/services/postgres/diff_service.py +57 -5
  48. rem/services/postgres/programmable_diff_service.py +635 -0
  49. rem/services/postgres/pydantic_to_sqlalchemy.py +2 -2
  50. rem/services/postgres/register_type.py +11 -10
  51. rem/services/postgres/repository.py +39 -28
  52. rem/services/postgres/schema_generator.py +5 -5
  53. rem/services/postgres/sql_builder.py +6 -5
  54. rem/services/rem/README.md +4 -3
  55. rem/services/rem/parser.py +7 -10
  56. rem/services/rem/service.py +47 -0
  57. rem/services/session/__init__.py +8 -1
  58. rem/services/session/compression.py +47 -5
  59. rem/services/session/pydantic_messages.py +310 -0
  60. rem/services/session/reload.py +2 -1
  61. rem/settings.py +92 -7
  62. rem/sql/migrations/001_install.sql +125 -7
  63. rem/sql/migrations/002_install_models.sql +159 -149
  64. rem/sql/migrations/004_cache_system.sql +10 -276
  65. rem/sql/migrations/migrate_session_id_to_uuid.sql +45 -0
  66. rem/utils/schema_loader.py +180 -120
  67. {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/METADATA +7 -6
  68. {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/RECORD +70 -61
  69. {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/WHEEL +0 -0
  70. {remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,310 @@
1
+ """Convert stored session messages to pydantic-ai native message format.
2
+
3
+ This module enables proper conversation history replay by converting our simplified
4
+ storage format into pydantic-ai's native ModelRequest/ModelResponse types.
5
+
6
+ Key insight: When we store tool results, we only store the result (ToolReturnPart).
7
+ But LLM APIs require matching ToolCallPart for each ToolReturnPart. So we synthesize
8
+ the ToolCallPart from stored metadata (tool_name, tool_call_id) and arguments.
9
+
10
+ Tool arguments can come from two places:
11
+ - Parent tool calls (ask_agent): tool_arguments stored in metadata (content = result)
12
+ - Child tool calls (register_metadata): arguments parsed from content (content = args as JSON)
13
+
14
+ Storage format (our simplified format):
15
+ {"role": "user", "content": "..."}
16
+ {"role": "assistant", "content": "..."}
17
+ {"role": "tool", "content": "{...}", "tool_name": "...", "tool_call_id": "...", "tool_arguments": {...}} # optional
18
+
19
+ Pydantic-ai format (what the LLM expects):
20
+ ModelRequest(parts=[UserPromptPart(content="...")])
21
+ ModelResponse(parts=[TextPart(content="..."), ToolCallPart(...)]) # Call
22
+ ModelRequest(parts=[ToolReturnPart(...)]) # Result
23
+
24
+ Example usage:
25
+ from rem.services.session.pydantic_messages import session_to_pydantic_messages
26
+
27
+ # Load session history
28
+ session_history = await store.load_session_messages(session_id)
29
+
30
+ # Convert to pydantic-ai format
31
+ message_history = session_to_pydantic_messages(session_history)
32
+
33
+ # Use with agent.run()
34
+ result = await agent.run(user_prompt, message_history=message_history)
35
+ """
36
+
37
+ import json
38
+ import re
39
+ from typing import Any
40
+
41
+ from loguru import logger
42
+ from pydantic_ai.messages import (
43
+ ModelMessage,
44
+ ModelRequest,
45
+ ModelResponse,
46
+ SystemPromptPart,
47
+ TextPart,
48
+ ToolCallPart,
49
+ ToolReturnPart,
50
+ UserPromptPart,
51
+ )
52
+
53
+
54
+ def _sanitize_tool_name(tool_name: str) -> str:
55
+ """Sanitize tool name for OpenAI API compatibility.
56
+
57
+ OpenAI requires tool names to match pattern: ^[a-zA-Z0-9_-]+$
58
+ This replaces invalid characters (like colons) with underscores.
59
+ """
60
+ return re.sub(r'[^a-zA-Z0-9_-]', '_', tool_name)
61
+
62
+
63
+ def session_to_pydantic_messages(
64
+ session_history: list[dict[str, Any]],
65
+ system_prompt: str | None = None,
66
+ ) -> list[ModelMessage]:
67
+ """Convert stored session messages to pydantic-ai ModelMessage format.
68
+
69
+ Handles the conversion of our simplified storage format to pydantic-ai's
70
+ native message types, including synthesizing ToolCallPart for tool results.
71
+
72
+ IMPORTANT: pydantic-ai only auto-adds system prompts when message_history is empty.
73
+ When passing message_history to agent.run(), you MUST include the system prompt
74
+ via the system_prompt parameter here.
75
+
76
+ Args:
77
+ session_history: List of message dicts from SessionMessageStore.load_session_messages()
78
+ Each dict has: role, content, and optionally tool_name, tool_call_id, tool_arguments
79
+ system_prompt: The agent's system prompt (from schema description). This is REQUIRED
80
+ for proper agent behavior on subsequent turns, as pydantic-ai won't add it
81
+ automatically when message_history is provided.
82
+
83
+ Returns:
84
+ List of ModelMessage (ModelRequest | ModelResponse) ready for agent.run(message_history=...)
85
+
86
+ Note:
87
+ - System prompts ARE included as SystemPromptPart when system_prompt is provided
88
+ - Tool results require synthesized ToolCallPart to satisfy LLM API requirements
89
+ - The first message in session_history should be "user" role (from context builder)
90
+ """
91
+ messages: list[ModelMessage] = []
92
+
93
+ # CRITICAL: Prepend agent's system prompt if provided
94
+ # This ensures the agent's instructions are present on every turn
95
+ # pydantic-ai only auto-adds system prompts when message_history is empty
96
+ if system_prompt:
97
+ messages.append(ModelRequest(parts=[SystemPromptPart(content=system_prompt)]))
98
+ logger.debug(f"Prepended agent system prompt ({len(system_prompt)} chars) to message history")
99
+
100
+ # Track pending tool results to batch them with assistant responses
101
+ # When we see a tool message, we need to:
102
+ # 1. Add a ModelResponse with ToolCallPart (synthesized)
103
+ # 2. Add a ModelRequest with ToolReturnPart (actual result)
104
+
105
+ i = 0
106
+ while i < len(session_history):
107
+ msg = session_history[i]
108
+ role = msg.get("role", "")
109
+ content = msg.get("content") or ""
110
+
111
+ if role == "user":
112
+ # User messages become ModelRequest with UserPromptPart
113
+ messages.append(ModelRequest(parts=[UserPromptPart(content=content)]))
114
+
115
+ elif role == "assistant":
116
+ # Assistant text becomes ModelResponse with TextPart
117
+ # Check if there are following tool messages that should be grouped
118
+ tool_calls = []
119
+ tool_returns = []
120
+
121
+ # Look ahead for tool messages that follow this assistant message
122
+ j = i + 1
123
+ while j < len(session_history) and session_history[j].get("role") == "tool":
124
+ tool_msg = session_history[j]
125
+ tool_name = tool_msg.get("tool_name", "unknown_tool")
126
+ tool_call_id = tool_msg.get("tool_call_id", f"call_{j}")
127
+ tool_content = tool_msg.get("content") or "{}"
128
+
129
+ # tool_arguments: prefer explicit field, fallback to parsing content
130
+ tool_arguments = tool_msg.get("tool_arguments")
131
+ if tool_arguments is None and isinstance(tool_content, str) and tool_content:
132
+ try:
133
+ tool_arguments = json.loads(tool_content)
134
+ except json.JSONDecodeError:
135
+ tool_arguments = {}
136
+
137
+ # Parse tool content if it's a JSON string
138
+ if isinstance(tool_content, str):
139
+ try:
140
+ tool_result = json.loads(tool_content)
141
+ except json.JSONDecodeError:
142
+ tool_result = {"raw": tool_content}
143
+ else:
144
+ tool_result = tool_content
145
+
146
+ # Sanitize tool name for OpenAI API compatibility
147
+ safe_tool_name = _sanitize_tool_name(tool_name)
148
+
149
+ # Synthesize ToolCallPart (what the model "called")
150
+ tool_calls.append(ToolCallPart(
151
+ tool_name=safe_tool_name,
152
+ args=tool_arguments if tool_arguments else {},
153
+ tool_call_id=tool_call_id,
154
+ ))
155
+
156
+ # Create ToolReturnPart (the actual result)
157
+ tool_returns.append(ToolReturnPart(
158
+ tool_name=safe_tool_name,
159
+ content=tool_result,
160
+ tool_call_id=tool_call_id,
161
+ ))
162
+
163
+ j += 1
164
+
165
+ # Build the assistant's ModelResponse
166
+ response_parts = []
167
+
168
+ # Add tool calls first (if any)
169
+ response_parts.extend(tool_calls)
170
+
171
+ # Add text content (if any)
172
+ if content:
173
+ response_parts.append(TextPart(content=content))
174
+
175
+ # Only add ModelResponse if we have parts
176
+ if response_parts:
177
+ messages.append(ModelResponse(
178
+ parts=response_parts,
179
+ model_name="recovered", # We don't store model name
180
+ ))
181
+
182
+ # Add tool returns as ModelRequest (required by LLM API)
183
+ if tool_returns:
184
+ messages.append(ModelRequest(parts=tool_returns))
185
+
186
+ # Skip the tool messages we just processed
187
+ i = j - 1
188
+
189
+ elif role == "tool":
190
+ # Orphan tool message (no preceding assistant) - synthesize both parts
191
+ tool_name = msg.get("tool_name", "unknown_tool")
192
+ tool_call_id = msg.get("tool_call_id", f"call_{i}")
193
+ tool_content = msg.get("content") or "{}"
194
+
195
+ # tool_arguments: prefer explicit field, fallback to parsing content
196
+ tool_arguments = msg.get("tool_arguments")
197
+ if tool_arguments is None and isinstance(tool_content, str) and tool_content:
198
+ try:
199
+ tool_arguments = json.loads(tool_content)
200
+ except json.JSONDecodeError:
201
+ tool_arguments = {}
202
+
203
+ # Parse tool content
204
+ if isinstance(tool_content, str):
205
+ try:
206
+ tool_result = json.loads(tool_content)
207
+ except json.JSONDecodeError:
208
+ tool_result = {"raw": tool_content}
209
+ else:
210
+ tool_result = tool_content
211
+
212
+ # Sanitize tool name for OpenAI API compatibility
213
+ safe_tool_name = _sanitize_tool_name(tool_name)
214
+
215
+ # Synthesize the tool call (ModelResponse with ToolCallPart)
216
+ messages.append(ModelResponse(
217
+ parts=[ToolCallPart(
218
+ tool_name=safe_tool_name,
219
+ args=tool_arguments if tool_arguments else {},
220
+ tool_call_id=tool_call_id,
221
+ )],
222
+ model_name="recovered",
223
+ ))
224
+
225
+ # Add the tool return (ModelRequest with ToolReturnPart)
226
+ messages.append(ModelRequest(
227
+ parts=[ToolReturnPart(
228
+ tool_name=safe_tool_name,
229
+ content=tool_result,
230
+ tool_call_id=tool_call_id,
231
+ )]
232
+ ))
233
+
234
+ elif role == "system":
235
+ # Skip system messages - pydantic-ai handles these via Agent.system_prompt
236
+ logger.debug("Skipping system message in session history (handled by Agent)")
237
+
238
+ else:
239
+ logger.warning(f"Unknown message role in session history: {role}")
240
+
241
+ i += 1
242
+
243
+ logger.debug(f"Converted {len(session_history)} stored messages to {len(messages)} pydantic-ai messages")
244
+ return messages
245
+
246
+
247
+ def audit_session_history(
248
+ session_id: str,
249
+ agent_name: str,
250
+ prompt: str,
251
+ raw_session_history: list[dict[str, Any]],
252
+ pydantic_messages_count: int,
253
+ ) -> None:
254
+ """
255
+ Dump session history to a YAML file for debugging.
256
+
257
+ Only runs when DEBUG__AUDIT_SESSION=true. Writes to DEBUG__AUDIT_DIR (default /tmp).
258
+ Appends to the same file for a session, so all agent invocations are in one place.
259
+
260
+ Args:
261
+ session_id: The session identifier
262
+ agent_name: Name of the agent being invoked
263
+ prompt: The prompt being sent to the agent
264
+ raw_session_history: The raw session messages from the database
265
+ pydantic_messages_count: Count of converted pydantic-ai messages
266
+ """
267
+ from ...settings import settings
268
+
269
+ if not settings.debug.audit_session:
270
+ return
271
+
272
+ try:
273
+ import yaml
274
+ from pathlib import Path
275
+ from ...utils.date_utils import utc_now, to_iso
276
+
277
+ audit_dir = Path(settings.debug.audit_dir)
278
+ audit_dir.mkdir(parents=True, exist_ok=True)
279
+ audit_file = audit_dir / f"{session_id}.yaml"
280
+
281
+ # Create entry for this agent invocation
282
+ entry = {
283
+ "timestamp": to_iso(utc_now()),
284
+ "agent_name": agent_name,
285
+ "prompt": prompt,
286
+ "raw_history_count": len(raw_session_history),
287
+ "pydantic_messages_count": pydantic_messages_count,
288
+ "raw_session_history": raw_session_history,
289
+ }
290
+
291
+ # Load existing data or create new
292
+ existing_data: dict[str, Any] = {"session_id": session_id, "invocations": []}
293
+ if audit_file.exists():
294
+ with open(audit_file) as f:
295
+ loaded = yaml.safe_load(f)
296
+ if loaded:
297
+ # Ensure session_id is always present (backfill if missing)
298
+ existing_data = {
299
+ "session_id": loaded.get("session_id", session_id),
300
+ "invocations": loaded.get("invocations", []),
301
+ }
302
+
303
+ # Append this invocation
304
+ existing_data["invocations"].append(entry)
305
+
306
+ with open(audit_file, "w") as f:
307
+ yaml.dump(existing_data, f, default_flow_style=False, allow_unicode=True)
308
+ logger.info(f"DEBUG: Session audit updated: {audit_file}")
309
+ except Exception as e:
310
+ logger.warning(f"DEBUG: Failed to dump session audit: {e}")
@@ -12,7 +12,8 @@ Design Pattern:
12
12
 
13
13
  Message Types on Reload:
14
14
  - user: Returned as-is
15
- - tool: Returned as-is with metadata (tool_call_id, tool_name, tool_arguments)
15
+ - tool: Returned with metadata (tool_call_id, tool_name). tool_arguments may be in
16
+ metadata (parent calls) or parsed from content (child calls) by pydantic_messages.py
16
17
  - assistant: Compressed on load if long (>400 chars), with REM LOOKUP for recovery
17
18
  """
18
19
 
rem/settings.py CHANGED
@@ -424,6 +424,49 @@ class AuthSettings(BaseSettings):
424
424
  google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
425
425
  microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)
426
426
 
427
+ # Pre-approved login codes (bypass email verification)
428
+ # Format: comma-separated codes with prefix A=admin, B=normal user
429
+ # Example: "A12345,A67890,B11111,B22222"
430
+ preapproved_codes: str = Field(
431
+ default="",
432
+ description=(
433
+ "Comma-separated list of pre-approved login codes. "
434
+ "Prefix A = admin user, B = normal user. "
435
+ "Example: 'A12345,A67890,B11111'. "
436
+ "Users can login with these codes without email verification."
437
+ ),
438
+ )
439
+
440
+ def check_preapproved_code(self, code: str) -> dict | None:
441
+ """
442
+ Check if a code is in the pre-approved list.
443
+
444
+ Args:
445
+ code: The code to check (including prefix)
446
+
447
+ Returns:
448
+ Dict with 'role' key if valid, None if not found.
449
+ - A prefix -> role='admin'
450
+ - B prefix -> role='user'
451
+ """
452
+ if not self.preapproved_codes:
453
+ return None
454
+
455
+ codes = [c.strip().upper() for c in self.preapproved_codes.split(",") if c.strip()]
456
+ code_upper = code.strip().upper()
457
+
458
+ if code_upper not in codes:
459
+ return None
460
+
461
+ # Parse prefix to determine role
462
+ if code_upper.startswith("A"):
463
+ return {"role": "admin", "code": code_upper}
464
+ elif code_upper.startswith("B"):
465
+ return {"role": "user", "code": code_upper}
466
+ else:
467
+ # Unknown prefix, treat as user
468
+ return {"role": "user", "code": code_upper}
469
+
427
470
  @field_validator("session_secret", mode="before")
428
471
  @classmethod
429
472
  def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
@@ -722,7 +765,7 @@ class DataLakeSettings(BaseSettings):
722
765
  │ └── cpt/ # CPT codes
723
766
  └── calibration/ # Agent calibration
724
767
  ├── experiments/ # Experiment configs + results
725
- │ └── {agent}/{task}/ # e.g., siggy/risk-assessment
768
+ │ └── {agent}/{task}/ # e.g., rem/risk-assessment
726
769
  └── datasets/ # Shared evaluation datasets
727
770
 
728
771
  Experiment Storage:
@@ -1598,7 +1641,7 @@ class EmailSettings(BaseSettings):
1598
1641
  "Existing users can always login regardless of domain. "
1599
1642
  "New users must have an email from a trusted domain. "
1600
1643
  "Empty string means all domains are allowed. "
1601
- "Example: 'siggymd.ai,example.com'"
1644
+ "Example: 'mycompany.com,example.com'"
1602
1645
  ),
1603
1646
  )
1604
1647
 
@@ -1651,6 +1694,33 @@ class EmailSettings(BaseSettings):
1651
1694
  return kwargs
1652
1695
 
1653
1696
 
1697
+ class DebugSettings(BaseSettings):
1698
+ """
1699
+ Debug settings for development and troubleshooting.
1700
+
1701
+ Environment variables:
1702
+ DEBUG__AUDIT_SESSION - Dump session history to /tmp/{session_id}.yaml
1703
+ DEBUG__AUDIT_DIR - Directory for session audit files (default: /tmp)
1704
+ """
1705
+
1706
+ model_config = SettingsConfigDict(
1707
+ env_prefix="DEBUG__",
1708
+ env_file=".env",
1709
+ env_file_encoding="utf-8",
1710
+ extra="ignore",
1711
+ )
1712
+
1713
+ audit_session: bool = Field(
1714
+ default=False,
1715
+ description="When true, dump full session history to audit files for debugging",
1716
+ )
1717
+
1718
+ audit_dir: str = Field(
1719
+ default="/tmp",
1720
+ description="Directory for session audit files",
1721
+ )
1722
+
1723
+
1654
1724
  class TestSettings(BaseSettings):
1655
1725
  """
1656
1726
  Test environment settings.
@@ -1767,16 +1837,31 @@ class Settings(BaseSettings):
1767
1837
  schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
1768
1838
  email: EmailSettings = Field(default_factory=EmailSettings)
1769
1839
  test: TestSettings = Field(default_factory=TestSettings)
1840
+ debug: DebugSettings = Field(default_factory=DebugSettings)
1770
1841
 
1771
1842
 
1772
- # Auto-load .env file from current directory if it exists
1773
- # This happens BEFORE config file loading, so .env takes precedence
1843
+ # Auto-load .env file from current directory or parent directories
1844
+ # This happens BEFORE config file loading, so .env takes precedence over shell env vars
1774
1845
  from pathlib import Path
1775
1846
  from dotenv import load_dotenv
1776
1847
 
1777
- _dotenv_path = Path(".env")
1778
- if _dotenv_path.exists():
1779
- load_dotenv(_dotenv_path, override=False) # Don't override existing env vars
1848
+
1849
+ def _find_dotenv() -> Path | None:
1850
+ """Search for .env in current dir and up to 3 parent directories."""
1851
+ current = Path.cwd()
1852
+ for _ in range(4): # Current + 3 parents
1853
+ env_path = current / ".env"
1854
+ if env_path.exists():
1855
+ return env_path
1856
+ if current.parent == current: # Reached root
1857
+ break
1858
+ current = current.parent
1859
+ return None
1860
+
1861
+
1862
+ _dotenv_path = _find_dotenv()
1863
+ if _dotenv_path:
1864
+ load_dotenv(_dotenv_path, override=True) # .env takes precedence over shell env vars
1780
1865
  logger.debug(f"Loaded environment from {_dotenv_path.resolve()}")
1781
1866
 
1782
1867
  # Load configuration from ~/.rem/config.yaml before initializing settings
@@ -121,18 +121,18 @@ CREATE UNLOGGED TABLE IF NOT EXISTS kv_store (
121
121
  entity_key VARCHAR(255) NOT NULL,
122
122
  entity_type VARCHAR(100) NOT NULL,
123
123
  entity_id UUID NOT NULL,
124
- tenant_id VARCHAR(100) NOT NULL,
124
+ tenant_id VARCHAR(100), -- NULL = public/shared data
125
125
  user_id VARCHAR(100),
126
126
  content_summary TEXT,
127
127
  metadata JSONB DEFAULT '{}',
128
128
  graph_edges JSONB DEFAULT '[]'::jsonb, -- Cached edges for fast graph traversal
129
129
  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
130
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
131
-
132
- -- Composite primary key: entity_key unique per tenant
133
- PRIMARY KEY (tenant_id, entity_key)
130
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
134
131
  );
135
132
 
133
+ -- Unique constraint on (tenant_id, entity_key) using COALESCE to handle NULL tenant_id
134
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_kv_store_tenant_key ON kv_store (COALESCE(tenant_id, ''), entity_key);
135
+
136
136
  -- Index for user-scoped lookups (when user_id IS NOT NULL)
137
137
  CREATE INDEX IF NOT EXISTS idx_kv_store_user ON kv_store (tenant_id, user_id)
138
138
  WHERE user_id IS NOT NULL;
@@ -173,7 +173,7 @@ COMMENT ON COLUMN kv_store.entity_id IS
173
173
  'UUID from primary table for reverse lookup';
174
174
 
175
175
  COMMENT ON COLUMN kv_store.tenant_id IS
176
- 'Tenant identifier for multi-tenancy isolation';
176
+ 'Tenant identifier for multi-tenancy isolation. NULL = public/shared data visible to all.';
177
177
 
178
178
  COMMENT ON COLUMN kv_store.user_id IS
179
179
  'Optional user scoping. NULL = system-level entity, visible to all users in tenant';
@@ -271,8 +271,12 @@ BEGIN
271
271
  AND kv.entity_key = normalize_key(p_entity_key)
272
272
  LIMIT 1;
273
273
 
274
- -- If not found, return empty
274
+ -- If not found, check if cache is empty and maybe trigger rebuild
275
275
  IF entity_table IS NULL THEN
276
+ -- SELF-HEALING: Check if this is because cache is empty
277
+ IF rem_kv_store_empty(effective_user_id) THEN
278
+ PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_lookup');
279
+ END IF;
276
280
  RETURN;
277
281
  END IF;
278
282
 
@@ -357,6 +361,7 @@ DECLARE
357
361
  entities_by_table JSONB := '{}'::jsonb;
358
362
  table_keys JSONB;
359
363
  effective_user_id VARCHAR(100);
364
+ v_found_any BOOLEAN := FALSE;
360
365
  BEGIN
361
366
  effective_user_id := COALESCE(p_user_id, p_tenant_id);
362
367
 
@@ -373,6 +378,7 @@ BEGIN
373
378
  ORDER BY sim_score DESC
374
379
  LIMIT p_limit
375
380
  LOOP
381
+ v_found_any := TRUE;
376
382
  -- Build JSONB mapping {table: [keys]}
377
383
  IF entities_by_table ? kv_matches.entity_type THEN
378
384
  table_keys := entities_by_table->kv_matches.entity_type;
@@ -390,6 +396,11 @@ BEGIN
390
396
  END IF;
391
397
  END LOOP;
392
398
 
399
+ -- SELF-HEALING: If no matches and cache is empty, trigger rebuild
400
+ IF NOT v_found_any AND rem_kv_store_empty(effective_user_id) THEN
401
+ PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_fuzzy');
402
+ END IF;
403
+
393
404
  -- Fetch full records using rem_fetch (which now supports NULL user_id)
394
405
  RETURN QUERY
395
406
  SELECT
@@ -436,9 +447,25 @@ DECLARE
436
447
  entities_by_table JSONB := '{}'::jsonb;
437
448
  table_keys JSONB;
438
449
  effective_user_id VARCHAR(100);
450
+ v_found_start BOOLEAN := FALSE;
439
451
  BEGIN
440
452
  effective_user_id := COALESCE(p_user_id, p_tenant_id);
441
453
 
454
+ -- Check if start entity exists in kv_store
455
+ SELECT TRUE INTO v_found_start
456
+ FROM kv_store kv
457
+ WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
458
+ AND kv.entity_key = normalize_key(p_entity_key)
459
+ LIMIT 1;
460
+
461
+ -- SELF-HEALING: If start not found and cache is empty, trigger rebuild
462
+ IF NOT COALESCE(v_found_start, FALSE) THEN
463
+ IF rem_kv_store_empty(effective_user_id) THEN
464
+ PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_traverse');
465
+ END IF;
466
+ RETURN;
467
+ END IF;
468
+
442
469
  FOR graph_keys IN
443
470
  WITH RECURSIVE graph_traversal AS (
444
471
  -- Base case: Find starting entity (user-owned OR public)
@@ -789,6 +816,97 @@ $$ LANGUAGE plpgsql STABLE;
789
816
  COMMENT ON FUNCTION fn_get_shared_messages IS
790
817
  'Get messages from sessions shared by a specific user with the recipient.';
791
818
 
819
+ -- ============================================================================
820
+ -- SESSIONS WITH USER INFO
821
+ -- ============================================================================
822
+ -- Function to list sessions with user details (name, email) for admin views
823
+
824
+ -- List sessions with user info, CTE pagination
825
+ -- Note: messages.session_id stores the session UUID (sessions.id)
826
+ CREATE OR REPLACE FUNCTION fn_list_sessions_with_user(
827
+ p_user_id VARCHAR(256) DEFAULT NULL, -- Filter by user_id (NULL = all users, admin only)
828
+ p_user_name VARCHAR(256) DEFAULT NULL, -- Filter by user name (partial match, admin only)
829
+ p_user_email VARCHAR(256) DEFAULT NULL, -- Filter by user email (partial match, admin only)
830
+ p_mode VARCHAR(50) DEFAULT NULL, -- Filter by session mode
831
+ p_page INTEGER DEFAULT 1,
832
+ p_page_size INTEGER DEFAULT 50
833
+ )
834
+ RETURNS TABLE(
835
+ id UUID,
836
+ name VARCHAR(256),
837
+ mode TEXT,
838
+ description TEXT,
839
+ user_id VARCHAR(256),
840
+ user_name VARCHAR(256),
841
+ user_email VARCHAR(256),
842
+ message_count INTEGER,
843
+ total_tokens INTEGER,
844
+ created_at TIMESTAMP,
845
+ updated_at TIMESTAMP,
846
+ metadata JSONB,
847
+ total_count BIGINT
848
+ ) AS $$
849
+ BEGIN
850
+ RETURN QUERY
851
+ WITH session_msg_counts AS (
852
+ -- Count messages per session (joining on session UUID)
853
+ SELECT
854
+ m.session_id,
855
+ COUNT(*)::INTEGER as actual_message_count
856
+ FROM messages m
857
+ GROUP BY m.session_id
858
+ ),
859
+ filtered_sessions AS (
860
+ SELECT
861
+ s.id,
862
+ s.name,
863
+ s.mode,
864
+ s.description,
865
+ s.user_id,
866
+ COALESCE(u.name, s.user_id)::VARCHAR(256) AS user_name,
867
+ u.email::VARCHAR(256) AS user_email,
868
+ COALESCE(mc.actual_message_count, 0) AS message_count,
869
+ s.total_tokens,
870
+ s.created_at,
871
+ s.updated_at,
872
+ s.metadata
873
+ FROM sessions s
874
+ LEFT JOIN users u ON u.id::text = s.user_id
875
+ LEFT JOIN session_msg_counts mc ON mc.session_id = s.id::text
876
+ WHERE s.deleted_at IS NULL
877
+ AND (p_user_id IS NULL OR s.user_id = p_user_id)
878
+ AND (p_user_name IS NULL OR u.name ILIKE '%' || p_user_name || '%')
879
+ AND (p_user_email IS NULL OR u.email ILIKE '%' || p_user_email || '%')
880
+ AND (p_mode IS NULL OR s.mode = p_mode)
881
+ ),
882
+ counted AS (
883
+ SELECT *, COUNT(*) OVER () AS total_count
884
+ FROM filtered_sessions
885
+ )
886
+ SELECT
887
+ c.id,
888
+ c.name,
889
+ c.mode,
890
+ c.description,
891
+ c.user_id,
892
+ c.user_name,
893
+ c.user_email,
894
+ c.message_count,
895
+ c.total_tokens,
896
+ c.created_at,
897
+ c.updated_at,
898
+ c.metadata,
899
+ c.total_count
900
+ FROM counted c
901
+ ORDER BY c.created_at DESC
902
+ LIMIT p_page_size
903
+ OFFSET (p_page - 1) * p_page_size;
904
+ END;
905
+ $$ LANGUAGE plpgsql STABLE;
906
+
907
+ COMMENT ON FUNCTION fn_list_sessions_with_user IS
908
+ 'List sessions with user details and computed message counts. Joins messages on session name.';
909
+
792
910
  -- ============================================================================
793
911
  -- RECORD INSTALLATION
794
912
  -- ============================================================================