remdb 0.3.14__py3-none-any.whl → 0.3.157__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. rem/agentic/README.md +76 -0
  2. rem/agentic/__init__.py +15 -0
  3. rem/agentic/agents/__init__.py +32 -2
  4. rem/agentic/agents/agent_manager.py +310 -0
  5. rem/agentic/agents/sse_simulator.py +502 -0
  6. rem/agentic/context.py +51 -27
  7. rem/agentic/context_builder.py +5 -3
  8. rem/agentic/llm_provider_models.py +301 -0
  9. rem/agentic/mcp/tool_wrapper.py +155 -18
  10. rem/agentic/otel/setup.py +93 -4
  11. rem/agentic/providers/phoenix.py +371 -108
  12. rem/agentic/providers/pydantic_ai.py +280 -57
  13. rem/agentic/schema.py +361 -21
  14. rem/agentic/tools/rem_tools.py +3 -3
  15. rem/api/README.md +215 -1
  16. rem/api/deps.py +255 -0
  17. rem/api/main.py +132 -40
  18. rem/api/mcp_router/resources.py +1 -1
  19. rem/api/mcp_router/server.py +28 -5
  20. rem/api/mcp_router/tools.py +555 -7
  21. rem/api/routers/admin.py +494 -0
  22. rem/api/routers/auth.py +278 -4
  23. rem/api/routers/chat/completions.py +402 -20
  24. rem/api/routers/chat/models.py +88 -10
  25. rem/api/routers/chat/otel_utils.py +33 -0
  26. rem/api/routers/chat/sse_events.py +542 -0
  27. rem/api/routers/chat/streaming.py +697 -45
  28. rem/api/routers/dev.py +81 -0
  29. rem/api/routers/feedback.py +268 -0
  30. rem/api/routers/messages.py +473 -0
  31. rem/api/routers/models.py +78 -0
  32. rem/api/routers/query.py +360 -0
  33. rem/api/routers/shared_sessions.py +406 -0
  34. rem/auth/__init__.py +13 -3
  35. rem/auth/middleware.py +186 -22
  36. rem/auth/providers/__init__.py +4 -1
  37. rem/auth/providers/email.py +215 -0
  38. rem/cli/commands/README.md +237 -64
  39. rem/cli/commands/cluster.py +1808 -0
  40. rem/cli/commands/configure.py +4 -7
  41. rem/cli/commands/db.py +386 -143
  42. rem/cli/commands/experiments.py +468 -76
  43. rem/cli/commands/process.py +14 -8
  44. rem/cli/commands/schema.py +97 -50
  45. rem/cli/commands/session.py +336 -0
  46. rem/cli/dreaming.py +2 -2
  47. rem/cli/main.py +29 -6
  48. rem/config.py +10 -3
  49. rem/models/core/core_model.py +7 -1
  50. rem/models/core/experiment.py +58 -14
  51. rem/models/core/rem_query.py +5 -2
  52. rem/models/entities/__init__.py +25 -0
  53. rem/models/entities/domain_resource.py +38 -0
  54. rem/models/entities/feedback.py +123 -0
  55. rem/models/entities/message.py +30 -1
  56. rem/models/entities/ontology.py +1 -1
  57. rem/models/entities/ontology_config.py +1 -1
  58. rem/models/entities/session.py +83 -0
  59. rem/models/entities/shared_session.py +180 -0
  60. rem/models/entities/subscriber.py +175 -0
  61. rem/models/entities/user.py +1 -0
  62. rem/registry.py +10 -4
  63. rem/schemas/agents/core/agent-builder.yaml +134 -0
  64. rem/schemas/agents/examples/contract-analyzer.yaml +1 -1
  65. rem/schemas/agents/examples/contract-extractor.yaml +1 -1
  66. rem/schemas/agents/examples/cv-parser.yaml +1 -1
  67. rem/schemas/agents/rem.yaml +7 -3
  68. rem/services/__init__.py +3 -1
  69. rem/services/content/service.py +92 -19
  70. rem/services/email/__init__.py +10 -0
  71. rem/services/email/service.py +459 -0
  72. rem/services/email/templates.py +360 -0
  73. rem/services/embeddings/api.py +4 -4
  74. rem/services/embeddings/worker.py +16 -16
  75. rem/services/phoenix/client.py +154 -14
  76. rem/services/postgres/README.md +197 -15
  77. rem/services/postgres/__init__.py +2 -1
  78. rem/services/postgres/diff_service.py +547 -0
  79. rem/services/postgres/pydantic_to_sqlalchemy.py +470 -140
  80. rem/services/postgres/repository.py +132 -0
  81. rem/services/postgres/schema_generator.py +205 -4
  82. rem/services/postgres/service.py +6 -6
  83. rem/services/rem/parser.py +44 -9
  84. rem/services/rem/service.py +36 -2
  85. rem/services/session/compression.py +137 -51
  86. rem/services/session/reload.py +15 -8
  87. rem/settings.py +515 -27
  88. rem/sql/background_indexes.sql +21 -16
  89. rem/sql/migrations/001_install.sql +387 -54
  90. rem/sql/migrations/002_install_models.sql +2304 -377
  91. rem/sql/migrations/003_optional_extensions.sql +326 -0
  92. rem/sql/migrations/004_cache_system.sql +548 -0
  93. rem/sql/migrations/005_schema_update.sql +145 -0
  94. rem/utils/README.md +45 -0
  95. rem/utils/__init__.py +18 -0
  96. rem/utils/date_utils.py +2 -2
  97. rem/utils/files.py +157 -1
  98. rem/utils/model_helpers.py +156 -1
  99. rem/utils/schema_loader.py +220 -22
  100. rem/utils/sql_paths.py +146 -0
  101. rem/utils/sql_types.py +3 -1
  102. rem/utils/vision.py +1 -1
  103. rem/workers/__init__.py +3 -1
  104. rem/workers/db_listener.py +579 -0
  105. rem/workers/unlogged_maintainer.py +463 -0
  106. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/METADATA +340 -229
  107. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/RECORD +109 -80
  108. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/WHEEL +1 -1
  109. rem/sql/002_install_models.sql +0 -1068
  110. rem/sql/install_models.sql +0 -1051
  111. rem/sql/migrations/003_seed_default_user.sql +0 -48
  112. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/entry_points.txt +0 -0
rem/api/routers/dev.py ADDED
@@ -0,0 +1,81 @@
1
+ """
2
+ Development utilities router (non-production only).
3
+
4
+ Provides testing endpoints that are available in development/staging environments
5
+ regardless of auth configuration. These endpoints are NEVER available in production.
6
+
7
+ Endpoints:
8
+ - GET /api/dev/token - Get a dev token for test-user
9
+ """
10
+
11
+ from fastapi import APIRouter, HTTPException, Request
12
+ from loguru import logger
13
+
14
+ from ...settings import settings
15
+
16
+ router = APIRouter(prefix="/api/dev", tags=["dev"])
17
+
18
+
19
+ def generate_dev_token() -> str:
20
+ """
21
+ Generate a dev token for testing.
22
+
23
+ Token format: dev_<hmac_signature>
24
+ The signature is based on the session secret to ensure only valid tokens work.
25
+ """
26
+ import hashlib
27
+ import hmac
28
+
29
+ # Use session secret as key
30
+ secret = settings.auth.session_secret or "dev-secret"
31
+ message = "test-user:dev-token"
32
+
33
+ signature = hmac.new(
34
+ secret.encode(),
35
+ message.encode(),
36
+ hashlib.sha256
37
+ ).hexdigest()[:32]
38
+
39
+ return f"dev_{signature}"
40
+
41
+
42
+ def verify_dev_token(token: str) -> bool:
43
+ """Verify a dev token is valid."""
44
+ expected = generate_dev_token()
45
+ return token == expected
46
+
47
+
48
+ @router.get("/token")
49
+ async def get_dev_token(request: Request):
50
+ """
51
+ Get a development token for testing (non-production only).
52
+
53
+ This token can be used as a Bearer token to authenticate as the
54
+ test user (test-user / test@rem.local) without going through OAuth.
55
+
56
+ Usage:
57
+ curl -H "Authorization: Bearer <token>" http://localhost:8000/api/v1/...
58
+
59
+ Returns:
60
+ 401 if in production environment
61
+ Token and usage instructions otherwise
62
+ """
63
+ if settings.environment == "production":
64
+ raise HTTPException(
65
+ status_code=401,
66
+ detail="Dev tokens are not available in production"
67
+ )
68
+
69
+ token = generate_dev_token()
70
+
71
+ return {
72
+ "token": token,
73
+ "type": "Bearer",
74
+ "user": {
75
+ "id": "test-user",
76
+ "email": "test@rem.local",
77
+ "name": "Test User",
78
+ },
79
+ "usage": f'curl -H "Authorization: Bearer {token}" http://localhost:8000/api/v1/...',
80
+ "warning": "This token is for development/testing only and will not work in production.",
81
+ }
@@ -0,0 +1,268 @@
1
+ """
2
+ Message feedback endpoint.
3
+
4
+ Provides endpoint for submitting feedback on messages.
5
+
6
+ Endpoints:
7
+ POST /api/v1/messages/feedback - Submit feedback on a message
8
+
9
+ Trace Integration:
10
+ - Feedback auto-resolves trace_id/span_id from the message in the database
11
+ - Phoenix sync attaches feedback as span annotations when trace info is available
12
+
13
+ HTTP Status Codes:
14
+ - 201: Feedback saved AND synced to Phoenix as annotation (phoenix_synced=true)
15
+ - 200: Feedback accepted and saved to DB, but NOT synced to Phoenix
16
+ (missing trace_id/span_id, Phoenix disabled, or sync failed)
17
+
18
+ IMPORTANT - Testing Requirements:
19
+ ╔════════════════════════════════════════════════════════════════════════════════════════════════════╗
20
+ ║ 1. Use 'rem' agent (NOT 'simulator') - only real agents capture traces ║
21
+ ║ 2. Session IDs MUST be UUIDs - use python3 -c "import uuid; print(uuid.uuid4())" ║
22
+ ║ 3. Port-forward OTEL collector: kubectl port-forward -n observability ║
23
+ ║ svc/otel-collector-collector 4318:4318 ║
24
+ ║ 4. Port-forward Phoenix: kubectl port-forward -n siggy svc/phoenix 6006:6006 ║
25
+ ║ 5. Set environment variables when starting the API: ║
26
+ ║ OTEL__ENABLED=true PHOENIX__ENABLED=true PHOENIX_API_KEY=<jwt> uvicorn ... ║
27
+ ║ 6. Get PHOENIX_API_KEY: ║
28
+ ║ kubectl get secret -n siggy rem-phoenix-api-key -o jsonpath='{.data.PHOENIX_API_KEY}' ║
29
+ ║ | base64 -d ║
30
+ ╚════════════════════════════════════════════════════════════════════════════════════════════════════╝
31
+
32
+ Usage:
33
+ # 1. Send a chat message with X-Session-Id header (MUST be UUID!)
34
+ SESSION_ID=$(python3 -c "import uuid; print(uuid.uuid4())")
35
+ curl -X POST http://localhost:8000/api/v1/chat/completions \\
36
+ -H "Content-Type: application/json" \\
37
+ -H "X-Session-Id: $SESSION_ID" \\
38
+ -H "X-Agent-Schema: rem" \\
39
+ -d '{"messages": [{"role": "user", "content": "hello"}], "stream": true}'
40
+
41
+ # 2. Extract message_id from the 'metadata' SSE event:
42
+ # event: metadata
43
+ # data: {"message_id": "728882f8-...", "trace_id": "e53c701c...", ...}
44
+
45
+ # 3. Submit feedback referencing that message (trace_id auto-resolved from DB)
46
+ curl -X POST http://localhost:8000/api/v1/messages/feedback \\
47
+ -H "Content-Type: application/json" \\
48
+ -H "X-Tenant-Id: default" \\
49
+ -d '{
50
+ "session_id": "'$SESSION_ID'",
51
+ "message_id": "<message-id-from-metadata>",
52
+ "rating": 1,
53
+ "categories": ["helpful"],
54
+ "comment": "Great response!"
55
+ }'
56
+
57
+ # 4. Check response:
58
+ # - 201 + phoenix_synced=true = annotation synced to Phoenix (check Phoenix UI at :6006)
59
+ # - 200 + phoenix_synced=false = feedback saved but not synced (missing trace info)
60
+ """
61
+
62
+ from fastapi import APIRouter, Header, HTTPException, Request, Response
63
+ from loguru import logger
64
+ from pydantic import BaseModel, Field
65
+
66
+ from ..deps import get_user_id_from_request
67
+ from ...models.entities import Feedback
68
+ from ...services.postgres import Repository
69
+ from ...settings import settings
70
+
71
+ router = APIRouter(prefix="/api/v1", tags=["messages"])
72
+
73
+
74
+ # =============================================================================
75
+ # Request/Response Models
76
+ # =============================================================================
77
+
78
+
79
+ class FeedbackCreateRequest(BaseModel):
80
+ """Request to submit feedback."""
81
+
82
+ session_id: str = Field(description="Session ID this feedback relates to")
83
+ message_id: str | None = Field(
84
+ default=None, description="Specific message ID (null for session-level)"
85
+ )
86
+ rating: int | None = Field(
87
+ default=None,
88
+ ge=-1,
89
+ le=5,
90
+ description="Rating: -1 (thumbs down), 1 (thumbs up), or 1-5 scale",
91
+ )
92
+ categories: list[str] = Field(
93
+ default_factory=list, description="Feedback categories"
94
+ )
95
+ comment: str | None = Field(default=None, description="Free-text comment")
96
+ trace_id: str | None = Field(
97
+ default=None, description="OTEL trace ID (auto-resolved if message has it)"
98
+ )
99
+ span_id: str | None = Field(
100
+ default=None, description="OTEL span ID (auto-resolved if message has it)"
101
+ )
102
+
103
+
104
+ class FeedbackResponse(BaseModel):
105
+ """Response after submitting feedback."""
106
+
107
+ id: str
108
+ session_id: str
109
+ message_id: str | None
110
+ rating: int | None
111
+ categories: list[str]
112
+ comment: str | None
113
+ trace_id: str | None
114
+ span_id: str | None
115
+ phoenix_synced: bool
116
+ created_at: str
117
+
118
+
119
+ # =============================================================================
120
+ # Feedback Endpoint
121
+ # =============================================================================
122
+
123
+
124
+ @router.post("/messages/feedback", response_model=FeedbackResponse)
125
+ async def submit_feedback(
126
+ request: Request,
127
+ response: Response,
128
+ request_body: FeedbackCreateRequest,
129
+ x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
130
+ ) -> FeedbackResponse:
131
+ """
132
+ Submit feedback on a message or session.
133
+
134
+ If message_id is provided, feedback is attached to that specific message.
135
+ If only session_id is provided, feedback applies to the entire session.
136
+
137
+ Trace IDs (trace_id, span_id) can be:
138
+ - Provided explicitly in the request
139
+ - Auto-resolved from the message if message_id is provided
140
+
141
+ HTTP Status Codes:
142
+ - 201: Feedback saved AND synced to Phoenix (phoenix_synced=true)
143
+ - 200: Feedback accepted but NOT synced (missing trace info, disabled, or failed)
144
+
145
+ Returns:
146
+ Created feedback object with phoenix_synced indicating sync status
147
+ """
148
+ if not settings.postgres.enabled:
149
+ raise HTTPException(status_code=503, detail="Database not enabled")
150
+
151
+ effective_user_id = get_user_id_from_request(request)
152
+
153
+ # Resolve trace_id/span_id from message if not provided
154
+ trace_id = request_body.trace_id
155
+ span_id = request_body.span_id
156
+
157
+ if request_body.message_id and (not trace_id or not span_id):
158
+ # Look up message by ID to get trace context
159
+ # Note: Messages are stored with tenant_id=user_id (not x_tenant_id header)
160
+ # so we query by ID only - UUIDs are globally unique
161
+ from ...services.postgres import PostgresService
162
+ import uuid
163
+
164
+ logger.info(f"Looking up trace context for message_id={request_body.message_id}")
165
+
166
+ # Convert message_id string to UUID for database query
167
+ try:
168
+ message_uuid = uuid.UUID(request_body.message_id)
169
+ except ValueError as e:
170
+ logger.warning(f"Invalid message_id format '{request_body.message_id}': {e}")
171
+ message_uuid = None
172
+
173
+ if message_uuid:
174
+ db = PostgresService()
175
+ # Ensure connection (same pattern as Repository)
176
+ if not db.pool:
177
+ await db.connect()
178
+
179
+ if db.pool:
180
+ query = """
181
+ SELECT trace_id, span_id FROM messages
182
+ WHERE id = $1 AND deleted_at IS NULL
183
+ LIMIT 1
184
+ """
185
+ async with db.pool.acquire() as conn:
186
+ row = await conn.fetchrow(query, message_uuid)
187
+ logger.info(f"Database query result for message {request_body.message_id}: row={row}")
188
+ if row:
189
+ trace_id = trace_id or row["trace_id"]
190
+ span_id = span_id or row["span_id"]
191
+ logger.info(f"Found trace context for message {request_body.message_id}: trace_id={trace_id}, span_id={span_id}")
192
+ else:
193
+ logger.warning(f"No message found in database with id={request_body.message_id}")
194
+ else:
195
+ logger.warning(f"Database pool not available for message lookup after connect attempt")
196
+
197
+ feedback = Feedback(
198
+ session_id=request_body.session_id,
199
+ message_id=request_body.message_id,
200
+ rating=request_body.rating,
201
+ categories=request_body.categories,
202
+ comment=request_body.comment,
203
+ trace_id=trace_id,
204
+ span_id=span_id,
205
+ phoenix_synced=False,
206
+ annotator_kind="HUMAN",
207
+ user_id=effective_user_id,
208
+ tenant_id=x_tenant_id,
209
+ )
210
+
211
+ repo = Repository(Feedback, table_name="feedbacks")
212
+ result = await repo.upsert(feedback)
213
+
214
+ logger.info(
215
+ f"Feedback submitted: session={request_body.session_id}, "
216
+ f"message={request_body.message_id}, rating={request_body.rating}"
217
+ )
218
+
219
+ # Sync to Phoenix if trace_id/span_id available and Phoenix is enabled
220
+ phoenix_synced = False
221
+ phoenix_annotation_id = None
222
+
223
+ if trace_id and span_id and settings.phoenix.enabled:
224
+ try:
225
+ from ...services.phoenix import PhoenixClient
226
+
227
+ phoenix_client = PhoenixClient()
228
+ phoenix_annotation_id = phoenix_client.sync_user_feedback(
229
+ span_id=span_id,
230
+ rating=request_body.rating,
231
+ categories=request_body.categories,
232
+ comment=request_body.comment,
233
+ feedback_id=str(result.id),
234
+ trace_id=trace_id,
235
+ )
236
+
237
+ if phoenix_annotation_id:
238
+ phoenix_synced = True
239
+ # Update the feedback record with sync status
240
+ result.phoenix_synced = True
241
+ result.phoenix_annotation_id = phoenix_annotation_id
242
+ await repo.upsert(result)
243
+ logger.info(f"Feedback synced to Phoenix: annotation_id={phoenix_annotation_id}")
244
+ else:
245
+ logger.warning(f"Phoenix sync returned no annotation ID for feedback {result.id}")
246
+
247
+ except Exception as e:
248
+ logger.error(f"Failed to sync feedback to Phoenix: {e}")
249
+ # Don't fail the request if Phoenix sync fails
250
+ elif trace_id and span_id:
251
+ logger.debug(f"Feedback has trace info but Phoenix disabled: trace={trace_id}, span={span_id}")
252
+
253
+ # Set HTTP status code based on Phoenix sync result
254
+ # 201 = synced to Phoenix, 200 = accepted but not synced
255
+ response.status_code = 201 if phoenix_synced else 200
256
+
257
+ return FeedbackResponse(
258
+ id=str(result.id),
259
+ session_id=result.session_id,
260
+ message_id=result.message_id,
261
+ rating=result.rating,
262
+ categories=result.categories,
263
+ comment=result.comment,
264
+ trace_id=result.trace_id,
265
+ span_id=result.span_id,
266
+ phoenix_synced=result.phoenix_synced,
267
+ created_at=result.created_at.isoformat() if result.created_at else "",
268
+ )