remdb 0.2.6__py3-none-any.whl → 0.3.103__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of remdb might be problematic. Click here for more details.

Files changed (82) hide show
  1. rem/__init__.py +129 -2
  2. rem/agentic/README.md +76 -0
  3. rem/agentic/__init__.py +15 -0
  4. rem/agentic/agents/__init__.py +16 -2
  5. rem/agentic/agents/sse_simulator.py +500 -0
  6. rem/agentic/context.py +7 -5
  7. rem/agentic/llm_provider_models.py +301 -0
  8. rem/agentic/providers/phoenix.py +32 -43
  9. rem/agentic/providers/pydantic_ai.py +84 -10
  10. rem/api/README.md +238 -1
  11. rem/api/deps.py +255 -0
  12. rem/api/main.py +70 -22
  13. rem/api/mcp_router/server.py +8 -1
  14. rem/api/mcp_router/tools.py +80 -0
  15. rem/api/middleware/tracking.py +172 -0
  16. rem/api/routers/admin.py +277 -0
  17. rem/api/routers/auth.py +124 -0
  18. rem/api/routers/chat/completions.py +123 -14
  19. rem/api/routers/chat/models.py +7 -3
  20. rem/api/routers/chat/sse_events.py +526 -0
  21. rem/api/routers/chat/streaming.py +468 -45
  22. rem/api/routers/dev.py +81 -0
  23. rem/api/routers/feedback.py +455 -0
  24. rem/api/routers/messages.py +473 -0
  25. rem/api/routers/models.py +78 -0
  26. rem/api/routers/shared_sessions.py +406 -0
  27. rem/auth/middleware.py +126 -27
  28. rem/cli/commands/ask.py +15 -11
  29. rem/cli/commands/configure.py +169 -94
  30. rem/cli/commands/db.py +53 -7
  31. rem/cli/commands/experiments.py +278 -96
  32. rem/cli/commands/process.py +8 -7
  33. rem/cli/commands/scaffold.py +47 -0
  34. rem/cli/commands/schema.py +9 -9
  35. rem/cli/main.py +10 -0
  36. rem/config.py +2 -2
  37. rem/models/core/core_model.py +7 -1
  38. rem/models/entities/__init__.py +21 -0
  39. rem/models/entities/domain_resource.py +38 -0
  40. rem/models/entities/feedback.py +123 -0
  41. rem/models/entities/message.py +30 -1
  42. rem/models/entities/session.py +83 -0
  43. rem/models/entities/shared_session.py +206 -0
  44. rem/models/entities/user.py +10 -3
  45. rem/registry.py +367 -0
  46. rem/schemas/agents/rem.yaml +7 -3
  47. rem/services/content/providers.py +94 -140
  48. rem/services/content/service.py +85 -16
  49. rem/services/dreaming/affinity_service.py +2 -16
  50. rem/services/dreaming/moment_service.py +2 -15
  51. rem/services/embeddings/api.py +20 -13
  52. rem/services/phoenix/EXPERIMENT_DESIGN.md +3 -3
  53. rem/services/phoenix/client.py +252 -19
  54. rem/services/postgres/README.md +29 -10
  55. rem/services/postgres/repository.py +132 -0
  56. rem/services/postgres/schema_generator.py +86 -5
  57. rem/services/rate_limit.py +113 -0
  58. rem/services/rem/README.md +14 -0
  59. rem/services/session/compression.py +17 -1
  60. rem/services/user_service.py +98 -0
  61. rem/settings.py +115 -17
  62. rem/sql/background_indexes.sql +10 -0
  63. rem/sql/migrations/001_install.sql +152 -2
  64. rem/sql/migrations/002_install_models.sql +580 -231
  65. rem/sql/migrations/003_seed_default_user.sql +48 -0
  66. rem/utils/constants.py +97 -0
  67. rem/utils/date_utils.py +228 -0
  68. rem/utils/embeddings.py +17 -4
  69. rem/utils/files.py +167 -0
  70. rem/utils/mime_types.py +158 -0
  71. rem/utils/model_helpers.py +156 -1
  72. rem/utils/schema_loader.py +273 -14
  73. rem/utils/sql_types.py +3 -1
  74. rem/utils/vision.py +9 -14
  75. rem/workers/README.md +14 -14
  76. rem/workers/db_maintainer.py +74 -0
  77. {remdb-0.2.6.dist-info → remdb-0.3.103.dist-info}/METADATA +486 -132
  78. {remdb-0.2.6.dist-info → remdb-0.3.103.dist-info}/RECORD +80 -57
  79. {remdb-0.2.6.dist-info → remdb-0.3.103.dist-info}/WHEEL +1 -1
  80. rem/sql/002_install_models.sql +0 -1068
  81. rem/sql/install_models.sql +0 -1038
  82. {remdb-0.2.6.dist-info → remdb-0.3.103.dist-info}/entry_points.txt +0 -0
rem/api/routers/dev.py ADDED
@@ -0,0 +1,81 @@
1
+ """
2
+ Development utilities router (non-production only).
3
+
4
+ Provides testing endpoints that are available in development/staging environments
5
+ regardless of auth configuration. These endpoints are NEVER available in production.
6
+
7
+ Endpoints:
8
+ - GET /api/dev/token - Get a dev token for test-user
9
+ """
10
+
11
+ from fastapi import APIRouter, HTTPException, Request
12
+ from loguru import logger
13
+
14
+ from ...settings import settings
15
+
16
+ router = APIRouter(prefix="/api/dev", tags=["dev"])
17
+
18
+
19
+ def generate_dev_token() -> str:
20
+ """
21
+ Generate a dev token for testing.
22
+
23
+ Token format: dev_<hmac_signature>
24
+ The signature is based on the session secret to ensure only valid tokens work.
25
+ """
26
+ import hashlib
27
+ import hmac
28
+
29
+ # Use session secret as key
30
+ secret = settings.auth.session_secret or "dev-secret"
31
+ message = "test-user:dev-token"
32
+
33
+ signature = hmac.new(
34
+ secret.encode(),
35
+ message.encode(),
36
+ hashlib.sha256
37
+ ).hexdigest()[:32]
38
+
39
+ return f"dev_{signature}"
40
+
41
+
42
+ def verify_dev_token(token: str) -> bool:
43
+ """Verify a dev token is valid."""
44
+ expected = generate_dev_token()
45
+ return token == expected
46
+
47
+
48
+ @router.get("/token")
49
+ async def get_dev_token(request: Request):
50
+ """
51
+ Get a development token for testing (non-production only).
52
+
53
+ This token can be used as a Bearer token to authenticate as the
54
+ test user (test-user / test@rem.local) without going through OAuth.
55
+
56
+ Usage:
57
+ curl -H "Authorization: Bearer <token>" http://localhost:8000/api/v1/...
58
+
59
+ Returns:
60
+ 401 if in production environment
61
+ Token and usage instructions otherwise
62
+ """
63
+ if settings.environment == "production":
64
+ raise HTTPException(
65
+ status_code=401,
66
+ detail="Dev tokens are not available in production"
67
+ )
68
+
69
+ token = generate_dev_token()
70
+
71
+ return {
72
+ "token": token,
73
+ "type": "Bearer",
74
+ "user": {
75
+ "id": "test-user",
76
+ "email": "test@rem.local",
77
+ "name": "Test User",
78
+ },
79
+ "usage": f'curl -H "Authorization: Bearer {token}" http://localhost:8000/api/v1/...',
80
+ "warning": "This token is for development/testing only and will not work in production.",
81
+ }
@@ -0,0 +1,455 @@
1
+ """
2
+ Feedback endpoints for chat message and session feedback.
3
+
4
+ Provides endpoints for:
5
+ - Submitting feedback on messages or sessions
6
+ - Listing feedback with filters
7
+ - Syncing feedback to Phoenix as annotations (async)
8
+
9
+ Endpoints:
10
+ POST /api/v1/feedback - Submit feedback
11
+ GET /api/v1/feedback - List feedback with filters
12
+ GET /api/v1/feedback/{id} - Get specific feedback
13
+ GET /api/v1/feedback/categories - List available categories
14
+
15
+ Trace Integration:
16
+ - Feedback can reference trace_id/span_id for OTEL integration
17
+ - Phoenix sync attaches feedback as span annotations
18
+ """
19
+
20
+ from typing import Literal
21
+
22
+ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request
23
+ from loguru import logger
24
+ from pydantic import BaseModel, Field
25
+
26
+ from ..deps import get_current_user, get_user_filter, get_user_id_from_request, is_admin
27
+ from ...models.entities import Feedback, FeedbackCategory, Message
28
+ from ...services.postgres import Repository
29
+ from ...settings import settings
30
+ from ...utils.date_utils import utc_now
31
+
32
+ router = APIRouter(prefix="/api/v1", tags=["feedback"])
33
+
34
+
35
+ # =============================================================================
36
+ # Request/Response Models
37
+ # =============================================================================
38
+
39
+
40
+ class FeedbackCreateRequest(BaseModel):
41
+ """Request to submit feedback."""
42
+
43
+ session_id: str = Field(description="Session ID this feedback relates to")
44
+ message_id: str | None = Field(
45
+ default=None, description="Specific message ID (null for session-level)"
46
+ )
47
+ rating: int | None = Field(
48
+ default=None,
49
+ ge=-1,
50
+ le=5,
51
+ description="Rating: -1 (thumbs down), 1 (thumbs up), or 1-5 scale",
52
+ )
53
+ categories: list[str] = Field(
54
+ default_factory=list, description="Selected feedback categories"
55
+ )
56
+ comment: str | None = Field(default=None, description="Free-text comment")
57
+ # Optional trace reference (can be auto-resolved from message)
58
+ trace_id: str | None = Field(
59
+ default=None, description="OTEL trace ID (auto-resolved if message has it)"
60
+ )
61
+ span_id: str | None = Field(
62
+ default=None, description="OTEL span ID (auto-resolved if message has it)"
63
+ )
64
+
65
+
66
+ class FeedbackResponse(BaseModel):
67
+ """Response after submitting feedback."""
68
+
69
+ id: str
70
+ session_id: str
71
+ message_id: str | None
72
+ rating: int | None
73
+ categories: list[str]
74
+ comment: str | None
75
+ trace_id: str | None
76
+ span_id: str | None
77
+ phoenix_synced: bool
78
+ created_at: str
79
+
80
+
81
+ class FeedbackListResponse(BaseModel):
82
+ """Response for feedback list endpoint."""
83
+
84
+ object: Literal["list"] = "list"
85
+ data: list[Feedback]
86
+ total: int
87
+ has_more: bool
88
+
89
+
90
+ class CategoryInfo(BaseModel):
91
+ """Information about a feedback category."""
92
+
93
+ value: str
94
+ label: str
95
+ description: str
96
+ sentiment: Literal["positive", "negative", "neutral"]
97
+
98
+
99
+ class CategoriesResponse(BaseModel):
100
+ """Response for categories endpoint."""
101
+
102
+ categories: list[CategoryInfo]
103
+
104
+
105
+ # =============================================================================
106
+ # Category Definitions
107
+ # =============================================================================
108
+
109
+ CATEGORY_INFO: dict[str, CategoryInfo] = {
110
+ FeedbackCategory.INCOMPLETE.value: CategoryInfo(
111
+ value=FeedbackCategory.INCOMPLETE.value,
112
+ label="Incomplete",
113
+ description="Response lacks expected information",
114
+ sentiment="negative",
115
+ ),
116
+ FeedbackCategory.INACCURATE.value: CategoryInfo(
117
+ value=FeedbackCategory.INACCURATE.value,
118
+ label="Inaccurate",
119
+ description="Response contains factual errors",
120
+ sentiment="negative",
121
+ ),
122
+ FeedbackCategory.POOR_TONE.value: CategoryInfo(
123
+ value=FeedbackCategory.POOR_TONE.value,
124
+ label="Poor Tone",
125
+ description="Inappropriate or unprofessional tone",
126
+ sentiment="negative",
127
+ ),
128
+ FeedbackCategory.OFF_TOPIC.value: CategoryInfo(
129
+ value=FeedbackCategory.OFF_TOPIC.value,
130
+ label="Off Topic",
131
+ description="Response doesn't address the question",
132
+ sentiment="negative",
133
+ ),
134
+ FeedbackCategory.TOO_VERBOSE.value: CategoryInfo(
135
+ value=FeedbackCategory.TOO_VERBOSE.value,
136
+ label="Too Verbose",
137
+ description="Unnecessarily long response",
138
+ sentiment="negative",
139
+ ),
140
+ FeedbackCategory.TOO_BRIEF.value: CategoryInfo(
141
+ value=FeedbackCategory.TOO_BRIEF.value,
142
+ label="Too Brief",
143
+ description="Insufficiently detailed response",
144
+ sentiment="negative",
145
+ ),
146
+ FeedbackCategory.CONFUSING.value: CategoryInfo(
147
+ value=FeedbackCategory.CONFUSING.value,
148
+ label="Confusing",
149
+ description="Hard to understand or unclear",
150
+ sentiment="negative",
151
+ ),
152
+ FeedbackCategory.UNSAFE.value: CategoryInfo(
153
+ value=FeedbackCategory.UNSAFE.value,
154
+ label="Unsafe",
155
+ description="Contains potentially harmful content",
156
+ sentiment="negative",
157
+ ),
158
+ FeedbackCategory.HELPFUL.value: CategoryInfo(
159
+ value=FeedbackCategory.HELPFUL.value,
160
+ label="Helpful",
161
+ description="Response was useful and addressed the need",
162
+ sentiment="positive",
163
+ ),
164
+ FeedbackCategory.EXCELLENT.value: CategoryInfo(
165
+ value=FeedbackCategory.EXCELLENT.value,
166
+ label="Excellent",
167
+ description="Exceptionally good response",
168
+ sentiment="positive",
169
+ ),
170
+ FeedbackCategory.ACCURATE.value: CategoryInfo(
171
+ value=FeedbackCategory.ACCURATE.value,
172
+ label="Accurate",
173
+ description="Factually correct and precise",
174
+ sentiment="positive",
175
+ ),
176
+ FeedbackCategory.WELL_WRITTEN.value: CategoryInfo(
177
+ value=FeedbackCategory.WELL_WRITTEN.value,
178
+ label="Well Written",
179
+ description="Clear, well-structured response",
180
+ sentiment="positive",
181
+ ),
182
+ FeedbackCategory.OTHER.value: CategoryInfo(
183
+ value=FeedbackCategory.OTHER.value,
184
+ label="Other",
185
+ description="Other feedback not covered by categories",
186
+ sentiment="neutral",
187
+ ),
188
+ }
189
+
190
+
191
+ # =============================================================================
192
+ # Feedback Endpoints
193
+ # =============================================================================
194
+
195
+
196
+ @router.get("/feedback/categories", response_model=CategoriesResponse)
197
+ async def list_categories() -> CategoriesResponse:
198
+ """
199
+ List available feedback categories.
200
+
201
+ Returns predefined categories with labels, descriptions, and sentiment.
202
+ """
203
+ return CategoriesResponse(categories=list(CATEGORY_INFO.values()))
204
+
205
+
206
+ @router.post("/feedback", response_model=FeedbackResponse, status_code=201)
207
+ async def submit_feedback(
208
+ request: Request,
209
+ request_body: FeedbackCreateRequest,
210
+ x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
211
+ ) -> FeedbackResponse:
212
+ """
213
+ Submit feedback on a message or session.
214
+
215
+ If message_id is provided, feedback is attached to that specific message.
216
+ If only session_id is provided, feedback applies to the entire session.
217
+
218
+ Trace IDs (trace_id, span_id) can be:
219
+ - Provided explicitly in the request
220
+ - Auto-resolved from the message if message_id is provided
221
+
222
+ Phoenix sync happens asynchronously after feedback is stored.
223
+
224
+ Returns:
225
+ Created feedback object
226
+ """
227
+ if not settings.postgres.enabled:
228
+ raise HTTPException(status_code=503, detail="Database not enabled")
229
+
230
+ # Get effective user_id from auth or anonymous tracking
231
+ effective_user_id = get_user_id_from_request(request)
232
+
233
+ # Resolve trace_id/span_id from message if not provided
234
+ trace_id = request_body.trace_id
235
+ span_id = request_body.span_id
236
+
237
+ if request_body.message_id and (not trace_id or not span_id):
238
+ # Try to get trace info from the message
239
+ message_repo = Repository(Message, table_name="messages")
240
+ message = await message_repo.get_by_id(request_body.message_id, x_tenant_id)
241
+ if message:
242
+ trace_id = trace_id or message.trace_id
243
+ span_id = span_id or message.span_id
244
+
245
+ # Create feedback entity
246
+ feedback = Feedback(
247
+ session_id=request_body.session_id,
248
+ message_id=request_body.message_id,
249
+ rating=request_body.rating,
250
+ categories=request_body.categories,
251
+ comment=request_body.comment,
252
+ trace_id=trace_id,
253
+ span_id=span_id,
254
+ phoenix_synced=False,
255
+ annotator_kind="HUMAN",
256
+ user_id=effective_user_id,
257
+ tenant_id=x_tenant_id,
258
+ )
259
+
260
+ # Store feedback (table is "feedbacks" - plural)
261
+ repo = Repository(Feedback, table_name="feedbacks")
262
+ result = await repo.upsert(feedback)
263
+
264
+ logger.info(
265
+ f"Feedback submitted: session={request_body.session_id}, "
266
+ f"message={request_body.message_id}, rating={request_body.rating}, "
267
+ f"categories={request_body.categories}"
268
+ )
269
+
270
+ # TODO: Async sync to Phoenix if trace_id/span_id available
271
+ # This would be done via a background task or queue
272
+ if trace_id and span_id:
273
+ logger.debug(f"Feedback has trace info: trace={trace_id}, span={span_id}")
274
+ # TODO: Queue for Phoenix annotation sync
275
+ # await sync_feedback_to_phoenix(feedback)
276
+
277
+ return FeedbackResponse(
278
+ id=str(result.id),
279
+ session_id=result.session_id,
280
+ message_id=result.message_id,
281
+ rating=result.rating,
282
+ categories=result.categories,
283
+ comment=result.comment,
284
+ trace_id=result.trace_id,
285
+ span_id=result.span_id,
286
+ phoenix_synced=result.phoenix_synced,
287
+ created_at=result.created_at.isoformat() if result.created_at else "",
288
+ )
289
+
290
+
291
+ @router.get("/feedback", response_model=FeedbackListResponse)
292
+ async def list_feedback(
293
+ request: Request,
294
+ session_id: str | None = Query(default=None, description="Filter by session ID"),
295
+ message_id: str | None = Query(default=None, description="Filter by message ID"),
296
+ rating: int | None = Query(default=None, description="Filter by rating"),
297
+ category: str | None = Query(default=None, description="Filter by category"),
298
+ phoenix_synced: bool | None = Query(
299
+ default=None, description="Filter by Phoenix sync status"
300
+ ),
301
+ limit: int = Query(default=50, ge=1, le=100, description="Max results"),
302
+ offset: int = Query(default=0, ge=0, description="Offset for pagination"),
303
+ x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
304
+ ) -> FeedbackListResponse:
305
+ """
306
+ List feedback with optional filters.
307
+
308
+ Access Control:
309
+ - Regular users: Only see feedback they submitted
310
+ - Admin users: Can see all feedback
311
+
312
+ Filters:
313
+ - session_id: Filter by session
314
+ - message_id: Filter by specific message
315
+ - rating: Filter by rating value
316
+ - category: Filter by category (checks if category in list)
317
+ - phoenix_synced: Filter by sync status
318
+ """
319
+ if not settings.postgres.enabled:
320
+ raise HTTPException(status_code=503, detail="Database not enabled")
321
+
322
+ repo = Repository(Feedback, table_name="feedbacks")
323
+
324
+ # Build user-scoped filters (uses anon_id for anonymous users)
325
+ filters = await get_user_filter(request, x_tenant_id=x_tenant_id)
326
+
327
+ # Apply optional filters
328
+ if session_id:
329
+ filters["session_id"] = session_id
330
+ if message_id:
331
+ filters["message_id"] = message_id
332
+ if rating is not None:
333
+ filters["rating"] = rating
334
+ if phoenix_synced is not None:
335
+ filters["phoenix_synced"] = phoenix_synced
336
+ # TODO: category filter requires array contains query
337
+
338
+ feedback_list = await repo.find(
339
+ filters,
340
+ order_by="created_at DESC",
341
+ limit=limit + 1,
342
+ offset=offset,
343
+ )
344
+
345
+ # Filter by category in Python if specified (until Repository supports array contains)
346
+ if category:
347
+ feedback_list = [f for f in feedback_list if category in f.categories]
348
+
349
+ has_more = len(feedback_list) > limit
350
+ if has_more:
351
+ feedback_list = feedback_list[:limit]
352
+
353
+ total = await repo.count(filters)
354
+
355
+ return FeedbackListResponse(data=feedback_list, total=total, has_more=has_more)
356
+
357
+
358
+ @router.get("/feedback/{feedback_id}", response_model=Feedback)
359
+ async def get_feedback(
360
+ request: Request,
361
+ feedback_id: str,
362
+ x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
363
+ ) -> Feedback:
364
+ """
365
+ Get specific feedback by ID.
366
+
367
+ Access Control:
368
+ - Regular users: Only access their own feedback
369
+ - Admin users: Can access any feedback
370
+ """
371
+ if not settings.postgres.enabled:
372
+ raise HTTPException(status_code=503, detail="Database not enabled")
373
+
374
+ repo = Repository(Feedback, table_name="feedbacks")
375
+ feedback = await repo.get_by_id(feedback_id, x_tenant_id)
376
+
377
+ if not feedback:
378
+ raise HTTPException(status_code=404, detail=f"Feedback '{feedback_id}' not found")
379
+
380
+ # Check access
381
+ current_user = get_current_user(request)
382
+ if not is_admin(current_user):
383
+ user_id = current_user.get("id") if current_user else None
384
+ if feedback.user_id and feedback.user_id != user_id:
385
+ raise HTTPException(status_code=403, detail="Access denied: not owner")
386
+
387
+ return feedback
388
+
389
+
390
+ # =============================================================================
391
+ # Phoenix Sync (Stub - TODO: Implement background task)
392
+ # =============================================================================
393
+
394
+
395
+ async def sync_feedback_to_phoenix(feedback: Feedback) -> bool:
396
+ """
397
+ Sync feedback to Phoenix as a span annotation.
398
+
399
+ TODO: Implement this as a background task.
400
+
401
+ This should:
402
+ 1. Connect to Phoenix client
403
+ 2. Resolve trace/span from feedback
404
+ 3. Create annotation with feedback data
405
+ 4. Update feedback.phoenix_synced = True
406
+ 5. Store phoenix_annotation_id
407
+
408
+ Args:
409
+ feedback: Feedback entity to sync
410
+
411
+ Returns:
412
+ True if synced successfully
413
+ """
414
+ if not feedback.span_id:
415
+ logger.warning(f"Cannot sync feedback {feedback.id}: no span_id")
416
+ return False
417
+
418
+ try:
419
+ # TODO: Import and use Phoenix client
420
+ # from ...services.phoenix import PhoenixClient
421
+ # client = PhoenixClient()
422
+ #
423
+ # # Build annotation from feedback
424
+ # label = None
425
+ # if feedback.categories:
426
+ # label = feedback.categories[0] # Primary category
427
+ #
428
+ # score = None
429
+ # if feedback.rating:
430
+ # # Normalize to 0-1 scale
431
+ # if feedback.rating == -1:
432
+ # score = 0.0
433
+ # elif feedback.rating >= 1 and feedback.rating <= 5:
434
+ # score = feedback.rating / 5.0
435
+ #
436
+ # client.add_span_feedback(
437
+ # span_id=feedback.span_id,
438
+ # annotation_name="user_feedback",
439
+ # annotator_kind=feedback.annotator_kind,
440
+ # label=label,
441
+ # score=score,
442
+ # explanation=feedback.comment,
443
+ # )
444
+ #
445
+ # # Update feedback record
446
+ # feedback.phoenix_synced = True
447
+ # repo = Repository(Feedback, table_name="feedbacks")
448
+ # await repo.update(feedback)
449
+
450
+ logger.info(f"TODO: Sync feedback {feedback.id} to Phoenix span {feedback.span_id}")
451
+ return True
452
+
453
+ except Exception as e:
454
+ logger.error(f"Failed to sync feedback to Phoenix: {e}")
455
+ return False