remdb 0.2.6__py3-none-any.whl → 0.3.118__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of remdb might be problematic. Click here for more details.

Files changed (104) hide show
  1. rem/__init__.py +129 -2
  2. rem/agentic/README.md +76 -0
  3. rem/agentic/__init__.py +15 -0
  4. rem/agentic/agents/__init__.py +16 -2
  5. rem/agentic/agents/sse_simulator.py +500 -0
  6. rem/agentic/context.py +28 -22
  7. rem/agentic/llm_provider_models.py +301 -0
  8. rem/agentic/mcp/tool_wrapper.py +29 -3
  9. rem/agentic/otel/setup.py +92 -4
  10. rem/agentic/providers/phoenix.py +32 -43
  11. rem/agentic/providers/pydantic_ai.py +168 -24
  12. rem/agentic/schema.py +358 -21
  13. rem/agentic/tools/rem_tools.py +3 -3
  14. rem/api/README.md +238 -1
  15. rem/api/deps.py +255 -0
  16. rem/api/main.py +154 -37
  17. rem/api/mcp_router/resources.py +1 -1
  18. rem/api/mcp_router/server.py +26 -5
  19. rem/api/mcp_router/tools.py +454 -7
  20. rem/api/middleware/tracking.py +172 -0
  21. rem/api/routers/admin.py +494 -0
  22. rem/api/routers/auth.py +124 -0
  23. rem/api/routers/chat/completions.py +152 -16
  24. rem/api/routers/chat/models.py +7 -3
  25. rem/api/routers/chat/sse_events.py +526 -0
  26. rem/api/routers/chat/streaming.py +608 -45
  27. rem/api/routers/dev.py +81 -0
  28. rem/api/routers/feedback.py +148 -0
  29. rem/api/routers/messages.py +473 -0
  30. rem/api/routers/models.py +78 -0
  31. rem/api/routers/query.py +360 -0
  32. rem/api/routers/shared_sessions.py +406 -0
  33. rem/auth/middleware.py +126 -27
  34. rem/cli/commands/README.md +237 -64
  35. rem/cli/commands/ask.py +15 -11
  36. rem/cli/commands/cluster.py +1300 -0
  37. rem/cli/commands/configure.py +170 -97
  38. rem/cli/commands/db.py +396 -139
  39. rem/cli/commands/experiments.py +278 -96
  40. rem/cli/commands/process.py +22 -15
  41. rem/cli/commands/scaffold.py +47 -0
  42. rem/cli/commands/schema.py +97 -50
  43. rem/cli/main.py +37 -6
  44. rem/config.py +2 -2
  45. rem/models/core/core_model.py +7 -1
  46. rem/models/core/rem_query.py +5 -2
  47. rem/models/entities/__init__.py +21 -0
  48. rem/models/entities/domain_resource.py +38 -0
  49. rem/models/entities/feedback.py +123 -0
  50. rem/models/entities/message.py +30 -1
  51. rem/models/entities/session.py +83 -0
  52. rem/models/entities/shared_session.py +180 -0
  53. rem/models/entities/user.py +10 -3
  54. rem/registry.py +373 -0
  55. rem/schemas/agents/rem.yaml +7 -3
  56. rem/services/content/providers.py +94 -140
  57. rem/services/content/service.py +115 -24
  58. rem/services/dreaming/affinity_service.py +2 -16
  59. rem/services/dreaming/moment_service.py +2 -15
  60. rem/services/embeddings/api.py +24 -17
  61. rem/services/embeddings/worker.py +16 -16
  62. rem/services/phoenix/EXPERIMENT_DESIGN.md +3 -3
  63. rem/services/phoenix/client.py +252 -19
  64. rem/services/postgres/README.md +159 -15
  65. rem/services/postgres/__init__.py +2 -1
  66. rem/services/postgres/diff_service.py +531 -0
  67. rem/services/postgres/pydantic_to_sqlalchemy.py +427 -129
  68. rem/services/postgres/repository.py +132 -0
  69. rem/services/postgres/schema_generator.py +291 -9
  70. rem/services/postgres/service.py +6 -6
  71. rem/services/rate_limit.py +113 -0
  72. rem/services/rem/README.md +14 -0
  73. rem/services/rem/parser.py +44 -9
  74. rem/services/rem/service.py +36 -2
  75. rem/services/session/compression.py +17 -1
  76. rem/services/session/reload.py +1 -1
  77. rem/services/user_service.py +98 -0
  78. rem/settings.py +169 -22
  79. rem/sql/background_indexes.sql +21 -16
  80. rem/sql/migrations/001_install.sql +387 -54
  81. rem/sql/migrations/002_install_models.sql +2320 -393
  82. rem/sql/migrations/003_optional_extensions.sql +326 -0
  83. rem/sql/migrations/004_cache_system.sql +548 -0
  84. rem/utils/__init__.py +18 -0
  85. rem/utils/constants.py +97 -0
  86. rem/utils/date_utils.py +228 -0
  87. rem/utils/embeddings.py +17 -4
  88. rem/utils/files.py +167 -0
  89. rem/utils/mime_types.py +158 -0
  90. rem/utils/model_helpers.py +156 -1
  91. rem/utils/schema_loader.py +284 -21
  92. rem/utils/sql_paths.py +146 -0
  93. rem/utils/sql_types.py +3 -1
  94. rem/utils/vision.py +9 -14
  95. rem/workers/README.md +14 -14
  96. rem/workers/__init__.py +2 -1
  97. rem/workers/db_maintainer.py +74 -0
  98. rem/workers/unlogged_maintainer.py +463 -0
  99. {remdb-0.2.6.dist-info → remdb-0.3.118.dist-info}/METADATA +598 -171
  100. {remdb-0.2.6.dist-info → remdb-0.3.118.dist-info}/RECORD +102 -73
  101. {remdb-0.2.6.dist-info → remdb-0.3.118.dist-info}/WHEEL +1 -1
  102. rem/sql/002_install_models.sql +0 -1068
  103. rem/sql/install_models.sql +0 -1038
  104. {remdb-0.2.6.dist-info → remdb-0.3.118.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,123 @@
1
+ """
2
+ Feedback - User feedback on chat messages and sessions.
3
+
4
+ Feedback allows users to rate and categorize responses, providing
5
+ data for evaluation and model improvement. Feedback can be attached
6
+ to specific messages or entire sessions.
7
+
8
+ Trace Integration:
9
+ - Feedback references trace_id/span_id for OTEL/Phoenix integration
10
+ - Can attach annotations to Phoenix spans for unified observability
11
+
12
+ Predefined Categories (system-defined, extensible):
13
+ - INCOMPLETE: Response lacks expected information
14
+ - INACCURATE: Response contains factual errors
15
+ - POOR_TONE: Inappropriate or unprofessional tone
16
+ - OFF_TOPIC: Response doesn't address the question
17
+ - TOO_VERBOSE: Unnecessarily long response
18
+ - TOO_BRIEF: Insufficiently detailed response
19
+ - HELPFUL: Positive feedback marker
20
+ - EXCELLENT: Exceptionally good response
21
+ """
22
+
23
+ from enum import Enum
24
+ from typing import Any
25
+
26
+ from pydantic import Field
27
+
28
+ from ..core import CoreModel
29
+
30
+
31
+ class FeedbackCategory(str, Enum):
32
+ """Predefined feedback categories (system-defined)."""
33
+
34
+ # Negative categories
35
+ INCOMPLETE = "incomplete"
36
+ INACCURATE = "inaccurate"
37
+ POOR_TONE = "poor_tone"
38
+ OFF_TOPIC = "off_topic"
39
+ TOO_VERBOSE = "too_verbose"
40
+ TOO_BRIEF = "too_brief"
41
+ CONFUSING = "confusing"
42
+ UNSAFE = "unsafe"
43
+
44
+ # Positive categories
45
+ HELPFUL = "helpful"
46
+ EXCELLENT = "excellent"
47
+ ACCURATE = "accurate"
48
+ WELL_WRITTEN = "well_written"
49
+
50
+ # Neutral/Other
51
+ OTHER = "other"
52
+
53
+
54
+ class Feedback(CoreModel):
55
+ """
56
+ User feedback on a message or session.
57
+
58
+ Captures structured feedback including:
59
+ - Rating (1-5 scale or thumbs up/down)
60
+ - Categories (predefined or custom)
61
+ - Free-text comment
62
+ - Trace reference for OTEL/Phoenix integration
63
+
64
+ The feedback can be attached to:
65
+ - A specific message (message_id set)
66
+ - An entire session (session_id set, message_id null)
67
+ """
68
+
69
+ # Target reference (at least one required)
70
+ session_id: str = Field(
71
+ ...,
72
+ description="Session ID this feedback relates to",
73
+ )
74
+ message_id: str | None = Field(
75
+ default=None,
76
+ description="Specific message ID (null for session-level feedback)",
77
+ )
78
+
79
+ # Rating (flexible: 1-5, or -1/1 for thumbs)
80
+ rating: int | None = Field(
81
+ default=None,
82
+ ge=-1,
83
+ le=5,
84
+ description="Rating: -1 (thumbs down), 1 (thumbs up), or 1-5 scale",
85
+ )
86
+
87
+ # Categories (can select multiple)
88
+ categories: list[str] = Field(
89
+ default_factory=list,
90
+ description="Selected feedback categories (from FeedbackCategory or custom)",
91
+ )
92
+
93
+ # Free-text comment
94
+ comment: str | None = Field(
95
+ default=None,
96
+ description="Optional free-text feedback comment",
97
+ )
98
+
99
+ # Trace reference for OTEL/Phoenix integration
100
+ trace_id: str | None = Field(
101
+ default=None,
102
+ description="OTEL trace ID for linking to observability",
103
+ )
104
+ span_id: str | None = Field(
105
+ default=None,
106
+ description="OTEL span ID for specific span feedback",
107
+ )
108
+
109
+ # Phoenix annotation status
110
+ phoenix_synced: bool = Field(
111
+ default=False,
112
+ description="Whether feedback has been synced to Phoenix as annotation",
113
+ )
114
+ phoenix_annotation_id: str | None = Field(
115
+ default=None,
116
+ description="Phoenix annotation ID after sync",
117
+ )
118
+
119
+ # Annotator info
120
+ annotator_kind: str = Field(
121
+ default="HUMAN",
122
+ description="Annotator type: HUMAN, LLM, CODE",
123
+ )
@@ -6,6 +6,11 @@ that can be grouped into conversations or moments.
6
6
 
7
7
  Messages are simpler than Resources but share the same graph connectivity
8
8
  through CoreModel inheritance.
9
+
10
+ Trace Integration:
11
+ - trace_id: OTEL trace ID for linking to observability
12
+ - span_id: OTEL span ID for specific span reference
13
+ - These enable feedback to be attached to Phoenix annotations
9
14
  """
10
15
 
11
16
  from pydantic import Field
@@ -19,6 +24,9 @@ class Message(CoreModel):
19
24
 
20
25
  Represents individual messages in conversations, chats, or other
21
26
  communication contexts. Tenant isolation is provided via CoreModel.tenant_id field.
27
+
28
+ Trace fields (trace_id, span_id) enable integration with OTEL/Phoenix
29
+ for observability and feedback annotation.
22
30
  """
23
31
 
24
32
  content: str = Field(
@@ -27,9 +35,30 @@ class Message(CoreModel):
27
35
  )
28
36
  message_type: str | None = Field(
29
37
  default=None,
30
- description="Message type e.g role",
38
+ description="Message type e.g. role: 'user', 'assistant', 'system', 'tool'",
31
39
  )
32
40
  session_id: str | None = Field(
33
41
  default=None,
34
42
  description="Session identifier for tracking message context",
35
43
  )
44
+ prompt: str | None = Field(
45
+ default=None,
46
+ description="Custom prompt used for this message (if overridden from default)",
47
+ )
48
+ model: str | None = Field(
49
+ default=None,
50
+ description="Model used for generating this message (provider:model format)",
51
+ )
52
+ token_count: int | None = Field(
53
+ default=None,
54
+ description="Token count for this message",
55
+ )
56
+ # OTEL/Phoenix trace integration
57
+ trace_id: str | None = Field(
58
+ default=None,
59
+ description="OTEL trace ID for observability integration",
60
+ )
61
+ span_id: str | None = Field(
62
+ default=None,
63
+ description="OTEL span ID for specific span reference",
64
+ )
@@ -0,0 +1,83 @@
1
+ """
2
+ Session - Conversation sessions in REM.
3
+
4
+ Sessions group related messages together and can have different modes:
5
+ - normal: Standard conversation session
6
+ - evaluation: For LLM evaluation, stores original trace and overridden settings
7
+
8
+ Sessions allow overriding settings like model, temperature, and custom prompts
9
+ for evaluation and experimentation purposes.
10
+ """
11
+
12
+ from enum import Enum
13
+
14
+ from pydantic import Field
15
+
16
+ from ..core import CoreModel
17
+
18
+
19
+ class SessionMode(str, Enum):
20
+ """Session mode types."""
21
+
22
+ NORMAL = "normal"
23
+ EVALUATION = "evaluation"
24
+
25
+
26
+ class Session(CoreModel):
27
+ """
28
+ Conversation session container.
29
+
30
+ Groups messages together and supports different modes for normal conversations
31
+ and evaluation/experimentation scenarios.
32
+
33
+ For evaluation sessions, stores:
34
+ - original_trace_id: Reference to the original session being evaluated
35
+ - settings_overrides: Model, temperature, prompt overrides
36
+ - prompt: Custom prompt being tested
37
+
38
+ Default sessions are lightweight - just a session_id on messages.
39
+ Special sessions store additional metadata for experiments.
40
+ """
41
+
42
+ name: str = Field(
43
+ ...,
44
+ description="Session name/identifier",
45
+ json_schema_extra={"entity_key": True},
46
+ )
47
+ mode: SessionMode = Field(
48
+ default=SessionMode.NORMAL,
49
+ description="Session mode: 'normal' or 'evaluation'",
50
+ )
51
+ description: str | None = Field(
52
+ default=None,
53
+ description="Optional session description",
54
+ )
55
+ # Evaluation-specific fields
56
+ original_trace_id: str | None = Field(
57
+ default=None,
58
+ description="For evaluation mode: ID of the original session/trace being evaluated",
59
+ )
60
+ settings_overrides: dict | None = Field(
61
+ default=None,
62
+ description="Settings overrides (model, temperature, max_tokens, system_prompt)",
63
+ )
64
+ prompt: str | None = Field(
65
+ default=None,
66
+ description="Custom prompt for this session (can override agent prompt)",
67
+ )
68
+ # Agent context
69
+ agent_schema_uri: str | None = Field(
70
+ default=None,
71
+ description="Agent schema used for this session",
72
+ )
73
+ # Summary stats (updated as session progresses)
74
+ message_count: int = Field(
75
+ default=0,
76
+ description="Number of messages in this session",
77
+ )
78
+ total_tokens: int | None = Field(
79
+ default=None,
80
+ description="Total tokens used in this session",
81
+ )
82
+
83
+ model_config = {"use_enum_values": True}
@@ -0,0 +1,180 @@
1
+ """
2
+ SharedSession - Session sharing between users in REM.
3
+
4
+ SharedSessions enable collaborative access to conversation sessions. When a user
5
+ shares a session with another user, a SharedSession record is created to track
6
+ this relationship.
7
+
8
+ ## Design Philosophy
9
+
10
+ Messages already have a session_id field that links them to sessions. The Session
11
+ entity itself is optional and can be left-joined - we don't require explicit Session
12
+ records for sharing to work. What matters is the session_id on messages.
13
+
14
+ SharedSession is a lightweight linking table that:
15
+ 1. Records who shared which session with whom
16
+ 2. Enables soft deletion (deleted_at) so shares can be revoked without data loss
17
+ 3. Supports aggregation queries to see "who is sharing with me"
18
+
19
+ ## Data Model
20
+
21
+ SharedSession
22
+ ├── session_id: str # The session being shared (matches Message.session_id)
23
+ ├── owner_user_id: str # Who owns/created the session (the sharer)
24
+ ├── shared_with_user_id: str # Who the session is shared with (the recipient)
25
+ ├── tenant_id: str # Multi-tenancy isolation
26
+ ├── created_at: datetime # When the share was created
27
+ ├── updated_at: datetime # Last modification
28
+ └── deleted_at: datetime # Soft delete (null = active share)
29
+
30
+ ## Aggregation Query
31
+
32
+ The primary use case is answering: "Who is sharing messages with me?"
33
+
34
+ This is provided by a Postgres function that aggregates:
35
+ - Messages grouped by owner_user_id
36
+ - Joined with users table for name/email
37
+ - Counting messages with min/max dates
38
+ - Filtering out deleted shares
39
+
40
+ Result shape:
41
+ {
42
+ "user_id": "uuid",
43
+ "name": "John Doe",
44
+ "email": "john@example.com",
45
+ "message_count": 42,
46
+ "first_message_at": "2024-01-15T10:30:00Z",
47
+ "last_message_at": "2024-03-20T14:45:00Z",
48
+ "session_count": 3
49
+ }
50
+
51
+ ## API Endpoints
52
+
53
+ 1. POST /api/v1/sessions/{session_id}/share
54
+ - Share a session with another user
55
+ - Body: { "shared_with_user_id": "..." }
56
+ - Creates SharedSession record
57
+
58
+ 2. DELETE /api/v1/sessions/{session_id}/share/{shared_with_user_id}
59
+ - Revoke a share (soft delete)
60
+ - Sets deleted_at on SharedSession
61
+
62
+ 3. GET /api/v1/shared-with-me
63
+ - Get paginated aggregate of users sharing with you
64
+ - Query params: page, page_size (default 50)
65
+ - Returns: list of user summaries with message counts
66
+
67
+ 4. GET /api/v1/shared-with-me/{user_id}/messages
68
+ - Get messages from a specific user's shared sessions
69
+ - Uses existing session message loading
70
+ - Respects pagination
71
+
72
+ ## Soft Delete Pattern
73
+
74
+ Removing a share does NOT delete the SharedSession record. Instead:
75
+ - deleted_at is set to current timestamp
76
+ - All queries filter WHERE deleted_at IS NULL
77
+ - This preserves audit trail and allows "undo"
78
+
79
+ To permanently delete, an admin can run:
80
+ DELETE FROM shared_sessions WHERE deleted_at IS NOT NULL AND deleted_at < NOW() - INTERVAL '30 days'
81
+
82
+ ## Example Usage
83
+
84
+ # Share a session
85
+ POST /api/v1/sessions/abc-123/share
86
+ {"shared_with_user_id": "user-456"}
87
+
88
+ # See who's sharing with me
89
+ GET /api/v1/shared-with-me
90
+ {
91
+ "data": [
92
+ {
93
+ "user_id": "user-789",
94
+ "name": "Alice",
95
+ "email": "alice@example.com",
96
+ "message_count": 150,
97
+ "session_count": 5,
98
+ "first_message_at": "2024-01-01T00:00:00Z",
99
+ "last_message_at": "2024-03-15T12:00:00Z"
100
+ }
101
+ ],
102
+ "metadata": {"total": 1, "page": 1, "page_size": 50, ...}
103
+ }
104
+
105
+ # Get messages from Alice's shared sessions
106
+ GET /api/v1/shared-with-me/user-789/messages?page=1&page_size=50
107
+
108
+ # Revoke a share
109
+ DELETE /api/v1/sessions/abc-123/share/user-456
110
+ """
111
+
112
+ from datetime import datetime
113
+ from typing import Optional
114
+
115
+ from pydantic import BaseModel, Field
116
+
117
+ from ..core import CoreModel
118
+
119
+
120
+ class SharedSession(CoreModel):
121
+ """
122
+ Session sharing record between users.
123
+
124
+ Links a session (identified by session_id from Message records) to a
125
+ recipient user, enabling collaborative access to conversation history.
126
+ """
127
+
128
+ session_id: str = Field(
129
+ ...,
130
+ description="The session being shared (matches Message.session_id)",
131
+ )
132
+ owner_user_id: str = Field(
133
+ ...,
134
+ description="User ID of the session owner (the sharer)",
135
+ )
136
+ shared_with_user_id: str = Field(
137
+ ...,
138
+ description="User ID of the recipient (who can now view the session)",
139
+ )
140
+
141
+
142
+ class SharedSessionCreate(BaseModel):
143
+ """Request to create a session share."""
144
+
145
+ shared_with_user_id: str = Field(
146
+ ...,
147
+ description="User ID to share the session with",
148
+ )
149
+
150
+
151
+ class SharedWithMeSummary(BaseModel):
152
+ """
153
+ Aggregate summary of a user sharing sessions with you.
154
+
155
+ Returned by GET /api/v1/shared-with-me endpoint.
156
+ """
157
+
158
+ user_id: str = Field(description="User ID of the person sharing with you")
159
+ name: Optional[str] = Field(default=None, description="User's display name")
160
+ email: Optional[str] = Field(default=None, description="User's email address")
161
+ message_count: int = Field(description="Total messages across all shared sessions")
162
+ session_count: int = Field(description="Number of sessions shared with you")
163
+ first_message_at: Optional[datetime] = Field(
164
+ default=None,
165
+ description="Timestamp of earliest message in shared sessions",
166
+ )
167
+ last_message_at: Optional[datetime] = Field(
168
+ default=None,
169
+ description="Timestamp of most recent message in shared sessions",
170
+ )
171
+
172
+
173
+ class SharedWithMeResponse(BaseModel):
174
+ """Response for paginated shared-with-me query."""
175
+
176
+ object: str = "list"
177
+ data: list[SharedWithMeSummary] = Field(
178
+ description="List of users sharing sessions with you"
179
+ )
180
+ metadata: dict = Field(description="Pagination metadata")
@@ -22,9 +22,12 @@ from ..core import CoreModel
22
22
  class UserTier(str, Enum):
23
23
  """User subscription tier for feature gating."""
24
24
 
25
+ ANONYMOUS = "anonymous"
25
26
  FREE = "free"
26
- SILVER = "silver"
27
- GOLD = "gold"
27
+ BASIC = "basic"
28
+ PRO = "pro"
29
+ SILVER = "silver" # Deprecated? Keeping for backward compatibility if needed
30
+ GOLD = "gold" # Deprecated? Keeping for backward compatibility if needed
28
31
 
29
32
 
30
33
  class User(CoreModel):
@@ -57,7 +60,11 @@ class User(CoreModel):
57
60
  )
58
61
  tier: UserTier = Field(
59
62
  default=UserTier.FREE,
60
- description="User subscription tier (free, silver, gold) for feature gating",
63
+ description="User subscription tier (free, basic, pro) for feature gating",
64
+ )
65
+ anonymous_ids: list[str] = Field(
66
+ default_factory=list,
67
+ description="Linked anonymous session IDs used for merging history",
61
68
  )
62
69
  sec_policy: dict = Field(
63
70
  default_factory=dict,