remdb 0.3.146__py3-none-any.whl → 0.3.181__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of remdb might be problematic. Click here for more details.
- rem/agentic/agents/__init__.py +16 -0
- rem/agentic/agents/agent_manager.py +311 -0
- rem/agentic/context.py +81 -3
- rem/agentic/context_builder.py +36 -9
- rem/agentic/mcp/tool_wrapper.py +43 -14
- rem/agentic/providers/pydantic_ai.py +76 -34
- rem/agentic/schema.py +4 -3
- rem/agentic/tools/rem_tools.py +11 -0
- rem/api/deps.py +3 -5
- rem/api/main.py +22 -3
- rem/api/mcp_router/resources.py +75 -14
- rem/api/mcp_router/server.py +28 -23
- rem/api/mcp_router/tools.py +177 -2
- rem/api/middleware/tracking.py +5 -5
- rem/api/routers/auth.py +352 -6
- rem/api/routers/chat/completions.py +5 -3
- rem/api/routers/chat/streaming.py +95 -22
- rem/api/routers/messages.py +24 -15
- rem/auth/__init__.py +13 -3
- rem/auth/jwt.py +352 -0
- rem/auth/middleware.py +70 -30
- rem/auth/providers/__init__.py +4 -1
- rem/auth/providers/email.py +215 -0
- rem/cli/commands/ask.py +1 -1
- rem/cli/commands/db.py +118 -54
- rem/models/entities/__init__.py +4 -0
- rem/models/entities/ontology.py +93 -101
- rem/models/entities/subscriber.py +175 -0
- rem/models/entities/user.py +1 -0
- rem/schemas/agents/core/agent-builder.yaml +235 -0
- rem/services/__init__.py +3 -1
- rem/services/content/service.py +4 -3
- rem/services/email/__init__.py +10 -0
- rem/services/email/service.py +522 -0
- rem/services/email/templates.py +360 -0
- rem/services/embeddings/worker.py +26 -12
- rem/services/postgres/README.md +38 -0
- rem/services/postgres/diff_service.py +19 -3
- rem/services/postgres/pydantic_to_sqlalchemy.py +37 -2
- rem/services/postgres/register_type.py +1 -1
- rem/services/postgres/repository.py +37 -25
- rem/services/postgres/schema_generator.py +5 -5
- rem/services/postgres/sql_builder.py +6 -5
- rem/services/session/compression.py +113 -50
- rem/services/session/reload.py +14 -7
- rem/services/user_service.py +41 -9
- rem/settings.py +182 -1
- rem/sql/background_indexes.sql +5 -0
- rem/sql/migrations/001_install.sql +33 -4
- rem/sql/migrations/002_install_models.sql +204 -186
- rem/sql/migrations/005_schema_update.sql +145 -0
- rem/utils/model_helpers.py +101 -0
- rem/utils/schema_loader.py +45 -7
- {remdb-0.3.146.dist-info → remdb-0.3.181.dist-info}/METADATA +1 -1
- {remdb-0.3.146.dist-info → remdb-0.3.181.dist-info}/RECORD +57 -48
- {remdb-0.3.146.dist-info → remdb-0.3.181.dist-info}/WHEEL +0 -0
- {remdb-0.3.146.dist-info → remdb-0.3.181.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Email Templates.
|
|
3
|
+
|
|
4
|
+
HTML email templates for transactional emails.
|
|
5
|
+
Uses inline CSS for maximum email client compatibility.
|
|
6
|
+
|
|
7
|
+
Downstream apps can customize by:
|
|
8
|
+
1. Overriding COLORS dict
|
|
9
|
+
2. Setting custom LOGO_URL and TAGLINE
|
|
10
|
+
3. Creating custom templates using base_template()
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# Default colors - override in downstream apps
|
|
18
|
+
COLORS = {
|
|
19
|
+
"background": "#F5F5F5",
|
|
20
|
+
"foreground": "#333333",
|
|
21
|
+
"primary": "#4A90D9",
|
|
22
|
+
"accent": "#5CB85C",
|
|
23
|
+
"card": "#FFFFFF",
|
|
24
|
+
"muted": "#6b7280",
|
|
25
|
+
"border": "#E0E0E0",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
# Branding - override in downstream apps
|
|
29
|
+
LOGO_URL: str | None = None
|
|
30
|
+
APP_NAME = "REM"
|
|
31
|
+
TAGLINE = "Your AI-powered platform"
|
|
32
|
+
WEBSITE_URL = "https://rem.ai"
|
|
33
|
+
PRIVACY_URL = "https://rem.ai/privacy"
|
|
34
|
+
TERMS_URL = "https://rem.ai/terms"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class EmailTemplate:
|
|
39
|
+
"""Email template with subject and HTML body."""
|
|
40
|
+
|
|
41
|
+
subject: str
|
|
42
|
+
html_body: str
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def base_template(
|
|
46
|
+
content: str,
|
|
47
|
+
preheader: Optional[str] = None,
|
|
48
|
+
colors: dict | None = None,
|
|
49
|
+
logo_url: str | None = None,
|
|
50
|
+
app_name: str | None = None,
|
|
51
|
+
tagline: str | None = None,
|
|
52
|
+
website_url: str | None = None,
|
|
53
|
+
privacy_url: str | None = None,
|
|
54
|
+
terms_url: str | None = None,
|
|
55
|
+
) -> str:
|
|
56
|
+
"""
|
|
57
|
+
Base email template.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
content: The main content HTML
|
|
61
|
+
preheader: Optional preview text shown in email clients
|
|
62
|
+
colors: Color overrides (merges with COLORS)
|
|
63
|
+
logo_url: Logo image URL
|
|
64
|
+
app_name: Application name
|
|
65
|
+
tagline: Footer tagline
|
|
66
|
+
website_url: Main website URL
|
|
67
|
+
privacy_url: Privacy policy URL
|
|
68
|
+
terms_url: Terms of service URL
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
Complete HTML email
|
|
72
|
+
"""
|
|
73
|
+
# Merge colors
|
|
74
|
+
c = {**COLORS, **(colors or {})}
|
|
75
|
+
|
|
76
|
+
# Use provided values or module defaults
|
|
77
|
+
logo = logo_url or LOGO_URL
|
|
78
|
+
name = app_name or APP_NAME
|
|
79
|
+
tag = tagline or TAGLINE
|
|
80
|
+
web = website_url or WEBSITE_URL
|
|
81
|
+
privacy = privacy_url or PRIVACY_URL
|
|
82
|
+
terms = terms_url or TERMS_URL
|
|
83
|
+
|
|
84
|
+
preheader_html = ""
|
|
85
|
+
if preheader:
|
|
86
|
+
preheader_html = f'''
|
|
87
|
+
<div style="display: none; max-height: 0; overflow: hidden;">
|
|
88
|
+
{preheader}
|
|
89
|
+
</div>
|
|
90
|
+
'''
|
|
91
|
+
|
|
92
|
+
logo_html = ""
|
|
93
|
+
if logo:
|
|
94
|
+
logo_html = f'''
|
|
95
|
+
<img src="{logo}" alt="{name}" width="40" height="40" style="display: block; margin: 0 auto 16px auto; border-radius: 8px;">
|
|
96
|
+
'''
|
|
97
|
+
|
|
98
|
+
return f'''<!DOCTYPE html>
|
|
99
|
+
<html lang="en">
|
|
100
|
+
<head>
|
|
101
|
+
<meta charset="UTF-8">
|
|
102
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
103
|
+
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
104
|
+
<title>{name}</title>
|
|
105
|
+
<!--[if mso]>
|
|
106
|
+
<style type="text/css">
|
|
107
|
+
body, table, td {{font-family: Arial, sans-serif !important;}}
|
|
108
|
+
</style>
|
|
109
|
+
<![endif]-->
|
|
110
|
+
</head>
|
|
111
|
+
<body style="margin: 0; padding: 0; background-color: {c['background']}; font-family: 'Georgia', 'Times New Roman', serif;">
|
|
112
|
+
{preheader_html}
|
|
113
|
+
|
|
114
|
+
<!-- Email Container -->
|
|
115
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%" style="background-color: {c['background']};">
|
|
116
|
+
<tr>
|
|
117
|
+
<td style="padding: 40px 20px;">
|
|
118
|
+
<!-- Inner Container -->
|
|
119
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%" style="max-width: 560px; margin: 0 auto;">
|
|
120
|
+
|
|
121
|
+
<!-- Main Content Card -->
|
|
122
|
+
<tr>
|
|
123
|
+
<td style="background-color: {c['card']}; border-radius: 16px; box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05);">
|
|
124
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%">
|
|
125
|
+
<tr>
|
|
126
|
+
<td style="padding: 40px;">
|
|
127
|
+
{content}
|
|
128
|
+
</td>
|
|
129
|
+
</tr>
|
|
130
|
+
</table>
|
|
131
|
+
</td>
|
|
132
|
+
</tr>
|
|
133
|
+
|
|
134
|
+
<!-- Footer -->
|
|
135
|
+
<tr>
|
|
136
|
+
<td style="padding: 32px 20px; text-align: center;">
|
|
137
|
+
{logo_html}
|
|
138
|
+
|
|
139
|
+
<!-- Tagline -->
|
|
140
|
+
<p style="margin: 0 0 8px 0; font-family: 'Georgia', serif; font-size: 14px; color: {c['muted']}; font-style: italic;">
|
|
141
|
+
{tag}
|
|
142
|
+
</p>
|
|
143
|
+
|
|
144
|
+
<!-- Footer Links -->
|
|
145
|
+
<p style="margin: 0; font-family: Arial, sans-serif; font-size: 12px; color: {c['muted']};">
|
|
146
|
+
<a href="{web}" style="color: {c['primary']}; text-decoration: none;">{name}</a>
|
|
147
|
+
•
|
|
148
|
+
<a href="{privacy}" style="color: {c['primary']}; text-decoration: none;">Privacy</a>
|
|
149
|
+
•
|
|
150
|
+
<a href="{terms}" style="color: {c['primary']}; text-decoration: none;">Terms</a>
|
|
151
|
+
</p>
|
|
152
|
+
|
|
153
|
+
<!-- Copyright -->
|
|
154
|
+
<p style="margin: 16px 0 0 0; font-family: Arial, sans-serif; font-size: 11px; color: {c['muted']};">
|
|
155
|
+
© 2025 {name}. All rights reserved.
|
|
156
|
+
</p>
|
|
157
|
+
</td>
|
|
158
|
+
</tr>
|
|
159
|
+
|
|
160
|
+
</table>
|
|
161
|
+
</td>
|
|
162
|
+
</tr>
|
|
163
|
+
</table>
|
|
164
|
+
</body>
|
|
165
|
+
</html>'''
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def login_code_template(
|
|
169
|
+
code: str,
|
|
170
|
+
email: str,
|
|
171
|
+
colors: dict | None = None,
|
|
172
|
+
app_name: str | None = None,
|
|
173
|
+
**kwargs,
|
|
174
|
+
) -> EmailTemplate:
|
|
175
|
+
"""
|
|
176
|
+
Generate a login code email template.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
code: The 6-digit login code
|
|
180
|
+
email: The recipient's email address
|
|
181
|
+
colors: Color overrides
|
|
182
|
+
app_name: Application name
|
|
183
|
+
**kwargs: Additional arguments passed to base_template
|
|
184
|
+
|
|
185
|
+
Returns:
|
|
186
|
+
EmailTemplate with subject and HTML body
|
|
187
|
+
"""
|
|
188
|
+
c = {**COLORS, **(colors or {})}
|
|
189
|
+
name = app_name or APP_NAME
|
|
190
|
+
|
|
191
|
+
# Format code with spaces for readability (e.g., "123 456")
|
|
192
|
+
formatted_code = f"{code[:3]} {code[3:]}" if len(code) == 6 else code
|
|
193
|
+
|
|
194
|
+
content = f'''
|
|
195
|
+
<!-- Greeting -->
|
|
196
|
+
<h1 style="margin: 0 0 8px 0; font-family: 'Arial', sans-serif; font-size: 24px; font-weight: 600; color: {c['foreground']};">
|
|
197
|
+
Hi there!
|
|
198
|
+
</h1>
|
|
199
|
+
|
|
200
|
+
<p style="margin: 0 0 24px 0; font-family: 'Georgia', serif; font-size: 16px; line-height: 1.6; color: {c['foreground']};">
|
|
201
|
+
Here's your login code for {name}. Enter this code to securely access your account.
|
|
202
|
+
</p>
|
|
203
|
+
|
|
204
|
+
<!-- Code Box -->
|
|
205
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%" style="margin: 0 0 24px 0;">
|
|
206
|
+
<tr>
|
|
207
|
+
<td align="center">
|
|
208
|
+
<div style="
|
|
209
|
+
display: inline-block;
|
|
210
|
+
padding: 20px 40px;
|
|
211
|
+
background-color: {c['background']};
|
|
212
|
+
border: 2px solid {c['border']};
|
|
213
|
+
border-radius: 12px;
|
|
214
|
+
">
|
|
215
|
+
<span style="
|
|
216
|
+
font-family: 'Courier New', monospace;
|
|
217
|
+
font-size: 32px;
|
|
218
|
+
font-weight: 700;
|
|
219
|
+
letter-spacing: 6px;
|
|
220
|
+
color: {c['foreground']};
|
|
221
|
+
">{formatted_code}</span>
|
|
222
|
+
</div>
|
|
223
|
+
</td>
|
|
224
|
+
</tr>
|
|
225
|
+
</table>
|
|
226
|
+
|
|
227
|
+
<!-- Instructions -->
|
|
228
|
+
<p style="margin: 0 0 8px 0; font-family: 'Georgia', serif; font-size: 14px; line-height: 1.6; color: {c['muted']};">
|
|
229
|
+
This code expires in <strong>10 minutes</strong>.
|
|
230
|
+
</p>
|
|
231
|
+
|
|
232
|
+
<p style="margin: 0 0 24px 0; font-family: 'Georgia', serif; font-size: 14px; line-height: 1.6; color: {c['muted']};">
|
|
233
|
+
If you didn't request this code, you can safely ignore this email.
|
|
234
|
+
</p>
|
|
235
|
+
|
|
236
|
+
<!-- Divider -->
|
|
237
|
+
<hr style="border: none; border-top: 1px solid {c['border']}; margin: 24px 0;">
|
|
238
|
+
|
|
239
|
+
<!-- Security Note -->
|
|
240
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%">
|
|
241
|
+
<tr>
|
|
242
|
+
<td style="padding: 16px; background-color: {c['background']}; border-radius: 8px;">
|
|
243
|
+
<p style="margin: 0; font-family: Arial, sans-serif; font-size: 12px; color: {c['muted']};">
|
|
244
|
+
<strong style="color: {c['primary']};">Security tip:</strong>
|
|
245
|
+
Never share your login code with anyone. {name} will never ask for your code via phone or text.
|
|
246
|
+
</p>
|
|
247
|
+
</td>
|
|
248
|
+
</tr>
|
|
249
|
+
</table>
|
|
250
|
+
'''
|
|
251
|
+
|
|
252
|
+
return EmailTemplate(
|
|
253
|
+
subject=f"Your {name} Login Code",
|
|
254
|
+
html_body=base_template(
|
|
255
|
+
content=content,
|
|
256
|
+
preheader=f"Your login code is {formatted_code}",
|
|
257
|
+
colors=colors,
|
|
258
|
+
app_name=app_name,
|
|
259
|
+
**kwargs,
|
|
260
|
+
),
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def welcome_template(
|
|
265
|
+
name: Optional[str] = None,
|
|
266
|
+
colors: dict | None = None,
|
|
267
|
+
app_name: str | None = None,
|
|
268
|
+
features: list[str] | None = None,
|
|
269
|
+
cta_url: str | None = None,
|
|
270
|
+
cta_text: str = "Get Started",
|
|
271
|
+
**kwargs,
|
|
272
|
+
) -> EmailTemplate:
|
|
273
|
+
"""
|
|
274
|
+
Generate a welcome email template for new users.
|
|
275
|
+
|
|
276
|
+
Args:
|
|
277
|
+
name: Optional user's name
|
|
278
|
+
colors: Color overrides
|
|
279
|
+
app_name: Application name
|
|
280
|
+
features: List of feature descriptions
|
|
281
|
+
cta_url: Call-to-action button URL
|
|
282
|
+
cta_text: Call-to-action button text
|
|
283
|
+
**kwargs: Additional arguments passed to base_template
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
EmailTemplate with subject and HTML body
|
|
287
|
+
"""
|
|
288
|
+
c = {**COLORS, **(colors or {})}
|
|
289
|
+
app = app_name or APP_NAME
|
|
290
|
+
greeting = f"Hi {name}!" if name else "Welcome!"
|
|
291
|
+
url = cta_url or WEBSITE_URL
|
|
292
|
+
|
|
293
|
+
# Default features if not provided
|
|
294
|
+
default_features = [
|
|
295
|
+
"Access powerful AI capabilities",
|
|
296
|
+
"Store and organize your data",
|
|
297
|
+
"Collaborate with your team",
|
|
298
|
+
]
|
|
299
|
+
feature_list = features or default_features
|
|
300
|
+
|
|
301
|
+
features_html = "".join(f"<li>{f}</li>" for f in feature_list)
|
|
302
|
+
|
|
303
|
+
content = f'''
|
|
304
|
+
<!-- Greeting -->
|
|
305
|
+
<h1 style="margin: 0 0 8px 0; font-family: 'Arial', sans-serif; font-size: 24px; font-weight: 600; color: {c['foreground']};">
|
|
306
|
+
{greeting}
|
|
307
|
+
</h1>
|
|
308
|
+
|
|
309
|
+
<p style="margin: 0 0 24px 0; font-family: 'Georgia', serif; font-size: 16px; line-height: 1.6; color: {c['foreground']};">
|
|
310
|
+
Welcome to {app}! We're excited to have you on board.
|
|
311
|
+
</p>
|
|
312
|
+
|
|
313
|
+
<!-- Feature List -->
|
|
314
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%" style="margin: 0 0 24px 0;">
|
|
315
|
+
<tr>
|
|
316
|
+
<td style="padding: 16px; background-color: {c['background']}; border-radius: 8px;">
|
|
317
|
+
<p style="margin: 0 0 12px 0; font-family: Arial, sans-serif; font-size: 14px; font-weight: 600; color: {c['primary']};">
|
|
318
|
+
What you can do with {app}:
|
|
319
|
+
</p>
|
|
320
|
+
<ul style="margin: 0; padding-left: 20px; font-family: 'Georgia', serif; font-size: 14px; line-height: 1.8; color: {c['foreground']};">
|
|
321
|
+
{features_html}
|
|
322
|
+
</ul>
|
|
323
|
+
</td>
|
|
324
|
+
</tr>
|
|
325
|
+
</table>
|
|
326
|
+
|
|
327
|
+
<!-- CTA Button -->
|
|
328
|
+
<table role="presentation" cellspacing="0" cellpadding="0" border="0" width="100%" style="margin: 0 0 24px 0;">
|
|
329
|
+
<tr>
|
|
330
|
+
<td align="center">
|
|
331
|
+
<a href="{url}" style="
|
|
332
|
+
display: inline-block;
|
|
333
|
+
padding: 14px 32px;
|
|
334
|
+
background-color: {c['primary']};
|
|
335
|
+
color: #FFFFFF;
|
|
336
|
+
font-family: Arial, sans-serif;
|
|
337
|
+
font-size: 16px;
|
|
338
|
+
font-weight: 600;
|
|
339
|
+
text-decoration: none;
|
|
340
|
+
border-radius: 8px;
|
|
341
|
+
">{cta_text}</a>
|
|
342
|
+
</td>
|
|
343
|
+
</tr>
|
|
344
|
+
</table>
|
|
345
|
+
|
|
346
|
+
<p style="margin: 0; font-family: 'Georgia', serif; font-size: 14px; line-height: 1.6; color: {c['muted']}; text-align: center;">
|
|
347
|
+
We're glad you're here!
|
|
348
|
+
</p>
|
|
349
|
+
'''
|
|
350
|
+
|
|
351
|
+
return EmailTemplate(
|
|
352
|
+
subject=f"Welcome to {app}",
|
|
353
|
+
html_body=base_template(
|
|
354
|
+
content=content,
|
|
355
|
+
preheader=f"Welcome to {app}! Get started today.",
|
|
356
|
+
colors=colors,
|
|
357
|
+
app_name=app_name,
|
|
358
|
+
**kwargs,
|
|
359
|
+
),
|
|
360
|
+
)
|
|
@@ -23,6 +23,8 @@ Future:
|
|
|
23
23
|
import asyncio
|
|
24
24
|
import os
|
|
25
25
|
from typing import Any, Optional
|
|
26
|
+
import hashlib
|
|
27
|
+
import uuid
|
|
26
28
|
from uuid import uuid4
|
|
27
29
|
|
|
28
30
|
import httpx
|
|
@@ -108,6 +110,7 @@ class EmbeddingWorker:
|
|
|
108
110
|
self.task_queue: asyncio.Queue = asyncio.Queue()
|
|
109
111
|
self.workers: list[asyncio.Task] = []
|
|
110
112
|
self.running = False
|
|
113
|
+
self._in_flight_count = 0 # Track tasks being processed (not just in queue)
|
|
111
114
|
|
|
112
115
|
# Store API key for direct HTTP requests
|
|
113
116
|
from ...settings import settings
|
|
@@ -143,17 +146,18 @@ class EmbeddingWorker:
|
|
|
143
146
|
return
|
|
144
147
|
|
|
145
148
|
queue_size = self.task_queue.qsize()
|
|
146
|
-
|
|
149
|
+
in_flight = self._in_flight_count
|
|
150
|
+
logger.debug(f"Stopping EmbeddingWorker (queue={queue_size}, in_flight={in_flight})")
|
|
147
151
|
|
|
148
|
-
# Wait for queue to drain
|
|
152
|
+
# Wait for both queue to drain AND in-flight tasks to complete
|
|
149
153
|
max_wait = 30 # 30 seconds max
|
|
150
154
|
waited = 0.0
|
|
151
|
-
while not self.task_queue.empty() and waited < max_wait:
|
|
155
|
+
while (not self.task_queue.empty() or self._in_flight_count > 0) and waited < max_wait:
|
|
152
156
|
await asyncio.sleep(0.5)
|
|
153
157
|
waited += 0.5
|
|
154
158
|
|
|
155
|
-
if not self.task_queue.empty():
|
|
156
|
-
remaining = self.task_queue.qsize()
|
|
159
|
+
if not self.task_queue.empty() or self._in_flight_count > 0:
|
|
160
|
+
remaining = self.task_queue.qsize() + self._in_flight_count
|
|
157
161
|
logger.warning(
|
|
158
162
|
f"EmbeddingWorker timeout: {remaining} tasks remaining after {max_wait}s"
|
|
159
163
|
)
|
|
@@ -205,12 +209,18 @@ class EmbeddingWorker:
|
|
|
205
209
|
if not batch:
|
|
206
210
|
continue
|
|
207
211
|
|
|
208
|
-
|
|
212
|
+
# Track in-flight tasks
|
|
213
|
+
self._in_flight_count += len(batch)
|
|
209
214
|
|
|
210
|
-
|
|
211
|
-
await self._process_batch(batch)
|
|
215
|
+
logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
|
|
212
216
|
|
|
213
|
-
|
|
217
|
+
try:
|
|
218
|
+
# Generate embeddings for batch
|
|
219
|
+
await self._process_batch(batch)
|
|
220
|
+
logger.debug(f"Worker {worker_id} completed batch")
|
|
221
|
+
finally:
|
|
222
|
+
# Always decrement in-flight count, even on error
|
|
223
|
+
self._in_flight_count -= len(batch)
|
|
214
224
|
|
|
215
225
|
except asyncio.CancelledError:
|
|
216
226
|
logger.debug(f"Worker {worker_id} cancelled")
|
|
@@ -373,7 +383,11 @@ class EmbeddingWorker:
|
|
|
373
383
|
for task, embedding in zip(tasks, embeddings):
|
|
374
384
|
table_name = f"embeddings_{task.table_name}"
|
|
375
385
|
|
|
376
|
-
#
|
|
386
|
+
# Generate deterministic ID from key fields (entity_id, field_name, provider)
|
|
387
|
+
key_string = f"{task.entity_id}:{task.field_name}:{task.provider}"
|
|
388
|
+
embedding_id = str(uuid.UUID(hashlib.md5(key_string.encode()).hexdigest()))
|
|
389
|
+
|
|
390
|
+
# Build upsert SQL - conflict on deterministic ID
|
|
377
391
|
sql = f"""
|
|
378
392
|
INSERT INTO {table_name} (
|
|
379
393
|
id,
|
|
@@ -386,7 +400,7 @@ class EmbeddingWorker:
|
|
|
386
400
|
updated_at
|
|
387
401
|
)
|
|
388
402
|
VALUES ($1, $2, $3, $4, $5, $6, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
389
|
-
ON CONFLICT (
|
|
403
|
+
ON CONFLICT (id)
|
|
390
404
|
DO UPDATE SET
|
|
391
405
|
model = EXCLUDED.model,
|
|
392
406
|
embedding = EXCLUDED.embedding,
|
|
@@ -400,7 +414,7 @@ class EmbeddingWorker:
|
|
|
400
414
|
await self.postgres_service.execute(
|
|
401
415
|
sql,
|
|
402
416
|
(
|
|
403
|
-
|
|
417
|
+
embedding_id,
|
|
404
418
|
task.entity_id,
|
|
405
419
|
task.field_name,
|
|
406
420
|
task.provider,
|
rem/services/postgres/README.md
CHANGED
|
@@ -688,6 +688,44 @@ from rem.api.main import app # Use REM's FastAPI app
|
|
|
688
688
|
# Or build your own app using rem.services
|
|
689
689
|
```
|
|
690
690
|
|
|
691
|
+
## Adding Models & Migrations
|
|
692
|
+
|
|
693
|
+
Quick workflow for adding new database models:
|
|
694
|
+
|
|
695
|
+
1. **Create a model** in `models/__init__.py` (or a submodule):
|
|
696
|
+
```python
|
|
697
|
+
import rem
|
|
698
|
+
from rem.models.core import CoreModel
|
|
699
|
+
|
|
700
|
+
@rem.register_model
|
|
701
|
+
class MyEntity(CoreModel):
|
|
702
|
+
name: str
|
|
703
|
+
description: str # Auto-embedded (common field name)
|
|
704
|
+
```
|
|
705
|
+
|
|
706
|
+
2. **Check for schema drift** - REM auto-detects `./models` directory:
|
|
707
|
+
```bash
|
|
708
|
+
rem db diff # Show pending changes (additive only)
|
|
709
|
+
rem db diff --strategy full # Include destructive changes
|
|
710
|
+
```
|
|
711
|
+
|
|
712
|
+
3. **Generate migration** (optional - for version-controlled SQL):
|
|
713
|
+
```bash
|
|
714
|
+
rem db diff --generate # Creates numbered .sql file
|
|
715
|
+
```
|
|
716
|
+
|
|
717
|
+
4. **Apply changes**:
|
|
718
|
+
```bash
|
|
719
|
+
rem db migrate # Apply all pending migrations
|
|
720
|
+
```
|
|
721
|
+
|
|
722
|
+
**Key points:**
|
|
723
|
+
- Models in `./models/` are auto-discovered (must have `__init__.py`)
|
|
724
|
+
- Or set `MODELS__IMPORT_MODULES=myapp.models` for custom paths
|
|
725
|
+
- `CoreModel` provides: `id`, `tenant_id`, `user_id`, `created_at`, `updated_at`, `deleted_at`, `graph_edges`, `metadata`, `tags`
|
|
726
|
+
- Fields named `content`, `description`, `summary`, `text`, `body`, `message`, `notes` get embeddings by default
|
|
727
|
+
- Use `Field(json_schema_extra={"embed": True})` to embed other fields
|
|
728
|
+
|
|
691
729
|
## Configuration
|
|
692
730
|
|
|
693
731
|
Environment variables:
|
|
@@ -22,6 +22,7 @@ from alembic.runtime.migration import MigrationContext
|
|
|
22
22
|
from alembic.script import ScriptDirectory
|
|
23
23
|
from loguru import logger
|
|
24
24
|
from sqlalchemy import create_engine, text
|
|
25
|
+
from sqlalchemy.dialects import postgresql
|
|
25
26
|
|
|
26
27
|
from ...settings import settings
|
|
27
28
|
from .pydantic_to_sqlalchemy import get_target_metadata
|
|
@@ -472,6 +473,18 @@ class DiffService:
|
|
|
472
473
|
|
|
473
474
|
return "\n".join(lines) + "\n"
|
|
474
475
|
|
|
476
|
+
def _compile_type(self, col_type) -> str:
|
|
477
|
+
"""Compile SQLAlchemy type to PostgreSQL DDL string.
|
|
478
|
+
|
|
479
|
+
SQLAlchemy types like ARRAY(Text) need dialect-specific compilation
|
|
480
|
+
to render correctly (e.g., "TEXT[]" instead of just "ARRAY").
|
|
481
|
+
"""
|
|
482
|
+
try:
|
|
483
|
+
return col_type.compile(dialect=postgresql.dialect())
|
|
484
|
+
except Exception:
|
|
485
|
+
# Fallback to string representation if compilation fails
|
|
486
|
+
return str(col_type)
|
|
487
|
+
|
|
475
488
|
def _op_to_sql(self, op: ops.MigrateOperation) -> list[str]:
|
|
476
489
|
"""Convert operation to SQL statements."""
|
|
477
490
|
lines = []
|
|
@@ -481,7 +494,8 @@ class DiffService:
|
|
|
481
494
|
for col in op.columns:
|
|
482
495
|
if hasattr(col, 'name') and hasattr(col, 'type'):
|
|
483
496
|
nullable = "" if getattr(col, 'nullable', True) else " NOT NULL"
|
|
484
|
-
|
|
497
|
+
type_str = self._compile_type(col.type)
|
|
498
|
+
cols.append(f" {col.name} {type_str}{nullable}")
|
|
485
499
|
col_str = ",\n".join(cols)
|
|
486
500
|
lines.append(f"CREATE TABLE IF NOT EXISTS {op.table_name} (\n{col_str}\n);")
|
|
487
501
|
|
|
@@ -491,14 +505,16 @@ class DiffService:
|
|
|
491
505
|
elif isinstance(op, ops.AddColumnOp):
|
|
492
506
|
col = op.column
|
|
493
507
|
nullable = "" if getattr(col, 'nullable', True) else " NOT NULL"
|
|
494
|
-
|
|
508
|
+
type_str = self._compile_type(col.type)
|
|
509
|
+
lines.append(f"ALTER TABLE {op.table_name} ADD COLUMN IF NOT EXISTS {col.name} {type_str}{nullable};")
|
|
495
510
|
|
|
496
511
|
elif isinstance(op, ops.DropColumnOp):
|
|
497
512
|
lines.append(f"ALTER TABLE {op.table_name} DROP COLUMN IF EXISTS {op.column_name};")
|
|
498
513
|
|
|
499
514
|
elif isinstance(op, ops.AlterColumnOp):
|
|
500
515
|
if op.modify_type is not None:
|
|
501
|
-
|
|
516
|
+
type_str = self._compile_type(op.modify_type)
|
|
517
|
+
lines.append(f"ALTER TABLE {op.table_name} ALTER COLUMN {op.column_name} TYPE {type_str};")
|
|
502
518
|
if op.modify_nullable is not None:
|
|
503
519
|
if op.modify_nullable:
|
|
504
520
|
lines.append(f"ALTER TABLE {op.table_name} ALTER COLUMN {op.column_name} DROP NOT NULL;")
|
|
@@ -494,12 +494,13 @@ def _build_embeddings_table(base_table_name: str, metadata: MetaData) -> Table:
|
|
|
494
494
|
]
|
|
495
495
|
|
|
496
496
|
# Create table with unique constraint
|
|
497
|
-
#
|
|
497
|
+
# Truncate constraint name to fit PostgreSQL's 63-char identifier limit
|
|
498
|
+
constraint_name = f"uq_{base_table_name[:30]}_emb_entity_field_prov"
|
|
498
499
|
table = Table(
|
|
499
500
|
embeddings_table_name,
|
|
500
501
|
metadata,
|
|
501
502
|
*columns,
|
|
502
|
-
UniqueConstraint("entity_id", "field_name", "provider", name=
|
|
503
|
+
UniqueConstraint("entity_id", "field_name", "provider", name=constraint_name),
|
|
503
504
|
)
|
|
504
505
|
|
|
505
506
|
# Add indexes (matching register_type output)
|
|
@@ -509,6 +510,30 @@ def _build_embeddings_table(base_table_name: str, metadata: MetaData) -> Table:
|
|
|
509
510
|
return table
|
|
510
511
|
|
|
511
512
|
|
|
513
|
+
def _import_model_modules() -> list[str]:
|
|
514
|
+
"""
|
|
515
|
+
Import modules specified in MODELS__IMPORT_MODULES setting.
|
|
516
|
+
|
|
517
|
+
This ensures downstream models decorated with @rem.register_model
|
|
518
|
+
are registered before schema generation.
|
|
519
|
+
|
|
520
|
+
Returns:
|
|
521
|
+
List of successfully imported module names
|
|
522
|
+
"""
|
|
523
|
+
import importlib
|
|
524
|
+
from ...settings import settings
|
|
525
|
+
|
|
526
|
+
imported = []
|
|
527
|
+
for module_name in settings.models.module_list:
|
|
528
|
+
try:
|
|
529
|
+
importlib.import_module(module_name)
|
|
530
|
+
imported.append(module_name)
|
|
531
|
+
logger.debug(f"Imported model module: {module_name}")
|
|
532
|
+
except ImportError as e:
|
|
533
|
+
logger.warning(f"Failed to import model module '{module_name}': {e}")
|
|
534
|
+
return imported
|
|
535
|
+
|
|
536
|
+
|
|
512
537
|
def get_target_metadata() -> MetaData:
|
|
513
538
|
"""
|
|
514
539
|
Get SQLAlchemy metadata for Alembic autogenerate.
|
|
@@ -519,9 +544,19 @@ def get_target_metadata() -> MetaData:
|
|
|
519
544
|
- Core REM models (Resource, Message, User, etc.)
|
|
520
545
|
- User-registered models via @rem.register_model decorator
|
|
521
546
|
|
|
547
|
+
Before building metadata, imports model modules from settings to ensure
|
|
548
|
+
downstream models are registered. This supports:
|
|
549
|
+
- Auto-detection of ./models directory (convention)
|
|
550
|
+
- MODELS__IMPORT_MODULES env var (explicit configuration)
|
|
551
|
+
|
|
522
552
|
Returns:
|
|
523
553
|
SQLAlchemy MetaData object representing all registered Pydantic models
|
|
524
554
|
"""
|
|
555
|
+
# Import model modules first (auto-detects ./models or uses MODELS__IMPORT_MODULES)
|
|
556
|
+
imported = _import_model_modules()
|
|
557
|
+
if imported:
|
|
558
|
+
logger.info(f"Imported model modules: {imported}")
|
|
559
|
+
|
|
525
560
|
# build_sqlalchemy_metadata_from_pydantic uses the registry internally,
|
|
526
561
|
# so no directory path is needed (the parameter is kept for backwards compat)
|
|
527
562
|
return build_sqlalchemy_metadata_from_pydantic()
|