remdb 0.2.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of remdb might be problematic. Click here for more details.

Files changed (187) hide show
  1. rem/__init__.py +2 -0
  2. rem/agentic/README.md +650 -0
  3. rem/agentic/__init__.py +39 -0
  4. rem/agentic/agents/README.md +155 -0
  5. rem/agentic/agents/__init__.py +8 -0
  6. rem/agentic/context.py +148 -0
  7. rem/agentic/context_builder.py +329 -0
  8. rem/agentic/mcp/__init__.py +0 -0
  9. rem/agentic/mcp/tool_wrapper.py +107 -0
  10. rem/agentic/otel/__init__.py +5 -0
  11. rem/agentic/otel/setup.py +151 -0
  12. rem/agentic/providers/phoenix.py +674 -0
  13. rem/agentic/providers/pydantic_ai.py +572 -0
  14. rem/agentic/query.py +117 -0
  15. rem/agentic/query_helper.py +89 -0
  16. rem/agentic/schema.py +396 -0
  17. rem/agentic/serialization.py +245 -0
  18. rem/agentic/tools/__init__.py +5 -0
  19. rem/agentic/tools/rem_tools.py +231 -0
  20. rem/api/README.md +420 -0
  21. rem/api/main.py +324 -0
  22. rem/api/mcp_router/prompts.py +182 -0
  23. rem/api/mcp_router/resources.py +536 -0
  24. rem/api/mcp_router/server.py +213 -0
  25. rem/api/mcp_router/tools.py +584 -0
  26. rem/api/routers/auth.py +229 -0
  27. rem/api/routers/chat/__init__.py +5 -0
  28. rem/api/routers/chat/completions.py +281 -0
  29. rem/api/routers/chat/json_utils.py +76 -0
  30. rem/api/routers/chat/models.py +124 -0
  31. rem/api/routers/chat/streaming.py +185 -0
  32. rem/auth/README.md +258 -0
  33. rem/auth/__init__.py +26 -0
  34. rem/auth/middleware.py +100 -0
  35. rem/auth/providers/__init__.py +13 -0
  36. rem/auth/providers/base.py +376 -0
  37. rem/auth/providers/google.py +163 -0
  38. rem/auth/providers/microsoft.py +237 -0
  39. rem/cli/README.md +455 -0
  40. rem/cli/__init__.py +8 -0
  41. rem/cli/commands/README.md +126 -0
  42. rem/cli/commands/__init__.py +3 -0
  43. rem/cli/commands/ask.py +565 -0
  44. rem/cli/commands/configure.py +423 -0
  45. rem/cli/commands/db.py +493 -0
  46. rem/cli/commands/dreaming.py +324 -0
  47. rem/cli/commands/experiments.py +1124 -0
  48. rem/cli/commands/mcp.py +66 -0
  49. rem/cli/commands/process.py +245 -0
  50. rem/cli/commands/schema.py +183 -0
  51. rem/cli/commands/serve.py +106 -0
  52. rem/cli/dreaming.py +363 -0
  53. rem/cli/main.py +88 -0
  54. rem/config.py +237 -0
  55. rem/mcp_server.py +41 -0
  56. rem/models/core/__init__.py +49 -0
  57. rem/models/core/core_model.py +64 -0
  58. rem/models/core/engram.py +333 -0
  59. rem/models/core/experiment.py +628 -0
  60. rem/models/core/inline_edge.py +132 -0
  61. rem/models/core/rem_query.py +243 -0
  62. rem/models/entities/__init__.py +43 -0
  63. rem/models/entities/file.py +57 -0
  64. rem/models/entities/image_resource.py +88 -0
  65. rem/models/entities/message.py +35 -0
  66. rem/models/entities/moment.py +123 -0
  67. rem/models/entities/ontology.py +191 -0
  68. rem/models/entities/ontology_config.py +131 -0
  69. rem/models/entities/resource.py +95 -0
  70. rem/models/entities/schema.py +87 -0
  71. rem/models/entities/user.py +85 -0
  72. rem/py.typed +0 -0
  73. rem/schemas/README.md +507 -0
  74. rem/schemas/__init__.py +6 -0
  75. rem/schemas/agents/README.md +92 -0
  76. rem/schemas/agents/core/moment-builder.yaml +178 -0
  77. rem/schemas/agents/core/rem-query-agent.yaml +226 -0
  78. rem/schemas/agents/core/resource-affinity-assessor.yaml +99 -0
  79. rem/schemas/agents/core/simple-assistant.yaml +19 -0
  80. rem/schemas/agents/core/user-profile-builder.yaml +163 -0
  81. rem/schemas/agents/examples/contract-analyzer.yaml +317 -0
  82. rem/schemas/agents/examples/contract-extractor.yaml +134 -0
  83. rem/schemas/agents/examples/cv-parser.yaml +263 -0
  84. rem/schemas/agents/examples/hello-world.yaml +37 -0
  85. rem/schemas/agents/examples/query.yaml +54 -0
  86. rem/schemas/agents/examples/simple.yaml +21 -0
  87. rem/schemas/agents/examples/test.yaml +29 -0
  88. rem/schemas/agents/rem.yaml +128 -0
  89. rem/schemas/evaluators/hello-world/default.yaml +77 -0
  90. rem/schemas/evaluators/rem/faithfulness.yaml +219 -0
  91. rem/schemas/evaluators/rem/lookup-correctness.yaml +182 -0
  92. rem/schemas/evaluators/rem/retrieval-precision.yaml +199 -0
  93. rem/schemas/evaluators/rem/retrieval-recall.yaml +211 -0
  94. rem/schemas/evaluators/rem/search-correctness.yaml +192 -0
  95. rem/services/__init__.py +16 -0
  96. rem/services/audio/INTEGRATION.md +308 -0
  97. rem/services/audio/README.md +376 -0
  98. rem/services/audio/__init__.py +15 -0
  99. rem/services/audio/chunker.py +354 -0
  100. rem/services/audio/transcriber.py +259 -0
  101. rem/services/content/README.md +1269 -0
  102. rem/services/content/__init__.py +5 -0
  103. rem/services/content/providers.py +806 -0
  104. rem/services/content/service.py +657 -0
  105. rem/services/dreaming/README.md +230 -0
  106. rem/services/dreaming/__init__.py +53 -0
  107. rem/services/dreaming/affinity_service.py +336 -0
  108. rem/services/dreaming/moment_service.py +264 -0
  109. rem/services/dreaming/ontology_service.py +54 -0
  110. rem/services/dreaming/user_model_service.py +297 -0
  111. rem/services/dreaming/utils.py +39 -0
  112. rem/services/embeddings/__init__.py +11 -0
  113. rem/services/embeddings/api.py +120 -0
  114. rem/services/embeddings/worker.py +421 -0
  115. rem/services/fs/README.md +662 -0
  116. rem/services/fs/__init__.py +62 -0
  117. rem/services/fs/examples.py +206 -0
  118. rem/services/fs/examples_paths.py +204 -0
  119. rem/services/fs/git_provider.py +935 -0
  120. rem/services/fs/local_provider.py +760 -0
  121. rem/services/fs/parsing-hooks-examples.md +172 -0
  122. rem/services/fs/paths.py +276 -0
  123. rem/services/fs/provider.py +460 -0
  124. rem/services/fs/s3_provider.py +1042 -0
  125. rem/services/fs/service.py +186 -0
  126. rem/services/git/README.md +1075 -0
  127. rem/services/git/__init__.py +17 -0
  128. rem/services/git/service.py +469 -0
  129. rem/services/phoenix/EXPERIMENT_DESIGN.md +1146 -0
  130. rem/services/phoenix/README.md +453 -0
  131. rem/services/phoenix/__init__.py +46 -0
  132. rem/services/phoenix/client.py +686 -0
  133. rem/services/phoenix/config.py +88 -0
  134. rem/services/phoenix/prompt_labels.py +477 -0
  135. rem/services/postgres/README.md +575 -0
  136. rem/services/postgres/__init__.py +23 -0
  137. rem/services/postgres/migration_service.py +427 -0
  138. rem/services/postgres/pydantic_to_sqlalchemy.py +232 -0
  139. rem/services/postgres/register_type.py +352 -0
  140. rem/services/postgres/repository.py +337 -0
  141. rem/services/postgres/schema_generator.py +379 -0
  142. rem/services/postgres/service.py +802 -0
  143. rem/services/postgres/sql_builder.py +354 -0
  144. rem/services/rem/README.md +304 -0
  145. rem/services/rem/__init__.py +23 -0
  146. rem/services/rem/exceptions.py +71 -0
  147. rem/services/rem/executor.py +293 -0
  148. rem/services/rem/parser.py +145 -0
  149. rem/services/rem/queries.py +196 -0
  150. rem/services/rem/query.py +371 -0
  151. rem/services/rem/service.py +527 -0
  152. rem/services/session/README.md +374 -0
  153. rem/services/session/__init__.py +6 -0
  154. rem/services/session/compression.py +360 -0
  155. rem/services/session/reload.py +77 -0
  156. rem/settings.py +1235 -0
  157. rem/sql/002_install_models.sql +1068 -0
  158. rem/sql/background_indexes.sql +42 -0
  159. rem/sql/install_models.sql +1038 -0
  160. rem/sql/migrations/001_install.sql +503 -0
  161. rem/sql/migrations/002_install_models.sql +1202 -0
  162. rem/utils/AGENTIC_CHUNKING.md +597 -0
  163. rem/utils/README.md +583 -0
  164. rem/utils/__init__.py +43 -0
  165. rem/utils/agentic_chunking.py +622 -0
  166. rem/utils/batch_ops.py +343 -0
  167. rem/utils/chunking.py +108 -0
  168. rem/utils/clip_embeddings.py +276 -0
  169. rem/utils/dict_utils.py +98 -0
  170. rem/utils/embeddings.py +423 -0
  171. rem/utils/examples/embeddings_example.py +305 -0
  172. rem/utils/examples/sql_types_example.py +202 -0
  173. rem/utils/markdown.py +16 -0
  174. rem/utils/model_helpers.py +236 -0
  175. rem/utils/schema_loader.py +229 -0
  176. rem/utils/sql_types.py +348 -0
  177. rem/utils/user_id.py +81 -0
  178. rem/utils/vision.py +330 -0
  179. rem/workers/README.md +506 -0
  180. rem/workers/__init__.py +5 -0
  181. rem/workers/dreaming.py +502 -0
  182. rem/workers/engram_processor.py +312 -0
  183. rem/workers/sqs_file_processor.py +193 -0
  184. remdb-0.2.6.dist-info/METADATA +1191 -0
  185. remdb-0.2.6.dist-info/RECORD +187 -0
  186. remdb-0.2.6.dist-info/WHEEL +4 -0
  187. remdb-0.2.6.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,324 @@
1
+ """CLI commands for dreaming worker operations.
2
+
3
+ Commands:
4
+ rem dreaming user-model - Update user profiles
5
+ rem dreaming moments - Extract temporal narratives
6
+ rem dreaming affinity - Build resource relationships
7
+ rem dreaming custom - Run custom extractors
8
+ rem dreaming full - Run complete workflow
9
+ """
10
+
11
+ import asyncio
12
+ from typing import Optional
13
+
14
+ import click
15
+ from loguru import logger
16
+
17
+
18
+ def register_commands(dreaming: click.Group):
19
+ """Register dreaming commands."""
20
+
21
+ @dreaming.command("user-model")
22
+ @click.option("--user-id", required=True, help="User ID to process")
23
+ @click.option("--time-window-days", type=int, default=30, help="Days to look back")
24
+ @click.option("--max-sessions", type=int, default=100, help="Max sessions to analyze")
25
+ @click.option("--max-moments", type=int, default=20, help="Max moments to include")
26
+ @click.option("--max-resources", type=int, default=20, help="Max resources to include")
27
+ def user_model(
28
+ user_id: str,
29
+ time_window_days: int,
30
+ max_sessions: int,
31
+ max_moments: int,
32
+ max_resources: int,
33
+ ):
34
+ """Update user model from recent activity.
35
+
36
+ Example:
37
+ rem dreaming user-model --user-id user-123
38
+ rem dreaming user-model --user-id sarah-chen
39
+ """
40
+ from ...workers.dreaming import DreamingWorker
41
+
42
+ async def run():
43
+ worker = DreamingWorker()
44
+ try:
45
+ logger.info(f"Updating user model for user: {user_id}")
46
+ result = await worker.update_user_model(
47
+ user_id=user_id,
48
+ time_window_days=time_window_days,
49
+ max_sessions=max_sessions,
50
+ max_moments=max_moments,
51
+ max_resources=max_resources,
52
+ )
53
+
54
+ if result["status"] == "success":
55
+ logger.success(f"User model updated successfully!")
56
+ logger.info(f"Messages analyzed: {result.get('messages_analyzed', 0)}")
57
+ logger.info(f"Moments included: {result.get('moments_included', 0)}")
58
+ logger.info(f"Resources included: {result.get('resources_included', 0)}")
59
+ logger.info(f"Activity level: {result.get('activity_level', 'N/A')}")
60
+ else:
61
+ logger.warning(f"Status: {result['status']}")
62
+
63
+ finally:
64
+ await worker.close()
65
+
66
+ asyncio.run(run())
67
+
68
+ @dreaming.command("moments")
69
+ @click.option("--user-id", required=True, help="User ID to process")
70
+ @click.option("--lookback-hours", type=int, help="Hours to look back")
71
+ @click.option("--limit", type=int, help="Max resources to process")
72
+ def moments(
73
+ user_id: str,
74
+ lookback_hours: Optional[int],
75
+ limit: Optional[int],
76
+ ):
77
+ """Extract temporal narratives from resources.
78
+
79
+ Example:
80
+ rem dreaming moments --user-id user-123 --lookback-hours 48
81
+ rem dreaming moments --user-id sarah-chen
82
+ """
83
+ from ...workers.dreaming import DreamingWorker
84
+
85
+ async def run():
86
+ worker = DreamingWorker()
87
+ try:
88
+ logger.info(f"Constructing moments for user: {user_id}")
89
+ result = await worker.construct_moments(
90
+ user_id=user_id,
91
+ lookback_hours=lookback_hours,
92
+ limit=limit,
93
+ )
94
+
95
+ if result["status"] == "success":
96
+ logger.success(f"Moments constructed successfully!")
97
+ logger.info(f"Resources queried: {result['resources_queried']}")
98
+ logger.info(f"Sessions queried: {result['sessions_queried']}")
99
+ logger.info(f"Moments created: {result['moments_created']}")
100
+ logger.info(f"Graph edges added: {result['graph_edges_added']}")
101
+ logger.info(f"Analysis: {result.get('analysis_summary', 'N/A')[:200]}")
102
+ else:
103
+ logger.warning(f"Status: {result['status']}")
104
+
105
+ finally:
106
+ await worker.close()
107
+
108
+ asyncio.run(run())
109
+
110
+ @dreaming.command("affinity")
111
+ @click.option("--user-id", required=True, help="User ID to process")
112
+ @click.option("--use-llm", is_flag=True, help="Use LLM mode (expensive)")
113
+ @click.option("--lookback-hours", type=int, help="Hours to look back")
114
+ @click.option("--limit", type=int, help="Max resources (REQUIRED for LLM mode)")
115
+ @click.option("--similarity-threshold", type=float, default=0.7, help="Min similarity (semantic mode)")
116
+ @click.option("--top-k", type=int, default=3, help="Max similar resources per resource")
117
+ def affinity(
118
+ user_id: str,
119
+ use_llm: bool,
120
+ lookback_hours: Optional[int],
121
+ limit: Optional[int],
122
+ similarity_threshold: float,
123
+ top_k: int,
124
+ ):
125
+ """Build semantic relationships between resources.
126
+
127
+ Semantic mode (default): Fast vector similarity
128
+ LLM mode (--use-llm): Intelligent but expensive
129
+
130
+ Examples:
131
+ rem dreaming affinity --user-id user-123
132
+ rem dreaming affinity --user-id user-123 --use-llm --limit 100
133
+ rem dreaming affinity --user-id sarah-chen
134
+ """
135
+ from ...workers.dreaming import DreamingWorker
136
+ from ...services.dreaming.affinity_service import AffinityMode
137
+
138
+ if use_llm and not limit:
139
+ logger.error("--limit is REQUIRED when using --use-llm to control costs")
140
+ raise click.ClickException("--limit is required with --use-llm")
141
+
142
+ async def run():
143
+ worker = DreamingWorker()
144
+ try:
145
+ mode = AffinityMode.LLM if use_llm else AffinityMode.SEMANTIC
146
+ logger.info(f"Building {mode.value} affinity for user: {user_id}")
147
+
148
+ result = await worker.build_affinity(
149
+ user_id=user_id,
150
+ mode=mode,
151
+ lookback_hours=lookback_hours,
152
+ limit=limit,
153
+ similarity_threshold=similarity_threshold,
154
+ top_k=top_k,
155
+ )
156
+
157
+ if result["status"] == "success":
158
+ logger.success(f"Resource affinity built successfully!")
159
+ logger.info(f"Resources processed: {result['resources_processed']}")
160
+ logger.info(f"Edges created: {result['edges_created']}")
161
+ if mode == AffinityMode.LLM:
162
+ logger.info(f"LLM calls made: {result['llm_calls_made']}")
163
+ else:
164
+ logger.warning(f"Status: {result['status']}")
165
+
166
+ finally:
167
+ await worker.close()
168
+
169
+ asyncio.run(run())
170
+
171
+ @dreaming.command("custom")
172
+ @click.option("--user-id", required=True, help="User ID to process")
173
+ @click.option("--extractor", required=True, help="Extractor schema ID (e.g., cv-parser-v1)")
174
+ @click.option("--lookback-hours", type=int, help="Hours to look back")
175
+ @click.option("--limit", type=int, help="Max resources/files to process")
176
+ @click.option("--provider", help="Optional LLM provider override")
177
+ @click.option("--model", help="Optional model override")
178
+ def custom(
179
+ user_id: str,
180
+ extractor: str,
181
+ lookback_hours: Optional[int],
182
+ limit: Optional[int],
183
+ provider: Optional[str],
184
+ model: Optional[str],
185
+ ):
186
+ """Run custom extractor on user's resources and files.
187
+
188
+ Loads the user's recent resources/files and runs them through
189
+ a custom extractor agent to extract domain-specific knowledge.
190
+
191
+ Examples:
192
+ # Extract from CVs
193
+ rem dreaming custom --user-id user-123 --extractor cv-parser-v1
194
+
195
+ # Extract from contracts with lookback
196
+ rem dreaming custom --user-id user-123 --extractor contract-analyzer-v1 \\
197
+ --lookback-hours 168 --limit 50
198
+
199
+ # Override provider
200
+ rem dreaming custom --user-id user-123 --extractor cv-parser-v1 \\
201
+ --provider anthropic --model claude-sonnet-4-5
202
+ """
203
+ logger.warning("Not implemented yet")
204
+ logger.info(f"Would run extractor '{extractor}' for user: {user_id}")
205
+ if lookback_hours:
206
+ logger.info(f"Lookback: {lookback_hours} hours")
207
+ if limit:
208
+ logger.info(f"Limit: {limit} items")
209
+ if provider:
210
+ logger.info(f"Provider override: {provider}")
211
+ if model:
212
+ logger.info(f"Model override: {model}")
213
+
214
+ @dreaming.command("full")
215
+ @click.option("--user-id", help="User ID (or --all-users)")
216
+ @click.option("--all-users", is_flag=True, help="Process all active users")
217
+ @click.option("--use-llm-affinity", is_flag=True, help="Use LLM mode for affinity")
218
+ @click.option("--lookback-hours", type=int, help="Hours to look back")
219
+ @click.option("--skip-extractors", is_flag=True, help="Skip custom extractors")
220
+ def full(
221
+ user_id: Optional[str],
222
+ all_users: bool,
223
+ use_llm_affinity: bool,
224
+ lookback_hours: Optional[int],
225
+ skip_extractors: bool,
226
+ ):
227
+ """Run complete dreaming workflow.
228
+
229
+ Executes all operations in sequence:
230
+ 1. Custom extractors (if configs exist)
231
+ 2. User model update
232
+ 3. Moment construction
233
+ 4. Resource affinity
234
+
235
+ Examples:
236
+ # Process single user
237
+ rem dreaming full --user-id user-123
238
+
239
+ # Process user with LLM affinity
240
+ rem dreaming full --user-id sarah-chen --use-llm-affinity
241
+
242
+ # Process all active users (daily cron)
243
+ rem dreaming full --all-users
244
+
245
+ # Skip extractors (faster)
246
+ rem dreaming full --user-id user-123 --skip-extractors
247
+ """
248
+ from ...workers.dreaming import DreamingWorker
249
+
250
+ if not user_id and not all_users:
251
+ logger.error("Either --user-id or --all-users is required")
252
+ raise click.ClickException("Either --user-id or --all-users required")
253
+
254
+ if user_id and all_users:
255
+ logger.error("Cannot use both --user-id and --all-users")
256
+ raise click.ClickException("Cannot use both --user-id and --all-users")
257
+
258
+ async def run():
259
+ worker = DreamingWorker()
260
+ try:
261
+ if all_users:
262
+ logger.warning("--all-users not implemented yet")
263
+ logger.info("Would process all active users")
264
+ # TODO: Implement process_all_users() method
265
+ else:
266
+ logger.info(f"Running full dreaming workflow for user: {user_id}")
267
+ if use_llm_affinity:
268
+ logger.warning("Using LLM affinity mode (expensive)")
269
+
270
+ result = await worker.process_full(
271
+ user_id=user_id,
272
+ use_llm_affinity=use_llm_affinity,
273
+ lookback_hours=lookback_hours,
274
+ extract_ontologies=not skip_extractors,
275
+ )
276
+
277
+ logger.success("Dreaming workflow completed!")
278
+ logger.info("\n=== Results Summary ===")
279
+
280
+ # Ontologies
281
+ if not skip_extractors:
282
+ ont = result.get("ontologies", {})
283
+ if ont.get("error"):
284
+ logger.error(f"Ontologies: {ont['error']}")
285
+ else:
286
+ logger.info(
287
+ f"Ontologies: {ont.get('ontologies_created', 0)} created"
288
+ )
289
+
290
+ # User model
291
+ user = result.get("user_model", {})
292
+ if user.get("error"):
293
+ logger.error(f"User model: {user['error']}")
294
+ else:
295
+ logger.info(
296
+ f"User model: {user.get('sessions_analyzed', 0)} sessions, "
297
+ f"{user.get('moments_included', 0)} moments, "
298
+ f"{user.get('current_projects', 0)} projects"
299
+ )
300
+
301
+ # Moments
302
+ moments = result.get("moments", {})
303
+ if moments.get("error"):
304
+ logger.error(f"Moments: {moments['error']}")
305
+ else:
306
+ logger.info(
307
+ f"Moments: {moments.get('moments_created', 0)} created, "
308
+ f"{moments.get('graph_edges_added', 0)} edges"
309
+ )
310
+
311
+ # Affinity
312
+ aff = result.get("affinity", {})
313
+ if aff.get("error"):
314
+ logger.error(f"Affinity: {aff['error']}")
315
+ else:
316
+ logger.info(
317
+ f"Affinity: {aff.get('resources_processed', 0)} resources, "
318
+ f"{aff.get('edges_created', 0)} edges created"
319
+ )
320
+
321
+ finally:
322
+ await worker.close()
323
+
324
+ asyncio.run(run())