remdb 0.3.242__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of remdb might be problematic. Click here for more details.

Files changed (235) hide show
  1. rem/__init__.py +129 -0
  2. rem/agentic/README.md +760 -0
  3. rem/agentic/__init__.py +54 -0
  4. rem/agentic/agents/README.md +155 -0
  5. rem/agentic/agents/__init__.py +38 -0
  6. rem/agentic/agents/agent_manager.py +311 -0
  7. rem/agentic/agents/sse_simulator.py +502 -0
  8. rem/agentic/context.py +425 -0
  9. rem/agentic/context_builder.py +360 -0
  10. rem/agentic/llm_provider_models.py +301 -0
  11. rem/agentic/mcp/__init__.py +0 -0
  12. rem/agentic/mcp/tool_wrapper.py +273 -0
  13. rem/agentic/otel/__init__.py +5 -0
  14. rem/agentic/otel/setup.py +240 -0
  15. rem/agentic/providers/phoenix.py +926 -0
  16. rem/agentic/providers/pydantic_ai.py +854 -0
  17. rem/agentic/query.py +117 -0
  18. rem/agentic/query_helper.py +89 -0
  19. rem/agentic/schema.py +737 -0
  20. rem/agentic/serialization.py +245 -0
  21. rem/agentic/tools/__init__.py +5 -0
  22. rem/agentic/tools/rem_tools.py +242 -0
  23. rem/api/README.md +657 -0
  24. rem/api/deps.py +253 -0
  25. rem/api/main.py +460 -0
  26. rem/api/mcp_router/prompts.py +182 -0
  27. rem/api/mcp_router/resources.py +820 -0
  28. rem/api/mcp_router/server.py +243 -0
  29. rem/api/mcp_router/tools.py +1605 -0
  30. rem/api/middleware/tracking.py +172 -0
  31. rem/api/routers/admin.py +520 -0
  32. rem/api/routers/auth.py +898 -0
  33. rem/api/routers/chat/__init__.py +5 -0
  34. rem/api/routers/chat/child_streaming.py +394 -0
  35. rem/api/routers/chat/completions.py +702 -0
  36. rem/api/routers/chat/json_utils.py +76 -0
  37. rem/api/routers/chat/models.py +202 -0
  38. rem/api/routers/chat/otel_utils.py +33 -0
  39. rem/api/routers/chat/sse_events.py +546 -0
  40. rem/api/routers/chat/streaming.py +950 -0
  41. rem/api/routers/chat/streaming_utils.py +327 -0
  42. rem/api/routers/common.py +18 -0
  43. rem/api/routers/dev.py +87 -0
  44. rem/api/routers/feedback.py +276 -0
  45. rem/api/routers/messages.py +620 -0
  46. rem/api/routers/models.py +86 -0
  47. rem/api/routers/query.py +362 -0
  48. rem/api/routers/shared_sessions.py +422 -0
  49. rem/auth/README.md +258 -0
  50. rem/auth/__init__.py +36 -0
  51. rem/auth/jwt.py +367 -0
  52. rem/auth/middleware.py +318 -0
  53. rem/auth/providers/__init__.py +16 -0
  54. rem/auth/providers/base.py +376 -0
  55. rem/auth/providers/email.py +215 -0
  56. rem/auth/providers/google.py +163 -0
  57. rem/auth/providers/microsoft.py +237 -0
  58. rem/cli/README.md +517 -0
  59. rem/cli/__init__.py +8 -0
  60. rem/cli/commands/README.md +299 -0
  61. rem/cli/commands/__init__.py +3 -0
  62. rem/cli/commands/ask.py +549 -0
  63. rem/cli/commands/cluster.py +1808 -0
  64. rem/cli/commands/configure.py +495 -0
  65. rem/cli/commands/db.py +828 -0
  66. rem/cli/commands/dreaming.py +324 -0
  67. rem/cli/commands/experiments.py +1698 -0
  68. rem/cli/commands/mcp.py +66 -0
  69. rem/cli/commands/process.py +388 -0
  70. rem/cli/commands/query.py +109 -0
  71. rem/cli/commands/scaffold.py +47 -0
  72. rem/cli/commands/schema.py +230 -0
  73. rem/cli/commands/serve.py +106 -0
  74. rem/cli/commands/session.py +453 -0
  75. rem/cli/dreaming.py +363 -0
  76. rem/cli/main.py +123 -0
  77. rem/config.py +244 -0
  78. rem/mcp_server.py +41 -0
  79. rem/models/core/__init__.py +49 -0
  80. rem/models/core/core_model.py +70 -0
  81. rem/models/core/engram.py +333 -0
  82. rem/models/core/experiment.py +672 -0
  83. rem/models/core/inline_edge.py +132 -0
  84. rem/models/core/rem_query.py +246 -0
  85. rem/models/entities/__init__.py +68 -0
  86. rem/models/entities/domain_resource.py +38 -0
  87. rem/models/entities/feedback.py +123 -0
  88. rem/models/entities/file.py +57 -0
  89. rem/models/entities/image_resource.py +88 -0
  90. rem/models/entities/message.py +64 -0
  91. rem/models/entities/moment.py +123 -0
  92. rem/models/entities/ontology.py +181 -0
  93. rem/models/entities/ontology_config.py +131 -0
  94. rem/models/entities/resource.py +95 -0
  95. rem/models/entities/schema.py +87 -0
  96. rem/models/entities/session.py +84 -0
  97. rem/models/entities/shared_session.py +180 -0
  98. rem/models/entities/subscriber.py +175 -0
  99. rem/models/entities/user.py +93 -0
  100. rem/py.typed +0 -0
  101. rem/registry.py +373 -0
  102. rem/schemas/README.md +507 -0
  103. rem/schemas/__init__.py +6 -0
  104. rem/schemas/agents/README.md +92 -0
  105. rem/schemas/agents/core/agent-builder.yaml +235 -0
  106. rem/schemas/agents/core/moment-builder.yaml +178 -0
  107. rem/schemas/agents/core/rem-query-agent.yaml +226 -0
  108. rem/schemas/agents/core/resource-affinity-assessor.yaml +99 -0
  109. rem/schemas/agents/core/simple-assistant.yaml +19 -0
  110. rem/schemas/agents/core/user-profile-builder.yaml +163 -0
  111. rem/schemas/agents/examples/contract-analyzer.yaml +317 -0
  112. rem/schemas/agents/examples/contract-extractor.yaml +134 -0
  113. rem/schemas/agents/examples/cv-parser.yaml +263 -0
  114. rem/schemas/agents/examples/hello-world.yaml +37 -0
  115. rem/schemas/agents/examples/query.yaml +54 -0
  116. rem/schemas/agents/examples/simple.yaml +21 -0
  117. rem/schemas/agents/examples/test.yaml +29 -0
  118. rem/schemas/agents/rem.yaml +132 -0
  119. rem/schemas/evaluators/hello-world/default.yaml +77 -0
  120. rem/schemas/evaluators/rem/faithfulness.yaml +219 -0
  121. rem/schemas/evaluators/rem/lookup-correctness.yaml +182 -0
  122. rem/schemas/evaluators/rem/retrieval-precision.yaml +199 -0
  123. rem/schemas/evaluators/rem/retrieval-recall.yaml +211 -0
  124. rem/schemas/evaluators/rem/search-correctness.yaml +192 -0
  125. rem/services/__init__.py +18 -0
  126. rem/services/audio/INTEGRATION.md +308 -0
  127. rem/services/audio/README.md +376 -0
  128. rem/services/audio/__init__.py +15 -0
  129. rem/services/audio/chunker.py +354 -0
  130. rem/services/audio/transcriber.py +259 -0
  131. rem/services/content/README.md +1269 -0
  132. rem/services/content/__init__.py +5 -0
  133. rem/services/content/providers.py +760 -0
  134. rem/services/content/service.py +762 -0
  135. rem/services/dreaming/README.md +230 -0
  136. rem/services/dreaming/__init__.py +53 -0
  137. rem/services/dreaming/affinity_service.py +322 -0
  138. rem/services/dreaming/moment_service.py +251 -0
  139. rem/services/dreaming/ontology_service.py +54 -0
  140. rem/services/dreaming/user_model_service.py +297 -0
  141. rem/services/dreaming/utils.py +39 -0
  142. rem/services/email/__init__.py +10 -0
  143. rem/services/email/service.py +522 -0
  144. rem/services/email/templates.py +360 -0
  145. rem/services/embeddings/__init__.py +11 -0
  146. rem/services/embeddings/api.py +127 -0
  147. rem/services/embeddings/worker.py +435 -0
  148. rem/services/fs/README.md +662 -0
  149. rem/services/fs/__init__.py +62 -0
  150. rem/services/fs/examples.py +206 -0
  151. rem/services/fs/examples_paths.py +204 -0
  152. rem/services/fs/git_provider.py +935 -0
  153. rem/services/fs/local_provider.py +760 -0
  154. rem/services/fs/parsing-hooks-examples.md +172 -0
  155. rem/services/fs/paths.py +276 -0
  156. rem/services/fs/provider.py +460 -0
  157. rem/services/fs/s3_provider.py +1042 -0
  158. rem/services/fs/service.py +186 -0
  159. rem/services/git/README.md +1075 -0
  160. rem/services/git/__init__.py +17 -0
  161. rem/services/git/service.py +469 -0
  162. rem/services/phoenix/EXPERIMENT_DESIGN.md +1146 -0
  163. rem/services/phoenix/README.md +453 -0
  164. rem/services/phoenix/__init__.py +46 -0
  165. rem/services/phoenix/client.py +960 -0
  166. rem/services/phoenix/config.py +88 -0
  167. rem/services/phoenix/prompt_labels.py +477 -0
  168. rem/services/postgres/README.md +757 -0
  169. rem/services/postgres/__init__.py +49 -0
  170. rem/services/postgres/diff_service.py +599 -0
  171. rem/services/postgres/migration_service.py +427 -0
  172. rem/services/postgres/programmable_diff_service.py +635 -0
  173. rem/services/postgres/pydantic_to_sqlalchemy.py +562 -0
  174. rem/services/postgres/register_type.py +353 -0
  175. rem/services/postgres/repository.py +481 -0
  176. rem/services/postgres/schema_generator.py +661 -0
  177. rem/services/postgres/service.py +802 -0
  178. rem/services/postgres/sql_builder.py +355 -0
  179. rem/services/rate_limit.py +113 -0
  180. rem/services/rem/README.md +318 -0
  181. rem/services/rem/__init__.py +23 -0
  182. rem/services/rem/exceptions.py +71 -0
  183. rem/services/rem/executor.py +293 -0
  184. rem/services/rem/parser.py +180 -0
  185. rem/services/rem/queries.py +196 -0
  186. rem/services/rem/query.py +371 -0
  187. rem/services/rem/service.py +608 -0
  188. rem/services/session/README.md +374 -0
  189. rem/services/session/__init__.py +13 -0
  190. rem/services/session/compression.py +488 -0
  191. rem/services/session/pydantic_messages.py +310 -0
  192. rem/services/session/reload.py +85 -0
  193. rem/services/user_service.py +130 -0
  194. rem/settings.py +1877 -0
  195. rem/sql/background_indexes.sql +52 -0
  196. rem/sql/migrations/001_install.sql +983 -0
  197. rem/sql/migrations/002_install_models.sql +3157 -0
  198. rem/sql/migrations/003_optional_extensions.sql +326 -0
  199. rem/sql/migrations/004_cache_system.sql +282 -0
  200. rem/sql/migrations/005_schema_update.sql +145 -0
  201. rem/sql/migrations/migrate_session_id_to_uuid.sql +45 -0
  202. rem/utils/AGENTIC_CHUNKING.md +597 -0
  203. rem/utils/README.md +628 -0
  204. rem/utils/__init__.py +61 -0
  205. rem/utils/agentic_chunking.py +622 -0
  206. rem/utils/batch_ops.py +343 -0
  207. rem/utils/chunking.py +108 -0
  208. rem/utils/clip_embeddings.py +276 -0
  209. rem/utils/constants.py +97 -0
  210. rem/utils/date_utils.py +228 -0
  211. rem/utils/dict_utils.py +98 -0
  212. rem/utils/embeddings.py +436 -0
  213. rem/utils/examples/embeddings_example.py +305 -0
  214. rem/utils/examples/sql_types_example.py +202 -0
  215. rem/utils/files.py +323 -0
  216. rem/utils/markdown.py +16 -0
  217. rem/utils/mime_types.py +158 -0
  218. rem/utils/model_helpers.py +492 -0
  219. rem/utils/schema_loader.py +649 -0
  220. rem/utils/sql_paths.py +146 -0
  221. rem/utils/sql_types.py +350 -0
  222. rem/utils/user_id.py +81 -0
  223. rem/utils/vision.py +325 -0
  224. rem/workers/README.md +506 -0
  225. rem/workers/__init__.py +7 -0
  226. rem/workers/db_listener.py +579 -0
  227. rem/workers/db_maintainer.py +74 -0
  228. rem/workers/dreaming.py +502 -0
  229. rem/workers/engram_processor.py +312 -0
  230. rem/workers/sqs_file_processor.py +193 -0
  231. rem/workers/unlogged_maintainer.py +463 -0
  232. remdb-0.3.242.dist-info/METADATA +1632 -0
  233. remdb-0.3.242.dist-info/RECORD +235 -0
  234. remdb-0.3.242.dist-info/WHEEL +4 -0
  235. remdb-0.3.242.dist-info/entry_points.txt +2 -0
rem/agentic/query.py ADDED
@@ -0,0 +1,117 @@
1
+ """
2
+ Agent query model for structured agent input.
3
+
4
+ Design pattern for standardized agent query structure with:
5
+ - Primary query (user question/task)
6
+ - Knowledge context (retrieved context, documentation)
7
+ - Scratchpad (working memory, session state)
8
+
9
+ Key Design Pattern
10
+ - Separates query from retrieval (query is what user asks, knowledge is what we retrieve)
11
+ - Scratchpad enables multi-turn reasoning and state tracking
12
+ - Supports markdown + fenced JSON for structured data
13
+ - Converts to single prompt string for agent consumption
14
+ """
15
+
16
+ from typing import Any
17
+
18
+ from pydantic import BaseModel, Field
19
+
20
+
21
+ class AgentQuery(BaseModel):
22
+ """
23
+ Standard query structure for agent execution.
24
+
25
+ Provides consistent structure for queries, knowledge context, and
26
+ working memory across all agent types.
27
+
28
+ Design Pattern
29
+ - query: User's question/task (markdown + fenced JSON)
30
+ - knowledge: Retrieved context from REM queries (markdown + fenced JSON)
31
+ - scratchpad: Working memory for multi-turn reasoning (JSON or markdown)
32
+
33
+ Example:
34
+ query = AgentQuery(
35
+ query="Find all documents Sarah authored",
36
+ knowledge=\"\"\"
37
+ # Entity Information
38
+ Sarah Chen (person/employee)
39
+ - Role: Senior Engineer
40
+ - Projects: [Project Alpha, TiDB Migration]
41
+ \"\"\",
42
+ scratchpad={"current_case": "TAP-1234", "stage": "entity_lookup"}
43
+ )
44
+
45
+ prompt = query.to_prompt()
46
+ result = await agent.run(prompt)
47
+ """
48
+
49
+ query: str = Field(
50
+ ...,
51
+ description="Primary user query or task (markdown format, may include fenced JSON)",
52
+ examples=[
53
+ "Find all documents Sarah authored",
54
+ "What happened in Q4 retrospective?",
55
+ "TRAVERSE manages WITH LOOKUP sarah-chen DEPTH 2",
56
+ ],
57
+ )
58
+
59
+ knowledge: str = Field(
60
+ default="",
61
+ description="Background knowledge and context (markdown, may include fenced JSON/code)",
62
+ examples=[
63
+ "# Entity: sarah-chen\nType: person/employee\nRole: Senior Engineer",
64
+ "Retrieved resources:\n```json\n[{...}]\n```",
65
+ ],
66
+ )
67
+
68
+ scratchpad: str | dict[str, Any] = Field(
69
+ default="",
70
+ description="Working memory for session state (JSON object or markdown with fenced JSON)",
71
+ examples=[
72
+ {"stage": "lookup", "visited_entities": ["sarah-chen"]},
73
+ "# Session State\n\nStage: TRAVERSE depth 1\n\n```json\n{\"nodes\": [...]}\n```",
74
+ ],
75
+ )
76
+
77
+ def to_prompt(self) -> str:
78
+ """
79
+ Convert query components to single prompt string.
80
+
81
+ Combines query, knowledge, and scratchpad into formatted prompt
82
+ for agent consumption.
83
+
84
+ Returns:
85
+ Formatted prompt string with sections
86
+
87
+ Example:
88
+ # Query
89
+
90
+ Find all documents Sarah authored
91
+
92
+ # Knowledge
93
+
94
+ Entity: sarah-chen
95
+ Type: person/employee
96
+
97
+ # Scratchpad
98
+
99
+ ```json
100
+ {"stage": "lookup"}
101
+ ```
102
+ """
103
+ parts = [f"# Query\n\n{self.query}"]
104
+
105
+ if self.knowledge:
106
+ parts.append(f"\n# Knowledge\n\n{self.knowledge}")
107
+
108
+ if self.scratchpad:
109
+ if isinstance(self.scratchpad, dict):
110
+ import json
111
+
112
+ scratchpad_str = json.dumps(self.scratchpad, indent=2)
113
+ parts.append(f"\n# Scratchpad\n\n```json\n{scratchpad_str}\n```")
114
+ else:
115
+ parts.append(f"\n# Scratchpad\n\n{self.scratchpad}")
116
+
117
+ return "\n".join(parts)
@@ -0,0 +1,89 @@
1
+ """
2
+ Helper functions for REM Query Agent.
3
+
4
+ This module provides convenience functions that load the REM Query Agent
5
+ from YAML schema and execute queries.
6
+ """
7
+
8
+ from typing import Any
9
+
10
+ from pydantic import BaseModel
11
+
12
+ from .context import AgentContext
13
+ from .providers.pydantic_ai import create_agent_from_schema_file
14
+
15
+
16
+ class REMQueryOutput(BaseModel):
17
+ """
18
+ REM Query Agent structured output.
19
+
20
+ Matches the schema defined in schemas/agents/core/rem-query-agent.yaml
21
+ """
22
+
23
+ query: str
24
+ confidence: float
25
+ reasoning: str = ""
26
+
27
+
28
+ async def ask_rem(
29
+ natural_query: str,
30
+ user_id: str = "system",
31
+ llm_model: str | None = None,
32
+ ) -> REMQueryOutput:
33
+ """
34
+ Convert natural language query to structured REM query.
35
+
36
+ Loads the REM Query Agent from YAML schema and executes the query.
37
+
38
+ Args:
39
+ natural_query: User's question in natural language
40
+ user_id: User ID for context (defaults to "system")
41
+ llm_model: Optional LLM model override
42
+
43
+ Returns:
44
+ REMQueryOutput with query, confidence, and reasoning
45
+
46
+ Example:
47
+ result = await ask_rem("Show me Sarah Chen")
48
+ # REMQueryOutput(
49
+ # query="LOOKUP sarah-chen",
50
+ # confidence=1.0,
51
+ # reasoning=""
52
+ # )
53
+ """
54
+ # Create context (only pass default_model if llm_model is provided)
55
+ context_kwargs = {"user_id": user_id}
56
+ if llm_model is not None:
57
+ context_kwargs["default_model"] = llm_model
58
+ context = AgentContext(**context_kwargs)
59
+
60
+ # Load agent from YAML schema
61
+ agent = await create_agent_from_schema_file(
62
+ schema_name_or_path="rem-query-agent",
63
+ context=context,
64
+ model_override=llm_model, # type: ignore[arg-type]
65
+ )
66
+
67
+ # Run query
68
+ result = await agent.run(natural_query)
69
+
70
+ # Handle different Pydantic AI versions
71
+ if hasattr(result, "data"):
72
+ output = result.data
73
+ elif hasattr(result, "output"):
74
+ output = result.output
75
+ else:
76
+ output = result
77
+
78
+ # Convert to REMQueryOutput if not already
79
+ if isinstance(output, dict):
80
+ return REMQueryOutput(**output)
81
+ elif isinstance(output, REMQueryOutput):
82
+ return output
83
+ else:
84
+ # Fallback: try to extract fields
85
+ return REMQueryOutput(
86
+ query=getattr(output, "query", str(output)),
87
+ confidence=getattr(output, "confidence", 0.5),
88
+ reasoning=getattr(output, "reasoning", ""),
89
+ )