remdb 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. rem/__init__.py +2 -0
  2. rem/agentic/README.md +650 -0
  3. rem/agentic/__init__.py +39 -0
  4. rem/agentic/agents/README.md +155 -0
  5. rem/agentic/agents/__init__.py +8 -0
  6. rem/agentic/context.py +148 -0
  7. rem/agentic/context_builder.py +329 -0
  8. rem/agentic/mcp/__init__.py +0 -0
  9. rem/agentic/mcp/tool_wrapper.py +107 -0
  10. rem/agentic/otel/__init__.py +5 -0
  11. rem/agentic/otel/setup.py +151 -0
  12. rem/agentic/providers/phoenix.py +674 -0
  13. rem/agentic/providers/pydantic_ai.py +572 -0
  14. rem/agentic/query.py +117 -0
  15. rem/agentic/query_helper.py +89 -0
  16. rem/agentic/schema.py +396 -0
  17. rem/agentic/serialization.py +245 -0
  18. rem/agentic/tools/__init__.py +5 -0
  19. rem/agentic/tools/rem_tools.py +231 -0
  20. rem/api/README.md +420 -0
  21. rem/api/main.py +324 -0
  22. rem/api/mcp_router/prompts.py +182 -0
  23. rem/api/mcp_router/resources.py +536 -0
  24. rem/api/mcp_router/server.py +213 -0
  25. rem/api/mcp_router/tools.py +584 -0
  26. rem/api/routers/auth.py +229 -0
  27. rem/api/routers/chat/__init__.py +5 -0
  28. rem/api/routers/chat/completions.py +281 -0
  29. rem/api/routers/chat/json_utils.py +76 -0
  30. rem/api/routers/chat/models.py +124 -0
  31. rem/api/routers/chat/streaming.py +185 -0
  32. rem/auth/README.md +258 -0
  33. rem/auth/__init__.py +26 -0
  34. rem/auth/middleware.py +100 -0
  35. rem/auth/providers/__init__.py +13 -0
  36. rem/auth/providers/base.py +376 -0
  37. rem/auth/providers/google.py +163 -0
  38. rem/auth/providers/microsoft.py +237 -0
  39. rem/cli/README.md +455 -0
  40. rem/cli/__init__.py +8 -0
  41. rem/cli/commands/README.md +126 -0
  42. rem/cli/commands/__init__.py +3 -0
  43. rem/cli/commands/ask.py +566 -0
  44. rem/cli/commands/configure.py +497 -0
  45. rem/cli/commands/db.py +493 -0
  46. rem/cli/commands/dreaming.py +324 -0
  47. rem/cli/commands/experiments.py +1302 -0
  48. rem/cli/commands/mcp.py +66 -0
  49. rem/cli/commands/process.py +245 -0
  50. rem/cli/commands/schema.py +183 -0
  51. rem/cli/commands/serve.py +106 -0
  52. rem/cli/dreaming.py +363 -0
  53. rem/cli/main.py +96 -0
  54. rem/config.py +237 -0
  55. rem/mcp_server.py +41 -0
  56. rem/models/core/__init__.py +49 -0
  57. rem/models/core/core_model.py +64 -0
  58. rem/models/core/engram.py +333 -0
  59. rem/models/core/experiment.py +628 -0
  60. rem/models/core/inline_edge.py +132 -0
  61. rem/models/core/rem_query.py +243 -0
  62. rem/models/entities/__init__.py +43 -0
  63. rem/models/entities/file.py +57 -0
  64. rem/models/entities/image_resource.py +88 -0
  65. rem/models/entities/message.py +35 -0
  66. rem/models/entities/moment.py +123 -0
  67. rem/models/entities/ontology.py +191 -0
  68. rem/models/entities/ontology_config.py +131 -0
  69. rem/models/entities/resource.py +95 -0
  70. rem/models/entities/schema.py +87 -0
  71. rem/models/entities/user.py +85 -0
  72. rem/py.typed +0 -0
  73. rem/schemas/README.md +507 -0
  74. rem/schemas/__init__.py +6 -0
  75. rem/schemas/agents/README.md +92 -0
  76. rem/schemas/agents/core/moment-builder.yaml +178 -0
  77. rem/schemas/agents/core/rem-query-agent.yaml +226 -0
  78. rem/schemas/agents/core/resource-affinity-assessor.yaml +99 -0
  79. rem/schemas/agents/core/simple-assistant.yaml +19 -0
  80. rem/schemas/agents/core/user-profile-builder.yaml +163 -0
  81. rem/schemas/agents/examples/contract-analyzer.yaml +317 -0
  82. rem/schemas/agents/examples/contract-extractor.yaml +134 -0
  83. rem/schemas/agents/examples/cv-parser.yaml +263 -0
  84. rem/schemas/agents/examples/hello-world.yaml +37 -0
  85. rem/schemas/agents/examples/query.yaml +54 -0
  86. rem/schemas/agents/examples/simple.yaml +21 -0
  87. rem/schemas/agents/examples/test.yaml +29 -0
  88. rem/schemas/agents/rem.yaml +128 -0
  89. rem/schemas/evaluators/hello-world/default.yaml +77 -0
  90. rem/schemas/evaluators/rem/faithfulness.yaml +219 -0
  91. rem/schemas/evaluators/rem/lookup-correctness.yaml +182 -0
  92. rem/schemas/evaluators/rem/retrieval-precision.yaml +199 -0
  93. rem/schemas/evaluators/rem/retrieval-recall.yaml +211 -0
  94. rem/schemas/evaluators/rem/search-correctness.yaml +192 -0
  95. rem/services/__init__.py +16 -0
  96. rem/services/audio/INTEGRATION.md +308 -0
  97. rem/services/audio/README.md +376 -0
  98. rem/services/audio/__init__.py +15 -0
  99. rem/services/audio/chunker.py +354 -0
  100. rem/services/audio/transcriber.py +259 -0
  101. rem/services/content/README.md +1269 -0
  102. rem/services/content/__init__.py +5 -0
  103. rem/services/content/providers.py +801 -0
  104. rem/services/content/service.py +676 -0
  105. rem/services/dreaming/README.md +230 -0
  106. rem/services/dreaming/__init__.py +53 -0
  107. rem/services/dreaming/affinity_service.py +336 -0
  108. rem/services/dreaming/moment_service.py +264 -0
  109. rem/services/dreaming/ontology_service.py +54 -0
  110. rem/services/dreaming/user_model_service.py +297 -0
  111. rem/services/dreaming/utils.py +39 -0
  112. rem/services/embeddings/__init__.py +11 -0
  113. rem/services/embeddings/api.py +120 -0
  114. rem/services/embeddings/worker.py +421 -0
  115. rem/services/fs/README.md +662 -0
  116. rem/services/fs/__init__.py +62 -0
  117. rem/services/fs/examples.py +206 -0
  118. rem/services/fs/examples_paths.py +204 -0
  119. rem/services/fs/git_provider.py +935 -0
  120. rem/services/fs/local_provider.py +760 -0
  121. rem/services/fs/parsing-hooks-examples.md +172 -0
  122. rem/services/fs/paths.py +276 -0
  123. rem/services/fs/provider.py +460 -0
  124. rem/services/fs/s3_provider.py +1042 -0
  125. rem/services/fs/service.py +186 -0
  126. rem/services/git/README.md +1075 -0
  127. rem/services/git/__init__.py +17 -0
  128. rem/services/git/service.py +469 -0
  129. rem/services/phoenix/EXPERIMENT_DESIGN.md +1146 -0
  130. rem/services/phoenix/README.md +453 -0
  131. rem/services/phoenix/__init__.py +46 -0
  132. rem/services/phoenix/client.py +686 -0
  133. rem/services/phoenix/config.py +88 -0
  134. rem/services/phoenix/prompt_labels.py +477 -0
  135. rem/services/postgres/README.md +575 -0
  136. rem/services/postgres/__init__.py +23 -0
  137. rem/services/postgres/migration_service.py +427 -0
  138. rem/services/postgres/pydantic_to_sqlalchemy.py +232 -0
  139. rem/services/postgres/register_type.py +352 -0
  140. rem/services/postgres/repository.py +337 -0
  141. rem/services/postgres/schema_generator.py +379 -0
  142. rem/services/postgres/service.py +802 -0
  143. rem/services/postgres/sql_builder.py +354 -0
  144. rem/services/rem/README.md +304 -0
  145. rem/services/rem/__init__.py +23 -0
  146. rem/services/rem/exceptions.py +71 -0
  147. rem/services/rem/executor.py +293 -0
  148. rem/services/rem/parser.py +145 -0
  149. rem/services/rem/queries.py +196 -0
  150. rem/services/rem/query.py +371 -0
  151. rem/services/rem/service.py +527 -0
  152. rem/services/session/README.md +374 -0
  153. rem/services/session/__init__.py +6 -0
  154. rem/services/session/compression.py +360 -0
  155. rem/services/session/reload.py +77 -0
  156. rem/settings.py +1235 -0
  157. rem/sql/002_install_models.sql +1068 -0
  158. rem/sql/background_indexes.sql +42 -0
  159. rem/sql/install_models.sql +1038 -0
  160. rem/sql/migrations/001_install.sql +503 -0
  161. rem/sql/migrations/002_install_models.sql +1202 -0
  162. rem/utils/AGENTIC_CHUNKING.md +597 -0
  163. rem/utils/README.md +583 -0
  164. rem/utils/__init__.py +43 -0
  165. rem/utils/agentic_chunking.py +622 -0
  166. rem/utils/batch_ops.py +343 -0
  167. rem/utils/chunking.py +108 -0
  168. rem/utils/clip_embeddings.py +276 -0
  169. rem/utils/dict_utils.py +98 -0
  170. rem/utils/embeddings.py +423 -0
  171. rem/utils/examples/embeddings_example.py +305 -0
  172. rem/utils/examples/sql_types_example.py +202 -0
  173. rem/utils/markdown.py +16 -0
  174. rem/utils/model_helpers.py +236 -0
  175. rem/utils/schema_loader.py +336 -0
  176. rem/utils/sql_types.py +348 -0
  177. rem/utils/user_id.py +81 -0
  178. rem/utils/vision.py +330 -0
  179. rem/workers/README.md +506 -0
  180. rem/workers/__init__.py +5 -0
  181. rem/workers/dreaming.py +502 -0
  182. rem/workers/engram_processor.py +312 -0
  183. rem/workers/sqs_file_processor.py +193 -0
  184. remdb-0.3.7.dist-info/METADATA +1473 -0
  185. remdb-0.3.7.dist-info/RECORD +187 -0
  186. remdb-0.3.7.dist-info/WHEEL +4 -0
  187. remdb-0.3.7.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,107 @@
1
+ """
2
+ MCP Tool Wrappers for Pydantic AI.
3
+
4
+ This module provides functions to convert MCP tool functions and resources
5
+ into a format compatible with the Pydantic AI library.
6
+ """
7
+
8
+ from typing import Any, Callable
9
+
10
+ from loguru import logger
11
+ from pydantic_ai.tools import Tool
12
+
13
+
14
+ def create_pydantic_tool(func: Callable[..., Any]) -> Tool:
15
+ """
16
+ Create a Pydantic AI Tool from a given function.
17
+
18
+ This uses the Tool constructor, which inspects the
19
+ function's signature and docstring to create the tool schema.
20
+
21
+ Args:
22
+ func: The function to wrap as a tool.
23
+
24
+ Returns:
25
+ A Pydantic AI Tool instance.
26
+ """
27
+ logger.debug(f"Creating Pydantic tool from function: {func.__name__}")
28
+ return Tool(func)
29
+
30
+
31
+ def create_mcp_tool_wrapper(tool_name: str, mcp_tool: Any, user_id: str | None = None) -> Tool:
32
+ """
33
+ Create a Pydantic AI Tool from a FastMCP FunctionTool.
34
+
35
+ FastMCP tools are FunctionTool objects that wrap the actual async function.
36
+ We pass the function directly to Pydantic AI's Tool class, which will
37
+ inspect its signature properly. User ID injection happens in the wrapper.
38
+
39
+ Args:
40
+ tool_name: Name of the MCP tool
41
+ mcp_tool: The FastMCP FunctionTool object
42
+ user_id: Optional user_id to inject into tool calls
43
+
44
+ Returns:
45
+ A Pydantic AI Tool instance
46
+ """
47
+ # Extract the actual function from FastMCP FunctionTool
48
+ tool_func = mcp_tool.fn
49
+
50
+ # Check if function accepts user_id parameter
51
+ import inspect
52
+ sig = inspect.signature(tool_func)
53
+ has_user_id = "user_id" in sig.parameters
54
+
55
+ # If we need to inject user_id, create a wrapper
56
+ # Otherwise, use the function directly for better signature preservation
57
+ if user_id and has_user_id:
58
+ async def wrapped_tool(**kwargs) -> Any:
59
+ """Wrapper that injects user_id."""
60
+ if "user_id" not in kwargs:
61
+ kwargs["user_id"] = user_id
62
+ logger.debug(f"Injecting user_id={user_id} into tool {tool_name}")
63
+
64
+ # Filter kwargs to only include parameters that the function accepts
65
+ valid_params = set(sig.parameters.keys())
66
+ filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
67
+
68
+ return await tool_func(**filtered_kwargs)
69
+
70
+ # Copy signature from original function for Pydantic AI inspection
71
+ wrapped_tool.__name__ = tool_name
72
+ wrapped_tool.__doc__ = tool_func.__doc__
73
+ wrapped_tool.__annotations__ = tool_func.__annotations__
74
+ wrapped_tool.__signature__ = sig # Important: preserve full signature
75
+
76
+ logger.debug(f"Creating MCP tool wrapper with user_id injection: {tool_name}")
77
+ return Tool(wrapped_tool)
78
+ else:
79
+ # No injection needed - use original function directly
80
+ logger.debug(f"Creating MCP tool wrapper (no injection): {tool_name}")
81
+ return Tool(tool_func)
82
+
83
+
84
+ def create_resource_tool(uri: str, usage: str) -> Tool:
85
+ """
86
+ Build a Tool instance from an MCP resource URI.
87
+
88
+ This is a placeholder for now. A real implementation would create a
89
+ tool that reads the content of the resource URI.
90
+
91
+ Args:
92
+ uri: The resource URI (e.g., "rem://resources/some-id").
93
+ usage: The description of how to use the tool.
94
+
95
+ Returns:
96
+ A Pydantic AI Tool instance.
97
+ """
98
+ # Placeholder function that would read the resource
99
+ def read_resource():
100
+ """Reads content from a resource URI."""
101
+ return f"Content of {uri}"
102
+
103
+ read_resource.__name__ = f"read_{uri.replace('://', '_').replace('/', '_')}"
104
+ read_resource.__doc__ = usage
105
+
106
+ logger.info(f"Built resource tool: {read_resource.__name__} (uri: {uri})")
107
+ return Tool(read_resource)
@@ -0,0 +1,5 @@
1
+ """OpenTelemetry instrumentation for REM agents."""
2
+
3
+ from .setup import setup_instrumentation, set_agent_resource_attributes
4
+
5
+ __all__ = ["setup_instrumentation", "set_agent_resource_attributes"]
@@ -0,0 +1,151 @@
1
+ """
2
+ OpenTelemetry instrumentation setup for REM agents.
3
+
4
+ Provides:
5
+ - OTLP exporter configuration
6
+ - Phoenix integration (OpenInference conventions)
7
+ - Resource attributes for agent metadata
8
+ - Idempotent setup (safe to call multiple times)
9
+ """
10
+
11
+ from typing import Any
12
+
13
+ from loguru import logger
14
+
15
+ from ...settings import settings
16
+
17
+ # Global flag to track if instrumentation is initialized
18
+ _instrumentation_initialized = False
19
+
20
+
21
+ def setup_instrumentation() -> None:
22
+ """
23
+ Initialize OpenTelemetry instrumentation for REM agents.
24
+
25
+ Idempotent - safe to call multiple times, only initializes once.
26
+
27
+ Configures:
28
+ - OTLP exporter (HTTP or gRPC)
29
+ - Phoenix integration if enabled
30
+ - Pydantic AI instrumentation (automatic via agent.instrument=True)
31
+ - Resource attributes (service name, environment, etc.)
32
+
33
+ Environment variables:
34
+ OTEL__ENABLED - Enable instrumentation (default: false)
35
+ OTEL__SERVICE_NAME - Service name (default: rem-api)
36
+ OTEL__COLLECTOR_ENDPOINT - OTLP endpoint (default: http://localhost:4318)
37
+ OTEL__PROTOCOL - Protocol (http or grpc, default: http)
38
+ PHOENIX__ENABLED - Enable Phoenix (default: false)
39
+ PHOENIX__COLLECTOR_ENDPOINT - Phoenix endpoint (default: http://localhost:6006/v1/traces)
40
+ """
41
+ global _instrumentation_initialized
42
+
43
+ if _instrumentation_initialized:
44
+ logger.debug("OTEL instrumentation already initialized, skipping")
45
+ return
46
+
47
+ if not settings.otel.enabled:
48
+ logger.debug("OTEL instrumentation disabled (OTEL__ENABLED=false)")
49
+ return
50
+
51
+ logger.info("Initializing OpenTelemetry instrumentation...")
52
+
53
+ try:
54
+ from opentelemetry import trace
55
+ from opentelemetry.sdk.trace import TracerProvider
56
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
57
+ from opentelemetry.sdk.resources import Resource, SERVICE_NAME, DEPLOYMENT_ENVIRONMENT
58
+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HTTPExporter
59
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as GRPCExporter
60
+
61
+ # Create resource with service metadata
62
+ resource = Resource(
63
+ attributes={
64
+ SERVICE_NAME: settings.otel.service_name,
65
+ DEPLOYMENT_ENVIRONMENT: settings.environment,
66
+ "service.team": settings.team,
67
+ }
68
+ )
69
+
70
+ # Create tracer provider
71
+ tracer_provider = TracerProvider(resource=resource)
72
+
73
+ # Configure OTLP exporter based on protocol
74
+ if settings.otel.protocol == "grpc":
75
+ exporter = GRPCExporter(
76
+ endpoint=settings.otel.collector_endpoint,
77
+ timeout=settings.otel.export_timeout,
78
+ )
79
+ else: # http
80
+ exporter = HTTPExporter(
81
+ endpoint=f"{settings.otel.collector_endpoint}/v1/traces",
82
+ timeout=settings.otel.export_timeout,
83
+ )
84
+
85
+ # Add span processor
86
+ tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
87
+
88
+ # Set as global tracer provider
89
+ trace.set_tracer_provider(tracer_provider)
90
+
91
+ logger.info(
92
+ f"OTLP exporter configured: {settings.otel.collector_endpoint} ({settings.otel.protocol})"
93
+ )
94
+
95
+ # Add OpenInference span processor for Pydantic AI
96
+ # This adds rich attributes (openinference.span.kind, input/output, etc.) to ALL traces
97
+ # Phoenix receives these traces via the OTLP collector - no separate "Phoenix integration" needed
98
+ try:
99
+ from openinference.instrumentation.pydantic_ai import OpenInferenceSpanProcessor as PydanticAISpanProcessor
100
+
101
+ tracer_provider.add_span_processor(PydanticAISpanProcessor())
102
+ logger.info("Added OpenInference span processor for Pydantic AI")
103
+
104
+ except ImportError:
105
+ logger.warning(
106
+ "openinference-instrumentation-pydantic-ai not installed - traces will lack OpenInference attributes. "
107
+ "Install with: pip install openinference-instrumentation-pydantic-ai"
108
+ )
109
+
110
+ _instrumentation_initialized = True
111
+ logger.info("OpenTelemetry instrumentation initialized successfully")
112
+
113
+ except Exception as e:
114
+ logger.error(f"Failed to initialize OTEL instrumentation: {e}")
115
+ # Don't raise - allow application to continue without tracing
116
+
117
+
118
+ def set_agent_resource_attributes(agent_schema: dict[str, Any] | None = None) -> None:
119
+ """
120
+ Set resource attributes for agent execution.
121
+
122
+ Called before creating agent to set span attributes with agent metadata.
123
+
124
+ Args:
125
+ agent_schema: Agent schema with metadata (kind, name, version, etc.)
126
+ """
127
+ if not settings.otel.enabled or not agent_schema:
128
+ return
129
+
130
+ try:
131
+ from opentelemetry import trace
132
+
133
+ # Get current span and set attributes
134
+ span = trace.get_current_span()
135
+ if span.is_recording():
136
+ json_extra = agent_schema.get("json_schema_extra", {})
137
+ kind = json_extra.get("kind")
138
+ name = json_extra.get("name")
139
+ version = json_extra.get("version", "unknown")
140
+
141
+ if kind:
142
+ span.set_attribute("agent.kind", kind)
143
+ if name:
144
+ span.set_attribute("agent.name", name)
145
+ if version:
146
+ span.set_attribute("agent.version", version)
147
+
148
+ logger.debug(f"Set agent resource attributes: kind={kind}, name={name}, version={version}")
149
+
150
+ except Exception as e:
151
+ logger.warning(f"Failed to set agent resource attributes: {e}")