MindsDB 25.9.2.0a1__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +39 -20
- mindsdb/api/a2a/agent.py +7 -9
- mindsdb/api/a2a/common/server/server.py +3 -3
- mindsdb/api/a2a/common/server/task_manager.py +4 -4
- mindsdb/api/a2a/task_manager.py +15 -17
- mindsdb/api/common/middleware.py +9 -11
- mindsdb/api/executor/command_executor.py +2 -4
- mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +100 -48
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
- mindsdb/api/executor/datahub/datanodes/system_tables.py +1 -1
- mindsdb/api/executor/exceptions.py +29 -10
- mindsdb/api/executor/planner/plan_join.py +17 -3
- mindsdb/api/executor/sql_query/sql_query.py +74 -74
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
- mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
- mindsdb/api/executor/utilities/functions.py +6 -6
- mindsdb/api/executor/utilities/sql.py +32 -16
- mindsdb/api/http/gui.py +5 -11
- mindsdb/api/http/initialize.py +8 -10
- mindsdb/api/http/namespaces/agents.py +10 -12
- mindsdb/api/http/namespaces/analysis.py +13 -20
- mindsdb/api/http/namespaces/auth.py +1 -1
- mindsdb/api/http/namespaces/config.py +15 -11
- mindsdb/api/http/namespaces/databases.py +140 -201
- mindsdb/api/http/namespaces/file.py +15 -4
- mindsdb/api/http/namespaces/handlers.py +7 -2
- mindsdb/api/http/namespaces/knowledge_bases.py +8 -7
- mindsdb/api/http/namespaces/models.py +94 -126
- mindsdb/api/http/namespaces/projects.py +13 -22
- mindsdb/api/http/namespaces/sql.py +33 -25
- mindsdb/api/http/namespaces/tab.py +27 -37
- mindsdb/api/http/namespaces/views.py +1 -1
- mindsdb/api/http/start.py +14 -8
- mindsdb/api/mcp/__init__.py +2 -1
- mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
- mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
- mindsdb/api/postgres/postgres_proxy/executor/executor.py +6 -13
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +40 -28
- mindsdb/integrations/handlers/byom_handler/byom_handler.py +168 -185
- mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
- mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +13 -1
- mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
- mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
- mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
- mindsdb/integrations/libs/api_handler.py +10 -10
- mindsdb/integrations/libs/base.py +4 -4
- mindsdb/integrations/libs/llm/utils.py +2 -2
- mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
- mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
- mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
- mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
- mindsdb/integrations/libs/process_cache.py +132 -140
- mindsdb/integrations/libs/response.py +18 -12
- mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
- mindsdb/integrations/utilities/files/file_reader.py +6 -7
- mindsdb/integrations/utilities/rag/config_loader.py +37 -26
- mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +59 -9
- mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
- mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
- mindsdb/integrations/utilities/rag/settings.py +58 -133
- mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
- mindsdb/interfaces/agents/agents_controller.py +2 -1
- mindsdb/interfaces/agents/constants.py +0 -2
- mindsdb/interfaces/agents/litellm_server.py +34 -58
- mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
- mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
- mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
- mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
- mindsdb/interfaces/chatbot/polling.py +30 -18
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +10 -10
- mindsdb/interfaces/database/integrations.py +19 -2
- mindsdb/interfaces/file/file_controller.py +6 -6
- mindsdb/interfaces/functions/controller.py +1 -1
- mindsdb/interfaces/functions/to_markdown.py +2 -2
- mindsdb/interfaces/jobs/jobs_controller.py +5 -5
- mindsdb/interfaces/jobs/scheduler.py +3 -8
- mindsdb/interfaces/knowledge_base/controller.py +50 -23
- mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
- mindsdb/interfaces/model/model_controller.py +170 -166
- mindsdb/interfaces/query_context/context_controller.py +14 -2
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +6 -4
- mindsdb/interfaces/skills/retrieval_tool.py +43 -50
- mindsdb/interfaces/skills/skill_tool.py +2 -2
- mindsdb/interfaces/skills/sql_agent.py +25 -19
- mindsdb/interfaces/storage/fs.py +114 -169
- mindsdb/interfaces/storage/json.py +19 -18
- mindsdb/interfaces/tabs/tabs_controller.py +49 -72
- mindsdb/interfaces/tasks/task_monitor.py +3 -9
- mindsdb/interfaces/tasks/task_thread.py +7 -9
- mindsdb/interfaces/triggers/trigger_task.py +7 -13
- mindsdb/interfaces/triggers/triggers_controller.py +47 -50
- mindsdb/migrations/migrate.py +16 -16
- mindsdb/utilities/api_status.py +58 -0
- mindsdb/utilities/config.py +49 -0
- mindsdb/utilities/exception.py +40 -1
- mindsdb/utilities/fs.py +0 -1
- mindsdb/utilities/hooks/profiling.py +17 -14
- mindsdb/utilities/langfuse.py +40 -45
- mindsdb/utilities/log.py +272 -0
- mindsdb/utilities/ml_task_queue/consumer.py +52 -58
- mindsdb/utilities/ml_task_queue/producer.py +26 -30
- mindsdb/utilities/render/sqlalchemy_render.py +7 -6
- mindsdb/utilities/utils.py +2 -2
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/METADATA +269 -264
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/RECORD +115 -115
- mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/WHEEL +0 -0
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.9.3rc1.dist-info}/top_level.txt +0 -0
|
@@ -54,8 +54,8 @@ class MCPQueryTool(BaseTool):
|
|
|
54
54
|
return f"Query executed successfully: {json.dumps(result.content)}"
|
|
55
55
|
|
|
56
56
|
except Exception as e:
|
|
57
|
-
logger.error(
|
|
58
|
-
return f"Error executing query: {
|
|
57
|
+
logger.error("Error executing MCP query:")
|
|
58
|
+
return f"Error executing query: {e}"
|
|
59
59
|
|
|
60
60
|
def _run(self, query: str) -> str:
|
|
61
61
|
"""Synchronous wrapper for async query function"""
|
|
@@ -112,8 +112,8 @@ class MCPLangchainAgent(LangchainAgent):
|
|
|
112
112
|
)
|
|
113
113
|
|
|
114
114
|
except Exception as e:
|
|
115
|
-
logger.
|
|
116
|
-
raise ConnectionError(f"Failed to connect to MCP server: {
|
|
115
|
+
logger.exception("Failed to connect to MCP server:")
|
|
116
|
+
raise ConnectionError(f"Failed to connect to MCP server: {e}") from e
|
|
117
117
|
|
|
118
118
|
def _langchain_tools_from_skills(self, llm):
|
|
119
119
|
"""Override to add MCP query tool along with other tools"""
|
|
@@ -131,8 +131,8 @@ class MCPLangchainAgent(LangchainAgent):
|
|
|
131
131
|
if self.session:
|
|
132
132
|
tools.append(MCPQueryTool(self.session))
|
|
133
133
|
logger.info("Added MCP query tool to agent tools")
|
|
134
|
-
except Exception
|
|
135
|
-
logger.
|
|
134
|
+
except Exception:
|
|
135
|
+
logger.exception("Failed to add MCP query tool:")
|
|
136
136
|
|
|
137
137
|
return tools
|
|
138
138
|
|
|
@@ -144,8 +144,8 @@ class MCPLangchainAgent(LangchainAgent):
|
|
|
144
144
|
# Using the event loop directly instead of asyncio.run()
|
|
145
145
|
loop = asyncio.get_event_loop()
|
|
146
146
|
loop.run_until_complete(self.connect_to_mcp())
|
|
147
|
-
except Exception
|
|
148
|
-
logger.
|
|
147
|
+
except Exception:
|
|
148
|
+
logger.exception("Failed to connect to MCP server:")
|
|
149
149
|
|
|
150
150
|
# Call parent implementation to get completion
|
|
151
151
|
response = super().get_completion(messages, stream)
|
|
@@ -224,8 +224,8 @@ class LiteLLMAgentWrapper:
|
|
|
224
224
|
}
|
|
225
225
|
# Allow async context switch
|
|
226
226
|
await asyncio.sleep(0)
|
|
227
|
-
except Exception
|
|
228
|
-
logger.
|
|
227
|
+
except Exception:
|
|
228
|
+
logger.exception("Streaming error:")
|
|
229
229
|
raise
|
|
230
230
|
|
|
231
231
|
async def cleanup(self):
|
|
@@ -3,11 +3,11 @@ Wrapper around MindsDB's executor and integration controller following the imple
|
|
|
3
3
|
langchain.sql_database.SQLDatabase class to partly replicate its behavior.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
-
import traceback
|
|
7
6
|
from typing import Any, Iterable, List, Optional
|
|
8
7
|
|
|
9
|
-
from mindsdb.utilities import log
|
|
10
8
|
from langchain_community.utilities import SQLDatabase
|
|
9
|
+
|
|
10
|
+
from mindsdb.utilities import log
|
|
11
11
|
from mindsdb.interfaces.skills.sql_agent import SQLAgent
|
|
12
12
|
|
|
13
13
|
logger = log.getLogger(__name__)
|
|
@@ -99,7 +99,7 @@ class MindsDBSQL(SQLDatabase):
|
|
|
99
99
|
return self._sql_agent.query(command)
|
|
100
100
|
|
|
101
101
|
except Exception as e:
|
|
102
|
-
logger.
|
|
102
|
+
logger.exception("Error executing SQL command:")
|
|
103
103
|
# If this is a knowledge base query, provide a more helpful error message
|
|
104
104
|
if "knowledge_base" in command.lower() or any(
|
|
105
105
|
kb in command for kb in self._sql_agent.get_usable_knowledge_base_names()
|
|
@@ -115,8 +115,8 @@ class MindsDBSQL(SQLDatabase):
|
|
|
115
115
|
"""
|
|
116
116
|
try:
|
|
117
117
|
return self._sql_agent.get_usable_knowledge_base_names()
|
|
118
|
-
except Exception
|
|
119
|
-
logger.
|
|
118
|
+
except Exception:
|
|
119
|
+
logger.exception("Error getting usable knowledge base names:")
|
|
120
120
|
return []
|
|
121
121
|
|
|
122
122
|
def check_knowledge_base_permission(self, name):
|
|
@@ -37,8 +37,8 @@ async def run_conversation(agent_wrapper, messages: List[Dict[str, str]], stream
|
|
|
37
37
|
# We still need to display the response to the user
|
|
38
38
|
sys.stdout.write(f"{content}\n")
|
|
39
39
|
sys.stdout.flush()
|
|
40
|
-
except Exception
|
|
41
|
-
logger.
|
|
40
|
+
except Exception:
|
|
41
|
+
logger.exception("Error during agent conversation:")
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
async def execute_direct_query(query):
|
|
@@ -48,11 +48,7 @@ async def execute_direct_query(query):
|
|
|
48
48
|
# Set up MCP client to connect to the running server
|
|
49
49
|
async with AsyncExitStack() as stack:
|
|
50
50
|
# Connect to MCP server
|
|
51
|
-
server_params = StdioServerParameters(
|
|
52
|
-
command="python",
|
|
53
|
-
args=["-m", "mindsdb", "--api=mcp"],
|
|
54
|
-
env=None
|
|
55
|
-
)
|
|
51
|
+
server_params = StdioServerParameters(command="python", args=["-m", "mindsdb", "--api=mcp"], env=None)
|
|
56
52
|
|
|
57
53
|
try:
|
|
58
54
|
stdio_transport = await stack.enter_async_context(stdio_client(server_params))
|
|
@@ -80,9 +76,9 @@ async def execute_direct_query(query):
|
|
|
80
76
|
# Execute query
|
|
81
77
|
result = await session.call_tool("query", {"query": query})
|
|
82
78
|
logger.info(f"Query result: {result.content}")
|
|
83
|
-
except Exception
|
|
84
|
-
logger.
|
|
85
|
-
logger.info("Make sure the MindsDB server is running with
|
|
79
|
+
except Exception:
|
|
80
|
+
logger.exception("Error executing query:")
|
|
81
|
+
logger.info("Make sure the MindsDB server is running with HTTP enabled: python -m mindsdb --api=http")
|
|
86
82
|
|
|
87
83
|
|
|
88
84
|
async def main():
|
|
@@ -100,6 +96,7 @@ async def main():
|
|
|
100
96
|
try:
|
|
101
97
|
# Initialize database connection
|
|
102
98
|
from mindsdb.interfaces.storage import db
|
|
99
|
+
|
|
103
100
|
db.init()
|
|
104
101
|
|
|
105
102
|
# Direct SQL execution mode (for testing MCP connection)
|
|
@@ -117,10 +114,7 @@ async def main():
|
|
|
117
114
|
logger.info("Make sure MindsDB server is running with MCP enabled: python -m mindsdb --api=mysql,mcp,http")
|
|
118
115
|
|
|
119
116
|
agent_wrapper = create_mcp_agent(
|
|
120
|
-
agent_name=args.agent,
|
|
121
|
-
project_name=args.project,
|
|
122
|
-
mcp_host=args.host,
|
|
123
|
-
mcp_port=args.port
|
|
117
|
+
agent_name=args.agent, project_name=args.project, mcp_host=args.host, mcp_port=args.port
|
|
124
118
|
)
|
|
125
119
|
|
|
126
120
|
# Run an example query if provided
|
|
@@ -173,7 +167,7 @@ async def main():
|
|
|
173
167
|
sys.stdout.write("Error: No active MCP session\n")
|
|
174
168
|
sys.stdout.flush()
|
|
175
169
|
except Exception as e:
|
|
176
|
-
logger.
|
|
170
|
+
logger.exception("SQL Error:")
|
|
177
171
|
sys.stdout.write(f"SQL Error: {str(e)}\n")
|
|
178
172
|
sys.stdout.flush()
|
|
179
173
|
continue
|
|
@@ -184,17 +178,14 @@ async def main():
|
|
|
184
178
|
# Add assistant's response to the conversation history
|
|
185
179
|
if not args.stream:
|
|
186
180
|
response = await agent_wrapper.acompletion(messages)
|
|
187
|
-
messages.append({
|
|
188
|
-
"role": "assistant",
|
|
189
|
-
"content": response["choices"][0]["message"]["content"]
|
|
190
|
-
})
|
|
181
|
+
messages.append({"role": "assistant", "content": response["choices"][0]["message"]["content"]})
|
|
191
182
|
|
|
192
183
|
# Clean up resources
|
|
193
184
|
logger.info("Cleaning up resources")
|
|
194
185
|
await agent_wrapper.cleanup()
|
|
195
186
|
|
|
196
|
-
except Exception
|
|
197
|
-
logger.
|
|
187
|
+
except Exception:
|
|
188
|
+
logger.exception("Error running MCP agent:")
|
|
198
189
|
logger.info("Make sure the MindsDB server is running with MCP enabled: python -m mindsdb --api=mysql,mcp,http")
|
|
199
190
|
return 1
|
|
200
191
|
|
|
@@ -21,7 +21,6 @@ HOLDING_MESSAGE = "Bot is typing..."
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class ChatBotTask(BaseTask):
|
|
24
|
-
|
|
25
24
|
def __init__(self, *args, **kwargs):
|
|
26
25
|
super().__init__(*args, **kwargs)
|
|
27
26
|
self.bot_id = self.object_id
|
|
@@ -40,7 +39,7 @@ class ChatBotTask(BaseTask):
|
|
|
40
39
|
self.agent_id = bot_record.agent_id
|
|
41
40
|
if self.agent_id is not None:
|
|
42
41
|
self.bot_executor_cls = AgentExecutor
|
|
43
|
-
elif self.bot_params.get(
|
|
42
|
+
elif self.bot_params.get("modes") is None:
|
|
44
43
|
self.bot_executor_cls = BotExecutor
|
|
45
44
|
else:
|
|
46
45
|
self.bot_executor_cls = MultiModeBotExecutor
|
|
@@ -52,36 +51,35 @@ class ChatBotTask(BaseTask):
|
|
|
52
51
|
raise Exception(f"Can't use chat database: {database_name}")
|
|
53
52
|
|
|
54
53
|
chat_params = self.chat_handler.get_chat_config()
|
|
55
|
-
polling = chat_params[
|
|
54
|
+
polling = chat_params["polling"]["type"]
|
|
56
55
|
|
|
57
|
-
memory = chat_params[
|
|
56
|
+
memory = chat_params["memory"]["type"] if "memory" in chat_params else None
|
|
58
57
|
memory_cls = None
|
|
59
58
|
if memory:
|
|
60
|
-
memory_cls = DBMemory if memory ==
|
|
59
|
+
memory_cls = DBMemory if memory == "db" else HandlerMemory
|
|
61
60
|
|
|
62
|
-
if polling ==
|
|
63
|
-
chat_params = chat_params[
|
|
61
|
+
if polling == "message_count":
|
|
62
|
+
chat_params = chat_params["tables"] if "tables" in chat_params else [chat_params]
|
|
64
63
|
self.chat_pooling = MessageCountPolling(self, chat_params)
|
|
65
64
|
# The default type for message count polling is HandlerMemory if not specified.
|
|
66
65
|
self.memory = HandlerMemory(self, chat_params) if memory_cls is None else memory_cls(self, chat_params)
|
|
67
66
|
|
|
68
|
-
elif polling ==
|
|
69
|
-
chat_params = chat_params[
|
|
67
|
+
elif polling == "realtime":
|
|
68
|
+
chat_params = chat_params["tables"] if "tables" in chat_params else [chat_params]
|
|
70
69
|
self.chat_pooling = RealtimePolling(self, chat_params)
|
|
71
70
|
# The default type for real-time polling is DBMemory if not specified.
|
|
72
71
|
self.memory = DBMemory(self, chat_params) if memory_cls is None else memory_cls(self, chat_params)
|
|
73
72
|
|
|
74
|
-
elif polling ==
|
|
73
|
+
elif polling == "webhook":
|
|
75
74
|
self.chat_pooling = WebhookPolling(self, chat_params)
|
|
76
75
|
self.memory = DBMemory(self, chat_params)
|
|
77
76
|
|
|
78
77
|
else:
|
|
79
78
|
raise Exception(f"Not supported polling: {polling}")
|
|
80
79
|
|
|
81
|
-
self.bot_params[
|
|
80
|
+
self.bot_params["bot_username"] = self.chat_handler.get_my_user_name()
|
|
82
81
|
|
|
83
82
|
def run(self, stop_event):
|
|
84
|
-
|
|
85
83
|
# TODO check deleted, raise errors
|
|
86
84
|
# TODO checks on delete predictor / project/ integration
|
|
87
85
|
|
|
@@ -89,7 +87,7 @@ class ChatBotTask(BaseTask):
|
|
|
89
87
|
|
|
90
88
|
def on_message(self, message: ChatBotMessage, chat_id=None, chat_memory=None, table_name=None):
|
|
91
89
|
if not chat_id and not chat_memory:
|
|
92
|
-
raise Exception(
|
|
90
|
+
raise Exception("chat_id or chat_memory should be provided")
|
|
93
91
|
|
|
94
92
|
try:
|
|
95
93
|
self._on_holding_message(chat_id, chat_memory, table_name)
|
|
@@ -97,9 +95,8 @@ class ChatBotTask(BaseTask):
|
|
|
97
95
|
except (SystemExit, KeyboardInterrupt):
|
|
98
96
|
raise
|
|
99
97
|
except Exception:
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
self.set_error(str(error))
|
|
98
|
+
logger.exception("Error during chatbot message processing:")
|
|
99
|
+
self.set_error(traceback.format_exc())
|
|
103
100
|
|
|
104
101
|
def _on_holding_message(self, chat_id: str = None, chat_memory: BaseMemory = None, table_name: str = None):
|
|
105
102
|
"""
|
|
@@ -117,14 +114,14 @@ class ChatBotTask(BaseTask):
|
|
|
117
114
|
ChatBotMessage.Type.DIRECT,
|
|
118
115
|
HOLDING_MESSAGE,
|
|
119
116
|
# In Slack direct messages are treated as channels themselves.
|
|
120
|
-
user=self.bot_params[
|
|
117
|
+
user=self.bot_params["bot_username"],
|
|
121
118
|
destination=chat_id,
|
|
122
|
-
sent_at=dt.datetime.now()
|
|
119
|
+
sent_at=dt.datetime.now(),
|
|
123
120
|
)
|
|
124
121
|
|
|
125
122
|
# send to chat adapter
|
|
126
123
|
self.chat_pooling.send_message(response_message, table_name=table_name)
|
|
127
|
-
logger.debug(f
|
|
124
|
+
logger.debug(f">>chatbot {chat_id} out (holding message): {response_message.text}")
|
|
128
125
|
|
|
129
126
|
def _on_message(self, message: ChatBotMessage, chat_id, chat_memory, table_name=None):
|
|
130
127
|
# add question to history
|
|
@@ -132,26 +129,26 @@ class ChatBotTask(BaseTask):
|
|
|
132
129
|
chat_memory = chat_memory if chat_memory else self.memory.get_chat(chat_id, table_name=table_name)
|
|
133
130
|
chat_memory.add_to_history(message)
|
|
134
131
|
|
|
135
|
-
logger.debug(f
|
|
132
|
+
logger.debug(f">>chatbot {chat_memory.chat_id} in: {message.text}")
|
|
136
133
|
|
|
137
134
|
# process
|
|
138
135
|
bot_executor = self.bot_executor_cls(self, chat_memory)
|
|
139
136
|
response_text = bot_executor.process()
|
|
140
137
|
|
|
141
138
|
chat_id = chat_memory.chat_id
|
|
142
|
-
bot_username = self.bot_params[
|
|
139
|
+
bot_username = self.bot_params["bot_username"]
|
|
143
140
|
response_message = ChatBotMessage(
|
|
144
141
|
ChatBotMessage.Type.DIRECT,
|
|
145
142
|
response_text,
|
|
146
143
|
# In Slack direct messages are treated as channels themselves.
|
|
147
144
|
user=bot_username,
|
|
148
145
|
destination=chat_id,
|
|
149
|
-
sent_at=dt.datetime.now()
|
|
146
|
+
sent_at=dt.datetime.now(),
|
|
150
147
|
)
|
|
151
148
|
|
|
152
149
|
# send to chat adapter
|
|
153
150
|
self.chat_pooling.send_message(response_message, table_name=table_name)
|
|
154
|
-
logger.debug(f
|
|
151
|
+
logger.debug(f">>chatbot {chat_id} out: {response_message.text}")
|
|
155
152
|
|
|
156
153
|
# send to history
|
|
157
154
|
chat_memory.add_to_history(response_message)
|
|
@@ -24,10 +24,14 @@ class BasePolling:
|
|
|
24
24
|
chat_id = message.destination if isinstance(message.destination, tuple) else (message.destination,)
|
|
25
25
|
text = message.text
|
|
26
26
|
|
|
27
|
-
t_params =
|
|
28
|
-
|
|
27
|
+
t_params = (
|
|
28
|
+
self.params["chat_table"]
|
|
29
|
+
if table_name is None
|
|
30
|
+
else next((t["chat_table"] for t in self.params if t["chat_table"]["name"] == table_name))
|
|
31
|
+
)
|
|
32
|
+
chat_id_cols = (
|
|
33
|
+
t_params["chat_id_col"] if isinstance(t_params["chat_id_col"], list) else [t_params["chat_id_col"]]
|
|
29
34
|
)
|
|
30
|
-
chat_id_cols = t_params["chat_id_col"] if isinstance(t_params["chat_id_col"], list) else [t_params["chat_id_col"]]
|
|
31
35
|
|
|
32
36
|
ast_query = Insert(
|
|
33
37
|
table=Identifier(t_params["name"]),
|
|
@@ -60,20 +64,22 @@ class MessageCountPolling(BasePolling):
|
|
|
60
64
|
chat_id,
|
|
61
65
|
table_name=chat_params["chat_table"]["name"],
|
|
62
66
|
)
|
|
63
|
-
except Exception
|
|
64
|
-
logger.
|
|
67
|
+
except Exception:
|
|
68
|
+
logger.exception("Problem retrieving chat memory:")
|
|
65
69
|
|
|
66
70
|
try:
|
|
67
71
|
message = self.get_last_message(chat_memory)
|
|
68
|
-
except Exception
|
|
69
|
-
logger.
|
|
72
|
+
except Exception:
|
|
73
|
+
logger.exception("Problem getting last message:")
|
|
70
74
|
message = None
|
|
71
75
|
|
|
72
76
|
if message:
|
|
73
|
-
self.chat_task.on_message(
|
|
77
|
+
self.chat_task.on_message(
|
|
78
|
+
message, chat_memory=chat_memory, table_name=chat_params["chat_table"]["name"]
|
|
79
|
+
)
|
|
74
80
|
|
|
75
|
-
except Exception
|
|
76
|
-
logger.error
|
|
81
|
+
except Exception:
|
|
82
|
+
logger.exception("Unexpected error")
|
|
77
83
|
|
|
78
84
|
if stop_event.is_set():
|
|
79
85
|
return
|
|
@@ -84,8 +90,8 @@ class MessageCountPolling(BasePolling):
|
|
|
84
90
|
# retrive from history
|
|
85
91
|
try:
|
|
86
92
|
history = chat_memory.get_history()
|
|
87
|
-
except Exception
|
|
88
|
-
logger.
|
|
93
|
+
except Exception:
|
|
94
|
+
logger.exception("Problem retrieving history:")
|
|
89
95
|
history = []
|
|
90
96
|
last_message = history[-1]
|
|
91
97
|
if last_message.user == self.chat_task.bot_params["bot_username"]:
|
|
@@ -151,7 +157,11 @@ class RealtimePolling(BasePolling):
|
|
|
151
157
|
# Identify the table relevant to this event based on the key.
|
|
152
158
|
event_keys = list(key.keys())
|
|
153
159
|
for param in self.params:
|
|
154
|
-
table_keys =
|
|
160
|
+
table_keys = (
|
|
161
|
+
[param["chat_table"]["chat_id_col"]]
|
|
162
|
+
if isinstance(param["chat_table"]["chat_id_col"], str)
|
|
163
|
+
else param["chat_table"]["chat_id_col"]
|
|
164
|
+
)
|
|
155
165
|
|
|
156
166
|
if sorted(event_keys) == sorted(table_keys):
|
|
157
167
|
t_params = param["chat_table"]
|
|
@@ -162,7 +172,11 @@ class RealtimePolling(BasePolling):
|
|
|
162
172
|
t_params = self.params[0]
|
|
163
173
|
|
|
164
174
|
# Get the chat ID from the row based on the chat ID column(s).
|
|
165
|
-
chat_id =
|
|
175
|
+
chat_id = (
|
|
176
|
+
tuple(row[key] for key in t_params["chat_id_col"])
|
|
177
|
+
if isinstance(t_params["chat_id_col"], list)
|
|
178
|
+
else row[t_params["chat_id_col"]]
|
|
179
|
+
)
|
|
166
180
|
|
|
167
181
|
message = ChatBotMessage(
|
|
168
182
|
ChatBotMessage.Type.DIRECT,
|
|
@@ -179,10 +193,7 @@ class RealtimePolling(BasePolling):
|
|
|
179
193
|
)
|
|
180
194
|
|
|
181
195
|
def run(self, stop_event):
|
|
182
|
-
self.chat_task.chat_handler.subscribe(
|
|
183
|
-
stop_event,
|
|
184
|
-
self._callback
|
|
185
|
-
)
|
|
196
|
+
self.chat_task.chat_handler.subscribe(stop_event, self._callback)
|
|
186
197
|
|
|
187
198
|
# def send_message(self, message: ChatBotMessage):
|
|
188
199
|
#
|
|
@@ -193,6 +204,7 @@ class WebhookPolling(BasePolling):
|
|
|
193
204
|
"""
|
|
194
205
|
Polling class for handling webhooks.
|
|
195
206
|
"""
|
|
207
|
+
|
|
196
208
|
def __init__(self, *args, **kwargs):
|
|
197
209
|
super().__init__(*args, **kwargs)
|
|
198
210
|
|
|
@@ -109,8 +109,8 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
109
109
|
|
|
110
110
|
db.session.add_all(tables)
|
|
111
111
|
db.session.commit()
|
|
112
|
-
except Exception
|
|
113
|
-
self.logger.
|
|
112
|
+
except Exception:
|
|
113
|
+
self.logger.exception("Failed to add tables:")
|
|
114
114
|
db.session.rollback()
|
|
115
115
|
raise
|
|
116
116
|
return tables
|
|
@@ -157,8 +157,8 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
157
157
|
|
|
158
158
|
db.session.add_all(columns)
|
|
159
159
|
db.session.commit()
|
|
160
|
-
except Exception
|
|
161
|
-
self.logger.
|
|
160
|
+
except Exception:
|
|
161
|
+
self.logger.exception("Failed to add columns:")
|
|
162
162
|
db.session.rollback()
|
|
163
163
|
raise
|
|
164
164
|
return columns
|
|
@@ -223,8 +223,8 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
223
223
|
|
|
224
224
|
db.session.add_all(column_statistics)
|
|
225
225
|
db.session.commit()
|
|
226
|
-
except Exception
|
|
227
|
-
self.logger.
|
|
226
|
+
except Exception:
|
|
227
|
+
self.logger.exception("Failed to add column statistics:")
|
|
228
228
|
db.session.rollback()
|
|
229
229
|
raise
|
|
230
230
|
|
|
@@ -275,8 +275,8 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
275
275
|
|
|
276
276
|
db.session.add_all(primary_keys)
|
|
277
277
|
db.session.commit()
|
|
278
|
-
except Exception
|
|
279
|
-
self.logger.
|
|
278
|
+
except Exception:
|
|
279
|
+
self.logger.exception("Failed to add primary keys:")
|
|
280
280
|
db.session.rollback()
|
|
281
281
|
raise
|
|
282
282
|
|
|
@@ -344,8 +344,8 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
344
344
|
|
|
345
345
|
db.session.add_all(foreign_keys)
|
|
346
346
|
db.session.commit()
|
|
347
|
-
except Exception
|
|
348
|
-
self.logger.
|
|
347
|
+
except Exception:
|
|
348
|
+
self.logger.exception("Failed to add foreign keys:")
|
|
349
349
|
db.session.rollback()
|
|
350
350
|
raise
|
|
351
351
|
|
|
@@ -71,22 +71,33 @@ class HandlersCache:
|
|
|
71
71
|
def set(self, handler: DatabaseHandler):
|
|
72
72
|
"""add (or replace) handler in cache
|
|
73
73
|
|
|
74
|
+
NOTE: If the handler is not thread-safe, then use a lock when making connection. Otherwise, make connection in
|
|
75
|
+
the same thread without using a lock to speed up parallel queries. (They don't need to wait for a connection in
|
|
76
|
+
another thread.)
|
|
77
|
+
|
|
74
78
|
Args:
|
|
75
79
|
handler (DatabaseHandler)
|
|
76
80
|
"""
|
|
81
|
+
thread_safe = getattr(handler, "thread_safe", False)
|
|
77
82
|
with self._lock:
|
|
78
83
|
try:
|
|
79
84
|
# If the handler is defined to be thread safe, set 0 as the last element of the key, otherwise set the thrad ID.
|
|
80
85
|
key = (
|
|
81
86
|
handler.name,
|
|
82
87
|
ctx.company_id,
|
|
83
|
-
0 if
|
|
88
|
+
0 if thread_safe else threading.get_native_id(),
|
|
84
89
|
)
|
|
85
|
-
handler.connect()
|
|
86
90
|
self.handlers[key] = {"handler": handler, "expired_at": time.time() + self.ttl}
|
|
91
|
+
if thread_safe:
|
|
92
|
+
handler.connect()
|
|
87
93
|
except Exception:
|
|
88
94
|
pass
|
|
89
95
|
self._start_clean()
|
|
96
|
+
try:
|
|
97
|
+
if not thread_safe:
|
|
98
|
+
handler.connect()
|
|
99
|
+
except Exception:
|
|
100
|
+
pass
|
|
90
101
|
|
|
91
102
|
def get(self, name: str) -> Optional[DatabaseHandler]:
|
|
92
103
|
"""get handler from cache by name
|
|
@@ -866,6 +877,8 @@ class IntegrationController:
|
|
|
866
877
|
|
|
867
878
|
def import_handler(self, handler_name: str, base_import: str = None):
|
|
868
879
|
with self._import_lock:
|
|
880
|
+
time_before_import = time.perf_counter()
|
|
881
|
+
logger.debug(f"Importing handler '{handler_name}'")
|
|
869
882
|
handler_meta = self.handlers_import_status[handler_name]
|
|
870
883
|
handler_dir = handler_meta["path"]
|
|
871
884
|
|
|
@@ -877,9 +890,13 @@ class IntegrationController:
|
|
|
877
890
|
handler_module = importlib.import_module(f"{base_import}{handler_folder_name}")
|
|
878
891
|
self.handler_modules[handler_name] = handler_module
|
|
879
892
|
handler_meta = self._get_handler_meta(handler_name)
|
|
893
|
+
logger.debug(
|
|
894
|
+
f"Handler '{handler_name}' imported successfully in {(time.perf_counter() - time_before_import):.3f} seconds"
|
|
895
|
+
)
|
|
880
896
|
except Exception as e:
|
|
881
897
|
handler_meta["import"]["success"] = False
|
|
882
898
|
handler_meta["import"]["error_message"] = str(e)
|
|
899
|
+
logger.debug(f"Failed to import handler '{handler_name}': {e}")
|
|
883
900
|
|
|
884
901
|
self.handlers_import_status[handler_meta["name"]] = handler_meta
|
|
885
902
|
return handler_meta
|
|
@@ -107,8 +107,8 @@ class FileController:
|
|
|
107
107
|
self.fs_store.put(store_file_path, base_dir=self.dir)
|
|
108
108
|
db.session.commit()
|
|
109
109
|
|
|
110
|
-
except Exception
|
|
111
|
-
logger.error
|
|
110
|
+
except Exception:
|
|
111
|
+
logger.exception("An error occurred while saving the file:")
|
|
112
112
|
if file_dir is not None:
|
|
113
113
|
shutil.rmtree(file_dir)
|
|
114
114
|
raise
|
|
@@ -151,7 +151,7 @@ class FileController:
|
|
|
151
151
|
def delete_file(self, name):
|
|
152
152
|
file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
|
|
153
153
|
if file_record is None:
|
|
154
|
-
|
|
154
|
+
raise FileNotFoundError(f"File '{name}' does not exists")
|
|
155
155
|
file_id = file_record.id
|
|
156
156
|
db.session.delete(file_record)
|
|
157
157
|
db.session.commit()
|
|
@@ -161,7 +161,7 @@ class FileController:
|
|
|
161
161
|
def get_file_path(self, name):
|
|
162
162
|
file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
|
|
163
163
|
if file_record is None:
|
|
164
|
-
raise
|
|
164
|
+
raise FileNotFoundError(f"File '{name}' does not exists")
|
|
165
165
|
file_dir = f"file_{ctx.company_id}_{file_record.id}"
|
|
166
166
|
self.fs_store.get(file_dir, base_dir=self.dir)
|
|
167
167
|
return str(Path(self.dir).joinpath(file_dir).joinpath(Path(file_record.source_file_path).name))
|
|
@@ -176,7 +176,7 @@ class FileController:
|
|
|
176
176
|
"""
|
|
177
177
|
file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
|
|
178
178
|
if file_record is None:
|
|
179
|
-
raise
|
|
179
|
+
raise FileNotFoundError(f"File '{name}' does not exists")
|
|
180
180
|
|
|
181
181
|
file_dir = f"file_{ctx.company_id}_{file_record.id}"
|
|
182
182
|
self.fs_store.get(file_dir, base_dir=self.dir)
|
|
@@ -217,7 +217,7 @@ class FileController:
|
|
|
217
217
|
|
|
218
218
|
file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
|
|
219
219
|
if file_record is None:
|
|
220
|
-
raise
|
|
220
|
+
raise FileNotFoundError(f"File '{name}' does not exists")
|
|
221
221
|
|
|
222
222
|
file_dir = f"file_{ctx.company_id}_{file_record.id}"
|
|
223
223
|
self.fs_store.get(file_dir, base_dir=self.dir)
|
|
@@ -140,7 +140,7 @@ class FunctionController(BYOMFunctionsController):
|
|
|
140
140
|
|
|
141
141
|
llm = create_chat_model(chat_model_params)
|
|
142
142
|
except Exception as e:
|
|
143
|
-
raise RuntimeError(f"Unable to use LLM function, check ENV variables: {e}")
|
|
143
|
+
raise RuntimeError(f"Unable to use LLM function, check ENV variables: {e}") from e
|
|
144
144
|
|
|
145
145
|
def callback(question):
|
|
146
146
|
resp = llm([HumanMessage(question)])
|
|
@@ -69,8 +69,8 @@ class ToMarkdown:
|
|
|
69
69
|
ext = os.path.splitext(parsed_url.path)[1]
|
|
70
70
|
if ext:
|
|
71
71
|
return ext
|
|
72
|
-
except requests.RequestException:
|
|
73
|
-
raise RuntimeError(f"Unable to retrieve file extension from URL: {file_path_or_url}")
|
|
72
|
+
except requests.RequestException as e:
|
|
73
|
+
raise RuntimeError(f"Unable to retrieve file extension from URL: {file_path_or_url}") from e
|
|
74
74
|
else:
|
|
75
75
|
return os.path.splitext(file_path_or_url)[1]
|
|
76
76
|
|
|
@@ -152,7 +152,7 @@ class JobsController:
|
|
|
152
152
|
|
|
153
153
|
parse_sql(sql)
|
|
154
154
|
except ParsingException as e:
|
|
155
|
-
raise ParsingException(f"Unable to parse: {sql}: {e}")
|
|
155
|
+
raise ParsingException(f"Unable to parse: {sql}: {e}") from e
|
|
156
156
|
|
|
157
157
|
if if_query is not None:
|
|
158
158
|
for sql in split_sql(if_query):
|
|
@@ -162,7 +162,7 @@ class JobsController:
|
|
|
162
162
|
|
|
163
163
|
parse_sql(sql)
|
|
164
164
|
except ParsingException as e:
|
|
165
|
-
raise ParsingException(f"Unable to parse: {sql}: {e}")
|
|
165
|
+
raise ParsingException(f"Unable to parse: {sql}: {e}") from e
|
|
166
166
|
|
|
167
167
|
# plan next run
|
|
168
168
|
next_run_at = start_at
|
|
@@ -494,7 +494,7 @@ class JobsExecutor:
|
|
|
494
494
|
|
|
495
495
|
data = ret.data
|
|
496
496
|
except Exception as e:
|
|
497
|
-
logger.
|
|
497
|
+
logger.exception("Error to execute job`s condition query")
|
|
498
498
|
error = str(e)
|
|
499
499
|
break
|
|
500
500
|
|
|
@@ -518,7 +518,7 @@ class JobsExecutor:
|
|
|
518
518
|
error = ret.error_message
|
|
519
519
|
break
|
|
520
520
|
except Exception as e:
|
|
521
|
-
logger.
|
|
521
|
+
logger.exception("Error to execute job`s query")
|
|
522
522
|
error = str(e)
|
|
523
523
|
break
|
|
524
524
|
|
|
@@ -526,7 +526,7 @@ class JobsExecutor:
|
|
|
526
526
|
self.update_task_schedule(record)
|
|
527
527
|
except Exception as e:
|
|
528
528
|
db.session.rollback()
|
|
529
|
-
logger.
|
|
529
|
+
logger.exception("Error to update schedule:")
|
|
530
530
|
error += f"Error to update schedule: {e}"
|
|
531
531
|
|
|
532
532
|
# stop scheduling
|