MindsDB 25.9.1.2__py3-none-any.whl → 25.9.3rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of MindsDB might be problematic. Click here for more details.

Files changed (120) hide show
  1. mindsdb/__about__.py +1 -1
  2. mindsdb/__main__.py +39 -20
  3. mindsdb/api/a2a/agent.py +7 -9
  4. mindsdb/api/a2a/common/server/server.py +3 -3
  5. mindsdb/api/a2a/common/server/task_manager.py +4 -4
  6. mindsdb/api/a2a/task_manager.py +15 -17
  7. mindsdb/api/common/middleware.py +9 -11
  8. mindsdb/api/executor/command_executor.py +2 -4
  9. mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
  10. mindsdb/api/executor/datahub/datanodes/integration_datanode.py +100 -48
  11. mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
  12. mindsdb/api/executor/datahub/datanodes/system_tables.py +1 -1
  13. mindsdb/api/executor/exceptions.py +29 -10
  14. mindsdb/api/executor/planner/plan_join.py +17 -3
  15. mindsdb/api/executor/sql_query/sql_query.py +74 -74
  16. mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
  17. mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
  18. mindsdb/api/executor/utilities/functions.py +6 -6
  19. mindsdb/api/executor/utilities/sql.py +32 -16
  20. mindsdb/api/http/gui.py +5 -11
  21. mindsdb/api/http/initialize.py +8 -10
  22. mindsdb/api/http/namespaces/agents.py +10 -12
  23. mindsdb/api/http/namespaces/analysis.py +13 -20
  24. mindsdb/api/http/namespaces/auth.py +1 -1
  25. mindsdb/api/http/namespaces/config.py +15 -11
  26. mindsdb/api/http/namespaces/databases.py +140 -201
  27. mindsdb/api/http/namespaces/file.py +15 -4
  28. mindsdb/api/http/namespaces/handlers.py +7 -2
  29. mindsdb/api/http/namespaces/knowledge_bases.py +8 -7
  30. mindsdb/api/http/namespaces/models.py +94 -126
  31. mindsdb/api/http/namespaces/projects.py +13 -22
  32. mindsdb/api/http/namespaces/sql.py +33 -25
  33. mindsdb/api/http/namespaces/tab.py +27 -37
  34. mindsdb/api/http/namespaces/views.py +1 -1
  35. mindsdb/api/http/start.py +14 -8
  36. mindsdb/api/mcp/__init__.py +2 -1
  37. mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
  38. mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
  39. mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
  40. mindsdb/api/postgres/postgres_proxy/executor/executor.py +6 -13
  41. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +40 -28
  42. mindsdb/integrations/handlers/byom_handler/byom_handler.py +168 -185
  43. mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py +11 -5
  44. mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
  45. mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
  46. mindsdb/integrations/handlers/openai_handler/openai_handler.py +1 -1
  47. mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +20 -2
  48. mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +18 -3
  49. mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
  50. mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
  51. mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
  52. mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
  53. mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
  54. mindsdb/integrations/libs/api_handler.py +10 -10
  55. mindsdb/integrations/libs/base.py +4 -4
  56. mindsdb/integrations/libs/llm/utils.py +2 -2
  57. mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
  58. mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
  59. mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
  60. mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
  61. mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
  62. mindsdb/integrations/libs/process_cache.py +132 -140
  63. mindsdb/integrations/libs/response.py +18 -12
  64. mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
  65. mindsdb/integrations/utilities/files/file_reader.py +6 -7
  66. mindsdb/integrations/utilities/rag/config_loader.py +37 -26
  67. mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +59 -9
  68. mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
  69. mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
  70. mindsdb/integrations/utilities/rag/settings.py +58 -133
  71. mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
  72. mindsdb/interfaces/agents/agents_controller.py +2 -1
  73. mindsdb/interfaces/agents/constants.py +0 -2
  74. mindsdb/interfaces/agents/litellm_server.py +34 -58
  75. mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
  76. mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
  77. mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
  78. mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
  79. mindsdb/interfaces/chatbot/polling.py +30 -18
  80. mindsdb/interfaces/data_catalog/data_catalog_loader.py +10 -10
  81. mindsdb/interfaces/database/integrations.py +19 -2
  82. mindsdb/interfaces/file/file_controller.py +6 -6
  83. mindsdb/interfaces/functions/controller.py +1 -1
  84. mindsdb/interfaces/functions/to_markdown.py +2 -2
  85. mindsdb/interfaces/jobs/jobs_controller.py +5 -5
  86. mindsdb/interfaces/jobs/scheduler.py +3 -8
  87. mindsdb/interfaces/knowledge_base/controller.py +54 -25
  88. mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
  89. mindsdb/interfaces/model/model_controller.py +170 -166
  90. mindsdb/interfaces/query_context/context_controller.py +14 -2
  91. mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +6 -4
  92. mindsdb/interfaces/skills/retrieval_tool.py +43 -50
  93. mindsdb/interfaces/skills/skill_tool.py +2 -2
  94. mindsdb/interfaces/skills/sql_agent.py +25 -19
  95. mindsdb/interfaces/storage/fs.py +114 -169
  96. mindsdb/interfaces/storage/json.py +19 -18
  97. mindsdb/interfaces/storage/model_fs.py +54 -92
  98. mindsdb/interfaces/tabs/tabs_controller.py +49 -72
  99. mindsdb/interfaces/tasks/task_monitor.py +3 -9
  100. mindsdb/interfaces/tasks/task_thread.py +7 -9
  101. mindsdb/interfaces/triggers/trigger_task.py +7 -13
  102. mindsdb/interfaces/triggers/triggers_controller.py +47 -50
  103. mindsdb/migrations/migrate.py +16 -16
  104. mindsdb/utilities/api_status.py +58 -0
  105. mindsdb/utilities/config.py +49 -0
  106. mindsdb/utilities/exception.py +40 -1
  107. mindsdb/utilities/fs.py +0 -1
  108. mindsdb/utilities/hooks/profiling.py +17 -14
  109. mindsdb/utilities/langfuse.py +40 -45
  110. mindsdb/utilities/log.py +272 -0
  111. mindsdb/utilities/ml_task_queue/consumer.py +52 -58
  112. mindsdb/utilities/ml_task_queue/producer.py +26 -30
  113. mindsdb/utilities/render/sqlalchemy_render.py +8 -7
  114. mindsdb/utilities/utils.py +2 -2
  115. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/METADATA +266 -261
  116. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/RECORD +119 -119
  117. mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
  118. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/WHEEL +0 -0
  119. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/licenses/LICENSE +0 -0
  120. {mindsdb-25.9.1.2.dist-info → mindsdb-25.9.3rc1.dist-info}/top_level.txt +0 -0
@@ -54,8 +54,8 @@ class MCPQueryTool(BaseTool):
54
54
  return f"Query executed successfully: {json.dumps(result.content)}"
55
55
 
56
56
  except Exception as e:
57
- logger.error(f"Error executing MCP query: {str(e)}")
58
- return f"Error executing query: {str(e)}"
57
+ logger.error("Error executing MCP query:")
58
+ return f"Error executing query: {e}"
59
59
 
60
60
  def _run(self, query: str) -> str:
61
61
  """Synchronous wrapper for async query function"""
@@ -112,8 +112,8 @@ class MCPLangchainAgent(LangchainAgent):
112
112
  )
113
113
 
114
114
  except Exception as e:
115
- logger.error(f"Failed to connect to MCP server: {str(e)}")
116
- raise ConnectionError(f"Failed to connect to MCP server: {str(e)}")
115
+ logger.exception("Failed to connect to MCP server:")
116
+ raise ConnectionError(f"Failed to connect to MCP server: {e}") from e
117
117
 
118
118
  def _langchain_tools_from_skills(self, llm):
119
119
  """Override to add MCP query tool along with other tools"""
@@ -131,8 +131,8 @@ class MCPLangchainAgent(LangchainAgent):
131
131
  if self.session:
132
132
  tools.append(MCPQueryTool(self.session))
133
133
  logger.info("Added MCP query tool to agent tools")
134
- except Exception as e:
135
- logger.error(f"Failed to add MCP query tool: {str(e)}")
134
+ except Exception:
135
+ logger.exception("Failed to add MCP query tool:")
136
136
 
137
137
  return tools
138
138
 
@@ -144,8 +144,8 @@ class MCPLangchainAgent(LangchainAgent):
144
144
  # Using the event loop directly instead of asyncio.run()
145
145
  loop = asyncio.get_event_loop()
146
146
  loop.run_until_complete(self.connect_to_mcp())
147
- except Exception as e:
148
- logger.error(f"Failed to connect to MCP server: {str(e)}")
147
+ except Exception:
148
+ logger.exception("Failed to connect to MCP server:")
149
149
 
150
150
  # Call parent implementation to get completion
151
151
  response = super().get_completion(messages, stream)
@@ -224,8 +224,8 @@ class LiteLLMAgentWrapper:
224
224
  }
225
225
  # Allow async context switch
226
226
  await asyncio.sleep(0)
227
- except Exception as e:
228
- logger.error(f"Streaming error: {str(e)}")
227
+ except Exception:
228
+ logger.exception("Streaming error:")
229
229
  raise
230
230
 
231
231
  async def cleanup(self):
@@ -3,11 +3,11 @@ Wrapper around MindsDB's executor and integration controller following the imple
3
3
  langchain.sql_database.SQLDatabase class to partly replicate its behavior.
4
4
  """
5
5
 
6
- import traceback
7
6
  from typing import Any, Iterable, List, Optional
8
7
 
9
- from mindsdb.utilities import log
10
8
  from langchain_community.utilities import SQLDatabase
9
+
10
+ from mindsdb.utilities import log
11
11
  from mindsdb.interfaces.skills.sql_agent import SQLAgent
12
12
 
13
13
  logger = log.getLogger(__name__)
@@ -99,7 +99,7 @@ class MindsDBSQL(SQLDatabase):
99
99
  return self._sql_agent.query(command)
100
100
 
101
101
  except Exception as e:
102
- logger.error(f"Error executing SQL command: {str(e)}\n{traceback.format_exc()}")
102
+ logger.exception("Error executing SQL command:")
103
103
  # If this is a knowledge base query, provide a more helpful error message
104
104
  if "knowledge_base" in command.lower() or any(
105
105
  kb in command for kb in self._sql_agent.get_usable_knowledge_base_names()
@@ -115,8 +115,8 @@ class MindsDBSQL(SQLDatabase):
115
115
  """
116
116
  try:
117
117
  return self._sql_agent.get_usable_knowledge_base_names()
118
- except Exception as e:
119
- logger.error(f"Error getting usable knowledge base names: {str(e)}")
118
+ except Exception:
119
+ logger.exception("Error getting usable knowledge base names:")
120
120
  return []
121
121
 
122
122
  def check_knowledge_base_permission(self, name):
@@ -37,8 +37,8 @@ async def run_conversation(agent_wrapper, messages: List[Dict[str, str]], stream
37
37
  # We still need to display the response to the user
38
38
  sys.stdout.write(f"{content}\n")
39
39
  sys.stdout.flush()
40
- except Exception as e:
41
- logger.error(f"Error during agent conversation: {str(e)}")
40
+ except Exception:
41
+ logger.exception("Error during agent conversation:")
42
42
 
43
43
 
44
44
  async def execute_direct_query(query):
@@ -48,11 +48,7 @@ async def execute_direct_query(query):
48
48
  # Set up MCP client to connect to the running server
49
49
  async with AsyncExitStack() as stack:
50
50
  # Connect to MCP server
51
- server_params = StdioServerParameters(
52
- command="python",
53
- args=["-m", "mindsdb", "--api=mcp"],
54
- env=None
55
- )
51
+ server_params = StdioServerParameters(command="python", args=["-m", "mindsdb", "--api=mcp"], env=None)
56
52
 
57
53
  try:
58
54
  stdio_transport = await stack.enter_async_context(stdio_client(server_params))
@@ -80,9 +76,9 @@ async def execute_direct_query(query):
80
76
  # Execute query
81
77
  result = await session.call_tool("query", {"query": query})
82
78
  logger.info(f"Query result: {result.content}")
83
- except Exception as e:
84
- logger.error(f"Error executing query: {str(e)}")
85
- logger.info("Make sure the MindsDB server is running with MCP enabled: python -m mindsdb --api=mysql,mcp,http")
79
+ except Exception:
80
+ logger.exception("Error executing query:")
81
+ logger.info("Make sure the MindsDB server is running with HTTP enabled: python -m mindsdb --api=http")
86
82
 
87
83
 
88
84
  async def main():
@@ -100,6 +96,7 @@ async def main():
100
96
  try:
101
97
  # Initialize database connection
102
98
  from mindsdb.interfaces.storage import db
99
+
103
100
  db.init()
104
101
 
105
102
  # Direct SQL execution mode (for testing MCP connection)
@@ -117,10 +114,7 @@ async def main():
117
114
  logger.info("Make sure MindsDB server is running with MCP enabled: python -m mindsdb --api=mysql,mcp,http")
118
115
 
119
116
  agent_wrapper = create_mcp_agent(
120
- agent_name=args.agent,
121
- project_name=args.project,
122
- mcp_host=args.host,
123
- mcp_port=args.port
117
+ agent_name=args.agent, project_name=args.project, mcp_host=args.host, mcp_port=args.port
124
118
  )
125
119
 
126
120
  # Run an example query if provided
@@ -173,7 +167,7 @@ async def main():
173
167
  sys.stdout.write("Error: No active MCP session\n")
174
168
  sys.stdout.flush()
175
169
  except Exception as e:
176
- logger.error(f"SQL Error: {str(e)}")
170
+ logger.exception("SQL Error:")
177
171
  sys.stdout.write(f"SQL Error: {str(e)}\n")
178
172
  sys.stdout.flush()
179
173
  continue
@@ -184,17 +178,14 @@ async def main():
184
178
  # Add assistant's response to the conversation history
185
179
  if not args.stream:
186
180
  response = await agent_wrapper.acompletion(messages)
187
- messages.append({
188
- "role": "assistant",
189
- "content": response["choices"][0]["message"]["content"]
190
- })
181
+ messages.append({"role": "assistant", "content": response["choices"][0]["message"]["content"]})
191
182
 
192
183
  # Clean up resources
193
184
  logger.info("Cleaning up resources")
194
185
  await agent_wrapper.cleanup()
195
186
 
196
- except Exception as e:
197
- logger.error(f"Error running MCP agent: {str(e)}")
187
+ except Exception:
188
+ logger.exception("Error running MCP agent:")
198
189
  logger.info("Make sure the MindsDB server is running with MCP enabled: python -m mindsdb --api=mysql,mcp,http")
199
190
  return 1
200
191
 
@@ -21,7 +21,6 @@ HOLDING_MESSAGE = "Bot is typing..."
21
21
 
22
22
 
23
23
  class ChatBotTask(BaseTask):
24
-
25
24
  def __init__(self, *args, **kwargs):
26
25
  super().__init__(*args, **kwargs)
27
26
  self.bot_id = self.object_id
@@ -40,7 +39,7 @@ class ChatBotTask(BaseTask):
40
39
  self.agent_id = bot_record.agent_id
41
40
  if self.agent_id is not None:
42
41
  self.bot_executor_cls = AgentExecutor
43
- elif self.bot_params.get('modes') is None:
42
+ elif self.bot_params.get("modes") is None:
44
43
  self.bot_executor_cls = BotExecutor
45
44
  else:
46
45
  self.bot_executor_cls = MultiModeBotExecutor
@@ -52,36 +51,35 @@ class ChatBotTask(BaseTask):
52
51
  raise Exception(f"Can't use chat database: {database_name}")
53
52
 
54
53
  chat_params = self.chat_handler.get_chat_config()
55
- polling = chat_params['polling']['type']
54
+ polling = chat_params["polling"]["type"]
56
55
 
57
- memory = chat_params['memory']['type'] if 'memory' in chat_params else None
56
+ memory = chat_params["memory"]["type"] if "memory" in chat_params else None
58
57
  memory_cls = None
59
58
  if memory:
60
- memory_cls = DBMemory if memory == 'db' else HandlerMemory
59
+ memory_cls = DBMemory if memory == "db" else HandlerMemory
61
60
 
62
- if polling == 'message_count':
63
- chat_params = chat_params['tables'] if 'tables' in chat_params else [chat_params]
61
+ if polling == "message_count":
62
+ chat_params = chat_params["tables"] if "tables" in chat_params else [chat_params]
64
63
  self.chat_pooling = MessageCountPolling(self, chat_params)
65
64
  # The default type for message count polling is HandlerMemory if not specified.
66
65
  self.memory = HandlerMemory(self, chat_params) if memory_cls is None else memory_cls(self, chat_params)
67
66
 
68
- elif polling == 'realtime':
69
- chat_params = chat_params['tables'] if 'tables' in chat_params else [chat_params]
67
+ elif polling == "realtime":
68
+ chat_params = chat_params["tables"] if "tables" in chat_params else [chat_params]
70
69
  self.chat_pooling = RealtimePolling(self, chat_params)
71
70
  # The default type for real-time polling is DBMemory if not specified.
72
71
  self.memory = DBMemory(self, chat_params) if memory_cls is None else memory_cls(self, chat_params)
73
72
 
74
- elif polling == 'webhook':
73
+ elif polling == "webhook":
75
74
  self.chat_pooling = WebhookPolling(self, chat_params)
76
75
  self.memory = DBMemory(self, chat_params)
77
76
 
78
77
  else:
79
78
  raise Exception(f"Not supported polling: {polling}")
80
79
 
81
- self.bot_params['bot_username'] = self.chat_handler.get_my_user_name()
80
+ self.bot_params["bot_username"] = self.chat_handler.get_my_user_name()
82
81
 
83
82
  def run(self, stop_event):
84
-
85
83
  # TODO check deleted, raise errors
86
84
  # TODO checks on delete predictor / project/ integration
87
85
 
@@ -89,7 +87,7 @@ class ChatBotTask(BaseTask):
89
87
 
90
88
  def on_message(self, message: ChatBotMessage, chat_id=None, chat_memory=None, table_name=None):
91
89
  if not chat_id and not chat_memory:
92
- raise Exception('chat_id or chat_memory should be provided')
90
+ raise Exception("chat_id or chat_memory should be provided")
93
91
 
94
92
  try:
95
93
  self._on_holding_message(chat_id, chat_memory, table_name)
@@ -97,9 +95,8 @@ class ChatBotTask(BaseTask):
97
95
  except (SystemExit, KeyboardInterrupt):
98
96
  raise
99
97
  except Exception:
100
- error = traceback.format_exc()
101
- logger.error(error)
102
- self.set_error(str(error))
98
+ logger.exception("Error during chatbot message processing:")
99
+ self.set_error(traceback.format_exc())
103
100
 
104
101
  def _on_holding_message(self, chat_id: str = None, chat_memory: BaseMemory = None, table_name: str = None):
105
102
  """
@@ -117,14 +114,14 @@ class ChatBotTask(BaseTask):
117
114
  ChatBotMessage.Type.DIRECT,
118
115
  HOLDING_MESSAGE,
119
116
  # In Slack direct messages are treated as channels themselves.
120
- user=self.bot_params['bot_username'],
117
+ user=self.bot_params["bot_username"],
121
118
  destination=chat_id,
122
- sent_at=dt.datetime.now()
119
+ sent_at=dt.datetime.now(),
123
120
  )
124
121
 
125
122
  # send to chat adapter
126
123
  self.chat_pooling.send_message(response_message, table_name=table_name)
127
- logger.debug(f'>>chatbot {chat_id} out (holding message): {response_message.text}')
124
+ logger.debug(f">>chatbot {chat_id} out (holding message): {response_message.text}")
128
125
 
129
126
  def _on_message(self, message: ChatBotMessage, chat_id, chat_memory, table_name=None):
130
127
  # add question to history
@@ -132,26 +129,26 @@ class ChatBotTask(BaseTask):
132
129
  chat_memory = chat_memory if chat_memory else self.memory.get_chat(chat_id, table_name=table_name)
133
130
  chat_memory.add_to_history(message)
134
131
 
135
- logger.debug(f'>>chatbot {chat_memory.chat_id} in: {message.text}')
132
+ logger.debug(f">>chatbot {chat_memory.chat_id} in: {message.text}")
136
133
 
137
134
  # process
138
135
  bot_executor = self.bot_executor_cls(self, chat_memory)
139
136
  response_text = bot_executor.process()
140
137
 
141
138
  chat_id = chat_memory.chat_id
142
- bot_username = self.bot_params['bot_username']
139
+ bot_username = self.bot_params["bot_username"]
143
140
  response_message = ChatBotMessage(
144
141
  ChatBotMessage.Type.DIRECT,
145
142
  response_text,
146
143
  # In Slack direct messages are treated as channels themselves.
147
144
  user=bot_username,
148
145
  destination=chat_id,
149
- sent_at=dt.datetime.now()
146
+ sent_at=dt.datetime.now(),
150
147
  )
151
148
 
152
149
  # send to chat adapter
153
150
  self.chat_pooling.send_message(response_message, table_name=table_name)
154
- logger.debug(f'>>chatbot {chat_id} out: {response_message.text}')
151
+ logger.debug(f">>chatbot {chat_id} out: {response_message.text}")
155
152
 
156
153
  # send to history
157
154
  chat_memory.add_to_history(response_message)
@@ -24,10 +24,14 @@ class BasePolling:
24
24
  chat_id = message.destination if isinstance(message.destination, tuple) else (message.destination,)
25
25
  text = message.text
26
26
 
27
- t_params = self.params["chat_table"] if table_name is None else next(
28
- (t["chat_table"] for t in self.params if t["chat_table"]["name"] == table_name)
27
+ t_params = (
28
+ self.params["chat_table"]
29
+ if table_name is None
30
+ else next((t["chat_table"] for t in self.params if t["chat_table"]["name"] == table_name))
31
+ )
32
+ chat_id_cols = (
33
+ t_params["chat_id_col"] if isinstance(t_params["chat_id_col"], list) else [t_params["chat_id_col"]]
29
34
  )
30
- chat_id_cols = t_params["chat_id_col"] if isinstance(t_params["chat_id_col"], list) else [t_params["chat_id_col"]]
31
35
 
32
36
  ast_query = Insert(
33
37
  table=Identifier(t_params["name"]),
@@ -60,20 +64,22 @@ class MessageCountPolling(BasePolling):
60
64
  chat_id,
61
65
  table_name=chat_params["chat_table"]["name"],
62
66
  )
63
- except Exception as e:
64
- logger.error(f"Problem retrieving chat memory: {e}")
67
+ except Exception:
68
+ logger.exception("Problem retrieving chat memory:")
65
69
 
66
70
  try:
67
71
  message = self.get_last_message(chat_memory)
68
- except Exception as e:
69
- logger.error(f"Problem getting last message: {e}")
72
+ except Exception:
73
+ logger.exception("Problem getting last message:")
70
74
  message = None
71
75
 
72
76
  if message:
73
- self.chat_task.on_message(message, chat_memory=chat_memory, table_name=chat_params["chat_table"]["name"])
77
+ self.chat_task.on_message(
78
+ message, chat_memory=chat_memory, table_name=chat_params["chat_table"]["name"]
79
+ )
74
80
 
75
- except Exception as e:
76
- logger.error(e)
81
+ except Exception:
82
+ logger.exception("Unexpected error")
77
83
 
78
84
  if stop_event.is_set():
79
85
  return
@@ -84,8 +90,8 @@ class MessageCountPolling(BasePolling):
84
90
  # retrive from history
85
91
  try:
86
92
  history = chat_memory.get_history()
87
- except Exception as e:
88
- logger.error(f"Problem retrieving history: {e}")
93
+ except Exception:
94
+ logger.exception("Problem retrieving history:")
89
95
  history = []
90
96
  last_message = history[-1]
91
97
  if last_message.user == self.chat_task.bot_params["bot_username"]:
@@ -151,7 +157,11 @@ class RealtimePolling(BasePolling):
151
157
  # Identify the table relevant to this event based on the key.
152
158
  event_keys = list(key.keys())
153
159
  for param in self.params:
154
- table_keys = [param["chat_table"]["chat_id_col"]] if isinstance(param["chat_table"]["chat_id_col"], str) else param["chat_table"]["chat_id_col"]
160
+ table_keys = (
161
+ [param["chat_table"]["chat_id_col"]]
162
+ if isinstance(param["chat_table"]["chat_id_col"], str)
163
+ else param["chat_table"]["chat_id_col"]
164
+ )
155
165
 
156
166
  if sorted(event_keys) == sorted(table_keys):
157
167
  t_params = param["chat_table"]
@@ -162,7 +172,11 @@ class RealtimePolling(BasePolling):
162
172
  t_params = self.params[0]
163
173
 
164
174
  # Get the chat ID from the row based on the chat ID column(s).
165
- chat_id = tuple(row[key] for key in t_params["chat_id_col"]) if isinstance(t_params["chat_id_col"], list) else row[t_params["chat_id_col"]]
175
+ chat_id = (
176
+ tuple(row[key] for key in t_params["chat_id_col"])
177
+ if isinstance(t_params["chat_id_col"], list)
178
+ else row[t_params["chat_id_col"]]
179
+ )
166
180
 
167
181
  message = ChatBotMessage(
168
182
  ChatBotMessage.Type.DIRECT,
@@ -179,10 +193,7 @@ class RealtimePolling(BasePolling):
179
193
  )
180
194
 
181
195
  def run(self, stop_event):
182
- self.chat_task.chat_handler.subscribe(
183
- stop_event,
184
- self._callback
185
- )
196
+ self.chat_task.chat_handler.subscribe(stop_event, self._callback)
186
197
 
187
198
  # def send_message(self, message: ChatBotMessage):
188
199
  #
@@ -193,6 +204,7 @@ class WebhookPolling(BasePolling):
193
204
  """
194
205
  Polling class for handling webhooks.
195
206
  """
207
+
196
208
  def __init__(self, *args, **kwargs):
197
209
  super().__init__(*args, **kwargs)
198
210
 
@@ -109,8 +109,8 @@ class DataCatalogLoader(BaseDataCatalog):
109
109
 
110
110
  db.session.add_all(tables)
111
111
  db.session.commit()
112
- except Exception as e:
113
- self.logger.error(f"Failed to add tables: {e}")
112
+ except Exception:
113
+ self.logger.exception("Failed to add tables:")
114
114
  db.session.rollback()
115
115
  raise
116
116
  return tables
@@ -157,8 +157,8 @@ class DataCatalogLoader(BaseDataCatalog):
157
157
 
158
158
  db.session.add_all(columns)
159
159
  db.session.commit()
160
- except Exception as e:
161
- self.logger.error(f"Failed to add columns: {e}")
160
+ except Exception:
161
+ self.logger.exception("Failed to add columns:")
162
162
  db.session.rollback()
163
163
  raise
164
164
  return columns
@@ -223,8 +223,8 @@ class DataCatalogLoader(BaseDataCatalog):
223
223
 
224
224
  db.session.add_all(column_statistics)
225
225
  db.session.commit()
226
- except Exception as e:
227
- self.logger.error(f"Failed to add column statistics: {e}")
226
+ except Exception:
227
+ self.logger.exception("Failed to add column statistics:")
228
228
  db.session.rollback()
229
229
  raise
230
230
 
@@ -275,8 +275,8 @@ class DataCatalogLoader(BaseDataCatalog):
275
275
 
276
276
  db.session.add_all(primary_keys)
277
277
  db.session.commit()
278
- except Exception as e:
279
- self.logger.error(f"Failed to add primary keys: {e}")
278
+ except Exception:
279
+ self.logger.exception("Failed to add primary keys:")
280
280
  db.session.rollback()
281
281
  raise
282
282
 
@@ -344,8 +344,8 @@ class DataCatalogLoader(BaseDataCatalog):
344
344
 
345
345
  db.session.add_all(foreign_keys)
346
346
  db.session.commit()
347
- except Exception as e:
348
- self.logger.error(f"Failed to add foreign keys: {e}")
347
+ except Exception:
348
+ self.logger.exception("Failed to add foreign keys:")
349
349
  db.session.rollback()
350
350
  raise
351
351
 
@@ -71,22 +71,33 @@ class HandlersCache:
71
71
  def set(self, handler: DatabaseHandler):
72
72
  """add (or replace) handler in cache
73
73
 
74
+ NOTE: If the handler is not thread-safe, then use a lock when making connection. Otherwise, make connection in
75
+ the same thread without using a lock to speed up parallel queries. (They don't need to wait for a connection in
76
+ another thread.)
77
+
74
78
  Args:
75
79
  handler (DatabaseHandler)
76
80
  """
81
+ thread_safe = getattr(handler, "thread_safe", False)
77
82
  with self._lock:
78
83
  try:
79
84
  # If the handler is defined to be thread safe, set 0 as the last element of the key, otherwise set the thrad ID.
80
85
  key = (
81
86
  handler.name,
82
87
  ctx.company_id,
83
- 0 if getattr(handler, "thread_safe", False) else threading.get_native_id(),
88
+ 0 if thread_safe else threading.get_native_id(),
84
89
  )
85
- handler.connect()
86
90
  self.handlers[key] = {"handler": handler, "expired_at": time.time() + self.ttl}
91
+ if thread_safe:
92
+ handler.connect()
87
93
  except Exception:
88
94
  pass
89
95
  self._start_clean()
96
+ try:
97
+ if not thread_safe:
98
+ handler.connect()
99
+ except Exception:
100
+ pass
90
101
 
91
102
  def get(self, name: str) -> Optional[DatabaseHandler]:
92
103
  """get handler from cache by name
@@ -866,6 +877,8 @@ class IntegrationController:
866
877
 
867
878
  def import_handler(self, handler_name: str, base_import: str = None):
868
879
  with self._import_lock:
880
+ time_before_import = time.perf_counter()
881
+ logger.debug(f"Importing handler '{handler_name}'")
869
882
  handler_meta = self.handlers_import_status[handler_name]
870
883
  handler_dir = handler_meta["path"]
871
884
 
@@ -877,9 +890,13 @@ class IntegrationController:
877
890
  handler_module = importlib.import_module(f"{base_import}{handler_folder_name}")
878
891
  self.handler_modules[handler_name] = handler_module
879
892
  handler_meta = self._get_handler_meta(handler_name)
893
+ logger.debug(
894
+ f"Handler '{handler_name}' imported successfully in {(time.perf_counter() - time_before_import):.3f} seconds"
895
+ )
880
896
  except Exception as e:
881
897
  handler_meta["import"]["success"] = False
882
898
  handler_meta["import"]["error_message"] = str(e)
899
+ logger.debug(f"Failed to import handler '{handler_name}': {e}")
883
900
 
884
901
  self.handlers_import_status[handler_meta["name"]] = handler_meta
885
902
  return handler_meta
@@ -107,8 +107,8 @@ class FileController:
107
107
  self.fs_store.put(store_file_path, base_dir=self.dir)
108
108
  db.session.commit()
109
109
 
110
- except Exception as e:
111
- logger.error(e)
110
+ except Exception:
111
+ logger.exception("An error occurred while saving the file:")
112
112
  if file_dir is not None:
113
113
  shutil.rmtree(file_dir)
114
114
  raise
@@ -151,7 +151,7 @@ class FileController:
151
151
  def delete_file(self, name):
152
152
  file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
153
153
  if file_record is None:
154
- return None
154
+ raise FileNotFoundError(f"File '{name}' does not exists")
155
155
  file_id = file_record.id
156
156
  db.session.delete(file_record)
157
157
  db.session.commit()
@@ -161,7 +161,7 @@ class FileController:
161
161
  def get_file_path(self, name):
162
162
  file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
163
163
  if file_record is None:
164
- raise Exception(f"File '{name}' does not exists")
164
+ raise FileNotFoundError(f"File '{name}' does not exists")
165
165
  file_dir = f"file_{ctx.company_id}_{file_record.id}"
166
166
  self.fs_store.get(file_dir, base_dir=self.dir)
167
167
  return str(Path(self.dir).joinpath(file_dir).joinpath(Path(file_record.source_file_path).name))
@@ -176,7 +176,7 @@ class FileController:
176
176
  """
177
177
  file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
178
178
  if file_record is None:
179
- raise Exception(f"File '{name}' does not exists")
179
+ raise FileNotFoundError(f"File '{name}' does not exists")
180
180
 
181
181
  file_dir = f"file_{ctx.company_id}_{file_record.id}"
182
182
  self.fs_store.get(file_dir, base_dir=self.dir)
@@ -217,7 +217,7 @@ class FileController:
217
217
 
218
218
  file_record = db.session.query(db.File).filter_by(company_id=ctx.company_id, name=name).first()
219
219
  if file_record is None:
220
- raise Exception(f"File '{name}' does not exists")
220
+ raise FileNotFoundError(f"File '{name}' does not exists")
221
221
 
222
222
  file_dir = f"file_{ctx.company_id}_{file_record.id}"
223
223
  self.fs_store.get(file_dir, base_dir=self.dir)
@@ -140,7 +140,7 @@ class FunctionController(BYOMFunctionsController):
140
140
 
141
141
  llm = create_chat_model(chat_model_params)
142
142
  except Exception as e:
143
- raise RuntimeError(f"Unable to use LLM function, check ENV variables: {e}")
143
+ raise RuntimeError(f"Unable to use LLM function, check ENV variables: {e}") from e
144
144
 
145
145
  def callback(question):
146
146
  resp = llm([HumanMessage(question)])
@@ -69,8 +69,8 @@ class ToMarkdown:
69
69
  ext = os.path.splitext(parsed_url.path)[1]
70
70
  if ext:
71
71
  return ext
72
- except requests.RequestException:
73
- raise RuntimeError(f"Unable to retrieve file extension from URL: {file_path_or_url}")
72
+ except requests.RequestException as e:
73
+ raise RuntimeError(f"Unable to retrieve file extension from URL: {file_path_or_url}") from e
74
74
  else:
75
75
  return os.path.splitext(file_path_or_url)[1]
76
76
 
@@ -152,7 +152,7 @@ class JobsController:
152
152
 
153
153
  parse_sql(sql)
154
154
  except ParsingException as e:
155
- raise ParsingException(f"Unable to parse: {sql}: {e}")
155
+ raise ParsingException(f"Unable to parse: {sql}: {e}") from e
156
156
 
157
157
  if if_query is not None:
158
158
  for sql in split_sql(if_query):
@@ -162,7 +162,7 @@ class JobsController:
162
162
 
163
163
  parse_sql(sql)
164
164
  except ParsingException as e:
165
- raise ParsingException(f"Unable to parse: {sql}: {e}")
165
+ raise ParsingException(f"Unable to parse: {sql}: {e}") from e
166
166
 
167
167
  # plan next run
168
168
  next_run_at = start_at
@@ -494,7 +494,7 @@ class JobsExecutor:
494
494
 
495
495
  data = ret.data
496
496
  except Exception as e:
497
- logger.error(e)
497
+ logger.exception("Error to execute job`s condition query")
498
498
  error = str(e)
499
499
  break
500
500
 
@@ -518,7 +518,7 @@ class JobsExecutor:
518
518
  error = ret.error_message
519
519
  break
520
520
  except Exception as e:
521
- logger.error(e)
521
+ logger.exception("Error to execute job`s query")
522
522
  error = str(e)
523
523
  break
524
524
 
@@ -526,7 +526,7 @@ class JobsExecutor:
526
526
  self.update_task_schedule(record)
527
527
  except Exception as e:
528
528
  db.session.rollback()
529
- logger.error(f"Error to update schedule: {e}")
529
+ logger.exception("Error to update schedule:")
530
530
  error += f"Error to update schedule: {e}"
531
531
 
532
532
  # stop scheduling