MindsDB 25.9.2.0a1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of MindsDB might be problematic. Click here for more details.

Files changed (163) hide show
  1. mindsdb/__about__.py +1 -1
  2. mindsdb/__main__.py +40 -29
  3. mindsdb/api/a2a/__init__.py +1 -1
  4. mindsdb/api/a2a/agent.py +16 -10
  5. mindsdb/api/a2a/common/server/server.py +7 -3
  6. mindsdb/api/a2a/common/server/task_manager.py +12 -5
  7. mindsdb/api/a2a/common/types.py +66 -0
  8. mindsdb/api/a2a/task_manager.py +65 -17
  9. mindsdb/api/common/middleware.py +10 -12
  10. mindsdb/api/executor/command_executor.py +51 -40
  11. mindsdb/api/executor/datahub/datanodes/datanode.py +2 -2
  12. mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py +7 -13
  13. mindsdb/api/executor/datahub/datanodes/integration_datanode.py +101 -49
  14. mindsdb/api/executor/datahub/datanodes/project_datanode.py +8 -4
  15. mindsdb/api/executor/datahub/datanodes/system_tables.py +3 -2
  16. mindsdb/api/executor/exceptions.py +29 -10
  17. mindsdb/api/executor/planner/plan_join.py +17 -3
  18. mindsdb/api/executor/planner/query_prepare.py +2 -20
  19. mindsdb/api/executor/sql_query/sql_query.py +74 -74
  20. mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +1 -2
  21. mindsdb/api/executor/sql_query/steps/subselect_step.py +0 -1
  22. mindsdb/api/executor/utilities/functions.py +6 -6
  23. mindsdb/api/executor/utilities/sql.py +37 -20
  24. mindsdb/api/http/gui.py +5 -11
  25. mindsdb/api/http/initialize.py +75 -61
  26. mindsdb/api/http/namespaces/agents.py +10 -15
  27. mindsdb/api/http/namespaces/analysis.py +13 -20
  28. mindsdb/api/http/namespaces/auth.py +1 -1
  29. mindsdb/api/http/namespaces/chatbots.py +0 -5
  30. mindsdb/api/http/namespaces/config.py +15 -11
  31. mindsdb/api/http/namespaces/databases.py +140 -201
  32. mindsdb/api/http/namespaces/file.py +17 -4
  33. mindsdb/api/http/namespaces/handlers.py +17 -7
  34. mindsdb/api/http/namespaces/knowledge_bases.py +28 -7
  35. mindsdb/api/http/namespaces/models.py +94 -126
  36. mindsdb/api/http/namespaces/projects.py +13 -22
  37. mindsdb/api/http/namespaces/sql.py +33 -25
  38. mindsdb/api/http/namespaces/tab.py +27 -37
  39. mindsdb/api/http/namespaces/views.py +1 -1
  40. mindsdb/api/http/start.py +16 -10
  41. mindsdb/api/mcp/__init__.py +2 -1
  42. mindsdb/api/mysql/mysql_proxy/executor/mysql_executor.py +15 -20
  43. mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +26 -50
  44. mindsdb/api/mysql/mysql_proxy/utilities/__init__.py +0 -1
  45. mindsdb/api/mysql/mysql_proxy/utilities/dump.py +8 -2
  46. mindsdb/integrations/handlers/byom_handler/byom_handler.py +165 -190
  47. mindsdb/integrations/handlers/databricks_handler/databricks_handler.py +98 -46
  48. mindsdb/integrations/handlers/druid_handler/druid_handler.py +32 -40
  49. mindsdb/integrations/handlers/file_handler/file_handler.py +7 -0
  50. mindsdb/integrations/handlers/gitlab_handler/gitlab_handler.py +5 -2
  51. mindsdb/integrations/handlers/lightwood_handler/functions.py +45 -79
  52. mindsdb/integrations/handlers/mssql_handler/mssql_handler.py +438 -100
  53. mindsdb/integrations/handlers/mssql_handler/requirements_odbc.txt +3 -0
  54. mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +235 -3
  55. mindsdb/integrations/handlers/oracle_handler/__init__.py +2 -0
  56. mindsdb/integrations/handlers/oracle_handler/connection_args.py +7 -1
  57. mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +321 -16
  58. mindsdb/integrations/handlers/oracle_handler/requirements.txt +1 -1
  59. mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +14 -2
  60. mindsdb/integrations/handlers/shopify_handler/shopify_handler.py +25 -12
  61. mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +2 -1
  62. mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +1 -0
  63. mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +1 -0
  64. mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +4 -4
  65. mindsdb/integrations/handlers/zendesk_handler/zendesk_tables.py +144 -111
  66. mindsdb/integrations/libs/api_handler.py +10 -10
  67. mindsdb/integrations/libs/base.py +4 -4
  68. mindsdb/integrations/libs/llm/utils.py +2 -2
  69. mindsdb/integrations/libs/ml_handler_process/create_engine_process.py +4 -7
  70. mindsdb/integrations/libs/ml_handler_process/func_call_process.py +2 -7
  71. mindsdb/integrations/libs/ml_handler_process/learn_process.py +37 -47
  72. mindsdb/integrations/libs/ml_handler_process/update_engine_process.py +4 -7
  73. mindsdb/integrations/libs/ml_handler_process/update_process.py +2 -7
  74. mindsdb/integrations/libs/process_cache.py +132 -140
  75. mindsdb/integrations/libs/response.py +18 -12
  76. mindsdb/integrations/libs/vectordatabase_handler.py +26 -0
  77. mindsdb/integrations/utilities/files/file_reader.py +6 -7
  78. mindsdb/integrations/utilities/handlers/auth_utilities/snowflake/__init__.py +1 -0
  79. mindsdb/integrations/utilities/handlers/auth_utilities/snowflake/snowflake_jwt_gen.py +151 -0
  80. mindsdb/integrations/utilities/rag/config_loader.py +37 -26
  81. mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +83 -30
  82. mindsdb/integrations/utilities/rag/rerankers/reranker_compressor.py +4 -4
  83. mindsdb/integrations/utilities/rag/retrievers/sql_retriever.py +55 -133
  84. mindsdb/integrations/utilities/rag/settings.py +58 -133
  85. mindsdb/integrations/utilities/rag/splitters/file_splitter.py +5 -15
  86. mindsdb/interfaces/agents/agents_controller.py +2 -3
  87. mindsdb/interfaces/agents/constants.py +0 -2
  88. mindsdb/interfaces/agents/litellm_server.py +34 -58
  89. mindsdb/interfaces/agents/mcp_client_agent.py +10 -10
  90. mindsdb/interfaces/agents/mindsdb_database_agent.py +5 -5
  91. mindsdb/interfaces/agents/run_mcp_agent.py +12 -21
  92. mindsdb/interfaces/chatbot/chatbot_task.py +20 -23
  93. mindsdb/interfaces/chatbot/polling.py +30 -18
  94. mindsdb/interfaces/data_catalog/data_catalog_loader.py +16 -17
  95. mindsdb/interfaces/data_catalog/data_catalog_reader.py +15 -4
  96. mindsdb/interfaces/database/data_handlers_cache.py +190 -0
  97. mindsdb/interfaces/database/database.py +3 -3
  98. mindsdb/interfaces/database/integrations.py +7 -110
  99. mindsdb/interfaces/database/projects.py +2 -6
  100. mindsdb/interfaces/database/views.py +1 -4
  101. mindsdb/interfaces/file/file_controller.py +6 -6
  102. mindsdb/interfaces/functions/controller.py +1 -1
  103. mindsdb/interfaces/functions/to_markdown.py +2 -2
  104. mindsdb/interfaces/jobs/jobs_controller.py +5 -9
  105. mindsdb/interfaces/jobs/scheduler.py +3 -9
  106. mindsdb/interfaces/knowledge_base/controller.py +244 -128
  107. mindsdb/interfaces/knowledge_base/evaluate.py +36 -41
  108. mindsdb/interfaces/knowledge_base/executor.py +11 -0
  109. mindsdb/interfaces/knowledge_base/llm_client.py +51 -17
  110. mindsdb/interfaces/knowledge_base/preprocessing/json_chunker.py +40 -61
  111. mindsdb/interfaces/model/model_controller.py +172 -168
  112. mindsdb/interfaces/query_context/context_controller.py +14 -2
  113. mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +10 -14
  114. mindsdb/interfaces/skills/retrieval_tool.py +43 -50
  115. mindsdb/interfaces/skills/skill_tool.py +2 -2
  116. mindsdb/interfaces/skills/skills_controller.py +1 -4
  117. mindsdb/interfaces/skills/sql_agent.py +25 -19
  118. mindsdb/interfaces/storage/db.py +16 -6
  119. mindsdb/interfaces/storage/fs.py +114 -169
  120. mindsdb/interfaces/storage/json.py +19 -18
  121. mindsdb/interfaces/tabs/tabs_controller.py +49 -72
  122. mindsdb/interfaces/tasks/task_monitor.py +3 -9
  123. mindsdb/interfaces/tasks/task_thread.py +7 -9
  124. mindsdb/interfaces/triggers/trigger_task.py +7 -13
  125. mindsdb/interfaces/triggers/triggers_controller.py +47 -52
  126. mindsdb/migrations/migrate.py +16 -16
  127. mindsdb/utilities/api_status.py +58 -0
  128. mindsdb/utilities/config.py +68 -2
  129. mindsdb/utilities/exception.py +40 -1
  130. mindsdb/utilities/fs.py +0 -1
  131. mindsdb/utilities/hooks/profiling.py +17 -14
  132. mindsdb/utilities/json_encoder.py +24 -10
  133. mindsdb/utilities/langfuse.py +40 -45
  134. mindsdb/utilities/log.py +272 -0
  135. mindsdb/utilities/ml_task_queue/consumer.py +52 -58
  136. mindsdb/utilities/ml_task_queue/producer.py +26 -30
  137. mindsdb/utilities/render/sqlalchemy_render.py +22 -20
  138. mindsdb/utilities/starters.py +0 -10
  139. mindsdb/utilities/utils.py +2 -2
  140. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0rc1.dist-info}/METADATA +293 -276
  141. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0rc1.dist-info}/RECORD +144 -158
  142. mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -14
  143. mindsdb/api/postgres/__init__.py +0 -0
  144. mindsdb/api/postgres/postgres_proxy/__init__.py +0 -0
  145. mindsdb/api/postgres/postgres_proxy/executor/__init__.py +0 -1
  146. mindsdb/api/postgres/postgres_proxy/executor/executor.py +0 -189
  147. mindsdb/api/postgres/postgres_proxy/postgres_packets/__init__.py +0 -0
  148. mindsdb/api/postgres/postgres_proxy/postgres_packets/errors.py +0 -322
  149. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_fields.py +0 -34
  150. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message.py +0 -31
  151. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_formats.py +0 -1265
  152. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_identifiers.py +0 -31
  153. mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +0 -253
  154. mindsdb/api/postgres/postgres_proxy/postgres_proxy.py +0 -477
  155. mindsdb/api/postgres/postgres_proxy/utilities/__init__.py +0 -10
  156. mindsdb/api/postgres/start.py +0 -11
  157. mindsdb/integrations/handlers/mssql_handler/tests/__init__.py +0 -0
  158. mindsdb/integrations/handlers/mssql_handler/tests/test_mssql_handler.py +0 -169
  159. mindsdb/integrations/handlers/oracle_handler/tests/__init__.py +0 -0
  160. mindsdb/integrations/handlers/oracle_handler/tests/test_oracle_handler.py +0 -32
  161. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0rc1.dist-info}/WHEEL +0 -0
  162. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0rc1.dist-info}/licenses/LICENSE +0 -0
  163. {mindsdb-25.9.2.0a1.dist-info → mindsdb-25.10.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,477 +0,0 @@
1
- import base64
2
- import datetime
3
- import os
4
- import json
5
- import select
6
- import socketserver
7
- import struct
8
- import sys
9
- from functools import partial
10
- import socket
11
- from typing import Callable, Dict, Type, Any, Iterable, Sequence
12
-
13
- from mindsdb.api.executor.controllers import SessionController
14
- from mindsdb.api.postgres.postgres_proxy.executor import Executor
15
- from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import CHARSET_NUMBERS
16
- from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
17
- from mindsdb.api.common.middleware import check_auth
18
- from mindsdb.api.mysql.mysql_proxy.mysql_proxy import SQLAnswer
19
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.errors import POSTGRES_SYNTAX_ERROR_CODE
20
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import GenericField, PostgresField
21
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_formats import (
22
- Terminate,
23
- Query,
24
- AuthenticationClearTextPassword,
25
- AuthenticationOk,
26
- RowDescriptions,
27
- DataRow,
28
- CommandComplete,
29
- ReadyForQuery,
30
- ConnectionFailure,
31
- ParameterStatus,
32
- Error,
33
- Execute,
34
- Bind,
35
- Parse,
36
- Sync,
37
- ParseComplete,
38
- InvalidSQLStatementName,
39
- BindComplete,
40
- Describe,
41
- DataException,
42
- ParameterDescription,
43
- )
44
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message import PostgresMessage
45
- from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_packets import (
46
- PostgresPacketReader,
47
- PostgresPacketBuilder,
48
- )
49
- from mindsdb.api.postgres.postgres_proxy.utilities import strip_null_byte
50
- from mindsdb.utilities.config import config
51
- from mindsdb.utilities.context import context as ctx
52
- from mindsdb.utilities import log
53
- from mindsdb.api.mysql.mysql_proxy.external_libs.mysql_scramble import scramble as scramble_func
54
-
55
-
56
- class PostgresProxyHandler(socketserver.StreamRequestHandler):
57
- client_buffer: PostgresPacketReader
58
- user_parameters: Dict[bytes, bytes]
59
-
60
- def __init__(self, request, client_address, server):
61
- self.logger = log.getLogger(__name__)
62
- self.charset = "utf8"
63
- self.charset_text_type = CHARSET_NUMBERS["utf8_general_ci"]
64
- self.session = None
65
- self.client_capabilities = None
66
- self.user_parameters = None
67
- self.named_statements = {}
68
- self.unnamed_statement = None
69
- self.is_cloud = False
70
- self.named_portals = {}
71
- self.unnamed_portal = None
72
- self.transaction_status = b"I" # I: Idle, T: Transaction Block, E: Failed Transaction Block
73
- super().__init__(request, client_address, server)
74
-
75
- def handle(self) -> None:
76
- ctx.set_default()
77
- self.init_session()
78
- self.logger.debug("handle new incoming connection")
79
- cloud_connection = self.is_cloud_connection()
80
- if cloud_connection["is_cloud"]:
81
- ctx.company_id = cloud_connection.get("company_id")
82
- self.is_cloud = True
83
-
84
- self.message_map: Dict[Type[PostgresMessage], Callable[[Any], bool]] = {
85
- Terminate: self.terminate,
86
- Query: self.query,
87
- Parse: self.parse,
88
- Bind: self.bind,
89
- Execute: self.execute,
90
- Describe: self.describe,
91
- Sync: self.sync,
92
- }
93
- self.client_buffer = PostgresPacketReader(self.rfile)
94
- if self.is_cloud:
95
- # We already have a connection started through the gateway.
96
- started = True
97
- self.handshake()
98
- else:
99
- started = self.start_connection()
100
- if started:
101
- self.logger.debug("connection started")
102
- self.send_initial_data()
103
- self.main_loop()
104
-
105
- def is_cloud_connection(self):
106
- """Determine source of connection. Must be call before handshake.
107
- Idea based on: real mysql connection does not send anything before server handshake, so
108
- socket should be in 'out' state. In opposite, clout connection sends '0000' right after
109
- connection. '0000' selected because in real mysql connection it should be length of package,
110
- and it can not be 0.
111
-
112
- Copied from mysql_proxy
113
- TODO: Extract method into common
114
- """
115
- is_cloud = config.get("cloud", False)
116
-
117
- if sys.platform != "linux" or is_cloud is False:
118
- return {"is_cloud": False}
119
-
120
- read_poller = select.poll()
121
- read_poller.register(self.request, select.POLLIN)
122
- events = read_poller.poll(30)
123
-
124
- if len(events) == 0:
125
- return {"is_cloud": False}
126
-
127
- first_byte = self.request.recv(4, socket.MSG_PEEK)
128
- if first_byte == b"\x00\x00\x00\x00":
129
- self.request.recv(4)
130
- client_capabilities = self.request.recv(8)
131
- client_capabilities = struct.unpack("L", client_capabilities)[0]
132
-
133
- company_id = self.request.recv(4)
134
- company_id = struct.unpack("I", company_id)[0]
135
-
136
- user_class = self.request.recv(1)
137
- user_class = struct.unpack("B", user_class)[0]
138
-
139
- database_name_len = self.request.recv(2)
140
- database_name_len = struct.unpack("H", database_name_len)[0]
141
-
142
- database_name = ""
143
- if database_name_len > 0:
144
- database_name = self.request.recv(database_name_len).decode()
145
-
146
- return {
147
- "is_cloud": True,
148
- "client_capabilities": client_capabilities,
149
- "company_id": company_id,
150
- "user_class": user_class,
151
- "database": database_name,
152
- }
153
-
154
- return {"is_cloud": False}
155
-
156
- def parse(self, message: Parse):
157
- self.logger.info("Postgres_Proxy: Parsing")
158
- executor = Executor(session=self.session, proxy_server=self, charset=self.charset)
159
- # TODO: Remove comment if unneeded ot use session since we're storing in this proxy class per session anyway
160
- # stmt_id = self.session.register_stmt(executor)
161
- executor.stmt_prepare(sql=message.query)
162
- statement = {"executor": executor, "parse": message}
163
- if message.name:
164
- self.named_statements[message.name] = statement
165
- else:
166
- self.unnamed_statement = statement
167
- self.send(ParseComplete())
168
- return True
169
-
170
- def bind(self, message: Bind):
171
- self.logger.info(f"Postgres_Proxy: Binding {message.name} with params {message.parameters}")
172
- if message.statement_name:
173
- statement = self.named_statements[message.statement_name]
174
- elif self.unnamed_statement:
175
- statement = self.unnamed_statement
176
- else:
177
- self.send(InvalidSQLStatementName())
178
- return True
179
-
180
- # TODO Should check validity of statement here and not at parse stage
181
- portal = statement.copy()
182
- portal["bind"] = message
183
- if message.name:
184
- self.named_portals[message.name] = portal
185
- else:
186
- self.unnamed_portal = portal
187
- self.send(BindComplete())
188
- return True
189
-
190
- def describe(self, message: Describe):
191
- self.logger.info("Postgres_Proxy: Describing")
192
- if message.describe_type == b"P":
193
- if message.name:
194
- describing = self.named_portals[message.name]
195
- elif self.unnamed_statement:
196
- describing = self.unnamed_portal
197
- else:
198
- self.send(InvalidSQLStatementName("Portal Does not Exist"))
199
- return True
200
- elif message.describe_type == b"S":
201
- if message.name:
202
- describing = self.named_statements[message.name]
203
- elif self.unnamed_statement:
204
- describing = self.unnamed_statement
205
- else:
206
- self.send(InvalidSQLStatementName())
207
- return True
208
- self.send(ParameterDescription(parameters=describing["parse"]["parameters"]))
209
- else:
210
- self.send(DataException(message="Describe did not have correct type. Can be 'P' or 'S'"))
211
- return True
212
-
213
- fields = self.to_postgres_fields(describing["executor"].columns)
214
- self.send(RowDescriptions(fields=fields))
215
- return True
216
-
217
- def execute(self, message: Execute):
218
- self.logger.info("Postgres_Proxy: Executing")
219
- if message.name:
220
- portal = self.named_portals[message.name]
221
- elif self.unnamed_portal:
222
- portal = self.unnamed_portal
223
- else:
224
- self.send(InvalidSQLStatementName("Portal does not exist"))
225
-
226
- executor = portal["executor"]
227
- params = portal["bind"].parameters
228
- executor.stmt_execute(param_values=params)
229
- sql_answer = self.return_executor_data(executor)
230
- self.respond_from_sql_answer(sql=executor.sql, sql_answer=sql_answer, row_descs=False)
231
- return True
232
-
233
- def sync(self, message: Sync):
234
- self.logger.info("Postgres_Proxy: Syncing")
235
- # TODO: Close/commit transaction if outside of a block. Maybe no collaries since Proxy
236
- self.send_ready()
237
- return True
238
-
239
- def init_session(self):
240
- self.logger.info("New connection [{ip}:{port}]".format(ip=self.client_address[0], port=self.client_address[1]))
241
- self.logger.debug(self.__dict__)
242
-
243
- if self.server.connection_id >= 65025:
244
- self.server.connection_id = 0
245
- self.server.connection_id += 1
246
- self.connection_id = self.server.connection_id
247
- self.session = SessionController()
248
- self.session.database = config.get("default_project")
249
-
250
- if hasattr(self.server, "salt") and isinstance(self.server.salt, str):
251
- self.salt = self.server.salt
252
- else:
253
- self.salt = base64.b64encode(os.urandom(15)).decode()
254
-
255
- self.socket = self.request
256
-
257
- self.current_transaction = None
258
-
259
- self.logger.debug("session salt: {salt}".format(salt=self.salt))
260
-
261
- def process_query(self, sql):
262
- executor = Executor(session=self.session, proxy_server=self, charset=self.charset)
263
- self.logger.debug("processing query\n%s", sql)
264
- try:
265
- executor.query_execute(sql)
266
- except Exception as e:
267
- return SQLAnswer(
268
- resp_type=RESPONSE_TYPE.ERROR,
269
- error_message=str(e).encode(self.get_encoding()),
270
- error_code=POSTGRES_SYNTAX_ERROR_CODE.encode(self.get_encoding()),
271
- )
272
- return self.return_executor_data(executor)
273
-
274
- def return_executor_data(self, executor):
275
- if executor.data is None:
276
- resp = SQLAnswer(
277
- resp_type=RESPONSE_TYPE.OK,
278
- state_track=executor.state_track,
279
- )
280
- else:
281
- resp = SQLAnswer(
282
- resp_type=RESPONSE_TYPE.TABLE,
283
- state_track=executor.state_track,
284
- result_set=executor.data,
285
- status=executor.server_status,
286
- )
287
- return resp
288
-
289
- def start_connection(self):
290
- self.logger.debug("starting handshake")
291
- self.handshake()
292
- self.logger.debug("handshake complete, checking authentication")
293
- return self.authenticate()
294
-
295
- def send(self, message: PostgresMessage):
296
- self.logger.debug("Sending message of type %s" % message.__class__.__name__)
297
- message.send(self.wfile)
298
-
299
- def handshake(self):
300
- self.client_buffer.read_verify_ssl_request()
301
- # self.send(NoticeResponse()) -- Should Probably not send. Looks in protocol manual to be sent for warning
302
- self.logger.debug("Sending No to SSL Request")
303
- PostgresPacketBuilder().write_char(b"N", self.wfile)
304
- self.user_parameters = self.client_buffer.read_startup_message()
305
-
306
- def authenticate(self, ask_for_password=False):
307
- if ask_for_password:
308
- self.send(AuthenticationClearTextPassword())
309
- password = self.client_buffer.read_authentication(encoding=self.charset)
310
- else:
311
- password = ""
312
- username = self.user_parameters[b"user"].decode(encoding=self.charset)
313
- auth_data = self.server.check_auth(username, password, scramble_func, self.salt, ctx.company_id)
314
- if auth_data["success"]:
315
- self.logger.debug("Authentication succeeded")
316
- self.session.username = auth_data["username"]
317
- self.session.auth = True
318
- self.send(AuthenticationOk())
319
- return True
320
- else:
321
- if not ask_for_password: # try asking for password
322
- return self.authenticate(ask_for_password=True)
323
- self.logger.debug("Authentication failed")
324
- self.send(ConnectionFailure(message="Authentication failed."))
325
- return False
326
-
327
- def terminate(self, message: Terminate) -> bool:
328
- self.logger.info("Postgres_Proxy: Terminating")
329
- return False
330
-
331
- def get_encoding(self) -> str:
332
- return self.user_parameters.get(b"user_encoding", None) or self.charset
333
-
334
- def return_ok(self, sql, rows: int = 0):
335
- command = self.get_command(sql)
336
- if command == "BEGIN":
337
- self.transaction_status = b"T"
338
- if command == "COMMIT":
339
- self.transaction_status = b"I"
340
- if command == "CREATE":
341
- command = "SELECT"
342
- if command in ("INSERT", "DELETE", "UPDATE", "SELECT", "MOVE", "FETCH", "COPY"):
343
- command = f"{command} {rows}"
344
- self.send(CommandComplete(tag=command.encode(encoding=self.get_encoding())))
345
- return True
346
-
347
- def get_command(self, sql):
348
- sql = self.stringify_sql(sql)
349
- return sql.split(" ", 1)[0]
350
-
351
- def stringify_sql(self, sql):
352
- if type(sql) == bytes:
353
- encoding = self.get_encoding()
354
- sql: str = sql.decode(encoding)
355
- return strip_null_byte(sql).strip(";")
356
-
357
- def return_table(self, sql_answer: SQLAnswer, row_descs=True):
358
- fields = self.to_postgres_fields(sql_answer.result_set.columns)
359
- rows = self.to_postgres_rows(sql_answer.result_set)
360
- if row_descs:
361
- self.send(RowDescriptions(fields=fields))
362
- self.send(DataRow(rows=rows))
363
- encoding = self.get_encoding()
364
- tag = ("SELECT %s" % str(len(rows))).encode(encoding)
365
- self.send(CommandComplete(tag=tag))
366
- return True
367
-
368
- def return_error(self, sql_answer: SQLAnswer):
369
- self.send(Error.from_answer(error_code=sql_answer.error_code, error_message=sql_answer.error_message))
370
- return True
371
-
372
- def query(self, message: Query) -> bool:
373
- self.logger.debug("Postgres Proxy: Got query of:\n%s" % message.sql)
374
- sql = message.get_parsed_sql()
375
- sql_answer = self.process_query(sql)
376
- self.respond_from_sql_answer(sql=sql, sql_answer=sql_answer)
377
- self.send_ready()
378
- return True
379
-
380
- def respond_from_sql_answer(self, sql, sql_answer: SQLAnswer, row_descs=True) -> bool:
381
- # TODO Add command complete passthrough for Complex Queries that exceed row limit in one go
382
- rows = 0
383
- if sql_answer.result_set:
384
- rows = len(sql_answer.result_set)
385
- if RESPONSE_TYPE.OK == sql_answer.type:
386
- return self.return_ok(sql, rows=rows)
387
- elif RESPONSE_TYPE.TABLE == sql_answer.type:
388
- return self.return_table(sql_answer, row_descs=row_descs)
389
- elif RESPONSE_TYPE.ERROR == sql_answer.type:
390
- return self.return_error(sql_answer)
391
-
392
- @staticmethod
393
- def to_postgres_fields(columns: Iterable[Dict[str, Any]]) -> Sequence[PostgresField]:
394
- fields = []
395
- i = 0
396
- for column in columns:
397
- fields.append(GenericField(name=column["name"], object_id=column["type"].value, column_id=i))
398
- i += 1
399
- return fields
400
-
401
- def to_postgres_rows(self, rows: Iterable[Iterable[Any]]) -> Sequence[Sequence[bytes]]:
402
- p_rows = []
403
- for row in rows:
404
- p_row = []
405
- for column in row:
406
- if column is None:
407
- column = ""
408
- elif type(column) == int or type(column) == float:
409
- column = str(column)
410
- elif type(column) == list or type(column) == dict:
411
- column = json.dumps(column)
412
- if isinstance(column, datetime.date) or isinstance(column, datetime.datetime):
413
- try:
414
- column = datetime.datetime.strftime(column, "%Y-%m-%d")
415
- except ValueError:
416
- try:
417
- column = datetime.datetime.strftime(column, "%Y-%m-%dT%H:%M:%S")
418
- except ValueError:
419
- column = datetime.datetime.strptime(column, "%Y-%m-%dT%H:%M:%S.%f")
420
- if isinstance(column, bool):
421
- if column:
422
- column = "true"
423
- else:
424
- column = "false"
425
- p_row.append(column.encode(encoding=self.charset))
426
- p_rows.append(p_row)
427
- return p_rows
428
-
429
- def send_initial_data(self):
430
- server_encoding = self.charset.encode(self.charset)
431
- client_encoding = self.user_parameters.get(b"client_encoding", server_encoding)
432
- # TODO: Send BackendKeyData Here (55.2.1)
433
- self.send(ParameterStatus(name=b"server_version", value=b"14.6"))
434
- self.send(ParameterStatus(name=b"server_encoding", value=server_encoding))
435
- self.send(ParameterStatus(name=b"client_encoding", value=client_encoding))
436
- # TODO Send Parameters to complete set on 55.2.7 Asynchronous Operations E.G. - At present there is a
437
- # hard-wired set of parameters for which ParameterStatus will be generated: they are server_version,
438
- # server_encoding, client_encoding, application_name, default_transaction_read_only, in_hot_standby,
439
- # is_superuser, session_authorization, DateStyle, IntervalStyle, TimeZone, integer_datetimes,
440
- # and standard_conforming_strings
441
- return
442
-
443
- def send_ready(self):
444
- self.logger.debug("Ready for Query")
445
- self.send(ReadyForQuery(transaction_status=self.transaction_status))
446
-
447
- def main_loop(self):
448
- self.send_ready()
449
- while True:
450
- message: PostgresMessage = self.client_buffer.read_message()
451
- if message is None: # Empty Data, Buffer done
452
- break
453
- tof = type(message)
454
- if tof in self.message_map:
455
- res = self.message_map[tof](message)
456
- if not res:
457
- break
458
- else:
459
- self.logger.warning("Ignoring unsupported message type %s" % tof)
460
-
461
- @staticmethod
462
- def startProxy():
463
- host = config["api"]["postgres"]["host"]
464
- port = int(config["api"]["postgres"]["port"])
465
- server = TcpServer((host, port), PostgresProxyHandler)
466
- server.connection_id = 0
467
- server.mindsdb_config = config
468
- server.check_auth = partial(check_auth, config=config)
469
- server.serve_forever()
470
-
471
-
472
- class TcpServer(socketserver.ThreadingTCPServer):
473
- allow_reuse_address = True
474
-
475
-
476
- if __name__ == "__main__":
477
- PostgresProxyHandler.startProxy()
@@ -1,10 +0,0 @@
1
- import re
2
- from typing import Union
3
-
4
-
5
- def strip_null_byte(x: Union[str, bytes], encoding=None):
6
- if type(x) == bytes:
7
- if encoding is None:
8
- encoding = "UTF-8"
9
- x = x.decode(encoding=encoding)
10
- return re.sub(r'[\s\x00]+$', '', x)
@@ -1,11 +0,0 @@
1
- import mindsdb.interfaces.storage.db as db
2
- from mindsdb.api.postgres.postgres_proxy.postgres_proxy import PostgresProxyHandler
3
- from mindsdb.utilities import log
4
-
5
-
6
- def start(verbose=False):
7
- logger = log.getLogger(__name__)
8
- logger.info("Postgres API is starting..")
9
- db.init()
10
-
11
- PostgresProxyHandler.startProxy()
@@ -1,169 +0,0 @@
1
- import os
2
- import pytest
3
- import pymssql
4
- import pandas as pd
5
- from mindsdb.integrations.handlers.mssql_handler.mssql_handler import SqlServerHandler
6
- from mindsdb.integrations.libs.response import (
7
- HandlerResponse as Response,
8
- RESPONSE_TYPE,
9
- )
10
-
11
- HANDLER_KWARGS = {
12
- "connection_data": {
13
- "host": os.environ.get("MDB_TEST_MSSQL_HOST", "localhost"),
14
- "port": os.environ.get("MDB_TEST_MSSQL_PORT", "1433"),
15
- "user": os.environ.get("MDB_TEST_MSSQL_USER", "sa"),
16
- "password": os.environ.get("MDB_TEST_MSSQL_PASSWORD", "admin5678@"),
17
- "database": os.environ.get("MDB_TEST_MSSQL_DATABASE", "mdb_db_handler_test"),
18
- }
19
- }
20
-
21
-
22
- @pytest.fixture(scope="class")
23
- def sql_server_handler():
24
- seed_db()
25
- handler = SqlServerHandler("test_sqlserver_handler", **HANDLER_KWARGS)
26
- yield handler
27
- handler.disconnect()
28
-
29
-
30
- def seed_db():
31
- """Seed the test DB with some data"""
32
-
33
- # Connect to 'master' while we create our test DB
34
- conn_info = HANDLER_KWARGS["connection_data"].copy()
35
- conn_info["database"] = "master"
36
- db = pymssql.connect(**conn_info, autocommit=True)
37
- cursor = db.cursor()
38
-
39
- with open("mindsdb/integrations/handlers/mssql_handler/tests/seed.sql", "r") as f:
40
- for line in f.readlines():
41
- cursor.execute(line)
42
- cursor.close()
43
- db.close()
44
-
45
-
46
- def check_valid_response(res):
47
- if res.resp_type == RESPONSE_TYPE.TABLE:
48
- assert res.data_frame is not None, "expected to have some data, but got None"
49
- assert (
50
- res.error_code == 0
51
- ), f"expected to have zero error_code, but got {res.error_code}"
52
- assert (
53
- res.error_message is None
54
- ), f"expected to have None in error message, but got {res.error_message}"
55
-
56
-
57
- def get_table_names(sql_server_handler):
58
- res = sql_server_handler.get_tables()
59
- tables = res.data_frame
60
- assert tables is not None, "expected to have some tables in the db, but got None"
61
- assert (
62
- "table_name" in tables
63
- ), f"expected to get 'table_name' column in the response:\n{tables}"
64
- return list(tables["table_name"])
65
-
66
-
67
- @pytest.mark.usefixtures("sql_server_handler")
68
- class TestMSSQLHandlerConnect:
69
- def test_connect(self, sql_server_handler):
70
- sql_server_handler.connect()
71
- assert sql_server_handler.is_connected, "the handler has failed to connect"
72
-
73
- def test_check_connection(self, sql_server_handler):
74
- res = sql_server_handler.check_connection()
75
- assert res.success, res.error_message
76
-
77
-
78
- class TestMSSQLHandlerDisconnect:
79
- def test_disconnect(self, sql_server_handler):
80
- sql_server_handler.disconnect()
81
- assert sql_server_handler.is_connected is False, "failed to disconnect"
82
-
83
- def test_check_connection(self, sql_server_handler):
84
- res = sql_server_handler.check_connection()
85
- assert res.success, res.error_message
86
-
87
-
88
- @pytest.mark.usefixtures("sql_server_handler")
89
- class TestMSSQLHandlerTables:
90
- table_for_creation = "test_mdb"
91
-
92
- def test_get_tables(self, sql_server_handler):
93
- res = sql_server_handler.get_tables()
94
- tables = res.data_frame
95
- assert (
96
- tables is not None
97
- ), "expected to have some tables in the db, but got None"
98
- assert (
99
- "table_name" in tables
100
- ), f"expected to get 'table_name' in the response but got: {tables}"
101
- assert (
102
- "test" in tables["table_name"].values
103
- ), "expected to have 'test' in the response."
104
-
105
- def test_get_columns(self, sql_server_handler):
106
- response = sql_server_handler.get_columns("test")
107
- assert response.type == RESPONSE_TYPE.TABLE, "expected a TABLE"
108
- assert len(response.data_frame) > 0, "expected > O columns"
109
-
110
- expected_columns = {
111
- "Field": ["col_one", "col_two", "col_three", "col_four"],
112
- "Type": ["int", "int", "float", "varchar"],
113
- }
114
- expected_df = pd.DataFrame(expected_columns)
115
- assert response.data_frame.equals(
116
- expected_df
117
- ), "response does not contain the expected columns"
118
-
119
- def test_create_table(self, sql_server_handler):
120
- query = f"""
121
- IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = '{self.table_for_creation}')
122
- BEGIN
123
- CREATE TABLE {self.table_for_creation} (test_col INT)
124
- END
125
- """
126
- res = sql_server_handler.native_query(query)
127
- check_valid_response(res)
128
- tables = get_table_names(sql_server_handler)
129
- assert (
130
- self.table_for_creation in tables
131
- ), f"expected to have {self.table_for_creation} in database, but got: {tables}"
132
-
133
- def test_drop_table(self, sql_server_handler):
134
- query = f"DROP TABLE IF EXISTS {self.table_for_creation}"
135
- res = sql_server_handler.native_query(query)
136
- check_valid_response(res)
137
- tables = get_table_names(sql_server_handler)
138
- assert self.table_for_creation not in tables
139
-
140
-
141
- @pytest.mark.usefixtures("sql_server_handler")
142
- class TestMSSQLHandlerQuery:
143
- def test_select_native_query(self, sql_server_handler):
144
- query = "SELECT * FROM test"
145
- response = sql_server_handler.native_query(query)
146
- assert type(response) is Response
147
- assert response.resp_type == RESPONSE_TYPE.TABLE
148
-
149
- expected_data = {
150
- "col_one": [1, 2, 3],
151
- "col_two": [-1, -2, -3],
152
- "col_three": [0.1, 0.2, 0.3],
153
- "col_four": ["A", "B", "C"],
154
- }
155
- expected_df = pd.DataFrame(expected_data)
156
- assert response.data_frame.equals(
157
- expected_df
158
- ), "response does not contain the expected data"
159
-
160
- def test_select_query(self, sql_server_handler):
161
- limit = 3
162
- query = "SELECT * FROM test"
163
- res = sql_server_handler.query(query)
164
- check_valid_response(res)
165
- got_rows = res.data_frame.shape[0]
166
- want_rows = limit
167
- assert (
168
- got_rows == want_rows
169
- ), f"expected to have {want_rows} rows in response but got: {got_rows}"