MindsDB 25.9.3rc1__py3-none-any.whl → 25.10.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +1 -9
- mindsdb/api/a2a/__init__.py +1 -1
- mindsdb/api/a2a/agent.py +9 -1
- mindsdb/api/a2a/common/server/server.py +4 -0
- mindsdb/api/a2a/common/server/task_manager.py +8 -1
- mindsdb/api/a2a/common/types.py +66 -0
- mindsdb/api/a2a/task_manager.py +50 -0
- mindsdb/api/common/middleware.py +1 -1
- mindsdb/api/executor/command_executor.py +49 -36
- mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py +7 -13
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +2 -2
- mindsdb/api/executor/datahub/datanodes/system_tables.py +2 -1
- mindsdb/api/executor/planner/query_prepare.py +2 -20
- mindsdb/api/executor/utilities/sql.py +5 -4
- mindsdb/api/http/initialize.py +76 -60
- mindsdb/api/http/namespaces/agents.py +0 -3
- mindsdb/api/http/namespaces/chatbots.py +0 -5
- mindsdb/api/http/namespaces/file.py +2 -0
- mindsdb/api/http/namespaces/handlers.py +10 -5
- mindsdb/api/http/namespaces/knowledge_bases.py +20 -0
- mindsdb/api/http/namespaces/sql.py +2 -2
- mindsdb/api/http/start.py +2 -2
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +8 -2
- mindsdb/integrations/handlers/byom_handler/byom_handler.py +2 -10
- mindsdb/integrations/handlers/databricks_handler/databricks_handler.py +98 -46
- mindsdb/integrations/handlers/druid_handler/druid_handler.py +32 -40
- mindsdb/integrations/handlers/gitlab_handler/gitlab_handler.py +5 -2
- mindsdb/integrations/handlers/mssql_handler/mssql_handler.py +438 -100
- mindsdb/integrations/handlers/mssql_handler/requirements_odbc.txt +3 -0
- mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +235 -3
- mindsdb/integrations/handlers/oracle_handler/__init__.py +2 -0
- mindsdb/integrations/handlers/oracle_handler/connection_args.py +7 -1
- mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +321 -16
- mindsdb/integrations/handlers/oracle_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +2 -2
- mindsdb/integrations/handlers/zendesk_handler/zendesk_tables.py +144 -111
- mindsdb/integrations/libs/response.py +2 -2
- mindsdb/integrations/utilities/handlers/auth_utilities/snowflake/__init__.py +1 -0
- mindsdb/integrations/utilities/handlers/auth_utilities/snowflake/snowflake_jwt_gen.py +151 -0
- mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +24 -21
- mindsdb/interfaces/agents/agents_controller.py +0 -2
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +6 -7
- mindsdb/interfaces/data_catalog/data_catalog_reader.py +15 -4
- mindsdb/interfaces/database/data_handlers_cache.py +190 -0
- mindsdb/interfaces/database/database.py +3 -3
- mindsdb/interfaces/database/integrations.py +1 -121
- mindsdb/interfaces/database/projects.py +2 -6
- mindsdb/interfaces/database/views.py +1 -4
- mindsdb/interfaces/jobs/jobs_controller.py +0 -4
- mindsdb/interfaces/jobs/scheduler.py +0 -1
- mindsdb/interfaces/knowledge_base/controller.py +197 -108
- mindsdb/interfaces/knowledge_base/evaluate.py +36 -41
- mindsdb/interfaces/knowledge_base/executor.py +11 -0
- mindsdb/interfaces/knowledge_base/llm_client.py +51 -17
- mindsdb/interfaces/model/model_controller.py +4 -4
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_sql_toolkit.py +4 -10
- mindsdb/interfaces/skills/skills_controller.py +1 -4
- mindsdb/interfaces/storage/db.py +16 -6
- mindsdb/interfaces/triggers/triggers_controller.py +1 -3
- mindsdb/utilities/config.py +19 -2
- mindsdb/utilities/exception.py +2 -2
- mindsdb/utilities/json_encoder.py +24 -10
- mindsdb/utilities/render/sqlalchemy_render.py +15 -14
- mindsdb/utilities/starters.py +0 -10
- {mindsdb-25.9.3rc1.dist-info → mindsdb-25.10.0rc1.dist-info}/METADATA +276 -264
- {mindsdb-25.9.3rc1.dist-info → mindsdb-25.10.0rc1.dist-info}/RECORD +70 -84
- mindsdb/api/postgres/__init__.py +0 -0
- mindsdb/api/postgres/postgres_proxy/__init__.py +0 -0
- mindsdb/api/postgres/postgres_proxy/executor/__init__.py +0 -1
- mindsdb/api/postgres/postgres_proxy/executor/executor.py +0 -182
- mindsdb/api/postgres/postgres_proxy/postgres_packets/__init__.py +0 -0
- mindsdb/api/postgres/postgres_proxy/postgres_packets/errors.py +0 -322
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_fields.py +0 -34
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message.py +0 -31
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_formats.py +0 -1265
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_identifiers.py +0 -31
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_packets.py +0 -265
- mindsdb/api/postgres/postgres_proxy/postgres_proxy.py +0 -477
- mindsdb/api/postgres/postgres_proxy/utilities/__init__.py +0 -10
- mindsdb/api/postgres/start.py +0 -11
- mindsdb/integrations/handlers/mssql_handler/tests/__init__.py +0 -0
- mindsdb/integrations/handlers/mssql_handler/tests/test_mssql_handler.py +0 -169
- mindsdb/integrations/handlers/oracle_handler/tests/__init__.py +0 -0
- mindsdb/integrations/handlers/oracle_handler/tests/test_oracle_handler.py +0 -32
- {mindsdb-25.9.3rc1.dist-info → mindsdb-25.10.0rc1.dist-info}/WHEEL +0 -0
- {mindsdb-25.9.3rc1.dist-info → mindsdb-25.10.0rc1.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.9.3rc1.dist-info → mindsdb-25.10.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,477 +0,0 @@
|
|
|
1
|
-
import base64
|
|
2
|
-
import datetime
|
|
3
|
-
import os
|
|
4
|
-
import json
|
|
5
|
-
import select
|
|
6
|
-
import socketserver
|
|
7
|
-
import struct
|
|
8
|
-
import sys
|
|
9
|
-
from functools import partial
|
|
10
|
-
import socket
|
|
11
|
-
from typing import Callable, Dict, Type, Any, Iterable, Sequence
|
|
12
|
-
|
|
13
|
-
from mindsdb.api.executor.controllers import SessionController
|
|
14
|
-
from mindsdb.api.postgres.postgres_proxy.executor import Executor
|
|
15
|
-
from mindsdb.api.mysql.mysql_proxy.libs.constants.mysql import CHARSET_NUMBERS
|
|
16
|
-
from mindsdb.api.executor.data_types.response_type import RESPONSE_TYPE
|
|
17
|
-
from mindsdb.api.common.middleware import check_auth
|
|
18
|
-
from mindsdb.api.mysql.mysql_proxy.mysql_proxy import SQLAnswer
|
|
19
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.errors import POSTGRES_SYNTAX_ERROR_CODE
|
|
20
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import GenericField, PostgresField
|
|
21
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_formats import (
|
|
22
|
-
Terminate,
|
|
23
|
-
Query,
|
|
24
|
-
AuthenticationClearTextPassword,
|
|
25
|
-
AuthenticationOk,
|
|
26
|
-
RowDescriptions,
|
|
27
|
-
DataRow,
|
|
28
|
-
CommandComplete,
|
|
29
|
-
ReadyForQuery,
|
|
30
|
-
ConnectionFailure,
|
|
31
|
-
ParameterStatus,
|
|
32
|
-
Error,
|
|
33
|
-
Execute,
|
|
34
|
-
Bind,
|
|
35
|
-
Parse,
|
|
36
|
-
Sync,
|
|
37
|
-
ParseComplete,
|
|
38
|
-
InvalidSQLStatementName,
|
|
39
|
-
BindComplete,
|
|
40
|
-
Describe,
|
|
41
|
-
DataException,
|
|
42
|
-
ParameterDescription,
|
|
43
|
-
)
|
|
44
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message import PostgresMessage
|
|
45
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_packets import (
|
|
46
|
-
PostgresPacketReader,
|
|
47
|
-
PostgresPacketBuilder,
|
|
48
|
-
)
|
|
49
|
-
from mindsdb.api.postgres.postgres_proxy.utilities import strip_null_byte
|
|
50
|
-
from mindsdb.utilities.config import config
|
|
51
|
-
from mindsdb.utilities.context import context as ctx
|
|
52
|
-
from mindsdb.utilities import log
|
|
53
|
-
from mindsdb.api.mysql.mysql_proxy.external_libs.mysql_scramble import scramble as scramble_func
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
class PostgresProxyHandler(socketserver.StreamRequestHandler):
|
|
57
|
-
client_buffer: PostgresPacketReader
|
|
58
|
-
user_parameters: Dict[bytes, bytes]
|
|
59
|
-
|
|
60
|
-
def __init__(self, request, client_address, server):
|
|
61
|
-
self.logger = log.getLogger(__name__)
|
|
62
|
-
self.charset = "utf8"
|
|
63
|
-
self.charset_text_type = CHARSET_NUMBERS["utf8_general_ci"]
|
|
64
|
-
self.session = None
|
|
65
|
-
self.client_capabilities = None
|
|
66
|
-
self.user_parameters = None
|
|
67
|
-
self.named_statements = {}
|
|
68
|
-
self.unnamed_statement = None
|
|
69
|
-
self.is_cloud = False
|
|
70
|
-
self.named_portals = {}
|
|
71
|
-
self.unnamed_portal = None
|
|
72
|
-
self.transaction_status = b"I" # I: Idle, T: Transaction Block, E: Failed Transaction Block
|
|
73
|
-
super().__init__(request, client_address, server)
|
|
74
|
-
|
|
75
|
-
def handle(self) -> None:
|
|
76
|
-
ctx.set_default()
|
|
77
|
-
self.init_session()
|
|
78
|
-
self.logger.debug("handle new incoming connection")
|
|
79
|
-
cloud_connection = self.is_cloud_connection()
|
|
80
|
-
if cloud_connection["is_cloud"]:
|
|
81
|
-
ctx.company_id = cloud_connection.get("company_id")
|
|
82
|
-
self.is_cloud = True
|
|
83
|
-
|
|
84
|
-
self.message_map: Dict[Type[PostgresMessage], Callable[[Any], bool]] = {
|
|
85
|
-
Terminate: self.terminate,
|
|
86
|
-
Query: self.query,
|
|
87
|
-
Parse: self.parse,
|
|
88
|
-
Bind: self.bind,
|
|
89
|
-
Execute: self.execute,
|
|
90
|
-
Describe: self.describe,
|
|
91
|
-
Sync: self.sync,
|
|
92
|
-
}
|
|
93
|
-
self.client_buffer = PostgresPacketReader(self.rfile)
|
|
94
|
-
if self.is_cloud:
|
|
95
|
-
# We already have a connection started through the gateway.
|
|
96
|
-
started = True
|
|
97
|
-
self.handshake()
|
|
98
|
-
else:
|
|
99
|
-
started = self.start_connection()
|
|
100
|
-
if started:
|
|
101
|
-
self.logger.debug("connection started")
|
|
102
|
-
self.send_initial_data()
|
|
103
|
-
self.main_loop()
|
|
104
|
-
|
|
105
|
-
def is_cloud_connection(self):
|
|
106
|
-
"""Determine source of connection. Must be call before handshake.
|
|
107
|
-
Idea based on: real mysql connection does not send anything before server handshake, so
|
|
108
|
-
socket should be in 'out' state. In opposite, clout connection sends '0000' right after
|
|
109
|
-
connection. '0000' selected because in real mysql connection it should be length of package,
|
|
110
|
-
and it can not be 0.
|
|
111
|
-
|
|
112
|
-
Copied from mysql_proxy
|
|
113
|
-
TODO: Extract method into common
|
|
114
|
-
"""
|
|
115
|
-
is_cloud = config.get("cloud", False)
|
|
116
|
-
|
|
117
|
-
if sys.platform != "linux" or is_cloud is False:
|
|
118
|
-
return {"is_cloud": False}
|
|
119
|
-
|
|
120
|
-
read_poller = select.poll()
|
|
121
|
-
read_poller.register(self.request, select.POLLIN)
|
|
122
|
-
events = read_poller.poll(30)
|
|
123
|
-
|
|
124
|
-
if len(events) == 0:
|
|
125
|
-
return {"is_cloud": False}
|
|
126
|
-
|
|
127
|
-
first_byte = self.request.recv(4, socket.MSG_PEEK)
|
|
128
|
-
if first_byte == b"\x00\x00\x00\x00":
|
|
129
|
-
self.request.recv(4)
|
|
130
|
-
client_capabilities = self.request.recv(8)
|
|
131
|
-
client_capabilities = struct.unpack("L", client_capabilities)[0]
|
|
132
|
-
|
|
133
|
-
company_id = self.request.recv(4)
|
|
134
|
-
company_id = struct.unpack("I", company_id)[0]
|
|
135
|
-
|
|
136
|
-
user_class = self.request.recv(1)
|
|
137
|
-
user_class = struct.unpack("B", user_class)[0]
|
|
138
|
-
|
|
139
|
-
database_name_len = self.request.recv(2)
|
|
140
|
-
database_name_len = struct.unpack("H", database_name_len)[0]
|
|
141
|
-
|
|
142
|
-
database_name = ""
|
|
143
|
-
if database_name_len > 0:
|
|
144
|
-
database_name = self.request.recv(database_name_len).decode()
|
|
145
|
-
|
|
146
|
-
return {
|
|
147
|
-
"is_cloud": True,
|
|
148
|
-
"client_capabilities": client_capabilities,
|
|
149
|
-
"company_id": company_id,
|
|
150
|
-
"user_class": user_class,
|
|
151
|
-
"database": database_name,
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
return {"is_cloud": False}
|
|
155
|
-
|
|
156
|
-
def parse(self, message: Parse):
|
|
157
|
-
self.logger.info("Postgres_Proxy: Parsing")
|
|
158
|
-
executor = Executor(session=self.session, proxy_server=self, charset=self.charset)
|
|
159
|
-
# TODO: Remove comment if unneeded ot use session since we're storing in this proxy class per session anyway
|
|
160
|
-
# stmt_id = self.session.register_stmt(executor)
|
|
161
|
-
executor.stmt_prepare(sql=message.query)
|
|
162
|
-
statement = {"executor": executor, "parse": message}
|
|
163
|
-
if message.name:
|
|
164
|
-
self.named_statements[message.name] = statement
|
|
165
|
-
else:
|
|
166
|
-
self.unnamed_statement = statement
|
|
167
|
-
self.send(ParseComplete())
|
|
168
|
-
return True
|
|
169
|
-
|
|
170
|
-
def bind(self, message: Bind):
|
|
171
|
-
self.logger.info(f"Postgres_Proxy: Binding {message.name} with params {message.parameters}")
|
|
172
|
-
if message.statement_name:
|
|
173
|
-
statement = self.named_statements[message.statement_name]
|
|
174
|
-
elif self.unnamed_statement:
|
|
175
|
-
statement = self.unnamed_statement
|
|
176
|
-
else:
|
|
177
|
-
self.send(InvalidSQLStatementName())
|
|
178
|
-
return True
|
|
179
|
-
|
|
180
|
-
# TODO Should check validity of statement here and not at parse stage
|
|
181
|
-
portal = statement.copy()
|
|
182
|
-
portal["bind"] = message
|
|
183
|
-
if message.name:
|
|
184
|
-
self.named_portals[message.name] = portal
|
|
185
|
-
else:
|
|
186
|
-
self.unnamed_portal = portal
|
|
187
|
-
self.send(BindComplete())
|
|
188
|
-
return True
|
|
189
|
-
|
|
190
|
-
def describe(self, message: Describe):
|
|
191
|
-
self.logger.info("Postgres_Proxy: Describing")
|
|
192
|
-
if message.describe_type == b"P":
|
|
193
|
-
if message.name:
|
|
194
|
-
describing = self.named_portals[message.name]
|
|
195
|
-
elif self.unnamed_statement:
|
|
196
|
-
describing = self.unnamed_portal
|
|
197
|
-
else:
|
|
198
|
-
self.send(InvalidSQLStatementName("Portal Does not Exist"))
|
|
199
|
-
return True
|
|
200
|
-
elif message.describe_type == b"S":
|
|
201
|
-
if message.name:
|
|
202
|
-
describing = self.named_statements[message.name]
|
|
203
|
-
elif self.unnamed_statement:
|
|
204
|
-
describing = self.unnamed_statement
|
|
205
|
-
else:
|
|
206
|
-
self.send(InvalidSQLStatementName())
|
|
207
|
-
return True
|
|
208
|
-
self.send(ParameterDescription(parameters=describing["parse"]["parameters"]))
|
|
209
|
-
else:
|
|
210
|
-
self.send(DataException(message="Describe did not have correct type. Can be 'P' or 'S'"))
|
|
211
|
-
return True
|
|
212
|
-
|
|
213
|
-
fields = self.to_postgres_fields(describing["executor"].columns)
|
|
214
|
-
self.send(RowDescriptions(fields=fields))
|
|
215
|
-
return True
|
|
216
|
-
|
|
217
|
-
def execute(self, message: Execute):
|
|
218
|
-
self.logger.info("Postgres_Proxy: Executing")
|
|
219
|
-
if message.name:
|
|
220
|
-
portal = self.named_portals[message.name]
|
|
221
|
-
elif self.unnamed_portal:
|
|
222
|
-
portal = self.unnamed_portal
|
|
223
|
-
else:
|
|
224
|
-
self.send(InvalidSQLStatementName("Portal does not exist"))
|
|
225
|
-
|
|
226
|
-
executor = portal["executor"]
|
|
227
|
-
params = portal["bind"].parameters
|
|
228
|
-
executor.stmt_execute(param_values=params)
|
|
229
|
-
sql_answer = self.return_executor_data(executor)
|
|
230
|
-
self.respond_from_sql_answer(sql=executor.sql, sql_answer=sql_answer, row_descs=False)
|
|
231
|
-
return True
|
|
232
|
-
|
|
233
|
-
def sync(self, message: Sync):
|
|
234
|
-
self.logger.info("Postgres_Proxy: Syncing")
|
|
235
|
-
# TODO: Close/commit transaction if outside of a block. Maybe no collaries since Proxy
|
|
236
|
-
self.send_ready()
|
|
237
|
-
return True
|
|
238
|
-
|
|
239
|
-
def init_session(self):
|
|
240
|
-
self.logger.info("New connection [{ip}:{port}]".format(ip=self.client_address[0], port=self.client_address[1]))
|
|
241
|
-
self.logger.debug(self.__dict__)
|
|
242
|
-
|
|
243
|
-
if self.server.connection_id >= 65025:
|
|
244
|
-
self.server.connection_id = 0
|
|
245
|
-
self.server.connection_id += 1
|
|
246
|
-
self.connection_id = self.server.connection_id
|
|
247
|
-
self.session = SessionController()
|
|
248
|
-
self.session.database = config.get("default_project")
|
|
249
|
-
|
|
250
|
-
if hasattr(self.server, "salt") and isinstance(self.server.salt, str):
|
|
251
|
-
self.salt = self.server.salt
|
|
252
|
-
else:
|
|
253
|
-
self.salt = base64.b64encode(os.urandom(15)).decode()
|
|
254
|
-
|
|
255
|
-
self.socket = self.request
|
|
256
|
-
|
|
257
|
-
self.current_transaction = None
|
|
258
|
-
|
|
259
|
-
self.logger.debug("session salt: {salt}".format(salt=self.salt))
|
|
260
|
-
|
|
261
|
-
def process_query(self, sql):
|
|
262
|
-
executor = Executor(session=self.session, proxy_server=self, charset=self.charset)
|
|
263
|
-
self.logger.debug("processing query\n%s", sql)
|
|
264
|
-
try:
|
|
265
|
-
executor.query_execute(sql)
|
|
266
|
-
except Exception as e:
|
|
267
|
-
return SQLAnswer(
|
|
268
|
-
resp_type=RESPONSE_TYPE.ERROR,
|
|
269
|
-
error_message=str(e).encode(self.get_encoding()),
|
|
270
|
-
error_code=POSTGRES_SYNTAX_ERROR_CODE.encode(self.get_encoding()),
|
|
271
|
-
)
|
|
272
|
-
return self.return_executor_data(executor)
|
|
273
|
-
|
|
274
|
-
def return_executor_data(self, executor):
|
|
275
|
-
if executor.data is None:
|
|
276
|
-
resp = SQLAnswer(
|
|
277
|
-
resp_type=RESPONSE_TYPE.OK,
|
|
278
|
-
state_track=executor.state_track,
|
|
279
|
-
)
|
|
280
|
-
else:
|
|
281
|
-
resp = SQLAnswer(
|
|
282
|
-
resp_type=RESPONSE_TYPE.TABLE,
|
|
283
|
-
state_track=executor.state_track,
|
|
284
|
-
result_set=executor.data,
|
|
285
|
-
status=executor.server_status,
|
|
286
|
-
)
|
|
287
|
-
return resp
|
|
288
|
-
|
|
289
|
-
def start_connection(self):
|
|
290
|
-
self.logger.debug("starting handshake")
|
|
291
|
-
self.handshake()
|
|
292
|
-
self.logger.debug("handshake complete, checking authentication")
|
|
293
|
-
return self.authenticate()
|
|
294
|
-
|
|
295
|
-
def send(self, message: PostgresMessage):
|
|
296
|
-
self.logger.debug("Sending message of type %s" % message.__class__.__name__)
|
|
297
|
-
message.send(self.wfile)
|
|
298
|
-
|
|
299
|
-
def handshake(self):
|
|
300
|
-
self.client_buffer.read_verify_ssl_request()
|
|
301
|
-
# self.send(NoticeResponse()) -- Should Probably not send. Looks in protocol manual to be sent for warning
|
|
302
|
-
self.logger.debug("Sending No to SSL Request")
|
|
303
|
-
PostgresPacketBuilder().write_char(b"N", self.wfile)
|
|
304
|
-
self.user_parameters = self.client_buffer.read_startup_message()
|
|
305
|
-
|
|
306
|
-
def authenticate(self, ask_for_password=False):
|
|
307
|
-
if ask_for_password:
|
|
308
|
-
self.send(AuthenticationClearTextPassword())
|
|
309
|
-
password = self.client_buffer.read_authentication(encoding=self.charset)
|
|
310
|
-
else:
|
|
311
|
-
password = ""
|
|
312
|
-
username = self.user_parameters[b"user"].decode(encoding=self.charset)
|
|
313
|
-
auth_data = self.server.check_auth(username, password, scramble_func, self.salt, ctx.company_id)
|
|
314
|
-
if auth_data["success"]:
|
|
315
|
-
self.logger.debug("Authentication succeeded")
|
|
316
|
-
self.session.username = auth_data["username"]
|
|
317
|
-
self.session.auth = True
|
|
318
|
-
self.send(AuthenticationOk())
|
|
319
|
-
return True
|
|
320
|
-
else:
|
|
321
|
-
if not ask_for_password: # try asking for password
|
|
322
|
-
return self.authenticate(ask_for_password=True)
|
|
323
|
-
self.logger.debug("Authentication failed")
|
|
324
|
-
self.send(ConnectionFailure(message="Authentication failed."))
|
|
325
|
-
return False
|
|
326
|
-
|
|
327
|
-
def terminate(self, message: Terminate) -> bool:
|
|
328
|
-
self.logger.info("Postgres_Proxy: Terminating")
|
|
329
|
-
return False
|
|
330
|
-
|
|
331
|
-
def get_encoding(self) -> str:
|
|
332
|
-
return self.user_parameters.get(b"user_encoding", None) or self.charset
|
|
333
|
-
|
|
334
|
-
def return_ok(self, sql, rows: int = 0):
|
|
335
|
-
command = self.get_command(sql)
|
|
336
|
-
if command == "BEGIN":
|
|
337
|
-
self.transaction_status = b"T"
|
|
338
|
-
if command == "COMMIT":
|
|
339
|
-
self.transaction_status = b"I"
|
|
340
|
-
if command == "CREATE":
|
|
341
|
-
command = "SELECT"
|
|
342
|
-
if command in ("INSERT", "DELETE", "UPDATE", "SELECT", "MOVE", "FETCH", "COPY"):
|
|
343
|
-
command = f"{command} {rows}"
|
|
344
|
-
self.send(CommandComplete(tag=command.encode(encoding=self.get_encoding())))
|
|
345
|
-
return True
|
|
346
|
-
|
|
347
|
-
def get_command(self, sql):
|
|
348
|
-
sql = self.stringify_sql(sql)
|
|
349
|
-
return sql.split(" ", 1)[0]
|
|
350
|
-
|
|
351
|
-
def stringify_sql(self, sql):
|
|
352
|
-
if type(sql) == bytes:
|
|
353
|
-
encoding = self.get_encoding()
|
|
354
|
-
sql: str = sql.decode(encoding)
|
|
355
|
-
return strip_null_byte(sql).strip(";")
|
|
356
|
-
|
|
357
|
-
def return_table(self, sql_answer: SQLAnswer, row_descs=True):
|
|
358
|
-
fields = self.to_postgres_fields(sql_answer.result_set.columns)
|
|
359
|
-
rows = self.to_postgres_rows(sql_answer.result_set)
|
|
360
|
-
if row_descs:
|
|
361
|
-
self.send(RowDescriptions(fields=fields))
|
|
362
|
-
self.send(DataRow(rows=rows))
|
|
363
|
-
encoding = self.get_encoding()
|
|
364
|
-
tag = ("SELECT %s" % str(len(rows))).encode(encoding)
|
|
365
|
-
self.send(CommandComplete(tag=tag))
|
|
366
|
-
return True
|
|
367
|
-
|
|
368
|
-
def return_error(self, sql_answer: SQLAnswer):
|
|
369
|
-
self.send(Error.from_answer(error_code=sql_answer.error_code, error_message=sql_answer.error_message))
|
|
370
|
-
return True
|
|
371
|
-
|
|
372
|
-
def query(self, message: Query) -> bool:
|
|
373
|
-
self.logger.debug("Postgres Proxy: Got query of:\n%s" % message.sql)
|
|
374
|
-
sql = message.get_parsed_sql()
|
|
375
|
-
sql_answer = self.process_query(sql)
|
|
376
|
-
self.respond_from_sql_answer(sql=sql, sql_answer=sql_answer)
|
|
377
|
-
self.send_ready()
|
|
378
|
-
return True
|
|
379
|
-
|
|
380
|
-
def respond_from_sql_answer(self, sql, sql_answer: SQLAnswer, row_descs=True) -> bool:
|
|
381
|
-
# TODO Add command complete passthrough for Complex Queries that exceed row limit in one go
|
|
382
|
-
rows = 0
|
|
383
|
-
if sql_answer.result_set:
|
|
384
|
-
rows = len(sql_answer.result_set)
|
|
385
|
-
if RESPONSE_TYPE.OK == sql_answer.type:
|
|
386
|
-
return self.return_ok(sql, rows=rows)
|
|
387
|
-
elif RESPONSE_TYPE.TABLE == sql_answer.type:
|
|
388
|
-
return self.return_table(sql_answer, row_descs=row_descs)
|
|
389
|
-
elif RESPONSE_TYPE.ERROR == sql_answer.type:
|
|
390
|
-
return self.return_error(sql_answer)
|
|
391
|
-
|
|
392
|
-
@staticmethod
|
|
393
|
-
def to_postgres_fields(columns: Iterable[Dict[str, Any]]) -> Sequence[PostgresField]:
|
|
394
|
-
fields = []
|
|
395
|
-
i = 0
|
|
396
|
-
for column in columns:
|
|
397
|
-
fields.append(GenericField(name=column["name"], object_id=column["type"].value, column_id=i))
|
|
398
|
-
i += 1
|
|
399
|
-
return fields
|
|
400
|
-
|
|
401
|
-
def to_postgres_rows(self, rows: Iterable[Iterable[Any]]) -> Sequence[Sequence[bytes]]:
|
|
402
|
-
p_rows = []
|
|
403
|
-
for row in rows:
|
|
404
|
-
p_row = []
|
|
405
|
-
for column in row:
|
|
406
|
-
if column is None:
|
|
407
|
-
column = ""
|
|
408
|
-
elif type(column) == int or type(column) == float:
|
|
409
|
-
column = str(column)
|
|
410
|
-
elif type(column) == list or type(column) == dict:
|
|
411
|
-
column = json.dumps(column)
|
|
412
|
-
if isinstance(column, datetime.date) or isinstance(column, datetime.datetime):
|
|
413
|
-
try:
|
|
414
|
-
column = datetime.datetime.strftime(column, "%Y-%m-%d")
|
|
415
|
-
except ValueError:
|
|
416
|
-
try:
|
|
417
|
-
column = datetime.datetime.strftime(column, "%Y-%m-%dT%H:%M:%S")
|
|
418
|
-
except ValueError:
|
|
419
|
-
column = datetime.datetime.strptime(column, "%Y-%m-%dT%H:%M:%S.%f")
|
|
420
|
-
if isinstance(column, bool):
|
|
421
|
-
if column:
|
|
422
|
-
column = "true"
|
|
423
|
-
else:
|
|
424
|
-
column = "false"
|
|
425
|
-
p_row.append(column.encode(encoding=self.charset))
|
|
426
|
-
p_rows.append(p_row)
|
|
427
|
-
return p_rows
|
|
428
|
-
|
|
429
|
-
def send_initial_data(self):
|
|
430
|
-
server_encoding = self.charset.encode(self.charset)
|
|
431
|
-
client_encoding = self.user_parameters.get(b"client_encoding", server_encoding)
|
|
432
|
-
# TODO: Send BackendKeyData Here (55.2.1)
|
|
433
|
-
self.send(ParameterStatus(name=b"server_version", value=b"14.6"))
|
|
434
|
-
self.send(ParameterStatus(name=b"server_encoding", value=server_encoding))
|
|
435
|
-
self.send(ParameterStatus(name=b"client_encoding", value=client_encoding))
|
|
436
|
-
# TODO Send Parameters to complete set on 55.2.7 Asynchronous Operations E.G. - At present there is a
|
|
437
|
-
# hard-wired set of parameters for which ParameterStatus will be generated: they are server_version,
|
|
438
|
-
# server_encoding, client_encoding, application_name, default_transaction_read_only, in_hot_standby,
|
|
439
|
-
# is_superuser, session_authorization, DateStyle, IntervalStyle, TimeZone, integer_datetimes,
|
|
440
|
-
# and standard_conforming_strings
|
|
441
|
-
return
|
|
442
|
-
|
|
443
|
-
def send_ready(self):
|
|
444
|
-
self.logger.debug("Ready for Query")
|
|
445
|
-
self.send(ReadyForQuery(transaction_status=self.transaction_status))
|
|
446
|
-
|
|
447
|
-
def main_loop(self):
|
|
448
|
-
self.send_ready()
|
|
449
|
-
while True:
|
|
450
|
-
message: PostgresMessage = self.client_buffer.read_message()
|
|
451
|
-
if message is None: # Empty Data, Buffer done
|
|
452
|
-
break
|
|
453
|
-
tof = type(message)
|
|
454
|
-
if tof in self.message_map:
|
|
455
|
-
res = self.message_map[tof](message)
|
|
456
|
-
if not res:
|
|
457
|
-
break
|
|
458
|
-
else:
|
|
459
|
-
self.logger.warning("Ignoring unsupported message type %s" % tof)
|
|
460
|
-
|
|
461
|
-
@staticmethod
|
|
462
|
-
def startProxy():
|
|
463
|
-
host = config["api"]["postgres"]["host"]
|
|
464
|
-
port = int(config["api"]["postgres"]["port"])
|
|
465
|
-
server = TcpServer((host, port), PostgresProxyHandler)
|
|
466
|
-
server.connection_id = 0
|
|
467
|
-
server.mindsdb_config = config
|
|
468
|
-
server.check_auth = partial(check_auth, config=config)
|
|
469
|
-
server.serve_forever()
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
class TcpServer(socketserver.ThreadingTCPServer):
|
|
473
|
-
allow_reuse_address = True
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
if __name__ == "__main__":
|
|
477
|
-
PostgresProxyHandler.startProxy()
|
mindsdb/api/postgres/start.py
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import mindsdb.interfaces.storage.db as db
|
|
2
|
-
from mindsdb.api.postgres.postgres_proxy.postgres_proxy import PostgresProxyHandler
|
|
3
|
-
from mindsdb.utilities import log
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def start(verbose=False):
|
|
7
|
-
logger = log.getLogger(__name__)
|
|
8
|
-
logger.info("Postgres API is starting..")
|
|
9
|
-
db.init()
|
|
10
|
-
|
|
11
|
-
PostgresProxyHandler.startProxy()
|
|
File without changes
|
|
@@ -1,169 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import pytest
|
|
3
|
-
import pymssql
|
|
4
|
-
import pandas as pd
|
|
5
|
-
from mindsdb.integrations.handlers.mssql_handler.mssql_handler import SqlServerHandler
|
|
6
|
-
from mindsdb.integrations.libs.response import (
|
|
7
|
-
HandlerResponse as Response,
|
|
8
|
-
RESPONSE_TYPE,
|
|
9
|
-
)
|
|
10
|
-
|
|
11
|
-
HANDLER_KWARGS = {
|
|
12
|
-
"connection_data": {
|
|
13
|
-
"host": os.environ.get("MDB_TEST_MSSQL_HOST", "localhost"),
|
|
14
|
-
"port": os.environ.get("MDB_TEST_MSSQL_PORT", "1433"),
|
|
15
|
-
"user": os.environ.get("MDB_TEST_MSSQL_USER", "sa"),
|
|
16
|
-
"password": os.environ.get("MDB_TEST_MSSQL_PASSWORD", "admin5678@"),
|
|
17
|
-
"database": os.environ.get("MDB_TEST_MSSQL_DATABASE", "mdb_db_handler_test"),
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@pytest.fixture(scope="class")
|
|
23
|
-
def sql_server_handler():
|
|
24
|
-
seed_db()
|
|
25
|
-
handler = SqlServerHandler("test_sqlserver_handler", **HANDLER_KWARGS)
|
|
26
|
-
yield handler
|
|
27
|
-
handler.disconnect()
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def seed_db():
|
|
31
|
-
"""Seed the test DB with some data"""
|
|
32
|
-
|
|
33
|
-
# Connect to 'master' while we create our test DB
|
|
34
|
-
conn_info = HANDLER_KWARGS["connection_data"].copy()
|
|
35
|
-
conn_info["database"] = "master"
|
|
36
|
-
db = pymssql.connect(**conn_info, autocommit=True)
|
|
37
|
-
cursor = db.cursor()
|
|
38
|
-
|
|
39
|
-
with open("mindsdb/integrations/handlers/mssql_handler/tests/seed.sql", "r") as f:
|
|
40
|
-
for line in f.readlines():
|
|
41
|
-
cursor.execute(line)
|
|
42
|
-
cursor.close()
|
|
43
|
-
db.close()
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def check_valid_response(res):
|
|
47
|
-
if res.resp_type == RESPONSE_TYPE.TABLE:
|
|
48
|
-
assert res.data_frame is not None, "expected to have some data, but got None"
|
|
49
|
-
assert (
|
|
50
|
-
res.error_code == 0
|
|
51
|
-
), f"expected to have zero error_code, but got {res.error_code}"
|
|
52
|
-
assert (
|
|
53
|
-
res.error_message is None
|
|
54
|
-
), f"expected to have None in error message, but got {res.error_message}"
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def get_table_names(sql_server_handler):
|
|
58
|
-
res = sql_server_handler.get_tables()
|
|
59
|
-
tables = res.data_frame
|
|
60
|
-
assert tables is not None, "expected to have some tables in the db, but got None"
|
|
61
|
-
assert (
|
|
62
|
-
"table_name" in tables
|
|
63
|
-
), f"expected to get 'table_name' column in the response:\n{tables}"
|
|
64
|
-
return list(tables["table_name"])
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
@pytest.mark.usefixtures("sql_server_handler")
|
|
68
|
-
class TestMSSQLHandlerConnect:
|
|
69
|
-
def test_connect(self, sql_server_handler):
|
|
70
|
-
sql_server_handler.connect()
|
|
71
|
-
assert sql_server_handler.is_connected, "the handler has failed to connect"
|
|
72
|
-
|
|
73
|
-
def test_check_connection(self, sql_server_handler):
|
|
74
|
-
res = sql_server_handler.check_connection()
|
|
75
|
-
assert res.success, res.error_message
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
class TestMSSQLHandlerDisconnect:
|
|
79
|
-
def test_disconnect(self, sql_server_handler):
|
|
80
|
-
sql_server_handler.disconnect()
|
|
81
|
-
assert sql_server_handler.is_connected is False, "failed to disconnect"
|
|
82
|
-
|
|
83
|
-
def test_check_connection(self, sql_server_handler):
|
|
84
|
-
res = sql_server_handler.check_connection()
|
|
85
|
-
assert res.success, res.error_message
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
@pytest.mark.usefixtures("sql_server_handler")
|
|
89
|
-
class TestMSSQLHandlerTables:
|
|
90
|
-
table_for_creation = "test_mdb"
|
|
91
|
-
|
|
92
|
-
def test_get_tables(self, sql_server_handler):
|
|
93
|
-
res = sql_server_handler.get_tables()
|
|
94
|
-
tables = res.data_frame
|
|
95
|
-
assert (
|
|
96
|
-
tables is not None
|
|
97
|
-
), "expected to have some tables in the db, but got None"
|
|
98
|
-
assert (
|
|
99
|
-
"table_name" in tables
|
|
100
|
-
), f"expected to get 'table_name' in the response but got: {tables}"
|
|
101
|
-
assert (
|
|
102
|
-
"test" in tables["table_name"].values
|
|
103
|
-
), "expected to have 'test' in the response."
|
|
104
|
-
|
|
105
|
-
def test_get_columns(self, sql_server_handler):
|
|
106
|
-
response = sql_server_handler.get_columns("test")
|
|
107
|
-
assert response.type == RESPONSE_TYPE.TABLE, "expected a TABLE"
|
|
108
|
-
assert len(response.data_frame) > 0, "expected > O columns"
|
|
109
|
-
|
|
110
|
-
expected_columns = {
|
|
111
|
-
"Field": ["col_one", "col_two", "col_three", "col_four"],
|
|
112
|
-
"Type": ["int", "int", "float", "varchar"],
|
|
113
|
-
}
|
|
114
|
-
expected_df = pd.DataFrame(expected_columns)
|
|
115
|
-
assert response.data_frame.equals(
|
|
116
|
-
expected_df
|
|
117
|
-
), "response does not contain the expected columns"
|
|
118
|
-
|
|
119
|
-
def test_create_table(self, sql_server_handler):
|
|
120
|
-
query = f"""
|
|
121
|
-
IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = '{self.table_for_creation}')
|
|
122
|
-
BEGIN
|
|
123
|
-
CREATE TABLE {self.table_for_creation} (test_col INT)
|
|
124
|
-
END
|
|
125
|
-
"""
|
|
126
|
-
res = sql_server_handler.native_query(query)
|
|
127
|
-
check_valid_response(res)
|
|
128
|
-
tables = get_table_names(sql_server_handler)
|
|
129
|
-
assert (
|
|
130
|
-
self.table_for_creation in tables
|
|
131
|
-
), f"expected to have {self.table_for_creation} in database, but got: {tables}"
|
|
132
|
-
|
|
133
|
-
def test_drop_table(self, sql_server_handler):
|
|
134
|
-
query = f"DROP TABLE IF EXISTS {self.table_for_creation}"
|
|
135
|
-
res = sql_server_handler.native_query(query)
|
|
136
|
-
check_valid_response(res)
|
|
137
|
-
tables = get_table_names(sql_server_handler)
|
|
138
|
-
assert self.table_for_creation not in tables
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
@pytest.mark.usefixtures("sql_server_handler")
|
|
142
|
-
class TestMSSQLHandlerQuery:
|
|
143
|
-
def test_select_native_query(self, sql_server_handler):
|
|
144
|
-
query = "SELECT * FROM test"
|
|
145
|
-
response = sql_server_handler.native_query(query)
|
|
146
|
-
assert type(response) is Response
|
|
147
|
-
assert response.resp_type == RESPONSE_TYPE.TABLE
|
|
148
|
-
|
|
149
|
-
expected_data = {
|
|
150
|
-
"col_one": [1, 2, 3],
|
|
151
|
-
"col_two": [-1, -2, -3],
|
|
152
|
-
"col_three": [0.1, 0.2, 0.3],
|
|
153
|
-
"col_four": ["A", "B", "C"],
|
|
154
|
-
}
|
|
155
|
-
expected_df = pd.DataFrame(expected_data)
|
|
156
|
-
assert response.data_frame.equals(
|
|
157
|
-
expected_df
|
|
158
|
-
), "response does not contain the expected data"
|
|
159
|
-
|
|
160
|
-
def test_select_query(self, sql_server_handler):
|
|
161
|
-
limit = 3
|
|
162
|
-
query = "SELECT * FROM test"
|
|
163
|
-
res = sql_server_handler.query(query)
|
|
164
|
-
check_valid_response(res)
|
|
165
|
-
got_rows = res.data_frame.shape[0]
|
|
166
|
-
want_rows = limit
|
|
167
|
-
assert (
|
|
168
|
-
got_rows == want_rows
|
|
169
|
-
), f"expected to have {want_rows} rows in response but got: {got_rows}"
|
|
File without changes
|