MindsDB 25.5.4.2__py3-none-any.whl → 25.6.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/api/a2a/agent.py +28 -25
- mindsdb/api/a2a/common/server/server.py +32 -26
- mindsdb/api/executor/command_executor.py +69 -14
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +49 -65
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +29 -48
- mindsdb/api/executor/datahub/datanodes/system_tables.py +35 -61
- mindsdb/api/executor/planner/plan_join.py +67 -77
- mindsdb/api/executor/planner/query_planner.py +176 -155
- mindsdb/api/executor/planner/steps.py +37 -12
- mindsdb/api/executor/sql_query/result_set.py +45 -64
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +14 -18
- mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py +17 -18
- mindsdb/api/executor/sql_query/steps/insert_step.py +13 -33
- mindsdb/api/executor/sql_query/steps/subselect_step.py +43 -35
- mindsdb/api/executor/utilities/sql.py +42 -48
- mindsdb/api/http/namespaces/config.py +1 -1
- mindsdb/api/http/namespaces/file.py +14 -23
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_datum.py +12 -28
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/binary_resultset_row_package.py +59 -50
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/resultset_row_package.py +9 -8
- mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py +449 -461
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +87 -36
- mindsdb/integrations/handlers/file_handler/file_handler.py +15 -9
- mindsdb/integrations/handlers/file_handler/tests/test_file_handler.py +43 -24
- mindsdb/integrations/handlers/litellm_handler/litellm_handler.py +10 -3
- mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +26 -33
- mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +74 -51
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +305 -98
- mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py +53 -34
- mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py +136 -6
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +334 -83
- mindsdb/integrations/libs/api_handler.py +261 -57
- mindsdb/integrations/libs/base.py +100 -29
- mindsdb/integrations/utilities/files/file_reader.py +99 -73
- mindsdb/integrations/utilities/handler_utils.py +23 -8
- mindsdb/integrations/utilities/sql_utils.py +35 -40
- mindsdb/interfaces/agents/agents_controller.py +196 -192
- mindsdb/interfaces/agents/constants.py +7 -1
- mindsdb/interfaces/agents/langchain_agent.py +42 -11
- mindsdb/interfaces/agents/mcp_client_agent.py +29 -21
- mindsdb/interfaces/data_catalog/__init__.py +0 -0
- mindsdb/interfaces/data_catalog/base_data_catalog.py +54 -0
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +359 -0
- mindsdb/interfaces/data_catalog/data_catalog_reader.py +34 -0
- mindsdb/interfaces/database/database.py +81 -57
- mindsdb/interfaces/database/integrations.py +220 -234
- mindsdb/interfaces/database/log.py +72 -104
- mindsdb/interfaces/database/projects.py +156 -193
- mindsdb/interfaces/file/file_controller.py +21 -65
- mindsdb/interfaces/knowledge_base/controller.py +63 -10
- mindsdb/interfaces/knowledge_base/evaluate.py +519 -0
- mindsdb/interfaces/knowledge_base/llm_client.py +75 -0
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_kb_tools.py +83 -43
- mindsdb/interfaces/skills/skills_controller.py +54 -36
- mindsdb/interfaces/skills/sql_agent.py +109 -86
- mindsdb/interfaces/storage/db.py +223 -79
- mindsdb/migrations/versions/2025-05-28_a44643042fe8_added_data_catalog_tables.py +118 -0
- mindsdb/migrations/versions/2025-06-09_608e376c19a7_updated_data_catalog_data_types.py +58 -0
- mindsdb/utilities/config.py +9 -2
- mindsdb/utilities/log.py +35 -26
- mindsdb/utilities/ml_task_queue/task.py +19 -22
- mindsdb/utilities/render/sqlalchemy_render.py +129 -181
- mindsdb/utilities/starters.py +40 -0
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.2.0.dist-info}/METADATA +253 -253
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.2.0.dist-info}/RECORD +69 -61
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.2.0.dist-info}/WHEEL +0 -0
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.2.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.2.0.dist-info}/top_level.txt +0 -0
|
@@ -14,24 +14,22 @@ from sqlalchemy.sql import functions as sa_fnc
|
|
|
14
14
|
from mindsdb_sql_parser import ast
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
RESERVED_WORDS = {
|
|
18
|
-
"collation"
|
|
19
|
-
}
|
|
17
|
+
RESERVED_WORDS = {"collation"}
|
|
20
18
|
|
|
21
19
|
sa_type_names = [
|
|
22
|
-
key
|
|
23
|
-
|
|
20
|
+
key
|
|
21
|
+
for key, val in sa.types.__dict__.items()
|
|
22
|
+
if hasattr(val, "__module__") and val.__module__ in ("sqlalchemy.sql.sqltypes", "sqlalchemy.sql.type_api")
|
|
24
23
|
]
|
|
25
24
|
|
|
26
25
|
types_map = {}
|
|
27
26
|
for type_name in sa_type_names:
|
|
28
27
|
types_map[type_name.upper()] = getattr(sa.types, type_name)
|
|
29
|
-
types_map[
|
|
30
|
-
types_map[
|
|
28
|
+
types_map["BOOL"] = types_map["BOOLEAN"]
|
|
29
|
+
types_map["DEC"] = types_map["DECIMAL"]
|
|
31
30
|
|
|
32
31
|
|
|
33
|
-
class RenderError(Exception):
|
|
34
|
-
...
|
|
32
|
+
class RenderError(Exception): ...
|
|
35
33
|
|
|
36
34
|
|
|
37
35
|
# https://github.com/sqlalchemy/sqlalchemy/discussions/9483?sort=old#discussioncomment-5312979
|
|
@@ -43,13 +41,13 @@ class INTERVAL(ColumnElement):
|
|
|
43
41
|
|
|
44
42
|
@compiles(INTERVAL)
|
|
45
43
|
def _compile_interval(element, compiler, **kw):
|
|
46
|
-
items = element.info.split(
|
|
47
|
-
if compiler.dialect.name ==
|
|
44
|
+
items = element.info.split(" ", maxsplit=1)
|
|
45
|
+
if compiler.dialect.name == "oracle" and len(items) == 2:
|
|
48
46
|
# replace to singular names (remove leading S if exists)
|
|
49
|
-
if items[1].upper().endswith(
|
|
47
|
+
if items[1].upper().endswith("S"):
|
|
50
48
|
items[1] = items[1][:-1]
|
|
51
49
|
|
|
52
|
-
if compiler.dialect.driver in [
|
|
50
|
+
if compiler.dialect.driver in ["snowflake"]:
|
|
53
51
|
# quote all
|
|
54
52
|
args = " ".join(map(str, items))
|
|
55
53
|
args = f"'{args}'"
|
|
@@ -62,8 +60,9 @@ def _compile_interval(element, compiler, **kw):
|
|
|
62
60
|
|
|
63
61
|
class AttributedStr(str):
|
|
64
62
|
"""
|
|
65
|
-
|
|
63
|
+
Custom str-like object to pass it to `_requires_quotes` method with `is_quoted` flag
|
|
66
64
|
"""
|
|
65
|
+
|
|
67
66
|
def __new__(cls, string, is_quoted: bool):
|
|
68
67
|
obj = str.__new__(cls, string)
|
|
69
68
|
obj.is_quoted = is_quoted
|
|
@@ -75,23 +74,22 @@ class AttributedStr(str):
|
|
|
75
74
|
|
|
76
75
|
|
|
77
76
|
def get_is_quoted(identifier: ast.Identifier):
|
|
78
|
-
quoted = getattr(identifier,
|
|
77
|
+
quoted = getattr(identifier, "is_quoted", [])
|
|
79
78
|
# len can be different
|
|
80
79
|
quoted = quoted + [None] * (len(identifier.parts) - len(quoted))
|
|
81
80
|
return quoted
|
|
82
81
|
|
|
83
82
|
|
|
84
83
|
class SqlalchemyRender:
|
|
85
|
-
|
|
86
84
|
def __init__(self, dialect_name):
|
|
87
85
|
dialects = {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
86
|
+
"mysql": mysql,
|
|
87
|
+
"postgresql": postgresql,
|
|
88
|
+
"postgres": postgresql,
|
|
89
|
+
"sqlite": sqlite,
|
|
90
|
+
"mssql": mssql,
|
|
91
|
+
"oracle": oracle,
|
|
92
|
+
"Snowflake": oracle,
|
|
95
93
|
}
|
|
96
94
|
|
|
97
95
|
if isinstance(dialect_name, str):
|
|
@@ -100,9 +98,9 @@ class SqlalchemyRender:
|
|
|
100
98
|
dialect = dialect_name
|
|
101
99
|
|
|
102
100
|
# override dialect's preparer
|
|
103
|
-
if hasattr(dialect,
|
|
104
|
-
class Preparer(dialect.preparer):
|
|
101
|
+
if hasattr(dialect, "preparer"):
|
|
105
102
|
|
|
103
|
+
class Preparer(dialect.preparer):
|
|
106
104
|
def _requires_quotes(self, value: str) -> bool:
|
|
107
105
|
# check force-quote flag
|
|
108
106
|
if isinstance(value, AttributedStr):
|
|
@@ -117,6 +115,7 @@ class SqlalchemyRender:
|
|
|
117
115
|
# Override sqlalchemy behavior: don't require to quote mixed- or upper-case
|
|
118
116
|
# or (lc_value != value)
|
|
119
117
|
)
|
|
118
|
+
|
|
120
119
|
dialect.preparer = Preparer
|
|
121
120
|
|
|
122
121
|
# remove double percent signs
|
|
@@ -126,11 +125,11 @@ class SqlalchemyRender:
|
|
|
126
125
|
|
|
127
126
|
self.selects_stack = []
|
|
128
127
|
|
|
129
|
-
if dialect_name ==
|
|
128
|
+
if dialect_name == "mssql":
|
|
130
129
|
# update version to MS_2008_VERSION for supports_multivalues_insert
|
|
131
130
|
self.dialect.server_version_info = (10,)
|
|
132
131
|
self.dialect._setup_version_attributes()
|
|
133
|
-
elif dialect_name ==
|
|
132
|
+
elif dialect_name == "mysql":
|
|
134
133
|
# update version for support float cast
|
|
135
134
|
self.dialect.server_version_info = (8, 0, 17)
|
|
136
135
|
|
|
@@ -142,7 +141,7 @@ class SqlalchemyRender:
|
|
|
142
141
|
quoted = get_is_quoted(identifier)
|
|
143
142
|
for i, is_quoted in zip(identifier.parts, quoted):
|
|
144
143
|
if isinstance(i, ast.Star):
|
|
145
|
-
part =
|
|
144
|
+
part = "*"
|
|
146
145
|
elif is_quoted or i.lower() in RESERVED_WORDS:
|
|
147
146
|
# quote anyway
|
|
148
147
|
part = self.dialect.identifier_preparer.quote_identifier(i)
|
|
@@ -151,45 +150,48 @@ class SqlalchemyRender:
|
|
|
151
150
|
part = self.dialect.identifier_preparer.quote(i)
|
|
152
151
|
|
|
153
152
|
parts2.append(part)
|
|
154
|
-
text =
|
|
155
|
-
if identifier.is_outer and self.dialect.name ==
|
|
156
|
-
text +=
|
|
153
|
+
text = ".".join(parts2)
|
|
154
|
+
if identifier.is_outer and self.dialect.name == "oracle":
|
|
155
|
+
text += "(+)"
|
|
157
156
|
return sa.column(text, is_literal=True)
|
|
158
157
|
|
|
159
158
|
def get_alias(self, alias):
|
|
160
159
|
if alias is None or len(alias.parts) == 0:
|
|
161
160
|
return None
|
|
162
161
|
if len(alias.parts) > 1:
|
|
163
|
-
raise NotImplementedError(f
|
|
162
|
+
raise NotImplementedError(f"Multiple alias {alias.parts}")
|
|
164
163
|
|
|
165
164
|
if self.selects_stack:
|
|
166
|
-
self.selects_stack[-1][
|
|
165
|
+
self.selects_stack[-1]["aliases"].append(alias)
|
|
167
166
|
|
|
168
167
|
is_quoted = get_is_quoted(alias)[0]
|
|
169
168
|
return AttributedStr(alias.parts[0], is_quoted)
|
|
170
169
|
|
|
171
|
-
def
|
|
170
|
+
def make_unique_alias(self, name):
|
|
171
|
+
if self.selects_stack:
|
|
172
|
+
aliases = self.selects_stack[-1]["aliases"]
|
|
173
|
+
for i in range(10):
|
|
174
|
+
name2 = f"{name}_{i}"
|
|
175
|
+
if name2 not in aliases:
|
|
176
|
+
aliases.append(name2)
|
|
177
|
+
return name2
|
|
172
178
|
|
|
179
|
+
def to_expression(self, t):
|
|
173
180
|
# simple type
|
|
174
|
-
if (
|
|
175
|
-
isinstance(t, str)
|
|
176
|
-
or isinstance(t, int)
|
|
177
|
-
or isinstance(t, float)
|
|
178
|
-
or t is None
|
|
179
|
-
):
|
|
181
|
+
if isinstance(t, str) or isinstance(t, int) or isinstance(t, float) or t is None:
|
|
180
182
|
t = ast.Constant(t)
|
|
181
183
|
|
|
182
184
|
if isinstance(t, ast.Star):
|
|
183
|
-
col = sa.text(
|
|
185
|
+
col = sa.text("*")
|
|
184
186
|
elif isinstance(t, ast.Last):
|
|
185
|
-
col = self.to_column(ast.Identifier(parts=[
|
|
187
|
+
col = self.to_column(ast.Identifier(parts=["last"]))
|
|
186
188
|
elif isinstance(t, ast.Constant):
|
|
187
189
|
col = sa.literal(t.value)
|
|
188
190
|
if t.alias:
|
|
189
191
|
alias = self.get_alias(t.alias)
|
|
190
192
|
else:
|
|
191
193
|
if t.value is None:
|
|
192
|
-
alias =
|
|
194
|
+
alias = "NULL"
|
|
193
195
|
else:
|
|
194
196
|
alias = str(t.value)
|
|
195
197
|
col = col.label(alias)
|
|
@@ -199,13 +201,13 @@ class SqlalchemyRender:
|
|
|
199
201
|
if len(t.parts) == 1:
|
|
200
202
|
if isinstance(t.parts[0], str):
|
|
201
203
|
name = t.parts[0].upper()
|
|
202
|
-
if name ==
|
|
204
|
+
if name == "CURRENT_DATE":
|
|
203
205
|
col = sa_fnc.current_date()
|
|
204
|
-
elif name ==
|
|
206
|
+
elif name == "CURRENT_TIME":
|
|
205
207
|
col = sa_fnc.current_time()
|
|
206
|
-
elif name ==
|
|
208
|
+
elif name == "CURRENT_TIMESTAMP":
|
|
207
209
|
col = sa_fnc.current_timestamp()
|
|
208
|
-
elif name ==
|
|
210
|
+
elif name == "CURRENT_USER":
|
|
209
211
|
col = sa_fnc.current_user()
|
|
210
212
|
if col is None:
|
|
211
213
|
col = self.to_column(t)
|
|
@@ -223,12 +225,9 @@ class SqlalchemyRender:
|
|
|
223
225
|
alias = self.get_alias(t.alias)
|
|
224
226
|
col = col.label(alias)
|
|
225
227
|
else:
|
|
226
|
-
alias = str(t.op)
|
|
227
|
-
if
|
|
228
|
-
|
|
229
|
-
if alias not in aliases:
|
|
230
|
-
aliases.append(alias)
|
|
231
|
-
col = col.label(alias)
|
|
228
|
+
alias = self.make_unique_alias(str(t.op))
|
|
229
|
+
if alias:
|
|
230
|
+
col = col.label(alias)
|
|
232
231
|
|
|
233
232
|
elif isinstance(t, ast.BinaryOperation):
|
|
234
233
|
ops = {
|
|
@@ -261,11 +260,11 @@ class SqlalchemyRender:
|
|
|
261
260
|
arg1 = self.to_expression(t.args[1])
|
|
262
261
|
|
|
263
262
|
op = t.op.lower()
|
|
264
|
-
if op in (
|
|
263
|
+
if op in ("in", "not in"):
|
|
265
264
|
if t.args[1].parentheses:
|
|
266
265
|
arg1 = [arg1]
|
|
267
266
|
if isinstance(arg1, sa.sql.selectable.ColumnClause):
|
|
268
|
-
raise NotImplementedError(f
|
|
267
|
+
raise NotImplementedError(f"Required list argument for: {op}")
|
|
269
268
|
|
|
270
269
|
sa_op = ops.get(op)
|
|
271
270
|
|
|
@@ -275,7 +274,7 @@ class SqlalchemyRender:
|
|
|
275
274
|
col = arg1.reverse_operate(sa_op, arg0)
|
|
276
275
|
elif isinstance(arg1, sa.TextClause):
|
|
277
276
|
# both args are text, return text
|
|
278
|
-
col = sa.text(f
|
|
277
|
+
col = sa.text(f"{arg0.compile(dialect=self.dialect)} {op} {arg1.compile(dialect=self.dialect)}")
|
|
279
278
|
else:
|
|
280
279
|
col = arg0.operate(sa_op, arg1)
|
|
281
280
|
|
|
@@ -320,53 +319,44 @@ class SqlalchemyRender:
|
|
|
320
319
|
|
|
321
320
|
partition = None
|
|
322
321
|
if t.partition is not None:
|
|
323
|
-
partition = [
|
|
324
|
-
self.to_expression(i)
|
|
325
|
-
for i in t.partition
|
|
326
|
-
]
|
|
322
|
+
partition = [self.to_expression(i) for i in t.partition]
|
|
327
323
|
|
|
328
324
|
order_by = None
|
|
329
325
|
if t.order_by is not None:
|
|
330
326
|
order_by = []
|
|
331
327
|
for f in t.order_by:
|
|
332
328
|
col0 = self.to_expression(f.field)
|
|
333
|
-
if f.direction ==
|
|
329
|
+
if f.direction == "DESC":
|
|
334
330
|
col0 = col0.desc()
|
|
335
331
|
order_by.append(col0)
|
|
336
332
|
|
|
337
333
|
rows, range_ = None, None
|
|
338
334
|
if t.modifier is not None:
|
|
339
335
|
words = t.modifier.lower().split()
|
|
340
|
-
if words[1] ==
|
|
336
|
+
if words[1] == "between" and words[4] == "and":
|
|
341
337
|
# frame options
|
|
342
338
|
# rows/groups BETWEEN <> <> AND <> <>
|
|
343
339
|
# https://docs.sqlalchemy.org/en/20/core/sqlelement.html#sqlalchemy.sql.expression.over
|
|
344
340
|
items = []
|
|
345
341
|
for word1, word2 in (words[2:4], words[5:7]):
|
|
346
|
-
if word1 ==
|
|
342
|
+
if word1 == "unbounded":
|
|
347
343
|
items.append(None)
|
|
348
|
-
elif (word1, word2) == (
|
|
344
|
+
elif (word1, word2) == ("current", "row"):
|
|
349
345
|
items.append(0)
|
|
350
346
|
elif word1.isdigits():
|
|
351
347
|
val = int(word1)
|
|
352
|
-
if word2 ==
|
|
348
|
+
if word2 == "preceding":
|
|
353
349
|
val = -val
|
|
354
|
-
elif word2 !=
|
|
350
|
+
elif word2 != "following":
|
|
355
351
|
continue
|
|
356
352
|
items.append(val)
|
|
357
353
|
if len(items) == 2:
|
|
358
|
-
if words[0] ==
|
|
354
|
+
if words[0] == "rows":
|
|
359
355
|
rows = tuple(items)
|
|
360
|
-
elif words[0] ==
|
|
356
|
+
elif words[0] == "range":
|
|
361
357
|
range_ = tuple(items)
|
|
362
358
|
|
|
363
|
-
col = sa.over(
|
|
364
|
-
func,
|
|
365
|
-
partition_by=partition,
|
|
366
|
-
order_by=order_by,
|
|
367
|
-
range_=range_,
|
|
368
|
-
rows=rows
|
|
369
|
-
)
|
|
359
|
+
col = sa.over(func, partition_by=partition, order_by=order_by, range_=range_, rows=rows)
|
|
370
360
|
|
|
371
361
|
if t.alias:
|
|
372
362
|
col = col.label(self.get_alias(t.alias))
|
|
@@ -380,15 +370,16 @@ class SqlalchemyRender:
|
|
|
380
370
|
if t.alias:
|
|
381
371
|
alias = self.get_alias(t.alias)
|
|
382
372
|
col = col.label(alias)
|
|
373
|
+
else:
|
|
374
|
+
alias = self.make_unique_alias("cast")
|
|
375
|
+
if alias:
|
|
376
|
+
col = col.label(alias)
|
|
383
377
|
elif isinstance(t, ast.Parameter):
|
|
384
378
|
col = sa.column(t.value, is_literal=True)
|
|
385
379
|
if t.alias:
|
|
386
380
|
raise RenderError()
|
|
387
381
|
elif isinstance(t, ast.Tuple):
|
|
388
|
-
col = [
|
|
389
|
-
self.to_expression(i)
|
|
390
|
-
for i in t.items
|
|
391
|
-
]
|
|
382
|
+
col = [self.to_expression(i) for i in t.items]
|
|
392
383
|
elif isinstance(t, ast.Variable):
|
|
393
384
|
col = sa.column(t.to_string(), is_literal=True)
|
|
394
385
|
elif isinstance(t, ast.Latest):
|
|
@@ -403,16 +394,14 @@ class SqlalchemyRender:
|
|
|
403
394
|
col = self.prepare_case(t)
|
|
404
395
|
else:
|
|
405
396
|
# some other complex object?
|
|
406
|
-
raise NotImplementedError(f
|
|
397
|
+
raise NotImplementedError(f"Column {t}")
|
|
407
398
|
|
|
408
399
|
return col
|
|
409
400
|
|
|
410
401
|
def prepare_case(self, t: ast.Case):
|
|
411
402
|
conditions = []
|
|
412
403
|
for condition, result in t.rules:
|
|
413
|
-
conditions.append(
|
|
414
|
-
(self.to_expression(condition), self.to_expression(result))
|
|
415
|
-
)
|
|
404
|
+
conditions.append((self.to_expression(condition), self.to_expression(result)))
|
|
416
405
|
default = None
|
|
417
406
|
if t.default is not None:
|
|
418
407
|
default = self.to_expression(t.default)
|
|
@@ -434,10 +423,7 @@ class SqlalchemyRender:
|
|
|
434
423
|
|
|
435
424
|
fnc = op(arg, from_arg)
|
|
436
425
|
else:
|
|
437
|
-
args = [
|
|
438
|
-
self.to_expression(i)
|
|
439
|
-
for i in t.args
|
|
440
|
-
]
|
|
426
|
+
args = [self.to_expression(i) for i in t.args]
|
|
441
427
|
if t.distinct:
|
|
442
428
|
# set first argument to distinct
|
|
443
429
|
args[0] = args[0].distinct()
|
|
@@ -451,10 +437,10 @@ class SqlalchemyRender:
|
|
|
451
437
|
return typename
|
|
452
438
|
|
|
453
439
|
typename = typename.upper()
|
|
454
|
-
if re.match(r
|
|
455
|
-
typename =
|
|
456
|
-
if re.match(r
|
|
457
|
-
typename =
|
|
440
|
+
if re.match(r"^INT[\d]+$", typename):
|
|
441
|
+
typename = "BIGINT"
|
|
442
|
+
if re.match(r"^FLOAT[\d]+$", typename):
|
|
443
|
+
typename = "FLOAT"
|
|
458
444
|
|
|
459
445
|
return types_map[typename]
|
|
460
446
|
|
|
@@ -462,7 +448,7 @@ class SqlalchemyRender:
|
|
|
462
448
|
# join tree to table list
|
|
463
449
|
|
|
464
450
|
if isinstance(join.right, ast.Join):
|
|
465
|
-
raise NotImplementedError(
|
|
451
|
+
raise NotImplementedError("Wrong join AST")
|
|
466
452
|
|
|
467
453
|
items = []
|
|
468
454
|
|
|
@@ -471,17 +457,12 @@ class SqlalchemyRender:
|
|
|
471
457
|
items.extend(self.prepare_join(join.left))
|
|
472
458
|
else:
|
|
473
459
|
# this is first table
|
|
474
|
-
items.append(dict(
|
|
475
|
-
table=join.left
|
|
476
|
-
))
|
|
460
|
+
items.append(dict(table=join.left))
|
|
477
461
|
|
|
478
462
|
# all properties set to right table
|
|
479
|
-
items.append(
|
|
480
|
-
table=join.right,
|
|
481
|
-
|
|
482
|
-
is_implicit=join.implicit,
|
|
483
|
-
condition=join.condition
|
|
484
|
-
))
|
|
463
|
+
items.append(
|
|
464
|
+
dict(table=join.right, join_type=join.join_type, is_implicit=join.implicit, condition=join.condition)
|
|
465
|
+
)
|
|
485
466
|
|
|
486
467
|
return items
|
|
487
468
|
|
|
@@ -493,7 +474,7 @@ class SqlalchemyRender:
|
|
|
493
474
|
|
|
494
475
|
if len(parts) > 2:
|
|
495
476
|
# TODO tests is failing
|
|
496
|
-
raise NotImplementedError(f
|
|
477
|
+
raise NotImplementedError(f"Path to long: {table_name.parts}")
|
|
497
478
|
|
|
498
479
|
if len(parts) == 2:
|
|
499
480
|
schema = AttributedStr(parts[-2], quoted[-2])
|
|
@@ -523,7 +504,7 @@ class SqlalchemyRender:
|
|
|
523
504
|
|
|
524
505
|
else:
|
|
525
506
|
# TODO tests are failing
|
|
526
|
-
raise NotImplementedError(f
|
|
507
|
+
raise NotImplementedError(f"Table {node.__name__}")
|
|
527
508
|
|
|
528
509
|
return table
|
|
529
510
|
|
|
@@ -533,7 +514,7 @@ class SqlalchemyRender:
|
|
|
533
514
|
|
|
534
515
|
cols = []
|
|
535
516
|
|
|
536
|
-
self.selects_stack.append({
|
|
517
|
+
self.selects_stack.append({"aliases": []})
|
|
537
518
|
|
|
538
519
|
for t in node.targets:
|
|
539
520
|
col = self.to_expression(t)
|
|
@@ -544,7 +525,7 @@ class SqlalchemyRender:
|
|
|
544
525
|
if node.cte is not None:
|
|
545
526
|
for cte in node.cte:
|
|
546
527
|
if cte.columns is not None and len(cte.columns) > 0:
|
|
547
|
-
raise NotImplementedError(
|
|
528
|
+
raise NotImplementedError("CTE columns")
|
|
548
529
|
|
|
549
530
|
stmt = self.prepare_select(cte.query)
|
|
550
531
|
alias = cte.name
|
|
@@ -563,45 +544,41 @@ class SqlalchemyRender:
|
|
|
563
544
|
if isinstance(from_table, ast.Join):
|
|
564
545
|
join_list = self.prepare_join(from_table)
|
|
565
546
|
# first table
|
|
566
|
-
table = self.to_table(join_list[0][
|
|
547
|
+
table = self.to_table(join_list[0]["table"])
|
|
567
548
|
query = query.select_from(table)
|
|
568
549
|
|
|
569
550
|
# other tables
|
|
570
551
|
has_explicit_join = False
|
|
571
552
|
for item in join_list[1:]:
|
|
572
|
-
join_type = item[
|
|
573
|
-
table = self.to_table(item[
|
|
574
|
-
if item[
|
|
553
|
+
join_type = item["join_type"]
|
|
554
|
+
table = self.to_table(item["table"], is_lateral=("LATERAL" in join_type))
|
|
555
|
+
if item["is_implicit"]:
|
|
575
556
|
# add to from clause
|
|
576
557
|
if has_explicit_join:
|
|
577
558
|
# sqlalchemy doesn't support implicit join after explicit
|
|
578
559
|
# convert it to explicit
|
|
579
|
-
query = query.join(table, sa.text(
|
|
560
|
+
query = query.join(table, sa.text("1=1"))
|
|
580
561
|
else:
|
|
581
562
|
query = query.select_from(table)
|
|
582
563
|
else:
|
|
583
564
|
has_explicit_join = True
|
|
584
|
-
if item[
|
|
565
|
+
if item["condition"] is None:
|
|
585
566
|
# otherwise, sqlalchemy raises "Don't know how to join to ..."
|
|
586
|
-
condition = sa.text(
|
|
567
|
+
condition = sa.text("1=1")
|
|
587
568
|
else:
|
|
588
|
-
condition = self.to_expression(item[
|
|
569
|
+
condition = self.to_expression(item["condition"])
|
|
589
570
|
|
|
590
|
-
if
|
|
591
|
-
raise NotImplementedError(f
|
|
592
|
-
method =
|
|
571
|
+
if "ASOF" in join_type:
|
|
572
|
+
raise NotImplementedError(f"Unsupported join type: {join_type}")
|
|
573
|
+
method = "join"
|
|
593
574
|
is_full = False
|
|
594
|
-
if join_type ==
|
|
595
|
-
method =
|
|
596
|
-
if join_type ==
|
|
575
|
+
if join_type == "LEFT JOIN":
|
|
576
|
+
method = "outerjoin"
|
|
577
|
+
if join_type == "FULL JOIN":
|
|
597
578
|
is_full = True
|
|
598
579
|
|
|
599
580
|
# perform join
|
|
600
|
-
query = getattr(query, method)(
|
|
601
|
-
table,
|
|
602
|
-
condition,
|
|
603
|
-
full=is_full
|
|
604
|
-
)
|
|
581
|
+
query = getattr(query, method)(table, condition, full=is_full)
|
|
605
582
|
elif isinstance(from_table, (ast.Union, ast.Intersect, ast.Except)):
|
|
606
583
|
alias = None
|
|
607
584
|
if from_table.alias:
|
|
@@ -624,18 +601,13 @@ class SqlalchemyRender:
|
|
|
624
601
|
table = sa.text(from_table.query).columns().subquery(alias)
|
|
625
602
|
query = query.select_from(table)
|
|
626
603
|
else:
|
|
627
|
-
raise NotImplementedError(f
|
|
604
|
+
raise NotImplementedError(f"Select from {from_table}")
|
|
628
605
|
|
|
629
606
|
if node.where is not None:
|
|
630
|
-
query = query.filter(
|
|
631
|
-
self.to_expression(node.where)
|
|
632
|
-
)
|
|
607
|
+
query = query.filter(self.to_expression(node.where))
|
|
633
608
|
|
|
634
609
|
if node.group_by is not None:
|
|
635
|
-
cols = [
|
|
636
|
-
self.to_expression(i)
|
|
637
|
-
for i in node.group_by
|
|
638
|
-
]
|
|
610
|
+
cols = [self.to_expression(i) for i in node.group_by]
|
|
639
611
|
query = query.group_by(*cols)
|
|
640
612
|
|
|
641
613
|
if node.having is not None:
|
|
@@ -645,13 +617,13 @@ class SqlalchemyRender:
|
|
|
645
617
|
order_by = []
|
|
646
618
|
for f in node.order_by:
|
|
647
619
|
col0 = self.to_expression(f.field)
|
|
648
|
-
if f.direction.upper() ==
|
|
620
|
+
if f.direction.upper() == "DESC":
|
|
649
621
|
col0 = col0.desc()
|
|
650
|
-
elif f.direction.upper() ==
|
|
622
|
+
elif f.direction.upper() == "ASC":
|
|
651
623
|
col0 = col0.asc()
|
|
652
|
-
if f.nulls.upper() ==
|
|
624
|
+
if f.nulls.upper() == "NULLS FIRST":
|
|
653
625
|
col0 = sa.nullsfirst(col0)
|
|
654
|
-
elif f.nulls.upper() ==
|
|
626
|
+
elif f.nulls.upper() == "NULLS LAST":
|
|
655
627
|
col0 = sa.nullslast(col0)
|
|
656
628
|
order_by.append(col0)
|
|
657
629
|
|
|
@@ -664,10 +636,10 @@ class SqlalchemyRender:
|
|
|
664
636
|
query = query.offset(node.offset.value)
|
|
665
637
|
|
|
666
638
|
if node.mode is not None:
|
|
667
|
-
if node.mode ==
|
|
639
|
+
if node.mode == "FOR UPDATE":
|
|
668
640
|
query = query.with_for_update()
|
|
669
641
|
else:
|
|
670
|
-
raise NotImplementedError(f
|
|
642
|
+
raise NotImplementedError(f"Select mode: {node.mode}")
|
|
671
643
|
|
|
672
644
|
self.selects_stack.pop()
|
|
673
645
|
|
|
@@ -695,49 +667,34 @@ class SqlalchemyRender:
|
|
|
695
667
|
if isinstance(col.default, str):
|
|
696
668
|
default = sa.text(col.default)
|
|
697
669
|
|
|
698
|
-
if isinstance(col.type, str) and col.type.lower() ==
|
|
670
|
+
if isinstance(col.type, str) and col.type.lower() == "serial":
|
|
699
671
|
col.is_primary_key = True
|
|
700
|
-
col.type =
|
|
672
|
+
col.type = "INT"
|
|
701
673
|
|
|
702
674
|
kwargs = {
|
|
703
|
-
|
|
704
|
-
|
|
675
|
+
"primary_key": col.is_primary_key,
|
|
676
|
+
"server_default": default,
|
|
705
677
|
}
|
|
706
678
|
if col.nullable is not None:
|
|
707
|
-
kwargs[
|
|
679
|
+
kwargs["nullable"] = col.nullable
|
|
708
680
|
|
|
709
|
-
columns.append(
|
|
710
|
-
sa.Column(
|
|
711
|
-
col.name,
|
|
712
|
-
self.get_type(col.type),
|
|
713
|
-
**kwargs
|
|
714
|
-
)
|
|
715
|
-
)
|
|
681
|
+
columns.append(sa.Column(col.name, self.get_type(col.type), **kwargs))
|
|
716
682
|
|
|
717
683
|
schema, table_name = self.get_table_name(ast_query.name)
|
|
718
684
|
|
|
719
685
|
metadata = sa.MetaData()
|
|
720
|
-
table = sa.Table(
|
|
721
|
-
table_name,
|
|
722
|
-
metadata,
|
|
723
|
-
schema=schema,
|
|
724
|
-
*columns
|
|
725
|
-
)
|
|
686
|
+
table = sa.Table(table_name, metadata, schema=schema, *columns)
|
|
726
687
|
|
|
727
688
|
return CreateTable(table)
|
|
728
689
|
|
|
729
690
|
def prepare_drop_table(self, ast_query):
|
|
730
691
|
if len(ast_query.tables) != 1:
|
|
731
|
-
raise NotImplementedError(
|
|
692
|
+
raise NotImplementedError("Only one table is supported")
|
|
732
693
|
|
|
733
694
|
schema, table_name = self.get_table_name(ast_query.tables[0])
|
|
734
695
|
|
|
735
696
|
metadata = sa.MetaData()
|
|
736
|
-
table = sa.Table(
|
|
737
|
-
table_name,
|
|
738
|
-
metadata,
|
|
739
|
-
schema=schema
|
|
740
|
-
)
|
|
697
|
+
table = sa.Table(table_name, metadata, schema=schema)
|
|
741
698
|
return DropTable(table, if_exists=ast_query.if_exists)
|
|
742
699
|
|
|
743
700
|
def prepare_insert(self, ast_query, with_params=False):
|
|
@@ -748,7 +705,7 @@ class SqlalchemyRender:
|
|
|
748
705
|
columns = []
|
|
749
706
|
|
|
750
707
|
if ast_query.columns is None:
|
|
751
|
-
raise NotImplementedError(
|
|
708
|
+
raise NotImplementedError("Columns is required in insert query")
|
|
752
709
|
for col in ast_query.columns:
|
|
753
710
|
columns.append(
|
|
754
711
|
sa.Column(
|
|
@@ -758,7 +715,7 @@ class SqlalchemyRender:
|
|
|
758
715
|
)
|
|
759
716
|
# check doubles
|
|
760
717
|
if col.name in names:
|
|
761
|
-
raise RenderError(f
|
|
718
|
+
raise RenderError(f"Columns name double: {col.name}")
|
|
762
719
|
names.append(col.name)
|
|
763
720
|
|
|
764
721
|
table = sa.table(table_name, schema=schema, *columns)
|
|
@@ -767,19 +724,14 @@ class SqlalchemyRender:
|
|
|
767
724
|
values = []
|
|
768
725
|
|
|
769
726
|
if ast_query.is_plain and with_params:
|
|
770
|
-
|
|
771
727
|
for i in range(len(ast_query.columns)):
|
|
772
|
-
values.append(sa.column(
|
|
728
|
+
values.append(sa.column("%s", is_literal=True))
|
|
773
729
|
|
|
774
730
|
values = [values]
|
|
775
731
|
params = ast_query.values
|
|
776
732
|
else:
|
|
777
|
-
|
|
778
733
|
for row in ast_query.values:
|
|
779
|
-
row = [
|
|
780
|
-
self.to_expression(val)
|
|
781
|
-
for val in row
|
|
782
|
-
]
|
|
734
|
+
row = [self.to_expression(val) for val in row]
|
|
783
735
|
values.append(row)
|
|
784
736
|
|
|
785
737
|
stmt = table.insert().values(values)
|
|
@@ -792,7 +744,7 @@ class SqlalchemyRender:
|
|
|
792
744
|
|
|
793
745
|
def prepare_update(self, ast_query):
|
|
794
746
|
if ast_query.from_select is not None:
|
|
795
|
-
raise NotImplementedError(
|
|
747
|
+
raise NotImplementedError("Render of update with sub-select is not implemented")
|
|
796
748
|
|
|
797
749
|
schema, table_name = self.get_table_name(ast_query.table)
|
|
798
750
|
|
|
@@ -846,7 +798,7 @@ class SqlalchemyRender:
|
|
|
846
798
|
elif isinstance(ast_query, ast.DropTables):
|
|
847
799
|
stmt = self.prepare_drop_table(ast_query)
|
|
848
800
|
else:
|
|
849
|
-
raise NotImplementedError(f
|
|
801
|
+
raise NotImplementedError(f"Unknown statement: {ast_query.__class__.__name__}")
|
|
850
802
|
return stmt, params
|
|
851
803
|
|
|
852
804
|
def get_string(self, ast_query, with_failback=True):
|
|
@@ -885,32 +837,28 @@ class SqlalchemyRender:
|
|
|
885
837
|
raise e
|
|
886
838
|
|
|
887
839
|
sql_query = str(ast_query)
|
|
888
|
-
if self.dialect.name ==
|
|
889
|
-
sql_query = sql_query.replace(
|
|
840
|
+
if self.dialect.name == "postgresql":
|
|
841
|
+
sql_query = sql_query.replace("`", "")
|
|
890
842
|
return sql_query, None
|
|
891
843
|
|
|
892
844
|
|
|
893
845
|
def render_dml_query(statement, dialect):
|
|
894
|
-
|
|
895
846
|
class LiteralCompiler(dialect.statement_compiler):
|
|
896
|
-
|
|
897
847
|
def render_literal_value(self, value, type_):
|
|
898
|
-
|
|
899
848
|
if isinstance(value, (str, dt.date, dt.datetime, dt.timedelta)):
|
|
900
849
|
return "'{}'".format(str(value).replace("'", "''"))
|
|
901
850
|
|
|
902
851
|
return super(LiteralCompiler, self).render_literal_value(value, type_)
|
|
903
852
|
|
|
904
|
-
return str(LiteralCompiler(dialect, statement, compile_kwargs={
|
|
853
|
+
return str(LiteralCompiler(dialect, statement, compile_kwargs={"literal_binds": True}))
|
|
905
854
|
|
|
906
855
|
|
|
907
856
|
def render_ddl_query(statement, dialect):
|
|
908
857
|
class LiteralCompiler(dialect.ddl_compiler):
|
|
909
|
-
|
|
910
858
|
def render_literal_value(self, value, type_):
|
|
911
859
|
if isinstance(value, (str, dt.date, dt.datetime, dt.timedelta)):
|
|
912
860
|
return "'{}'".format(str(value).replace("'", "''"))
|
|
913
861
|
|
|
914
862
|
return super(LiteralCompiler, self).render_literal_value(value, type_)
|
|
915
863
|
|
|
916
|
-
return str(LiteralCompiler(dialect, statement, compile_kwargs={
|
|
864
|
+
return str(LiteralCompiler(dialect, statement, compile_kwargs={"literal_binds": True}))
|