MindsDB 25.8.3.0__py3-none-any.whl → 25.9.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +3 -45
- mindsdb/api/a2a/__init__.py +52 -0
- mindsdb/api/a2a/agent.py +11 -12
- mindsdb/api/a2a/common/server/server.py +17 -36
- mindsdb/api/a2a/common/server/task_manager.py +14 -28
- mindsdb/api/a2a/task_manager.py +20 -21
- mindsdb/api/a2a/utils.py +1 -1
- mindsdb/api/common/middleware.py +106 -0
- mindsdb/api/executor/utilities/mysql_to_duckdb_functions.py +466 -18
- mindsdb/api/executor/utilities/sql.py +9 -31
- mindsdb/api/http/initialize.py +34 -43
- mindsdb/api/http/namespaces/auth.py +6 -14
- mindsdb/api/http/namespaces/config.py +0 -2
- mindsdb/api/http/namespaces/default.py +74 -106
- mindsdb/api/http/namespaces/file.py +9 -3
- mindsdb/api/http/namespaces/handlers.py +77 -87
- mindsdb/api/http/start.py +29 -47
- mindsdb/api/litellm/start.py +11 -10
- mindsdb/api/mcp/__init__.py +165 -0
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +33 -64
- mindsdb/api/postgres/postgres_proxy/postgres_proxy.py +86 -85
- mindsdb/integrations/handlers/autogluon_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/autosklearn_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/crate_handler/crate_handler.py +3 -7
- mindsdb/integrations/handlers/derby_handler/derby_handler.py +32 -34
- mindsdb/integrations/handlers/documentdb_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/dummy_data_handler/dummy_data_handler.py +12 -13
- mindsdb/integrations/handlers/flaml_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/google_books_handler/google_books_handler.py +45 -44
- mindsdb/integrations/handlers/google_calendar_handler/google_calendar_handler.py +101 -95
- mindsdb/integrations/handlers/google_content_shopping_handler/google_content_shopping_handler.py +129 -129
- mindsdb/integrations/handlers/google_fit_handler/google_fit_handler.py +59 -43
- mindsdb/integrations/handlers/google_search_handler/google_search_handler.py +38 -39
- mindsdb/integrations/handlers/informix_handler/informix_handler.py +5 -18
- mindsdb/integrations/handlers/lightfm_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/lightwood_handler/requirements.txt +4 -4
- mindsdb/integrations/handlers/maxdb_handler/maxdb_handler.py +22 -28
- mindsdb/integrations/handlers/monetdb_handler/monetdb_handler.py +3 -7
- mindsdb/integrations/handlers/mongodb_handler/mongodb_handler.py +53 -67
- mindsdb/integrations/handlers/mongodb_handler/requirements.txt +1 -0
- mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_ast.py +43 -68
- mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_parser.py +17 -25
- mindsdb/{api/mongo/utilities → integrations/handlers/mongodb_handler/utils}/mongodb_query.py +10 -16
- mindsdb/integrations/handlers/mongodb_handler/utils/mongodb_render.py +43 -69
- mindsdb/integrations/handlers/tpot_handler/requirements.txt +1 -1
- mindsdb/integrations/libs/base.py +1 -1
- mindsdb/integrations/libs/llm/config.py +15 -0
- mindsdb/integrations/libs/llm/utils.py +15 -0
- mindsdb/interfaces/agents/constants.py +1 -0
- mindsdb/interfaces/agents/langchain_agent.py +4 -0
- mindsdb/interfaces/agents/providers.py +20 -0
- mindsdb/interfaces/knowledge_base/controller.py +25 -7
- mindsdb/utilities/config.py +15 -158
- mindsdb/utilities/log.py +0 -25
- mindsdb/utilities/render/sqlalchemy_render.py +7 -1
- mindsdb/utilities/starters.py +0 -39
- {mindsdb-25.8.3.0.dist-info → mindsdb-25.9.1.1.dist-info}/METADATA +269 -267
- {mindsdb-25.8.3.0.dist-info → mindsdb-25.9.1.1.dist-info}/RECORD +62 -105
- mindsdb/api/a2a/__main__.py +0 -144
- mindsdb/api/a2a/run_a2a.py +0 -86
- mindsdb/api/common/check_auth.py +0 -42
- mindsdb/api/http/gunicorn_wrapper.py +0 -17
- mindsdb/api/mcp/start.py +0 -205
- mindsdb/api/mongo/__init__.py +0 -0
- mindsdb/api/mongo/classes/__init__.py +0 -5
- mindsdb/api/mongo/classes/query_sql.py +0 -19
- mindsdb/api/mongo/classes/responder.py +0 -45
- mindsdb/api/mongo/classes/responder_collection.py +0 -34
- mindsdb/api/mongo/classes/scram.py +0 -86
- mindsdb/api/mongo/classes/session.py +0 -23
- mindsdb/api/mongo/functions/__init__.py +0 -19
- mindsdb/api/mongo/responders/__init__.py +0 -73
- mindsdb/api/mongo/responders/add_shard.py +0 -13
- mindsdb/api/mongo/responders/aggregate.py +0 -90
- mindsdb/api/mongo/responders/buildinfo.py +0 -17
- mindsdb/api/mongo/responders/coll_stats.py +0 -63
- mindsdb/api/mongo/responders/company_id.py +0 -25
- mindsdb/api/mongo/responders/connection_status.py +0 -22
- mindsdb/api/mongo/responders/count.py +0 -21
- mindsdb/api/mongo/responders/db_stats.py +0 -32
- mindsdb/api/mongo/responders/delete.py +0 -105
- mindsdb/api/mongo/responders/describe.py +0 -23
- mindsdb/api/mongo/responders/end_sessions.py +0 -13
- mindsdb/api/mongo/responders/find.py +0 -175
- mindsdb/api/mongo/responders/get_cmd_line_opts.py +0 -18
- mindsdb/api/mongo/responders/get_free_monitoring_status.py +0 -14
- mindsdb/api/mongo/responders/get_parameter.py +0 -23
- mindsdb/api/mongo/responders/getlog.py +0 -14
- mindsdb/api/mongo/responders/host_info.py +0 -28
- mindsdb/api/mongo/responders/insert.py +0 -270
- mindsdb/api/mongo/responders/is_master.py +0 -20
- mindsdb/api/mongo/responders/is_master_lower.py +0 -13
- mindsdb/api/mongo/responders/list_collections.py +0 -55
- mindsdb/api/mongo/responders/list_databases.py +0 -37
- mindsdb/api/mongo/responders/list_indexes.py +0 -22
- mindsdb/api/mongo/responders/ping.py +0 -13
- mindsdb/api/mongo/responders/recv_chunk_start.py +0 -13
- mindsdb/api/mongo/responders/replsetgetstatus.py +0 -13
- mindsdb/api/mongo/responders/sasl_continue.py +0 -34
- mindsdb/api/mongo/responders/sasl_start.py +0 -33
- mindsdb/api/mongo/responders/update_range_deletions.py +0 -12
- mindsdb/api/mongo/responders/whatsmyuri.py +0 -18
- mindsdb/api/mongo/server.py +0 -388
- mindsdb/api/mongo/start.py +0 -15
- mindsdb/api/mongo/utilities/__init__.py +0 -0
- {mindsdb-25.8.3.0.dist-info → mindsdb-25.9.1.1.dist-info}/WHEEL +0 -0
- {mindsdb-25.8.3.0.dist-info → mindsdb-25.9.1.1.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.8.3.0.dist-info → mindsdb-25.9.1.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
2
|
+
from starlette.responses import JSONResponse
|
|
3
|
+
from starlette.requests import Request
|
|
4
|
+
from http import HTTPStatus
|
|
5
|
+
from typing import Optional
|
|
6
|
+
import secrets
|
|
7
|
+
import hmac
|
|
8
|
+
import hashlib
|
|
9
|
+
import os
|
|
10
|
+
import traceback
|
|
11
|
+
|
|
12
|
+
from mindsdb.utilities import log
|
|
13
|
+
from mindsdb.utilities.config import config
|
|
14
|
+
|
|
15
|
+
logger = log.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
SECRET_KEY = os.environ.get("AUTH_SECRET_KEY") or secrets.token_urlsafe(32)
|
|
18
|
+
# We store token (fingerprints) in memory, which means everyone is logged out if the process restarts
|
|
19
|
+
TOKENS = []
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_pat_fingerprint(token: str) -> str:
|
|
23
|
+
"""Hash the token with HMAC-SHA256 using secret_key as pepper."""
|
|
24
|
+
return hmac.new(SECRET_KEY.encode(), token.encode(), hashlib.sha256).hexdigest()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def generate_pat() -> str:
|
|
28
|
+
logger.debug("Generating new auth token")
|
|
29
|
+
token = "pat_" + secrets.token_urlsafe(32)
|
|
30
|
+
TOKENS.append(get_pat_fingerprint(token))
|
|
31
|
+
return token
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def verify_pat(raw_token: str) -> bool:
|
|
35
|
+
"""Verify if the raw_token matches a stored fingerprint.
|
|
36
|
+
Returns token_id if valid, None if not.
|
|
37
|
+
"""
|
|
38
|
+
if not raw_token:
|
|
39
|
+
return False
|
|
40
|
+
fp = get_pat_fingerprint(raw_token)
|
|
41
|
+
for stored_fp in TOKENS:
|
|
42
|
+
if hmac.compare_digest(fp, stored_fp):
|
|
43
|
+
return True
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def revoke_pat(raw_token: str) -> bool:
|
|
48
|
+
"""Revoke raw_token from active tokens"""
|
|
49
|
+
if not raw_token:
|
|
50
|
+
return False
|
|
51
|
+
fp = get_pat_fingerprint(raw_token)
|
|
52
|
+
for stored_fp in TOKENS:
|
|
53
|
+
if hmac.compare_digest(fp, stored_fp):
|
|
54
|
+
TOKENS.remove(stored_fp)
|
|
55
|
+
return True
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class PATAuthMiddleware(BaseHTTPMiddleware):
|
|
60
|
+
def _extract_bearer(self, request: Request) -> Optional[str]:
|
|
61
|
+
h = request.headers.get("Authorization")
|
|
62
|
+
if not h or not h.startswith("Bearer "):
|
|
63
|
+
return None
|
|
64
|
+
return h.split(" ", 1)[1].strip() or None
|
|
65
|
+
|
|
66
|
+
async def dispatch(self, request: Request, call_next):
|
|
67
|
+
if config.get("auth", {}).get("http_auth_enabled", False) is False:
|
|
68
|
+
return await call_next(request)
|
|
69
|
+
|
|
70
|
+
token = self._extract_bearer(request)
|
|
71
|
+
if not token or not verify_pat(token):
|
|
72
|
+
return JSONResponse({"detail": "Unauthorized"}, status_code=HTTPStatus.UNAUTHORIZED)
|
|
73
|
+
|
|
74
|
+
request.state.user = config["auth"].get("username")
|
|
75
|
+
return await call_next(request)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# Used by mysql and postgres protocols
|
|
79
|
+
def check_auth(username, password, scramble_func, salt, company_id, config):
|
|
80
|
+
try:
|
|
81
|
+
hardcoded_user = config["auth"].get("username")
|
|
82
|
+
hardcoded_password = config["auth"].get("password")
|
|
83
|
+
if hardcoded_password is None:
|
|
84
|
+
hardcoded_password = ""
|
|
85
|
+
hardcoded_password_hash = scramble_func(hardcoded_password, salt)
|
|
86
|
+
hardcoded_password = hardcoded_password.encode()
|
|
87
|
+
|
|
88
|
+
if password is None:
|
|
89
|
+
password = ""
|
|
90
|
+
if isinstance(password, str):
|
|
91
|
+
password = password.encode()
|
|
92
|
+
|
|
93
|
+
if username != hardcoded_user:
|
|
94
|
+
logger.warning(f"Check auth, user={username}: user mismatch")
|
|
95
|
+
return {"success": False}
|
|
96
|
+
|
|
97
|
+
if password != hardcoded_password and password != hardcoded_password_hash:
|
|
98
|
+
logger.warning(f"check auth, user={username}: password mismatch")
|
|
99
|
+
return {"success": False}
|
|
100
|
+
|
|
101
|
+
logger.info(f"Check auth, user={username}: Ok")
|
|
102
|
+
return {"success": True, "username": username}
|
|
103
|
+
except Exception as e:
|
|
104
|
+
logger.error(f"Check auth, user={username}: ERROR")
|
|
105
|
+
logger.error(e)
|
|
106
|
+
logger.error(traceback.format_exc())
|
|
@@ -1,7 +1,33 @@
|
|
|
1
|
-
|
|
1
|
+
import re
|
|
2
|
+
from mindsdb_sql_parser.ast import Identifier, Function, Constant, BinaryOperation, Interval, ASTNode, UnaryOperation
|
|
2
3
|
|
|
3
4
|
|
|
4
|
-
|
|
5
|
+
# ---- helper -----
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def cast(node: ASTNode, typename: str) -> BinaryOperation:
|
|
9
|
+
return BinaryOperation("::", args=[node, Identifier(typename)])
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def date_part(node, part):
|
|
13
|
+
"""
|
|
14
|
+
Wrap element into DATE_PART function
|
|
15
|
+
|
|
16
|
+
Docs:
|
|
17
|
+
https://duckdb.org/docs/stable/sql/functions/date#date_partpart-date
|
|
18
|
+
"""
|
|
19
|
+
node.args = apply_nested_functions(node.args)
|
|
20
|
+
|
|
21
|
+
if len(node.args) != 1:
|
|
22
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
23
|
+
|
|
24
|
+
return Function("DATE_PART", args=[Constant(part), cast(node.args[0], "date")])
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ------------------------------
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def char_fn(node: Function) -> Function | None:
|
|
5
31
|
"""Replace MySQL's multy-arg CHAR call to chain of DuckDB's CHR calls
|
|
6
32
|
|
|
7
33
|
Example:
|
|
@@ -30,7 +56,7 @@ def adapt_char_fn(node: Function) -> Function | None:
|
|
|
30
56
|
return acc
|
|
31
57
|
|
|
32
58
|
|
|
33
|
-
def
|
|
59
|
+
def locate_fn(node: Function) -> Function | None:
|
|
34
60
|
"""Replace MySQL's LOCATE (or INSTR) call to DuckDB's STRPOS call
|
|
35
61
|
|
|
36
62
|
Example:
|
|
@@ -56,7 +82,7 @@ def adapt_locate_fn(node: Function) -> Function | None:
|
|
|
56
82
|
node.op = "strpos"
|
|
57
83
|
|
|
58
84
|
|
|
59
|
-
def
|
|
85
|
+
def unhex_fn(node: Function) -> None:
|
|
60
86
|
"""Check MySQL's UNHEX function call arguments to ensure they are strings,
|
|
61
87
|
because DuckDB's UNHEX accepts only string arguments, while MySQL's UNHEX can accept integer arguments.
|
|
62
88
|
NOTE: if return dataframe from duckdb then unhex values are array - this may be an issue
|
|
@@ -75,7 +101,7 @@ def adapt_unhex_fn(node: Function) -> None:
|
|
|
75
101
|
raise ValueError("MySQL UNHEX function argument must be a string")
|
|
76
102
|
|
|
77
103
|
|
|
78
|
-
def
|
|
104
|
+
def format_fn(node: Function) -> None:
|
|
79
105
|
"""Adapt MySQL's FORMAT function to DuckDB's FORMAT function
|
|
80
106
|
|
|
81
107
|
Example:
|
|
@@ -113,7 +139,7 @@ def adapt_format_fn(node: Function) -> None:
|
|
|
113
139
|
node.args[0] = Constant(f"{{:,.{decimal_places}f}}")
|
|
114
140
|
|
|
115
141
|
|
|
116
|
-
def
|
|
142
|
+
def sha2_fn(node: Function) -> None:
|
|
117
143
|
"""Adapt MySQL's SHA2 function to DuckDB's SHA256 function
|
|
118
144
|
|
|
119
145
|
Example:
|
|
@@ -134,7 +160,7 @@ def adapt_sha2_fn(node: Function) -> None:
|
|
|
134
160
|
node.args = [node.args[0]]
|
|
135
161
|
|
|
136
162
|
|
|
137
|
-
def
|
|
163
|
+
def length_fn(node: Function) -> None:
|
|
138
164
|
"""Adapt MySQL's LENGTH function to DuckDB's STRLEN function
|
|
139
165
|
NOTE: duckdb also have LENGTH, therefore it can not be used
|
|
140
166
|
|
|
@@ -150,7 +176,7 @@ def adapt_length_fn(node: Function) -> None:
|
|
|
150
176
|
node.op = "strlen"
|
|
151
177
|
|
|
152
178
|
|
|
153
|
-
def
|
|
179
|
+
def regexp_substr_fn(node: Function) -> None:
|
|
154
180
|
"""Adapt MySQL's REGEXP_SUBSTR function to DuckDB's REGEXP_EXTRACT function
|
|
155
181
|
|
|
156
182
|
Example:
|
|
@@ -177,7 +203,7 @@ def adapt_regexp_substr_fn(node: Function) -> None:
|
|
|
177
203
|
node.op = "regexp_extract"
|
|
178
204
|
|
|
179
205
|
|
|
180
|
-
def
|
|
206
|
+
def substring_index_fn(node: Function) -> BinaryOperation | Function:
|
|
181
207
|
"""Adapt MySQL's SUBSTRING_INDEX function to DuckDB's SPLIT_PART function
|
|
182
208
|
|
|
183
209
|
Example:
|
|
@@ -210,7 +236,7 @@ def adapt_substring_index_fn(node: Function) -> BinaryOperation | Function:
|
|
|
210
236
|
return acc
|
|
211
237
|
|
|
212
238
|
|
|
213
|
-
def
|
|
239
|
+
def curtime_fn(node: Function) -> BinaryOperation:
|
|
214
240
|
"""Adapt MySQL's CURTIME function to DuckDB's GET_CURRENT_TIME function.
|
|
215
241
|
To get the same type as MySQL's CURTIME function, we need to cast the result to time type.
|
|
216
242
|
|
|
@@ -223,10 +249,10 @@ def adapt_curtime_fn(node: Function) -> BinaryOperation:
|
|
|
223
249
|
Returns:
|
|
224
250
|
BinaryOperation: Binary operation node
|
|
225
251
|
"""
|
|
226
|
-
return
|
|
252
|
+
return cast(Function(op="get_current_time", args=[]), "time")
|
|
227
253
|
|
|
228
254
|
|
|
229
|
-
def
|
|
255
|
+
def timestampdiff_fn(node: Function) -> None:
|
|
230
256
|
"""Adapt MySQL's TIMESTAMPDIFF function to DuckDB's DATE_DIFF function
|
|
231
257
|
NOTE: Looks like cast string args to timestamp works in most cases, but there may be some exceptions.
|
|
232
258
|
|
|
@@ -241,11 +267,11 @@ def adapt_timestampdiff_fn(node: Function) -> None:
|
|
|
241
267
|
"""
|
|
242
268
|
node.op = "date_diff"
|
|
243
269
|
node.args[0] = Constant(node.args[0].parts[0])
|
|
244
|
-
node.args[1] =
|
|
245
|
-
node.args[2] =
|
|
270
|
+
node.args[1] = cast(node.args[1], "timestamp")
|
|
271
|
+
node.args[2] = cast(node.args[2], "timestamp")
|
|
246
272
|
|
|
247
273
|
|
|
248
|
-
def
|
|
274
|
+
def extract_fn(node: Function) -> None:
|
|
249
275
|
"""Adapt MySQL's EXTRACT function to DuckDB's EXTRACT function
|
|
250
276
|
TODO: multi-part args, like YEAR_MONTH, is not supported yet
|
|
251
277
|
NOTE: Looks like adding 'timestamp' works in most cases, but there may be some exceptions.
|
|
@@ -259,6 +285,428 @@ def adapt_extract_fn(node: Function) -> None:
|
|
|
259
285
|
Returns:
|
|
260
286
|
None
|
|
261
287
|
"""
|
|
262
|
-
|
|
263
|
-
if
|
|
264
|
-
node.
|
|
288
|
+
part = node.args[0].parts[0]
|
|
289
|
+
if part.upper() == "YEAR_MONTH":
|
|
290
|
+
node.args = apply_nested_functions([node.from_arg, Constant("%Y%m")])
|
|
291
|
+
node.from_arg = None
|
|
292
|
+
date_format_fn(node)
|
|
293
|
+
return cast(node, "int")
|
|
294
|
+
elif part.upper() == "DAY_MINUTE":
|
|
295
|
+
node.args = apply_nested_functions([node.from_arg, Constant("%e%H%i")])
|
|
296
|
+
node.from_arg = None
|
|
297
|
+
date_format_fn(node)
|
|
298
|
+
return cast(node, "int")
|
|
299
|
+
else:
|
|
300
|
+
node.args[0] = Constant(part)
|
|
301
|
+
if not isinstance(node.from_arg, Identifier):
|
|
302
|
+
node.from_arg = cast(node.from_arg, "timestamp")
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def get_format_fn(node: Function) -> Constant:
|
|
306
|
+
"""
|
|
307
|
+
Replace function with a constant according to table:
|
|
308
|
+
Important! The parameters can be only constants.
|
|
309
|
+
|
|
310
|
+
Example: GET_FORMAT(DATE, 'USA') => '%m.%d.%Y'
|
|
311
|
+
|
|
312
|
+
Docs:
|
|
313
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_get-format
|
|
314
|
+
"""
|
|
315
|
+
|
|
316
|
+
if len(node.args) != 2:
|
|
317
|
+
raise ValueError("MySQL GET_FORMAT supports only 2 arguments")
|
|
318
|
+
|
|
319
|
+
arg1, arg2 = node.args
|
|
320
|
+
|
|
321
|
+
if not isinstance(arg1, Identifier) and len(arg1.parts) != 1:
|
|
322
|
+
raise ValueError(f"Unknown type: {arg1}")
|
|
323
|
+
|
|
324
|
+
if not isinstance(arg2, Constant):
|
|
325
|
+
raise ValueError(f"Unknown format name: {arg2}")
|
|
326
|
+
|
|
327
|
+
match arg1.parts[0].upper(), arg2.value.upper():
|
|
328
|
+
case "DATE", "USA":
|
|
329
|
+
value = "%m.%d.%Y"
|
|
330
|
+
case "DATE", "JIS":
|
|
331
|
+
value = "%Y-%m-%d"
|
|
332
|
+
case "DATE", "ISO":
|
|
333
|
+
value = "%Y-%m-%d"
|
|
334
|
+
case "DATE", "EUR":
|
|
335
|
+
value = "%d.%m.%Y"
|
|
336
|
+
case "DATE", "INTERNAL":
|
|
337
|
+
value = "%Y%m%d"
|
|
338
|
+
|
|
339
|
+
case "DATETIME", "USA":
|
|
340
|
+
value = "%Y-%m-%d %H.%i.%s"
|
|
341
|
+
case "DATETIME", "JIS":
|
|
342
|
+
value = "%Y-%m-%d %H:%i:%s"
|
|
343
|
+
case "DATETIME", "ISO":
|
|
344
|
+
value = "%Y-%m-%d %H:%i:%s"
|
|
345
|
+
case "DATETIME", "EUR":
|
|
346
|
+
value = "%Y-%m-%d %H.%i.%s"
|
|
347
|
+
case "DATETIME", "INTERNAL":
|
|
348
|
+
value = "%Y%m%d%H%i%s"
|
|
349
|
+
|
|
350
|
+
case "TIME", "USA":
|
|
351
|
+
value = "%h:%i:%s %p"
|
|
352
|
+
case "TIME", "JIS":
|
|
353
|
+
value = "%H:%i:%s"
|
|
354
|
+
case "TIME", "ISO":
|
|
355
|
+
value = "%H:%i:%s"
|
|
356
|
+
case "TIME", "EUR":
|
|
357
|
+
value = "%H.%i.%s"
|
|
358
|
+
case "TIME", "INTERNAL":
|
|
359
|
+
value = "%H%i%s"
|
|
360
|
+
|
|
361
|
+
case _:
|
|
362
|
+
value = ""
|
|
363
|
+
|
|
364
|
+
return Constant(value)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def date_format_fn(node: Function):
|
|
368
|
+
"""
|
|
369
|
+
Adapt to strftime function and convert keys in format string.
|
|
370
|
+
|
|
371
|
+
DATE_FORMAT('2009-10-04 22:23:00', '%W %M %Y')
|
|
372
|
+
=>
|
|
373
|
+
strftime('2009-10-04 22:23:00'::datetime, '%A %B %Y')
|
|
374
|
+
|
|
375
|
+
Docs:
|
|
376
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_date-format
|
|
377
|
+
https://duckdb.org/docs/stable/sql/functions/timestamp.html#strftimetimestamp-format
|
|
378
|
+
https://duckdb.org/docs/stable/sql/functions/dateformat.html#format-specifiers
|
|
379
|
+
"""
|
|
380
|
+
specifiers_map = {
|
|
381
|
+
"%c": "%-m", # Month, numeric (0..12) -> Month as decimal
|
|
382
|
+
"%D": "%-d", # Day with English suffix -> Day as decimal (no suffix in DuckDB)
|
|
383
|
+
"%e": "%-d", # Day of month (0..31) -> Day as decimal
|
|
384
|
+
"%h": "%I", # Hour (01..12)
|
|
385
|
+
"%i": "%M", # Minutes
|
|
386
|
+
"%j": "%j", # Day of year
|
|
387
|
+
"%k": "%-H", # Hour (0..23) -> Hour as decimal
|
|
388
|
+
"%l": "%-I", # Hour (1..12) -> Hour as decimal
|
|
389
|
+
"%M": "%B", # Month name -> Full month name
|
|
390
|
+
"%r": "%I:%M:%S %p", # Time, 12-hour
|
|
391
|
+
"%s": "%S", # Seconds
|
|
392
|
+
"%T": "%X", # Time, 24-hour
|
|
393
|
+
"%u": "%V", # Week, mode 1, Monday is first day, can be wrong in the edges of year
|
|
394
|
+
"%v": "%V", # Week, mode 3, Monday is first day
|
|
395
|
+
"%V": "%U", # Week, mode 2, Sunday is first day, can be wrong in the edges of year
|
|
396
|
+
"%W": "%A", # Weekday name -> Full weekday name
|
|
397
|
+
"%X": "%G", # Year for week
|
|
398
|
+
"%x": "%G", # Year for week
|
|
399
|
+
}
|
|
400
|
+
node.op = "strftime"
|
|
401
|
+
|
|
402
|
+
node.args = apply_nested_functions(node.args)
|
|
403
|
+
|
|
404
|
+
if len(node.args) != 2 or not isinstance(node.args[1], Constant):
|
|
405
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
406
|
+
|
|
407
|
+
def repl_f(match):
|
|
408
|
+
specifier = match.group()
|
|
409
|
+
return specifiers_map.get(specifier, specifier)
|
|
410
|
+
|
|
411
|
+
# adapt format string
|
|
412
|
+
node.args[1].value = re.sub(r"%[a-zA-Z]", repl_f, node.args[1].value)
|
|
413
|
+
|
|
414
|
+
# add type casting
|
|
415
|
+
node.args[0] = cast(node.args[0], "timestamp")
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def from_unixtime_fn(node):
|
|
419
|
+
"""
|
|
420
|
+
Adapt to make_timestamp function
|
|
421
|
+
FROM_UNIXTIME(1447430881) => make_timestamp((1447430881::int8 *1000000))
|
|
422
|
+
|
|
423
|
+
Docs:
|
|
424
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_from-unixtime
|
|
425
|
+
https://duckdb.org/docs/stable/sql/functions/timestamp#make_timestampmicroseconds
|
|
426
|
+
"""
|
|
427
|
+
|
|
428
|
+
if len(node.args) != 1:
|
|
429
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
430
|
+
|
|
431
|
+
node.op = "make_timestamp"
|
|
432
|
+
|
|
433
|
+
node.args[0] = BinaryOperation("*", args=[cast(node.args[0], "int8"), Constant(1_000_000)])
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def from_days_fn(node):
|
|
437
|
+
"""
|
|
438
|
+
Adapt to converting days to interval and adding to first day of the 0 year:
|
|
439
|
+
FROM_DAYS(735669) => '0000-01-01'::date + (735669 * INTERVAL '1 day')
|
|
440
|
+
|
|
441
|
+
Docs:
|
|
442
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_from-days
|
|
443
|
+
"""
|
|
444
|
+
node.args = apply_nested_functions(node.args)
|
|
445
|
+
|
|
446
|
+
if len(node.args) != 1:
|
|
447
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
448
|
+
|
|
449
|
+
return BinaryOperation(
|
|
450
|
+
op="+",
|
|
451
|
+
args=[
|
|
452
|
+
BinaryOperation("::", args=[Constant("0000-01-01"), Identifier("date")]),
|
|
453
|
+
BinaryOperation("*", args=[node.args[0], Interval("1 day")]),
|
|
454
|
+
],
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def dayofyear_fn(node):
|
|
459
|
+
"""
|
|
460
|
+
Addapt to DATE_PART:
|
|
461
|
+
DAYOFYEAR('2007-02-03') => DATE_PART('doy', '2007-02-03'::date)
|
|
462
|
+
|
|
463
|
+
Docs:
|
|
464
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_dayofyear
|
|
465
|
+
"""
|
|
466
|
+
|
|
467
|
+
return date_part(node, "doy")
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def dayofweek_fn(node):
|
|
471
|
+
"""
|
|
472
|
+
Addapt to DATE_PART:
|
|
473
|
+
DAYOFWEEK('2007-02-03'); => DATE_PART('dow', '2007-02-03'::date) + 1;
|
|
474
|
+
|
|
475
|
+
Docs:
|
|
476
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_dayofweek
|
|
477
|
+
"""
|
|
478
|
+
return BinaryOperation("+", args=[date_part(node, "dow"), Constant(1)])
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def dayofmonth_fn(node):
|
|
482
|
+
"""
|
|
483
|
+
Addapt to DATE_PART:
|
|
484
|
+
DAYOFMONTH('2007-02-03') => DATE_PART('day', '2007-02-03'::date)
|
|
485
|
+
|
|
486
|
+
Docs:
|
|
487
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_dayofmonth
|
|
488
|
+
"""
|
|
489
|
+
|
|
490
|
+
return date_part(node, "day")
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def dayname_fn(node):
|
|
494
|
+
"""
|
|
495
|
+
Use the same function with type casting
|
|
496
|
+
DAYNAME('2007-02-03') => DAYNAME('2007-02-03'::date)
|
|
497
|
+
|
|
498
|
+
Docs:
|
|
499
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_dayname
|
|
500
|
+
"""
|
|
501
|
+
if len(node.args) != 1:
|
|
502
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
503
|
+
|
|
504
|
+
node.args[0] = cast(node.args[0], "date")
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def curdate_fn(node):
|
|
508
|
+
"""
|
|
509
|
+
Replace the name of the function
|
|
510
|
+
CURDATE() => CURRENT_DATE()
|
|
511
|
+
|
|
512
|
+
Docs:
|
|
513
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_curdate
|
|
514
|
+
https://duckdb.org/docs/stable/sql/functions/date.html#current_date
|
|
515
|
+
"""
|
|
516
|
+
node.op = "CURRENT_DATE"
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def datediff_fn(node):
|
|
520
|
+
"""
|
|
521
|
+
Change argument's order and cast to date:
|
|
522
|
+
DATEDIFF('2007-12-31 23:59:59','2007-11-30') => datediff('day',DATE '2007-11-30', DATE '2007-12-31 23:59:59')
|
|
523
|
+
|
|
524
|
+
Docs:
|
|
525
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_datediff
|
|
526
|
+
https://duckdb.org/docs/stable/sql/functions/date#date_diffpart-startdate-enddate
|
|
527
|
+
|
|
528
|
+
"""
|
|
529
|
+
if len(node.args) != 2:
|
|
530
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
531
|
+
|
|
532
|
+
arg1, arg2 = node.args
|
|
533
|
+
node.args = [Constant("day"), cast(arg2, "date"), cast(arg1, "date")]
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def adddate_fn(node):
|
|
537
|
+
"""
|
|
538
|
+
Replace the name of the function and add type casting
|
|
539
|
+
Important! The second parameter can be only interval (not count of days).
|
|
540
|
+
SELECT ADDDATE('2008-01-02', INTERVAL 31 DAY) => SELECT DATE_ADD('2008-01-02'::date, INTERVAL 31 DAY)
|
|
541
|
+
|
|
542
|
+
Docs:
|
|
543
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_adddate
|
|
544
|
+
https://duckdb.org/docs/stable/sql/functions/date.html#date_adddate-interval
|
|
545
|
+
"""
|
|
546
|
+
if len(node.args) != 2:
|
|
547
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
548
|
+
|
|
549
|
+
node.op = "DATE_ADD"
|
|
550
|
+
node.args[0] = cast(node.args[0], "timestamp")
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def date_sub_fn(node):
|
|
554
|
+
"""
|
|
555
|
+
Use DATE_ADD with negative interval
|
|
556
|
+
SELECT DATE_SUB('1998-01-02', INTERVAL 31 DAY) => select DATE_ADD('1998-01-02'::date, -INTERVAL 31 DAY)
|
|
557
|
+
|
|
558
|
+
Docs:
|
|
559
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_date-add
|
|
560
|
+
https://duckdb.org/docs/stable/sql/functions/date.html#date_adddate-interval
|
|
561
|
+
"""
|
|
562
|
+
if len(node.args) != 2:
|
|
563
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
564
|
+
|
|
565
|
+
node.op = "DATE_ADD"
|
|
566
|
+
node.args[0] = cast(node.args[0], "timestamp")
|
|
567
|
+
node.args[1] = UnaryOperation("-", args=[node.args[1]])
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def addtime_fn(node):
|
|
571
|
+
"""
|
|
572
|
+
Convert second parameter into interval.
|
|
573
|
+
Important!
|
|
574
|
+
- The second parameter can be only a constant.
|
|
575
|
+
- The first parameter can be only date/datetime (not just time)
|
|
576
|
+
|
|
577
|
+
ADDTIME('2007-12-31', '1 1:1:1.2')
|
|
578
|
+
=>
|
|
579
|
+
DATE_ADD('2007-12-31'::timestamp, INTERVAL '1 day 1 hour 1 minute 1.2 second')
|
|
580
|
+
|
|
581
|
+
Docs:
|
|
582
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_addtime
|
|
583
|
+
https://duckdb.org/docs/stable/sql/functions/date.html#date_adddate-interval
|
|
584
|
+
"""
|
|
585
|
+
node.args = apply_nested_functions(node.args)
|
|
586
|
+
|
|
587
|
+
if len(node.args) != 2:
|
|
588
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
589
|
+
|
|
590
|
+
interval = node.args[1]
|
|
591
|
+
if not isinstance(interval, Constant) or not isinstance(interval.value, str):
|
|
592
|
+
raise ValueError(f"The second argument have to be string: {node.args[1]}")
|
|
593
|
+
|
|
594
|
+
pattern = r"^(?:(\d+)\s+)?(?:(\d+):)?(?:(\d+):)?(\d+)(?:\.(\d+))?$"
|
|
595
|
+
|
|
596
|
+
match = re.match(pattern, interval.value)
|
|
597
|
+
if not match:
|
|
598
|
+
raise ValueError(f"Invalid MySQL time format: {interval.value}")
|
|
599
|
+
|
|
600
|
+
# Extract components
|
|
601
|
+
days, hours, minutes, seconds, fractional = match.groups()
|
|
602
|
+
# Build interval string
|
|
603
|
+
parts = []
|
|
604
|
+
if days and int(days) > 0:
|
|
605
|
+
parts.append(f"{days} day")
|
|
606
|
+
|
|
607
|
+
if hours and int(hours) > 0:
|
|
608
|
+
parts.append(f"{int(hours)} hour")
|
|
609
|
+
|
|
610
|
+
if minutes and int(minutes) > 0:
|
|
611
|
+
parts.append(f"{int(minutes)} minute")
|
|
612
|
+
|
|
613
|
+
seconds = int(seconds) if seconds else 0
|
|
614
|
+
fractional = float(f"0.{fractional}") if fractional else 0.0
|
|
615
|
+
total_seconds = seconds + fractional
|
|
616
|
+
if total_seconds > 0:
|
|
617
|
+
seconds_str = str(total_seconds).rstrip("0").rstrip(".")
|
|
618
|
+
parts.append(f"{seconds_str} second")
|
|
619
|
+
|
|
620
|
+
# If all components are zero, return 0 seconds
|
|
621
|
+
if not parts:
|
|
622
|
+
interval_str = "0 second"
|
|
623
|
+
else:
|
|
624
|
+
interval_str = " ".join(parts)
|
|
625
|
+
|
|
626
|
+
return Function(
|
|
627
|
+
"DATE_ADD",
|
|
628
|
+
args=[
|
|
629
|
+
cast(node.args[0], "timestamp"),
|
|
630
|
+
Interval(interval_str),
|
|
631
|
+
],
|
|
632
|
+
)
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def convert_tz_fn(node):
|
|
636
|
+
"""
|
|
637
|
+
Concatenate timezone to first argument and cast it as timestamptz. Then use `timezone` function
|
|
638
|
+
Important! Duckdb doesn't recognize timezones in digital formats: +10:00
|
|
639
|
+
|
|
640
|
+
CONVERT_TZ('2004-01-01 12:00:00','GMT','MET')
|
|
641
|
+
=>
|
|
642
|
+
timezone('MET', ('2004-01-01 12:00:00' || ' ' || 'GMT')::timestamptz);
|
|
643
|
+
|
|
644
|
+
Docs:
|
|
645
|
+
https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_convert-tz
|
|
646
|
+
https://duckdb.org/docs/stable/sql/functions/timestamptz.html#timezonetext-timestamp
|
|
647
|
+
"""
|
|
648
|
+
node.args = apply_nested_functions(node.args)
|
|
649
|
+
|
|
650
|
+
if len(node.args) != 3:
|
|
651
|
+
raise ValueError(f"Wrong arguments: {node.args}")
|
|
652
|
+
|
|
653
|
+
date, tzfrom, tzto = node.args
|
|
654
|
+
|
|
655
|
+
# concatenate tz name: date || ' ' || tzfrom
|
|
656
|
+
tzdate = BinaryOperation("||", args=[BinaryOperation("||", args=[date, Constant(" ")]), tzfrom], parentheses=True)
|
|
657
|
+
|
|
658
|
+
return Function(
|
|
659
|
+
"timezone",
|
|
660
|
+
args=[
|
|
661
|
+
tzto,
|
|
662
|
+
cast(tzdate, "timestamptz"),
|
|
663
|
+
],
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def apply_nested_functions(args):
|
|
668
|
+
args2 = []
|
|
669
|
+
for arg in args:
|
|
670
|
+
if isinstance(arg, Function):
|
|
671
|
+
fnc = mysql_to_duckdb_fnc(arg)
|
|
672
|
+
if args2 is not None:
|
|
673
|
+
arg = fnc(arg)
|
|
674
|
+
args2.append(arg)
|
|
675
|
+
return args2
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def mysql_to_duckdb_fnc(node):
|
|
679
|
+
fnc_name = node.op.lower()
|
|
680
|
+
|
|
681
|
+
mysql_to_duck_fn_map = {
|
|
682
|
+
"char": char_fn,
|
|
683
|
+
"locate": locate_fn,
|
|
684
|
+
"insrt": locate_fn,
|
|
685
|
+
"unhex": unhex_fn,
|
|
686
|
+
"format": format_fn,
|
|
687
|
+
"sha2": sha2_fn,
|
|
688
|
+
"length": length_fn,
|
|
689
|
+
"regexp_substr": regexp_substr_fn,
|
|
690
|
+
"substring_index": substring_index_fn,
|
|
691
|
+
"curtime": curtime_fn,
|
|
692
|
+
"timestampdiff": timestampdiff_fn,
|
|
693
|
+
"extract": extract_fn,
|
|
694
|
+
"get_format": get_format_fn,
|
|
695
|
+
"date_format": date_format_fn,
|
|
696
|
+
"from_unixtime": from_unixtime_fn,
|
|
697
|
+
"from_days": from_days_fn,
|
|
698
|
+
"dayofyear": dayofyear_fn,
|
|
699
|
+
"dayofweek": dayofweek_fn,
|
|
700
|
+
"day": dayofmonth_fn,
|
|
701
|
+
"dayofmonth": dayofmonth_fn,
|
|
702
|
+
"dayname": dayname_fn,
|
|
703
|
+
"curdate": curdate_fn,
|
|
704
|
+
"datediff": datediff_fn,
|
|
705
|
+
"adddate": adddate_fn,
|
|
706
|
+
"date_sub": date_sub_fn,
|
|
707
|
+
"date_add": adddate_fn,
|
|
708
|
+
"addtime": addtime_fn,
|
|
709
|
+
"convert_tz": convert_tz_fn,
|
|
710
|
+
}
|
|
711
|
+
if fnc_name in mysql_to_duck_fn_map:
|
|
712
|
+
return mysql_to_duck_fn_map[fnc_name]
|