MindsDB 25.4.5.0__py3-none-any.whl → 25.5.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +107 -125
- mindsdb/api/executor/command_executor.py +2 -1
- mindsdb/api/executor/datahub/datanodes/information_schema_datanode.py +8 -0
- mindsdb/api/executor/datahub/datanodes/system_tables.py +10 -13
- mindsdb/api/executor/planner/query_planner.py +4 -1
- mindsdb/api/executor/sql_query/steps/apply_predictor_step.py +2 -1
- mindsdb/api/http/initialize.py +20 -3
- mindsdb/api/http/namespaces/analysis.py +14 -1
- mindsdb/api/http/namespaces/tree.py +1 -1
- mindsdb/api/http/start.py +7 -2
- mindsdb/api/mysql/mysql_proxy/mysql_proxy.py +4 -8
- mindsdb/api/mysql/mysql_proxy/utilities/exceptions.py +0 -4
- mindsdb/api/postgres/postgres_proxy/postgres_packets/postgres_message_formats.py +2 -2
- mindsdb/integrations/handlers/bigquery_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/chromadb_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/gmail_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/google_analytics_handler/requirements.txt +2 -1
- mindsdb/integrations/handlers/google_books_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/google_calendar_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/google_content_shopping_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/google_fit_handler/requirements.txt +2 -0
- mindsdb/integrations/handlers/google_search_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/jira_handler/jira_handler.archived.py +75 -0
- mindsdb/integrations/handlers/jira_handler/jira_handler.py +113 -38
- mindsdb/integrations/handlers/jira_handler/jira_tables.py +229 -0
- mindsdb/integrations/handlers/jira_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/lightfm_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/lightwood_handler/lightwood_handler.py +0 -2
- mindsdb/integrations/handlers/lightwood_handler/requirements.txt +4 -4
- mindsdb/integrations/handlers/lindorm_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/ms_one_drive_handler/requirements.txt +2 -0
- mindsdb/integrations/handlers/ms_teams_handler/requirements.txt +3 -1
- mindsdb/integrations/handlers/openai_handler/openai_handler.py +5 -4
- mindsdb/integrations/handlers/snowflake_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/vertex_handler/requirements.txt +1 -0
- mindsdb/integrations/handlers/youtube_handler/requirements.txt +1 -0
- mindsdb/integrations/utilities/files/file_reader.py +5 -2
- mindsdb/interfaces/agents/constants.py +14 -2
- mindsdb/interfaces/agents/langchain_agent.py +2 -4
- mindsdb/interfaces/database/projects.py +1 -7
- mindsdb/interfaces/functions/controller.py +11 -14
- mindsdb/interfaces/functions/to_markdown.py +9 -124
- mindsdb/interfaces/knowledge_base/controller.py +22 -19
- mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py +28 -5
- mindsdb/interfaces/knowledge_base/utils.py +10 -15
- mindsdb/interfaces/model/model_controller.py +0 -2
- mindsdb/interfaces/skills/sql_agent.py +33 -11
- mindsdb/migrations/migrate.py +0 -2
- mindsdb/utilities/config.py +3 -2
- mindsdb/utilities/context.py +1 -1
- mindsdb/utilities/functions.py +0 -36
- mindsdb/utilities/langfuse.py +19 -10
- mindsdb/utilities/otel/__init__.py +9 -193
- mindsdb/utilities/otel/metric_handlers/__init__.py +5 -1
- mindsdb/utilities/otel/prepare.py +198 -0
- mindsdb/utilities/sql.py +83 -0
- {mindsdb-25.4.5.0.dist-info → mindsdb-25.5.3.0.dist-info}/METADATA +663 -596
- {mindsdb-25.4.5.0.dist-info → mindsdb-25.5.3.0.dist-info}/RECORD +62 -57
- {mindsdb-25.4.5.0.dist-info → mindsdb-25.5.3.0.dist-info}/WHEEL +1 -1
- mindsdb/api/mysql/mysql_proxy/classes/sql_statement_parser.py +0 -151
- {mindsdb-25.4.5.0.dist-info → mindsdb-25.5.3.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.4.5.0.dist-info → mindsdb-25.5.3.0.dist-info}/top_level.txt +0 -0
mindsdb/__about__.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
__title__ = 'MindsDB'
|
|
2
2
|
__package_name__ = 'mindsdb'
|
|
3
|
-
__version__ = '25.
|
|
3
|
+
__version__ = '25.5.3.0'
|
|
4
4
|
__description__ = "MindsDB's AI SQL Server enables developers to build AI tools that need access to real-time data to perform their tasks"
|
|
5
5
|
__email__ = "jorge@mindsdb.com"
|
|
6
6
|
__author__ = 'MindsDB Inc'
|
mindsdb/__main__.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
gc.disable()
|
|
1
3
|
import os
|
|
2
4
|
import sys
|
|
3
5
|
import time
|
|
@@ -12,7 +14,7 @@ from enum import Enum
|
|
|
12
14
|
from dataclasses import dataclass, field
|
|
13
15
|
from typing import Callable, Optional, Tuple, List
|
|
14
16
|
|
|
15
|
-
from
|
|
17
|
+
from sqlalchemy import func
|
|
16
18
|
from sqlalchemy.orm.attributes import flag_modified
|
|
17
19
|
|
|
18
20
|
from mindsdb.utilities import log
|
|
@@ -22,17 +24,12 @@ logger.debug("Starting MindsDB...")
|
|
|
22
24
|
|
|
23
25
|
from mindsdb.__about__ import __version__ as mindsdb_version
|
|
24
26
|
from mindsdb.utilities.config import config
|
|
25
|
-
from mindsdb.utilities.exception import EntityNotExistsError
|
|
26
27
|
from mindsdb.utilities.starters import (
|
|
27
|
-
start_http, start_mysql, start_mongo, start_postgres, start_ml_task_queue,
|
|
28
|
-
start_mcp, start_litellm
|
|
28
|
+
start_http, start_mysql, start_mongo, start_postgres, start_ml_task_queue,
|
|
29
|
+
start_scheduler, start_tasks, start_mcp, start_litellm
|
|
29
30
|
)
|
|
30
31
|
from mindsdb.utilities.ps import is_pid_listen_port, get_child_pids
|
|
31
|
-
from mindsdb.utilities.functions import get_versions_where_predictors_become_obsolete
|
|
32
|
-
from mindsdb.interfaces.database.integrations import integration_controller
|
|
33
|
-
from mindsdb.interfaces.database.projects import ProjectController
|
|
34
32
|
import mindsdb.interfaces.storage.db as db
|
|
35
|
-
from mindsdb.integrations.utilities.install import install_dependencies
|
|
36
33
|
from mindsdb.utilities.fs import clean_process_marks, clean_unlinked_process_marks
|
|
37
34
|
from mindsdb.utilities.context import context as ctx
|
|
38
35
|
from mindsdb.utilities.auth import register_oauth_client, get_aws_meta_data
|
|
@@ -47,6 +44,8 @@ try:
|
|
|
47
44
|
except RuntimeError:
|
|
48
45
|
logger.info('Torch multiprocessing context already set, ignoring...')
|
|
49
46
|
|
|
47
|
+
gc.enable()
|
|
48
|
+
|
|
50
49
|
_stop_event = threading.Event()
|
|
51
50
|
|
|
52
51
|
|
|
@@ -213,7 +212,96 @@ def do_clean_process_marks():
|
|
|
213
212
|
set_error_model_status_by_pids(unexisting_pids)
|
|
214
213
|
|
|
215
214
|
|
|
215
|
+
def create_permanent_integrations():
|
|
216
|
+
"""
|
|
217
|
+
Create permanent integrations, for now only the 'files' integration.
|
|
218
|
+
NOTE: this is intentional to avoid importing integration_controller
|
|
219
|
+
"""
|
|
220
|
+
integration_name = 'files'
|
|
221
|
+
existing = db.session.query(db.Integration).filter_by(name=integration_name, company_id=None).first()
|
|
222
|
+
if existing is None:
|
|
223
|
+
integration_record = db.Integration(
|
|
224
|
+
name=integration_name,
|
|
225
|
+
data={},
|
|
226
|
+
engine=integration_name,
|
|
227
|
+
company_id=None,
|
|
228
|
+
)
|
|
229
|
+
db.session.add(integration_record)
|
|
230
|
+
try:
|
|
231
|
+
db.session.commit()
|
|
232
|
+
except Exception as e:
|
|
233
|
+
logger.error(f"Failed to commit permanent integration {integration_name}: {e}")
|
|
234
|
+
db.session.rollback()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def validate_default_project() -> None:
|
|
238
|
+
"""Handle 'default_project' config option.
|
|
239
|
+
Project with the name specified in 'default_project' must exists and be marked with
|
|
240
|
+
'is_default' metadata. If it is not possible, then terminate the process with error.
|
|
241
|
+
Note: this can be done using 'project_controller', but we want to save init time and used RAM.
|
|
242
|
+
"""
|
|
243
|
+
new_default_project_name = config.get('default_project')
|
|
244
|
+
logger.debug(f"Checking if default project {new_default_project_name} exists")
|
|
245
|
+
filter_company_id = ctx.company_id if ctx.company_id is not None else 0
|
|
246
|
+
|
|
247
|
+
current_default_project: db.Project | None = (
|
|
248
|
+
db.Project.query.filter(
|
|
249
|
+
db.Project.company_id == filter_company_id,
|
|
250
|
+
db.Project.metadata_['is_default'].as_boolean() == True # noqa
|
|
251
|
+
).first()
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
if current_default_project is None:
|
|
255
|
+
# Legacy: If the default project does not exist, mark the new one as default.
|
|
256
|
+
existing_project = db.Project.query.filter(
|
|
257
|
+
db.Project.company_id == filter_company_id,
|
|
258
|
+
func.lower(db.Project.name) == func.lower(new_default_project_name)
|
|
259
|
+
).first()
|
|
260
|
+
if existing_project is None:
|
|
261
|
+
logger.critical(f"A project with the name '{new_default_project_name}' does not exist")
|
|
262
|
+
sys.exit(1)
|
|
263
|
+
|
|
264
|
+
existing_project.metadata_ = {'is_default': True}
|
|
265
|
+
flag_modified(existing_project, 'metadata_')
|
|
266
|
+
db.session.commit()
|
|
267
|
+
elif current_default_project.name != new_default_project_name:
|
|
268
|
+
# If the default project exists, but the name is different, update the name.
|
|
269
|
+
existing_project = db.Project.query.filter(
|
|
270
|
+
db.Project.company_id == filter_company_id,
|
|
271
|
+
func.lower(db.Project.name) == func.lower(new_default_project_name)
|
|
272
|
+
).first()
|
|
273
|
+
if existing_project is not None:
|
|
274
|
+
logger.critical(f"A project with the name '{new_default_project_name}' already exists")
|
|
275
|
+
sys.exit(1)
|
|
276
|
+
current_default_project.name = new_default_project_name
|
|
277
|
+
db.session.commit()
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def start_process(trunc_process_data: TrunkProcessData) -> None:
|
|
281
|
+
"""Start a process.
|
|
282
|
+
|
|
283
|
+
Args:
|
|
284
|
+
trunc_process_data (TrunkProcessData): The data of the process to start.
|
|
285
|
+
"""
|
|
286
|
+
mp_ctx = mp.get_context("spawn")
|
|
287
|
+
logger.info(f"{trunc_process_data.name} API: starting...")
|
|
288
|
+
try:
|
|
289
|
+
trunc_process_data.process = mp_ctx.Process(
|
|
290
|
+
target=trunc_process_data.entrypoint,
|
|
291
|
+
args=trunc_process_data.args,
|
|
292
|
+
name=trunc_process_data.name
|
|
293
|
+
)
|
|
294
|
+
trunc_process_data.process.start()
|
|
295
|
+
except Exception as e:
|
|
296
|
+
logger.error(
|
|
297
|
+
f"Failed to start {trunc_process_data.name} API with exception {e}\n{traceback.format_exc()}"
|
|
298
|
+
)
|
|
299
|
+
close_api_gracefully(trunc_processes_struct)
|
|
300
|
+
raise e
|
|
301
|
+
|
|
302
|
+
|
|
216
303
|
if __name__ == '__main__':
|
|
304
|
+
mp.freeze_support()
|
|
217
305
|
# warn if less than 1Gb of free RAM
|
|
218
306
|
if psutil.virtual_memory().available < (1 << 30):
|
|
219
307
|
logger.warning(
|
|
@@ -270,7 +358,6 @@ if __name__ == '__main__':
|
|
|
270
358
|
pass
|
|
271
359
|
|
|
272
360
|
db.init()
|
|
273
|
-
mp.freeze_support()
|
|
274
361
|
|
|
275
362
|
environment = config["environment"]
|
|
276
363
|
if environment == "aws_marketplace":
|
|
@@ -287,52 +374,6 @@ if __name__ == '__main__':
|
|
|
287
374
|
except Exception:
|
|
288
375
|
pass
|
|
289
376
|
|
|
290
|
-
is_cloud = config.is_cloud
|
|
291
|
-
|
|
292
|
-
if not is_cloud:
|
|
293
|
-
logger.debug("Applying database migrations")
|
|
294
|
-
try:
|
|
295
|
-
from mindsdb.migrations import migrate
|
|
296
|
-
migrate.migrate_to_head()
|
|
297
|
-
except Exception as e:
|
|
298
|
-
logger.error(f"Error! Something went wrong during DB migrations: {e}")
|
|
299
|
-
|
|
300
|
-
logger.debug(f"Checking if default project {config.get('default_project')} exists")
|
|
301
|
-
project_controller = ProjectController()
|
|
302
|
-
|
|
303
|
-
try:
|
|
304
|
-
current_default_project = project_controller.get(is_default=True)
|
|
305
|
-
except EntityNotExistsError:
|
|
306
|
-
# In previous versions, the default project could be deleted. This is no longer possible.
|
|
307
|
-
current_default_project = None
|
|
308
|
-
|
|
309
|
-
if current_default_project:
|
|
310
|
-
if current_default_project.record.name != config.get('default_project'):
|
|
311
|
-
try:
|
|
312
|
-
project_controller.get(name=config.get('default_project'))
|
|
313
|
-
log.critical(f"A project with the name '{config.get('default_project')}' already exists")
|
|
314
|
-
sys.exit(1)
|
|
315
|
-
except EntityNotExistsError:
|
|
316
|
-
pass
|
|
317
|
-
project_controller.update(current_default_project.record.id, new_name=config.get('default_project'))
|
|
318
|
-
|
|
319
|
-
# Legacy: If the default project does not exist, mark the new one as default.
|
|
320
|
-
else:
|
|
321
|
-
try:
|
|
322
|
-
project_controller.get(name=config.get('default_project'))
|
|
323
|
-
except EntityNotExistsError:
|
|
324
|
-
log.critical(
|
|
325
|
-
f"A project with the name '{config.get('default_project')}' does not exist"
|
|
326
|
-
)
|
|
327
|
-
raise
|
|
328
|
-
|
|
329
|
-
project_controller.update(
|
|
330
|
-
name=config.get('default_project'),
|
|
331
|
-
new_metadata={
|
|
332
|
-
"is_default": True
|
|
333
|
-
}
|
|
334
|
-
)
|
|
335
|
-
|
|
336
377
|
apis = os.getenv('MINDSDB_APIS') or config.cmd_args.api
|
|
337
378
|
|
|
338
379
|
if apis is None: # If "--api" option is not specified, start the default APIs
|
|
@@ -342,27 +383,6 @@ if __name__ == '__main__':
|
|
|
342
383
|
else: # The user has provided a list of APIs to start
|
|
343
384
|
api_arr = [TrunkProcessEnum(name) for name in apis.split(',')]
|
|
344
385
|
|
|
345
|
-
if config.cmd_args.install_handlers is not None:
|
|
346
|
-
handlers_list = [s.strip() for s in config.cmd_args.install_handlers.split(",")]
|
|
347
|
-
# import_meta = handler_meta.get('import', {})
|
|
348
|
-
for handler_name, handler_meta in integration_controller.get_handlers_import_status().items():
|
|
349
|
-
if handler_name not in handlers_list:
|
|
350
|
-
continue
|
|
351
|
-
import_meta = handler_meta.get("import", {})
|
|
352
|
-
if import_meta.get("success") is True:
|
|
353
|
-
logger.info(f"{'{0: <18}'.format(handler_name)} - already installed")
|
|
354
|
-
continue
|
|
355
|
-
result = install_dependencies(import_meta.get("dependencies", []))
|
|
356
|
-
if result.get("success") is True:
|
|
357
|
-
logger.info(
|
|
358
|
-
f"{'{0: <18}'.format(handler_name)} - successfully installed"
|
|
359
|
-
)
|
|
360
|
-
else:
|
|
361
|
-
logger.info(
|
|
362
|
-
f"{'{0: <18}'.format(handler_name)} - error during dependencies installation: {result.get('error_message', 'unknown error')}"
|
|
363
|
-
)
|
|
364
|
-
sys.exit(0)
|
|
365
|
-
|
|
366
386
|
logger.info(f"Version: {mindsdb_version}")
|
|
367
387
|
logger.info(f"Configuration file: {config.config_path or 'absent'}")
|
|
368
388
|
logger.info(f"Storage path: {config.paths['root']}")
|
|
@@ -370,42 +390,22 @@ if __name__ == '__main__':
|
|
|
370
390
|
logger.debug(f"System config: {config.auto_config}")
|
|
371
391
|
logger.debug(f"Env config: {config.env_config}")
|
|
372
392
|
|
|
393
|
+
is_cloud = config.is_cloud
|
|
373
394
|
unexisting_pids = clean_unlinked_process_marks()
|
|
374
395
|
if not is_cloud:
|
|
396
|
+
logger.debug("Applying database migrations")
|
|
397
|
+
try:
|
|
398
|
+
from mindsdb.migrations import migrate
|
|
399
|
+
migrate.migrate_to_head()
|
|
400
|
+
except Exception as e:
|
|
401
|
+
logger.error(f"Error! Something went wrong during DB migrations: {e}")
|
|
402
|
+
|
|
403
|
+
validate_default_project()
|
|
404
|
+
|
|
375
405
|
if len(unexisting_pids) > 0:
|
|
376
406
|
set_error_model_status_by_pids(unexisting_pids)
|
|
377
407
|
set_error_model_status_for_unfinished()
|
|
378
|
-
|
|
379
|
-
integration_controller.create_permanent_integrations()
|
|
380
|
-
|
|
381
|
-
# region Mark old predictors as outdated
|
|
382
|
-
is_modified = False
|
|
383
|
-
predictor_records = (
|
|
384
|
-
db.session.query(db.Predictor)
|
|
385
|
-
.filter(db.Predictor.deleted_at.is_(None))
|
|
386
|
-
.all()
|
|
387
|
-
)
|
|
388
|
-
if len(predictor_records) > 0:
|
|
389
|
-
(
|
|
390
|
-
sucess,
|
|
391
|
-
compatible_versions,
|
|
392
|
-
) = get_versions_where_predictors_become_obsolete()
|
|
393
|
-
if sucess is True:
|
|
394
|
-
compatible_versions = [version.parse(x) for x in compatible_versions]
|
|
395
|
-
mindsdb_version_parsed = version.parse(mindsdb_version)
|
|
396
|
-
compatible_versions = [x for x in compatible_versions if x <= mindsdb_version_parsed]
|
|
397
|
-
if len(compatible_versions) > 0:
|
|
398
|
-
last_compatible_version = compatible_versions[-1]
|
|
399
|
-
for predictor_record in predictor_records:
|
|
400
|
-
if (
|
|
401
|
-
isinstance(predictor_record.mindsdb_version, str)
|
|
402
|
-
and version.parse(predictor_record.mindsdb_version) < last_compatible_version
|
|
403
|
-
):
|
|
404
|
-
predictor_record.update_status = "available"
|
|
405
|
-
is_modified = True
|
|
406
|
-
if is_modified is True:
|
|
407
|
-
db.session.commit()
|
|
408
|
-
# endregion
|
|
408
|
+
create_permanent_integrations()
|
|
409
409
|
|
|
410
410
|
clean_process_marks()
|
|
411
411
|
|
|
@@ -503,24 +503,6 @@ if __name__ == '__main__':
|
|
|
503
503
|
if config.cmd_args.ml_task_queue_consumer is True:
|
|
504
504
|
trunc_processes_struct[TrunkProcessEnum.ML_TASK_QUEUE].need_to_run = True
|
|
505
505
|
|
|
506
|
-
def start_process(trunc_process_data):
|
|
507
|
-
# TODO this 'ctx' is eclipsing 'context' class imported as 'ctx'
|
|
508
|
-
ctx = mp.get_context("spawn")
|
|
509
|
-
logger.info(f"{trunc_process_data.name} API: starting...")
|
|
510
|
-
try:
|
|
511
|
-
trunc_process_data.process = ctx.Process(
|
|
512
|
-
target=trunc_process_data.entrypoint,
|
|
513
|
-
args=trunc_process_data.args,
|
|
514
|
-
name=trunc_process_data.name
|
|
515
|
-
)
|
|
516
|
-
trunc_process_data.process.start()
|
|
517
|
-
except Exception as e:
|
|
518
|
-
logger.error(
|
|
519
|
-
f"Failed to start {trunc_process_data.name} API with exception {e}\n{traceback.format_exc()}"
|
|
520
|
-
)
|
|
521
|
-
close_api_gracefully(trunc_processes_struct)
|
|
522
|
-
raise e
|
|
523
|
-
|
|
524
506
|
for trunc_process_data in trunc_processes_struct.values():
|
|
525
507
|
if trunc_process_data.started is True or trunc_process_data.need_to_run is False:
|
|
526
508
|
continue
|
|
@@ -5,7 +5,6 @@ from typing import Optional
|
|
|
5
5
|
from functools import reduce
|
|
6
6
|
|
|
7
7
|
import pandas as pd
|
|
8
|
-
from mindsdb_evaluator.accuracy.general import evaluate_accuracy
|
|
9
8
|
from mindsdb_sql_parser import parse_sql
|
|
10
9
|
from mindsdb_sql_parser.ast import (
|
|
11
10
|
Alter,
|
|
@@ -814,6 +813,8 @@ class ExecuteCommands:
|
|
|
814
813
|
return ExecuteAnswer()
|
|
815
814
|
|
|
816
815
|
def answer_evaluate_metric(self, statement, database_name):
|
|
816
|
+
# heavy import, so we do it here on-demand
|
|
817
|
+
from mindsdb_evaluator.accuracy.general import evaluate_accuracy
|
|
817
818
|
try:
|
|
818
819
|
sqlquery = SQLQuery(statement.data, session=self.session, database=database_name)
|
|
819
820
|
except Exception as e:
|
|
@@ -22,6 +22,8 @@ from .mindsdb_tables import (
|
|
|
22
22
|
ModelsTable, DatabasesTable, MLEnginesTable, HandlersTable, JobsTable, QueriesTable,
|
|
23
23
|
ChatbotsTable, KBTable, SkillsTable, AgentsTable, ViewsTable, TriggersTable)
|
|
24
24
|
|
|
25
|
+
from mindsdb.api.executor.datahub.classes.tables_row import TablesRow
|
|
26
|
+
|
|
25
27
|
|
|
26
28
|
logger = log.getLogger(__name__)
|
|
27
29
|
|
|
@@ -166,6 +168,12 @@ class InformationSchemaDataNode(DataNode):
|
|
|
166
168
|
return [x.lower() for x in projects]
|
|
167
169
|
|
|
168
170
|
def get_tables(self):
|
|
171
|
+
return [
|
|
172
|
+
TablesRow(TABLE_NAME=name)
|
|
173
|
+
for name in self.tables.keys()
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
def get_tree_tables(self):
|
|
169
177
|
return {
|
|
170
178
|
name: table
|
|
171
179
|
for name, table in self.tables.items()
|
|
@@ -311,20 +311,17 @@ class ColumnsTable(Table):
|
|
|
311
311
|
result = []
|
|
312
312
|
for db_name in databases:
|
|
313
313
|
tables = {}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
314
|
+
|
|
315
|
+
dn = inf_schema.get(db_name)
|
|
316
|
+
if dn is None:
|
|
317
|
+
continue
|
|
318
|
+
|
|
319
|
+
if tables_names is None:
|
|
320
|
+
list_tables = [t.TABLE_NAME for t in dn.get_tables()]
|
|
319
321
|
else:
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
if tables_names is None:
|
|
325
|
-
tables_names = [t.TABLE_NAME for t in dn.get_tables()]
|
|
326
|
-
for table_name in tables_names:
|
|
327
|
-
tables[table_name] = dn.get_table_columns_df(table_name)
|
|
322
|
+
list_tables = tables_names
|
|
323
|
+
for table_name in list_tables:
|
|
324
|
+
tables[table_name] = dn.get_table_columns_df(table_name)
|
|
328
325
|
|
|
329
326
|
for table_name, table_columns_df in tables.items():
|
|
330
327
|
for _, row in table_columns_df.iterrows():
|
|
@@ -29,6 +29,9 @@ from mindsdb.utilities.config import config
|
|
|
29
29
|
|
|
30
30
|
default_project = config.get('default_project')
|
|
31
31
|
|
|
32
|
+
# This includes built-in MindsDB SQL functions and functions to be executed via DuckDB consistently.
|
|
33
|
+
MINDSDB_SQL_FUNCTIONS = {'llm', 'to_markdown', 'hash'}
|
|
34
|
+
|
|
32
35
|
|
|
33
36
|
class QueryPlanner:
|
|
34
37
|
|
|
@@ -237,7 +240,7 @@ class QueryPlanner:
|
|
|
237
240
|
|
|
238
241
|
def find_objects(node, is_table, **kwargs):
|
|
239
242
|
if isinstance(node, Function):
|
|
240
|
-
if node.namespace is not None or node.op.lower() in
|
|
243
|
+
if node.namespace is not None or node.op.lower() in MINDSDB_SQL_FUNCTIONS:
|
|
241
244
|
user_functions.append(node)
|
|
242
245
|
|
|
243
246
|
if is_table:
|
|
@@ -2,7 +2,6 @@ import datetime as dt
|
|
|
2
2
|
import re
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
|
-
import dateparser
|
|
6
5
|
|
|
7
6
|
from mindsdb_sql_parser.ast import (
|
|
8
7
|
BinaryOperation,
|
|
@@ -283,6 +282,8 @@ class ApplyPredictorStepCall(ApplyPredictorBaseCall):
|
|
|
283
282
|
# Use dateparser as fallback and infer format
|
|
284
283
|
try:
|
|
285
284
|
# Parse the first sample to get its format
|
|
285
|
+
# The import is heavy, so we do it here on-demand
|
|
286
|
+
import dateparser
|
|
286
287
|
parsed_date = dateparser.parse(samples[0])
|
|
287
288
|
if parsed_date is None:
|
|
288
289
|
raise ValueError("Could not parse date")
|
mindsdb/api/http/initialize.py
CHANGED
|
@@ -50,12 +50,29 @@ from mindsdb.utilities.json_encoder import CustomJSONProvider
|
|
|
50
50
|
from mindsdb.utilities.ps import is_pid_listen_port, wait_func_is_true
|
|
51
51
|
from mindsdb.utilities.sentry import sentry_sdk # noqa: F401
|
|
52
52
|
from mindsdb.utilities.otel import trace # noqa: F401
|
|
53
|
-
from opentelemetry.instrumentation.flask import FlaskInstrumentor # noqa: F401
|
|
54
|
-
from opentelemetry.instrumentation.requests import RequestsInstrumentor # noqa: F401
|
|
55
53
|
|
|
56
54
|
logger = log.getLogger(__name__)
|
|
57
55
|
|
|
58
56
|
|
|
57
|
+
class _NoOpFlaskInstrumentor:
|
|
58
|
+
def instrument_app(self, app):
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class _NoOpRequestsInstrumentor:
|
|
63
|
+
def instrument(self):
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
from opentelemetry.instrumentation.flask import FlaskInstrumentor
|
|
69
|
+
from opentelemetry.instrumentation.requests import RequestsInstrumentor
|
|
70
|
+
except ImportError:
|
|
71
|
+
logger.debug("OpenTelemetry is not avaiable. Please run `pip install -r requirements/requirements-opentelemetry.txt` to use it.")
|
|
72
|
+
FlaskInstrumentor = _NoOpFlaskInstrumentor
|
|
73
|
+
RequestsInstrumentor = _NoOpRequestsInstrumentor
|
|
74
|
+
|
|
75
|
+
|
|
59
76
|
class Swagger_Api(Api):
|
|
60
77
|
"""
|
|
61
78
|
This is a modification of the base Flask Restplus Api class due to the issue described here
|
|
@@ -376,7 +393,7 @@ def initialize_flask(config, init_static_thread, no_studio):
|
|
|
376
393
|
app = Flask(__name__, **kwargs)
|
|
377
394
|
init_metrics(app)
|
|
378
395
|
|
|
379
|
-
# Instrument Flask app
|
|
396
|
+
# Instrument Flask app and requests using either real or no-op instrumentors
|
|
380
397
|
FlaskInstrumentor().instrument_app(app)
|
|
381
398
|
RequestsInstrumentor().instrument()
|
|
382
399
|
|
|
@@ -79,7 +79,14 @@ class QueryAnalysis(Resource):
|
|
|
79
79
|
|
|
80
80
|
column_names = [x["name"] for x in result.columns]
|
|
81
81
|
df = DataFrame(result.data, columns=column_names)
|
|
82
|
-
|
|
82
|
+
try:
|
|
83
|
+
analysis = analyze_df(df)
|
|
84
|
+
except ImportError:
|
|
85
|
+
return {
|
|
86
|
+
'analysis': {},
|
|
87
|
+
'timestamp': time.time(),
|
|
88
|
+
'error': 'To use this feature, please install the "dataprep_ml" package.'
|
|
89
|
+
}
|
|
83
90
|
|
|
84
91
|
query_tables = [
|
|
85
92
|
table.to_string() for table in get_query_tables(ast)
|
|
@@ -107,6 +114,12 @@ class DataAnalysis(Resource):
|
|
|
107
114
|
try:
|
|
108
115
|
analysis = analyze_df(DataFrame(data, columns=column_names))
|
|
109
116
|
return {"analysis": analysis, "timestamp": time.time()}
|
|
117
|
+
except ImportError:
|
|
118
|
+
return {
|
|
119
|
+
'analysis': {},
|
|
120
|
+
'timestamp': timestamp,
|
|
121
|
+
'error': 'To use this feature, please install the "dataprep_ml" package.'
|
|
122
|
+
}
|
|
110
123
|
except Exception as e:
|
|
111
124
|
# Don't want analysis exceptions to show up on UI.
|
|
112
125
|
# TODO: Fix analysis so it doesn't throw exceptions at all.
|
|
@@ -94,7 +94,7 @@ class GetLeaf(Resource):
|
|
|
94
94
|
} for key, val in schemas.items()]
|
|
95
95
|
elif db['type'] == 'system':
|
|
96
96
|
system_db = ca.database_controller.get_system_db(db_name)
|
|
97
|
-
tables = system_db.
|
|
97
|
+
tables = system_db.get_tree_tables()
|
|
98
98
|
tables = [{
|
|
99
99
|
'name': table.name,
|
|
100
100
|
'class': table.kind,
|
mindsdb/api/http/start.py
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
gc.disable()
|
|
3
|
+
|
|
1
4
|
from flask import Flask
|
|
2
5
|
from waitress import serve
|
|
3
6
|
|
|
@@ -8,6 +11,8 @@ from mindsdb.utilities.config import config
|
|
|
8
11
|
from mindsdb.utilities.functions import init_lexer_parsers
|
|
9
12
|
from mindsdb.integrations.libs.ml_exec_base import process_cache
|
|
10
13
|
|
|
14
|
+
gc.enable()
|
|
15
|
+
|
|
11
16
|
logger = log.getLogger(__name__)
|
|
12
17
|
|
|
13
18
|
|
|
@@ -26,7 +31,7 @@ def start(verbose, no_studio, app: Flask = None):
|
|
|
26
31
|
process_cache.init()
|
|
27
32
|
|
|
28
33
|
if server_type == "waitress":
|
|
29
|
-
logger.debug("Serving HTTP app with waitress
|
|
34
|
+
logger.debug("Serving HTTP app with waitress...")
|
|
30
35
|
serve(
|
|
31
36
|
app,
|
|
32
37
|
host='*' if host in ('', '0.0.0.0') else host,
|
|
@@ -34,7 +39,7 @@ def start(verbose, no_studio, app: Flask = None):
|
|
|
34
39
|
**server_config
|
|
35
40
|
)
|
|
36
41
|
elif server_type == "flask":
|
|
37
|
-
logger.debug("Serving HTTP app with flask
|
|
42
|
+
logger.debug("Serving HTTP app with flask...")
|
|
38
43
|
# that will 'disable access' log in console
|
|
39
44
|
|
|
40
45
|
app.run(debug=False, port=port, host=host, **server_config)
|
|
@@ -32,13 +32,11 @@ from mindsdb.api.mysql.mysql_proxy.data_types.mysql_datum import Datum
|
|
|
32
32
|
|
|
33
33
|
import mindsdb.utilities.hooks as hooks
|
|
34
34
|
import mindsdb.utilities.profiler as profiler
|
|
35
|
+
from mindsdb.utilities.sql import clear_sql
|
|
35
36
|
from mindsdb.api.mysql.mysql_proxy.classes.client_capabilities import ClentCapabilities
|
|
36
37
|
from mindsdb.api.mysql.mysql_proxy.classes.server_capabilities import (
|
|
37
38
|
server_capabilities,
|
|
38
39
|
)
|
|
39
|
-
from mindsdb.api.mysql.mysql_proxy.classes.sql_statement_parser import (
|
|
40
|
-
SqlStatementParser,
|
|
41
|
-
)
|
|
42
40
|
from mindsdb.api.executor.controllers import SessionController
|
|
43
41
|
from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packet import Packet
|
|
44
42
|
from mindsdb.api.mysql.mysql_proxy.data_types.mysql_packets import (
|
|
@@ -85,7 +83,7 @@ from mindsdb.api.mysql.mysql_proxy.utilities.lightwood_dtype import dtype
|
|
|
85
83
|
from mindsdb.utilities import log
|
|
86
84
|
from mindsdb.utilities.config import config
|
|
87
85
|
from mindsdb.utilities.context import context as ctx
|
|
88
|
-
from mindsdb.utilities.otel
|
|
86
|
+
from mindsdb.utilities.otel import increment_otel_query_request_counter
|
|
89
87
|
from mindsdb.utilities.wizards import make_ssl_cert
|
|
90
88
|
|
|
91
89
|
logger = log.getLogger(__name__)
|
|
@@ -560,9 +558,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
560
558
|
)
|
|
561
559
|
|
|
562
560
|
# Increment the counter and include metadata in attributes
|
|
563
|
-
|
|
564
|
-
query_request_counter = get_query_request_counter()
|
|
565
|
-
query_request_counter.add(1, metadata)
|
|
561
|
+
increment_otel_query_request_counter(ctx.get_metadata(query=sql))
|
|
566
562
|
|
|
567
563
|
return resp
|
|
568
564
|
|
|
@@ -741,7 +737,7 @@ class MysqlProxy(SocketServer.BaseRequestHandler):
|
|
|
741
737
|
try:
|
|
742
738
|
if p.type.value == COMMANDS.COM_QUERY:
|
|
743
739
|
sql = self.decode_utf(p.sql.value)
|
|
744
|
-
sql =
|
|
740
|
+
sql = clear_sql(sql)
|
|
745
741
|
logger.debug(f'Incoming query: {sql}')
|
|
746
742
|
profiler.set_meta(
|
|
747
743
|
query=sql, api="mysql", environment=config.get("environment")
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from typing import BinaryIO, Sequence, Dict, Type
|
|
2
2
|
|
|
3
|
-
from mindsdb.api.mysql.mysql_proxy.classes.sql_statement_parser import SqlStatementParser
|
|
4
3
|
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_fields import PostgresField
|
|
5
4
|
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message import PostgresMessage
|
|
6
5
|
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_identifiers import \
|
|
@@ -8,6 +7,7 @@ from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_message_ident
|
|
|
8
7
|
|
|
9
8
|
from mindsdb.api.postgres.postgres_proxy.postgres_packets.postgres_packets import PostgresPacketReader
|
|
10
9
|
from mindsdb.api.postgres.postgres_proxy.utilities import strip_null_byte
|
|
10
|
+
from mindsdb.utilities.sql import clear_sql
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
# All docstrings for Messages are taken from
|
|
@@ -507,7 +507,7 @@ class Query(PostgresMessage):
|
|
|
507
507
|
raise Exception(f'SQL contains non {encoding} values: {self.sql}')
|
|
508
508
|
# Remove null bytes from end of sql statement. This is important.
|
|
509
509
|
sql = strip_null_byte(sql)
|
|
510
|
-
sql =
|
|
510
|
+
sql = clear_sql(sql)
|
|
511
511
|
return sql
|
|
512
512
|
|
|
513
513
|
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
google-api-python-client
|
|
2
|
-
google-auth
|
|
2
|
+
google-auth
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
google-api-python-client
|
|
2
|
-
google-auth
|
|
2
|
+
google-auth
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
google-api-python-client
|
|
2
|
-
google-auth
|
|
2
|
+
google-auth
|