MindsDB 25.7.3.0__py3-none-any.whl → 25.7.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/api/a2a/common/server/server.py +16 -6
- mindsdb/api/executor/command_executor.py +206 -135
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +14 -3
- mindsdb/api/executor/planner/plan_join.py +3 -0
- mindsdb/api/executor/planner/plan_join_ts.py +117 -100
- mindsdb/api/executor/planner/query_planner.py +1 -0
- mindsdb/api/executor/sql_query/steps/apply_predictor_step.py +54 -85
- mindsdb/api/http/initialize.py +16 -43
- mindsdb/api/http/namespaces/agents.py +23 -20
- mindsdb/api/http/namespaces/chatbots.py +83 -120
- mindsdb/api/http/namespaces/file.py +1 -1
- mindsdb/api/http/namespaces/jobs.py +38 -60
- mindsdb/api/http/namespaces/tree.py +69 -61
- mindsdb/api/mcp/start.py +2 -0
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +3 -2
- mindsdb/integrations/handlers/autogluon_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/autosklearn_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/bigquery_handler/bigquery_handler.py +25 -5
- mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py +3 -3
- mindsdb/integrations/handlers/flaml_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/google_calendar_handler/google_calendar_tables.py +82 -73
- mindsdb/integrations/handlers/hubspot_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/langchain_handler/langchain_handler.py +83 -76
- mindsdb/integrations/handlers/lightwood_handler/requirements.txt +4 -4
- mindsdb/integrations/handlers/litellm_handler/litellm_handler.py +5 -2
- mindsdb/integrations/handlers/litellm_handler/settings.py +2 -1
- mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +106 -90
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +41 -39
- mindsdb/integrations/handlers/salesforce_handler/constants.py +208 -0
- mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py +141 -80
- mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py +0 -1
- mindsdb/integrations/handlers/tpot_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +32 -17
- mindsdb/integrations/handlers/web_handler/web_handler.py +19 -22
- mindsdb/integrations/libs/vectordatabase_handler.py +10 -1
- mindsdb/integrations/utilities/handler_utils.py +32 -12
- mindsdb/interfaces/agents/agents_controller.py +167 -108
- mindsdb/interfaces/agents/langchain_agent.py +10 -3
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +4 -4
- mindsdb/interfaces/database/database.py +38 -13
- mindsdb/interfaces/database/integrations.py +20 -5
- mindsdb/interfaces/database/projects.py +63 -16
- mindsdb/interfaces/database/views.py +86 -60
- mindsdb/interfaces/jobs/jobs_controller.py +103 -110
- mindsdb/interfaces/knowledge_base/controller.py +26 -5
- mindsdb/interfaces/knowledge_base/evaluate.py +2 -1
- mindsdb/interfaces/knowledge_base/executor.py +24 -0
- mindsdb/interfaces/query_context/context_controller.py +100 -133
- mindsdb/interfaces/skills/skills_controller.py +18 -6
- mindsdb/interfaces/storage/db.py +40 -6
- mindsdb/interfaces/variables/variables_controller.py +8 -15
- mindsdb/utilities/config.py +3 -3
- mindsdb/utilities/functions.py +72 -60
- mindsdb/utilities/log.py +38 -6
- mindsdb/utilities/ps.py +7 -7
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.7.4.0.dist-info}/METADATA +246 -247
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.7.4.0.dist-info}/RECORD +61 -60
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.7.4.0.dist-info}/WHEEL +0 -0
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.7.4.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.7.4.0.dist-info}/top_level.txt +0 -0
|
@@ -20,7 +20,7 @@ from mindsdb.utilities import log
|
|
|
20
20
|
|
|
21
21
|
from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
|
|
22
22
|
|
|
23
|
-
from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
|
|
23
|
+
from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
|
|
24
24
|
from .langchain_agent import get_llm_provider
|
|
25
25
|
|
|
26
26
|
logger = log.getLogger(__name__)
|
|
@@ -165,16 +165,21 @@ class AgentsController:
|
|
|
165
165
|
with one of keys is "name", and other is additional parameters for relationship agent<>skill
|
|
166
166
|
provider (str): The provider of the model
|
|
167
167
|
params (Dict[str, str]): Parameters to use when running the agent
|
|
168
|
+
data: Dict, data sources for an agent, keys:
|
|
169
|
+
- knowledge_bases: List of KBs to use
|
|
170
|
+
- tables: list of tables to use
|
|
171
|
+
model: Dict, parameters for the model to use
|
|
172
|
+
- provider: The provider of the model (e.g., 'openai', 'google')
|
|
173
|
+
- Other model-specific parameters like 'api_key', 'model_name', etc.
|
|
174
|
+
<provider>_api_key: API key for the provider (e.g., openai_api_key)
|
|
175
|
+
|
|
176
|
+
# Deprecated parameters:
|
|
168
177
|
database: The database to use for text2sql skills (default is 'mindsdb')
|
|
169
178
|
knowledge_base_database: The database to use for knowledge base queries (default is 'mindsdb')
|
|
170
179
|
include_tables: List of tables to include for text2sql skills
|
|
171
180
|
ignore_tables: List of tables to ignore for text2sql skills
|
|
172
181
|
include_knowledge_bases: List of knowledge bases to include for text2sql skills
|
|
173
182
|
ignore_knowledge_bases: List of knowledge bases to ignore for text2sql skills
|
|
174
|
-
<provider>_api_key: API key for the provider (e.g., openai_api_key)
|
|
175
|
-
data: Dict, data sources for an agent, keys:
|
|
176
|
-
- knowledge_bases: List of KBs to use (alternative to `include_knowledge_bases`)
|
|
177
|
-
- tables: list of tables to use (alternative to `include_tables`)
|
|
178
183
|
|
|
179
184
|
Returns:
|
|
180
185
|
agent (db.Agents): The created agent
|
|
@@ -223,48 +228,31 @@ class AgentsController:
|
|
|
223
228
|
# It will be picked up by get_api_key() in handler_utils.py
|
|
224
229
|
pass
|
|
225
230
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
params["database"] = database
|
|
231
|
+
depreciated_params = [
|
|
232
|
+
"database",
|
|
233
|
+
"knowledge_base_database",
|
|
234
|
+
"include_tables",
|
|
235
|
+
"ignore_tables",
|
|
236
|
+
"include_knowledge_bases",
|
|
237
|
+
"ignore_knowledge_bases",
|
|
238
|
+
]
|
|
239
|
+
if any(param in params for param in depreciated_params):
|
|
240
|
+
raise ValueError(
|
|
241
|
+
f"Parameters {', '.join(depreciated_params)} are deprecated. "
|
|
242
|
+
"Use 'data' parameter with 'tables' and 'knowledge_bases' keys instead."
|
|
243
|
+
)
|
|
240
244
|
|
|
245
|
+
include_tables = None
|
|
246
|
+
include_knowledge_bases = None
|
|
241
247
|
if "data" in params:
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
if include_tables is None:
|
|
245
|
-
include_tables = params["data"].get("tables")
|
|
246
|
-
|
|
247
|
-
if "knowledge_base_database" in params or include_knowledge_bases or ignore_knowledge_bases:
|
|
248
|
-
params["knowledge_base_database"] = knowledge_base_database
|
|
249
|
-
|
|
250
|
-
if include_tables is not None:
|
|
251
|
-
params["include_tables"] = include_tables
|
|
252
|
-
if ignore_tables is not None:
|
|
253
|
-
params["ignore_tables"] = ignore_tables
|
|
254
|
-
if include_knowledge_bases is not None:
|
|
255
|
-
params["include_knowledge_bases"] = include_knowledge_bases
|
|
256
|
-
if ignore_knowledge_bases is not None:
|
|
257
|
-
params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
248
|
+
include_knowledge_bases = params["data"].get("knowledge_bases")
|
|
249
|
+
include_tables = params["data"].get("tables")
|
|
258
250
|
|
|
259
251
|
# Convert string parameters to lists if needed
|
|
260
252
|
if isinstance(include_tables, str):
|
|
261
253
|
include_tables = [t.strip() for t in include_tables.split(",")]
|
|
262
|
-
if isinstance(ignore_tables, str):
|
|
263
|
-
ignore_tables = [t.strip() for t in ignore_tables.split(",")]
|
|
264
254
|
if isinstance(include_knowledge_bases, str):
|
|
265
255
|
include_knowledge_bases = [kb.strip() for kb in include_knowledge_bases.split(",")]
|
|
266
|
-
if isinstance(ignore_knowledge_bases, str):
|
|
267
|
-
ignore_knowledge_bases = [kb.strip() for kb in ignore_knowledge_bases.split(",")]
|
|
268
256
|
|
|
269
257
|
# Auto-create SQL skill if no skills are provided but include_tables or include_knowledge_bases params are provided
|
|
270
258
|
if not skills and (include_tables or include_knowledge_bases):
|
|
@@ -272,23 +260,15 @@ class AgentsController:
|
|
|
272
260
|
skill_name = f"{name}_sql_skill"
|
|
273
261
|
skill_params = {
|
|
274
262
|
"type": "sql",
|
|
275
|
-
"database": database,
|
|
276
263
|
"description": f"Auto-generated SQL skill for agent {name}",
|
|
277
264
|
}
|
|
278
265
|
|
|
279
|
-
# Add
|
|
266
|
+
# Add restrictions provided
|
|
280
267
|
if include_tables:
|
|
281
268
|
skill_params["include_tables"] = include_tables
|
|
282
|
-
if ignore_tables:
|
|
283
|
-
skill_params["ignore_tables"] = ignore_tables
|
|
284
|
-
|
|
285
|
-
# Add knowledge base parameters if provided
|
|
286
|
-
if knowledge_base_database:
|
|
287
|
-
skill_params["knowledge_base_database"] = knowledge_base_database
|
|
288
269
|
if include_knowledge_bases:
|
|
289
270
|
skill_params["include_knowledge_bases"] = include_knowledge_bases
|
|
290
|
-
|
|
291
|
-
skill_params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
271
|
+
|
|
292
272
|
try:
|
|
293
273
|
# Check if skill already exists
|
|
294
274
|
existing_skill = self.skills_controller.get_skill(skill_name, project_name)
|
|
@@ -340,44 +320,16 @@ class AgentsController:
|
|
|
340
320
|
db.session.rollback()
|
|
341
321
|
raise ValueError(f"Skill with name does not exist: {skill_name}")
|
|
342
322
|
|
|
343
|
-
if
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
for table in include_tables:
|
|
349
|
-
parts = table.split(".", 1)
|
|
350
|
-
database_table_map[parts[0]] = database_table_map.get(parts[0], []) + [parts[1]]
|
|
351
|
-
|
|
352
|
-
for database_name, table_names in database_table_map.items():
|
|
353
|
-
data_catalog_loader = DataCatalogLoader(
|
|
354
|
-
database_name=database_name, table_names=table_names
|
|
355
|
-
)
|
|
356
|
-
data_catalog_loader.load_metadata()
|
|
357
|
-
|
|
358
|
-
elif "database" in existing_skill.params:
|
|
359
|
-
data_catalog_loader = DataCatalogLoader(
|
|
360
|
-
database_name=existing_skill.params["database"],
|
|
361
|
-
table_names=parameters["tables"] if "tables" in parameters else None,
|
|
362
|
-
)
|
|
363
|
-
data_catalog_loader.load_metadata()
|
|
364
|
-
|
|
365
|
-
else:
|
|
366
|
-
raise ValueError(
|
|
367
|
-
"Data Catalog loading is enabled, but the provided parameters are insufficient to load metadata. "
|
|
368
|
-
)
|
|
323
|
+
# Run Data Catalog loader if enabled.
|
|
324
|
+
if include_tables:
|
|
325
|
+
self._run_data_catalog_loader_for_table_entries(include_tables, project_name, skill=existing_skill)
|
|
326
|
+
else:
|
|
327
|
+
self._run_data_catalog_loader_for_skill(existing_skill, project_name, tables=parameters.get("tables"))
|
|
369
328
|
|
|
329
|
+
if existing_skill.type == "sql":
|
|
370
330
|
# Add table restrictions if this is a text2sql skill
|
|
371
|
-
if include_tables
|
|
372
|
-
parameters["tables"] = include_tables
|
|
373
|
-
|
|
374
|
-
# Pass database parameter if provided
|
|
375
|
-
if database and "database" not in parameters:
|
|
376
|
-
parameters["database"] = database
|
|
377
|
-
|
|
378
|
-
# Pass knowledge base database parameter if provided
|
|
379
|
-
if knowledge_base_database and "knowledge_base_database" not in parameters:
|
|
380
|
-
parameters["knowledge_base_database"] = knowledge_base_database
|
|
331
|
+
if include_tables:
|
|
332
|
+
parameters["tables"] = include_tables
|
|
381
333
|
|
|
382
334
|
# Add knowledge base parameters to both the skill and the association parameters
|
|
383
335
|
if include_knowledge_bases:
|
|
@@ -385,16 +337,6 @@ class AgentsController:
|
|
|
385
337
|
if "include_knowledge_bases" not in existing_skill.params:
|
|
386
338
|
existing_skill.params["include_knowledge_bases"] = include_knowledge_bases
|
|
387
339
|
flag_modified(existing_skill, "params")
|
|
388
|
-
elif ignore_knowledge_bases:
|
|
389
|
-
parameters["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
390
|
-
if "ignore_knowledge_bases" not in existing_skill.params:
|
|
391
|
-
existing_skill.params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
392
|
-
flag_modified(existing_skill, "params")
|
|
393
|
-
|
|
394
|
-
# Ensure knowledge_base_database is set in the skill's params
|
|
395
|
-
if knowledge_base_database and "knowledge_base_database" not in existing_skill.params:
|
|
396
|
-
existing_skill.params["knowledge_base_database"] = knowledge_base_database
|
|
397
|
-
flag_modified(existing_skill, "params")
|
|
398
340
|
|
|
399
341
|
association = db.AgentSkillsAssociation(parameters=parameters, agent=agent, skill=existing_skill)
|
|
400
342
|
db.session.add(association)
|
|
@@ -462,6 +404,8 @@ class AgentsController:
|
|
|
462
404
|
raise ValueError("It is forbidden to change properties of the demo object")
|
|
463
405
|
|
|
464
406
|
if name is not None and name != agent_name:
|
|
407
|
+
if not name.islower():
|
|
408
|
+
raise ValueError(f"The name must be in lower case: {name}")
|
|
465
409
|
# Check to see if updated name already exists
|
|
466
410
|
agent_with_new_name = self.get_agent(name, project_name=project_name)
|
|
467
411
|
if agent_with_new_name is not None:
|
|
@@ -503,12 +447,20 @@ class AgentsController:
|
|
|
503
447
|
|
|
504
448
|
# add skills
|
|
505
449
|
for skill_name in set(skills_to_add_names) - set(existing_agent_skills_names):
|
|
450
|
+
# Run Data Catalog loader if enabled for the new skill
|
|
451
|
+
self._run_data_catalog_loader_for_skill(
|
|
452
|
+
skill_name,
|
|
453
|
+
project_name,
|
|
454
|
+
tables=next((x for x in skills_to_add if x["name"] == skill_name), {}).get("tables"),
|
|
455
|
+
)
|
|
456
|
+
|
|
506
457
|
skill_parameters = next(x for x in skills_to_add if x["name"] == skill_name).copy()
|
|
507
458
|
del skill_parameters["name"]
|
|
508
459
|
association = db.AgentSkillsAssociation(
|
|
509
460
|
parameters=skill_parameters, agent=existing_agent, skill=skill_name_to_record_map[skill_name]
|
|
510
461
|
)
|
|
511
462
|
db.session.add(association)
|
|
463
|
+
|
|
512
464
|
elif len(skills_to_rewrite) > 0:
|
|
513
465
|
skill_name_to_parameters = {
|
|
514
466
|
x["name"]: {k: v for k, v in x.items() if k != "name"} for x in skills_to_rewrite
|
|
@@ -519,9 +471,23 @@ class AgentsController:
|
|
|
519
471
|
db.session.delete(rel)
|
|
520
472
|
else:
|
|
521
473
|
existing_skill_names.add(rel.skill.name)
|
|
522
|
-
|
|
474
|
+
skill_parameters = skill_name_to_parameters[rel.skill.name]
|
|
475
|
+
|
|
476
|
+
# Run Data Catalog loader if enabled for the updated skill
|
|
477
|
+
self._run_data_catalog_loader_for_skill(
|
|
478
|
+
rel.skill.name, project_name, tables=skill_parameters.get("tables")
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
rel.parameters = skill_parameters
|
|
523
482
|
flag_modified(rel, "parameters")
|
|
524
483
|
for new_skill_name in set(skill_name_to_parameters) - existing_skill_names:
|
|
484
|
+
# Run Data Catalog loader if enabled for the new skill
|
|
485
|
+
self._run_data_catalog_loader_for_skill(
|
|
486
|
+
new_skill_name,
|
|
487
|
+
project_name,
|
|
488
|
+
tables=skill_name_to_parameters[new_skill_name].get("tables"),
|
|
489
|
+
)
|
|
490
|
+
|
|
525
491
|
association = db.AgentSkillsAssociation(
|
|
526
492
|
parameters=skill_name_to_parameters[new_skill_name],
|
|
527
493
|
agent=existing_agent,
|
|
@@ -530,8 +496,17 @@ class AgentsController:
|
|
|
530
496
|
db.session.add(association)
|
|
531
497
|
|
|
532
498
|
if params is not None:
|
|
533
|
-
# Merge params on update
|
|
534
499
|
existing_params = existing_agent.params or {}
|
|
500
|
+
|
|
501
|
+
if params.get("data", {}).get("tables"):
|
|
502
|
+
new_table_entries = set(params["data"]["tables"]) - set(
|
|
503
|
+
existing_params.get("data", {}).get("tables", [])
|
|
504
|
+
)
|
|
505
|
+
if new_table_entries:
|
|
506
|
+
# Run Data Catalog loader for new table entries if enabled.
|
|
507
|
+
self._run_data_catalog_loader_for_table_entries(new_table_entries, project_name)
|
|
508
|
+
|
|
509
|
+
# Merge params on update
|
|
535
510
|
existing_params.update(params)
|
|
536
511
|
# Remove None values entirely.
|
|
537
512
|
params = {k: v for k, v in existing_params.items() if v is not None}
|
|
@@ -543,6 +518,86 @@ class AgentsController:
|
|
|
543
518
|
|
|
544
519
|
return existing_agent
|
|
545
520
|
|
|
521
|
+
def _run_data_catalog_loader_for_skill(
|
|
522
|
+
self,
|
|
523
|
+
skill: Union[str, db.Skills],
|
|
524
|
+
project_name: str,
|
|
525
|
+
tables: List[str] = None,
|
|
526
|
+
):
|
|
527
|
+
"""
|
|
528
|
+
Runs Data Catalog loader for a skill if enabled in the config.
|
|
529
|
+
This is used to load metadata for SQL skills when they are added or updated.
|
|
530
|
+
"""
|
|
531
|
+
if not config.get("data_catalog", {}).get("enabled", False):
|
|
532
|
+
return
|
|
533
|
+
|
|
534
|
+
skill = skill if isinstance(skill, db.Skills) else self.skills_controller.get_skill(skill, project_name)
|
|
535
|
+
if skill.type == "sql":
|
|
536
|
+
if "database" in skill.params:
|
|
537
|
+
valid_table_names = skill.params.get("tables") if skill.params.get("tables") else tables
|
|
538
|
+
data_catalog_loader = DataCatalogLoader(
|
|
539
|
+
database_name=skill.params["database"], table_names=valid_table_names
|
|
540
|
+
)
|
|
541
|
+
data_catalog_loader.load_metadata()
|
|
542
|
+
else:
|
|
543
|
+
raise ValueError(
|
|
544
|
+
"Data Catalog loading is enabled, but the provided parameters for the new skills are insufficient to load metadata. "
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
def _run_data_catalog_loader_for_table_entries(
|
|
548
|
+
self,
|
|
549
|
+
table_entries: List[str],
|
|
550
|
+
project_name: str,
|
|
551
|
+
skill: Union[str, db.Skills] = None,
|
|
552
|
+
):
|
|
553
|
+
"""
|
|
554
|
+
Runs Data Catalog loader for a list of table entries if enabled in the config.
|
|
555
|
+
This is used to load metadata for SQL skills when they are added or updated.
|
|
556
|
+
"""
|
|
557
|
+
if not config.get("data_catalog", {}).get("enabled", False):
|
|
558
|
+
return
|
|
559
|
+
|
|
560
|
+
skill = skill if isinstance(skill, db.Skills) else self.skills_controller.get_skill(skill, project_name)
|
|
561
|
+
if not skill or skill.type == "sql":
|
|
562
|
+
database_table_map = {}
|
|
563
|
+
for table_entry in table_entries:
|
|
564
|
+
parts = table_entry.split(".", 1)
|
|
565
|
+
|
|
566
|
+
# Ensure the table name is in 'database.table' format.
|
|
567
|
+
if len(parts) != 2:
|
|
568
|
+
logger.warning(
|
|
569
|
+
f"Invalid table name format: {table_entry}. Expected 'database.table' format."
|
|
570
|
+
"Metadata will not be loaded for this entry."
|
|
571
|
+
)
|
|
572
|
+
continue
|
|
573
|
+
|
|
574
|
+
database, table = parts[0], parts[1]
|
|
575
|
+
|
|
576
|
+
# Wildcards in database names are not supported at the moment by data catalog loader.
|
|
577
|
+
if "*" in database:
|
|
578
|
+
logger.warning(
|
|
579
|
+
f"Invalid database name format: {database}. Wildcards are not supported."
|
|
580
|
+
"Metadata will not be loaded for this entry."
|
|
581
|
+
)
|
|
582
|
+
continue
|
|
583
|
+
|
|
584
|
+
# Wildcards in table names are supported either.
|
|
585
|
+
# However, the table name itself can be a wildcard representing all tables.
|
|
586
|
+
if table == "*":
|
|
587
|
+
table = None
|
|
588
|
+
elif "*" in table:
|
|
589
|
+
logger.warning(
|
|
590
|
+
f"Invalid table name format: {table}. Wildcards are not supported."
|
|
591
|
+
"Metadata will not be loaded for this entry."
|
|
592
|
+
)
|
|
593
|
+
continue
|
|
594
|
+
|
|
595
|
+
database_table_map[database] = database_table_map.get(database, []) + [table]
|
|
596
|
+
|
|
597
|
+
for database_name, table_names in database_table_map.items():
|
|
598
|
+
data_catalog_loader = DataCatalogLoader(database_name=database_name, table_names=table_names)
|
|
599
|
+
data_catalog_loader.load_metadata()
|
|
600
|
+
|
|
546
601
|
def delete_agent(self, agent_name: str, project_name: str = default_project):
|
|
547
602
|
"""
|
|
548
603
|
Deletes an agent by name.
|
|
@@ -584,20 +639,22 @@ class AgentsController:
|
|
|
584
639
|
def get_completion(
|
|
585
640
|
self,
|
|
586
641
|
agent: db.Agents,
|
|
587
|
-
messages:
|
|
642
|
+
messages: list[Dict[str, str]],
|
|
588
643
|
project_name: str = default_project,
|
|
589
|
-
tools:
|
|
644
|
+
tools: list[BaseTool] = None,
|
|
590
645
|
stream: bool = False,
|
|
646
|
+
params: dict | None = None,
|
|
591
647
|
) -> Union[Iterator[object], pd.DataFrame]:
|
|
592
648
|
"""
|
|
593
649
|
Queries an agent to get a completion.
|
|
594
650
|
|
|
595
651
|
Parameters:
|
|
596
652
|
agent (db.Agents): Existing agent to get completion from
|
|
597
|
-
messages (
|
|
653
|
+
messages (list[Dict[str, str]]): Chat history to send to the agent
|
|
598
654
|
project_name (str): Project the agent belongs to (default mindsdb)
|
|
599
|
-
tools (
|
|
655
|
+
tools (list[BaseTool]): Tools to use while getting the completion
|
|
600
656
|
stream (bool): Whether to stream the response
|
|
657
|
+
params (dict | None): params to redefine agent params
|
|
601
658
|
|
|
602
659
|
Returns:
|
|
603
660
|
response (Union[Iterator[object], pd.DataFrame]): Completion as a DataFrame or iterator of completion chunks
|
|
@@ -606,7 +663,7 @@ class AgentsController:
|
|
|
606
663
|
ValueError: Agent's model does not exist.
|
|
607
664
|
"""
|
|
608
665
|
if stream:
|
|
609
|
-
return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools)
|
|
666
|
+
return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools, params=params)
|
|
610
667
|
from .langchain_agent import LangchainAgent
|
|
611
668
|
|
|
612
669
|
model, provider = self.check_model_provider(agent.model_name, agent.provider)
|
|
@@ -619,25 +676,27 @@ class AgentsController:
|
|
|
619
676
|
llm_params = self.get_agent_llm_params(agent.params)
|
|
620
677
|
|
|
621
678
|
lang_agent = LangchainAgent(agent, model, llm_params=llm_params)
|
|
622
|
-
return lang_agent.get_completion(messages)
|
|
679
|
+
return lang_agent.get_completion(messages, params=params)
|
|
623
680
|
|
|
624
681
|
def _get_completion_stream(
|
|
625
682
|
self,
|
|
626
683
|
agent: db.Agents,
|
|
627
|
-
messages:
|
|
684
|
+
messages: list[Dict[str, str]],
|
|
628
685
|
project_name: str = default_project,
|
|
629
|
-
tools:
|
|
686
|
+
tools: list[BaseTool] = None,
|
|
687
|
+
params: dict | None = None,
|
|
630
688
|
) -> Iterator[object]:
|
|
631
689
|
"""
|
|
632
690
|
Queries an agent to get a stream of completion chunks.
|
|
633
691
|
|
|
634
692
|
Parameters:
|
|
635
693
|
agent (db.Agents): Existing agent to get completion from
|
|
636
|
-
messages (
|
|
694
|
+
messages (list[Dict[str, str]]): Chat history to send to the agent
|
|
637
695
|
trace_id (str): ID of Langfuse trace to use
|
|
638
696
|
observation_id (str): ID of parent Langfuse observation to use
|
|
639
697
|
project_name (str): Project the agent belongs to (default mindsdb)
|
|
640
|
-
tools (
|
|
698
|
+
tools (list[BaseTool]): Tools to use while getting the completion
|
|
699
|
+
params (dict | None): params to redefine agent params
|
|
641
700
|
|
|
642
701
|
Returns:
|
|
643
702
|
chunks (Iterator[object]): Completion chunks as an iterator
|
|
@@ -659,4 +718,4 @@ class AgentsController:
|
|
|
659
718
|
llm_params = self.get_agent_llm_params(agent.params)
|
|
660
719
|
|
|
661
720
|
lang_agent = LangchainAgent(agent, model=model, llm_params=llm_params)
|
|
662
|
-
return lang_agent.get_completion(messages, stream=True)
|
|
721
|
+
return lang_agent.get_completion(messages, stream=True, params=params)
|
|
@@ -321,7 +321,7 @@ class LangchainAgent:
|
|
|
321
321
|
self.provider,
|
|
322
322
|
]
|
|
323
323
|
|
|
324
|
-
def get_completion(self, messages, stream: bool = False):
|
|
324
|
+
def get_completion(self, messages, stream: bool = False, params: dict | None = None):
|
|
325
325
|
# Get metadata and tags to be used in the trace
|
|
326
326
|
metadata = self.get_metadata()
|
|
327
327
|
tags = self.get_tags()
|
|
@@ -342,7 +342,9 @@ class LangchainAgent:
|
|
|
342
342
|
if stream:
|
|
343
343
|
return self._get_completion_stream(messages)
|
|
344
344
|
|
|
345
|
-
args =
|
|
345
|
+
args = {}
|
|
346
|
+
args.update(self.args)
|
|
347
|
+
args.update(params or {})
|
|
346
348
|
|
|
347
349
|
df = pd.DataFrame(messages)
|
|
348
350
|
|
|
@@ -598,7 +600,12 @@ AI: {response}"""
|
|
|
598
600
|
completions.append(result[ASSISTANT_COLUMN])
|
|
599
601
|
contexts.append(result[CONTEXT_COLUMN])
|
|
600
602
|
except TimeoutError:
|
|
601
|
-
timeout_message =
|
|
603
|
+
timeout_message = (
|
|
604
|
+
f"I'm sorry! I couldn't generate a response within the allotted time ({agent_timeout_seconds} seconds). "
|
|
605
|
+
"If you need more time for processing, you can adjust the timeout settings. "
|
|
606
|
+
"Please refer to the documentation for instructions on how to change the timeout value. "
|
|
607
|
+
"Feel free to try your request again."
|
|
608
|
+
)
|
|
602
609
|
logger.warning(f"Agent execution timed out after {agent_timeout_seconds} seconds")
|
|
603
610
|
for _ in range(len(futures) - len(completions)):
|
|
604
611
|
completions.append(timeout_message)
|
|
@@ -120,7 +120,7 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
120
120
|
Load the column metadata from the handler.
|
|
121
121
|
"""
|
|
122
122
|
self.logger.info(f"Loading columns for {self.database_name}")
|
|
123
|
-
response = self.data_handler.meta_get_columns(
|
|
123
|
+
response = self.data_handler.meta_get_columns([table.name for table in tables])
|
|
124
124
|
if response.resp_type == RESPONSE_TYPE.ERROR:
|
|
125
125
|
self.logger.error(f"Failed to load columns for {self.database_name}: {response.error_message}")
|
|
126
126
|
return []
|
|
@@ -168,7 +168,7 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
168
168
|
Load the column statistics metadata from the handler.
|
|
169
169
|
"""
|
|
170
170
|
self.logger.info(f"Loading column statistics for {self.database_name}")
|
|
171
|
-
response = self.data_handler.meta_get_column_statistics(
|
|
171
|
+
response = self.data_handler.meta_get_column_statistics([table.name for table in tables])
|
|
172
172
|
if response.resp_type == RESPONSE_TYPE.ERROR:
|
|
173
173
|
self.logger.error(f"Failed to load column statistics for {self.database_name}: {response.error_message}")
|
|
174
174
|
return
|
|
@@ -233,7 +233,7 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
233
233
|
Load the primary keys metadata from the handler.
|
|
234
234
|
"""
|
|
235
235
|
self.logger.info(f"Loading primary keys for {self.database_name}")
|
|
236
|
-
response = self.data_handler.meta_get_primary_keys(
|
|
236
|
+
response = self.data_handler.meta_get_primary_keys([table.name for table in tables])
|
|
237
237
|
if response.resp_type == RESPONSE_TYPE.ERROR:
|
|
238
238
|
self.logger.error(f"Failed to load primary keys for {self.database_name}: {response.error_message}")
|
|
239
239
|
return
|
|
@@ -285,7 +285,7 @@ class DataCatalogLoader(BaseDataCatalog):
|
|
|
285
285
|
Load the foreign keys metadata from the handler.
|
|
286
286
|
"""
|
|
287
287
|
self.logger.info(f"Loading foreign keys for {self.database_name}")
|
|
288
|
-
response = self.data_handler.meta_get_foreign_keys(
|
|
288
|
+
response = self.data_handler.meta_get_foreign_keys([table.name for table in tables])
|
|
289
289
|
if response.resp_type == RESPONSE_TYPE.ERROR:
|
|
290
290
|
self.logger.error(f"Failed to foreign keys for {self.database_name}: {response.error_message}")
|
|
291
291
|
return
|
|
@@ -18,18 +18,30 @@ class DatabaseController:
|
|
|
18
18
|
self.logs_db_controller = LogDBController()
|
|
19
19
|
self.information_schema_controller = None
|
|
20
20
|
|
|
21
|
-
def delete(self, name: str):
|
|
21
|
+
def delete(self, name: str, strict_case: bool = False) -> None:
|
|
22
|
+
"""Delete a database (project or integration) by name.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
name (str): The name of the database to delete.
|
|
26
|
+
strict_case (bool, optional): If True, the database name is case-sensitive. Defaults to False.
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
EntityNotExistsError: If the database does not exist.
|
|
30
|
+
Exception: If the database cannot be deleted.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
None
|
|
34
|
+
"""
|
|
22
35
|
databases = self.get_dict()
|
|
23
|
-
|
|
24
|
-
if name not in databases:
|
|
36
|
+
if name.lower() not in databases:
|
|
25
37
|
raise EntityNotExistsError("Database does not exists", name)
|
|
26
|
-
db_type = databases[name]["type"]
|
|
38
|
+
db_type = databases[name.lower()]["type"]
|
|
27
39
|
if db_type == "project":
|
|
28
|
-
project = self.get_project(name)
|
|
40
|
+
project = self.get_project(name, strict_case)
|
|
29
41
|
project.delete()
|
|
30
42
|
return
|
|
31
43
|
elif db_type == "data":
|
|
32
|
-
self.integration_controller.delete(name)
|
|
44
|
+
self.integration_controller.delete(name, strict_case)
|
|
33
45
|
return
|
|
34
46
|
else:
|
|
35
47
|
raise Exception(f"Database with type '{db_type}' cannot be deleted")
|
|
@@ -81,9 +93,9 @@ class DatabaseController:
|
|
|
81
93
|
|
|
82
94
|
return result
|
|
83
95
|
|
|
84
|
-
def get_dict(self, filter_type: Optional[str] = None):
|
|
96
|
+
def get_dict(self, filter_type: Optional[str] = None, lowercase: bool = True):
|
|
85
97
|
return OrderedDict(
|
|
86
|
-
(x["name"].lower(), {"type": x["type"], "engine": x["engine"], "id": x["id"]})
|
|
98
|
+
(x["name"].lower() if lowercase else x["name"], {"type": x["type"], "engine": x["engine"], "id": x["id"]})
|
|
87
99
|
for x in self.get_list(filter_type=filter_type)
|
|
88
100
|
)
|
|
89
101
|
|
|
@@ -98,8 +110,17 @@ class DatabaseController:
|
|
|
98
110
|
def exists(self, db_name: str) -> bool:
|
|
99
111
|
return db_name.lower() in self.get_dict()
|
|
100
112
|
|
|
101
|
-
def get_project(self, name: str):
|
|
102
|
-
|
|
113
|
+
def get_project(self, name: str, strict_case: bool = False):
|
|
114
|
+
"""Get a project by name.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
name (str): The name of the project to retrieve.
|
|
118
|
+
strict_case (bool, optional): If True, the project name is case-sensitive. Defaults to False.
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
Project: The project instance matching the given name.
|
|
122
|
+
"""
|
|
123
|
+
return self.project_controller.get(name=name, strict_case=strict_case)
|
|
103
124
|
|
|
104
125
|
def get_system_db(self, name: str):
|
|
105
126
|
if name == "log":
|
|
@@ -112,19 +133,21 @@ class DatabaseController:
|
|
|
112
133
|
else:
|
|
113
134
|
raise Exception(f"Database '{name}' does not exists")
|
|
114
135
|
|
|
115
|
-
def update(self, name: str, data: dict):
|
|
136
|
+
def update(self, name: str, data: dict, strict_case: bool = False):
|
|
116
137
|
"""
|
|
117
138
|
Updates the database with the given name using the provided data.
|
|
118
139
|
|
|
119
140
|
Parameters:
|
|
120
141
|
name (str): The name of the database to update.
|
|
121
142
|
data (dict): The data to update the database with.
|
|
143
|
+
strict_case (bool): if True, then name is case-sesitive
|
|
122
144
|
|
|
123
145
|
Raises:
|
|
124
146
|
EntityNotExistsError: If the database does not exist.
|
|
125
147
|
"""
|
|
126
|
-
databases = self.get_dict()
|
|
127
|
-
|
|
148
|
+
databases = self.get_dict(lowercase=(not strict_case))
|
|
149
|
+
if not strict_case:
|
|
150
|
+
name = name.lower()
|
|
128
151
|
if name not in databases:
|
|
129
152
|
raise EntityNotExistsError("Database does not exist.", name)
|
|
130
153
|
|
|
@@ -133,6 +156,8 @@ class DatabaseController:
|
|
|
133
156
|
# Only the name of the project can be updated.
|
|
134
157
|
if {"name"} != set(data):
|
|
135
158
|
raise ValueError("Only the 'name' field can be updated for projects.")
|
|
159
|
+
if not data["name"].islower():
|
|
160
|
+
raise ValueError("New name must be in lower case.")
|
|
136
161
|
self.project_controller.update(name=name, new_name=str(data["name"]))
|
|
137
162
|
return
|
|
138
163
|
|
|
@@ -161,7 +161,7 @@ class IntegrationController:
|
|
|
161
161
|
db.session.commit()
|
|
162
162
|
return integration_record.id
|
|
163
163
|
|
|
164
|
-
def add(self, name, engine, connection_args):
|
|
164
|
+
def add(self, name: str, engine, connection_args):
|
|
165
165
|
logger.debug(
|
|
166
166
|
"%s: add method calling name=%s, engine=%s, connection_args=%s, company_id=%s",
|
|
167
167
|
self.__class__.__name__,
|
|
@@ -172,6 +172,9 @@ class IntegrationController:
|
|
|
172
172
|
)
|
|
173
173
|
handler_meta = self.get_handler_meta(engine)
|
|
174
174
|
|
|
175
|
+
if not name.islower():
|
|
176
|
+
raise ValueError(f"The name must be in lower case: {name}")
|
|
177
|
+
|
|
175
178
|
accept_connection_args = handler_meta.get("connection_args")
|
|
176
179
|
logger.debug("%s: accept_connection_args - %s", self.__class__.__name__, accept_connection_args)
|
|
177
180
|
|
|
@@ -210,20 +213,32 @@ class IntegrationController:
|
|
|
210
213
|
integration_record.data = data
|
|
211
214
|
db.session.commit()
|
|
212
215
|
|
|
213
|
-
def delete(self, name):
|
|
214
|
-
|
|
216
|
+
def delete(self, name: str, strict_case: bool = False) -> None:
|
|
217
|
+
"""Delete an integration by name.
|
|
218
|
+
|
|
219
|
+
Args:
|
|
220
|
+
name (str): The name of the integration to delete.
|
|
221
|
+
strict_case (bool, optional): If True, the integration name is case-sensitive. Defaults to False.
|
|
222
|
+
|
|
223
|
+
Raises:
|
|
224
|
+
Exception: If the integration cannot be deleted (system, permanent, demo, in use, or has active models).
|
|
225
|
+
|
|
226
|
+
Returns:
|
|
227
|
+
None
|
|
228
|
+
"""
|
|
229
|
+
if name == "files":
|
|
215
230
|
raise Exception("Unable to drop: is system database")
|
|
216
231
|
|
|
217
232
|
self.handlers_cache.delete(name)
|
|
218
233
|
|
|
219
234
|
# check permanent integration
|
|
220
|
-
if name in self.handler_modules:
|
|
235
|
+
if name.lower() in self.handler_modules:
|
|
221
236
|
handler = self.handler_modules[name]
|
|
222
237
|
|
|
223
238
|
if getattr(handler, "permanent", False) is True:
|
|
224
239
|
raise Exception("Unable to drop permanent integration")
|
|
225
240
|
|
|
226
|
-
integration_record = self._get_integration_record(name)
|
|
241
|
+
integration_record = self._get_integration_record(name, case_sensitive=strict_case)
|
|
227
242
|
if isinstance(integration_record.data, dict) and integration_record.data.get("is_demo") is True:
|
|
228
243
|
raise Exception("Unable to drop demo object")
|
|
229
244
|
|