MindsDB 25.7.3.0__py3-none-any.whl → 25.8.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of MindsDB might be problematic. Click here for more details.

Files changed (102) hide show
  1. mindsdb/__about__.py +1 -1
  2. mindsdb/__main__.py +11 -1
  3. mindsdb/api/a2a/common/server/server.py +16 -6
  4. mindsdb/api/executor/command_executor.py +215 -150
  5. mindsdb/api/executor/datahub/datanodes/project_datanode.py +14 -3
  6. mindsdb/api/executor/planner/plan_join.py +3 -0
  7. mindsdb/api/executor/planner/plan_join_ts.py +117 -100
  8. mindsdb/api/executor/planner/query_planner.py +1 -0
  9. mindsdb/api/executor/sql_query/steps/apply_predictor_step.py +54 -85
  10. mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +21 -24
  11. mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py +9 -3
  12. mindsdb/api/executor/sql_query/steps/subselect_step.py +11 -8
  13. mindsdb/api/executor/utilities/mysql_to_duckdb_functions.py +264 -0
  14. mindsdb/api/executor/utilities/sql.py +30 -0
  15. mindsdb/api/http/initialize.py +18 -44
  16. mindsdb/api/http/namespaces/agents.py +23 -20
  17. mindsdb/api/http/namespaces/chatbots.py +83 -120
  18. mindsdb/api/http/namespaces/file.py +1 -1
  19. mindsdb/api/http/namespaces/jobs.py +38 -60
  20. mindsdb/api/http/namespaces/tree.py +69 -61
  21. mindsdb/api/http/namespaces/views.py +56 -72
  22. mindsdb/api/mcp/start.py +2 -0
  23. mindsdb/api/mysql/mysql_proxy/utilities/dump.py +3 -2
  24. mindsdb/integrations/handlers/autogluon_handler/requirements.txt +1 -1
  25. mindsdb/integrations/handlers/autosklearn_handler/requirements.txt +1 -1
  26. mindsdb/integrations/handlers/bigquery_handler/bigquery_handler.py +25 -5
  27. mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py +3 -3
  28. mindsdb/integrations/handlers/db2_handler/db2_handler.py +19 -23
  29. mindsdb/integrations/handlers/flaml_handler/requirements.txt +1 -1
  30. mindsdb/integrations/handlers/gong_handler/__about__.py +2 -0
  31. mindsdb/integrations/handlers/gong_handler/__init__.py +30 -0
  32. mindsdb/integrations/handlers/gong_handler/connection_args.py +37 -0
  33. mindsdb/integrations/handlers/gong_handler/gong_handler.py +164 -0
  34. mindsdb/integrations/handlers/gong_handler/gong_tables.py +508 -0
  35. mindsdb/integrations/handlers/gong_handler/icon.svg +25 -0
  36. mindsdb/integrations/handlers/gong_handler/test_gong_handler.py +125 -0
  37. mindsdb/integrations/handlers/google_calendar_handler/google_calendar_tables.py +82 -73
  38. mindsdb/integrations/handlers/hubspot_handler/requirements.txt +1 -1
  39. mindsdb/integrations/handlers/huggingface_handler/__init__.py +8 -12
  40. mindsdb/integrations/handlers/huggingface_handler/finetune.py +203 -223
  41. mindsdb/integrations/handlers/huggingface_handler/huggingface_handler.py +360 -383
  42. mindsdb/integrations/handlers/huggingface_handler/requirements.txt +7 -7
  43. mindsdb/integrations/handlers/huggingface_handler/requirements_cpu.txt +7 -7
  44. mindsdb/integrations/handlers/huggingface_handler/settings.py +25 -25
  45. mindsdb/integrations/handlers/langchain_handler/langchain_handler.py +83 -77
  46. mindsdb/integrations/handlers/lightwood_handler/requirements.txt +4 -4
  47. mindsdb/integrations/handlers/litellm_handler/litellm_handler.py +5 -2
  48. mindsdb/integrations/handlers/litellm_handler/settings.py +2 -1
  49. mindsdb/integrations/handlers/openai_handler/constants.py +11 -30
  50. mindsdb/integrations/handlers/openai_handler/helpers.py +27 -34
  51. mindsdb/integrations/handlers/openai_handler/openai_handler.py +14 -12
  52. mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +106 -90
  53. mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +41 -39
  54. mindsdb/integrations/handlers/salesforce_handler/constants.py +215 -0
  55. mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py +141 -80
  56. mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py +0 -1
  57. mindsdb/integrations/handlers/tpot_handler/requirements.txt +1 -1
  58. mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +32 -17
  59. mindsdb/integrations/handlers/web_handler/web_handler.py +19 -22
  60. mindsdb/integrations/libs/llm/config.py +0 -14
  61. mindsdb/integrations/libs/llm/utils.py +0 -15
  62. mindsdb/integrations/libs/vectordatabase_handler.py +10 -1
  63. mindsdb/integrations/utilities/files/file_reader.py +5 -19
  64. mindsdb/integrations/utilities/handler_utils.py +32 -12
  65. mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +1 -1
  66. mindsdb/interfaces/agents/agents_controller.py +246 -149
  67. mindsdb/interfaces/agents/constants.py +0 -1
  68. mindsdb/interfaces/agents/langchain_agent.py +11 -6
  69. mindsdb/interfaces/data_catalog/data_catalog_loader.py +4 -4
  70. mindsdb/interfaces/database/database.py +38 -13
  71. mindsdb/interfaces/database/integrations.py +20 -5
  72. mindsdb/interfaces/database/projects.py +174 -23
  73. mindsdb/interfaces/database/views.py +86 -60
  74. mindsdb/interfaces/jobs/jobs_controller.py +103 -110
  75. mindsdb/interfaces/knowledge_base/controller.py +33 -6
  76. mindsdb/interfaces/knowledge_base/evaluate.py +2 -1
  77. mindsdb/interfaces/knowledge_base/executor.py +24 -0
  78. mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py +6 -10
  79. mindsdb/interfaces/knowledge_base/preprocessing/text_splitter.py +73 -0
  80. mindsdb/interfaces/query_context/context_controller.py +111 -145
  81. mindsdb/interfaces/skills/skills_controller.py +18 -6
  82. mindsdb/interfaces/storage/db.py +40 -6
  83. mindsdb/interfaces/variables/variables_controller.py +8 -15
  84. mindsdb/utilities/config.py +5 -3
  85. mindsdb/utilities/fs.py +54 -17
  86. mindsdb/utilities/functions.py +72 -60
  87. mindsdb/utilities/log.py +38 -6
  88. mindsdb/utilities/ps.py +7 -7
  89. {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/METADATA +282 -268
  90. {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/RECORD +94 -92
  91. mindsdb/integrations/handlers/anyscale_endpoints_handler/__about__.py +0 -9
  92. mindsdb/integrations/handlers/anyscale_endpoints_handler/__init__.py +0 -20
  93. mindsdb/integrations/handlers/anyscale_endpoints_handler/anyscale_endpoints_handler.py +0 -290
  94. mindsdb/integrations/handlers/anyscale_endpoints_handler/creation_args.py +0 -14
  95. mindsdb/integrations/handlers/anyscale_endpoints_handler/icon.svg +0 -4
  96. mindsdb/integrations/handlers/anyscale_endpoints_handler/requirements.txt +0 -2
  97. mindsdb/integrations/handlers/anyscale_endpoints_handler/settings.py +0 -51
  98. mindsdb/integrations/handlers/anyscale_endpoints_handler/tests/test_anyscale_endpoints_handler.py +0 -212
  99. /mindsdb/integrations/handlers/{anyscale_endpoints_handler/tests/__init__.py → gong_handler/requirements.txt} +0 -0
  100. {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/WHEEL +0 -0
  101. {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/licenses/LICENSE +0 -0
  102. {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/top_level.txt +0 -0
@@ -20,7 +20,7 @@ from mindsdb.utilities import log
20
20
 
21
21
  from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
22
22
 
23
- from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS, DEFAULT_TEXT2SQL_DATABASE
23
+ from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
24
24
  from .langchain_agent import get_llm_provider
25
25
 
26
26
  logger = log.getLogger(__name__)
@@ -145,11 +145,60 @@ class AgentsController:
145
145
 
146
146
  return all_agents.all()
147
147
 
148
+ def _create_default_sql_skill(
149
+ self,
150
+ name,
151
+ project_name,
152
+ include_tables: List[str] = None,
153
+ include_knowledge_bases: List[str] = None,
154
+ ):
155
+ # Create a default SQL skill
156
+ skill_name = f"{name}_sql_skill"
157
+ skill_params = {
158
+ "type": "sql",
159
+ "description": f"Auto-generated SQL skill for agent {name}",
160
+ }
161
+
162
+ # Add restrictions provided
163
+ if include_tables:
164
+ skill_params["include_tables"] = include_tables
165
+ if include_knowledge_bases:
166
+ skill_params["include_knowledge_bases"] = include_knowledge_bases
167
+
168
+ try:
169
+ # Check if skill already exists
170
+ existing_skill = self.skills_controller.get_skill(skill_name, project_name)
171
+ if existing_skill is None:
172
+ # Create the skill
173
+ skill_type = skill_params.pop("type")
174
+ self.skills_controller.add_skill(
175
+ name=skill_name, project_name=project_name, type=skill_type, params=skill_params
176
+ )
177
+ else:
178
+ # Update the skill if parameters have changed
179
+ params_changed = False
180
+
181
+ # Check if skill parameters need to be updated
182
+ for param_key, param_value in skill_params.items():
183
+ if existing_skill.params.get(param_key) != param_value:
184
+ existing_skill.params[param_key] = param_value
185
+ params_changed = True
186
+
187
+ # Update the skill if needed
188
+ if params_changed:
189
+ flag_modified(existing_skill, "params")
190
+ db.session.commit()
191
+
192
+ except Exception as e:
193
+ raise ValueError(f"Failed to auto-create or update SQL skill: {str(e)}")
194
+
195
+ return skill_name
196
+
148
197
  def add_agent(
149
198
  self,
150
199
  name: str,
151
200
  project_name: str = None,
152
- model_name: str = None,
201
+ model_name: Union[str, dict] = None,
153
202
  skills: List[Union[str, dict]] = None,
154
203
  provider: str = None,
155
204
  params: Dict[str, Any] = None,
@@ -165,16 +214,21 @@ class AgentsController:
165
214
  with one of keys is "name", and other is additional parameters for relationship agent<>skill
166
215
  provider (str): The provider of the model
167
216
  params (Dict[str, str]): Parameters to use when running the agent
217
+ data: Dict, data sources for an agent, keys:
218
+ - knowledge_bases: List of KBs to use
219
+ - tables: list of tables to use
220
+ model: Dict, parameters for the model to use
221
+ - provider: The provider of the model (e.g., 'openai', 'google')
222
+ - Other model-specific parameters like 'api_key', 'model_name', etc.
223
+ <provider>_api_key: API key for the provider (e.g., openai_api_key)
224
+
225
+ # Deprecated parameters:
168
226
  database: The database to use for text2sql skills (default is 'mindsdb')
169
227
  knowledge_base_database: The database to use for knowledge base queries (default is 'mindsdb')
170
228
  include_tables: List of tables to include for text2sql skills
171
229
  ignore_tables: List of tables to ignore for text2sql skills
172
230
  include_knowledge_bases: List of knowledge bases to include for text2sql skills
173
231
  ignore_knowledge_bases: List of knowledge bases to ignore for text2sql skills
174
- <provider>_api_key: API key for the provider (e.g., openai_api_key)
175
- data: Dict, data sources for an agent, keys:
176
- - knowledge_bases: List of KBs to use (alternative to `include_knowledge_bases`)
177
- - tables: list of tables to use (alternative to `include_tables`)
178
232
 
179
233
  Returns:
180
234
  agent (db.Agents): The created agent
@@ -223,99 +277,41 @@ class AgentsController:
223
277
  # It will be picked up by get_api_key() in handler_utils.py
224
278
  pass
225
279
 
226
- # Extract table and knowledge base parameters from params
227
- database = params.pop("database", None)
228
- knowledge_base_database = params.pop("knowledge_base_database", DEFAULT_TEXT2SQL_DATABASE)
229
- include_tables = params.pop("include_tables", None)
230
- ignore_tables = params.pop("ignore_tables", None)
231
- include_knowledge_bases = params.pop("include_knowledge_bases", None)
232
- ignore_knowledge_bases = params.pop("ignore_knowledge_bases", None)
233
-
234
- # Save the extracted parameters back to params for API responses only if they were explicitly provided
235
- # or if they're needed for skills
236
- need_params = include_tables or ignore_tables or include_knowledge_bases or ignore_knowledge_bases
237
-
238
- if "database" in params or need_params:
239
- params["database"] = database
280
+ depreciated_params = [
281
+ "database",
282
+ "knowledge_base_database",
283
+ "include_tables",
284
+ "ignore_tables",
285
+ "include_knowledge_bases",
286
+ "ignore_knowledge_bases",
287
+ ]
288
+ if any(param in params for param in depreciated_params):
289
+ raise ValueError(
290
+ f"Parameters {', '.join(depreciated_params)} are deprecated. "
291
+ "Use 'data' parameter with 'tables' and 'knowledge_bases' keys instead."
292
+ )
240
293
 
294
+ include_tables = None
295
+ include_knowledge_bases = None
241
296
  if "data" in params:
242
- if include_knowledge_bases is None:
243
- include_knowledge_bases = params["data"].get("knowledge_bases")
244
- if include_tables is None:
245
- include_tables = params["data"].get("tables")
246
-
247
- if "knowledge_base_database" in params or include_knowledge_bases or ignore_knowledge_bases:
248
- params["knowledge_base_database"] = knowledge_base_database
249
-
250
- if include_tables is not None:
251
- params["include_tables"] = include_tables
252
- if ignore_tables is not None:
253
- params["ignore_tables"] = ignore_tables
254
- if include_knowledge_bases is not None:
255
- params["include_knowledge_bases"] = include_knowledge_bases
256
- if ignore_knowledge_bases is not None:
257
- params["ignore_knowledge_bases"] = ignore_knowledge_bases
297
+ include_knowledge_bases = params["data"].get("knowledge_bases")
298
+ include_tables = params["data"].get("tables")
258
299
 
259
300
  # Convert string parameters to lists if needed
260
301
  if isinstance(include_tables, str):
261
302
  include_tables = [t.strip() for t in include_tables.split(",")]
262
- if isinstance(ignore_tables, str):
263
- ignore_tables = [t.strip() for t in ignore_tables.split(",")]
264
303
  if isinstance(include_knowledge_bases, str):
265
304
  include_knowledge_bases = [kb.strip() for kb in include_knowledge_bases.split(",")]
266
- if isinstance(ignore_knowledge_bases, str):
267
- ignore_knowledge_bases = [kb.strip() for kb in ignore_knowledge_bases.split(",")]
268
305
 
269
306
  # Auto-create SQL skill if no skills are provided but include_tables or include_knowledge_bases params are provided
270
307
  if not skills and (include_tables or include_knowledge_bases):
271
- # Create a default SQL skill
272
- skill_name = f"{name}_sql_skill"
273
- skill_params = {
274
- "type": "sql",
275
- "database": database,
276
- "description": f"Auto-generated SQL skill for agent {name}",
277
- }
278
-
279
- # Add table restrictions if provided
280
- if include_tables:
281
- skill_params["include_tables"] = include_tables
282
- if ignore_tables:
283
- skill_params["ignore_tables"] = ignore_tables
284
-
285
- # Add knowledge base parameters if provided
286
- if knowledge_base_database:
287
- skill_params["knowledge_base_database"] = knowledge_base_database
288
- if include_knowledge_bases:
289
- skill_params["include_knowledge_bases"] = include_knowledge_bases
290
- if ignore_knowledge_bases:
291
- skill_params["ignore_knowledge_bases"] = ignore_knowledge_bases
292
- try:
293
- # Check if skill already exists
294
- existing_skill = self.skills_controller.get_skill(skill_name, project_name)
295
- if existing_skill is None:
296
- # Create the skill
297
- skill_type = skill_params.pop("type")
298
- self.skills_controller.add_skill(
299
- name=skill_name, project_name=project_name, type=skill_type, params=skill_params
300
- )
301
- else:
302
- # Update the skill if parameters have changed
303
- params_changed = False
304
-
305
- # Check if skill parameters need to be updated
306
- for param_key, param_value in skill_params.items():
307
- if existing_skill.params.get(param_key) != param_value:
308
- existing_skill.params[param_key] = param_value
309
- params_changed = True
310
-
311
- # Update the skill if needed
312
- if params_changed:
313
- flag_modified(existing_skill, "params")
314
- db.session.commit()
315
-
316
- skills = [skill_name]
317
- except Exception as e:
318
- raise ValueError(f"Failed to auto-create or update SQL skill: {str(e)}")
308
+ skill = self._create_default_sql_skill(
309
+ name,
310
+ project_name,
311
+ include_tables=include_tables,
312
+ include_knowledge_bases=include_knowledge_bases,
313
+ )
314
+ skills = [skill]
319
315
 
320
316
  agent = db.Agents(
321
317
  name=name,
@@ -340,44 +336,16 @@ class AgentsController:
340
336
  db.session.rollback()
341
337
  raise ValueError(f"Skill with name does not exist: {skill_name}")
342
338
 
343
- if existing_skill.type == "sql":
344
- # Run Data Catalog loader if enabled
345
- if config.get("data_catalog", {}).get("enabled", False):
346
- if include_tables:
347
- database_table_map = {}
348
- for table in include_tables:
349
- parts = table.split(".", 1)
350
- database_table_map[parts[0]] = database_table_map.get(parts[0], []) + [parts[1]]
351
-
352
- for database_name, table_names in database_table_map.items():
353
- data_catalog_loader = DataCatalogLoader(
354
- database_name=database_name, table_names=table_names
355
- )
356
- data_catalog_loader.load_metadata()
357
-
358
- elif "database" in existing_skill.params:
359
- data_catalog_loader = DataCatalogLoader(
360
- database_name=existing_skill.params["database"],
361
- table_names=parameters["tables"] if "tables" in parameters else None,
362
- )
363
- data_catalog_loader.load_metadata()
364
-
365
- else:
366
- raise ValueError(
367
- "Data Catalog loading is enabled, but the provided parameters are insufficient to load metadata. "
368
- )
339
+ # Run Data Catalog loader if enabled.
340
+ if include_tables:
341
+ self._run_data_catalog_loader_for_table_entries(include_tables, project_name, skill=existing_skill)
342
+ else:
343
+ self._run_data_catalog_loader_for_skill(existing_skill, project_name, tables=parameters.get("tables"))
369
344
 
345
+ if existing_skill.type == "sql":
370
346
  # Add table restrictions if this is a text2sql skill
371
- if include_tables or ignore_tables:
372
- parameters["tables"] = include_tables or ignore_tables
373
-
374
- # Pass database parameter if provided
375
- if database and "database" not in parameters:
376
- parameters["database"] = database
377
-
378
- # Pass knowledge base database parameter if provided
379
- if knowledge_base_database and "knowledge_base_database" not in parameters:
380
- parameters["knowledge_base_database"] = knowledge_base_database
347
+ if include_tables:
348
+ parameters["tables"] = include_tables
381
349
 
382
350
  # Add knowledge base parameters to both the skill and the association parameters
383
351
  if include_knowledge_bases:
@@ -385,16 +353,6 @@ class AgentsController:
385
353
  if "include_knowledge_bases" not in existing_skill.params:
386
354
  existing_skill.params["include_knowledge_bases"] = include_knowledge_bases
387
355
  flag_modified(existing_skill, "params")
388
- elif ignore_knowledge_bases:
389
- parameters["ignore_knowledge_bases"] = ignore_knowledge_bases
390
- if "ignore_knowledge_bases" not in existing_skill.params:
391
- existing_skill.params["ignore_knowledge_bases"] = ignore_knowledge_bases
392
- flag_modified(existing_skill, "params")
393
-
394
- # Ensure knowledge_base_database is set in the skill's params
395
- if knowledge_base_database and "knowledge_base_database" not in existing_skill.params:
396
- existing_skill.params["knowledge_base_database"] = knowledge_base_database
397
- flag_modified(existing_skill, "params")
398
356
 
399
357
  association = db.AgentSkillsAssociation(parameters=parameters, agent=agent, skill=existing_skill)
400
358
  db.session.add(association)
@@ -409,7 +367,7 @@ class AgentsController:
409
367
  agent_name: str,
410
368
  project_name: str = default_project,
411
369
  name: str = None,
412
- model_name: str = None,
370
+ model_name: Union[str, dict] = None,
413
371
  skills_to_add: List[Union[str, dict]] = None,
414
372
  skills_to_remove: List[str] = None,
415
373
  skills_to_rewrite: List[Union[str, dict]] = None,
@@ -423,7 +381,7 @@ class AgentsController:
423
381
  agent_name (str): The name of the new agent, or existing agent to update
424
382
  project_name (str): The containing project
425
383
  name (str): The updated name of the agent
426
- model_name (str): The name of the existing ML model the agent will use
384
+ model_name (str | dict): The name of the existing ML model the agent will use
427
385
  skills_to_add (List[Union[str, dict]]): List of skill names to add to the agent, or list of dicts
428
386
  with one of keys is "name", and other is additional parameters for relationship agent<>skill
429
387
  skills_to_remove (List[str]): List of skill names to remove from the agent
@@ -452,6 +410,8 @@ class AgentsController:
452
410
  existing_agent = self.get_agent(agent_name, project_name=project_name)
453
411
  if existing_agent is None:
454
412
  raise EntityNotExistsError(f"Agent with name not found: {agent_name}")
413
+ existing_params = existing_agent.params or {}
414
+
455
415
  is_demo = (existing_agent.params or {}).get("is_demo", False)
456
416
  if is_demo and (
457
417
  (name is not None and name != agent_name)
@@ -462,6 +422,8 @@ class AgentsController:
462
422
  raise ValueError("It is forbidden to change properties of the demo object")
463
423
 
464
424
  if name is not None and name != agent_name:
425
+ if not name.islower():
426
+ raise ValueError(f"The name must be in lower case: {name}")
465
427
  # Check to see if updated name already exists
466
428
  agent_with_new_name = self.get_agent(name, project_name=project_name)
467
429
  if agent_with_new_name is not None:
@@ -469,12 +431,34 @@ class AgentsController:
469
431
  existing_agent.name = name
470
432
 
471
433
  if model_name or provider:
434
+ if isinstance(model_name, dict):
435
+ # move into params
436
+ existing_params["model"] = model_name
437
+ model_name = None
438
+
472
439
  # check model and provider
473
440
  model, provider = self.check_model_provider(model_name, provider)
474
441
  # Update model and provider
475
442
  existing_agent.model_name = model_name
476
443
  existing_agent.provider = provider
477
444
 
445
+ if "data" in params:
446
+ if len(skills_to_add) > 0 or len(skills_to_remove) > 0:
447
+ raise ValueError(
448
+ "'data' parameter cannot be used with 'skills_to_remove' or 'skills_to_add' parameters"
449
+ )
450
+
451
+ include_knowledge_bases = params["data"].get("knowledge_bases")
452
+ include_tables = params["data"].get("tables")
453
+
454
+ skill = self._create_default_sql_skill(
455
+ agent_name,
456
+ project_name,
457
+ include_tables=include_tables,
458
+ include_knowledge_bases=include_knowledge_bases,
459
+ )
460
+ skills_to_rewrite = [{"name": skill}]
461
+
478
462
  # check that all skills exist
479
463
  skill_name_to_record_map = {}
480
464
  for skill_meta in skills_to_add + skills_to_remove + skills_to_rewrite:
@@ -503,12 +487,20 @@ class AgentsController:
503
487
 
504
488
  # add skills
505
489
  for skill_name in set(skills_to_add_names) - set(existing_agent_skills_names):
490
+ # Run Data Catalog loader if enabled for the new skill
491
+ self._run_data_catalog_loader_for_skill(
492
+ skill_name,
493
+ project_name,
494
+ tables=next((x for x in skills_to_add if x["name"] == skill_name), {}).get("tables"),
495
+ )
496
+
506
497
  skill_parameters = next(x for x in skills_to_add if x["name"] == skill_name).copy()
507
498
  del skill_parameters["name"]
508
499
  association = db.AgentSkillsAssociation(
509
500
  parameters=skill_parameters, agent=existing_agent, skill=skill_name_to_record_map[skill_name]
510
501
  )
511
502
  db.session.add(association)
503
+
512
504
  elif len(skills_to_rewrite) > 0:
513
505
  skill_name_to_parameters = {
514
506
  x["name"]: {k: v for k, v in x.items() if k != "name"} for x in skills_to_rewrite
@@ -519,9 +511,23 @@ class AgentsController:
519
511
  db.session.delete(rel)
520
512
  else:
521
513
  existing_skill_names.add(rel.skill.name)
522
- rel.parameters = skill_name_to_parameters[rel.skill.name]
514
+ skill_parameters = skill_name_to_parameters[rel.skill.name]
515
+
516
+ # Run Data Catalog loader if enabled for the updated skill
517
+ self._run_data_catalog_loader_for_skill(
518
+ rel.skill.name, project_name, tables=skill_parameters.get("tables")
519
+ )
520
+
521
+ rel.parameters = skill_parameters
523
522
  flag_modified(rel, "parameters")
524
523
  for new_skill_name in set(skill_name_to_parameters) - existing_skill_names:
524
+ # Run Data Catalog loader if enabled for the new skill
525
+ self._run_data_catalog_loader_for_skill(
526
+ new_skill_name,
527
+ project_name,
528
+ tables=skill_name_to_parameters[new_skill_name].get("tables"),
529
+ )
530
+
525
531
  association = db.AgentSkillsAssociation(
526
532
  parameters=skill_name_to_parameters[new_skill_name],
527
533
  agent=existing_agent,
@@ -530,8 +536,15 @@ class AgentsController:
530
536
  db.session.add(association)
531
537
 
532
538
  if params is not None:
539
+ if params.get("data", {}).get("tables"):
540
+ new_table_entries = set(params["data"]["tables"]) - set(
541
+ existing_params.get("data", {}).get("tables", [])
542
+ )
543
+ if new_table_entries:
544
+ # Run Data Catalog loader for new table entries if enabled.
545
+ self._run_data_catalog_loader_for_table_entries(new_table_entries, project_name)
546
+
533
547
  # Merge params on update
534
- existing_params = existing_agent.params or {}
535
548
  existing_params.update(params)
536
549
  # Remove None values entirely.
537
550
  params = {k: v for k, v in existing_params.items() if v is not None}
@@ -543,6 +556,86 @@ class AgentsController:
543
556
 
544
557
  return existing_agent
545
558
 
559
+ def _run_data_catalog_loader_for_skill(
560
+ self,
561
+ skill: Union[str, db.Skills],
562
+ project_name: str,
563
+ tables: List[str] = None,
564
+ ):
565
+ """
566
+ Runs Data Catalog loader for a skill if enabled in the config.
567
+ This is used to load metadata for SQL skills when they are added or updated.
568
+ """
569
+ if not config.get("data_catalog", {}).get("enabled", False):
570
+ return
571
+
572
+ skill = skill if isinstance(skill, db.Skills) else self.skills_controller.get_skill(skill, project_name)
573
+ if skill.type == "sql":
574
+ if "database" in skill.params:
575
+ valid_table_names = skill.params.get("tables") if skill.params.get("tables") else tables
576
+ data_catalog_loader = DataCatalogLoader(
577
+ database_name=skill.params["database"], table_names=valid_table_names
578
+ )
579
+ data_catalog_loader.load_metadata()
580
+ else:
581
+ raise ValueError(
582
+ "Data Catalog loading is enabled, but the provided parameters for the new skills are insufficient to load metadata. "
583
+ )
584
+
585
+ def _run_data_catalog_loader_for_table_entries(
586
+ self,
587
+ table_entries: List[str],
588
+ project_name: str,
589
+ skill: Union[str, db.Skills] = None,
590
+ ):
591
+ """
592
+ Runs Data Catalog loader for a list of table entries if enabled in the config.
593
+ This is used to load metadata for SQL skills when they are added or updated.
594
+ """
595
+ if not config.get("data_catalog", {}).get("enabled", False):
596
+ return
597
+
598
+ skill = skill if isinstance(skill, db.Skills) else self.skills_controller.get_skill(skill, project_name)
599
+ if not skill or skill.type == "sql":
600
+ database_table_map = {}
601
+ for table_entry in table_entries:
602
+ parts = table_entry.split(".", 1)
603
+
604
+ # Ensure the table name is in 'database.table' format.
605
+ if len(parts) != 2:
606
+ logger.warning(
607
+ f"Invalid table name format: {table_entry}. Expected 'database.table' format."
608
+ "Metadata will not be loaded for this entry."
609
+ )
610
+ continue
611
+
612
+ database, table = parts[0], parts[1]
613
+
614
+ # Wildcards in database names are not supported at the moment by data catalog loader.
615
+ if "*" in database:
616
+ logger.warning(
617
+ f"Invalid database name format: {database}. Wildcards are not supported."
618
+ "Metadata will not be loaded for this entry."
619
+ )
620
+ continue
621
+
622
+ # Wildcards in table names are supported either.
623
+ # However, the table name itself can be a wildcard representing all tables.
624
+ if table == "*":
625
+ table = None
626
+ elif "*" in table:
627
+ logger.warning(
628
+ f"Invalid table name format: {table}. Wildcards are not supported."
629
+ "Metadata will not be loaded for this entry."
630
+ )
631
+ continue
632
+
633
+ database_table_map[database] = database_table_map.get(database, []) + [table]
634
+
635
+ for database_name, table_names in database_table_map.items():
636
+ data_catalog_loader = DataCatalogLoader(database_name=database_name, table_names=table_names)
637
+ data_catalog_loader.load_metadata()
638
+
546
639
  def delete_agent(self, agent_name: str, project_name: str = default_project):
547
640
  """
548
641
  Deletes an agent by name.
@@ -584,20 +677,22 @@ class AgentsController:
584
677
  def get_completion(
585
678
  self,
586
679
  agent: db.Agents,
587
- messages: List[Dict[str, str]],
680
+ messages: list[Dict[str, str]],
588
681
  project_name: str = default_project,
589
- tools: List[BaseTool] = None,
682
+ tools: list[BaseTool] = None,
590
683
  stream: bool = False,
684
+ params: dict | None = None,
591
685
  ) -> Union[Iterator[object], pd.DataFrame]:
592
686
  """
593
687
  Queries an agent to get a completion.
594
688
 
595
689
  Parameters:
596
690
  agent (db.Agents): Existing agent to get completion from
597
- messages (List[Dict[str, str]]): Chat history to send to the agent
691
+ messages (list[Dict[str, str]]): Chat history to send to the agent
598
692
  project_name (str): Project the agent belongs to (default mindsdb)
599
- tools (List[BaseTool]): Tools to use while getting the completion
693
+ tools (list[BaseTool]): Tools to use while getting the completion
600
694
  stream (bool): Whether to stream the response
695
+ params (dict | None): params to redefine agent params
601
696
 
602
697
  Returns:
603
698
  response (Union[Iterator[object], pd.DataFrame]): Completion as a DataFrame or iterator of completion chunks
@@ -606,7 +701,7 @@ class AgentsController:
606
701
  ValueError: Agent's model does not exist.
607
702
  """
608
703
  if stream:
609
- return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools)
704
+ return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools, params=params)
610
705
  from .langchain_agent import LangchainAgent
611
706
 
612
707
  model, provider = self.check_model_provider(agent.model_name, agent.provider)
@@ -619,25 +714,27 @@ class AgentsController:
619
714
  llm_params = self.get_agent_llm_params(agent.params)
620
715
 
621
716
  lang_agent = LangchainAgent(agent, model, llm_params=llm_params)
622
- return lang_agent.get_completion(messages)
717
+ return lang_agent.get_completion(messages, params=params)
623
718
 
624
719
  def _get_completion_stream(
625
720
  self,
626
721
  agent: db.Agents,
627
- messages: List[Dict[str, str]],
722
+ messages: list[Dict[str, str]],
628
723
  project_name: str = default_project,
629
- tools: List[BaseTool] = None,
724
+ tools: list[BaseTool] = None,
725
+ params: dict | None = None,
630
726
  ) -> Iterator[object]:
631
727
  """
632
728
  Queries an agent to get a stream of completion chunks.
633
729
 
634
730
  Parameters:
635
731
  agent (db.Agents): Existing agent to get completion from
636
- messages (List[Dict[str, str]]): Chat history to send to the agent
732
+ messages (list[Dict[str, str]]): Chat history to send to the agent
637
733
  trace_id (str): ID of Langfuse trace to use
638
734
  observation_id (str): ID of parent Langfuse observation to use
639
735
  project_name (str): Project the agent belongs to (default mindsdb)
640
- tools (List[BaseTool]): Tools to use while getting the completion
736
+ tools (list[BaseTool]): Tools to use while getting the completion
737
+ params (dict | None): params to redefine agent params
641
738
 
642
739
  Returns:
643
740
  chunks (Iterator[object]): Completion chunks as an iterator
@@ -659,4 +756,4 @@ class AgentsController:
659
756
  llm_params = self.get_agent_llm_params(agent.params)
660
757
 
661
758
  lang_agent = LangchainAgent(agent, model=model, llm_params=llm_params)
662
- return lang_agent.get_completion(messages, stream=True)
759
+ return lang_agent.get_completion(messages, stream=True, params=params)
@@ -26,7 +26,6 @@ OPEN_AI_CHAT_MODELS = (
26
26
  SUPPORTED_PROVIDERS = {
27
27
  "openai",
28
28
  "anthropic",
29
- "anyscale",
30
29
  "litellm",
31
30
  "ollama",
32
31
  "nvidia_nim",
@@ -11,7 +11,7 @@ import pandas as pd
11
11
  from langchain.agents import AgentExecutor
12
12
  from langchain.agents.initialize import initialize_agent
13
13
  from langchain.chains.conversation.memory import ConversationSummaryBufferMemory
14
- from langchain_community.chat_models import ChatAnyscale, ChatLiteLLM, ChatOllama
14
+ from langchain_community.chat_models import ChatLiteLLM, ChatOllama
15
15
  from langchain_writer import ChatWriter
16
16
  from langchain_google_genai import ChatGoogleGenerativeAI
17
17
  from langchain_core.agents import AgentAction, AgentStep
@@ -165,8 +165,6 @@ def create_chat_model(args: Dict):
165
165
  except NotImplementedError:
166
166
  chat_open_ai.tiktoken_model_name = DEFAULT_TIKTOKEN_MODEL_NAME
167
167
  return chat_open_ai
168
- if args["provider"] == "anyscale":
169
- return ChatAnyscale(**model_kwargs)
170
168
  if args["provider"] == "litellm":
171
169
  return ChatLiteLLM(**model_kwargs)
172
170
  if args["provider"] == "ollama":
@@ -321,7 +319,7 @@ class LangchainAgent:
321
319
  self.provider,
322
320
  ]
323
321
 
324
- def get_completion(self, messages, stream: bool = False):
322
+ def get_completion(self, messages, stream: bool = False, params: dict | None = None):
325
323
  # Get metadata and tags to be used in the trace
326
324
  metadata = self.get_metadata()
327
325
  tags = self.get_tags()
@@ -342,7 +340,9 @@ class LangchainAgent:
342
340
  if stream:
343
341
  return self._get_completion_stream(messages)
344
342
 
345
- args = self.args
343
+ args = {}
344
+ args.update(self.args)
345
+ args.update(params or {})
346
346
 
347
347
  df = pd.DataFrame(messages)
348
348
 
@@ -598,7 +598,12 @@ AI: {response}"""
598
598
  completions.append(result[ASSISTANT_COLUMN])
599
599
  contexts.append(result[CONTEXT_COLUMN])
600
600
  except TimeoutError:
601
- timeout_message = "I'm sorry! I couldn't come up with a response in time. Please try again."
601
+ timeout_message = (
602
+ f"I'm sorry! I couldn't generate a response within the allotted time ({agent_timeout_seconds} seconds). "
603
+ "If you need more time for processing, you can adjust the timeout settings. "
604
+ "Please refer to the documentation for instructions on how to change the timeout value. "
605
+ "Feel free to try your request again."
606
+ )
602
607
  logger.warning(f"Agent execution timed out after {agent_timeout_seconds} seconds")
603
608
  for _ in range(len(futures) - len(completions)):
604
609
  completions.append(timeout_message)