MindsDB 25.7.3.0__py3-none-any.whl → 25.8.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/__main__.py +11 -1
- mindsdb/api/a2a/common/server/server.py +16 -6
- mindsdb/api/executor/command_executor.py +215 -150
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +14 -3
- mindsdb/api/executor/planner/plan_join.py +3 -0
- mindsdb/api/executor/planner/plan_join_ts.py +117 -100
- mindsdb/api/executor/planner/query_planner.py +1 -0
- mindsdb/api/executor/sql_query/steps/apply_predictor_step.py +54 -85
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +21 -24
- mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py +9 -3
- mindsdb/api/executor/sql_query/steps/subselect_step.py +11 -8
- mindsdb/api/executor/utilities/mysql_to_duckdb_functions.py +264 -0
- mindsdb/api/executor/utilities/sql.py +30 -0
- mindsdb/api/http/initialize.py +18 -44
- mindsdb/api/http/namespaces/agents.py +23 -20
- mindsdb/api/http/namespaces/chatbots.py +83 -120
- mindsdb/api/http/namespaces/file.py +1 -1
- mindsdb/api/http/namespaces/jobs.py +38 -60
- mindsdb/api/http/namespaces/tree.py +69 -61
- mindsdb/api/http/namespaces/views.py +56 -72
- mindsdb/api/mcp/start.py +2 -0
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +3 -2
- mindsdb/integrations/handlers/autogluon_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/autosklearn_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/bigquery_handler/bigquery_handler.py +25 -5
- mindsdb/integrations/handlers/chromadb_handler/chromadb_handler.py +3 -3
- mindsdb/integrations/handlers/db2_handler/db2_handler.py +19 -23
- mindsdb/integrations/handlers/flaml_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/gong_handler/__about__.py +2 -0
- mindsdb/integrations/handlers/gong_handler/__init__.py +30 -0
- mindsdb/integrations/handlers/gong_handler/connection_args.py +37 -0
- mindsdb/integrations/handlers/gong_handler/gong_handler.py +164 -0
- mindsdb/integrations/handlers/gong_handler/gong_tables.py +508 -0
- mindsdb/integrations/handlers/gong_handler/icon.svg +25 -0
- mindsdb/integrations/handlers/gong_handler/test_gong_handler.py +125 -0
- mindsdb/integrations/handlers/google_calendar_handler/google_calendar_tables.py +82 -73
- mindsdb/integrations/handlers/hubspot_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/huggingface_handler/__init__.py +8 -12
- mindsdb/integrations/handlers/huggingface_handler/finetune.py +203 -223
- mindsdb/integrations/handlers/huggingface_handler/huggingface_handler.py +360 -383
- mindsdb/integrations/handlers/huggingface_handler/requirements.txt +7 -7
- mindsdb/integrations/handlers/huggingface_handler/requirements_cpu.txt +7 -7
- mindsdb/integrations/handlers/huggingface_handler/settings.py +25 -25
- mindsdb/integrations/handlers/langchain_handler/langchain_handler.py +83 -77
- mindsdb/integrations/handlers/lightwood_handler/requirements.txt +4 -4
- mindsdb/integrations/handlers/litellm_handler/litellm_handler.py +5 -2
- mindsdb/integrations/handlers/litellm_handler/settings.py +2 -1
- mindsdb/integrations/handlers/openai_handler/constants.py +11 -30
- mindsdb/integrations/handlers/openai_handler/helpers.py +27 -34
- mindsdb/integrations/handlers/openai_handler/openai_handler.py +14 -12
- mindsdb/integrations/handlers/pgvector_handler/pgvector_handler.py +106 -90
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +41 -39
- mindsdb/integrations/handlers/salesforce_handler/constants.py +215 -0
- mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py +141 -80
- mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py +0 -1
- mindsdb/integrations/handlers/tpot_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/web_handler/urlcrawl_helpers.py +32 -17
- mindsdb/integrations/handlers/web_handler/web_handler.py +19 -22
- mindsdb/integrations/libs/llm/config.py +0 -14
- mindsdb/integrations/libs/llm/utils.py +0 -15
- mindsdb/integrations/libs/vectordatabase_handler.py +10 -1
- mindsdb/integrations/utilities/files/file_reader.py +5 -19
- mindsdb/integrations/utilities/handler_utils.py +32 -12
- mindsdb/integrations/utilities/rag/rerankers/base_reranker.py +1 -1
- mindsdb/interfaces/agents/agents_controller.py +246 -149
- mindsdb/interfaces/agents/constants.py +0 -1
- mindsdb/interfaces/agents/langchain_agent.py +11 -6
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +4 -4
- mindsdb/interfaces/database/database.py +38 -13
- mindsdb/interfaces/database/integrations.py +20 -5
- mindsdb/interfaces/database/projects.py +174 -23
- mindsdb/interfaces/database/views.py +86 -60
- mindsdb/interfaces/jobs/jobs_controller.py +103 -110
- mindsdb/interfaces/knowledge_base/controller.py +33 -6
- mindsdb/interfaces/knowledge_base/evaluate.py +2 -1
- mindsdb/interfaces/knowledge_base/executor.py +24 -0
- mindsdb/interfaces/knowledge_base/preprocessing/document_preprocessor.py +6 -10
- mindsdb/interfaces/knowledge_base/preprocessing/text_splitter.py +73 -0
- mindsdb/interfaces/query_context/context_controller.py +111 -145
- mindsdb/interfaces/skills/skills_controller.py +18 -6
- mindsdb/interfaces/storage/db.py +40 -6
- mindsdb/interfaces/variables/variables_controller.py +8 -15
- mindsdb/utilities/config.py +5 -3
- mindsdb/utilities/fs.py +54 -17
- mindsdb/utilities/functions.py +72 -60
- mindsdb/utilities/log.py +38 -6
- mindsdb/utilities/ps.py +7 -7
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/METADATA +282 -268
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/RECORD +94 -92
- mindsdb/integrations/handlers/anyscale_endpoints_handler/__about__.py +0 -9
- mindsdb/integrations/handlers/anyscale_endpoints_handler/__init__.py +0 -20
- mindsdb/integrations/handlers/anyscale_endpoints_handler/anyscale_endpoints_handler.py +0 -290
- mindsdb/integrations/handlers/anyscale_endpoints_handler/creation_args.py +0 -14
- mindsdb/integrations/handlers/anyscale_endpoints_handler/icon.svg +0 -4
- mindsdb/integrations/handlers/anyscale_endpoints_handler/requirements.txt +0 -2
- mindsdb/integrations/handlers/anyscale_endpoints_handler/settings.py +0 -51
- mindsdb/integrations/handlers/anyscale_endpoints_handler/tests/test_anyscale_endpoints_handler.py +0 -212
- /mindsdb/integrations/handlers/{anyscale_endpoints_handler/tests/__init__.py → gong_handler/requirements.txt} +0 -0
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/WHEEL +0 -0
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.7.3.0.dist-info → mindsdb-25.8.2.0.dist-info}/top_level.txt +0 -0
|
@@ -20,7 +20,7 @@ from mindsdb.utilities import log
|
|
|
20
20
|
|
|
21
21
|
from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
|
|
22
22
|
|
|
23
|
-
from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
|
|
23
|
+
from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
|
|
24
24
|
from .langchain_agent import get_llm_provider
|
|
25
25
|
|
|
26
26
|
logger = log.getLogger(__name__)
|
|
@@ -145,11 +145,60 @@ class AgentsController:
|
|
|
145
145
|
|
|
146
146
|
return all_agents.all()
|
|
147
147
|
|
|
148
|
+
def _create_default_sql_skill(
|
|
149
|
+
self,
|
|
150
|
+
name,
|
|
151
|
+
project_name,
|
|
152
|
+
include_tables: List[str] = None,
|
|
153
|
+
include_knowledge_bases: List[str] = None,
|
|
154
|
+
):
|
|
155
|
+
# Create a default SQL skill
|
|
156
|
+
skill_name = f"{name}_sql_skill"
|
|
157
|
+
skill_params = {
|
|
158
|
+
"type": "sql",
|
|
159
|
+
"description": f"Auto-generated SQL skill for agent {name}",
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
# Add restrictions provided
|
|
163
|
+
if include_tables:
|
|
164
|
+
skill_params["include_tables"] = include_tables
|
|
165
|
+
if include_knowledge_bases:
|
|
166
|
+
skill_params["include_knowledge_bases"] = include_knowledge_bases
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
# Check if skill already exists
|
|
170
|
+
existing_skill = self.skills_controller.get_skill(skill_name, project_name)
|
|
171
|
+
if existing_skill is None:
|
|
172
|
+
# Create the skill
|
|
173
|
+
skill_type = skill_params.pop("type")
|
|
174
|
+
self.skills_controller.add_skill(
|
|
175
|
+
name=skill_name, project_name=project_name, type=skill_type, params=skill_params
|
|
176
|
+
)
|
|
177
|
+
else:
|
|
178
|
+
# Update the skill if parameters have changed
|
|
179
|
+
params_changed = False
|
|
180
|
+
|
|
181
|
+
# Check if skill parameters need to be updated
|
|
182
|
+
for param_key, param_value in skill_params.items():
|
|
183
|
+
if existing_skill.params.get(param_key) != param_value:
|
|
184
|
+
existing_skill.params[param_key] = param_value
|
|
185
|
+
params_changed = True
|
|
186
|
+
|
|
187
|
+
# Update the skill if needed
|
|
188
|
+
if params_changed:
|
|
189
|
+
flag_modified(existing_skill, "params")
|
|
190
|
+
db.session.commit()
|
|
191
|
+
|
|
192
|
+
except Exception as e:
|
|
193
|
+
raise ValueError(f"Failed to auto-create or update SQL skill: {str(e)}")
|
|
194
|
+
|
|
195
|
+
return skill_name
|
|
196
|
+
|
|
148
197
|
def add_agent(
|
|
149
198
|
self,
|
|
150
199
|
name: str,
|
|
151
200
|
project_name: str = None,
|
|
152
|
-
model_name: str = None,
|
|
201
|
+
model_name: Union[str, dict] = None,
|
|
153
202
|
skills: List[Union[str, dict]] = None,
|
|
154
203
|
provider: str = None,
|
|
155
204
|
params: Dict[str, Any] = None,
|
|
@@ -165,16 +214,21 @@ class AgentsController:
|
|
|
165
214
|
with one of keys is "name", and other is additional parameters for relationship agent<>skill
|
|
166
215
|
provider (str): The provider of the model
|
|
167
216
|
params (Dict[str, str]): Parameters to use when running the agent
|
|
217
|
+
data: Dict, data sources for an agent, keys:
|
|
218
|
+
- knowledge_bases: List of KBs to use
|
|
219
|
+
- tables: list of tables to use
|
|
220
|
+
model: Dict, parameters for the model to use
|
|
221
|
+
- provider: The provider of the model (e.g., 'openai', 'google')
|
|
222
|
+
- Other model-specific parameters like 'api_key', 'model_name', etc.
|
|
223
|
+
<provider>_api_key: API key for the provider (e.g., openai_api_key)
|
|
224
|
+
|
|
225
|
+
# Deprecated parameters:
|
|
168
226
|
database: The database to use for text2sql skills (default is 'mindsdb')
|
|
169
227
|
knowledge_base_database: The database to use for knowledge base queries (default is 'mindsdb')
|
|
170
228
|
include_tables: List of tables to include for text2sql skills
|
|
171
229
|
ignore_tables: List of tables to ignore for text2sql skills
|
|
172
230
|
include_knowledge_bases: List of knowledge bases to include for text2sql skills
|
|
173
231
|
ignore_knowledge_bases: List of knowledge bases to ignore for text2sql skills
|
|
174
|
-
<provider>_api_key: API key for the provider (e.g., openai_api_key)
|
|
175
|
-
data: Dict, data sources for an agent, keys:
|
|
176
|
-
- knowledge_bases: List of KBs to use (alternative to `include_knowledge_bases`)
|
|
177
|
-
- tables: list of tables to use (alternative to `include_tables`)
|
|
178
232
|
|
|
179
233
|
Returns:
|
|
180
234
|
agent (db.Agents): The created agent
|
|
@@ -223,99 +277,41 @@ class AgentsController:
|
|
|
223
277
|
# It will be picked up by get_api_key() in handler_utils.py
|
|
224
278
|
pass
|
|
225
279
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
params["database"] = database
|
|
280
|
+
depreciated_params = [
|
|
281
|
+
"database",
|
|
282
|
+
"knowledge_base_database",
|
|
283
|
+
"include_tables",
|
|
284
|
+
"ignore_tables",
|
|
285
|
+
"include_knowledge_bases",
|
|
286
|
+
"ignore_knowledge_bases",
|
|
287
|
+
]
|
|
288
|
+
if any(param in params for param in depreciated_params):
|
|
289
|
+
raise ValueError(
|
|
290
|
+
f"Parameters {', '.join(depreciated_params)} are deprecated. "
|
|
291
|
+
"Use 'data' parameter with 'tables' and 'knowledge_bases' keys instead."
|
|
292
|
+
)
|
|
240
293
|
|
|
294
|
+
include_tables = None
|
|
295
|
+
include_knowledge_bases = None
|
|
241
296
|
if "data" in params:
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
if include_tables is None:
|
|
245
|
-
include_tables = params["data"].get("tables")
|
|
246
|
-
|
|
247
|
-
if "knowledge_base_database" in params or include_knowledge_bases or ignore_knowledge_bases:
|
|
248
|
-
params["knowledge_base_database"] = knowledge_base_database
|
|
249
|
-
|
|
250
|
-
if include_tables is not None:
|
|
251
|
-
params["include_tables"] = include_tables
|
|
252
|
-
if ignore_tables is not None:
|
|
253
|
-
params["ignore_tables"] = ignore_tables
|
|
254
|
-
if include_knowledge_bases is not None:
|
|
255
|
-
params["include_knowledge_bases"] = include_knowledge_bases
|
|
256
|
-
if ignore_knowledge_bases is not None:
|
|
257
|
-
params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
297
|
+
include_knowledge_bases = params["data"].get("knowledge_bases")
|
|
298
|
+
include_tables = params["data"].get("tables")
|
|
258
299
|
|
|
259
300
|
# Convert string parameters to lists if needed
|
|
260
301
|
if isinstance(include_tables, str):
|
|
261
302
|
include_tables = [t.strip() for t in include_tables.split(",")]
|
|
262
|
-
if isinstance(ignore_tables, str):
|
|
263
|
-
ignore_tables = [t.strip() for t in ignore_tables.split(",")]
|
|
264
303
|
if isinstance(include_knowledge_bases, str):
|
|
265
304
|
include_knowledge_bases = [kb.strip() for kb in include_knowledge_bases.split(",")]
|
|
266
|
-
if isinstance(ignore_knowledge_bases, str):
|
|
267
|
-
ignore_knowledge_bases = [kb.strip() for kb in ignore_knowledge_bases.split(",")]
|
|
268
305
|
|
|
269
306
|
# Auto-create SQL skill if no skills are provided but include_tables or include_knowledge_bases params are provided
|
|
270
307
|
if not skills and (include_tables or include_knowledge_bases):
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
# Add table restrictions if provided
|
|
280
|
-
if include_tables:
|
|
281
|
-
skill_params["include_tables"] = include_tables
|
|
282
|
-
if ignore_tables:
|
|
283
|
-
skill_params["ignore_tables"] = ignore_tables
|
|
284
|
-
|
|
285
|
-
# Add knowledge base parameters if provided
|
|
286
|
-
if knowledge_base_database:
|
|
287
|
-
skill_params["knowledge_base_database"] = knowledge_base_database
|
|
288
|
-
if include_knowledge_bases:
|
|
289
|
-
skill_params["include_knowledge_bases"] = include_knowledge_bases
|
|
290
|
-
if ignore_knowledge_bases:
|
|
291
|
-
skill_params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
292
|
-
try:
|
|
293
|
-
# Check if skill already exists
|
|
294
|
-
existing_skill = self.skills_controller.get_skill(skill_name, project_name)
|
|
295
|
-
if existing_skill is None:
|
|
296
|
-
# Create the skill
|
|
297
|
-
skill_type = skill_params.pop("type")
|
|
298
|
-
self.skills_controller.add_skill(
|
|
299
|
-
name=skill_name, project_name=project_name, type=skill_type, params=skill_params
|
|
300
|
-
)
|
|
301
|
-
else:
|
|
302
|
-
# Update the skill if parameters have changed
|
|
303
|
-
params_changed = False
|
|
304
|
-
|
|
305
|
-
# Check if skill parameters need to be updated
|
|
306
|
-
for param_key, param_value in skill_params.items():
|
|
307
|
-
if existing_skill.params.get(param_key) != param_value:
|
|
308
|
-
existing_skill.params[param_key] = param_value
|
|
309
|
-
params_changed = True
|
|
310
|
-
|
|
311
|
-
# Update the skill if needed
|
|
312
|
-
if params_changed:
|
|
313
|
-
flag_modified(existing_skill, "params")
|
|
314
|
-
db.session.commit()
|
|
315
|
-
|
|
316
|
-
skills = [skill_name]
|
|
317
|
-
except Exception as e:
|
|
318
|
-
raise ValueError(f"Failed to auto-create or update SQL skill: {str(e)}")
|
|
308
|
+
skill = self._create_default_sql_skill(
|
|
309
|
+
name,
|
|
310
|
+
project_name,
|
|
311
|
+
include_tables=include_tables,
|
|
312
|
+
include_knowledge_bases=include_knowledge_bases,
|
|
313
|
+
)
|
|
314
|
+
skills = [skill]
|
|
319
315
|
|
|
320
316
|
agent = db.Agents(
|
|
321
317
|
name=name,
|
|
@@ -340,44 +336,16 @@ class AgentsController:
|
|
|
340
336
|
db.session.rollback()
|
|
341
337
|
raise ValueError(f"Skill with name does not exist: {skill_name}")
|
|
342
338
|
|
|
343
|
-
if
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
for table in include_tables:
|
|
349
|
-
parts = table.split(".", 1)
|
|
350
|
-
database_table_map[parts[0]] = database_table_map.get(parts[0], []) + [parts[1]]
|
|
351
|
-
|
|
352
|
-
for database_name, table_names in database_table_map.items():
|
|
353
|
-
data_catalog_loader = DataCatalogLoader(
|
|
354
|
-
database_name=database_name, table_names=table_names
|
|
355
|
-
)
|
|
356
|
-
data_catalog_loader.load_metadata()
|
|
357
|
-
|
|
358
|
-
elif "database" in existing_skill.params:
|
|
359
|
-
data_catalog_loader = DataCatalogLoader(
|
|
360
|
-
database_name=existing_skill.params["database"],
|
|
361
|
-
table_names=parameters["tables"] if "tables" in parameters else None,
|
|
362
|
-
)
|
|
363
|
-
data_catalog_loader.load_metadata()
|
|
364
|
-
|
|
365
|
-
else:
|
|
366
|
-
raise ValueError(
|
|
367
|
-
"Data Catalog loading is enabled, but the provided parameters are insufficient to load metadata. "
|
|
368
|
-
)
|
|
339
|
+
# Run Data Catalog loader if enabled.
|
|
340
|
+
if include_tables:
|
|
341
|
+
self._run_data_catalog_loader_for_table_entries(include_tables, project_name, skill=existing_skill)
|
|
342
|
+
else:
|
|
343
|
+
self._run_data_catalog_loader_for_skill(existing_skill, project_name, tables=parameters.get("tables"))
|
|
369
344
|
|
|
345
|
+
if existing_skill.type == "sql":
|
|
370
346
|
# Add table restrictions if this is a text2sql skill
|
|
371
|
-
if include_tables
|
|
372
|
-
parameters["tables"] = include_tables
|
|
373
|
-
|
|
374
|
-
# Pass database parameter if provided
|
|
375
|
-
if database and "database" not in parameters:
|
|
376
|
-
parameters["database"] = database
|
|
377
|
-
|
|
378
|
-
# Pass knowledge base database parameter if provided
|
|
379
|
-
if knowledge_base_database and "knowledge_base_database" not in parameters:
|
|
380
|
-
parameters["knowledge_base_database"] = knowledge_base_database
|
|
347
|
+
if include_tables:
|
|
348
|
+
parameters["tables"] = include_tables
|
|
381
349
|
|
|
382
350
|
# Add knowledge base parameters to both the skill and the association parameters
|
|
383
351
|
if include_knowledge_bases:
|
|
@@ -385,16 +353,6 @@ class AgentsController:
|
|
|
385
353
|
if "include_knowledge_bases" not in existing_skill.params:
|
|
386
354
|
existing_skill.params["include_knowledge_bases"] = include_knowledge_bases
|
|
387
355
|
flag_modified(existing_skill, "params")
|
|
388
|
-
elif ignore_knowledge_bases:
|
|
389
|
-
parameters["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
390
|
-
if "ignore_knowledge_bases" not in existing_skill.params:
|
|
391
|
-
existing_skill.params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
392
|
-
flag_modified(existing_skill, "params")
|
|
393
|
-
|
|
394
|
-
# Ensure knowledge_base_database is set in the skill's params
|
|
395
|
-
if knowledge_base_database and "knowledge_base_database" not in existing_skill.params:
|
|
396
|
-
existing_skill.params["knowledge_base_database"] = knowledge_base_database
|
|
397
|
-
flag_modified(existing_skill, "params")
|
|
398
356
|
|
|
399
357
|
association = db.AgentSkillsAssociation(parameters=parameters, agent=agent, skill=existing_skill)
|
|
400
358
|
db.session.add(association)
|
|
@@ -409,7 +367,7 @@ class AgentsController:
|
|
|
409
367
|
agent_name: str,
|
|
410
368
|
project_name: str = default_project,
|
|
411
369
|
name: str = None,
|
|
412
|
-
model_name: str = None,
|
|
370
|
+
model_name: Union[str, dict] = None,
|
|
413
371
|
skills_to_add: List[Union[str, dict]] = None,
|
|
414
372
|
skills_to_remove: List[str] = None,
|
|
415
373
|
skills_to_rewrite: List[Union[str, dict]] = None,
|
|
@@ -423,7 +381,7 @@ class AgentsController:
|
|
|
423
381
|
agent_name (str): The name of the new agent, or existing agent to update
|
|
424
382
|
project_name (str): The containing project
|
|
425
383
|
name (str): The updated name of the agent
|
|
426
|
-
model_name (str): The name of the existing ML model the agent will use
|
|
384
|
+
model_name (str | dict): The name of the existing ML model the agent will use
|
|
427
385
|
skills_to_add (List[Union[str, dict]]): List of skill names to add to the agent, or list of dicts
|
|
428
386
|
with one of keys is "name", and other is additional parameters for relationship agent<>skill
|
|
429
387
|
skills_to_remove (List[str]): List of skill names to remove from the agent
|
|
@@ -452,6 +410,8 @@ class AgentsController:
|
|
|
452
410
|
existing_agent = self.get_agent(agent_name, project_name=project_name)
|
|
453
411
|
if existing_agent is None:
|
|
454
412
|
raise EntityNotExistsError(f"Agent with name not found: {agent_name}")
|
|
413
|
+
existing_params = existing_agent.params or {}
|
|
414
|
+
|
|
455
415
|
is_demo = (existing_agent.params or {}).get("is_demo", False)
|
|
456
416
|
if is_demo and (
|
|
457
417
|
(name is not None and name != agent_name)
|
|
@@ -462,6 +422,8 @@ class AgentsController:
|
|
|
462
422
|
raise ValueError("It is forbidden to change properties of the demo object")
|
|
463
423
|
|
|
464
424
|
if name is not None and name != agent_name:
|
|
425
|
+
if not name.islower():
|
|
426
|
+
raise ValueError(f"The name must be in lower case: {name}")
|
|
465
427
|
# Check to see if updated name already exists
|
|
466
428
|
agent_with_new_name = self.get_agent(name, project_name=project_name)
|
|
467
429
|
if agent_with_new_name is not None:
|
|
@@ -469,12 +431,34 @@ class AgentsController:
|
|
|
469
431
|
existing_agent.name = name
|
|
470
432
|
|
|
471
433
|
if model_name or provider:
|
|
434
|
+
if isinstance(model_name, dict):
|
|
435
|
+
# move into params
|
|
436
|
+
existing_params["model"] = model_name
|
|
437
|
+
model_name = None
|
|
438
|
+
|
|
472
439
|
# check model and provider
|
|
473
440
|
model, provider = self.check_model_provider(model_name, provider)
|
|
474
441
|
# Update model and provider
|
|
475
442
|
existing_agent.model_name = model_name
|
|
476
443
|
existing_agent.provider = provider
|
|
477
444
|
|
|
445
|
+
if "data" in params:
|
|
446
|
+
if len(skills_to_add) > 0 or len(skills_to_remove) > 0:
|
|
447
|
+
raise ValueError(
|
|
448
|
+
"'data' parameter cannot be used with 'skills_to_remove' or 'skills_to_add' parameters"
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
include_knowledge_bases = params["data"].get("knowledge_bases")
|
|
452
|
+
include_tables = params["data"].get("tables")
|
|
453
|
+
|
|
454
|
+
skill = self._create_default_sql_skill(
|
|
455
|
+
agent_name,
|
|
456
|
+
project_name,
|
|
457
|
+
include_tables=include_tables,
|
|
458
|
+
include_knowledge_bases=include_knowledge_bases,
|
|
459
|
+
)
|
|
460
|
+
skills_to_rewrite = [{"name": skill}]
|
|
461
|
+
|
|
478
462
|
# check that all skills exist
|
|
479
463
|
skill_name_to_record_map = {}
|
|
480
464
|
for skill_meta in skills_to_add + skills_to_remove + skills_to_rewrite:
|
|
@@ -503,12 +487,20 @@ class AgentsController:
|
|
|
503
487
|
|
|
504
488
|
# add skills
|
|
505
489
|
for skill_name in set(skills_to_add_names) - set(existing_agent_skills_names):
|
|
490
|
+
# Run Data Catalog loader if enabled for the new skill
|
|
491
|
+
self._run_data_catalog_loader_for_skill(
|
|
492
|
+
skill_name,
|
|
493
|
+
project_name,
|
|
494
|
+
tables=next((x for x in skills_to_add if x["name"] == skill_name), {}).get("tables"),
|
|
495
|
+
)
|
|
496
|
+
|
|
506
497
|
skill_parameters = next(x for x in skills_to_add if x["name"] == skill_name).copy()
|
|
507
498
|
del skill_parameters["name"]
|
|
508
499
|
association = db.AgentSkillsAssociation(
|
|
509
500
|
parameters=skill_parameters, agent=existing_agent, skill=skill_name_to_record_map[skill_name]
|
|
510
501
|
)
|
|
511
502
|
db.session.add(association)
|
|
503
|
+
|
|
512
504
|
elif len(skills_to_rewrite) > 0:
|
|
513
505
|
skill_name_to_parameters = {
|
|
514
506
|
x["name"]: {k: v for k, v in x.items() if k != "name"} for x in skills_to_rewrite
|
|
@@ -519,9 +511,23 @@ class AgentsController:
|
|
|
519
511
|
db.session.delete(rel)
|
|
520
512
|
else:
|
|
521
513
|
existing_skill_names.add(rel.skill.name)
|
|
522
|
-
|
|
514
|
+
skill_parameters = skill_name_to_parameters[rel.skill.name]
|
|
515
|
+
|
|
516
|
+
# Run Data Catalog loader if enabled for the updated skill
|
|
517
|
+
self._run_data_catalog_loader_for_skill(
|
|
518
|
+
rel.skill.name, project_name, tables=skill_parameters.get("tables")
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
rel.parameters = skill_parameters
|
|
523
522
|
flag_modified(rel, "parameters")
|
|
524
523
|
for new_skill_name in set(skill_name_to_parameters) - existing_skill_names:
|
|
524
|
+
# Run Data Catalog loader if enabled for the new skill
|
|
525
|
+
self._run_data_catalog_loader_for_skill(
|
|
526
|
+
new_skill_name,
|
|
527
|
+
project_name,
|
|
528
|
+
tables=skill_name_to_parameters[new_skill_name].get("tables"),
|
|
529
|
+
)
|
|
530
|
+
|
|
525
531
|
association = db.AgentSkillsAssociation(
|
|
526
532
|
parameters=skill_name_to_parameters[new_skill_name],
|
|
527
533
|
agent=existing_agent,
|
|
@@ -530,8 +536,15 @@ class AgentsController:
|
|
|
530
536
|
db.session.add(association)
|
|
531
537
|
|
|
532
538
|
if params is not None:
|
|
539
|
+
if params.get("data", {}).get("tables"):
|
|
540
|
+
new_table_entries = set(params["data"]["tables"]) - set(
|
|
541
|
+
existing_params.get("data", {}).get("tables", [])
|
|
542
|
+
)
|
|
543
|
+
if new_table_entries:
|
|
544
|
+
# Run Data Catalog loader for new table entries if enabled.
|
|
545
|
+
self._run_data_catalog_loader_for_table_entries(new_table_entries, project_name)
|
|
546
|
+
|
|
533
547
|
# Merge params on update
|
|
534
|
-
existing_params = existing_agent.params or {}
|
|
535
548
|
existing_params.update(params)
|
|
536
549
|
# Remove None values entirely.
|
|
537
550
|
params = {k: v for k, v in existing_params.items() if v is not None}
|
|
@@ -543,6 +556,86 @@ class AgentsController:
|
|
|
543
556
|
|
|
544
557
|
return existing_agent
|
|
545
558
|
|
|
559
|
+
def _run_data_catalog_loader_for_skill(
|
|
560
|
+
self,
|
|
561
|
+
skill: Union[str, db.Skills],
|
|
562
|
+
project_name: str,
|
|
563
|
+
tables: List[str] = None,
|
|
564
|
+
):
|
|
565
|
+
"""
|
|
566
|
+
Runs Data Catalog loader for a skill if enabled in the config.
|
|
567
|
+
This is used to load metadata for SQL skills when they are added or updated.
|
|
568
|
+
"""
|
|
569
|
+
if not config.get("data_catalog", {}).get("enabled", False):
|
|
570
|
+
return
|
|
571
|
+
|
|
572
|
+
skill = skill if isinstance(skill, db.Skills) else self.skills_controller.get_skill(skill, project_name)
|
|
573
|
+
if skill.type == "sql":
|
|
574
|
+
if "database" in skill.params:
|
|
575
|
+
valid_table_names = skill.params.get("tables") if skill.params.get("tables") else tables
|
|
576
|
+
data_catalog_loader = DataCatalogLoader(
|
|
577
|
+
database_name=skill.params["database"], table_names=valid_table_names
|
|
578
|
+
)
|
|
579
|
+
data_catalog_loader.load_metadata()
|
|
580
|
+
else:
|
|
581
|
+
raise ValueError(
|
|
582
|
+
"Data Catalog loading is enabled, but the provided parameters for the new skills are insufficient to load metadata. "
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
def _run_data_catalog_loader_for_table_entries(
|
|
586
|
+
self,
|
|
587
|
+
table_entries: List[str],
|
|
588
|
+
project_name: str,
|
|
589
|
+
skill: Union[str, db.Skills] = None,
|
|
590
|
+
):
|
|
591
|
+
"""
|
|
592
|
+
Runs Data Catalog loader for a list of table entries if enabled in the config.
|
|
593
|
+
This is used to load metadata for SQL skills when they are added or updated.
|
|
594
|
+
"""
|
|
595
|
+
if not config.get("data_catalog", {}).get("enabled", False):
|
|
596
|
+
return
|
|
597
|
+
|
|
598
|
+
skill = skill if isinstance(skill, db.Skills) else self.skills_controller.get_skill(skill, project_name)
|
|
599
|
+
if not skill or skill.type == "sql":
|
|
600
|
+
database_table_map = {}
|
|
601
|
+
for table_entry in table_entries:
|
|
602
|
+
parts = table_entry.split(".", 1)
|
|
603
|
+
|
|
604
|
+
# Ensure the table name is in 'database.table' format.
|
|
605
|
+
if len(parts) != 2:
|
|
606
|
+
logger.warning(
|
|
607
|
+
f"Invalid table name format: {table_entry}. Expected 'database.table' format."
|
|
608
|
+
"Metadata will not be loaded for this entry."
|
|
609
|
+
)
|
|
610
|
+
continue
|
|
611
|
+
|
|
612
|
+
database, table = parts[0], parts[1]
|
|
613
|
+
|
|
614
|
+
# Wildcards in database names are not supported at the moment by data catalog loader.
|
|
615
|
+
if "*" in database:
|
|
616
|
+
logger.warning(
|
|
617
|
+
f"Invalid database name format: {database}. Wildcards are not supported."
|
|
618
|
+
"Metadata will not be loaded for this entry."
|
|
619
|
+
)
|
|
620
|
+
continue
|
|
621
|
+
|
|
622
|
+
# Wildcards in table names are supported either.
|
|
623
|
+
# However, the table name itself can be a wildcard representing all tables.
|
|
624
|
+
if table == "*":
|
|
625
|
+
table = None
|
|
626
|
+
elif "*" in table:
|
|
627
|
+
logger.warning(
|
|
628
|
+
f"Invalid table name format: {table}. Wildcards are not supported."
|
|
629
|
+
"Metadata will not be loaded for this entry."
|
|
630
|
+
)
|
|
631
|
+
continue
|
|
632
|
+
|
|
633
|
+
database_table_map[database] = database_table_map.get(database, []) + [table]
|
|
634
|
+
|
|
635
|
+
for database_name, table_names in database_table_map.items():
|
|
636
|
+
data_catalog_loader = DataCatalogLoader(database_name=database_name, table_names=table_names)
|
|
637
|
+
data_catalog_loader.load_metadata()
|
|
638
|
+
|
|
546
639
|
def delete_agent(self, agent_name: str, project_name: str = default_project):
|
|
547
640
|
"""
|
|
548
641
|
Deletes an agent by name.
|
|
@@ -584,20 +677,22 @@ class AgentsController:
|
|
|
584
677
|
def get_completion(
|
|
585
678
|
self,
|
|
586
679
|
agent: db.Agents,
|
|
587
|
-
messages:
|
|
680
|
+
messages: list[Dict[str, str]],
|
|
588
681
|
project_name: str = default_project,
|
|
589
|
-
tools:
|
|
682
|
+
tools: list[BaseTool] = None,
|
|
590
683
|
stream: bool = False,
|
|
684
|
+
params: dict | None = None,
|
|
591
685
|
) -> Union[Iterator[object], pd.DataFrame]:
|
|
592
686
|
"""
|
|
593
687
|
Queries an agent to get a completion.
|
|
594
688
|
|
|
595
689
|
Parameters:
|
|
596
690
|
agent (db.Agents): Existing agent to get completion from
|
|
597
|
-
messages (
|
|
691
|
+
messages (list[Dict[str, str]]): Chat history to send to the agent
|
|
598
692
|
project_name (str): Project the agent belongs to (default mindsdb)
|
|
599
|
-
tools (
|
|
693
|
+
tools (list[BaseTool]): Tools to use while getting the completion
|
|
600
694
|
stream (bool): Whether to stream the response
|
|
695
|
+
params (dict | None): params to redefine agent params
|
|
601
696
|
|
|
602
697
|
Returns:
|
|
603
698
|
response (Union[Iterator[object], pd.DataFrame]): Completion as a DataFrame or iterator of completion chunks
|
|
@@ -606,7 +701,7 @@ class AgentsController:
|
|
|
606
701
|
ValueError: Agent's model does not exist.
|
|
607
702
|
"""
|
|
608
703
|
if stream:
|
|
609
|
-
return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools)
|
|
704
|
+
return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools, params=params)
|
|
610
705
|
from .langchain_agent import LangchainAgent
|
|
611
706
|
|
|
612
707
|
model, provider = self.check_model_provider(agent.model_name, agent.provider)
|
|
@@ -619,25 +714,27 @@ class AgentsController:
|
|
|
619
714
|
llm_params = self.get_agent_llm_params(agent.params)
|
|
620
715
|
|
|
621
716
|
lang_agent = LangchainAgent(agent, model, llm_params=llm_params)
|
|
622
|
-
return lang_agent.get_completion(messages)
|
|
717
|
+
return lang_agent.get_completion(messages, params=params)
|
|
623
718
|
|
|
624
719
|
def _get_completion_stream(
|
|
625
720
|
self,
|
|
626
721
|
agent: db.Agents,
|
|
627
|
-
messages:
|
|
722
|
+
messages: list[Dict[str, str]],
|
|
628
723
|
project_name: str = default_project,
|
|
629
|
-
tools:
|
|
724
|
+
tools: list[BaseTool] = None,
|
|
725
|
+
params: dict | None = None,
|
|
630
726
|
) -> Iterator[object]:
|
|
631
727
|
"""
|
|
632
728
|
Queries an agent to get a stream of completion chunks.
|
|
633
729
|
|
|
634
730
|
Parameters:
|
|
635
731
|
agent (db.Agents): Existing agent to get completion from
|
|
636
|
-
messages (
|
|
732
|
+
messages (list[Dict[str, str]]): Chat history to send to the agent
|
|
637
733
|
trace_id (str): ID of Langfuse trace to use
|
|
638
734
|
observation_id (str): ID of parent Langfuse observation to use
|
|
639
735
|
project_name (str): Project the agent belongs to (default mindsdb)
|
|
640
|
-
tools (
|
|
736
|
+
tools (list[BaseTool]): Tools to use while getting the completion
|
|
737
|
+
params (dict | None): params to redefine agent params
|
|
641
738
|
|
|
642
739
|
Returns:
|
|
643
740
|
chunks (Iterator[object]): Completion chunks as an iterator
|
|
@@ -659,4 +756,4 @@ class AgentsController:
|
|
|
659
756
|
llm_params = self.get_agent_llm_params(agent.params)
|
|
660
757
|
|
|
661
758
|
lang_agent = LangchainAgent(agent, model=model, llm_params=llm_params)
|
|
662
|
-
return lang_agent.get_completion(messages, stream=True)
|
|
759
|
+
return lang_agent.get_completion(messages, stream=True, params=params)
|
|
@@ -11,7 +11,7 @@ import pandas as pd
|
|
|
11
11
|
from langchain.agents import AgentExecutor
|
|
12
12
|
from langchain.agents.initialize import initialize_agent
|
|
13
13
|
from langchain.chains.conversation.memory import ConversationSummaryBufferMemory
|
|
14
|
-
from langchain_community.chat_models import
|
|
14
|
+
from langchain_community.chat_models import ChatLiteLLM, ChatOllama
|
|
15
15
|
from langchain_writer import ChatWriter
|
|
16
16
|
from langchain_google_genai import ChatGoogleGenerativeAI
|
|
17
17
|
from langchain_core.agents import AgentAction, AgentStep
|
|
@@ -165,8 +165,6 @@ def create_chat_model(args: Dict):
|
|
|
165
165
|
except NotImplementedError:
|
|
166
166
|
chat_open_ai.tiktoken_model_name = DEFAULT_TIKTOKEN_MODEL_NAME
|
|
167
167
|
return chat_open_ai
|
|
168
|
-
if args["provider"] == "anyscale":
|
|
169
|
-
return ChatAnyscale(**model_kwargs)
|
|
170
168
|
if args["provider"] == "litellm":
|
|
171
169
|
return ChatLiteLLM(**model_kwargs)
|
|
172
170
|
if args["provider"] == "ollama":
|
|
@@ -321,7 +319,7 @@ class LangchainAgent:
|
|
|
321
319
|
self.provider,
|
|
322
320
|
]
|
|
323
321
|
|
|
324
|
-
def get_completion(self, messages, stream: bool = False):
|
|
322
|
+
def get_completion(self, messages, stream: bool = False, params: dict | None = None):
|
|
325
323
|
# Get metadata and tags to be used in the trace
|
|
326
324
|
metadata = self.get_metadata()
|
|
327
325
|
tags = self.get_tags()
|
|
@@ -342,7 +340,9 @@ class LangchainAgent:
|
|
|
342
340
|
if stream:
|
|
343
341
|
return self._get_completion_stream(messages)
|
|
344
342
|
|
|
345
|
-
args =
|
|
343
|
+
args = {}
|
|
344
|
+
args.update(self.args)
|
|
345
|
+
args.update(params or {})
|
|
346
346
|
|
|
347
347
|
df = pd.DataFrame(messages)
|
|
348
348
|
|
|
@@ -598,7 +598,12 @@ AI: {response}"""
|
|
|
598
598
|
completions.append(result[ASSISTANT_COLUMN])
|
|
599
599
|
contexts.append(result[CONTEXT_COLUMN])
|
|
600
600
|
except TimeoutError:
|
|
601
|
-
timeout_message =
|
|
601
|
+
timeout_message = (
|
|
602
|
+
f"I'm sorry! I couldn't generate a response within the allotted time ({agent_timeout_seconds} seconds). "
|
|
603
|
+
"If you need more time for processing, you can adjust the timeout settings. "
|
|
604
|
+
"Please refer to the documentation for instructions on how to change the timeout value. "
|
|
605
|
+
"Feel free to try your request again."
|
|
606
|
+
)
|
|
602
607
|
logger.warning(f"Agent execution timed out after {agent_timeout_seconds} seconds")
|
|
603
608
|
for _ in range(len(futures) - len(completions)):
|
|
604
609
|
completions.append(timeout_message)
|