MindsDB 25.5.4.2__py3-none-any.whl → 25.6.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of MindsDB might be problematic. Click here for more details.
- mindsdb/__about__.py +1 -1
- mindsdb/api/a2a/agent.py +50 -26
- mindsdb/api/a2a/common/server/server.py +32 -26
- mindsdb/api/a2a/task_manager.py +68 -6
- mindsdb/api/executor/command_executor.py +69 -14
- mindsdb/api/executor/datahub/datanodes/integration_datanode.py +49 -65
- mindsdb/api/executor/datahub/datanodes/mindsdb_tables.py +91 -84
- mindsdb/api/executor/datahub/datanodes/project_datanode.py +29 -48
- mindsdb/api/executor/datahub/datanodes/system_tables.py +35 -61
- mindsdb/api/executor/planner/plan_join.py +67 -77
- mindsdb/api/executor/planner/query_planner.py +176 -155
- mindsdb/api/executor/planner/steps.py +37 -12
- mindsdb/api/executor/sql_query/result_set.py +45 -64
- mindsdb/api/executor/sql_query/steps/fetch_dataframe.py +14 -18
- mindsdb/api/executor/sql_query/steps/fetch_dataframe_partition.py +17 -18
- mindsdb/api/executor/sql_query/steps/insert_step.py +13 -33
- mindsdb/api/executor/sql_query/steps/subselect_step.py +43 -35
- mindsdb/api/executor/utilities/sql.py +42 -48
- mindsdb/api/http/namespaces/config.py +1 -1
- mindsdb/api/http/namespaces/file.py +14 -23
- mindsdb/api/http/namespaces/knowledge_bases.py +132 -154
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_datum.py +12 -28
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/binary_resultset_row_package.py +59 -50
- mindsdb/api/mysql/mysql_proxy/data_types/mysql_packets/resultset_row_package.py +9 -8
- mindsdb/api/mysql/mysql_proxy/libs/constants/mysql.py +449 -461
- mindsdb/api/mysql/mysql_proxy/utilities/dump.py +87 -36
- mindsdb/integrations/handlers/bigquery_handler/bigquery_handler.py +219 -28
- mindsdb/integrations/handlers/file_handler/file_handler.py +15 -9
- mindsdb/integrations/handlers/file_handler/tests/test_file_handler.py +43 -24
- mindsdb/integrations/handlers/litellm_handler/litellm_handler.py +10 -3
- mindsdb/integrations/handlers/llama_index_handler/requirements.txt +1 -1
- mindsdb/integrations/handlers/mysql_handler/mysql_handler.py +29 -33
- mindsdb/integrations/handlers/openai_handler/openai_handler.py +277 -356
- mindsdb/integrations/handlers/oracle_handler/oracle_handler.py +74 -51
- mindsdb/integrations/handlers/postgres_handler/postgres_handler.py +305 -98
- mindsdb/integrations/handlers/salesforce_handler/salesforce_handler.py +145 -40
- mindsdb/integrations/handlers/salesforce_handler/salesforce_tables.py +136 -6
- mindsdb/integrations/handlers/snowflake_handler/snowflake_handler.py +352 -83
- mindsdb/integrations/libs/api_handler.py +279 -57
- mindsdb/integrations/libs/base.py +185 -30
- mindsdb/integrations/utilities/files/file_reader.py +99 -73
- mindsdb/integrations/utilities/handler_utils.py +23 -8
- mindsdb/integrations/utilities/sql_utils.py +35 -40
- mindsdb/interfaces/agents/agents_controller.py +226 -196
- mindsdb/interfaces/agents/constants.py +8 -1
- mindsdb/interfaces/agents/langchain_agent.py +42 -11
- mindsdb/interfaces/agents/mcp_client_agent.py +29 -21
- mindsdb/interfaces/agents/mindsdb_database_agent.py +23 -18
- mindsdb/interfaces/data_catalog/__init__.py +0 -0
- mindsdb/interfaces/data_catalog/base_data_catalog.py +54 -0
- mindsdb/interfaces/data_catalog/data_catalog_loader.py +375 -0
- mindsdb/interfaces/data_catalog/data_catalog_reader.py +38 -0
- mindsdb/interfaces/database/database.py +81 -57
- mindsdb/interfaces/database/integrations.py +222 -234
- mindsdb/interfaces/database/log.py +72 -104
- mindsdb/interfaces/database/projects.py +156 -193
- mindsdb/interfaces/file/file_controller.py +21 -65
- mindsdb/interfaces/knowledge_base/controller.py +66 -25
- mindsdb/interfaces/knowledge_base/evaluate.py +516 -0
- mindsdb/interfaces/knowledge_base/llm_client.py +75 -0
- mindsdb/interfaces/skills/custom/text2sql/mindsdb_kb_tools.py +83 -43
- mindsdb/interfaces/skills/skills_controller.py +31 -36
- mindsdb/interfaces/skills/sql_agent.py +113 -86
- mindsdb/interfaces/storage/db.py +242 -82
- mindsdb/migrations/versions/2025-05-28_a44643042fe8_added_data_catalog_tables.py +118 -0
- mindsdb/migrations/versions/2025-06-09_608e376c19a7_updated_data_catalog_data_types.py +58 -0
- mindsdb/utilities/config.py +13 -2
- mindsdb/utilities/log.py +35 -26
- mindsdb/utilities/ml_task_queue/task.py +19 -22
- mindsdb/utilities/render/sqlalchemy_render.py +129 -181
- mindsdb/utilities/starters.py +40 -0
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.3.0.dist-info}/METADATA +257 -257
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.3.0.dist-info}/RECORD +76 -68
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.3.0.dist-info}/WHEEL +0 -0
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.3.0.dist-info}/licenses/LICENSE +0 -0
- {mindsdb-25.5.4.2.dist-info → mindsdb-25.6.3.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import datetime
|
|
2
|
-
from typing import Dict, Iterator, List, Union, Tuple, Optional
|
|
2
|
+
from typing import Dict, Iterator, List, Union, Tuple, Optional, Any
|
|
3
|
+
import copy
|
|
3
4
|
|
|
4
5
|
from langchain_core.tools import BaseTool
|
|
5
6
|
from sqlalchemy.orm.attributes import flag_modified
|
|
@@ -9,23 +10,26 @@ import pandas as pd
|
|
|
9
10
|
from mindsdb.interfaces.storage import db
|
|
10
11
|
from mindsdb.interfaces.storage.db import Predictor
|
|
11
12
|
from mindsdb.utilities.context import context as ctx
|
|
13
|
+
from mindsdb.interfaces.data_catalog.data_catalog_loader import DataCatalogLoader
|
|
12
14
|
from mindsdb.interfaces.database.projects import ProjectController
|
|
13
15
|
from mindsdb.interfaces.model.functions import PredictorRecordNotFound
|
|
14
16
|
from mindsdb.interfaces.model.model_controller import ModelController
|
|
15
17
|
from mindsdb.interfaces.skills.skills_controller import SkillsController
|
|
16
18
|
from mindsdb.utilities.config import config
|
|
19
|
+
from mindsdb.utilities import log
|
|
20
|
+
|
|
17
21
|
from mindsdb.utilities.exception import EntityExistsError, EntityNotExistsError
|
|
18
22
|
|
|
19
|
-
from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS
|
|
23
|
+
from .constants import ASSISTANT_COLUMN, SUPPORTED_PROVIDERS, PROVIDER_TO_MODELS, DEFAULT_TEXT2SQL_DATABASE
|
|
20
24
|
from .langchain_agent import get_llm_provider
|
|
21
25
|
|
|
22
|
-
|
|
26
|
+
logger = log.getLogger(__name__)
|
|
23
27
|
|
|
24
|
-
|
|
28
|
+
default_project = config.get("default_project")
|
|
25
29
|
|
|
26
30
|
|
|
27
31
|
class AgentsController:
|
|
28
|
-
|
|
32
|
+
"""Handles CRUD operations at the database level for Agents"""
|
|
29
33
|
|
|
30
34
|
assistant_column = ASSISTANT_COLUMN
|
|
31
35
|
|
|
@@ -33,7 +37,7 @@ class AgentsController:
|
|
|
33
37
|
self,
|
|
34
38
|
project_controller: ProjectController = None,
|
|
35
39
|
skills_controller: SkillsController = None,
|
|
36
|
-
model_controller: ModelController = None
|
|
40
|
+
model_controller: ModelController = None,
|
|
37
41
|
):
|
|
38
42
|
if project_controller is None:
|
|
39
43
|
project_controller = ProjectController()
|
|
@@ -46,10 +50,10 @@ class AgentsController:
|
|
|
46
50
|
self.model_controller = model_controller
|
|
47
51
|
|
|
48
52
|
def check_model_provider(self, model_name: str, provider: str = None) -> Tuple[dict, str]:
|
|
49
|
-
|
|
53
|
+
"""
|
|
50
54
|
Checks if a model exists, and gets the provider of the model.
|
|
51
55
|
|
|
52
|
-
The provider is either the provider of the model
|
|
56
|
+
The provider is either the provider of the model or the provider given as an argument.
|
|
53
57
|
|
|
54
58
|
Parameters:
|
|
55
59
|
model_name (str): The name of the model
|
|
@@ -58,25 +62,29 @@ class AgentsController:
|
|
|
58
62
|
Returns:
|
|
59
63
|
model (dict): The model object
|
|
60
64
|
provider (str): The provider of the model
|
|
61
|
-
|
|
65
|
+
"""
|
|
62
66
|
model = None
|
|
63
67
|
|
|
68
|
+
# Handle the case when model_name is None (using default LLM)
|
|
69
|
+
if model_name is None:
|
|
70
|
+
return model, provider
|
|
71
|
+
|
|
64
72
|
try:
|
|
65
73
|
model_name_no_version, model_version = Predictor.get_name_and_version(model_name)
|
|
66
74
|
model = self.model_controller.get_model(model_name_no_version, version=model_version)
|
|
67
|
-
provider =
|
|
75
|
+
provider = "mindsdb" if model.get("provider") is None else model.get("provider")
|
|
68
76
|
except PredictorRecordNotFound:
|
|
69
77
|
if not provider:
|
|
70
78
|
# If provider is not given, get it from the model name
|
|
71
79
|
provider = get_llm_provider({"model_name": model_name})
|
|
72
80
|
|
|
73
81
|
elif provider not in SUPPORTED_PROVIDERS and model_name not in PROVIDER_TO_MODELS.get(provider, []):
|
|
74
|
-
raise ValueError(f
|
|
82
|
+
raise ValueError(f"Model with name does not exist for provider {provider}: {model_name}")
|
|
75
83
|
|
|
76
84
|
return model, provider
|
|
77
85
|
|
|
78
86
|
def get_agent(self, agent_name: str, project_name: str = default_project) -> Optional[db.Agents]:
|
|
79
|
-
|
|
87
|
+
"""
|
|
80
88
|
Gets an agent by name.
|
|
81
89
|
|
|
82
90
|
Parameters:
|
|
@@ -85,19 +93,19 @@ class AgentsController:
|
|
|
85
93
|
|
|
86
94
|
Returns:
|
|
87
95
|
agent (Optional[db.Agents]): The database agent object
|
|
88
|
-
|
|
96
|
+
"""
|
|
89
97
|
|
|
90
98
|
project = self.project_controller.get(name=project_name)
|
|
91
99
|
agent = db.Agents.query.filter(
|
|
92
100
|
db.Agents.name == agent_name,
|
|
93
101
|
db.Agents.project_id == project.id,
|
|
94
102
|
db.Agents.company_id == ctx.company_id,
|
|
95
|
-
db.Agents.deleted_at == null()
|
|
103
|
+
db.Agents.deleted_at == null(),
|
|
96
104
|
).first()
|
|
97
105
|
return agent
|
|
98
106
|
|
|
99
107
|
def get_agent_by_id(self, id: int, project_name: str = default_project) -> db.Agents:
|
|
100
|
-
|
|
108
|
+
"""
|
|
101
109
|
Gets an agent by id.
|
|
102
110
|
|
|
103
111
|
Parameters:
|
|
@@ -106,14 +114,14 @@ class AgentsController:
|
|
|
106
114
|
|
|
107
115
|
Returns:
|
|
108
116
|
agent (db.Agents): The database agent object
|
|
109
|
-
|
|
117
|
+
"""
|
|
110
118
|
|
|
111
119
|
project = self.project_controller.get(name=project_name)
|
|
112
120
|
agent = db.Agents.query.filter(
|
|
113
121
|
db.Agents.id == id,
|
|
114
122
|
db.Agents.project_id == project.id,
|
|
115
123
|
db.Agents.company_id == ctx.company_id,
|
|
116
|
-
db.Agents.deleted_at == null()
|
|
124
|
+
db.Agents.deleted_at == null(),
|
|
117
125
|
).first()
|
|
118
126
|
return agent
|
|
119
127
|
|
|
@@ -128,10 +136,7 @@ class AgentsController:
|
|
|
128
136
|
all-agents (List[db.Agents]): List of database agent object
|
|
129
137
|
"""
|
|
130
138
|
|
|
131
|
-
all_agents = db.Agents.query.filter(
|
|
132
|
-
db.Agents.company_id == ctx.company_id,
|
|
133
|
-
db.Agents.deleted_at == null()
|
|
134
|
-
)
|
|
139
|
+
all_agents = db.Agents.query.filter(db.Agents.company_id == ctx.company_id, db.Agents.deleted_at == null())
|
|
135
140
|
|
|
136
141
|
if project_name is not None:
|
|
137
142
|
project = self.project_controller.get(name=project_name)
|
|
@@ -141,14 +146,15 @@ class AgentsController:
|
|
|
141
146
|
return all_agents.all()
|
|
142
147
|
|
|
143
148
|
def add_agent(
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
self,
|
|
150
|
+
name: str,
|
|
151
|
+
project_name: str = None,
|
|
152
|
+
model_name: str = None,
|
|
153
|
+
skills: List[Union[str, dict]] = None,
|
|
154
|
+
provider: str = None,
|
|
155
|
+
params: Dict[str, Any] = None,
|
|
156
|
+
) -> db.Agents:
|
|
157
|
+
"""
|
|
152
158
|
Adds an agent to the database.
|
|
153
159
|
|
|
154
160
|
Parameters:
|
|
@@ -172,7 +178,7 @@ class AgentsController:
|
|
|
172
178
|
|
|
173
179
|
Raises:
|
|
174
180
|
ValueError: Agent with given name already exists, or skill/model with given name does not exist.
|
|
175
|
-
|
|
181
|
+
"""
|
|
176
182
|
if project_name is None:
|
|
177
183
|
project_name = default_project
|
|
178
184
|
project = self.project_controller.get(name=project_name)
|
|
@@ -180,126 +186,117 @@ class AgentsController:
|
|
|
180
186
|
agent = self.get_agent(name, project_name)
|
|
181
187
|
|
|
182
188
|
if agent is not None:
|
|
183
|
-
raise ValueError(f
|
|
189
|
+
raise ValueError(f"Agent with name already exists: {name}")
|
|
190
|
+
|
|
191
|
+
if model_name is not None:
|
|
192
|
+
_, provider = self.check_model_provider(model_name, provider)
|
|
193
|
+
|
|
194
|
+
# No need to copy params since we're not preserving the original reference
|
|
195
|
+
params = params or {}
|
|
184
196
|
|
|
185
|
-
|
|
197
|
+
if model_name is None:
|
|
198
|
+
logger.warning("'model_name' param is not provided. Using default global llm model at runtime.")
|
|
199
|
+
|
|
200
|
+
# If model_name is not provided, we use default global llm model at runtime
|
|
201
|
+
# Default parameters will be applied at runtime via get_agent_llm_params
|
|
202
|
+
# This allows global default updates to apply to all agents immediately
|
|
186
203
|
|
|
187
204
|
# Extract API key if provided in the format <provider>_api_key
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
205
|
+
if provider is not None:
|
|
206
|
+
provider_api_key_param = f"{provider.lower()}_api_key"
|
|
207
|
+
if provider_api_key_param in params:
|
|
208
|
+
# Keep the API key in params for the agent to use
|
|
209
|
+
# It will be picked up by get_api_key() in handler_utils.py
|
|
210
|
+
pass
|
|
211
|
+
|
|
212
|
+
# Handle generic api_key parameter if provided
|
|
213
|
+
if "api_key" in params:
|
|
214
|
+
# Keep the generic API key in params for the agent to use
|
|
191
215
|
# It will be picked up by get_api_key() in handler_utils.py
|
|
192
216
|
pass
|
|
193
217
|
|
|
194
218
|
# Extract table and knowledge base parameters from params
|
|
195
|
-
database = params.pop(
|
|
196
|
-
knowledge_base_database = params.pop(
|
|
197
|
-
include_tables = params.pop(
|
|
198
|
-
ignore_tables = params.pop(
|
|
199
|
-
include_knowledge_bases = params.pop(
|
|
200
|
-
ignore_knowledge_bases = params.pop(
|
|
219
|
+
database = params.pop("database", None)
|
|
220
|
+
knowledge_base_database = params.pop("knowledge_base_database", DEFAULT_TEXT2SQL_DATABASE)
|
|
221
|
+
include_tables = params.pop("include_tables", None)
|
|
222
|
+
ignore_tables = params.pop("ignore_tables", None)
|
|
223
|
+
include_knowledge_bases = params.pop("include_knowledge_bases", None)
|
|
224
|
+
ignore_knowledge_bases = params.pop("ignore_knowledge_bases", None)
|
|
201
225
|
|
|
202
226
|
# Save the extracted parameters back to params for API responses only if they were explicitly provided
|
|
203
227
|
# or if they're needed for skills
|
|
204
228
|
need_params = include_tables or ignore_tables or include_knowledge_bases or ignore_knowledge_bases
|
|
205
229
|
|
|
206
|
-
if
|
|
207
|
-
params[
|
|
230
|
+
if "database" in params or need_params:
|
|
231
|
+
params["database"] = database
|
|
208
232
|
|
|
209
|
-
if
|
|
210
|
-
params[
|
|
233
|
+
if "knowledge_base_database" in params or include_knowledge_bases or ignore_knowledge_bases:
|
|
234
|
+
params["knowledge_base_database"] = knowledge_base_database
|
|
211
235
|
|
|
212
236
|
if include_tables is not None:
|
|
213
|
-
params[
|
|
237
|
+
params["include_tables"] = include_tables
|
|
214
238
|
if ignore_tables is not None:
|
|
215
|
-
params[
|
|
239
|
+
params["ignore_tables"] = ignore_tables
|
|
216
240
|
if include_knowledge_bases is not None:
|
|
217
|
-
params[
|
|
241
|
+
params["include_knowledge_bases"] = include_knowledge_bases
|
|
218
242
|
if ignore_knowledge_bases is not None:
|
|
219
|
-
params[
|
|
243
|
+
params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
220
244
|
|
|
221
245
|
# Convert string parameters to lists if needed
|
|
222
246
|
if isinstance(include_tables, str):
|
|
223
|
-
include_tables = [t.strip() for t in include_tables.split(
|
|
247
|
+
include_tables = [t.strip() for t in include_tables.split(",")]
|
|
224
248
|
if isinstance(ignore_tables, str):
|
|
225
|
-
ignore_tables = [t.strip() for t in ignore_tables.split(
|
|
249
|
+
ignore_tables = [t.strip() for t in ignore_tables.split(",")]
|
|
226
250
|
if isinstance(include_knowledge_bases, str):
|
|
227
|
-
include_knowledge_bases = [kb.strip() for kb in include_knowledge_bases.split(
|
|
251
|
+
include_knowledge_bases = [kb.strip() for kb in include_knowledge_bases.split(",")]
|
|
228
252
|
if isinstance(ignore_knowledge_bases, str):
|
|
229
|
-
ignore_knowledge_bases = [kb.strip() for kb in ignore_knowledge_bases.split(
|
|
253
|
+
ignore_knowledge_bases = [kb.strip() for kb in ignore_knowledge_bases.split(",")]
|
|
230
254
|
|
|
231
255
|
# Auto-create SQL skill if no skills are provided but include_tables or include_knowledge_bases params are provided
|
|
232
256
|
if not skills and (include_tables or include_knowledge_bases):
|
|
233
|
-
# Determine database to use (default to 'mindsdb')
|
|
234
|
-
db_name = database
|
|
235
|
-
kb_db_name = knowledge_base_database
|
|
236
|
-
|
|
237
|
-
# If database is not explicitly provided but tables are, try to extract the database name from the first table
|
|
238
|
-
if not database and include_tables and len(include_tables) > 0:
|
|
239
|
-
parts = include_tables[0].split('.')
|
|
240
|
-
if len(parts) >= 2:
|
|
241
|
-
db_name = parts[0]
|
|
242
|
-
elif not database and ignore_tables and len(ignore_tables) > 0:
|
|
243
|
-
parts = ignore_tables[0].split('.')
|
|
244
|
-
if len(parts) >= 2:
|
|
245
|
-
db_name = parts[0]
|
|
246
|
-
|
|
247
257
|
# Create a default SQL skill
|
|
248
258
|
skill_name = f"{name}_sql_skill"
|
|
249
259
|
skill_params = {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
260
|
+
"type": "sql",
|
|
261
|
+
"database": database,
|
|
262
|
+
"description": f"Auto-generated SQL skill for agent {name}",
|
|
253
263
|
}
|
|
254
264
|
|
|
255
|
-
# Add
|
|
256
|
-
if
|
|
257
|
-
skill_params[
|
|
265
|
+
# Add table restrictions if provided
|
|
266
|
+
if include_tables:
|
|
267
|
+
skill_params["include_tables"] = include_tables
|
|
268
|
+
if ignore_tables:
|
|
269
|
+
skill_params["ignore_tables"] = ignore_tables
|
|
258
270
|
|
|
259
271
|
# Add knowledge base parameters if provided
|
|
272
|
+
if knowledge_base_database:
|
|
273
|
+
skill_params["knowledge_base_database"] = knowledge_base_database
|
|
260
274
|
if include_knowledge_bases:
|
|
261
|
-
skill_params[
|
|
275
|
+
skill_params["include_knowledge_bases"] = include_knowledge_bases
|
|
262
276
|
if ignore_knowledge_bases:
|
|
263
|
-
skill_params[
|
|
264
|
-
|
|
277
|
+
skill_params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
265
278
|
try:
|
|
266
279
|
# Check if skill already exists
|
|
267
280
|
existing_skill = self.skills_controller.get_skill(skill_name, project_name)
|
|
268
281
|
if existing_skill is None:
|
|
269
282
|
# Create the skill
|
|
270
|
-
skill_type = skill_params.pop(
|
|
283
|
+
skill_type = skill_params.pop("type")
|
|
271
284
|
self.skills_controller.add_skill(
|
|
272
|
-
name=skill_name,
|
|
273
|
-
project_name=project_name,
|
|
274
|
-
type=skill_type,
|
|
275
|
-
params=skill_params
|
|
285
|
+
name=skill_name, project_name=project_name, type=skill_type, params=skill_params
|
|
276
286
|
)
|
|
277
287
|
else:
|
|
278
288
|
# Update the skill if parameters have changed
|
|
279
289
|
params_changed = False
|
|
280
290
|
|
|
281
|
-
# Check if
|
|
282
|
-
|
|
283
|
-
existing_skill.params
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
# Check if knowledge base database has changed
|
|
287
|
-
if knowledge_base_database and existing_skill.params.get('knowledge_base_database') != kb_db_name:
|
|
288
|
-
existing_skill.params['knowledge_base_database'] = kb_db_name
|
|
289
|
-
params_changed = True
|
|
290
|
-
|
|
291
|
-
# Check if knowledge base parameters have changed
|
|
292
|
-
if include_knowledge_bases and existing_skill.params.get('include_knowledge_bases') != include_knowledge_bases:
|
|
293
|
-
existing_skill.params['include_knowledge_bases'] = include_knowledge_bases
|
|
294
|
-
params_changed = True
|
|
295
|
-
|
|
296
|
-
if ignore_knowledge_bases and existing_skill.params.get('ignore_knowledge_bases') != ignore_knowledge_bases:
|
|
297
|
-
existing_skill.params['ignore_knowledge_bases'] = ignore_knowledge_bases
|
|
298
|
-
params_changed = True
|
|
291
|
+
# Check if skill parameters need to be updated
|
|
292
|
+
for param_key, param_value in skill_params.items():
|
|
293
|
+
if existing_skill.params.get(param_key) != param_value:
|
|
294
|
+
existing_skill.params[param_key] = param_value
|
|
295
|
+
params_changed = True
|
|
299
296
|
|
|
300
297
|
# Update the skill if needed
|
|
301
298
|
if params_changed:
|
|
302
|
-
flag_modified(existing_skill,
|
|
299
|
+
flag_modified(existing_skill, "params")
|
|
303
300
|
db.session.commit()
|
|
304
301
|
|
|
305
302
|
skills = [skill_name]
|
|
@@ -322,49 +319,70 @@ class AgentsController:
|
|
|
322
319
|
parameters = {}
|
|
323
320
|
else:
|
|
324
321
|
parameters = skill.copy()
|
|
325
|
-
skill_name = parameters.pop(
|
|
322
|
+
skill_name = parameters.pop("name")
|
|
326
323
|
|
|
327
324
|
existing_skill = self.skills_controller.get_skill(skill_name, project_name)
|
|
328
325
|
if existing_skill is None:
|
|
329
326
|
db.session.rollback()
|
|
330
|
-
raise ValueError(f
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
327
|
+
raise ValueError(f"Skill with name does not exist: {skill_name}")
|
|
328
|
+
|
|
329
|
+
if existing_skill.type == "sql":
|
|
330
|
+
# Run Data Catalog loader if enabled
|
|
331
|
+
if config.get("data_catalog", {}).get("enabled", False):
|
|
332
|
+
if include_tables:
|
|
333
|
+
database_table_map = {}
|
|
334
|
+
for table in include_tables:
|
|
335
|
+
parts = table.split(".", 1)
|
|
336
|
+
database_table_map[parts[0]] = database_table_map.get(parts[0], []) + [parts[1]]
|
|
337
|
+
|
|
338
|
+
for database_name, table_names in database_table_map.items():
|
|
339
|
+
data_catalog_loader = DataCatalogLoader(
|
|
340
|
+
database_name=database_name, table_names=table_names
|
|
341
|
+
)
|
|
342
|
+
data_catalog_loader.load_metadata()
|
|
343
|
+
|
|
344
|
+
elif "database" in existing_skill.params:
|
|
345
|
+
data_catalog_loader = DataCatalogLoader(
|
|
346
|
+
database_name=existing_skill.params["database"],
|
|
347
|
+
table_names=parameters["tables"] if "tables" in parameters else None,
|
|
348
|
+
)
|
|
349
|
+
data_catalog_loader.load_metadata()
|
|
350
|
+
|
|
351
|
+
else:
|
|
352
|
+
raise ValueError(
|
|
353
|
+
"Data Catalog loading is enabled, but the provided parameters are insufficient to load metadata. "
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
# Add table restrictions if this is a text2sql skill
|
|
357
|
+
if include_tables or ignore_tables:
|
|
358
|
+
parameters["tables"] = include_tables or ignore_tables
|
|
335
359
|
|
|
336
|
-
# Add knowledge base restrictions if this is a text2sql skill
|
|
337
|
-
if existing_skill.type == 'sql':
|
|
338
360
|
# Pass database parameter if provided
|
|
339
|
-
if database and
|
|
340
|
-
parameters[
|
|
361
|
+
if database and "database" not in parameters:
|
|
362
|
+
parameters["database"] = database
|
|
341
363
|
|
|
342
364
|
# Pass knowledge base database parameter if provided
|
|
343
|
-
if knowledge_base_database and
|
|
344
|
-
parameters[
|
|
365
|
+
if knowledge_base_database and "knowledge_base_database" not in parameters:
|
|
366
|
+
parameters["knowledge_base_database"] = knowledge_base_database
|
|
345
367
|
|
|
346
368
|
# Add knowledge base parameters to both the skill and the association parameters
|
|
347
369
|
if include_knowledge_bases:
|
|
348
|
-
parameters[
|
|
349
|
-
if
|
|
350
|
-
existing_skill.params[
|
|
351
|
-
flag_modified(existing_skill,
|
|
370
|
+
parameters["include_knowledge_bases"] = include_knowledge_bases
|
|
371
|
+
if "include_knowledge_bases" not in existing_skill.params:
|
|
372
|
+
existing_skill.params["include_knowledge_bases"] = include_knowledge_bases
|
|
373
|
+
flag_modified(existing_skill, "params")
|
|
352
374
|
elif ignore_knowledge_bases:
|
|
353
|
-
parameters[
|
|
354
|
-
if
|
|
355
|
-
existing_skill.params[
|
|
356
|
-
flag_modified(existing_skill,
|
|
375
|
+
parameters["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
376
|
+
if "ignore_knowledge_bases" not in existing_skill.params:
|
|
377
|
+
existing_skill.params["ignore_knowledge_bases"] = ignore_knowledge_bases
|
|
378
|
+
flag_modified(existing_skill, "params")
|
|
357
379
|
|
|
358
380
|
# Ensure knowledge_base_database is set in the skill's params
|
|
359
|
-
if knowledge_base_database and
|
|
360
|
-
existing_skill.params[
|
|
361
|
-
flag_modified(existing_skill,
|
|
362
|
-
|
|
363
|
-
association = db.AgentSkillsAssociation(
|
|
364
|
-
parameters=parameters,
|
|
365
|
-
agent=agent,
|
|
366
|
-
skill=existing_skill
|
|
367
|
-
)
|
|
381
|
+
if knowledge_base_database and "knowledge_base_database" not in existing_skill.params:
|
|
382
|
+
existing_skill.params["knowledge_base_database"] = knowledge_base_database
|
|
383
|
+
flag_modified(existing_skill, "params")
|
|
384
|
+
|
|
385
|
+
association = db.AgentSkillsAssociation(parameters=parameters, agent=agent, skill=existing_skill)
|
|
368
386
|
db.session.add(association)
|
|
369
387
|
|
|
370
388
|
db.session.add(agent)
|
|
@@ -373,17 +391,18 @@ class AgentsController:
|
|
|
373
391
|
return agent
|
|
374
392
|
|
|
375
393
|
def update_agent(
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
394
|
+
self,
|
|
395
|
+
agent_name: str,
|
|
396
|
+
project_name: str = default_project,
|
|
397
|
+
name: str = None,
|
|
398
|
+
model_name: str = None,
|
|
399
|
+
skills_to_add: List[Union[str, dict]] = None,
|
|
400
|
+
skills_to_remove: List[str] = None,
|
|
401
|
+
skills_to_rewrite: List[Union[str, dict]] = None,
|
|
402
|
+
provider: str = None,
|
|
403
|
+
params: Dict[str, str] = None,
|
|
404
|
+
):
|
|
405
|
+
"""
|
|
387
406
|
Updates an agent in the database.
|
|
388
407
|
|
|
389
408
|
Parameters:
|
|
@@ -405,7 +424,7 @@ class AgentsController:
|
|
|
405
424
|
EntityExistsError: if agent with new name already exists
|
|
406
425
|
EntityNotExistsError: if agent with name or skill not found
|
|
407
426
|
ValueError: if conflict in skills list
|
|
408
|
-
|
|
427
|
+
"""
|
|
409
428
|
|
|
410
429
|
skills_to_add = skills_to_add or []
|
|
411
430
|
skills_to_remove = skills_to_remove or []
|
|
@@ -418,15 +437,13 @@ class AgentsController:
|
|
|
418
437
|
|
|
419
438
|
existing_agent = self.get_agent(agent_name, project_name=project_name)
|
|
420
439
|
if existing_agent is None:
|
|
421
|
-
raise EntityNotExistsError(f
|
|
422
|
-
is_demo = (existing_agent.params or {}).get(
|
|
423
|
-
if (
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
or (isinstance(params, dict) and len(params) > 0 and 'prompt_template' not in params)
|
|
429
|
-
)
|
|
440
|
+
raise EntityNotExistsError(f"Agent with name not found: {agent_name}")
|
|
441
|
+
is_demo = (existing_agent.params or {}).get("is_demo", False)
|
|
442
|
+
if is_demo and (
|
|
443
|
+
(name is not None and name != agent_name)
|
|
444
|
+
or (model_name is not None and existing_agent.model_name != model_name)
|
|
445
|
+
or (provider is not None and existing_agent.provider != provider)
|
|
446
|
+
or (isinstance(params, dict) and len(params) > 0 and "prompt_template" not in params)
|
|
430
447
|
):
|
|
431
448
|
raise ValueError("It is forbidden to change properties of the demo object")
|
|
432
449
|
|
|
@@ -434,7 +451,7 @@ class AgentsController:
|
|
|
434
451
|
# Check to see if updated name already exists
|
|
435
452
|
agent_with_new_name = self.get_agent(name, project_name=project_name)
|
|
436
453
|
if agent_with_new_name is not None:
|
|
437
|
-
raise EntityExistsError(f
|
|
454
|
+
raise EntityExistsError(f"Agent with updated name already exists: {name}")
|
|
438
455
|
existing_agent.name = name
|
|
439
456
|
|
|
440
457
|
if model_name or provider:
|
|
@@ -446,21 +463,21 @@ class AgentsController:
|
|
|
446
463
|
|
|
447
464
|
# check that all skills exist
|
|
448
465
|
skill_name_to_record_map = {}
|
|
449
|
-
for skill_meta in
|
|
450
|
-
skill_name = skill_meta[
|
|
466
|
+
for skill_meta in skills_to_add + skills_to_remove + skills_to_rewrite:
|
|
467
|
+
skill_name = skill_meta["name"] if isinstance(skill_meta, dict) else skill_meta
|
|
451
468
|
if skill_name not in skill_name_to_record_map:
|
|
452
469
|
skill_record = self.skills_controller.get_skill(skill_name, project_name)
|
|
453
470
|
if skill_record is None:
|
|
454
|
-
raise EntityNotExistsError(f
|
|
471
|
+
raise EntityNotExistsError(f"Skill with name does not exist: {skill_name}")
|
|
455
472
|
skill_name_to_record_map[skill_name] = skill_record
|
|
456
473
|
|
|
457
474
|
if len(skills_to_add) > 0 or len(skills_to_remove) > 0:
|
|
458
|
-
skills_to_add = [{
|
|
459
|
-
skills_to_add_names = [x[
|
|
475
|
+
skills_to_add = [{"name": x} if isinstance(x, str) else x for x in skills_to_add]
|
|
476
|
+
skills_to_add_names = [x["name"] for x in skills_to_add]
|
|
460
477
|
|
|
461
478
|
# there are no intersection between lists
|
|
462
479
|
if not set(skills_to_add_names).isdisjoint(set(skills_to_remove)):
|
|
463
|
-
raise ValueError(
|
|
480
|
+
raise ValueError("Conflict between skills to add and skills to remove.")
|
|
464
481
|
|
|
465
482
|
existing_agent_skills_names = [rel.skill.name for rel in existing_agent.skills_relationships]
|
|
466
483
|
|
|
@@ -471,19 +488,17 @@ class AgentsController:
|
|
|
471
488
|
db.session.delete(rel)
|
|
472
489
|
|
|
473
490
|
# add skills
|
|
474
|
-
for skill_name in
|
|
475
|
-
skill_parameters = next(x for x in skills_to_add if x[
|
|
476
|
-
del skill_parameters[
|
|
491
|
+
for skill_name in set(skills_to_add_names) - set(existing_agent_skills_names):
|
|
492
|
+
skill_parameters = next(x for x in skills_to_add if x["name"] == skill_name).copy()
|
|
493
|
+
del skill_parameters["name"]
|
|
477
494
|
association = db.AgentSkillsAssociation(
|
|
478
|
-
parameters=skill_parameters,
|
|
479
|
-
agent=existing_agent,
|
|
480
|
-
skill=skill_name_to_record_map[skill_name]
|
|
495
|
+
parameters=skill_parameters, agent=existing_agent, skill=skill_name_to_record_map[skill_name]
|
|
481
496
|
)
|
|
482
497
|
db.session.add(association)
|
|
483
498
|
elif len(skills_to_rewrite) > 0:
|
|
484
|
-
skill_name_to_parameters = {
|
|
485
|
-
k: v for k, v in x.items() if k !=
|
|
486
|
-
}
|
|
499
|
+
skill_name_to_parameters = {
|
|
500
|
+
x["name"]: {k: v for k, v in x.items() if k != "name"} for x in skills_to_rewrite
|
|
501
|
+
}
|
|
487
502
|
existing_skill_names = set()
|
|
488
503
|
for rel in existing_agent.skills_relationships:
|
|
489
504
|
if rel.skill.name not in skill_name_to_parameters:
|
|
@@ -491,12 +506,12 @@ class AgentsController:
|
|
|
491
506
|
else:
|
|
492
507
|
existing_skill_names.add(rel.skill.name)
|
|
493
508
|
rel.parameters = skill_name_to_parameters[rel.skill.name]
|
|
494
|
-
flag_modified(rel,
|
|
495
|
-
for new_skill_name in
|
|
509
|
+
flag_modified(rel, "parameters")
|
|
510
|
+
for new_skill_name in set(skill_name_to_parameters) - existing_skill_names:
|
|
496
511
|
association = db.AgentSkillsAssociation(
|
|
497
512
|
parameters=skill_name_to_parameters[new_skill_name],
|
|
498
513
|
agent=existing_agent,
|
|
499
|
-
skill=skill_name_to_record_map[new_skill_name]
|
|
514
|
+
skill=skill_name_to_record_map[new_skill_name],
|
|
500
515
|
)
|
|
501
516
|
db.session.add(association)
|
|
502
517
|
|
|
@@ -509,13 +524,13 @@ class AgentsController:
|
|
|
509
524
|
existing_agent.params = params
|
|
510
525
|
# Some versions of SQL Alchemy won't handle JSON updates correctly without this.
|
|
511
526
|
# See: https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.attributes.flag_modified
|
|
512
|
-
flag_modified(existing_agent,
|
|
527
|
+
flag_modified(existing_agent, "params")
|
|
513
528
|
db.session.commit()
|
|
514
529
|
|
|
515
530
|
return existing_agent
|
|
516
531
|
|
|
517
532
|
def delete_agent(self, agent_name: str, project_name: str = default_project):
|
|
518
|
-
|
|
533
|
+
"""
|
|
519
534
|
Deletes an agent by name.
|
|
520
535
|
|
|
521
536
|
Parameters:
|
|
@@ -524,23 +539,36 @@ class AgentsController:
|
|
|
524
539
|
|
|
525
540
|
Raises:
|
|
526
541
|
ValueError: Agent does not exist.
|
|
527
|
-
|
|
542
|
+
"""
|
|
528
543
|
|
|
529
544
|
agent = self.get_agent(agent_name, project_name)
|
|
530
545
|
if agent is None:
|
|
531
|
-
raise ValueError(f
|
|
532
|
-
if isinstance(agent.params, dict) and agent.params.get(
|
|
533
|
-
raise ValueError(
|
|
546
|
+
raise ValueError(f"Agent with name does not exist: {agent_name}")
|
|
547
|
+
if isinstance(agent.params, dict) and agent.params.get("is_demo") is True:
|
|
548
|
+
raise ValueError("Unable to delete demo object")
|
|
534
549
|
agent.deleted_at = datetime.datetime.now()
|
|
535
550
|
db.session.commit()
|
|
536
551
|
|
|
552
|
+
def get_agent_llm_params(self, model_params: dict):
|
|
553
|
+
"""
|
|
554
|
+
Get agent LLM parameters by combining default config with user provided parameters.
|
|
555
|
+
Similar to how knowledge bases handle default parameters.
|
|
556
|
+
"""
|
|
557
|
+
combined_model_params = copy.deepcopy(config.get("default_llm", {}))
|
|
558
|
+
|
|
559
|
+
if model_params:
|
|
560
|
+
combined_model_params.update(model_params)
|
|
561
|
+
|
|
562
|
+
return combined_model_params
|
|
563
|
+
|
|
537
564
|
def get_completion(
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
565
|
+
self,
|
|
566
|
+
agent: db.Agents,
|
|
567
|
+
messages: List[Dict[str, str]],
|
|
568
|
+
project_name: str = default_project,
|
|
569
|
+
tools: List[BaseTool] = None,
|
|
570
|
+
stream: bool = False,
|
|
571
|
+
) -> Union[Iterator[object], pd.DataFrame]:
|
|
544
572
|
"""
|
|
545
573
|
Queries an agent to get a completion.
|
|
546
574
|
|
|
@@ -558,12 +586,7 @@ class AgentsController:
|
|
|
558
586
|
ValueError: Agent's model does not exist.
|
|
559
587
|
"""
|
|
560
588
|
if stream:
|
|
561
|
-
return self._get_completion_stream(
|
|
562
|
-
agent,
|
|
563
|
-
messages,
|
|
564
|
-
project_name=project_name,
|
|
565
|
-
tools=tools
|
|
566
|
-
)
|
|
589
|
+
return self._get_completion_stream(agent, messages, project_name=project_name, tools=tools)
|
|
567
590
|
from .langchain_agent import LangchainAgent
|
|
568
591
|
|
|
569
592
|
model, provider = self.check_model_provider(agent.model_name, agent.provider)
|
|
@@ -572,16 +595,20 @@ class AgentsController:
|
|
|
572
595
|
agent.provider = provider
|
|
573
596
|
db.session.commit()
|
|
574
597
|
|
|
575
|
-
|
|
598
|
+
# Get agent parameters and combine with default LLM parameters at runtime
|
|
599
|
+
agent_params = self.get_agent_llm_params(agent.params)
|
|
600
|
+
|
|
601
|
+
lang_agent = LangchainAgent(agent, model, params=agent_params)
|
|
576
602
|
return lang_agent.get_completion(messages)
|
|
577
603
|
|
|
578
604
|
def _get_completion_stream(
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
605
|
+
self,
|
|
606
|
+
agent: db.Agents,
|
|
607
|
+
messages: List[Dict[str, str]],
|
|
608
|
+
project_name: str = default_project,
|
|
609
|
+
tools: List[BaseTool] = None,
|
|
610
|
+
) -> Iterator[object]:
|
|
611
|
+
"""
|
|
585
612
|
Queries an agent to get a stream of completion chunks.
|
|
586
613
|
|
|
587
614
|
Parameters:
|
|
@@ -597,7 +624,7 @@ class AgentsController:
|
|
|
597
624
|
|
|
598
625
|
Raises:
|
|
599
626
|
ValueError: Agent's model does not exist.
|
|
600
|
-
|
|
627
|
+
"""
|
|
601
628
|
# For circular dependency.
|
|
602
629
|
from .langchain_agent import LangchainAgent
|
|
603
630
|
|
|
@@ -608,5 +635,8 @@ class AgentsController:
|
|
|
608
635
|
agent.provider = provider
|
|
609
636
|
db.session.commit()
|
|
610
637
|
|
|
611
|
-
|
|
638
|
+
# Get agent parameters and combine with default LLM parameters at runtime
|
|
639
|
+
agent_params = self.get_agent_llm_params(agent.params)
|
|
640
|
+
|
|
641
|
+
lang_agent = LangchainAgent(agent, model=model, params=agent_params)
|
|
612
642
|
return lang_agent.get_completion(messages, stream=True)
|