ibm-watsonx-orchestrate 1.10.0b0__py3-none-any.whl → 1.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ibm_watsonx_orchestrate/__init__.py +1 -2
- ibm_watsonx_orchestrate/agent_builder/connections/__init__.py +1 -1
- ibm_watsonx_orchestrate/agent_builder/connections/connections.py +6 -3
- ibm_watsonx_orchestrate/agent_builder/connections/types.py +68 -17
- ibm_watsonx_orchestrate/agent_builder/knowledge_bases/types.py +47 -3
- ibm_watsonx_orchestrate/agent_builder/toolkits/types.py +18 -15
- ibm_watsonx_orchestrate/agent_builder/tools/types.py +1 -1
- ibm_watsonx_orchestrate/cli/commands/connections/connections_command.py +40 -11
- ibm_watsonx_orchestrate/cli/commands/connections/connections_controller.py +96 -30
- ibm_watsonx_orchestrate/cli/commands/knowledge_bases/knowledge_bases_controller.py +32 -10
- ibm_watsonx_orchestrate/cli/commands/server/server_command.py +95 -17
- ibm_watsonx_orchestrate/cli/commands/server/types.py +14 -6
- ibm_watsonx_orchestrate/cli/commands/toolkit/toolkit_command.py +43 -10
- ibm_watsonx_orchestrate/cli/commands/toolkit/toolkit_controller.py +52 -25
- ibm_watsonx_orchestrate/client/connections/connections_client.py +4 -3
- ibm_watsonx_orchestrate/client/knowledge_bases/knowledge_base_client.py +4 -4
- ibm_watsonx_orchestrate/docker/compose-lite.yml +52 -13
- ibm_watsonx_orchestrate/docker/default.env +21 -14
- ibm_watsonx_orchestrate/flow_builder/data_map.py +4 -1
- ibm_watsonx_orchestrate/flow_builder/flows/__init__.py +2 -0
- ibm_watsonx_orchestrate/flow_builder/flows/flow.py +204 -17
- ibm_watsonx_orchestrate/flow_builder/node.py +114 -19
- ibm_watsonx_orchestrate/flow_builder/types.py +206 -34
- ibm_watsonx_orchestrate/run/connections.py +2 -2
- {ibm_watsonx_orchestrate-1.10.0b0.dist-info → ibm_watsonx_orchestrate-1.10.1.dist-info}/METADATA +1 -1
- {ibm_watsonx_orchestrate-1.10.0b0.dist-info → ibm_watsonx_orchestrate-1.10.1.dist-info}/RECORD +29 -29
- {ibm_watsonx_orchestrate-1.10.0b0.dist-info → ibm_watsonx_orchestrate-1.10.1.dist-info}/WHEEL +0 -0
- {ibm_watsonx_orchestrate-1.10.0b0.dist-info → ibm_watsonx_orchestrate-1.10.1.dist-info}/entry_points.txt +0 -0
- {ibm_watsonx_orchestrate-1.10.0b0.dist-info → ibm_watsonx_orchestrate-1.10.1.dist-info}/licenses/LICENSE +0 -0
@@ -19,15 +19,18 @@ from ibm_watsonx_orchestrate.agent_builder.connections.types import (
|
|
19
19
|
BasicAuthCredentials,
|
20
20
|
BearerTokenAuthCredentials,
|
21
21
|
APIKeyAuthCredentials,
|
22
|
-
|
22
|
+
OAuth2AuthCodeCredentials,
|
23
23
|
OAuth2ClientCredentials,
|
24
24
|
# OAuth2ImplicitCredentials,
|
25
|
-
|
25
|
+
OAuth2PasswordCredentials,
|
26
26
|
OAuthOnBehalfOfCredentials,
|
27
27
|
KeyValueConnectionCredentials,
|
28
28
|
CREDENTIALS,
|
29
29
|
IdentityProviderCredentials,
|
30
|
-
OAUTH_CONNECTION_TYPES,
|
30
|
+
OAUTH_CONNECTION_TYPES,
|
31
|
+
ConnectionCredentialsEntryLocation,
|
32
|
+
ConnectionCredentialsEntry,
|
33
|
+
ConnectionCredentialsCustomFields
|
31
34
|
|
32
35
|
)
|
33
36
|
|
@@ -115,7 +118,7 @@ def _format_token_headers(header_list: List) -> dict:
|
|
115
118
|
|
116
119
|
def _validate_connection_params(type: ConnectionType, **args) -> None:
|
117
120
|
|
118
|
-
if type
|
121
|
+
if type in {ConnectionType.BASIC_AUTH, ConnectionType.OAUTH2_PASSWORD} and (
|
119
122
|
args.get('username') is None or args.get('password') is None
|
120
123
|
):
|
121
124
|
raise typer.BadParameter(
|
@@ -136,7 +139,7 @@ def _validate_connection_params(type: ConnectionType, **args) -> None:
|
|
136
139
|
f"Missing flags --api-key is required for type {type}"
|
137
140
|
)
|
138
141
|
|
139
|
-
if type in {ConnectionType.OAUTH2_CLIENT_CREDS, ConnectionType.OAUTH2_AUTH_CODE} and args.get('client_secret') is None:
|
142
|
+
if type in {ConnectionType.OAUTH2_CLIENT_CREDS, ConnectionType.OAUTH2_AUTH_CODE, ConnectionType.OAUTH2_PASSWORD} and args.get('client_secret') is None:
|
140
143
|
raise typer.BadParameter(
|
141
144
|
f"Missing flags --client-secret is required for type {type}"
|
142
145
|
)
|
@@ -146,14 +149,14 @@ def _validate_connection_params(type: ConnectionType, **args) -> None:
|
|
146
149
|
f"Missing flags --auth-url is required for type {type}"
|
147
150
|
)
|
148
151
|
|
149
|
-
if type in {ConnectionType.OAUTH_ON_BEHALF_OF_FLOW, ConnectionType.OAUTH2_CLIENT_CREDS, ConnectionType.OAUTH2_AUTH_CODE} and (
|
152
|
+
if type in {ConnectionType.OAUTH_ON_BEHALF_OF_FLOW, ConnectionType.OAUTH2_CLIENT_CREDS, ConnectionType.OAUTH2_AUTH_CODE, ConnectionType.OAUTH2_PASSWORD} and (
|
150
153
|
args.get('client_id') is None
|
151
154
|
):
|
152
155
|
raise typer.BadParameter(
|
153
156
|
f"Missing flags --client-id is required for type {type}"
|
154
157
|
)
|
155
158
|
|
156
|
-
if type in {ConnectionType.OAUTH_ON_BEHALF_OF_FLOW, ConnectionType.OAUTH2_CLIENT_CREDS, ConnectionType.OAUTH2_AUTH_CODE} and (
|
159
|
+
if type in {ConnectionType.OAUTH_ON_BEHALF_OF_FLOW, ConnectionType.OAUTH2_CLIENT_CREDS, ConnectionType.OAUTH2_AUTH_CODE, ConnectionType.OAUTH2_PASSWORD} and (
|
157
160
|
args.get('token_url') is None
|
158
161
|
):
|
159
162
|
raise typer.BadParameter(
|
@@ -167,6 +170,13 @@ def _validate_connection_params(type: ConnectionType, **args) -> None:
|
|
167
170
|
f"Missing flags --grant-type is required for type {type}"
|
168
171
|
)
|
169
172
|
|
173
|
+
if type != ConnectionType.OAUTH2_AUTH_CODE and (
|
174
|
+
args.get('auth_entries')
|
175
|
+
):
|
176
|
+
raise typer.BadParameter(
|
177
|
+
f"The flag --auth-entries is only supported by type {type}"
|
178
|
+
)
|
179
|
+
|
170
180
|
|
171
181
|
def _parse_entry(entry: str) -> dict[str,str]:
|
172
182
|
split_entry = entry.split('=', 1)
|
@@ -176,6 +186,19 @@ def _parse_entry(entry: str) -> dict[str,str]:
|
|
176
186
|
exit(1)
|
177
187
|
return {split_entry[0]: split_entry[1]}
|
178
188
|
|
189
|
+
def _get_oauth_custom_fields(token_entries: List[ConnectionCredentialsEntry] | None, auth_entries: List[ConnectionCredentialsEntry] | None) -> dict:
|
190
|
+
custom_fields = ConnectionCredentialsCustomFields()
|
191
|
+
|
192
|
+
if token_entries:
|
193
|
+
for entry in token_entries:
|
194
|
+
custom_fields.add_field(entry, is_token=True)
|
195
|
+
|
196
|
+
if auth_entries:
|
197
|
+
for entry in auth_entries:
|
198
|
+
custom_fields.add_field(entry, is_token=False)
|
199
|
+
|
200
|
+
return custom_fields.model_dump(exclude_none=True)
|
201
|
+
|
179
202
|
def _get_credentials(type: ConnectionType, **kwargs):
|
180
203
|
match type:
|
181
204
|
case ConnectionType.BASIC_AUTH:
|
@@ -192,35 +215,39 @@ def _get_credentials(type: ConnectionType, **kwargs):
|
|
192
215
|
api_key=kwargs.get("api_key")
|
193
216
|
)
|
194
217
|
case ConnectionType.OAUTH2_AUTH_CODE:
|
218
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
195
219
|
return OAuth2AuthCodeCredentials(
|
196
220
|
authorization_url=kwargs.get("auth_url"),
|
197
221
|
client_id=kwargs.get("client_id"),
|
198
222
|
client_secret=kwargs.get("client_secret"),
|
199
223
|
token_url=kwargs.get("token_url"),
|
200
|
-
scope=kwargs.get("scope")
|
224
|
+
scope=kwargs.get("scope"),
|
225
|
+
**custom_fields
|
201
226
|
)
|
202
227
|
case ConnectionType.OAUTH2_CLIENT_CREDS:
|
203
228
|
# using filtered args as default values will not be set if 'None' is passed, causing validation errors
|
204
229
|
keys = ["client_id","client_secret","token_url","grant_type","send_via", "scope"]
|
205
230
|
filtered_args = { key_name: kwargs[key_name] for key_name in keys if kwargs.get(key_name) }
|
206
|
-
|
231
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
232
|
+
return OAuth2ClientCredentials(**filtered_args, **custom_fields)
|
207
233
|
# case ConnectionType.OAUTH2_IMPLICIT:
|
208
234
|
# return OAuth2ImplicitCredentials(
|
209
235
|
# authorization_url=kwargs.get("auth_url"),
|
210
236
|
# client_id=kwargs.get("client_id"),
|
211
237
|
# )
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
# )
|
238
|
+
case ConnectionType.OAUTH2_PASSWORD:
|
239
|
+
keys = ["username", "password", "client_id","client_secret","token_url","grant_type", "scope"]
|
240
|
+
filtered_args = { key_name: kwargs[key_name] for key_name in keys if kwargs.get(key_name) }
|
241
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
242
|
+
return OAuth2PasswordCredentials(**filtered_args, **custom_fields)
|
243
|
+
|
219
244
|
case ConnectionType.OAUTH_ON_BEHALF_OF_FLOW:
|
245
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
220
246
|
return OAuthOnBehalfOfCredentials(
|
221
247
|
client_id=kwargs.get("client_id"),
|
222
248
|
access_token_url=kwargs.get("token_url"),
|
223
|
-
grant_type=kwargs.get("grant_type")
|
249
|
+
grant_type=kwargs.get("grant_type"),
|
250
|
+
**custom_fields
|
224
251
|
)
|
225
252
|
case ConnectionType.KEY_VALUE:
|
226
253
|
env = {}
|
@@ -233,6 +260,23 @@ def _get_credentials(type: ConnectionType, **kwargs):
|
|
233
260
|
case _:
|
234
261
|
raise ValueError(f"Invalid type '{type}' selected")
|
235
262
|
|
263
|
+
def _connection_credentials_parse_entry(text: str, default_location: ConnectionCredentialsEntryLocation) -> ConnectionCredentialsEntry:
|
264
|
+
location_kv_pair = text.split(":", 1)
|
265
|
+
key_value = location_kv_pair[-1]
|
266
|
+
location = location_kv_pair[0] if len(location_kv_pair)>1 else default_location
|
267
|
+
|
268
|
+
valid_locations = [item.value for item in ConnectionCredentialsEntryLocation]
|
269
|
+
if location not in valid_locations:
|
270
|
+
raise typer.BadParameter(f"The provided location '{location}' is not in the allowed values {valid_locations}.")
|
271
|
+
|
272
|
+
key_value_pair = key_value.split('=', 1)
|
273
|
+
if len(key_value_pair) != 2:
|
274
|
+
message = f"The entry '{text}' is not in the expected form '<location>:<key>=<value>' or '<key>=<value>'"
|
275
|
+
raise typer.BadParameter(message)
|
276
|
+
key, value = key_value_pair[0], key_value_pair[1]
|
277
|
+
|
278
|
+
return ConnectionCredentialsEntry(key=key, value=value, location=location)
|
279
|
+
|
236
280
|
|
237
281
|
def add_configuration(config: ConnectionConfiguration) -> None:
|
238
282
|
client = get_connections_client()
|
@@ -283,25 +327,24 @@ def add_configuration(config: ConnectionConfiguration) -> None:
|
|
283
327
|
logger.error(response_text)
|
284
328
|
exit(1)
|
285
329
|
|
286
|
-
def add_credentials(app_id: str, environment: ConnectionEnvironment, use_app_credentials: bool, credentials: CREDENTIALS) -> None:
|
330
|
+
def add_credentials(app_id: str, environment: ConnectionEnvironment, use_app_credentials: bool, credentials: CREDENTIALS, payload: dict = None) -> None:
|
287
331
|
client = get_connections_client()
|
288
332
|
try:
|
289
333
|
existing_credentials = client.get_credentials(app_id=app_id, env=environment, use_app_credentials=use_app_credentials)
|
290
|
-
if
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
334
|
+
if not payload:
|
335
|
+
if use_app_credentials:
|
336
|
+
payload = {
|
337
|
+
"app_credentials": credentials.model_dump(exclude_none=True)
|
338
|
+
}
|
339
|
+
else:
|
340
|
+
payload = {
|
341
|
+
"runtime_credentials": credentials.model_dump(exclude_none=True)
|
342
|
+
}
|
298
343
|
|
299
|
-
logger.info(f"Setting credentials for environment '{environment}' on connection '{app_id}'")
|
300
344
|
if existing_credentials:
|
301
345
|
client.update_credentials(app_id=app_id, env=environment, use_app_credentials=use_app_credentials, payload=payload)
|
302
346
|
else:
|
303
347
|
client.create_credentials(app_id=app_id,env=environment, use_app_credentials=use_app_credentials, payload=payload)
|
304
|
-
logger.info(f"Credentials successfully set for '{environment}' environment of connection '{app_id}'")
|
305
348
|
except requests.HTTPError as e:
|
306
349
|
response = e.response
|
307
350
|
response_text = response.text
|
@@ -489,7 +532,20 @@ def set_credentials_connection(
|
|
489
532
|
_validate_connection_params(type=conn_type, **kwargs)
|
490
533
|
credentials = _get_credentials(type=conn_type, **kwargs)
|
491
534
|
|
492
|
-
|
535
|
+
# Special handling for oauth2 password flow as it sends both app_creds and runtime_creds
|
536
|
+
logger.info(f"Setting credentials for environment '{environment}' on connection '{app_id}'")
|
537
|
+
if conn_type == ConnectionType.OAUTH2_PASSWORD:
|
538
|
+
credentials_model = credentials.model_dump(exclude_none=True)
|
539
|
+
runtime_cred_keys = {"username", "password"}
|
540
|
+
app_creds = {"app_credentials": {k: credentials_model[k] for k in credentials_model if k not in runtime_cred_keys}}
|
541
|
+
runtime_creds = {"runtime_credentials": {k: credentials_model[k] for k in credentials_model if k in runtime_cred_keys}}
|
542
|
+
|
543
|
+
add_credentials(app_id=app_id, environment=environment, use_app_credentials=True, credentials=credentials, payload=app_creds)
|
544
|
+
add_credentials(app_id=app_id, environment=environment, use_app_credentials=False, credentials=credentials, payload=runtime_creds)
|
545
|
+
else:
|
546
|
+
add_credentials(app_id=app_id, environment=environment, use_app_credentials=use_app_credentials, credentials=credentials)
|
547
|
+
|
548
|
+
logger.info(f"Credentials successfully set for '{environment}' environment of connection '{app_id}'")
|
493
549
|
|
494
550
|
def set_identity_provider_connection(
|
495
551
|
app_id: str,
|
@@ -514,5 +570,15 @@ def set_identity_provider_connection(
|
|
514
570
|
logger.error(f"Cannot set Identity Provider when 'sso' is false in configuration. Please enable sso for connection '{app_id}' in environment '{environment}' and try again.")
|
515
571
|
sys.exit(1)
|
516
572
|
|
517
|
-
|
573
|
+
custom_fields = _get_oauth_custom_fields(token_entries=kwargs.get("token_entries"), auth_entries=None)
|
574
|
+
idp = IdentityProviderCredentials(**kwargs, **custom_fields)
|
518
575
|
add_identity_provider(app_id=app_id, environment=environment, idp=idp)
|
576
|
+
|
577
|
+
def token_entry_connection_credentials_parse(text: str) -> ConnectionCredentialsEntry:
|
578
|
+
return _connection_credentials_parse_entry(text=text, default_location=ConnectionCredentialsEntryLocation.HEADER)
|
579
|
+
|
580
|
+
def auth_entry_connection_credentials_parse(text: str) -> ConnectionCredentialsEntry:
|
581
|
+
entry = _connection_credentials_parse_entry(text=text, default_location=ConnectionCredentialsEntryLocation.QUERY)
|
582
|
+
if entry.location != ConnectionCredentialsEntryLocation.QUERY:
|
583
|
+
raise typer.BadParameter(f"Only location '{ConnectionCredentialsEntryLocation.QUERY}' is supported for --auth-entry")
|
584
|
+
return entry
|
@@ -13,6 +13,7 @@ from ibm_watsonx_orchestrate.client.knowledge_bases.knowledge_base_client import
|
|
13
13
|
from ibm_watsonx_orchestrate.client.base_api_client import ClientAPIException
|
14
14
|
from ibm_watsonx_orchestrate.client.connections import get_connections_client
|
15
15
|
from ibm_watsonx_orchestrate.client.utils import instantiate_client
|
16
|
+
from ibm_watsonx_orchestrate.agent_builder.knowledge_bases.types import FileUpload
|
16
17
|
|
17
18
|
logger = logging.getLogger(__name__)
|
18
19
|
|
@@ -43,7 +44,8 @@ def parse_file(file: str) -> List[KnowledgeBase]:
|
|
43
44
|
def to_column_name(col: str):
|
44
45
|
return " ".join([word.capitalize() if not word[0].isupper() else word for word in col.split("_")])
|
45
46
|
|
46
|
-
def get_file_name(
|
47
|
+
def get_file_name(file: str | FileUpload):
|
48
|
+
path = file.path if isinstance(file, FileUpload) else file
|
47
49
|
# This name prettifying currently screws up file type detection on ingestion
|
48
50
|
# return to_column_name(path.split("/")[-1].split(".")[0])
|
49
51
|
return path.split("/")[-1]
|
@@ -55,7 +57,11 @@ def get_relative_file_path(path, dir):
|
|
55
57
|
return f"{dir}{path.removeprefix('.')}"
|
56
58
|
else:
|
57
59
|
return f"{dir}/{path}"
|
58
|
-
|
60
|
+
|
61
|
+
def build_file_object(file_dir: str, file: str | FileUpload):
|
62
|
+
if isinstance(file, FileUpload):
|
63
|
+
return ('files', (get_file_name(file.path), open(get_relative_file_path(file.path, file_dir), 'rb')))
|
64
|
+
return ('files', (get_file_name(file), open(get_relative_file_path(file, file_dir), 'rb')))
|
59
65
|
|
60
66
|
class KnowledgeBaseController:
|
61
67
|
def __init__(self):
|
@@ -101,13 +107,19 @@ class KnowledgeBaseController:
|
|
101
107
|
|
102
108
|
kb.validate_documents_or_index_exists()
|
103
109
|
if kb.documents:
|
104
|
-
files = [(
|
110
|
+
files = [build_file_object(file_dir, file) for file in kb.documents]
|
111
|
+
file_urls = { get_file_name(file): file.url for file in kb.documents if isinstance(file, FileUpload) and file.url }
|
105
112
|
|
106
113
|
kb.prioritize_built_in_index = True
|
107
114
|
payload = kb.model_dump(exclude_none=True);
|
108
115
|
payload.pop('documents');
|
109
116
|
|
110
|
-
|
117
|
+
data = {
|
118
|
+
'knowledge_base': json.dumps(payload),
|
119
|
+
'file_urls': json.dumps(file_urls)
|
120
|
+
}
|
121
|
+
|
122
|
+
client.create_built_in(payload=data, files=files)
|
111
123
|
else:
|
112
124
|
if len(kb.conversational_search_tool.index_config) != 1:
|
113
125
|
raise ValueError(f"Must provide exactly one conversational_search_tool.index_config. Provided {len(kb.conversational_search_tool.index_config)}.")
|
@@ -118,7 +130,9 @@ class KnowledgeBaseController:
|
|
118
130
|
raise ValueError(f"Must provide credentials (via --app-id) when using milvus or elastic_search.")
|
119
131
|
|
120
132
|
kb.prioritize_built_in_index = False
|
121
|
-
|
133
|
+
data = { 'knowledge_base': json.dumps(kb.model_dump(exclude_none=True)) }
|
134
|
+
|
135
|
+
client.create(payload=data)
|
122
136
|
|
123
137
|
logger.info(f"Successfully imported knowledge base '{kb.name}'")
|
124
138
|
except ClientAPIException as e:
|
@@ -151,8 +165,8 @@ class KnowledgeBaseController:
|
|
151
165
|
existing_docs = [doc.get("metadata", {}).get("original_file_name", "") for doc in status.get("documents", [])]
|
152
166
|
|
153
167
|
removed_docs = existing_docs[:]
|
154
|
-
for
|
155
|
-
filename = get_file_name(
|
168
|
+
for file in kb.documents:
|
169
|
+
filename = get_file_name(file)
|
156
170
|
|
157
171
|
if filename in existing_docs:
|
158
172
|
logger.warning(f'Document \"{filename}\" already exists in knowledge base. Updating...')
|
@@ -162,17 +176,25 @@ class KnowledgeBaseController:
|
|
162
176
|
logger.warning(f'Document \"{filename}\" removed from knowledge base.')
|
163
177
|
|
164
178
|
|
165
|
-
files = [(
|
179
|
+
files = [build_file_object(file_dir, file) for file in kb.documents]
|
180
|
+
file_urls = { get_file_name(file): file.url for file in kb.documents if isinstance(file, FileUpload) and file.url }
|
166
181
|
|
167
182
|
kb.prioritize_built_in_index = True
|
168
183
|
payload = kb.model_dump(exclude_none=True);
|
169
184
|
payload.pop('documents');
|
170
185
|
|
171
|
-
|
186
|
+
data = {
|
187
|
+
'knowledge_base': json.dumps(payload),
|
188
|
+
'file_urls': json.dumps(file_urls)
|
189
|
+
}
|
190
|
+
|
191
|
+
self.get_client().update_with_documents(knowledge_base_id, payload=data, files=files)
|
172
192
|
else:
|
173
193
|
if kb.conversational_search_tool and kb.conversational_search_tool.index_config:
|
174
194
|
kb.prioritize_built_in_index = False
|
175
|
-
|
195
|
+
|
196
|
+
data = { 'knowledge_base': json.dumps(kb.model_dump(exclude_none=True)) }
|
197
|
+
self.get_client().update(knowledge_base_id, payload=data)
|
176
198
|
|
177
199
|
logger.info(f"Knowledge base '{kb.name}' updated successfully")
|
178
200
|
|
@@ -191,9 +191,6 @@ def get_default_registry_env_vars_by_dev_edition_source(default_env: dict, user_
|
|
191
191
|
parsed = urlparse(wo_url)
|
192
192
|
hostname = parsed.hostname
|
193
193
|
|
194
|
-
if not hostname or not hostname.startswith("api."):
|
195
|
-
raise ValueError(f"Invalid WO_INSTANCE URL: '{wo_url}'. It should starts with 'api.'")
|
196
|
-
|
197
194
|
registry_url = f"registry.{hostname[4:]}/cp/wxo-lite"
|
198
195
|
else:
|
199
196
|
raise ValueError(f"Unknown value for developer edition source: {source}. Must be one of ['internal', 'myibm', 'orchestrate'].")
|
@@ -332,7 +329,6 @@ def write_merged_env_file(merged_env: dict, target_path: str = None) -> Path:
|
|
332
329
|
file.write(f"{key}={val}\n")
|
333
330
|
return Path(file.name)
|
334
331
|
|
335
|
-
|
336
332
|
def get_dbtag_from_architecture(merged_env_dict: dict) -> str:
|
337
333
|
"""Detects system architecture and returns the corresponding DBTAG."""
|
338
334
|
arch = platform.machine()
|
@@ -375,7 +371,8 @@ def run_compose_lite(
|
|
375
371
|
experimental_with_langfuse=False,
|
376
372
|
experimental_with_ibm_telemetry=False,
|
377
373
|
with_doc_processing=False,
|
378
|
-
with_voice=False
|
374
|
+
with_voice=False,
|
375
|
+
experimental_with_langflow=False,
|
379
376
|
) -> None:
|
380
377
|
compose_path = get_compose_file()
|
381
378
|
|
@@ -404,7 +401,11 @@ def run_compose_lite(
|
|
404
401
|
logger.info("Database container started successfully. Now starting other services...")
|
405
402
|
|
406
403
|
|
407
|
-
# Step 2:
|
404
|
+
# Step 2: Create Langflow DB (if enabled)
|
405
|
+
if experimental_with_langflow:
|
406
|
+
create_langflow_db()
|
407
|
+
|
408
|
+
# Step 3: Start all remaining services (except DB)
|
408
409
|
profiles = []
|
409
410
|
if experimental_with_langfuse:
|
410
411
|
profiles.append("langfuse")
|
@@ -414,6 +415,8 @@ def run_compose_lite(
|
|
414
415
|
profiles.append("docproc")
|
415
416
|
if with_voice:
|
416
417
|
profiles.append("voice")
|
418
|
+
if experimental_with_langflow:
|
419
|
+
profiles.append("langflow")
|
417
420
|
|
418
421
|
command = compose_command[:]
|
419
422
|
for profile in profiles:
|
@@ -665,7 +668,6 @@ def run_compose_lite_down(final_env_file: Path, is_reset: bool = False) -> None:
|
|
665
668
|
)
|
666
669
|
sys.exit(1)
|
667
670
|
|
668
|
-
|
669
671
|
def run_compose_lite_logs(final_env_file: Path, is_reset: bool = False) -> None:
|
670
672
|
compose_path = get_compose_file()
|
671
673
|
compose_command = ensure_docker_compose_installed()
|
@@ -866,6 +868,12 @@ def server_start(
|
|
866
868
|
'--with-voice', '-v',
|
867
869
|
help='Enable voice controller to interact with the chat via voice channels'
|
868
870
|
),
|
871
|
+
experimental_with_langflow: bool = typer.Option(
|
872
|
+
False,
|
873
|
+
'--experimental-with-langflow',
|
874
|
+
help='(Experimental) Enable Langflow UI, available at http://localhost:7861',
|
875
|
+
hidden=True
|
876
|
+
),
|
869
877
|
):
|
870
878
|
confirm_accepts_license_agreement(accept_terms_and_conditions)
|
871
879
|
|
@@ -907,6 +915,9 @@ def server_start(
|
|
907
915
|
if experimental_with_ibm_telemetry:
|
908
916
|
merged_env_dict['USE_IBM_TELEMETRY'] = 'true'
|
909
917
|
|
918
|
+
if experimental_with_langflow:
|
919
|
+
merged_env_dict['LANGFLOW_ENABLED'] = 'true'
|
920
|
+
|
910
921
|
|
911
922
|
try:
|
912
923
|
dev_edition_source = get_dev_edition_source(merged_env_dict)
|
@@ -921,7 +932,8 @@ def server_start(
|
|
921
932
|
experimental_with_langfuse=experimental_with_langfuse,
|
922
933
|
experimental_with_ibm_telemetry=experimental_with_ibm_telemetry,
|
923
934
|
with_doc_processing=with_doc_processing,
|
924
|
-
with_voice=with_voice
|
935
|
+
with_voice=with_voice,
|
936
|
+
experimental_with_langflow=experimental_with_langflow)
|
925
937
|
|
926
938
|
run_db_migration()
|
927
939
|
|
@@ -951,6 +963,8 @@ def server_start(
|
|
951
963
|
logger.info(f"You can access the observability platform Langfuse at http://localhost:3010, username: orchestrate@ibm.com, password: orchestrate")
|
952
964
|
if with_doc_processing:
|
953
965
|
logger.info(f"Document processing in Flows (Public Preview) has been enabled.")
|
966
|
+
if experimental_with_langflow:
|
967
|
+
logger.info("Langflow has been enabled, the Langflow UI is available at http://localhost:7861")
|
954
968
|
|
955
969
|
@server_app.command(name="stop")
|
956
970
|
def server_stop(
|
@@ -1031,15 +1045,11 @@ def run_db_migration() -> None:
|
|
1031
1045
|
merged_env_dict['ROUTING_LLM_API_KEY'] = ''
|
1032
1046
|
merged_env_dict['ASSISTANT_LLM_API_KEY'] = ''
|
1033
1047
|
final_env_file = write_merged_env_file(merged_env_dict)
|
1048
|
+
|
1034
1049
|
|
1035
|
-
|
1036
|
-
|
1037
|
-
|
1038
|
-
"exec",
|
1039
|
-
"wxo-server-db",
|
1040
|
-
"bash",
|
1041
|
-
"-c",
|
1042
|
-
'''
|
1050
|
+
pg_user = merged_env_dict.get("POSTGRES_USER","postgres")
|
1051
|
+
|
1052
|
+
migration_command = f'''
|
1043
1053
|
APPLIED_MIGRATIONS_FILE="/var/lib/postgresql/applied_migrations/applied_migrations.txt"
|
1044
1054
|
touch "$APPLIED_MIGRATIONS_FILE"
|
1045
1055
|
|
@@ -1050,7 +1060,7 @@ def run_db_migration() -> None:
|
|
1050
1060
|
echo "Skipping already applied migration: $filename"
|
1051
1061
|
else
|
1052
1062
|
echo "Applying migration: $filename"
|
1053
|
-
if psql -U
|
1063
|
+
if psql -U {pg_user} -d postgres -q -f "$file" > /dev/null 2>&1; then
|
1054
1064
|
echo "$filename" >> "$APPLIED_MIGRATIONS_FILE"
|
1055
1065
|
else
|
1056
1066
|
echo "Error applying $filename. Stopping migrations."
|
@@ -1059,6 +1069,15 @@ def run_db_migration() -> None:
|
|
1059
1069
|
fi
|
1060
1070
|
done
|
1061
1071
|
'''
|
1072
|
+
|
1073
|
+
command = compose_command + [
|
1074
|
+
"-f", str(compose_path),
|
1075
|
+
"--env-file", str(final_env_file),
|
1076
|
+
"exec",
|
1077
|
+
"wxo-server-db",
|
1078
|
+
"bash",
|
1079
|
+
"-c",
|
1080
|
+
migration_command
|
1062
1081
|
]
|
1063
1082
|
|
1064
1083
|
logger.info("Running Database Migration...")
|
@@ -1073,6 +1092,65 @@ def run_db_migration() -> None:
|
|
1073
1092
|
)
|
1074
1093
|
sys.exit(1)
|
1075
1094
|
|
1095
|
+
def create_langflow_db() -> None:
|
1096
|
+
compose_path = get_compose_file()
|
1097
|
+
compose_command = ensure_docker_compose_installed()
|
1098
|
+
default_env_path = get_default_env_file()
|
1099
|
+
merged_env_dict = merge_env(default_env_path, user_env_path=None)
|
1100
|
+
merged_env_dict['WATSONX_SPACE_ID']='X'
|
1101
|
+
merged_env_dict['WATSONX_APIKEY']='X'
|
1102
|
+
merged_env_dict['WXAI_API_KEY'] = ''
|
1103
|
+
merged_env_dict['ASSISTANT_EMBEDDINGS_API_KEY'] = ''
|
1104
|
+
merged_env_dict['ASSISTANT_LLM_SPACE_ID'] = ''
|
1105
|
+
merged_env_dict['ROUTING_LLM_SPACE_ID'] = ''
|
1106
|
+
merged_env_dict['USE_SAAS_ML_TOOLS_RUNTIME'] = ''
|
1107
|
+
merged_env_dict['BAM_API_KEY'] = ''
|
1108
|
+
merged_env_dict['ASSISTANT_EMBEDDINGS_SPACE_ID'] = ''
|
1109
|
+
merged_env_dict['ROUTING_LLM_API_KEY'] = ''
|
1110
|
+
merged_env_dict['ASSISTANT_LLM_API_KEY'] = ''
|
1111
|
+
final_env_file = write_merged_env_file(merged_env_dict)
|
1112
|
+
|
1113
|
+
pg_timeout = merged_env_dict.get('POSTGRES_READY_TIMEOUT','10')
|
1114
|
+
|
1115
|
+
pg_user = merged_env_dict.get("POSTGRES_USER","postgres")
|
1116
|
+
|
1117
|
+
creation_command = f"""
|
1118
|
+
echo 'Waiting for pg to initialize...'
|
1119
|
+
|
1120
|
+
timeout={pg_timeout}
|
1121
|
+
while [[ -z `pg_isready | grep 'accepting connections'` ]] && (( timeout > 0 )); do
|
1122
|
+
((timeout-=1)) && sleep 1;
|
1123
|
+
done
|
1124
|
+
|
1125
|
+
if psql -U {pg_user} -lqt | cut -d \\| -f 1 | grep -qw langflow; then
|
1126
|
+
echo 'Existing Langflow DB found'
|
1127
|
+
else
|
1128
|
+
echo 'Creating Langflow DB'
|
1129
|
+
createdb -U "{pg_user}" -O "{pg_user}" langflow;
|
1130
|
+
psql -U {pg_user} -q -d postgres -c "GRANT CONNECT ON DATABASE langflow TO {pg_user}";
|
1131
|
+
fi
|
1132
|
+
"""
|
1133
|
+
command = compose_command + [
|
1134
|
+
"-f", str(compose_path),
|
1135
|
+
"--env-file", str(final_env_file),
|
1136
|
+
"exec",
|
1137
|
+
"wxo-server-db",
|
1138
|
+
"bash",
|
1139
|
+
"-c",
|
1140
|
+
creation_command
|
1141
|
+
]
|
1142
|
+
|
1143
|
+
logger.info("Preparing Langflow resources...")
|
1144
|
+
result = subprocess.run(command, capture_output=False)
|
1145
|
+
|
1146
|
+
if result.returncode == 0:
|
1147
|
+
logger.info("Langflow resources sucessfully created")
|
1148
|
+
else:
|
1149
|
+
error_message = result.stderr.decode('utf-8') if result.stderr else "Error occurred."
|
1150
|
+
logger.error(
|
1151
|
+
f"Failed to create Langflow resources\n{error_message}"
|
1152
|
+
)
|
1153
|
+
sys.exit(1)
|
1076
1154
|
|
1077
1155
|
def bump_file_iteration(filename: str) -> str:
|
1078
1156
|
regex = re.compile(f"^(?P<name>[^\\(\\s\\.\\)]+)(\\((?P<num>\\d+)\\))?(?P<type>\\.(?:{'|'.join(_EXPORT_FILE_TYPES)}))?$")
|
@@ -1,5 +1,6 @@
|
|
1
1
|
import logging
|
2
2
|
import sys
|
3
|
+
import uuid
|
3
4
|
from enum import Enum
|
4
5
|
from pydantic import BaseModel, model_validator, ConfigDict
|
5
6
|
|
@@ -43,9 +44,6 @@ class WatsonXAIEnvConfig(BaseModel):
|
|
43
44
|
if not config.get("WATSONX_SPACE_ID") and not config.get("WATSONX_APIKEY"):
|
44
45
|
raise ValueError("Missing configuration requirements 'WATSONX_SPACE_ID' and 'WATSONX_APIKEY'")
|
45
46
|
|
46
|
-
if config.get("WATSONX_SPACE_ID") and not config.get("WATSONX_APIKEY"):
|
47
|
-
logger.error("Cannot use env var 'WATSONX_SPACE_ID' without setting the corresponding 'WATSONX_APIKEY'")
|
48
|
-
sys.exit(1)
|
49
47
|
|
50
48
|
if not config.get("WATSONX_SPACE_ID") and config.get("WATSONX_APIKEY"):
|
51
49
|
logger.error("Cannot use env var 'WATSONX_APIKEY' without setting the corresponding 'WATSONX_SPACE_ID'")
|
@@ -54,6 +52,12 @@ class WatsonXAIEnvConfig(BaseModel):
|
|
54
52
|
config["USE_SAAS_ML_TOOLS_RUNTIME"] = False
|
55
53
|
return config
|
56
54
|
|
55
|
+
def is_valid_uuid(value) -> bool:
|
56
|
+
try:
|
57
|
+
uuid.UUID(str(value))
|
58
|
+
return True
|
59
|
+
except (ValueError, TypeError, AttributeError):
|
60
|
+
return False
|
57
61
|
|
58
62
|
class ModelGatewayEnvConfig(BaseModel):
|
59
63
|
WO_API_KEY: str | None = None
|
@@ -84,7 +88,10 @@ class ModelGatewayEnvConfig(BaseModel):
|
|
84
88
|
if not config.get("AUTHORIZATION_URL"):
|
85
89
|
inferred_auth_url = AUTH_TYPE_DEFAULT_URL_MAPPING.get(auth_type)
|
86
90
|
if not inferred_auth_url:
|
87
|
-
|
91
|
+
if auth_type == WoAuthType.CPD:
|
92
|
+
inferred_auth_url = config.get("WO_INSTANCE") + '/icp4d-api/v1/authorize'
|
93
|
+
else:
|
94
|
+
logger.error(f"No 'AUTHORIZATION_URL' found. Auth type '{auth_type}' does not support defaulting. Please set the 'AUTHORIZATION_URL' explictly")
|
88
95
|
sys.exit(1)
|
89
96
|
config["AUTHORIZATION_URL"] = inferred_auth_url
|
90
97
|
|
@@ -101,6 +108,7 @@ class ModelGatewayEnvConfig(BaseModel):
|
|
101
108
|
sys.exit(1)
|
102
109
|
|
103
110
|
config["USE_SAAS_ML_TOOLS_RUNTIME"] = True
|
104
|
-
|
105
|
-
|
111
|
+
if not is_valid_uuid(config.get("WATSONX_SPACE_ID")):
|
112
|
+
# Fake (but valid) UUIDv4 for knowledgebase check
|
113
|
+
config["WATSONX_SPACE_ID"] = "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa"
|
106
114
|
return config
|