mlrun 1.10.0rc20__py3-none-any.whl → 1.10.0rc22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/artifacts/llm_prompt.py +11 -10
- mlrun/artifacts/model.py +3 -3
- mlrun/common/schemas/auth.py +2 -0
- mlrun/db/base.py +9 -0
- mlrun/db/httpdb.py +21 -1
- mlrun/db/nopdb.py +8 -0
- mlrun/execution.py +52 -10
- mlrun/model_monitoring/applications/__init__.py +1 -1
- mlrun/model_monitoring/applications/base.py +86 -33
- mlrun/model_monitoring/db/_schedules.py +21 -0
- mlrun/model_monitoring/db/tsdb/base.py +14 -5
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +4 -5
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +53 -20
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +39 -1
- mlrun/projects/project.py +50 -7
- mlrun/run.py +38 -5
- mlrun/serving/states.py +169 -16
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.10.0rc20.dist-info → mlrun-1.10.0rc22.dist-info}/METADATA +4 -3
- {mlrun-1.10.0rc20.dist-info → mlrun-1.10.0rc22.dist-info}/RECORD +24 -24
- {mlrun-1.10.0rc20.dist-info → mlrun-1.10.0rc22.dist-info}/WHEEL +0 -0
- {mlrun-1.10.0rc20.dist-info → mlrun-1.10.0rc22.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc20.dist-info → mlrun-1.10.0rc22.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc20.dist-info → mlrun-1.10.0rc22.dist-info}/top_level.txt +0 -0
|
@@ -22,7 +22,6 @@ import taosws
|
|
|
22
22
|
import mlrun.common.schemas.model_monitoring as mm_schemas
|
|
23
23
|
import mlrun.common.types
|
|
24
24
|
import mlrun.model_monitoring.db.tsdb.tdengine.schemas as tdengine_schemas
|
|
25
|
-
import mlrun.model_monitoring.db.tsdb.tdengine.stream_graph_steps
|
|
26
25
|
from mlrun.datastore.datastore_profile import DatastoreProfile
|
|
27
26
|
from mlrun.model_monitoring.db import TSDBConnector
|
|
28
27
|
from mlrun.model_monitoring.db.tsdb.tdengine.tdengine_connection import (
|
|
@@ -205,7 +204,7 @@ class TDEngineConnector(TSDBConnector):
|
|
|
205
204
|
@staticmethod
|
|
206
205
|
def _generate_filter_query(
|
|
207
206
|
filter_column: str, filter_values: Union[str, list[Union[str, int]]]
|
|
208
|
-
) ->
|
|
207
|
+
) -> str:
|
|
209
208
|
"""
|
|
210
209
|
Generate a filter query for TDEngine based on the provided column and values.
|
|
211
210
|
|
|
@@ -213,15 +212,14 @@ class TDEngineConnector(TSDBConnector):
|
|
|
213
212
|
:param filter_values: A single value or a list of values to filter by.
|
|
214
213
|
|
|
215
214
|
:return: A string representing the filter query.
|
|
216
|
-
:raise:
|
|
215
|
+
:raise: ``MLRunValueError`` if the filter values are not of type string or list.
|
|
217
216
|
"""
|
|
218
|
-
|
|
219
217
|
if isinstance(filter_values, str):
|
|
220
218
|
return f"{filter_column}='{filter_values}'"
|
|
221
219
|
elif isinstance(filter_values, list):
|
|
222
220
|
return f"{filter_column} IN ({', '.join(repr(v) for v in filter_values)}) "
|
|
223
221
|
else:
|
|
224
|
-
raise mlrun.errors.
|
|
222
|
+
raise mlrun.errors.MLRunValueError(
|
|
225
223
|
f"Invalid filter values {filter_values}: must be a string or a list, "
|
|
226
224
|
f"got {type(filter_values).__name__}; filter values: {filter_values}"
|
|
227
225
|
)
|
|
@@ -311,10 +309,7 @@ class TDEngineConnector(TSDBConnector):
|
|
|
311
309
|
flush_after_seconds=tsdb_batching_timeout_secs,
|
|
312
310
|
)
|
|
313
311
|
|
|
314
|
-
def delete_tsdb_records(
|
|
315
|
-
self,
|
|
316
|
-
endpoint_ids: list[str],
|
|
317
|
-
):
|
|
312
|
+
def delete_tsdb_records(self, endpoint_ids: list[str]) -> None:
|
|
318
313
|
"""
|
|
319
314
|
To delete subtables within TDEngine, we first query the subtables names with the provided endpoint_ids.
|
|
320
315
|
Then, we drop each subtable.
|
|
@@ -332,9 +327,7 @@ class TDEngineConnector(TSDBConnector):
|
|
|
332
327
|
get_subtable_query = self.tables[table]._get_subtables_query_by_tag(
|
|
333
328
|
filter_tag="endpoint_id", filter_values=endpoint_ids
|
|
334
329
|
)
|
|
335
|
-
subtables_result = self.connection.run(
|
|
336
|
-
query=get_subtable_query,
|
|
337
|
-
)
|
|
330
|
+
subtables_result = self.connection.run(query=get_subtable_query)
|
|
338
331
|
subtables.extend([subtable[0] for subtable in subtables_result.data])
|
|
339
332
|
except Exception as e:
|
|
340
333
|
logger.warning(
|
|
@@ -346,15 +339,13 @@ class TDEngineConnector(TSDBConnector):
|
|
|
346
339
|
)
|
|
347
340
|
|
|
348
341
|
# Prepare the drop statements
|
|
349
|
-
drop_statements = [
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
)
|
|
342
|
+
drop_statements = [
|
|
343
|
+
self.tables[table].drop_subtable_query(subtable=subtable)
|
|
344
|
+
for subtable in subtables
|
|
345
|
+
]
|
|
354
346
|
try:
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
)
|
|
347
|
+
logger.debug("Dropping subtables", drop_statements=drop_statements)
|
|
348
|
+
self.connection.run(statements=drop_statements)
|
|
358
349
|
except Exception as e:
|
|
359
350
|
logger.warning(
|
|
360
351
|
"Failed to delete model endpoint resources. You may need to delete them manually. "
|
|
@@ -369,6 +360,48 @@ class TDEngineConnector(TSDBConnector):
|
|
|
369
360
|
number_of_endpoints_to_delete=len(endpoint_ids),
|
|
370
361
|
)
|
|
371
362
|
|
|
363
|
+
def delete_application_records(
|
|
364
|
+
self, application_name: str, endpoint_ids: Optional[list[str]] = None
|
|
365
|
+
) -> None:
|
|
366
|
+
"""
|
|
367
|
+
Delete application records from the TSDB for the given model endpoints or all if ``endpoint_ids`` is ``None``.
|
|
368
|
+
"""
|
|
369
|
+
logger.debug(
|
|
370
|
+
"Deleting application records",
|
|
371
|
+
project=self.project,
|
|
372
|
+
application_name=application_name,
|
|
373
|
+
endpoint_ids=endpoint_ids,
|
|
374
|
+
)
|
|
375
|
+
tables = [
|
|
376
|
+
self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS],
|
|
377
|
+
self.tables[mm_schemas.TDEngineSuperTables.METRICS],
|
|
378
|
+
]
|
|
379
|
+
|
|
380
|
+
filter_query = self._generate_filter_query(
|
|
381
|
+
filter_column=mm_schemas.ApplicationEvent.APPLICATION_NAME,
|
|
382
|
+
filter_values=application_name,
|
|
383
|
+
)
|
|
384
|
+
if endpoint_ids:
|
|
385
|
+
endpoint_ids_filter = self._generate_filter_query(
|
|
386
|
+
filter_column=mm_schemas.EventFieldType.ENDPOINT_ID,
|
|
387
|
+
filter_values=endpoint_ids,
|
|
388
|
+
)
|
|
389
|
+
filter_query += f" AND {endpoint_ids_filter}"
|
|
390
|
+
|
|
391
|
+
drop_statements: list[str] = []
|
|
392
|
+
for table in tables:
|
|
393
|
+
get_subtable_query = table._get_tables_query_by_condition(filter_query)
|
|
394
|
+
subtables_result = self.connection.run(query=get_subtable_query)
|
|
395
|
+
drop_statements.extend(
|
|
396
|
+
[
|
|
397
|
+
table.drop_subtable_query(subtable=subtable[0])
|
|
398
|
+
for subtable in subtables_result.data
|
|
399
|
+
]
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
logger.debug("Dropping application records", drop_statements=drop_statements)
|
|
403
|
+
self.connection.run(statements=drop_statements)
|
|
404
|
+
|
|
372
405
|
def delete_tsdb_resources(self):
|
|
373
406
|
"""
|
|
374
407
|
Delete all project resources in the TSDB connector, such as model endpoints data and drift results.
|
|
@@ -492,7 +492,8 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
492
492
|
# Split the endpoint ids into chunks to avoid exceeding the v3io-engine filter-expression limit
|
|
493
493
|
for i in range(0, len(endpoint_ids), V3IO_FRAMESD_MEPS_LIMIT):
|
|
494
494
|
endpoint_id_chunk = endpoint_ids[i : i + V3IO_FRAMESD_MEPS_LIMIT]
|
|
495
|
-
|
|
495
|
+
endpoints_list = "', '".join(endpoint_id_chunk)
|
|
496
|
+
filter_query = f"endpoint_id IN('{endpoints_list}')"
|
|
496
497
|
for table in tables:
|
|
497
498
|
try:
|
|
498
499
|
self.frames_client.delete(
|
|
@@ -532,6 +533,43 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
532
533
|
project=self.project,
|
|
533
534
|
)
|
|
534
535
|
|
|
536
|
+
def delete_application_records(
|
|
537
|
+
self, application_name: str, endpoint_ids: Optional[list[str]] = None
|
|
538
|
+
) -> None:
|
|
539
|
+
"""
|
|
540
|
+
Delete application records from the TSDB for the given model endpoints or all if ``endpoint_ids`` is ``None``.
|
|
541
|
+
"""
|
|
542
|
+
base_filter_query = f"application_name=='{application_name}'"
|
|
543
|
+
|
|
544
|
+
filter_queries: list[str] = []
|
|
545
|
+
if endpoint_ids:
|
|
546
|
+
for i in range(0, len(endpoint_ids), V3IO_FRAMESD_MEPS_LIMIT):
|
|
547
|
+
endpoint_id_chunk = endpoint_ids[i : i + V3IO_FRAMESD_MEPS_LIMIT]
|
|
548
|
+
endpoints_list = "', '".join(endpoint_id_chunk)
|
|
549
|
+
filter_queries.append(
|
|
550
|
+
f"{base_filter_query} AND endpoint_id IN ('{endpoints_list}')"
|
|
551
|
+
)
|
|
552
|
+
else:
|
|
553
|
+
filter_queries = [base_filter_query]
|
|
554
|
+
|
|
555
|
+
for table in [
|
|
556
|
+
self.tables[mm_schemas.V3IOTSDBTables.APP_RESULTS],
|
|
557
|
+
self.tables[mm_schemas.V3IOTSDBTables.METRICS],
|
|
558
|
+
]:
|
|
559
|
+
logger.debug(
|
|
560
|
+
"Deleting application records from TSDB",
|
|
561
|
+
table=table,
|
|
562
|
+
filter_queries=filter_queries,
|
|
563
|
+
project=self.project,
|
|
564
|
+
)
|
|
565
|
+
for filter_query in filter_queries:
|
|
566
|
+
self.frames_client.delete(
|
|
567
|
+
backend=_TSDB_BE,
|
|
568
|
+
table=table,
|
|
569
|
+
filter=filter_query,
|
|
570
|
+
start="0",
|
|
571
|
+
)
|
|
572
|
+
|
|
535
573
|
def get_model_endpoint_real_time_metrics(
|
|
536
574
|
self, endpoint_id: str, metrics: list[str], start: str, end: str
|
|
537
575
|
) -> dict[str, list[tuple[str, float]]]:
|
mlrun/projects/project.py
CHANGED
|
@@ -1908,13 +1908,51 @@ class MlrunProject(ModelObj):
|
|
|
1908
1908
|
|
|
1909
1909
|
Examples::
|
|
1910
1910
|
|
|
1911
|
+
# Log directly with an inline prompt template
|
|
1912
|
+
project.log_llm_prompt(
|
|
1913
|
+
key="customer_support_prompt",
|
|
1914
|
+
prompt_template=[
|
|
1915
|
+
{
|
|
1916
|
+
"role": "system",
|
|
1917
|
+
"content": "You are a helpful customer support assistant.",
|
|
1918
|
+
},
|
|
1919
|
+
{
|
|
1920
|
+
"role": "user",
|
|
1921
|
+
"content": "The customer reports: {issue_description}",
|
|
1922
|
+
},
|
|
1923
|
+
],
|
|
1924
|
+
prompt_legend={
|
|
1925
|
+
"issue_description": {
|
|
1926
|
+
"field": "user_issue",
|
|
1927
|
+
"description": "Detailed description of the customer's issue",
|
|
1928
|
+
},
|
|
1929
|
+
"solution": {
|
|
1930
|
+
"field": "proposed_solution",
|
|
1931
|
+
"description": "Suggested fix for the customer's issue",
|
|
1932
|
+
},
|
|
1933
|
+
},
|
|
1934
|
+
model_artifact=model,
|
|
1935
|
+
model_configuration={"temperature": 0.5, "max_tokens": 200},
|
|
1936
|
+
description="Prompt for handling customer support queries",
|
|
1937
|
+
tag="support-v1",
|
|
1938
|
+
labels={"domain": "support"},
|
|
1939
|
+
)
|
|
1940
|
+
|
|
1911
1941
|
# Log a prompt from file
|
|
1912
1942
|
project.log_llm_prompt(
|
|
1913
|
-
key="
|
|
1914
|
-
prompt_path="prompts/
|
|
1915
|
-
prompt_legend={
|
|
1943
|
+
key="qa_prompt",
|
|
1944
|
+
prompt_path="prompts/template.json",
|
|
1945
|
+
prompt_legend={
|
|
1946
|
+
"question": {
|
|
1947
|
+
"field": "user_question",
|
|
1948
|
+
"description": "The actual question asked by the user",
|
|
1949
|
+
}
|
|
1950
|
+
},
|
|
1916
1951
|
model_artifact=model,
|
|
1952
|
+
model_configuration={"temperature": 0.7, "max_tokens": 256},
|
|
1953
|
+
description="Q&A prompt template with user-provided question",
|
|
1917
1954
|
tag="v2",
|
|
1955
|
+
labels={"task": "qa", "stage": "experiment"},
|
|
1918
1956
|
)
|
|
1919
1957
|
|
|
1920
1958
|
:param key: Unique key for the prompt artifact.
|
|
@@ -1923,7 +1961,10 @@ class MlrunProject(ModelObj):
|
|
|
1923
1961
|
"role": "user", "content": "I need your help with {profession}"]. only "role" and "content" keys allow in any
|
|
1924
1962
|
str format (upper/lower case), keys will be modified to lower case.
|
|
1925
1963
|
Cannot be used with `prompt_path`.
|
|
1926
|
-
:param prompt_path: Path to a file containing the prompt.
|
|
1964
|
+
:param prompt_path: Path to a JSON file containing the prompt template.
|
|
1965
|
+
Cannot be used together with `prompt_template`.
|
|
1966
|
+
The file should define a list of dictionaries in the same format
|
|
1967
|
+
supported by `prompt_template`.
|
|
1927
1968
|
:param prompt_legend: A dictionary where each key is a placeholder in the prompt (e.g., ``{user_name}``)
|
|
1928
1969
|
and the value is a dictionary holding two keys, "field", "description". "field" points to the field in
|
|
1929
1970
|
the event where the value of the place-holder inside the event, if None or not exist will be replaced
|
|
@@ -1932,9 +1973,11 @@ class MlrunProject(ModelObj):
|
|
|
1932
1973
|
:param model_artifact: Reference to the parent model (either `ModelArtifact` or model URI string).
|
|
1933
1974
|
:param model_configuration: Configuration dictionary for model generation parameters
|
|
1934
1975
|
(e.g., temperature, max tokens).
|
|
1935
|
-
:param description:
|
|
1936
|
-
:param target_path:
|
|
1937
|
-
:param artifact_path:
|
|
1976
|
+
:param description: Optional description of the prompt.
|
|
1977
|
+
:param target_path: Absolute target path (instead of using artifact_path + local_path)
|
|
1978
|
+
:param artifact_path: Target artifact path (when not using the default)
|
|
1979
|
+
To define a subpath under the default location use:
|
|
1980
|
+
`artifact_path=context.artifact_subpath('data')`
|
|
1938
1981
|
:param tag: Version tag for the artifact (e.g., "v1", "latest").
|
|
1939
1982
|
:param labels: Labels to tag the artifact for filtering and organization.
|
|
1940
1983
|
:param upload: Whether to upload the artifact to a remote datastore. Defaults to True.
|
mlrun/run.py
CHANGED
|
@@ -141,7 +141,7 @@ def load_func_code(command="", workdir=None, secrets=None, name="name"):
|
|
|
141
141
|
else:
|
|
142
142
|
is_remote = "://" in command
|
|
143
143
|
data = get_object(command, secrets)
|
|
144
|
-
runtime = yaml.
|
|
144
|
+
runtime = yaml.safe_load(data)
|
|
145
145
|
runtime = new_function(runtime=runtime)
|
|
146
146
|
|
|
147
147
|
command = runtime.spec.command or ""
|
|
@@ -362,7 +362,10 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
|
|
|
362
362
|
return function
|
|
363
363
|
|
|
364
364
|
|
|
365
|
-
def import_function_to_dict(
|
|
365
|
+
def import_function_to_dict(
|
|
366
|
+
url: str,
|
|
367
|
+
secrets: Optional[dict] = None,
|
|
368
|
+
) -> dict:
|
|
366
369
|
"""Load function spec from local/remote YAML file"""
|
|
367
370
|
obj = get_object(url, secrets)
|
|
368
371
|
runtime = yaml.safe_load(obj)
|
|
@@ -388,6 +391,11 @@ def import_function_to_dict(url, secrets=None):
|
|
|
388
391
|
raise ValueError("exec path (spec.command) must be relative")
|
|
389
392
|
url = url[: url.rfind("/") + 1] + code_file
|
|
390
393
|
code = get_object(url, secrets)
|
|
394
|
+
code_file = _ensure_path_confined_to_base_dir(
|
|
395
|
+
base_directory=".",
|
|
396
|
+
relative_path=code_file,
|
|
397
|
+
error_message_on_escape="Path traversal detected in spec.command",
|
|
398
|
+
)
|
|
391
399
|
dir = path.dirname(code_file)
|
|
392
400
|
if dir:
|
|
393
401
|
makedirs(dir, exist_ok=True)
|
|
@@ -395,9 +403,16 @@ def import_function_to_dict(url, secrets=None):
|
|
|
395
403
|
fp.write(code)
|
|
396
404
|
elif cmd:
|
|
397
405
|
if not path.isfile(code_file):
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
406
|
+
slash_index = url.rfind("/")
|
|
407
|
+
if slash_index < 0:
|
|
408
|
+
raise ValueError(f"no file in exec path (spec.command={code_file})")
|
|
409
|
+
base_dir = os.path.normpath(url[: slash_index + 1])
|
|
410
|
+
candidate_path = _ensure_path_confined_to_base_dir(
|
|
411
|
+
base_directory=base_dir,
|
|
412
|
+
relative_path=code_file,
|
|
413
|
+
error_message_on_escape=f"exec file spec.command={code_file} is outside of allowed directory",
|
|
414
|
+
)
|
|
415
|
+
if path.isfile(candidate_path):
|
|
401
416
|
raise ValueError(
|
|
402
417
|
f"exec file spec.command={code_file} is relative, change working dir"
|
|
403
418
|
)
|
|
@@ -1258,3 +1273,21 @@ def wait_for_runs_completion(
|
|
|
1258
1273
|
runs = running
|
|
1259
1274
|
|
|
1260
1275
|
return completed
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def _ensure_path_confined_to_base_dir(
|
|
1279
|
+
base_directory: str,
|
|
1280
|
+
relative_path: str,
|
|
1281
|
+
error_message_on_escape: str,
|
|
1282
|
+
) -> str:
|
|
1283
|
+
"""
|
|
1284
|
+
Join `user_supplied_relative_path` to `allowed_base_directory`, normalise the result,
|
|
1285
|
+
and guarantee it stays inside `allowed_base_directory`.
|
|
1286
|
+
"""
|
|
1287
|
+
absolute_base_directory = path.abspath(base_directory)
|
|
1288
|
+
absolute_candidate_path = path.abspath(
|
|
1289
|
+
path.join(absolute_base_directory, relative_path)
|
|
1290
|
+
)
|
|
1291
|
+
if not absolute_candidate_path.startswith(absolute_base_directory + path.sep):
|
|
1292
|
+
raise ValueError(error_message_on_escape)
|
|
1293
|
+
return absolute_candidate_path
|
mlrun/serving/states.py
CHANGED
|
@@ -1210,6 +1210,55 @@ class Model(storey.ParallelExecutionRunnable, ModelObj):
|
|
|
1210
1210
|
|
|
1211
1211
|
|
|
1212
1212
|
class LLModel(Model):
|
|
1213
|
+
"""
|
|
1214
|
+
A model wrapper for handling LLM (Large Language Model) prompt-based inference.
|
|
1215
|
+
|
|
1216
|
+
This class extends the base `Model` to provide specialized handling for
|
|
1217
|
+
`LLMPromptArtifact` objects, enabling both synchronous and asynchronous
|
|
1218
|
+
invocation of language models.
|
|
1219
|
+
|
|
1220
|
+
**Model Invocation**:
|
|
1221
|
+
|
|
1222
|
+
- The execution of enriched prompts is delegated to the `model_provider`
|
|
1223
|
+
configured for the model (e.g., **Hugging Face** or **OpenAI**).
|
|
1224
|
+
- The `model_provider` is responsible for sending the prompt to the correct
|
|
1225
|
+
backend API and returning the generated output.
|
|
1226
|
+
- Users can override the `predict` and `predict_async` methods to customize
|
|
1227
|
+
the behavior of the model invocation.
|
|
1228
|
+
|
|
1229
|
+
**Prompt Enrichment Overview**:
|
|
1230
|
+
|
|
1231
|
+
- If an `LLMPromptArtifact` is found, load its prompt template and fill in
|
|
1232
|
+
placeholders using values from the request body.
|
|
1233
|
+
- If the artifact is not an `LLMPromptArtifact`, skip formatting and attempt
|
|
1234
|
+
to retrieve `messages` directly from the request body using the input path.
|
|
1235
|
+
|
|
1236
|
+
**Simplified Example**:
|
|
1237
|
+
|
|
1238
|
+
Input body::
|
|
1239
|
+
|
|
1240
|
+
{"city": "Paris", "days": 3}
|
|
1241
|
+
|
|
1242
|
+
Prompt template in artifact::
|
|
1243
|
+
|
|
1244
|
+
[
|
|
1245
|
+
{"role": "system", "content": "You are a travel planning assistant."},
|
|
1246
|
+
{"role": "user", "content": "Create a {{days}}-day itinerary for {{city}}."},
|
|
1247
|
+
]
|
|
1248
|
+
|
|
1249
|
+
Result after enrichment::
|
|
1250
|
+
|
|
1251
|
+
[
|
|
1252
|
+
{"role": "system", "content": "You are a travel planning assistant."},
|
|
1253
|
+
{"role": "user", "content": "Create a 3-day itinerary for Paris."},
|
|
1254
|
+
]
|
|
1255
|
+
|
|
1256
|
+
:param name: Name of the model.
|
|
1257
|
+
:param input_path: Path in the request body where input data is located.
|
|
1258
|
+
:param result_path: Path in the response body where model outputs and the statistics
|
|
1259
|
+
will be stored.
|
|
1260
|
+
"""
|
|
1261
|
+
|
|
1213
1262
|
def __init__(
|
|
1214
1263
|
self,
|
|
1215
1264
|
name: str,
|
|
@@ -1220,6 +1269,12 @@ class LLModel(Model):
|
|
|
1220
1269
|
super().__init__(name, **kwargs)
|
|
1221
1270
|
self._input_path = split_path(input_path)
|
|
1222
1271
|
self._result_path = split_path(result_path)
|
|
1272
|
+
logger.info(
|
|
1273
|
+
"LLModel initialized",
|
|
1274
|
+
model_name=name,
|
|
1275
|
+
input_path=input_path,
|
|
1276
|
+
result_path=result_path,
|
|
1277
|
+
)
|
|
1223
1278
|
|
|
1224
1279
|
def predict(
|
|
1225
1280
|
self,
|
|
@@ -1231,6 +1286,12 @@ class LLModel(Model):
|
|
|
1231
1286
|
if isinstance(
|
|
1232
1287
|
self.invocation_artifact, mlrun.artifacts.LLMPromptArtifact
|
|
1233
1288
|
) and isinstance(self.model_provider, ModelProvider):
|
|
1289
|
+
logger.debug(
|
|
1290
|
+
"Invoking model provider",
|
|
1291
|
+
model_name=self.name,
|
|
1292
|
+
messages=messages,
|
|
1293
|
+
model_configuration=model_configuration,
|
|
1294
|
+
)
|
|
1234
1295
|
response_with_stats = self.model_provider.invoke(
|
|
1235
1296
|
messages=messages,
|
|
1236
1297
|
invoke_response_format=InvokeResponseFormat.USAGE,
|
|
@@ -1239,6 +1300,19 @@ class LLModel(Model):
|
|
|
1239
1300
|
set_data_by_path(
|
|
1240
1301
|
path=self._result_path, data=body, value=response_with_stats
|
|
1241
1302
|
)
|
|
1303
|
+
logger.debug(
|
|
1304
|
+
"LLModel prediction completed",
|
|
1305
|
+
model_name=self.name,
|
|
1306
|
+
answer=response_with_stats.get("answer"),
|
|
1307
|
+
usage=response_with_stats.get("usage"),
|
|
1308
|
+
)
|
|
1309
|
+
else:
|
|
1310
|
+
logger.warning(
|
|
1311
|
+
"LLModel invocation artifact or model provider not set, skipping prediction",
|
|
1312
|
+
model_name=self.name,
|
|
1313
|
+
invocation_artifact_type=type(self.invocation_artifact).__name__,
|
|
1314
|
+
model_provider_type=type(self.model_provider).__name__,
|
|
1315
|
+
)
|
|
1242
1316
|
return body
|
|
1243
1317
|
|
|
1244
1318
|
async def predict_async(
|
|
@@ -1251,6 +1325,12 @@ class LLModel(Model):
|
|
|
1251
1325
|
if isinstance(
|
|
1252
1326
|
self.invocation_artifact, mlrun.artifacts.LLMPromptArtifact
|
|
1253
1327
|
) and isinstance(self.model_provider, ModelProvider):
|
|
1328
|
+
logger.debug(
|
|
1329
|
+
"Async invoking model provider",
|
|
1330
|
+
model_name=self.name,
|
|
1331
|
+
messages=messages,
|
|
1332
|
+
model_configuration=model_configuration,
|
|
1333
|
+
)
|
|
1254
1334
|
response_with_stats = await self.model_provider.async_invoke(
|
|
1255
1335
|
messages=messages,
|
|
1256
1336
|
invoke_response_format=InvokeResponseFormat.USAGE,
|
|
@@ -1259,10 +1339,29 @@ class LLModel(Model):
|
|
|
1259
1339
|
set_data_by_path(
|
|
1260
1340
|
path=self._result_path, data=body, value=response_with_stats
|
|
1261
1341
|
)
|
|
1342
|
+
logger.debug(
|
|
1343
|
+
"LLModel async prediction completed",
|
|
1344
|
+
model_name=self.name,
|
|
1345
|
+
answer=response_with_stats.get("answer"),
|
|
1346
|
+
usage=response_with_stats.get("usage"),
|
|
1347
|
+
)
|
|
1348
|
+
else:
|
|
1349
|
+
logger.warning(
|
|
1350
|
+
"LLModel invocation artifact or model provider not set, skipping async prediction",
|
|
1351
|
+
model_name=self.name,
|
|
1352
|
+
invocation_artifact_type=type(self.invocation_artifact).__name__,
|
|
1353
|
+
model_provider_type=type(self.model_provider).__name__,
|
|
1354
|
+
)
|
|
1262
1355
|
return body
|
|
1263
1356
|
|
|
1264
1357
|
def run(self, body: Any, path: str, origin_name: Optional[str] = None) -> Any:
|
|
1265
1358
|
messages, model_configuration = self.enrich_prompt(body, origin_name)
|
|
1359
|
+
logger.info(
|
|
1360
|
+
"Calling LLModel predict",
|
|
1361
|
+
model_name=self.name,
|
|
1362
|
+
model_endpoint_name=origin_name,
|
|
1363
|
+
messages_len=len(messages) if messages else 0,
|
|
1364
|
+
)
|
|
1266
1365
|
return self.predict(
|
|
1267
1366
|
body, messages=messages, model_configuration=model_configuration
|
|
1268
1367
|
)
|
|
@@ -1271,6 +1370,12 @@ class LLModel(Model):
|
|
|
1271
1370
|
self, body: Any, path: str, origin_name: Optional[str] = None
|
|
1272
1371
|
) -> Any:
|
|
1273
1372
|
messages, model_configuration = self.enrich_prompt(body, origin_name)
|
|
1373
|
+
logger.info(
|
|
1374
|
+
"Calling LLModel async predict",
|
|
1375
|
+
model_name=self.name,
|
|
1376
|
+
model_endpoint_name=origin_name,
|
|
1377
|
+
messages_len=len(messages) if messages else 0,
|
|
1378
|
+
)
|
|
1274
1379
|
return await self.predict_async(
|
|
1275
1380
|
body, messages=messages, model_configuration=model_configuration
|
|
1276
1381
|
)
|
|
@@ -1278,6 +1383,11 @@ class LLModel(Model):
|
|
|
1278
1383
|
def enrich_prompt(
|
|
1279
1384
|
self, body: dict, origin_name: str
|
|
1280
1385
|
) -> Union[tuple[list[dict], dict], tuple[None, None]]:
|
|
1386
|
+
logger.info(
|
|
1387
|
+
"Enriching prompt",
|
|
1388
|
+
model_name=self.name,
|
|
1389
|
+
model_endpoint_name=origin_name,
|
|
1390
|
+
)
|
|
1281
1391
|
if origin_name and self.shared_proxy_mapping:
|
|
1282
1392
|
llm_prompt_artifact = self.shared_proxy_mapping.get(origin_name)
|
|
1283
1393
|
if isinstance(llm_prompt_artifact, str):
|
|
@@ -1287,18 +1397,22 @@ class LLModel(Model):
|
|
|
1287
1397
|
llm_prompt_artifact = (
|
|
1288
1398
|
self.invocation_artifact or self._get_artifact_object()
|
|
1289
1399
|
)
|
|
1290
|
-
if not (
|
|
1400
|
+
if not llm_prompt_artifact or not (
|
|
1291
1401
|
llm_prompt_artifact and isinstance(llm_prompt_artifact, LLMPromptArtifact)
|
|
1292
1402
|
):
|
|
1293
1403
|
logger.warning(
|
|
1294
|
-
"
|
|
1404
|
+
"LLModel must be provided with LLMPromptArtifact",
|
|
1405
|
+
model_name=self.name,
|
|
1406
|
+
artifact_type=type(llm_prompt_artifact).__name__,
|
|
1295
1407
|
llm_prompt_artifact=llm_prompt_artifact,
|
|
1296
1408
|
)
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1409
|
+
prompt_legend, prompt_template, model_configuration = {}, [], {}
|
|
1410
|
+
else:
|
|
1411
|
+
prompt_legend = llm_prompt_artifact.spec.prompt_legend
|
|
1412
|
+
prompt_template = deepcopy(llm_prompt_artifact.read_prompt())
|
|
1413
|
+
model_configuration = llm_prompt_artifact.spec.model_configuration
|
|
1300
1414
|
input_data = copy(get_data_from_path(self._input_path, body))
|
|
1301
|
-
if isinstance(input_data, dict):
|
|
1415
|
+
if isinstance(input_data, dict) and prompt_template:
|
|
1302
1416
|
kwargs = (
|
|
1303
1417
|
{
|
|
1304
1418
|
place_holder: input_data.get(body_map["field"])
|
|
@@ -1315,23 +1429,40 @@ class LLModel(Model):
|
|
|
1315
1429
|
message["content"] = message["content"].format(**input_data)
|
|
1316
1430
|
except KeyError as e:
|
|
1317
1431
|
logger.warning(
|
|
1318
|
-
"Input data
|
|
1319
|
-
|
|
1432
|
+
"Input data missing placeholder, content stays unformatted",
|
|
1433
|
+
model_name=self.name,
|
|
1434
|
+
key_error=mlrun.errors.err_to_str(e),
|
|
1320
1435
|
)
|
|
1321
1436
|
message["content"] = message["content"].format_map(
|
|
1322
1437
|
default_place_holders
|
|
1323
1438
|
)
|
|
1439
|
+
elif isinstance(input_data, dict) and not prompt_template:
|
|
1440
|
+
# If there is no prompt template, we assume the input data is already in the correct format.
|
|
1441
|
+
logger.debug("Attempting to retrieve messages from the request body.")
|
|
1442
|
+
prompt_template = input_data.get("messages", [])
|
|
1324
1443
|
else:
|
|
1325
1444
|
logger.warning(
|
|
1326
|
-
|
|
1327
|
-
|
|
1445
|
+
"Expected input data to be a dict, prompt template stays unformatted",
|
|
1446
|
+
model_name=self.name,
|
|
1447
|
+
input_data_type=type(input_data).__name__,
|
|
1328
1448
|
)
|
|
1329
|
-
return prompt_template,
|
|
1449
|
+
return prompt_template, model_configuration
|
|
1330
1450
|
|
|
1331
1451
|
|
|
1332
|
-
class ModelSelector:
|
|
1452
|
+
class ModelSelector(ModelObj):
|
|
1333
1453
|
"""Used to select which models to run on each event."""
|
|
1334
1454
|
|
|
1455
|
+
def __init__(self, **kwargs):
|
|
1456
|
+
super().__init__()
|
|
1457
|
+
|
|
1458
|
+
def __init_subclass__(cls):
|
|
1459
|
+
super().__init_subclass__()
|
|
1460
|
+
cls._dict_fields = list(
|
|
1461
|
+
set(cls._dict_fields)
|
|
1462
|
+
| set(inspect.signature(cls.__init__).parameters.keys())
|
|
1463
|
+
)
|
|
1464
|
+
cls._dict_fields.remove("self")
|
|
1465
|
+
|
|
1335
1466
|
def select(
|
|
1336
1467
|
self, event, available_models: list[Model]
|
|
1337
1468
|
) -> Union[list[str], list[Model]]:
|
|
@@ -1442,15 +1573,33 @@ class ModelRunnerStep(MonitoredStep):
|
|
|
1442
1573
|
*args,
|
|
1443
1574
|
name: Optional[str] = None,
|
|
1444
1575
|
model_selector: Optional[Union[str, ModelSelector]] = None,
|
|
1576
|
+
model_selector_parameters: Optional[dict] = None,
|
|
1445
1577
|
raise_exception: bool = True,
|
|
1446
1578
|
**kwargs,
|
|
1447
1579
|
):
|
|
1580
|
+
if isinstance(model_selector, ModelSelector) and model_selector_parameters:
|
|
1581
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
1582
|
+
"Cannot provide a model_selector object as argument to `model_selector` and also provide "
|
|
1583
|
+
"`model_selector_parameters`."
|
|
1584
|
+
)
|
|
1585
|
+
if model_selector:
|
|
1586
|
+
model_selector_parameters = model_selector_parameters or (
|
|
1587
|
+
model_selector.to_dict()
|
|
1588
|
+
if isinstance(model_selector, ModelSelector)
|
|
1589
|
+
else {}
|
|
1590
|
+
)
|
|
1591
|
+
model_selector = (
|
|
1592
|
+
model_selector
|
|
1593
|
+
if isinstance(model_selector, str)
|
|
1594
|
+
else model_selector.__class__.__name__
|
|
1595
|
+
)
|
|
1596
|
+
|
|
1448
1597
|
super().__init__(
|
|
1449
1598
|
*args,
|
|
1450
1599
|
name=name,
|
|
1451
1600
|
raise_exception=raise_exception,
|
|
1452
1601
|
class_name="mlrun.serving.ModelRunner",
|
|
1453
|
-
class_args=dict(model_selector=model_selector),
|
|
1602
|
+
class_args=dict(model_selector=(model_selector, model_selector_parameters)),
|
|
1454
1603
|
**kwargs,
|
|
1455
1604
|
)
|
|
1456
1605
|
self.raise_exception = raise_exception
|
|
@@ -1827,13 +1976,17 @@ class ModelRunnerStep(MonitoredStep):
|
|
|
1827
1976
|
if not self._is_local_function(context):
|
|
1828
1977
|
# skip init of non local functions
|
|
1829
1978
|
return
|
|
1830
|
-
model_selector = self.class_args.get(
|
|
1979
|
+
model_selector, model_selector_params = self.class_args.get(
|
|
1980
|
+
"model_selector", (None, None)
|
|
1981
|
+
)
|
|
1831
1982
|
execution_mechanism_by_model_name = self.class_args.get(
|
|
1832
1983
|
schemas.ModelRunnerStepData.MODEL_TO_EXECUTION_MECHANISM
|
|
1833
1984
|
)
|
|
1834
1985
|
models = self.class_args.get(schemas.ModelRunnerStepData.MODELS, {})
|
|
1835
|
-
if
|
|
1836
|
-
model_selector = get_class(model_selector, namespace)(
|
|
1986
|
+
if model_selector:
|
|
1987
|
+
model_selector = get_class(model_selector, namespace).from_dict(
|
|
1988
|
+
model_selector_params, init_with_params=True
|
|
1989
|
+
)
|
|
1837
1990
|
model_objects = []
|
|
1838
1991
|
for model, model_params in models.values():
|
|
1839
1992
|
model_params[schemas.MonitoringData.INPUT_PATH] = (
|
mlrun/utils/version/version.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mlrun
|
|
3
|
-
Version: 1.10.
|
|
3
|
+
Version: 1.10.0rc22
|
|
4
4
|
Summary: Tracking and config of machine learning runs
|
|
5
5
|
Home-page: https://github.com/mlrun/mlrun
|
|
6
6
|
Author: Yaron Haviv
|
|
@@ -21,7 +21,8 @@ Classifier: Topic :: Software Development :: Libraries
|
|
|
21
21
|
Requires-Python: >=3.9, <3.12
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
License-File: LICENSE
|
|
24
|
-
Requires-Dist: urllib3
|
|
24
|
+
Requires-Dist: urllib3>=1.26.20; python_version < "3.11"
|
|
25
|
+
Requires-Dist: urllib3>=2.5.0; python_version >= "3.11"
|
|
25
26
|
Requires-Dist: GitPython>=3.1.41,~=3.1
|
|
26
27
|
Requires-Dist: aiohttp~=3.11
|
|
27
28
|
Requires-Dist: aiohttp-retry~=2.9
|
|
@@ -38,7 +39,7 @@ Requires-Dist: tabulate~=0.8.6
|
|
|
38
39
|
Requires-Dist: v3io~=0.7.0
|
|
39
40
|
Requires-Dist: pydantic>=1.10.15
|
|
40
41
|
Requires-Dist: mergedeep~=1.3
|
|
41
|
-
Requires-Dist: v3io-frames~=0.10.
|
|
42
|
+
Requires-Dist: v3io-frames~=0.10.15; python_version < "3.11"
|
|
42
43
|
Requires-Dist: v3io-frames>=0.13.0; python_version >= "3.11"
|
|
43
44
|
Requires-Dist: semver~=3.0
|
|
44
45
|
Requires-Dist: dependency-injector~=4.41
|