mlrun 1.10.0rc2__py3-none-any.whl → 1.10.0rc4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +2 -2
- mlrun/__main__.py +2 -2
- mlrun/artifacts/__init__.py +1 -0
- mlrun/artifacts/base.py +20 -8
- mlrun/artifacts/dataset.py +1 -1
- mlrun/artifacts/document.py +1 -1
- mlrun/artifacts/helpers.py +40 -0
- mlrun/artifacts/llm_prompt.py +165 -0
- mlrun/artifacts/manager.py +13 -1
- mlrun/artifacts/model.py +92 -12
- mlrun/artifacts/plots.py +2 -2
- mlrun/common/formatters/artifact.py +1 -0
- mlrun/common/runtimes/constants.py +0 -21
- mlrun/common/schemas/artifact.py +12 -12
- mlrun/common/schemas/pipeline.py +0 -16
- mlrun/common/schemas/project.py +0 -17
- mlrun/common/schemas/runs.py +0 -17
- mlrun/config.py +3 -3
- mlrun/datastore/base.py +2 -2
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/datastore_profile.py +3 -11
- mlrun/datastore/redis.py +2 -3
- mlrun/datastore/sources.py +0 -9
- mlrun/datastore/store_resources.py +3 -3
- mlrun/datastore/storeytargets.py +2 -5
- mlrun/datastore/targets.py +7 -57
- mlrun/datastore/utils.py +1 -11
- mlrun/db/base.py +7 -6
- mlrun/db/httpdb.py +72 -66
- mlrun/db/nopdb.py +1 -0
- mlrun/errors.py +22 -1
- mlrun/execution.py +87 -1
- mlrun/feature_store/common.py +5 -5
- mlrun/feature_store/feature_set.py +10 -6
- mlrun/feature_store/feature_vector.py +8 -6
- mlrun/launcher/base.py +1 -1
- mlrun/lists.py +1 -1
- mlrun/model.py +0 -5
- mlrun/model_monitoring/__init__.py +0 -1
- mlrun/model_monitoring/api.py +0 -44
- mlrun/model_monitoring/applications/evidently/base.py +3 -41
- mlrun/model_monitoring/controller.py +1 -1
- mlrun/model_monitoring/writer.py +1 -4
- mlrun/projects/operations.py +3 -3
- mlrun/projects/project.py +260 -23
- mlrun/run.py +9 -27
- mlrun/runtimes/base.py +6 -6
- mlrun/runtimes/kubejob.py +2 -2
- mlrun/runtimes/nuclio/function.py +3 -3
- mlrun/runtimes/nuclio/serving.py +13 -23
- mlrun/runtimes/remotesparkjob.py +6 -0
- mlrun/runtimes/sparkjob/spark3job.py +6 -0
- mlrun/serving/__init__.py +5 -1
- mlrun/serving/server.py +39 -3
- mlrun/serving/states.py +101 -4
- mlrun/serving/v2_serving.py +1 -1
- mlrun/utils/helpers.py +66 -9
- mlrun/utils/notifications/notification/slack.py +5 -1
- mlrun/utils/notifications/notification_pusher.py +2 -1
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.10.0rc2.dist-info → mlrun-1.10.0rc4.dist-info}/METADATA +22 -10
- {mlrun-1.10.0rc2.dist-info → mlrun-1.10.0rc4.dist-info}/RECORD +66 -65
- {mlrun-1.10.0rc2.dist-info → mlrun-1.10.0rc4.dist-info}/WHEEL +1 -1
- mlrun/model_monitoring/tracking_policy.py +0 -124
- {mlrun-1.10.0rc2.dist-info → mlrun-1.10.0rc4.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc2.dist-info → mlrun-1.10.0rc4.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc2.dist-info → mlrun-1.10.0rc4.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py
CHANGED
|
@@ -82,6 +82,7 @@ from ..artifacts import (
|
|
|
82
82
|
DatasetArtifact,
|
|
83
83
|
DocumentArtifact,
|
|
84
84
|
DocumentLoaderSpec,
|
|
85
|
+
LLMPromptArtifact,
|
|
85
86
|
ModelArtifact,
|
|
86
87
|
)
|
|
87
88
|
from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifact_path
|
|
@@ -278,7 +279,7 @@ def new_project(
|
|
|
278
279
|
for key, val in parameters.items():
|
|
279
280
|
project.spec.params[key] = val
|
|
280
281
|
|
|
281
|
-
|
|
282
|
+
_set_as_current_active_project(project)
|
|
282
283
|
|
|
283
284
|
if save and mlrun.mlconf.dbpath:
|
|
284
285
|
if overwrite:
|
|
@@ -451,7 +452,7 @@ def load_project(
|
|
|
451
452
|
if sync_functions:
|
|
452
453
|
project.sync_functions(save=to_save)
|
|
453
454
|
|
|
454
|
-
|
|
455
|
+
_set_as_current_active_project(project)
|
|
455
456
|
|
|
456
457
|
return project
|
|
457
458
|
|
|
@@ -471,7 +472,7 @@ def get_or_create_project(
|
|
|
471
472
|
allow_cross_project: Optional[bool] = None,
|
|
472
473
|
) -> "MlrunProject":
|
|
473
474
|
"""Load a project from MLRun DB, or create/import if it does not exist.
|
|
474
|
-
The project will become the
|
|
475
|
+
The project will become the active project for the current session.
|
|
475
476
|
|
|
476
477
|
MLRun looks for a project.yaml file with project definition and objects in the project root path
|
|
477
478
|
and use it to initialize the project, in addition it runs the project_setup.py file (if it exists)
|
|
@@ -757,10 +758,10 @@ def _project_instance_from_struct(struct, name, allow_cross_project):
|
|
|
757
758
|
)
|
|
758
759
|
|
|
759
760
|
if allow_cross_project is None:
|
|
760
|
-
# TODO: Remove this warning in version 1.
|
|
761
|
+
# TODO: Remove this warning in version 1.10.0 and also fix cli to support allow_cross_project
|
|
761
762
|
warnings.warn(
|
|
762
763
|
f"Project {name=} is different than specified on the context's project yaml. "
|
|
763
|
-
"This behavior is deprecated and will not be supported from version 1.
|
|
764
|
+
"This behavior is deprecated and will not be supported from version 1.10.0."
|
|
764
765
|
)
|
|
765
766
|
logger.warn(error_message)
|
|
766
767
|
elif allow_cross_project:
|
|
@@ -1799,6 +1800,8 @@ class MlrunProject(ModelObj):
|
|
|
1799
1800
|
training_set=None,
|
|
1800
1801
|
label_column=None,
|
|
1801
1802
|
extra_data=None,
|
|
1803
|
+
model_url: Optional[str] = None,
|
|
1804
|
+
default_config=None,
|
|
1802
1805
|
**kwargs,
|
|
1803
1806
|
) -> ModelArtifact:
|
|
1804
1807
|
"""Log a model artifact and optionally upload it to datastore
|
|
@@ -1841,7 +1844,9 @@ class MlrunProject(ModelObj):
|
|
|
1841
1844
|
:param label_column: which columns in the training set are the label (target) columns
|
|
1842
1845
|
:param extra_data: key/value list of extra files/charts to link with this dataset
|
|
1843
1846
|
value can be absolute path | relative path (to model dir) | bytes | artifact object
|
|
1844
|
-
|
|
1847
|
+
:param model_url: Remote model url.
|
|
1848
|
+
:param default_config: Default configuration for client building
|
|
1849
|
+
Saved as a sub-dictionary under the parameter.
|
|
1845
1850
|
:returns: model artifact object
|
|
1846
1851
|
"""
|
|
1847
1852
|
|
|
@@ -1864,6 +1869,8 @@ class MlrunProject(ModelObj):
|
|
|
1864
1869
|
feature_vector=feature_vector,
|
|
1865
1870
|
feature_weights=feature_weights,
|
|
1866
1871
|
extra_data=extra_data,
|
|
1872
|
+
model_url=model_url,
|
|
1873
|
+
default_config=default_config,
|
|
1867
1874
|
**kwargs,
|
|
1868
1875
|
)
|
|
1869
1876
|
if training_set is not None:
|
|
@@ -1881,6 +1888,87 @@ class MlrunProject(ModelObj):
|
|
|
1881
1888
|
)
|
|
1882
1889
|
return item
|
|
1883
1890
|
|
|
1891
|
+
def log_llm_prompt(
|
|
1892
|
+
self,
|
|
1893
|
+
key,
|
|
1894
|
+
prompt_string: Optional[str] = None,
|
|
1895
|
+
prompt_path: Optional[str] = None,
|
|
1896
|
+
prompt_legend: Optional[dict] = None,
|
|
1897
|
+
model_artifact: Union[ModelArtifact, str] = None,
|
|
1898
|
+
model_configuration: Optional[dict] = None,
|
|
1899
|
+
description: Optional[str] = None,
|
|
1900
|
+
target_path: Optional[str] = None,
|
|
1901
|
+
artifact_path: Optional[str] = None,
|
|
1902
|
+
tag: Optional[str] = None,
|
|
1903
|
+
labels: Optional[Union[list[str], str]] = None,
|
|
1904
|
+
upload: Optional[bool] = None,
|
|
1905
|
+
**kwargs,
|
|
1906
|
+
) -> LLMPromptArtifact:
|
|
1907
|
+
"""
|
|
1908
|
+
Log an LLM prompt artifact to the project.
|
|
1909
|
+
|
|
1910
|
+
This method creates and logs an `LLMPromptArtifact` which captures a prompt definition for large language model
|
|
1911
|
+
(LLM) interactions. The prompt can be provided as a string or a file, and may include metadata like generation
|
|
1912
|
+
parameters, a legend for variable injection, and references to a parent model artifact.
|
|
1913
|
+
|
|
1914
|
+
If the prompt content exceeds a certain length, it may be stored in a temporary file and logged accordingly.
|
|
1915
|
+
|
|
1916
|
+
Examples::
|
|
1917
|
+
|
|
1918
|
+
# Log a prompt from file
|
|
1919
|
+
project.log_llm_prompt(
|
|
1920
|
+
key="qa-prompt",
|
|
1921
|
+
prompt_path="prompts/qa_template.txt",
|
|
1922
|
+
prompt_legend={"question": "user_question"},
|
|
1923
|
+
model_artifact=model,
|
|
1924
|
+
tag="v2",
|
|
1925
|
+
)
|
|
1926
|
+
|
|
1927
|
+
:param key: Unique key for the prompt artifact.
|
|
1928
|
+
:param prompt_string: Raw prompt text. Mutually exclusive with `prompt_path`.
|
|
1929
|
+
:param prompt_path: Path to a file containing the prompt. Mutually exclusive with `prompt_string`.
|
|
1930
|
+
:param prompt_legend: A dictionary where each key is a placeholder in the prompt (e.g., ``{user_name}``)
|
|
1931
|
+
and the value is a description or explanation of what that placeholder represents.
|
|
1932
|
+
Useful for documenting and clarifying dynamic parts of the prompt.
|
|
1933
|
+
:param model_artifact: Reference to the parent model (either `ModelArtifact` or model URI string).
|
|
1934
|
+
:param model_configuration: Configuration dictionary for model generation parameters
|
|
1935
|
+
(e.g., temperature, max tokens).
|
|
1936
|
+
:param description: Optional description of the prompt.
|
|
1937
|
+
:param target_path: Optional local target path for saving prompt content.
|
|
1938
|
+
:param artifact_path: Storage path for the logged artifact.
|
|
1939
|
+
:param tag: Version tag for the artifact (e.g., "v1", "latest").
|
|
1940
|
+
:param labels: Labels to tag the artifact for filtering and organization.
|
|
1941
|
+
:param upload: Whether to upload the artifact to a remote datastore. Defaults to True.
|
|
1942
|
+
:param kwargs: Additional attributes to pass into the `LLMPromptArtifact`.
|
|
1943
|
+
|
|
1944
|
+
:returns: The logged `LLMPromptArtifact` object.
|
|
1945
|
+
"""
|
|
1946
|
+
|
|
1947
|
+
llm_prompt = LLMPromptArtifact(
|
|
1948
|
+
key=key,
|
|
1949
|
+
project=self.name,
|
|
1950
|
+
prompt_string=prompt_string,
|
|
1951
|
+
prompt_path=prompt_path,
|
|
1952
|
+
prompt_legend=prompt_legend,
|
|
1953
|
+
model_artifact=model_artifact,
|
|
1954
|
+
model_configuration=model_configuration,
|
|
1955
|
+
target_path=target_path,
|
|
1956
|
+
description=description,
|
|
1957
|
+
**kwargs,
|
|
1958
|
+
)
|
|
1959
|
+
|
|
1960
|
+
item = cast(
|
|
1961
|
+
LLMPromptArtifact,
|
|
1962
|
+
self.log_artifact(
|
|
1963
|
+
llm_prompt,
|
|
1964
|
+
artifact_path=artifact_path,
|
|
1965
|
+
tag=tag,
|
|
1966
|
+
upload=upload,
|
|
1967
|
+
labels=labels,
|
|
1968
|
+
),
|
|
1969
|
+
)
|
|
1970
|
+
return item
|
|
1971
|
+
|
|
1884
1972
|
def get_vector_store_collection(
|
|
1885
1973
|
self,
|
|
1886
1974
|
vector_store: "VectorStore", # noqa: F821
|
|
@@ -2479,9 +2567,9 @@ class MlrunProject(ModelObj):
|
|
|
2479
2567
|
:param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
|
|
2480
2568
|
"""
|
|
2481
2569
|
if default_controller_image != "mlrun/mlrun":
|
|
2482
|
-
# TODO: Remove this in 1.
|
|
2570
|
+
# TODO: Remove this in 1.10.0
|
|
2483
2571
|
warnings.warn(
|
|
2484
|
-
"'default_controller_image' is deprecated and will be removed in 1.
|
|
2572
|
+
"'default_controller_image' is deprecated in 1.7.0 and will be removed in 1.10.0, "
|
|
2485
2573
|
"use 'image' instead",
|
|
2486
2574
|
FutureWarning,
|
|
2487
2575
|
)
|
|
@@ -2892,9 +2980,9 @@ class MlrunProject(ModelObj):
|
|
|
2892
2980
|
|
|
2893
2981
|
:param name: name of the model-monitoring-function/s (under the project)
|
|
2894
2982
|
"""
|
|
2895
|
-
# TODO: Remove this in 1.
|
|
2983
|
+
# TODO: Remove this in 1.10.0
|
|
2896
2984
|
warnings.warn(
|
|
2897
|
-
"'remove_model_monitoring_function' is deprecated and will be removed in 1.
|
|
2985
|
+
"'remove_model_monitoring_function' is deprecated in 1.7.0 and will be removed in 1.10.0. "
|
|
2898
2986
|
"Please use `delete_model_monitoring_function` instead.",
|
|
2899
2987
|
FutureWarning,
|
|
2900
2988
|
)
|
|
@@ -4107,9 +4195,9 @@ class MlrunProject(ModelObj):
|
|
|
4107
4195
|
(by default `/home/mlrun_code`)
|
|
4108
4196
|
"""
|
|
4109
4197
|
if not overwrite_build_params:
|
|
4110
|
-
# TODO: change overwrite_build_params default to True in 1.
|
|
4198
|
+
# TODO: change overwrite_build_params default to True in 1.10.0
|
|
4111
4199
|
warnings.warn(
|
|
4112
|
-
"The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.
|
|
4200
|
+
"The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
|
|
4113
4201
|
mlrun.utils.OverwriteBuildParamsWarning,
|
|
4114
4202
|
)
|
|
4115
4203
|
default_image_name = mlrun.mlconf.default_project_image_name.format(
|
|
@@ -4186,9 +4274,9 @@ class MlrunProject(ModelObj):
|
|
|
4186
4274
|
)
|
|
4187
4275
|
|
|
4188
4276
|
if not overwrite_build_params:
|
|
4189
|
-
# TODO: change overwrite_build_params default to True in 1.
|
|
4277
|
+
# TODO: change overwrite_build_params default to True in 1.10.0
|
|
4190
4278
|
warnings.warn(
|
|
4191
|
-
"The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.
|
|
4279
|
+
"The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
|
|
4192
4280
|
mlrun.utils.OverwriteBuildParamsWarning,
|
|
4193
4281
|
)
|
|
4194
4282
|
|
|
@@ -4474,8 +4562,8 @@ class MlrunProject(ModelObj):
|
|
|
4474
4562
|
|
|
4475
4563
|
def list_models(
|
|
4476
4564
|
self,
|
|
4477
|
-
name=None,
|
|
4478
|
-
tag=None,
|
|
4565
|
+
name: Optional[str] = None,
|
|
4566
|
+
tag: Optional[str] = None,
|
|
4479
4567
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4480
4568
|
since=None,
|
|
4481
4569
|
until=None,
|
|
@@ -4486,7 +4574,7 @@ class MlrunProject(ModelObj):
|
|
|
4486
4574
|
format_: Optional[
|
|
4487
4575
|
mlrun.common.formatters.ArtifactFormat
|
|
4488
4576
|
] = mlrun.common.formatters.ArtifactFormat.full,
|
|
4489
|
-
):
|
|
4577
|
+
) -> list[ModelArtifact]:
|
|
4490
4578
|
"""List models in project, filtered by various parameters.
|
|
4491
4579
|
|
|
4492
4580
|
Examples::
|
|
@@ -4516,7 +4604,7 @@ class MlrunProject(ModelObj):
|
|
|
4516
4604
|
artifacts generated from a hyper-param run. If only a single iteration exists, will return the artifact
|
|
4517
4605
|
from that iteration. If using ``best_iter``, the ``iter`` parameter must not be used.
|
|
4518
4606
|
:param tree: Return artifacts of the requested tree.
|
|
4519
|
-
:param limit: Maximum number of artifacts to return.
|
|
4607
|
+
:param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
|
|
4520
4608
|
:param format_: The format in which to return the artifacts. Default is 'full'.
|
|
4521
4609
|
"""
|
|
4522
4610
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
@@ -4595,6 +4683,155 @@ class MlrunProject(ModelObj):
|
|
|
4595
4683
|
**kwargs,
|
|
4596
4684
|
)
|
|
4597
4685
|
|
|
4686
|
+
def list_llm_prompts(
|
|
4687
|
+
self,
|
|
4688
|
+
name: Optional[str] = None,
|
|
4689
|
+
tag: Optional[str] = None,
|
|
4690
|
+
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4691
|
+
since: Optional[datetime.datetime] = None,
|
|
4692
|
+
until: Optional[datetime.datetime] = None,
|
|
4693
|
+
iter: Optional[int] = None,
|
|
4694
|
+
best_iteration: bool = False,
|
|
4695
|
+
tree: Optional[str] = None,
|
|
4696
|
+
model: Optional[Union[str, Artifact]] = None,
|
|
4697
|
+
format_: Optional[
|
|
4698
|
+
mlrun.common.formatters.ArtifactFormat
|
|
4699
|
+
] = mlrun.common.formatters.ArtifactFormat.full,
|
|
4700
|
+
partition_by: Optional[
|
|
4701
|
+
Union[mlrun.common.schemas.ArtifactPartitionByField, str]
|
|
4702
|
+
] = None,
|
|
4703
|
+
rows_per_partition: int = 1,
|
|
4704
|
+
partition_sort_by: Optional[
|
|
4705
|
+
Union[mlrun.common.schemas.SortField, str]
|
|
4706
|
+
] = mlrun.common.schemas.SortField.updated,
|
|
4707
|
+
partition_order: Union[
|
|
4708
|
+
mlrun.common.schemas.OrderType, str
|
|
4709
|
+
] = mlrun.common.schemas.OrderType.desc,
|
|
4710
|
+
) -> list[mlrun.artifacts.llm_prompt.LLMPromptArtifact]:
|
|
4711
|
+
"""List LLM prompt artifacts in the project with support for filtering.
|
|
4712
|
+
|
|
4713
|
+
This method returns a list of LLM prompt artifacts, filtered by parameters such as name, tag, labels,
|
|
4714
|
+
model association, iteration, and more. It can be used to retrieve the latest, best, or specific versions
|
|
4715
|
+
of prompts tied to a model or general project context.
|
|
4716
|
+
|
|
4717
|
+
Examples::
|
|
4718
|
+
|
|
4719
|
+
# Get all latest tagged prompts
|
|
4720
|
+
prompts = project.list_llm_prompts(tag="latest")
|
|
4721
|
+
|
|
4722
|
+
# Get prompts associated with a specific model
|
|
4723
|
+
prompts = project.list_llm_prompts(model=ModelArtifact("m1"))
|
|
4724
|
+
|
|
4725
|
+
# Get prompts filtered by label
|
|
4726
|
+
prompts = project.list_llm_prompts(labels={"use_case": "chatbot"})
|
|
4727
|
+
|
|
4728
|
+
# Get prompts using a name wildcard
|
|
4729
|
+
prompts = project.list_llm_prompts(name="~chat")
|
|
4730
|
+
|
|
4731
|
+
:param name: Name of the prompt artifact. Prefix with '~' for wildcard search (case-insensitive).
|
|
4732
|
+
:param tag: Filter artifacts by this tag (e.g., 'latest', 'prod').
|
|
4733
|
+
:param labels: Filter llm-prompt artifacts by label key-value pairs or key existence. This can be provided as:
|
|
4734
|
+
|
|
4735
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
4736
|
+
or `{"label": None}` to check for key existence.
|
|
4737
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
4738
|
+
or just `"label"` for key existence.
|
|
4739
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4740
|
+
the specified key-value pairs or key existence.
|
|
4741
|
+
|
|
4742
|
+
:param since: Return artifacts updated after this date (as datetime object).
|
|
4743
|
+
:param until: Return artifacts updated before this date (as datetime object).
|
|
4744
|
+
:param iter: Retrieve a specific iteration. Use `0` for root; `None` for all.
|
|
4745
|
+
:param best_iteration: Returns the llm-prompt artifact which belongs to the best iteration of a given run,
|
|
4746
|
+
in the case of artifacts generated from a hyper-param run. If only a single iteration exists, will return
|
|
4747
|
+
the artifact from that iteration. If using ``best_iter``, the ``iter`` parameter must not be used.
|
|
4748
|
+
:param tree: Filter by artifact tree ID (e.g., for lineage filtering).
|
|
4749
|
+
:param model: Return prompts associated with this model (can be `Artifact` URI or `Artifact` object).
|
|
4750
|
+
:param format_: The format in which to return the artifacts. Default is 'full'.
|
|
4751
|
+
:param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
|
|
4752
|
+
parameter must be provided as well.
|
|
4753
|
+
:param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
|
|
4754
|
+
to return per group. Default value is 1.
|
|
4755
|
+
:param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
|
|
4756
|
+
Currently the only allowed values are `created` and `updated`.
|
|
4757
|
+
:param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
|
|
4758
|
+
|
|
4759
|
+
:returns: A list of filtered `LLMPromptArtifact` objects matching the given parameters.
|
|
4760
|
+
"""
|
|
4761
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
4762
|
+
return db.list_artifacts(
|
|
4763
|
+
name=name,
|
|
4764
|
+
project=self.metadata.name,
|
|
4765
|
+
tag=tag,
|
|
4766
|
+
labels=labels,
|
|
4767
|
+
since=since,
|
|
4768
|
+
until=until,
|
|
4769
|
+
iter=iter,
|
|
4770
|
+
best_iteration=best_iteration,
|
|
4771
|
+
kind=mlrun.artifacts.llm_prompt.LLMPromptArtifact.kind,
|
|
4772
|
+
tree=tree,
|
|
4773
|
+
parent=model.uri if isinstance(model, Artifact) else model,
|
|
4774
|
+
format_=format_,
|
|
4775
|
+
partition_by=partition_by,
|
|
4776
|
+
rows_per_partition=rows_per_partition,
|
|
4777
|
+
partition_sort_by=partition_sort_by,
|
|
4778
|
+
partition_order=partition_order,
|
|
4779
|
+
).to_objects()
|
|
4780
|
+
|
|
4781
|
+
def paginated_list_llm_prompts(
|
|
4782
|
+
self,
|
|
4783
|
+
*args,
|
|
4784
|
+
page: Optional[int] = None,
|
|
4785
|
+
page_size: Optional[int] = None,
|
|
4786
|
+
page_token: Optional[str] = None,
|
|
4787
|
+
**kwargs,
|
|
4788
|
+
) -> tuple[mlrun.lists.ArtifactList, Optional[str]]:
|
|
4789
|
+
"""Retrieve a paginated list of LLM prompt artifacts for the current project.
|
|
4790
|
+
|
|
4791
|
+
This method returns a list of LLM prompt artifacts, supporting both token-based and page-number-based
|
|
4792
|
+
pagination. You can filter and navigate through the results using the optional `page`, `page_size`, and
|
|
4793
|
+
`page_token` parameters.
|
|
4794
|
+
|
|
4795
|
+
Examples::
|
|
4796
|
+
|
|
4797
|
+
# Fetch the first page with up to 5 prompt artifacts
|
|
4798
|
+
prompts, token = project.paginated_list_llm_prompts(page_size=5)
|
|
4799
|
+
|
|
4800
|
+
# Fetch the next page using the page token
|
|
4801
|
+
prompts, token = project.paginated_list_llm_prompts(page_token=token)
|
|
4802
|
+
|
|
4803
|
+
# Fetch a specific page (e.g., page 3)
|
|
4804
|
+
prompts, token = project.paginated_list_llm_prompts(page=3, page_size=5)
|
|
4805
|
+
|
|
4806
|
+
# Retrieve all prompt artifacts across pages
|
|
4807
|
+
all_prompts = []
|
|
4808
|
+
token = None
|
|
4809
|
+
while True:
|
|
4810
|
+
page_prompts, token = project.paginated_list_llm_prompts(
|
|
4811
|
+
page_token=token, page_size=5
|
|
4812
|
+
)
|
|
4813
|
+
all_prompts.extend(page_prompts)
|
|
4814
|
+
if not token:
|
|
4815
|
+
break
|
|
4816
|
+
print(f"Total retrieved prompts: {len(all_prompts)}")
|
|
4817
|
+
|
|
4818
|
+
:param page: Page number to retrieve (alternative to page_token).
|
|
4819
|
+
:param page_size: Number of items per page. Defaults to `mlrun.mlconf.httpdb.pagination.default_page_size`.
|
|
4820
|
+
:param page_token: Token for retrieving the next page of results (used for continuous iteration).
|
|
4821
|
+
|
|
4822
|
+
:returns: A tuple of (ArtifactList of LLM prompts, next page_token or None if no more pages).
|
|
4823
|
+
"""
|
|
4824
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
4825
|
+
return db.paginated_list_artifacts(
|
|
4826
|
+
*args,
|
|
4827
|
+
project=self.metadata.name,
|
|
4828
|
+
kind=mlrun.artifacts.llm_prompt.LLMPromptArtifact.kind,
|
|
4829
|
+
page=page,
|
|
4830
|
+
page_size=page_size,
|
|
4831
|
+
page_token=page_token,
|
|
4832
|
+
**kwargs,
|
|
4833
|
+
)
|
|
4834
|
+
|
|
4598
4835
|
def list_functions(
|
|
4599
4836
|
self,
|
|
4600
4837
|
name: Optional[str] = None,
|
|
@@ -4789,7 +5026,7 @@ class MlrunProject(ModelObj):
|
|
|
4789
5026
|
:param states: List only runs whose state is one of the provided states.
|
|
4790
5027
|
:param sort: Whether to sort the result according to their start time. Otherwise, results will be
|
|
4791
5028
|
returned by their internal order in the DB (order will not be guaranteed).
|
|
4792
|
-
:param last: Deprecated - currently not used (will be removed in 1.
|
|
5029
|
+
:param last: Deprecated - currently not used (will be removed in 1.10.0).
|
|
4793
5030
|
:param iter: If ``True`` return runs from all iterations. Otherwise, return only runs whose ``iter`` is 0.
|
|
4794
5031
|
:param start_time_from: Filter by run start time in ``[start_time_from, start_time_to]``.
|
|
4795
5032
|
:param start_time_to: Filter by run start time in ``[start_time_from, start_time_to]``.
|
|
@@ -4800,9 +5037,9 @@ class MlrunProject(ModelObj):
|
|
|
4800
5037
|
:param end_time_to: Filter by run end time in ``[end_time_from, end_time_to]``.
|
|
4801
5038
|
"""
|
|
4802
5039
|
if state:
|
|
4803
|
-
# TODO: Remove this in 1.
|
|
5040
|
+
# TODO: Remove this in 1.10.0
|
|
4804
5041
|
warnings.warn(
|
|
4805
|
-
"'state' is deprecated and will be removed in 1.
|
|
5042
|
+
"'state' is deprecated in 1.7.0 and will be removed in 1.10.0. Use 'states' instead.",
|
|
4806
5043
|
FutureWarning,
|
|
4807
5044
|
)
|
|
4808
5045
|
|
|
@@ -5460,8 +5697,8 @@ class MlrunProject(ModelObj):
|
|
|
5460
5697
|
return os.getenv("V3IO_USERNAME") or self.spec.owner
|
|
5461
5698
|
|
|
5462
5699
|
|
|
5463
|
-
def
|
|
5464
|
-
mlrun.mlconf.
|
|
5700
|
+
def _set_as_current_active_project(project: MlrunProject):
|
|
5701
|
+
mlrun.mlconf.active_project = project.metadata.name
|
|
5465
5702
|
pipeline_context.set(project)
|
|
5466
5703
|
|
|
5467
5704
|
|
mlrun/run.py
CHANGED
|
@@ -21,7 +21,6 @@ import tempfile
|
|
|
21
21
|
import time
|
|
22
22
|
import typing
|
|
23
23
|
import uuid
|
|
24
|
-
import warnings
|
|
25
24
|
from base64 import b64decode
|
|
26
25
|
from copy import deepcopy
|
|
27
26
|
from os import environ, makedirs, path
|
|
@@ -206,7 +205,6 @@ def get_or_create_ctx(
|
|
|
206
205
|
rundb: Union[str, "mlrun.db.RunDBInterface"] = "",
|
|
207
206
|
project: str = "",
|
|
208
207
|
upload_artifacts: bool = False,
|
|
209
|
-
labels: Optional[dict] = None,
|
|
210
208
|
) -> MLClientCtx:
|
|
211
209
|
"""
|
|
212
210
|
Called from within the user program to obtain a run context.
|
|
@@ -223,10 +221,9 @@ def get_or_create_ctx(
|
|
|
223
221
|
:param spec: dictionary holding run spec
|
|
224
222
|
:param with_env: look for context in environment vars, default True
|
|
225
223
|
:param rundb: path/url to the metadata and artifact database
|
|
226
|
-
:param project: project to initiate the context in (by default `mlrun.mlconf.
|
|
224
|
+
:param project: project to initiate the context in (by default `mlrun.mlconf.active_project`)
|
|
227
225
|
:param upload_artifacts: when using local context (not as part of a job/run), upload artifacts to the
|
|
228
226
|
system default artifact path location
|
|
229
|
-
:param labels: (deprecated - use spec instead) dict of the context labels.
|
|
230
227
|
:return: execution context
|
|
231
228
|
|
|
232
229
|
Examples::
|
|
@@ -259,21 +256,6 @@ def get_or_create_ctx(
|
|
|
259
256
|
context.log_artifact("results.html", body=b"<b> Some HTML <b>", viewer="web-app")
|
|
260
257
|
|
|
261
258
|
"""
|
|
262
|
-
if labels:
|
|
263
|
-
warnings.warn(
|
|
264
|
-
"The `labels` argument is deprecated and will be removed in 1.9.0. "
|
|
265
|
-
"Please use `spec` instead, e.g.:\n"
|
|
266
|
-
"spec={'metadata': {'labels': {'key': 'value'}}}",
|
|
267
|
-
FutureWarning,
|
|
268
|
-
)
|
|
269
|
-
if spec is None:
|
|
270
|
-
spec = {}
|
|
271
|
-
if "metadata" not in spec:
|
|
272
|
-
spec["metadata"] = {}
|
|
273
|
-
if "labels" not in spec["metadata"]:
|
|
274
|
-
spec["metadata"]["labels"] = {}
|
|
275
|
-
spec["metadata"]["labels"].update(labels)
|
|
276
|
-
|
|
277
259
|
if global_context.get() and not spec and not event:
|
|
278
260
|
return global_context.get()
|
|
279
261
|
|
|
@@ -298,7 +280,7 @@ def get_or_create_ctx(
|
|
|
298
280
|
newspec = {}
|
|
299
281
|
if upload_artifacts:
|
|
300
282
|
artifact_path = mlrun.utils.helpers.template_artifact_path(
|
|
301
|
-
mlconf.artifact_path, project or mlconf.
|
|
283
|
+
mlconf.artifact_path, project or mlconf.active_project
|
|
302
284
|
)
|
|
303
285
|
update_in(newspec, ["spec", RunKeys.output_path], artifact_path)
|
|
304
286
|
|
|
@@ -312,7 +294,7 @@ def get_or_create_ctx(
|
|
|
312
294
|
logger.info(f"Logging run results to: {out}")
|
|
313
295
|
|
|
314
296
|
newspec["metadata"]["project"] = (
|
|
315
|
-
newspec["metadata"].get("project") or project or mlconf.
|
|
297
|
+
newspec["metadata"].get("project") or project or mlconf.active_project
|
|
316
298
|
)
|
|
317
299
|
|
|
318
300
|
newspec["metadata"].setdefault("labels", {})
|
|
@@ -369,9 +351,9 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
|
|
|
369
351
|
url, is_hub_uri = extend_hub_uri_if_needed(url)
|
|
370
352
|
runtime = import_function_to_dict(url, secrets)
|
|
371
353
|
function = new_function(runtime=runtime)
|
|
372
|
-
project = project or mlrun.mlconf.
|
|
354
|
+
project = project or mlrun.mlconf.active_project
|
|
373
355
|
# When we're importing from the hub we want to assign to a target project, otherwise any store on it will
|
|
374
|
-
# simply default to the
|
|
356
|
+
# simply default to the active project
|
|
375
357
|
if project and is_hub_uri:
|
|
376
358
|
function.metadata.project = project
|
|
377
359
|
if new_name:
|
|
@@ -464,7 +446,7 @@ def new_function(
|
|
|
464
446
|
f = new_function().run(task, handler=myfunction)
|
|
465
447
|
|
|
466
448
|
:param name: function name
|
|
467
|
-
:param project: function project (none for
|
|
449
|
+
:param project: function project (none for the active project)
|
|
468
450
|
:param tag: function version tag (none for 'latest')
|
|
469
451
|
|
|
470
452
|
:param kind: runtime type (local, job, nuclio, spark, mpijob, dask, ..)
|
|
@@ -523,7 +505,7 @@ def new_function(
|
|
|
523
505
|
|
|
524
506
|
runner.metadata.name = name
|
|
525
507
|
runner.metadata.project = (
|
|
526
|
-
runner.metadata.project or project or mlconf.
|
|
508
|
+
runner.metadata.project or project or mlconf.active_project
|
|
527
509
|
)
|
|
528
510
|
if tag:
|
|
529
511
|
runner.metadata.tag = tag
|
|
@@ -640,7 +622,7 @@ def code_to_function(
|
|
|
640
622
|
Learn more about :doc:`../../concepts/functions-overview`
|
|
641
623
|
|
|
642
624
|
:param name: function name, typically best to use hyphen-case
|
|
643
|
-
:param project: project used to namespace the function, defaults to
|
|
625
|
+
:param project: project used to namespace the function, defaults to the active project
|
|
644
626
|
:param tag: function tag to track multiple versions of the same function, defaults to 'latest'
|
|
645
627
|
:param filename: path to .py/.ipynb file, defaults to current jupyter notebook
|
|
646
628
|
:param handler: The default function handler to call for the job or nuclio function, in batch functions
|
|
@@ -729,7 +711,7 @@ def code_to_function(
|
|
|
729
711
|
fn.spec.volume_mounts.append(vol.get("volumeMount"))
|
|
730
712
|
|
|
731
713
|
fn.spec.description = description
|
|
732
|
-
fn.metadata.project = project or mlconf.
|
|
714
|
+
fn.metadata.project = project or mlconf.active_project
|
|
733
715
|
fn.metadata.tag = tag
|
|
734
716
|
fn.metadata.categories = categories
|
|
735
717
|
fn.metadata.labels = labels or fn.metadata.labels
|
mlrun/runtimes/base.py
CHANGED
|
@@ -148,10 +148,10 @@ class FunctionSpec(ModelObj):
|
|
|
148
148
|
|
|
149
149
|
@property
|
|
150
150
|
def clone_target_dir(self):
|
|
151
|
-
# TODO: remove this property in 1.
|
|
151
|
+
# TODO: remove this property in 1.10.0
|
|
152
152
|
if self.build.source_code_target_dir:
|
|
153
153
|
warnings.warn(
|
|
154
|
-
"The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.
|
|
154
|
+
"The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
|
|
155
155
|
"Use spec.build.source_code_target_dir instead.",
|
|
156
156
|
FutureWarning,
|
|
157
157
|
)
|
|
@@ -159,10 +159,10 @@ class FunctionSpec(ModelObj):
|
|
|
159
159
|
|
|
160
160
|
@clone_target_dir.setter
|
|
161
161
|
def clone_target_dir(self, clone_target_dir):
|
|
162
|
-
# TODO: remove this property in 1.
|
|
162
|
+
# TODO: remove this property in 1.10.0
|
|
163
163
|
if clone_target_dir:
|
|
164
164
|
warnings.warn(
|
|
165
|
-
"The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.
|
|
165
|
+
"The clone_target_dir attribute is deprecated in 1.6.2 and will be removed in 1.10.0. "
|
|
166
166
|
"Use spec.build.source_code_target_dir instead.",
|
|
167
167
|
FutureWarning,
|
|
168
168
|
)
|
|
@@ -470,14 +470,14 @@ class BaseRuntime(ModelObj):
|
|
|
470
470
|
:return: Dictionary with all the variables that could be parsed
|
|
471
471
|
"""
|
|
472
472
|
runtime_env = {
|
|
473
|
-
"
|
|
473
|
+
"MLRUN_ACTIVE_PROJECT": self.metadata.project or config.active_project
|
|
474
474
|
}
|
|
475
475
|
if runobj:
|
|
476
476
|
runtime_env["MLRUN_EXEC_CONFIG"] = runobj.to_json(
|
|
477
477
|
exclude_notifications_params=True
|
|
478
478
|
)
|
|
479
479
|
if runobj.metadata.project:
|
|
480
|
-
runtime_env["
|
|
480
|
+
runtime_env["MLRUN_ACTIVE_PROJECT"] = runobj.metadata.project
|
|
481
481
|
if runobj.spec.verbose:
|
|
482
482
|
runtime_env["MLRUN_LOG_LEVEL"] = "DEBUG"
|
|
483
483
|
if config.httpdb.api_url:
|
mlrun/runtimes/kubejob.py
CHANGED
|
@@ -114,9 +114,9 @@ class KubejobRuntime(KubeResource):
|
|
|
114
114
|
e.g. builder_env={"GIT_TOKEN": token}
|
|
115
115
|
"""
|
|
116
116
|
if not overwrite:
|
|
117
|
-
# TODO: change overwrite default to True in 1.
|
|
117
|
+
# TODO: change overwrite default to True in 1.10.0
|
|
118
118
|
warnings.warn(
|
|
119
|
-
"The `overwrite` parameter default will change from 'False' to 'True' in 1.
|
|
119
|
+
"The `overwrite` parameter default will change from 'False' to 'True' in 1.10.0.",
|
|
120
120
|
mlrun.utils.OverwriteBuildParamsWarning,
|
|
121
121
|
)
|
|
122
122
|
image = mlrun.utils.helpers.remove_image_protocol_prefix(image)
|
|
@@ -623,9 +623,9 @@ class RemoteRuntime(KubeResource):
|
|
|
623
623
|
:param force_build: set True for force building the image
|
|
624
624
|
"""
|
|
625
625
|
if auth_info:
|
|
626
|
-
# TODO: remove in 1.
|
|
626
|
+
# TODO: remove in 1.10.0
|
|
627
627
|
warnings.warn(
|
|
628
|
-
"'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.
|
|
628
|
+
"'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.10.0",
|
|
629
629
|
FutureWarning,
|
|
630
630
|
)
|
|
631
631
|
|
|
@@ -833,7 +833,7 @@ class RemoteRuntime(KubeResource):
|
|
|
833
833
|
def _get_runtime_env(self):
|
|
834
834
|
# for runtime specific env var enrichment (before deploy)
|
|
835
835
|
runtime_env = {
|
|
836
|
-
"
|
|
836
|
+
"MLRUN_ACTIVE_PROJECT": self.metadata.project or mlconf.active_project,
|
|
837
837
|
}
|
|
838
838
|
if mlconf.httpdb.api_url:
|
|
839
839
|
runtime_env["MLRUN_DBPATH"] = mlconf.httpdb.api_url
|