mlrun 1.8.0rc5__py3-none-any.whl → 1.8.0rc6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (60) hide show
  1. mlrun/artifacts/__init__.py +1 -1
  2. mlrun/artifacts/base.py +12 -1
  3. mlrun/artifacts/document.py +59 -38
  4. mlrun/common/model_monitoring/__init__.py +0 -2
  5. mlrun/common/model_monitoring/helpers.py +0 -28
  6. mlrun/common/schemas/__init__.py +1 -4
  7. mlrun/common/schemas/client_spec.py +0 -1
  8. mlrun/common/schemas/model_monitoring/__init__.py +0 -6
  9. mlrun/common/schemas/model_monitoring/constants.py +11 -9
  10. mlrun/common/schemas/model_monitoring/model_endpoints.py +77 -149
  11. mlrun/common/schemas/notification.py +6 -0
  12. mlrun/config.py +0 -2
  13. mlrun/datastore/datastore_profile.py +57 -17
  14. mlrun/datastore/vectorstore.py +67 -59
  15. mlrun/db/base.py +22 -18
  16. mlrun/db/httpdb.py +116 -148
  17. mlrun/db/nopdb.py +33 -17
  18. mlrun/execution.py +11 -4
  19. mlrun/model.py +3 -0
  20. mlrun/model_monitoring/__init__.py +3 -2
  21. mlrun/model_monitoring/api.py +40 -43
  22. mlrun/model_monitoring/applications/_application_steps.py +3 -1
  23. mlrun/model_monitoring/applications/context.py +15 -17
  24. mlrun/model_monitoring/controller.py +43 -37
  25. mlrun/model_monitoring/db/__init__.py +0 -2
  26. mlrun/model_monitoring/db/tsdb/base.py +2 -1
  27. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +2 -1
  28. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +43 -0
  29. mlrun/model_monitoring/helpers.py +12 -66
  30. mlrun/model_monitoring/stream_processing.py +83 -270
  31. mlrun/model_monitoring/writer.py +1 -10
  32. mlrun/projects/project.py +63 -55
  33. mlrun/runtimes/nuclio/function.py +7 -6
  34. mlrun/runtimes/nuclio/serving.py +7 -1
  35. mlrun/serving/routers.py +158 -145
  36. mlrun/serving/server.py +6 -0
  37. mlrun/serving/states.py +2 -0
  38. mlrun/serving/v2_serving.py +69 -60
  39. mlrun/utils/helpers.py +14 -30
  40. mlrun/utils/notifications/notification/mail.py +17 -6
  41. mlrun/utils/version/version.json +2 -2
  42. {mlrun-1.8.0rc5.dist-info → mlrun-1.8.0rc6.dist-info}/METADATA +1 -1
  43. {mlrun-1.8.0rc5.dist-info → mlrun-1.8.0rc6.dist-info}/RECORD +47 -60
  44. mlrun/common/schemas/model_monitoring/model_endpoint_v2.py +0 -149
  45. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  46. mlrun/model_monitoring/db/stores/base/__init__.py +0 -15
  47. mlrun/model_monitoring/db/stores/base/store.py +0 -154
  48. mlrun/model_monitoring/db/stores/sqldb/__init__.py +0 -13
  49. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -46
  50. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -93
  51. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -47
  52. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -25
  53. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -408
  54. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +0 -13
  55. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -464
  56. mlrun/model_monitoring/model_endpoint.py +0 -120
  57. {mlrun-1.8.0rc5.dist-info → mlrun-1.8.0rc6.dist-info}/LICENSE +0 -0
  58. {mlrun-1.8.0rc5.dist-info → mlrun-1.8.0rc6.dist-info}/WHEEL +0 -0
  59. {mlrun-1.8.0rc5.dist-info → mlrun-1.8.0rc6.dist-info}/entry_points.txt +0 -0
  60. {mlrun-1.8.0rc5.dist-info → mlrun-1.8.0rc6.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py CHANGED
@@ -62,7 +62,6 @@ from mlrun.alerts.alert import AlertConfig
62
62
  from mlrun.datastore.datastore_profile import (
63
63
  DatastoreProfile,
64
64
  DatastoreProfile2Json,
65
- VectorStoreProfile,
66
65
  datastore_profile_read,
67
66
  )
68
67
  from mlrun.datastore.vectorstore import VectorStoreCollection
@@ -1535,7 +1534,9 @@ class MlrunProject(ModelObj):
1535
1534
 
1536
1535
  def update_artifact(self, artifact_object: Artifact):
1537
1536
  artifacts_manager = self._get_artifact_manager()
1538
- artifacts_manager.update_artifact(artifact_object, artifact_object)
1537
+ project_tag = self._get_project_tag()
1538
+ producer, _ = self._resolve_artifact_producer(artifact_object, project_tag)
1539
+ artifacts_manager.update_artifact(producer, artifact_object)
1539
1540
 
1540
1541
  def _get_artifact_manager(self):
1541
1542
  if self._artifact_manager:
@@ -1862,70 +1863,60 @@ class MlrunProject(ModelObj):
1862
1863
  )
1863
1864
  return item
1864
1865
 
1865
- def get_or_create_vector_store_collection(
1866
+ def get_vector_store_collection(
1866
1867
  self,
1867
1868
  collection_name: str,
1868
- profile: Union[str, VectorStoreProfile],
1869
- **kwargs,
1869
+ vector_store: "VectorStore", # noqa: F821
1870
1870
  ) -> VectorStoreCollection:
1871
- """
1872
- Create or retrieve a VectorStoreCollection.
1873
-
1874
- :param collection_name: Name of the collection
1875
- :param profile: Name of the VectorStoreProfile or a VectorStoreProfile object
1876
- :param kwargs: Additional arguments for the VectorStoreCollection
1877
- :return: VectorStoreCollection object
1878
- """
1879
- if isinstance(profile, str):
1880
- profile = datastore_profile_read(f"ds://{profile}")
1881
-
1882
- if not isinstance(profile, VectorStoreProfile):
1883
- raise ValueError(
1884
- "Profile must be a VectorStoreProfile object or a profile name"
1885
- )
1886
1871
  return VectorStoreCollection(
1887
- profile.vector_store_class,
1888
1872
  self,
1889
- profile.name,
1890
1873
  collection_name,
1891
- **profile.attributes(kwargs),
1874
+ vector_store,
1892
1875
  )
1893
1876
 
1894
1877
  def log_document(
1895
1878
  self,
1896
1879
  key: str,
1897
- artifact_path: Optional[str] = None,
1898
- document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
1899
1880
  tag: str = "",
1881
+ local_path: str = "",
1882
+ artifact_path: Optional[str] = None,
1883
+ document_loader_spec: Optional[DocumentLoaderSpec] = None,
1900
1884
  upload: Optional[bool] = False,
1901
1885
  labels: Optional[dict[str, str]] = None,
1886
+ target_path: Optional[str] = None,
1902
1887
  **kwargs,
1903
1888
  ) -> DocumentArtifact:
1904
1889
  """
1905
1890
  Log a document as an artifact.
1906
1891
 
1907
1892
  :param key: Artifact key
1908
- :param target_path: Path to the local file
1909
- :param artifact_path: Target path for artifact storage
1910
- :param document_loader: Spec to use to load the artifact as langchain document
1911
1893
  :param tag: Version tag
1894
+ :param local_path: path to the local file we upload, will also be use
1895
+ as the destination subpath (under "artifact_path")
1896
+ :param artifact_path: Target path for artifact storage
1897
+ :param document_loader_spec: Spec to use to load the artifact as langchain document
1912
1898
  :param upload: Whether to upload the artifact
1913
1899
  :param labels: Key-value labels
1900
+ :param target_path: Target file path
1914
1901
  :param kwargs: Additional keyword arguments
1915
1902
  :return: DocumentArtifact object
1916
1903
  """
1917
1904
  doc_artifact = DocumentArtifact(
1918
1905
  key=key,
1919
- document_loader=document_loader,
1906
+ original_source=local_path or target_path,
1907
+ document_loader_spec=document_loader_spec
1908
+ if document_loader_spec
1909
+ else DocumentLoaderSpec(),
1920
1910
  **kwargs,
1921
1911
  )
1922
-
1923
1912
  return self.log_artifact(
1924
- doc_artifact,
1925
- artifact_path=artifact_path,
1913
+ item=doc_artifact,
1926
1914
  tag=tag,
1915
+ local_path=local_path,
1916
+ artifact_path=artifact_path,
1927
1917
  upload=upload,
1928
1918
  labels=labels,
1919
+ target_path=target_path,
1929
1920
  )
1930
1921
 
1931
1922
  def import_artifact(
@@ -3413,7 +3404,6 @@ class MlrunProject(ModelObj):
3413
3404
  def set_model_monitoring_credentials(
3414
3405
  self,
3415
3406
  access_key: Optional[str] = None,
3416
- endpoint_store_connection: Optional[str] = None,
3417
3407
  stream_path: Optional[str] = None,
3418
3408
  tsdb_connection: Optional[str] = None,
3419
3409
  replace_creds: bool = False,
@@ -3424,7 +3414,6 @@ class MlrunProject(ModelObj):
3424
3414
  model monitoring or serving function.
3425
3415
 
3426
3416
  :param access_key: Model monitoring access key for managing user permissions.
3427
- :param endpoint_store_connection: Endpoint store connection string. By default, None. Options:
3428
3417
 
3429
3418
  * None - will be set from the system configuration.
3430
3419
  * v3io - for v3io endpoint store, pass `v3io` and the system will generate the
@@ -3457,7 +3446,6 @@ class MlrunProject(ModelObj):
3457
3446
  project=self.name,
3458
3447
  credentials={
3459
3448
  "access_key": access_key,
3460
- "endpoint_store_connection": endpoint_store_connection,
3461
3449
  "stream_path": stream_path,
3462
3450
  "tsdb_connection": tsdb_connection,
3463
3451
  },
@@ -3475,29 +3463,33 @@ class MlrunProject(ModelObj):
3475
3463
 
3476
3464
  def list_model_endpoints(
3477
3465
  self,
3478
- model: Optional[str] = None,
3479
- function: Optional[str] = None,
3466
+ name: Optional[str] = None,
3467
+ model_name: Optional[str] = None,
3468
+ function_name: Optional[str] = None,
3480
3469
  labels: Optional[list[str]] = None,
3481
- start: str = "now-1h",
3482
- end: str = "now",
3470
+ start: Optional[datetime.datetime] = None,
3471
+ end: Optional[datetime.datetime] = None,
3483
3472
  top_level: bool = False,
3484
3473
  uids: Optional[list[str]] = None,
3485
- ) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
3474
+ ) -> mlrun.common.schemas.ModelEndpointList:
3486
3475
  """
3487
3476
  Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
3488
3477
  model endpoint. This functions supports filtering by the following parameters:
3489
- 1) model
3490
- 2) function
3491
- 3) labels
3492
- 4) top level
3493
- 5) uids
3478
+ 1) name
3479
+ 2) model_name
3480
+ 3) function_name
3481
+ 4) labels
3482
+ 5) top level
3483
+ 6) uids
3484
+ 7) start and end time, corresponding to the `created` field.
3494
3485
  By default, when no filters are applied, all available endpoints for the given project will be listed.
3495
3486
 
3496
3487
  In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
3497
3488
  and depends on the 'start' and 'end' parameters.
3498
3489
 
3499
- :param model: The name of the model to filter by
3500
- :param function: The name of the function to filter by
3490
+ :param name: The name of the model to filter by
3491
+ :param model_name: The name of the model to filter by
3492
+ :param function_name: The name of the function to filter by
3501
3493
  :param labels: Filter model endpoints by label key-value pairs or key existence. This can be provided as:
3502
3494
  - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
3503
3495
  or `{"label": None}` to check for key existence.
@@ -3505,12 +3497,8 @@ class MlrunProject(ModelObj):
3505
3497
  or just `"label"` for key existence.
3506
3498
  - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
3507
3499
  the specified key-value pairs or key existence.
3508
- :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
3509
- Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
3510
- `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3511
- :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time, a
3512
- Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
3513
- `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
3500
+ :param start: The start time to filter by.Corresponding to the `created` field.
3501
+ :param end: The end time to filter by. Corresponding to the `created` field.
3514
3502
  :param top_level: if true will return only routers and endpoint that are NOT children of any router
3515
3503
  :param uids: if passed will return a list `ModelEndpoint` object with uid in uids
3516
3504
 
@@ -3519,8 +3507,9 @@ class MlrunProject(ModelObj):
3519
3507
  db = mlrun.db.get_run_db(secrets=self._secrets)
3520
3508
  return db.list_model_endpoints(
3521
3509
  project=self.name,
3522
- model=model,
3523
- function=function,
3510
+ name=name,
3511
+ model_name=model_name,
3512
+ function_name=function_name,
3524
3513
  labels=labels,
3525
3514
  start=start,
3526
3515
  end=end,
@@ -4502,6 +4491,25 @@ class MlrunProject(ModelObj):
4502
4491
  profile, self.name
4503
4492
  )
4504
4493
 
4494
+ def get_config_profile_attributes(self, name: str) -> dict:
4495
+ """
4496
+ Get the merged attributes from a named configuration profile.
4497
+
4498
+ Retrieves a profile from the datastore using the provided name and returns its
4499
+ merged public and private attributes as a dictionary.
4500
+
4501
+ Args:
4502
+ name (str): Name of the configuration profile to retrieve. Will be prefixed
4503
+ with "ds://" to form the full profile path.
4504
+
4505
+ Returns:
4506
+ dict: The merged attributes dictionary containing both public and private
4507
+ configuration settings from the profile. Returns nested dictionaries if
4508
+ the profile contains nested configurations.
4509
+ """
4510
+ profile = datastore_profile_read(f"ds://{name}", self.name)
4511
+ return profile.attributes()
4512
+
4505
4513
  def delete_datastore_profile(self, profile: str):
4506
4514
  mlrun.db.get_run_db(secrets=self._secrets).delete_datastore_profile(
4507
4515
  profile, self.name
@@ -1192,9 +1192,6 @@ class RemoteRuntime(KubeResource):
1192
1192
  return results
1193
1193
 
1194
1194
  def _resolve_invocation_url(self, path, force_external_address):
1195
- if not path.startswith("/") and path != "":
1196
- path = f"/{path}"
1197
-
1198
1195
  # internal / external invocation urls is a nuclio >= 1.6.x feature
1199
1196
  # try to infer the invocation url from the internal and if not exists, use external.
1200
1197
  # $$$$ we do not want to use the external invocation url (e.g.: ingress, nodePort, etc.)
@@ -1203,12 +1200,16 @@ class RemoteRuntime(KubeResource):
1203
1200
  and self.status.internal_invocation_urls
1204
1201
  and mlrun.k8s_utils.is_running_inside_kubernetes_cluster()
1205
1202
  ):
1206
- return f"http://{self.status.internal_invocation_urls[0]}{path}"
1203
+ return mlrun.utils.helpers.join_urls(
1204
+ f"http://{self.status.internal_invocation_urls[0]}", path
1205
+ )
1207
1206
 
1208
1207
  if self.status.external_invocation_urls:
1209
- return f"http://{self.status.external_invocation_urls[0]}{path}"
1208
+ return mlrun.utils.helpers.join_urls(
1209
+ f"http://{self.status.external_invocation_urls[0]}", path
1210
+ )
1210
1211
  else:
1211
- return f"http://{self.status.address}{path}"
1212
+ return mlrun.utils.helpers.join_urls(f"http://{self.status.address}", path)
1212
1213
 
1213
1214
  def _update_credentials_from_remote_build(self, remote_data):
1214
1215
  self.metadata.credentials = remote_data.get("metadata", {}).get(
@@ -644,9 +644,12 @@ class ServingRuntime(RemoteRuntime):
644
644
 
645
645
  def _get_serving_spec(self):
646
646
  function_name_uri_map = {f.name: f.uri(self) for f in self.spec.function_refs}
647
-
648
647
  serving_spec = {
648
+ "function_name": self.metadata.name,
649
+ "function_tag": self.metadata.tag,
649
650
  "function_uri": self._function_uri(),
651
+ "function_hash": self.metadata.hash,
652
+ "project": self.metadata.project,
650
653
  "version": "v2",
651
654
  "parameters": self.spec.parameters,
652
655
  "graph": self.spec.graph.to_dict() if self.spec.graph else {},
@@ -707,6 +710,9 @@ class ServingRuntime(RemoteRuntime):
707
710
  function_uri=self._function_uri(),
708
711
  secret_sources=self.spec.secret_sources,
709
712
  default_content_type=self.spec.default_content_type,
713
+ function_name=self.metadata.name,
714
+ function_tag=self.metadata.tag,
715
+ project=self.metadata.project,
710
716
  **kwargs,
711
717
  )
712
718
  server.init_states(