mlrun 1.5.0rc12__py3-none-any.whl → 1.5.0rc13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (45) hide show
  1. mlrun/__main__.py +31 -2
  2. mlrun/api/api/endpoints/functions.py +110 -52
  3. mlrun/api/crud/model_monitoring/deployment.py +208 -38
  4. mlrun/api/crud/model_monitoring/helpers.py +19 -6
  5. mlrun/api/crud/model_monitoring/model_endpoints.py +14 -1
  6. mlrun/api/db/sqldb/db.py +3 -1
  7. mlrun/api/utils/builder.py +2 -4
  8. mlrun/common/model_monitoring/helpers.py +19 -5
  9. mlrun/common/schemas/model_monitoring/constants.py +69 -0
  10. mlrun/common/schemas/model_monitoring/model_endpoints.py +10 -0
  11. mlrun/config.py +30 -12
  12. mlrun/datastore/__init__.py +1 -0
  13. mlrun/datastore/sources.py +4 -30
  14. mlrun/datastore/targets.py +68 -31
  15. mlrun/db/httpdb.py +20 -6
  16. mlrun/feature_store/api.py +3 -31
  17. mlrun/feature_store/feature_vector.py +1 -1
  18. mlrun/feature_store/retrieval/base.py +8 -3
  19. mlrun/launcher/remote.py +3 -3
  20. mlrun/lists.py +11 -0
  21. mlrun/model_monitoring/__init__.py +0 -1
  22. mlrun/model_monitoring/api.py +1 -1
  23. mlrun/model_monitoring/application.py +313 -0
  24. mlrun/model_monitoring/batch_application.py +526 -0
  25. mlrun/model_monitoring/batch_application_handler.py +32 -0
  26. mlrun/model_monitoring/evidently_application.py +89 -0
  27. mlrun/model_monitoring/helpers.py +39 -3
  28. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +37 -0
  29. mlrun/model_monitoring/tracking_policy.py +4 -4
  30. mlrun/model_monitoring/writer.py +37 -0
  31. mlrun/projects/pipelines.py +38 -4
  32. mlrun/projects/project.py +257 -43
  33. mlrun/run.py +5 -2
  34. mlrun/runtimes/__init__.py +2 -0
  35. mlrun/runtimes/function.py +2 -1
  36. mlrun/utils/helpers.py +12 -0
  37. mlrun/utils/http.py +3 -0
  38. mlrun/utils/version/version.json +2 -2
  39. {mlrun-1.5.0rc12.dist-info → mlrun-1.5.0rc13.dist-info}/METADATA +5 -5
  40. {mlrun-1.5.0rc12.dist-info → mlrun-1.5.0rc13.dist-info}/RECORD +45 -40
  41. /mlrun/model_monitoring/{model_monitoring_batch.py → batch.py} +0 -0
  42. {mlrun-1.5.0rc12.dist-info → mlrun-1.5.0rc13.dist-info}/LICENSE +0 -0
  43. {mlrun-1.5.0rc12.dist-info → mlrun-1.5.0rc13.dist-info}/WHEEL +0 -0
  44. {mlrun-1.5.0rc12.dist-info → mlrun-1.5.0rc13.dist-info}/entry_points.txt +0 -0
  45. {mlrun-1.5.0rc12.dist-info → mlrun-1.5.0rc13.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py CHANGED
@@ -38,7 +38,9 @@ import requests
38
38
  import yaml
39
39
  from deprecated import deprecated
40
40
 
41
+ import mlrun.common.helpers
41
42
  import mlrun.common.schemas.model_monitoring
43
+ import mlrun.common.schemas.model_monitoring.constants as mm_constants
42
44
  import mlrun.db
43
45
  import mlrun.errors
44
46
  import mlrun.runtimes
@@ -52,7 +54,12 @@ from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifa
52
54
  from ..datastore import store_manager
53
55
  from ..features import Feature
54
56
  from ..model import EntrypointParam, ImageBuilder, ModelObj
57
+ from ..model_monitoring.application import (
58
+ ModelMonitoringApplication,
59
+ PushToMonitoringWriter,
60
+ )
55
61
  from ..run import code_to_function, get_object, import_function, new_function
62
+ from ..runtimes.function import RemoteRuntime
56
63
  from ..secrets import SecretsStore
57
64
  from ..utils import (
58
65
  is_ipython,
@@ -1767,6 +1774,102 @@ class MlrunProject(ModelObj):
1767
1774
  )
1768
1775
  return _run_project_setup(self, setup_file_path, save)
1769
1776
 
1777
+ def set_model_monitoring_application(
1778
+ self,
1779
+ func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
1780
+ application_class: typing.Union[str, ModelMonitoringApplication] = None,
1781
+ name: str = None,
1782
+ image: str = None,
1783
+ handler=None,
1784
+ with_repo: bool = None,
1785
+ tag: str = None,
1786
+ requirements: typing.Union[str, typing.List[str]] = None,
1787
+ requirements_file: str = "",
1788
+ **application_kwargs,
1789
+ ) -> mlrun.runtimes.BaseRuntime:
1790
+ """
1791
+ update or add a monitoring application to the project & deploy it.
1792
+ examples::
1793
+ project.set_model_monitoring_application(application_class_name="MyApp",
1794
+ image="davesh0812/mlrun-api:1.5.0", name="myApp")
1795
+ :param func: function object or spec/code url, None refers to current Notebook
1796
+ :param name: name of the function (under the project), can be specified with a tag to support
1797
+ versions (e.g. myfunc:v1)
1798
+ default: job
1799
+ :param image: docker image to be used, can also be specified in
1800
+ the function object/yaml
1801
+ :param handler: default function handler to invoke (can only be set with .py/.ipynb files)
1802
+ :param with_repo: add (clone) the current repo to the build source
1803
+ :param tag: function version tag (none for 'latest', can only be set with .py/.ipynb files)
1804
+ if tag is specified and name is empty, the function key (under the project)
1805
+ will be enriched with the tag value. (i.e. 'function-name:tag')
1806
+ :param requirements: a list of python packages
1807
+ :param requirements_file: path to a python requirements file
1808
+ :param application_class: Name or an Instance of a class that implementing the monitoring application.
1809
+ :param application_kwargs: Additional keyword arguments to be passed to the
1810
+ monitoring application's constructor.
1811
+ """
1812
+
1813
+ function_object: RemoteRuntime = None
1814
+ kind = None
1815
+ if (isinstance(func, str) or func is None) and application_class is not None:
1816
+ kind = "serving"
1817
+ func = mlrun.code_to_function(
1818
+ name=name,
1819
+ project=self.metadata.name,
1820
+ tag=tag,
1821
+ kind=kind,
1822
+ image=image,
1823
+ requirements=requirements,
1824
+ requirements_file=requirements_file,
1825
+ )
1826
+ graph = func.set_topology("flow")
1827
+ if isinstance(application_class, str):
1828
+ first_step = graph.to(
1829
+ class_name=application_class, **application_kwargs
1830
+ )
1831
+ else:
1832
+ first_step = graph.to(class_name=application_class)
1833
+ first_step.to(
1834
+ class_name=PushToMonitoringWriter(
1835
+ project=self.metadata.name,
1836
+ writer_application_name=mm_constants.MonitoringFunctionNames.WRITER,
1837
+ stream_uri=None,
1838
+ ),
1839
+ ).respond()
1840
+ elif isinstance(func, str) and isinstance(handler, str):
1841
+ kind = "nuclio"
1842
+
1843
+ resolved_function_name, function_object, func = self._resolved_function(
1844
+ func,
1845
+ name,
1846
+ kind,
1847
+ image,
1848
+ handler,
1849
+ with_repo,
1850
+ tag,
1851
+ requirements,
1852
+ requirements_file,
1853
+ )
1854
+ models_names = "all"
1855
+ function_object.set_label(
1856
+ mm_constants.ModelMonitoringAppTag.KEY,
1857
+ mm_constants.ModelMonitoringAppTag.VAL,
1858
+ )
1859
+ function_object.set_label("models", models_names)
1860
+
1861
+ if not mlrun.mlconf.is_ce_mode():
1862
+ function_object.apply(mlrun.mount_v3io())
1863
+ # Deploy & Add stream triggers
1864
+ self.deploy_function(
1865
+ function_object,
1866
+ )
1867
+
1868
+ # save to project spec
1869
+ self.spec.set_function(resolved_function_name, function_object, func)
1870
+
1871
+ return function_object
1872
+
1770
1873
  def set_function(
1771
1874
  self,
1772
1875
  func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
@@ -1805,29 +1908,56 @@ class MlrunProject(ModelObj):
1805
1908
  # by providing a path to a pip requirements file
1806
1909
  proj.set_function('my.py', requirements="requirements.txt")
1807
1910
 
1808
- :param func: function object or spec/code url, None refers to current Notebook
1809
- :param name: name of the function (under the project), can be specified with a tag to support
1810
- versions (e.g. myfunc:v1)
1811
- :param kind: runtime kind e.g. job, nuclio, spark, dask, mpijob
1812
- default: job
1813
- :param image: docker image to be used, can also be specified in
1814
- the function object/yaml
1815
- :param handler: default function handler to invoke (can only be set with .py/.ipynb files)
1816
- :param with_repo: add (clone) the current repo to the build source
1817
- :param tag: function version tag (none for 'latest', can only be set with .py/.ipynb files)
1818
- if tag is specified and name is empty, the function key (under the project)
1819
- will be enriched with the tag value. (i.e. 'function-name:tag')
1820
- :param requirements: a list of python packages
1821
- :param requirements_file: path to a python requirements file
1911
+ :param func: Function object or spec/code url, None refers to current Notebook
1912
+ :param name: Name of the function (under the project), can be specified with a tag to support
1913
+ Versions (e.g. myfunc:v1). If the `tag` parameter is provided, the tag in the name
1914
+ must match the tag parameter.
1915
+ Specifying a tag in the name will update the project's tagged function (myfunc:v1)
1916
+ :param kind: Runtime kind e.g. job, nuclio, spark, dask, mpijob
1917
+ Default: job
1918
+ :param image: Docker image to be used, can also be specified in the function object/yaml
1919
+ :param handler: Default function handler to invoke (can only be set with .py/.ipynb files)
1920
+ :param with_repo: Add (clone) the current repo to the build source
1921
+ :param tag: Function version tag to set (none for current or 'latest')
1922
+ Specifying a tag as a parameter will update the project's tagged function
1923
+ (myfunc:v1) and the untagged function (myfunc)
1924
+ :param requirements: A list of python packages
1925
+ :param requirements_file: Path to a python requirements file
1822
1926
 
1823
- :returns: project object
1927
+ :returns: function object
1824
1928
  """
1929
+ resolved_function_name, function_object, func = self._resolved_function(
1930
+ func,
1931
+ name,
1932
+ kind,
1933
+ image,
1934
+ handler,
1935
+ with_repo,
1936
+ tag,
1937
+ requirements,
1938
+ requirements_file,
1939
+ )
1940
+ self.spec.set_function(resolved_function_name, function_object, func)
1941
+ return function_object
1942
+
1943
+ def _resolved_function(
1944
+ self,
1945
+ func: typing.Union[str, mlrun.runtimes.BaseRuntime] = None,
1946
+ name: str = "",
1947
+ kind: str = "",
1948
+ image: str = None,
1949
+ handler=None,
1950
+ with_repo: bool = None,
1951
+ tag: str = None,
1952
+ requirements: typing.Union[str, typing.List[str]] = None,
1953
+ requirements_file: str = "",
1954
+ ):
1825
1955
  if func is None and not _has_module(handler, kind):
1826
1956
  # if function path is not provided and it is not a module (no ".")
1827
1957
  # use the current notebook as default
1828
1958
  if not is_ipython:
1829
1959
  raise ValueError(
1830
- "function path or module must be specified (when not running inside a Notebook)"
1960
+ "Function path or module must be specified (when not running inside a Notebook)"
1831
1961
  )
1832
1962
  from IPython import get_ipython
1833
1963
 
@@ -1837,14 +1967,31 @@ class MlrunProject(ModelObj):
1837
1967
  func = path.relpath(func, self.spec.context)
1838
1968
 
1839
1969
  func = func or ""
1970
+
1840
1971
  name = mlrun.utils.normalize_name(name) if name else name
1972
+ untagged_name = name
1973
+ # validate tag in name if specified
1974
+ if len(split_name := name.split(":")) == 2:
1975
+ untagged_name, name_tag = split_name
1976
+ if tag and name_tag and tag != name_tag:
1977
+ raise ValueError(
1978
+ f"Tag parameter ({tag}) and tag in function name ({name}) must match"
1979
+ )
1980
+
1981
+ tag = tag or name_tag
1982
+ elif len(split_name) > 2:
1983
+ raise ValueError(
1984
+ f"Function name ({name}) must be in the format <name>:<tag> or <name>"
1985
+ )
1986
+
1841
1987
  if isinstance(func, str):
1988
+
1842
1989
  # in hub or db functions name defaults to the function name
1843
1990
  if not name and not (func.startswith("db://") or func.startswith("hub://")):
1844
- raise ValueError("function name must be specified")
1991
+ raise ValueError("Function name must be specified")
1845
1992
  function_dict = {
1846
1993
  "url": func,
1847
- "name": name,
1994
+ "name": untagged_name,
1848
1995
  "kind": kind,
1849
1996
  "image": image,
1850
1997
  "handler": handler,
@@ -1857,13 +2004,14 @@ class MlrunProject(ModelObj):
1857
2004
  func, self
1858
2005
  )
1859
2006
  func["name"] = resolved_function_name
2007
+
1860
2008
  elif hasattr(func, "to_dict"):
1861
2009
  resolved_function_name, function_object = _init_function_from_obj(
1862
- func, self, name=name
2010
+ func, self, name=untagged_name
1863
2011
  )
1864
2012
  if handler:
1865
2013
  raise ValueError(
1866
- "default handler cannot be set for existing function object"
2014
+ "Default handler cannot be set for existing function object"
1867
2015
  )
1868
2016
  if image:
1869
2017
  function_object.spec.image = image
@@ -1875,18 +2023,21 @@ class MlrunProject(ModelObj):
1875
2023
  requirements, requirements_file=requirements_file
1876
2024
  )
1877
2025
  if not resolved_function_name:
1878
- raise ValueError("function name must be specified")
2026
+ raise ValueError("Function name must be specified")
1879
2027
  else:
1880
- raise ValueError("func must be a function url or object")
2028
+ raise ValueError("'func' parameter must be a function url or object")
1881
2029
 
1882
- if tag and not resolved_function_name.endswith(f":{tag}"):
1883
- # Update the tagged key as well for consistency
1884
- self.spec.set_function(
1885
- f"{resolved_function_name}:{tag}", function_object, func
1886
- )
2030
+ function_object.metadata.tag = tag or function_object.metadata.tag or "latest"
2031
+ # resolved_function_name is the name without the tag or the actual function name if it was not specified
2032
+ # if the name contains the tag we only update the tagged entry
2033
+ # if the name doesn't contain the tag (or was not specified) we update both the tagged and untagged entries
2034
+ # for consistency
2035
+ name = name or resolved_function_name
2036
+ if tag and not name.endswith(f":{tag}"):
2037
+ self.spec.set_function(f"{name}:{tag}", function_object, func)
1887
2038
 
1888
- self.spec.set_function(resolved_function_name, function_object, func)
1889
- return function_object
2039
+ self.spec.set_function(name, function_object, func)
2040
+ return name, function_object, func
1890
2041
 
1891
2042
  def remove_function(self, name):
1892
2043
  """remove a function from a project
@@ -1895,6 +2046,24 @@ class MlrunProject(ModelObj):
1895
2046
  """
1896
2047
  self.spec.remove_function(name)
1897
2048
 
2049
+ def remove_model_monitoring_application(self, name):
2050
+ """remove a function from a project and from the db.
2051
+
2052
+ :param name: name of the function (under the project)
2053
+ """
2054
+ application = self.get_function(key=name)
2055
+ if (
2056
+ application.metadata.labels.get(mm_constants.ModelMonitoringAppTag.KEY)
2057
+ == mm_constants.ModelMonitoringAppTag.VAL
2058
+ ):
2059
+ self.remove_function(name=name)
2060
+ mlrun.db.get_run_db().delete_function(name=name.lower())
2061
+ logger.info(f"{name} application has been removed from {self.name} project")
2062
+ else:
2063
+ raise logger.error(
2064
+ f"There is no model monitoring application with {name} name"
2065
+ )
2066
+
1898
2067
  def get_function(
1899
2068
  self,
1900
2069
  key,
@@ -2208,12 +2377,16 @@ class MlrunProject(ModelObj):
2208
2377
  sync: bool = False,
2209
2378
  watch: bool = False,
2210
2379
  dirty: bool = False,
2380
+ # TODO: deprecated, remove in 1.6.0
2381
+ ttl: int = None,
2211
2382
  engine: str = None,
2212
2383
  local: bool = None,
2213
2384
  schedule: typing.Union[
2214
2385
  str, mlrun.common.schemas.ScheduleCronTrigger, bool
2215
2386
  ] = None,
2216
2387
  timeout: int = None,
2388
+ # TODO: deprecated, remove in 1.6.0
2389
+ overwrite: bool = False,
2217
2390
  source: str = None,
2218
2391
  cleanup_ttl: int = None,
2219
2392
  ) -> _PipelineRunStatus:
@@ -2233,6 +2406,8 @@ class MlrunProject(ModelObj):
2233
2406
  :param sync: force functions sync before run
2234
2407
  :param watch: wait for pipeline completion
2235
2408
  :param dirty: allow running the workflow when the git repo is dirty
2409
+ :param ttl: pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
2410
+ workflow and all its resources are deleted) (deprecated, use cleanup_ttl instead)
2236
2411
  :param engine: workflow engine running the workflow.
2237
2412
  supported values are 'kfp' (default), 'local' or 'remote'.
2238
2413
  for setting engine for remote running use 'remote:local' or 'remote:kfp'.
@@ -2243,6 +2418,8 @@ class MlrunProject(ModelObj):
2243
2418
  https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
2244
2419
  for using the pre-defined workflow's schedule, set `schedule=True`
2245
2420
  :param timeout: timeout in seconds to wait for pipeline completion (watch will be activated)
2421
+ :param overwrite: (deprecated) replacing the schedule of the same workflow (under the same name) if exists
2422
+ with the new one.
2246
2423
  :param source: remote source to use instead of the actual `project.spec.source` (used when engine is remote).
2247
2424
  for other engines the source is to validate that the code is up-to-date
2248
2425
  :param cleanup_ttl:
@@ -2251,10 +2428,26 @@ class MlrunProject(ModelObj):
2251
2428
  :returns: run id
2252
2429
  """
2253
2430
 
2431
+ if ttl:
2432
+ warnings.warn(
2433
+ "'ttl' is deprecated, use 'cleanup_ttl' instead. "
2434
+ "This will be removed in 1.6.0",
2435
+ # TODO: Remove this in 1.6.0
2436
+ FutureWarning,
2437
+ )
2438
+
2439
+ if overwrite:
2440
+ warnings.warn(
2441
+ "'overwrite' is deprecated, running a schedule is now an upsert operation. "
2442
+ "This will be removed in 1.6.0",
2443
+ # TODO: Remove this in 1.6.0
2444
+ FutureWarning,
2445
+ )
2446
+
2254
2447
  arguments = arguments or {}
2255
2448
  need_repo = self.spec._need_repo()
2256
2449
  if self.spec.repo and self.spec.repo.is_dirty():
2257
- msg = "you seem to have uncommitted git changes, use .push()"
2450
+ msg = "You seem to have uncommitted git changes, use .push()"
2258
2451
  if dirty or not need_repo:
2259
2452
  logger.warning("WARNING!, " + msg)
2260
2453
  else:
@@ -2262,7 +2455,7 @@ class MlrunProject(ModelObj):
2262
2455
 
2263
2456
  if need_repo and self.spec.repo and not self.spec.source:
2264
2457
  raise ProjectError(
2265
- "remote repo is not defined, use .create_remote() + push()"
2458
+ "Remote repo is not defined, use .create_remote() + push()"
2266
2459
  )
2267
2460
 
2268
2461
  self.sync_functions(always=sync)
@@ -2276,14 +2469,16 @@ class MlrunProject(ModelObj):
2276
2469
  if self.spec.workflows:
2277
2470
  name = list(self.spec._workflows.keys())[0]
2278
2471
  else:
2279
- raise ValueError("workflow name or path must be specified")
2472
+ raise ValueError("Workflow name or path must be specified")
2280
2473
 
2281
2474
  if workflow_path or (workflow_handler and callable(workflow_handler)):
2282
2475
  workflow_spec = WorkflowSpec(path=workflow_path, args=arguments)
2283
2476
  else:
2284
2477
  workflow_spec = self.spec._workflows[name].copy()
2285
2478
  workflow_spec.merge_args(arguments)
2286
- workflow_spec.cleanup_ttl = cleanup_ttl or workflow_spec.cleanup_ttl
2479
+ workflow_spec.cleanup_ttl = (
2480
+ cleanup_ttl or ttl or workflow_spec.cleanup_ttl or workflow_spec.ttl
2481
+ )
2287
2482
  workflow_spec.run_local = local
2288
2483
 
2289
2484
  name = f"{self.metadata.name}-{name}" if name else self.metadata.name
@@ -2324,7 +2519,7 @@ class MlrunProject(ModelObj):
2324
2519
  if not workflow_spec.schedule:
2325
2520
  # Failure and schedule messages already logged
2326
2521
  logger.info(
2327
- f"started run workflow {name} with run id = '{run.run_id}' by {workflow_engine.engine} engine"
2522
+ f"Started run workflow {name} with run id = '{run.run_id}' by {workflow_engine.engine} engine"
2328
2523
  )
2329
2524
  workflow_spec.clear_tmp()
2330
2525
  if (timeout or watch) and not workflow_spec.schedule:
@@ -2957,6 +3152,18 @@ class MlrunProject(ModelObj):
2957
3152
  # convert dict to function objects
2958
3153
  return [mlrun.new_function(runtime=func) for func in functions]
2959
3154
 
3155
+ def list_model_monitoring_applications(self):
3156
+ """Retrieve a list of alll the model monitoring application.
3157
+ example::
3158
+ functions = project.list_model_monitoring_applications()
3159
+ :returns: List of function objects.
3160
+ """
3161
+ return self.list_functions(
3162
+ labels=[
3163
+ f"{mm_constants.ModelMonitoringAppTag.KEY}={mm_constants.ModelMonitoringAppTag.VAL}"
3164
+ ]
3165
+ )
3166
+
2960
3167
  def list_runs(
2961
3168
  self,
2962
3169
  name: Optional[str] = None,
@@ -3113,31 +3320,32 @@ def _init_function_from_dict(
3113
3320
  has_module = _has_module(handler, kind)
3114
3321
  if not url and "spec" not in f and not has_module:
3115
3322
  # function must point to a file or a module or have a spec
3116
- raise ValueError("function missing a url or a spec or a module")
3323
+ raise ValueError("Function missing a url or a spec or a module")
3117
3324
 
3118
3325
  relative_url = url
3119
3326
  url, in_context = project.get_item_absolute_path(url)
3120
3327
 
3121
3328
  if "spec" in f:
3122
- func = new_function(name, runtime=f["spec"])
3329
+ func = new_function(name, runtime=f, tag=tag)
3330
+
3123
3331
  elif not url and has_module:
3124
3332
  func = new_function(
3125
3333
  name, image=image, kind=kind or "job", handler=handler, tag=tag
3126
3334
  )
3127
3335
 
3128
3336
  elif is_yaml_path(url) or url.startswith("db://") or url.startswith("hub://"):
3129
- if tag:
3130
- raise ValueError(
3131
- "function with db:// or hub:// url or .yaml file, does not support tag value "
3132
- )
3133
3337
  func = import_function(url, new_name=name)
3134
3338
  if image:
3135
3339
  func.spec.image = image
3340
+ if tag:
3341
+ func.spec.tag = tag
3342
+
3136
3343
  elif url.endswith(".ipynb"):
3137
3344
  # not defaulting kind to job here cause kind might come from magic annotations in the notebook
3138
3345
  func = code_to_function(
3139
3346
  name, filename=url, image=image, kind=kind, handler=handler, tag=tag
3140
3347
  )
3348
+
3141
3349
  elif url.endswith(".py"):
3142
3350
  if not image and not project.default_image and kind != "local":
3143
3351
  raise ValueError(
@@ -3162,8 +3370,9 @@ def _init_function_from_dict(
3162
3370
  handler=handler,
3163
3371
  tag=tag,
3164
3372
  )
3373
+
3165
3374
  else:
3166
- raise ValueError(f"unsupported function url:handler {url}:{handler} or no spec")
3375
+ raise ValueError(f"Unsupported function url:handler {url}:{handler} or no spec")
3167
3376
 
3168
3377
  if with_repo:
3169
3378
  # mark source to be enriched before run with project source (enrich_function_object)
@@ -3171,7 +3380,7 @@ def _init_function_from_dict(
3171
3380
  if requirements:
3172
3381
  func.with_requirements(requirements)
3173
3382
 
3174
- return _init_function_from_obj(func, project, name)
3383
+ return _init_function_from_obj(func, project)
3175
3384
 
3176
3385
 
3177
3386
  def _init_function_from_obj(
@@ -3190,9 +3399,14 @@ def _init_function_from_obj(
3190
3399
  build.code_origin = origin
3191
3400
  if project.metadata.name:
3192
3401
  func.metadata.project = project.metadata.name
3402
+
3403
+ # TODO: deprecate project tag
3193
3404
  if project.spec.tag:
3194
3405
  func.metadata.tag = project.spec.tag
3195
- return name or func.metadata.name, func
3406
+
3407
+ if name:
3408
+ func.metadata.name = name
3409
+ return func.metadata.name, func
3196
3410
 
3197
3411
 
3198
3412
  def _has_module(handler, kind):
mlrun/run.py CHANGED
@@ -348,6 +348,7 @@ def get_or_create_ctx(
348
348
  rundb: str = "",
349
349
  project: str = "",
350
350
  upload_artifacts=False,
351
+ labels: dict = None,
351
352
  ):
352
353
  """called from within the user program to obtain a run context
353
354
 
@@ -366,7 +367,7 @@ def get_or_create_ctx(
366
367
  :param project: project to initiate the context in (by default mlrun.mlctx.default_project)
367
368
  :param upload_artifacts: when using local context (not as part of a job/run), upload artifacts to the
368
369
  system default artifact path location
369
-
370
+ :param labels: dict of the context labels
370
371
  :return: execution context
371
372
 
372
373
  Examples::
@@ -442,6 +443,9 @@ def get_or_create_ctx(
442
443
  ctx = MLClientCtx.from_dict(
443
444
  newspec, rundb=out, autocommit=autocommit, tmp=tmp, host=socket.gethostname()
444
445
  )
446
+ labels = labels or {}
447
+ for key, val in labels.items():
448
+ ctx.set_label(key=key, value=val)
445
449
  global_context.set(ctx)
446
450
  return ctx
447
451
 
@@ -679,7 +683,6 @@ def _process_runtime(command, runtime, kind):
679
683
  if not runtime:
680
684
  runtime = {}
681
685
  update_in(runtime, "spec.command", command)
682
- runtime["kind"] = kind
683
686
  if kind != RuntimeKinds.remote:
684
687
  if command:
685
688
  update_in(runtime, "spec.command", command)
@@ -132,6 +132,8 @@ class RuntimeKinds(object):
132
132
  RuntimeKinds.mpijob,
133
133
  RuntimeKinds.databricks,
134
134
  RuntimeKinds.local,
135
+ RuntimeKinds.handler,
136
+ "",
135
137
  ]
136
138
 
137
139
  @staticmethod
@@ -558,8 +558,9 @@ class RemoteRuntime(KubeResource):
558
558
  self.metadata.tag = tag
559
559
 
560
560
  if dashboard:
561
+ # TODO: remove in 1.6.0
561
562
  warnings.warn(
562
- "'dashboard' parameter is no longer supported on client side, "
563
+ "'dashboard' parameter is no longer supported on client side, and will be removed in 1.6.0."
563
564
  "it is being configured through the MLRun API.",
564
565
  )
565
566
 
mlrun/utils/helpers.py CHANGED
@@ -812,6 +812,18 @@ def new_pipe_metadata(
812
812
  return conf
813
813
 
814
814
 
815
+ # TODO: remove in 1.6.0
816
+ @deprecated(
817
+ version="1.3.0",
818
+ reason="'new_pipe_meta' will be removed in 1.6.0",
819
+ category=FutureWarning,
820
+ )
821
+ def new_pipe_meta(artifact_path=None, ttl=None, *args):
822
+ return new_pipe_metadata(
823
+ artifact_path=artifact_path, cleanup_ttl=ttl, op_transformers=args
824
+ )
825
+
826
+
815
827
  def _convert_python_package_version_to_image_tag(version: typing.Optional[str]):
816
828
  return (
817
829
  version.replace("+", "-").replace("0.0.0-", "") if version is not None else None
mlrun/utils/http.py CHANGED
@@ -58,6 +58,9 @@ class HTTPSessionWithRetry(requests.Session):
58
58
  requests.exceptions.ConnectTimeout,
59
59
  requests.exceptions.ReadTimeout,
60
60
  urllib3.exceptions.ReadTimeoutError,
61
+ # may occur when connection breaks during the request.
62
+ requests.exceptions.ChunkedEncodingError,
63
+ urllib3.exceptions.InvalidChunkLength,
61
64
  ]
62
65
 
63
66
  def __init__(
@@ -1,4 +1,4 @@
1
1
  {
2
- "git_commit": "9d380ba79abf98974cecd2b9619b426010ad8c73",
3
- "version": "1.5.0-rc12"
2
+ "git_commit": "ec9859b42dcd9bac01f726f06b0c0d482572946a",
3
+ "version": "1.5.0-rc13"
4
4
  }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mlrun
3
- Version: 1.5.0rc12
3
+ Version: 1.5.0rc13
4
4
  Summary: Tracking and config of machine learning runs
5
5
  Home-page: https://github.com/mlrun/mlrun
6
6
  Author: Yaron Haviv
@@ -64,7 +64,7 @@ Requires-Dist: avro ~=1.11 ; extra == 'all'
64
64
  Requires-Dist: azure-core ~=1.24 ; extra == 'all'
65
65
  Requires-Dist: azure-identity ~=1.5 ; extra == 'all'
66
66
  Requires-Dist: azure-keyvault-secrets ~=4.2 ; extra == 'all'
67
- Requires-Dist: azure-storage-blob ~=12.13 ; extra == 'all'
67
+ Requires-Dist: azure-storage-blob <12.18,>=12.13 ; extra == 'all'
68
68
  Requires-Dist: bokeh >=2.4.2,~=2.4 ; extra == 'all'
69
69
  Requires-Dist: boto3 <1.27,>=1.24.59 ; extra == 'all'
70
70
  Requires-Dist: databricks-sdk ~=0.3.0 ; extra == 'all'
@@ -95,7 +95,7 @@ Requires-Dist: alembic ~=1.9 ; extra == 'api'
95
95
  Provides-Extra: azure-blob-storage
96
96
  Requires-Dist: msrest ~=0.6.21 ; extra == 'azure-blob-storage'
97
97
  Requires-Dist: azure-core ~=1.24 ; extra == 'azure-blob-storage'
98
- Requires-Dist: azure-storage-blob ~=12.13 ; extra == 'azure-blob-storage'
98
+ Requires-Dist: azure-storage-blob <12.18,>=12.13 ; extra == 'azure-blob-storage'
99
99
  Requires-Dist: adlfs <2023.5,>=2022.2 ; extra == 'azure-blob-storage'
100
100
  Requires-Dist: pyopenssl >=23 ; extra == 'azure-blob-storage'
101
101
  Provides-Extra: azure-key-vault
@@ -111,7 +111,7 @@ Requires-Dist: avro ~=1.11 ; extra == 'complete'
111
111
  Requires-Dist: azure-core ~=1.24 ; extra == 'complete'
112
112
  Requires-Dist: azure-identity ~=1.5 ; extra == 'complete'
113
113
  Requires-Dist: azure-keyvault-secrets ~=4.2 ; extra == 'complete'
114
- Requires-Dist: azure-storage-blob ~=12.13 ; extra == 'complete'
114
+ Requires-Dist: azure-storage-blob <12.18,>=12.13 ; extra == 'complete'
115
115
  Requires-Dist: boto3 <1.27,>=1.24.59 ; extra == 'complete'
116
116
  Requires-Dist: databricks-sdk ~=0.3.0 ; extra == 'complete'
117
117
  Requires-Dist: gcsfs <2023.7,>=2023.1 ; extra == 'complete'
@@ -133,7 +133,7 @@ Requires-Dist: avro ~=1.11 ; extra == 'complete-api'
133
133
  Requires-Dist: azure-core ~=1.24 ; extra == 'complete-api'
134
134
  Requires-Dist: azure-identity ~=1.5 ; extra == 'complete-api'
135
135
  Requires-Dist: azure-keyvault-secrets ~=4.2 ; extra == 'complete-api'
136
- Requires-Dist: azure-storage-blob ~=12.13 ; extra == 'complete-api'
136
+ Requires-Dist: azure-storage-blob <12.18,>=12.13 ; extra == 'complete-api'
137
137
  Requires-Dist: boto3 <1.27,>=1.24.59 ; extra == 'complete-api'
138
138
  Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'complete-api'
139
139
  Requires-Dist: databricks-sdk ~=0.3.0 ; extra == 'complete-api'