mlrun 1.6.2rc3__py3-none-any.whl → 1.6.2rc6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (48) hide show
  1. mlrun/common/db/sql_session.py +0 -3
  2. mlrun/common/model_monitoring/helpers.py +2 -4
  3. mlrun/common/schemas/__init__.py +0 -1
  4. mlrun/common/schemas/project.py +0 -2
  5. mlrun/config.py +10 -29
  6. mlrun/datastore/azure_blob.py +9 -9
  7. mlrun/datastore/base.py +28 -0
  8. mlrun/datastore/google_cloud_storage.py +6 -6
  9. mlrun/datastore/v3io.py +46 -70
  10. mlrun/db/base.py +0 -18
  11. mlrun/db/httpdb.py +25 -28
  12. mlrun/execution.py +3 -3
  13. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +3 -3
  14. mlrun/frameworks/tf_keras/model_handler.py +7 -7
  15. mlrun/k8s_utils.py +5 -10
  16. mlrun/kfpops.py +10 -19
  17. mlrun/model.py +0 -5
  18. mlrun/model_monitoring/api.py +8 -8
  19. mlrun/model_monitoring/batch.py +1 -1
  20. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +13 -13
  21. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +1 -0
  22. mlrun/package/packagers/pandas_packagers.py +3 -3
  23. mlrun/package/utils/_archiver.py +1 -3
  24. mlrun/platforms/iguazio.py +65 -6
  25. mlrun/projects/pipelines.py +11 -21
  26. mlrun/projects/project.py +46 -65
  27. mlrun/runtimes/base.py +1 -24
  28. mlrun/runtimes/function.py +9 -9
  29. mlrun/runtimes/kubejob.py +3 -5
  30. mlrun/runtimes/local.py +2 -2
  31. mlrun/runtimes/mpijob/abstract.py +6 -6
  32. mlrun/runtimes/pod.py +3 -3
  33. mlrun/runtimes/serving.py +3 -3
  34. mlrun/runtimes/sparkjob/spark3job.py +3 -3
  35. mlrun/serving/remote.py +2 -4
  36. mlrun/utils/async_http.py +3 -3
  37. mlrun/utils/helpers.py +0 -8
  38. mlrun/utils/http.py +3 -3
  39. mlrun/utils/logger.py +2 -2
  40. mlrun/utils/notifications/notification_pusher.py +6 -6
  41. mlrun/utils/version/version.json +2 -2
  42. {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/METADATA +16 -14
  43. {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/RECORD +47 -48
  44. {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/WHEEL +1 -1
  45. mlrun/common/schemas/common.py +0 -40
  46. {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/LICENSE +0 -0
  47. {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/entry_points.txt +0 -0
  48. {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/top_level.txt +0 -0
mlrun/k8s_utils.py CHANGED
@@ -134,13 +134,13 @@ def sanitize_label_value(value: str) -> str:
134
134
  return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
135
135
 
136
136
 
137
- def verify_label_key(key: str):
138
- """
139
- Verify that the label key is valid for Kubernetes.
140
- Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
141
- """
137
+ def verify_label_key(key):
142
138
  if not key:
143
139
  raise mlrun.errors.MLRunInvalidArgumentError("label key cannot be empty")
140
+ if key.startswith("k8s.io") or key.startswith("kubernetes.io"):
141
+ raise mlrun.errors.MLRunInvalidArgumentError(
142
+ "Labels cannot start with 'k8s.io' or 'kubernetes.io'"
143
+ )
144
144
 
145
145
  mlrun.utils.helpers.verify_field_regex(
146
146
  f"project.metadata.labels.'{key}'",
@@ -148,11 +148,6 @@ def verify_label_key(key: str):
148
148
  mlrun.utils.regex.k8s_character_limit,
149
149
  )
150
150
 
151
- if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
152
- raise mlrun.errors.MLRunInvalidArgumentError(
153
- "Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
154
- )
155
-
156
151
  parts = key.split("/")
157
152
  if len(parts) == 1:
158
153
  name = parts[0]
mlrun/kfpops.py CHANGED
@@ -41,8 +41,8 @@ from .utils import (
41
41
 
42
42
  # default KFP artifacts and output (ui metadata, metrics etc.)
43
43
  # directories to /tmp to allow running with security context
44
- KFPMETA_DIR = "/tmp"
45
- KFP_ARTIFACTS_DIR = "/tmp"
44
+ KFPMETA_DIR = os.environ.get("KFPMETA_OUT_DIR", "/tmp")
45
+ KFP_ARTIFACTS_DIR = os.environ.get("KFP_ARTIFACTS_DIR", "/tmp")
46
46
 
47
47
  project_annotation = "mlrun/project"
48
48
  run_annotation = "mlrun/pipeline-step-type"
@@ -71,7 +71,7 @@ def write_kfpmeta(struct):
71
71
  {"name": k, "numberValue": v} for k, v in results.items() if is_num(v)
72
72
  ],
73
73
  }
74
- with open(os.path.join(KFPMETA_DIR, "mlpipeline-metrics.json"), "w") as f:
74
+ with open(KFPMETA_DIR + "/mlpipeline-metrics.json", "w") as f:
75
75
  json.dump(metrics, f)
76
76
 
77
77
  struct = deepcopy(struct)
@@ -91,14 +91,7 @@ def write_kfpmeta(struct):
91
91
  elif key in results:
92
92
  val = results[key]
93
93
  try:
94
- # NOTE: if key has "../x", it would fail on path traversal
95
- path = os.path.join(KFP_ARTIFACTS_DIR, key)
96
- if not mlrun.utils.helpers.is_safe_path(KFP_ARTIFACTS_DIR, path):
97
- logger.warning(
98
- "Path traversal is not allowed ignoring", path=path, key=key
99
- )
100
- continue
101
- path = os.path.abspath(path)
94
+ path = "/".join([KFP_ARTIFACTS_DIR, key])
102
95
  logger.info("Writing artifact output", path=path, val=val)
103
96
  with open(path, "w") as fp:
104
97
  fp.write(str(val))
@@ -116,7 +109,7 @@ def write_kfpmeta(struct):
116
109
  "outputs": output_artifacts
117
110
  + [{"type": "markdown", "storage": "inline", "source": text}]
118
111
  }
119
- with open(os.path.join(KFPMETA_DIR, "mlpipeline-ui-metadata.json"), "w") as f:
112
+ with open(KFPMETA_DIR + "/mlpipeline-ui-metadata.json", "w") as f:
120
113
  json.dump(metadata, f)
121
114
 
122
115
 
@@ -408,9 +401,9 @@ def mlrun_op(
408
401
  cmd += ["--label", f"{label}={val}"]
409
402
  for output in outputs:
410
403
  cmd += ["-o", str(output)]
411
- file_outputs[output.replace(".", "_")] = (
412
- f"/tmp/{output}" # not using path.join to avoid windows "\"
413
- )
404
+ file_outputs[
405
+ output.replace(".", "_")
406
+ ] = f"/tmp/{output}" # not using path.join to avoid windows "\"
414
407
  if project:
415
408
  cmd += ["--project", project]
416
409
  if handler:
@@ -457,10 +450,8 @@ def mlrun_op(
457
450
  command=cmd + [command],
458
451
  file_outputs=file_outputs,
459
452
  output_artifact_paths={
460
- "mlpipeline-ui-metadata": os.path.join(
461
- KFPMETA_DIR, "mlpipeline-ui-metadata.json"
462
- ),
463
- "mlpipeline-metrics": os.path.join(KFPMETA_DIR, "mlpipeline-metrics.json"),
453
+ "mlpipeline-ui-metadata": KFPMETA_DIR + "/mlpipeline-ui-metadata.json",
454
+ "mlpipeline-metrics": KFPMETA_DIR + "/mlpipeline-metrics.json",
464
455
  },
465
456
  )
466
457
  cop = add_default_function_resources(cop)
mlrun/model.py CHANGED
@@ -359,7 +359,6 @@ class ImageBuilder(ModelObj):
359
359
  requirements: list = None,
360
360
  extra_args=None,
361
361
  builder_env=None,
362
- source_code_target_dir=None,
363
362
  ):
364
363
  self.functionSourceCode = functionSourceCode #: functionSourceCode
365
364
  self.codeEntryType = "" #: codeEntryType
@@ -380,7 +379,6 @@ class ImageBuilder(ModelObj):
380
379
  self.auto_build = auto_build #: auto_build
381
380
  self.build_pod = None
382
381
  self.requirements = requirements or [] #: pip requirements
383
- self.source_code_target_dir = source_code_target_dir or None
384
382
 
385
383
  @property
386
384
  def source(self):
@@ -417,7 +415,6 @@ class ImageBuilder(ModelObj):
417
415
  overwrite=False,
418
416
  builder_env=None,
419
417
  extra_args=None,
420
- source_code_target_dir=None,
421
418
  ):
422
419
  if image:
423
420
  self.image = image
@@ -443,8 +440,6 @@ class ImageBuilder(ModelObj):
443
440
  self.builder_env = builder_env
444
441
  if extra_args:
445
442
  self.extra_args = extra_args
446
- if source_code_target_dir:
447
- self.source_code_target_dir = source_code_target_dir
448
443
 
449
444
  def with_commands(
450
445
  self,
@@ -436,9 +436,9 @@ def _generate_model_endpoint(
436
436
  ] = possible_drift_threshold
437
437
 
438
438
  model_endpoint.spec.monitoring_mode = monitoring_mode
439
- model_endpoint.status.first_request = model_endpoint.status.last_request = (
440
- datetime_now().isoformat()
441
- )
439
+ model_endpoint.status.first_request = (
440
+ model_endpoint.status.last_request
441
+ ) = datetime_now().isoformat()
442
442
  if sample_set_statistics:
443
443
  model_endpoint.status.feature_stats = sample_set_statistics
444
444
 
@@ -476,11 +476,11 @@ def trigger_drift_batch_job(
476
476
  db_session = mlrun.get_run_db()
477
477
 
478
478
  # Register the monitoring batch job (do nothing if already exist) and get the job function as a dictionary
479
- batch_function_dict: typing.Dict[str, typing.Any] = (
480
- db_session.deploy_monitoring_batch_job(
481
- project=project,
482
- default_batch_image=default_batch_image,
483
- )
479
+ batch_function_dict: typing.Dict[
480
+ str, typing.Any
481
+ ] = db_session.deploy_monitoring_batch_job(
482
+ project=project,
483
+ default_batch_image=default_batch_image,
484
484
  )
485
485
 
486
486
  # Prepare current run params
@@ -992,7 +992,7 @@ class BatchProcessor:
992
992
  """
993
993
  stream_http_path = (
994
994
  mlrun.mlconf.model_endpoint_monitoring.default_http_sink.format(
995
- project=self.project, namespace=mlrun.mlconf.namespace
995
+ project=self.project
996
996
  )
997
997
  )
998
998
 
@@ -540,24 +540,24 @@ class KVModelEndpointStore(ModelEndpointStore):
540
540
  and endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS]
541
541
  == "null"
542
542
  ):
543
- endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS] = (
544
- json.dumps(
545
- {
546
- mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
547
- mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
548
- mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
549
- }
543
+ endpoint[
544
+ mlrun.common.schemas.model_monitoring.EventFieldType.METRICS
545
+ ] = json.dumps(
546
+ {
547
+ mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
548
+ mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
549
+ mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
550
550
  }
551
- )
551
+ }
552
552
  )
553
553
  # Validate key `uid` instead of `endpoint_id`
554
554
  # For backwards compatibility reasons, we replace the `endpoint_id` with `uid` which is the updated key name
555
555
  if mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID in endpoint:
556
- endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.UID] = (
557
- endpoint[
558
- mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
559
- ]
560
- )
556
+ endpoint[
557
+ mlrun.common.schemas.model_monitoring.EventFieldType.UID
558
+ ] = endpoint[
559
+ mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
560
+ ]
561
561
 
562
562
  @staticmethod
563
563
  def _encode_field(field: typing.Union[str, bytes]) -> bytes:
@@ -31,6 +31,7 @@ from .models import get_model_endpoints_table
31
31
 
32
32
 
33
33
  class SQLModelEndpointStore(ModelEndpointStore):
34
+
34
35
  """
35
36
  Handles the DB operations when the DB target is from type SQL. For the SQL operations, we use SQLAlchemy, a Python
36
37
  SQL toolkit that handles the communication with the database. When using SQL for storing the model endpoints
@@ -838,9 +838,9 @@ class PandasDataFramePackager(DefaultPackager):
838
838
  """
839
839
  if isinstance(obj, dict):
840
840
  for key, value in obj.items():
841
- obj[PandasDataFramePackager._prepare_result(obj=key)] = (
842
- PandasDataFramePackager._prepare_result(obj=value)
843
- )
841
+ obj[
842
+ PandasDataFramePackager._prepare_result(obj=key)
843
+ ] = PandasDataFramePackager._prepare_result(obj=value)
844
844
  elif isinstance(obj, list):
845
845
  for i, value in enumerate(obj):
846
846
  obj[i] = PandasDataFramePackager._prepare_result(obj=value)
@@ -179,9 +179,7 @@ class _TarArchiver(_Archiver):
179
179
 
180
180
  # Extract:
181
181
  with tarfile.open(archive_path, f"r:{cls._MODE_STRING}") as tar_file:
182
- # use 'data' to ensure no security risks are imposed by the archive files
183
- # see: https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall
184
- tar_file.extractall(directory_path, filter="data")
182
+ tar_file.extractall(directory_path)
185
183
 
186
184
  return str(directory_path)
187
185
 
@@ -16,15 +16,19 @@ import json
16
16
  import os
17
17
  import urllib
18
18
  from collections import namedtuple
19
+ from datetime import datetime
20
+ from http import HTTPStatus
19
21
  from urllib.parse import urlparse
20
22
 
21
23
  import kfp.dsl
22
24
  import requests
23
25
  import semver
26
+ import urllib3
24
27
  import v3io
25
28
 
26
29
  import mlrun.errors
27
30
  from mlrun.config import config as mlconf
31
+ from mlrun.errors import err_to_str
28
32
  from mlrun.utils import dict_to_json
29
33
 
30
34
  _cached_control_session = None
@@ -484,6 +488,25 @@ class V3ioStreamClient:
484
488
  return response.output.records
485
489
 
486
490
 
491
+ def create_control_session(url, username, password):
492
+ # for systems without production cert - silence no cert verification WARN
493
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
494
+ if not username or not password:
495
+ raise ValueError("cannot create session key, missing username or password")
496
+
497
+ session = requests.Session()
498
+ session.auth = (username, password)
499
+ try:
500
+ auth = session.post(f"{url}/api/sessions", verify=False)
501
+ except OSError as exc:
502
+ raise OSError(f"error: cannot connect to {url}: {err_to_str(exc)}")
503
+
504
+ if not auth.ok:
505
+ raise OSError(f"failed to create session: {url}, {auth.text}")
506
+
507
+ return auth.json()["data"]["id"]
508
+
509
+
487
510
  def is_iguazio_endpoint(endpoint_url: str) -> bool:
488
511
  # TODO: find a better heuristic
489
512
  return ".default-tenant." in endpoint_url
@@ -510,6 +533,21 @@ def is_iguazio_session_cookie(session_cookie: str) -> bool:
510
533
  return False
511
534
 
512
535
 
536
+ def is_iguazio_system_2_10_or_above(dashboard_url):
537
+ # for systems without production cert - silence no cert verification WARN
538
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
539
+ response = requests.get(f"{dashboard_url}/api/external_versions", verify=False)
540
+
541
+ if not response.ok:
542
+ if response.status_code == HTTPStatus.NOT_FOUND.value:
543
+ # in iguazio systems prior to 2.10 this endpoint didn't exist, so the api returns 404 cause endpoint not
544
+ # found
545
+ return False
546
+ response.raise_for_status()
547
+
548
+ return True
549
+
550
+
513
551
  # we assign the control session or access key to the password since this is iguazio auth scheme
514
552
  # (requests should be sent with username:control_session/access_key as auth header)
515
553
  def add_or_refresh_credentials(
@@ -539,12 +577,33 @@ def add_or_refresh_credentials(
539
577
  # (ideally if we could identify we're in enterprise we would have verify here that token and username have value)
540
578
  if not is_iguazio_endpoint(api_url):
541
579
  return "", "", token
542
-
543
- if not username or not token:
544
- raise ValueError(
545
- "username and access key required to authenticate against iguazio system"
546
- )
547
- return username, token, ""
580
+ iguazio_dashboard_url = "https://dashboard" + api_url[api_url.find(".") :]
581
+
582
+ # in 2.8 mlrun api is protected with control session, from 2.10 it's protected with access key
583
+ is_access_key_auth = is_iguazio_system_2_10_or_above(iguazio_dashboard_url)
584
+ if is_access_key_auth:
585
+ if not username or not token:
586
+ raise ValueError(
587
+ "username and access key required to authenticate against iguazio system"
588
+ )
589
+ return username, token, ""
590
+
591
+ if not username or not password:
592
+ raise ValueError("username and password needed to create session")
593
+
594
+ global _cached_control_session
595
+ now = datetime.now()
596
+ if _cached_control_session:
597
+ if (
598
+ _cached_control_session[2] == username
599
+ and _cached_control_session[3] == password
600
+ and (now - _cached_control_session[1]).seconds < 20 * 60 * 60
601
+ ):
602
+ return _cached_control_session[2], _cached_control_session[0], ""
603
+
604
+ control_session = create_control_session(iguazio_dashboard_url, username, password)
605
+ _cached_control_session = (control_session, now, username, password)
606
+ return username, control_session, ""
548
607
 
549
608
 
550
609
  def parse_path(url, suffix="/"):
@@ -69,16 +69,16 @@ class WorkflowSpec(mlrun.model.ModelObj):
69
69
 
70
70
  def __init__(
71
71
  self,
72
- engine: typing.Optional[str] = None,
73
- code: typing.Optional[str] = None,
74
- path: typing.Optional[str] = None,
75
- args: typing.Optional[dict] = None,
76
- name: typing.Optional[str] = None,
77
- handler: typing.Optional[str] = None,
78
- args_schema: typing.Optional[dict] = None,
72
+ engine=None,
73
+ code=None,
74
+ path=None,
75
+ args=None,
76
+ name=None,
77
+ handler=None,
78
+ args_schema: dict = None,
79
79
  schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
80
- cleanup_ttl: typing.Optional[int] = None,
81
- image: typing.Optional[str] = None,
80
+ cleanup_ttl: int = None,
81
+ image: str = None,
82
82
  ):
83
83
  self.engine = engine
84
84
  self.code = code
@@ -401,9 +401,6 @@ def enrich_function_object(
401
401
  else:
402
402
  f.spec.build.source = project.spec.source
403
403
  f.spec.build.load_source_on_run = project.spec.load_source_on_run
404
- f.spec.build.source_code_target_dir = (
405
- project.spec.build.source_code_target_dir
406
- )
407
404
  f.spec.workdir = project.spec.workdir or project.spec.subpath
408
405
  f.prepare_image_for_deploy()
409
406
 
@@ -865,11 +862,6 @@ class _RemoteRunner(_PipelineRunner):
865
862
  )
866
863
  return
867
864
 
868
- logger.debug(
869
- "Workflow submitted, waiting for pipeline run to start",
870
- workflow_name=workflow_response.name,
871
- )
872
-
873
865
  # Getting workflow id from run:
874
866
  response = retry_until_successful(
875
867
  1,
@@ -996,7 +988,6 @@ def load_and_run(
996
988
  cleanup_ttl: int = None,
997
989
  load_only: bool = False,
998
990
  wait_for_completion: bool = False,
999
- project_context: str = None,
1000
991
  ):
1001
992
  """
1002
993
  Auxiliary function that the RemoteRunner run once or run every schedule.
@@ -1027,11 +1018,10 @@ def load_and_run(
1027
1018
  workflow and all its resources are deleted)
1028
1019
  :param load_only: for just loading the project, inner use.
1029
1020
  :param wait_for_completion: wait for workflow completion before returning
1030
- :param project_context: project context path (used for loading the project)
1031
1021
  """
1032
1022
  try:
1033
1023
  project = mlrun.load_project(
1034
- context=project_context or f"./{project_name}",
1024
+ context=f"./{project_name}",
1035
1025
  url=url,
1036
1026
  name=project_name,
1037
1027
  init_git=init_git,
@@ -1063,7 +1053,7 @@ def load_and_run(
1063
1053
 
1064
1054
  raise error
1065
1055
 
1066
- context.logger.info(f"Loaded project {project.name} successfully")
1056
+ context.logger.info(f"Loaded project {project.name} from remote successfully")
1067
1057
 
1068
1058
  if load_only:
1069
1059
  return
mlrun/projects/project.py CHANGED
@@ -24,7 +24,7 @@ import typing
24
24
  import uuid
25
25
  import warnings
26
26
  import zipfile
27
- from os import environ, makedirs, path
27
+ from os import environ, makedirs, path, remove
28
28
  from typing import Callable, Dict, List, Optional, Union
29
29
 
30
30
  import dotenv
@@ -605,14 +605,9 @@ def _load_project_dir(context, name="", subpath=""):
605
605
  # If there is a setup script do not force having project.yaml file
606
606
  project = MlrunProject()
607
607
  else:
608
- message = "Project or function YAML not found in path"
609
- logger.error(
610
- message,
611
- context=context,
612
- name=name,
613
- subpath=subpath,
608
+ raise mlrun.errors.MLRunNotFoundError(
609
+ "project or function YAML not found in path"
614
610
  )
615
- raise mlrun.errors.MLRunNotFoundError(message)
616
611
 
617
612
  project.spec.context = context
618
613
  project.metadata.name = name or project.metadata.name
@@ -1240,20 +1235,20 @@ class MlrunProject(ModelObj):
1240
1235
  self,
1241
1236
  name,
1242
1237
  workflow_path: str,
1243
- embed: bool = False,
1244
- engine: Optional[str] = None,
1245
- args_schema: list[EntrypointParam] = None,
1246
- handler: Optional[str] = None,
1238
+ embed=False,
1239
+ engine=None,
1240
+ args_schema: typing.List[EntrypointParam] = None,
1241
+ handler=None,
1247
1242
  schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
1248
- ttl: Optional[int] = None,
1249
- image: Optional[str] = None,
1243
+ ttl=None,
1244
+ image: str = None,
1250
1245
  **args,
1251
1246
  ):
1252
1247
  """Add or update a workflow, specify a name and the code path
1253
1248
 
1254
1249
  :param name: Name of the workflow
1255
1250
  :param workflow_path: URL (remote) / Path (absolute or relative to the project code path i.e.
1256
- <project.spec.get_code_path()>/<workflow_path>) for the workflow file.
1251
+ <project.spec.get_code_path()>/<workflow_path>) for the workflow file.
1257
1252
  :param embed: Add the workflow code into the project.yaml
1258
1253
  :param engine: Workflow processing engine ("kfp", "local", "remote" or "remote:local")
1259
1254
  :param args_schema: List of arg schema definitions (:py:class`~mlrun.model.EntrypointParam`)
@@ -2600,45 +2595,40 @@ class MlrunProject(ModelObj):
2600
2595
  cleanup_ttl: int = None,
2601
2596
  notifications: typing.List[mlrun.model.Notification] = None,
2602
2597
  ) -> _PipelineRunStatus:
2603
- """Run a workflow using kubeflow pipelines
2598
+ """run a workflow using kubeflow pipelines
2604
2599
 
2605
- :param name: Name of the workflow
2600
+ :param name: name of the workflow
2606
2601
  :param workflow_path:
2607
- URL to a workflow file, if not a project workflow
2602
+ url to a workflow file, if not a project workflow
2608
2603
  :param arguments:
2609
- Kubeflow pipelines arguments (parameters)
2604
+ kubeflow pipelines arguments (parameters)
2610
2605
  :param artifact_path:
2611
- Target path/url for workflow artifacts, the string
2606
+ target path/url for workflow artifacts, the string
2612
2607
  '{{workflow.uid}}' will be replaced by workflow id
2613
2608
  :param workflow_handler:
2614
- Workflow function handler (for running workflow function directly)
2615
- :param namespace: Kubernetes namespace if other than default
2616
- :param sync: Force functions sync before run
2617
- :param watch: Wait for pipeline completion
2618
- :param dirty: Allow running the workflow when the git repo is dirty
2619
- :param engine: Workflow engine running the workflow.
2620
- Supported values are 'kfp' (default), 'local' or 'remote'.
2621
- For setting engine for remote running use 'remote:local' or 'remote:kfp'.
2622
- :param local: Run local pipeline with local functions (set local=True in function.run())
2609
+ workflow function handler (for running workflow function directly)
2610
+ :param namespace: kubernetes namespace if other than default
2611
+ :param sync: force functions sync before run
2612
+ :param watch: wait for pipeline completion
2613
+ :param dirty: allow running the workflow when the git repo is dirty
2614
+ :param engine: workflow engine running the workflow.
2615
+ supported values are 'kfp' (default), 'local' or 'remote'.
2616
+ for setting engine for remote running use 'remote:local' or 'remote:kfp'.
2617
+ :param local: run local pipeline with local functions (set local=True in function.run())
2623
2618
  :param schedule: ScheduleCronTrigger class instance or a standard crontab expression string
2624
2619
  (which will be converted to the class using its `from_crontab` constructor),
2625
2620
  see this link for help:
2626
2621
  https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
2627
2622
  for using the pre-defined workflow's schedule, set `schedule=True`
2628
- :param timeout: Timeout in seconds to wait for pipeline completion (watch will be activated)
2629
- :param source: Source to use instead of the actual `project.spec.source` (used when engine is remote).
2630
- Can be a one of:
2631
- 1. Remote URL which is loaded dynamically to the workflow runner.
2632
- 2. A path to the project's context on the workflow runner's image.
2633
- Path can be absolute or relative to `project.spec.build.source_code_target_dir` if defined
2634
- (enriched when building a project image with source, see `MlrunProject.build_image`).
2635
- For other engines the source is used to validate that the code is up-to-date.
2623
+ :param timeout: timeout in seconds to wait for pipeline completion (watch will be activated)
2624
+ :param source: remote source to use instead of the actual `project.spec.source` (used when engine is remote).
2625
+ for other engines the source is to validate that the code is up-to-date
2636
2626
  :param cleanup_ttl:
2637
- Pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
2638
- Workflow and all its resources are deleted)
2627
+ pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
2628
+ workflow and all its resources are deleted)
2639
2629
  :param notifications:
2640
- List of notifications to send for workflow completion
2641
- :returns: Run id
2630
+ list of notifications to send for workflow completion
2631
+ :returns: run id
2642
2632
  """
2643
2633
 
2644
2634
  arguments = arguments or {}
@@ -2785,7 +2775,7 @@ class MlrunProject(ModelObj):
2785
2775
  def export(self, filepath=None, include_files: str = None):
2786
2776
  """save the project object into a yaml file or zip archive (default to project.yaml)
2787
2777
 
2788
- By default, the project object is exported to a yaml file, when the filepath suffix is '.zip'
2778
+ By default the project object is exported to a yaml file, when the filepath suffix is '.zip'
2789
2779
  the project context dir (code files) are also copied into the zip, the archive path can include
2790
2780
  DataItem urls (for remote object storage, e.g. s3://<bucket>/<path>).
2791
2781
 
@@ -2810,19 +2800,19 @@ class MlrunProject(ModelObj):
2810
2800
 
2811
2801
  if archive_code:
2812
2802
  files_filter = include_files or "**"
2813
- with tempfile.NamedTemporaryFile(suffix=".zip") as f:
2814
- remote_file = "://" in filepath
2815
- fpath = f.name if remote_file else filepath
2816
- with zipfile.ZipFile(fpath, "w") as zipf:
2817
- for file_path in glob.iglob(
2818
- f"{project_dir}/{files_filter}", recursive=True
2819
- ):
2820
- write_path = pathlib.Path(file_path)
2821
- zipf.write(
2822
- write_path, arcname=write_path.relative_to(project_dir)
2823
- )
2824
- if remote_file:
2825
- mlrun.get_dataitem(filepath).upload(zipf.filename)
2803
+ tmp_path = None
2804
+ if "://" in filepath:
2805
+ tmp_path = tempfile.mktemp(".zip")
2806
+ zipf = zipfile.ZipFile(tmp_path or filepath, "w")
2807
+ for file_path in glob.iglob(
2808
+ f"{project_dir}/{files_filter}", recursive=True
2809
+ ):
2810
+ write_path = pathlib.Path(file_path)
2811
+ zipf.write(write_path, arcname=write_path.relative_to(project_dir))
2812
+ zipf.close()
2813
+ if tmp_path:
2814
+ mlrun.get_dataitem(filepath).upload(tmp_path)
2815
+ remove(tmp_path)
2826
2816
 
2827
2817
  def set_model_monitoring_credentials(
2828
2818
  self,
@@ -3037,7 +3027,6 @@ class MlrunProject(ModelObj):
3037
3027
  requirements_file: str = None,
3038
3028
  builder_env: dict = None,
3039
3029
  extra_args: str = None,
3040
- source_code_target_dir: str = None,
3041
3030
  ):
3042
3031
  """specify builder configuration for the project
3043
3032
 
@@ -3058,8 +3047,6 @@ class MlrunProject(ModelObj):
3058
3047
  e.g. builder_env={"GIT_TOKEN": token}, does not work yet in KFP
3059
3048
  :param extra_args: A string containing additional builder arguments in the format of command-line options,
3060
3049
  e.g. extra_args="--skip-tls-verify --build-arg A=val"
3061
- :param source_code_target_dir: Path on the image where source code would be extracted
3062
- (by default `/home/mlrun_code`)
3063
3050
  """
3064
3051
  if not overwrite_build_params:
3065
3052
  # TODO: change overwrite_build_params default to True in 1.8.0
@@ -3083,7 +3070,6 @@ class MlrunProject(ModelObj):
3083
3070
  overwrite=overwrite_build_params,
3084
3071
  builder_env=builder_env,
3085
3072
  extra_args=extra_args,
3086
- source_code_target_dir=source_code_target_dir,
3087
3073
  )
3088
3074
 
3089
3075
  if set_as_default and image != self.default_image:
@@ -3130,7 +3116,7 @@ class MlrunProject(ModelObj):
3130
3116
  * False: The new params are merged with the existing
3131
3117
  * True: The existing params are replaced by the new ones
3132
3118
  :param extra_args: A string containing additional builder arguments in the format of command-line options,
3133
- e.g. extra_args="--skip-tls-verify --build-arg A=val"
3119
+ e.g. extra_args="--skip-tls-verify --build-arg A=val"r
3134
3120
  :param target_dir: Path on the image where source code would be extracted (by default `/home/mlrun_code`)
3135
3121
  """
3136
3122
  if not base_image:
@@ -3198,11 +3184,6 @@ class MlrunProject(ModelObj):
3198
3184
  force_build=True,
3199
3185
  )
3200
3186
 
3201
- # Get the enriched target dir from the function
3202
- self.spec.build.source_code_target_dir = (
3203
- function.spec.build.source_code_target_dir
3204
- )
3205
-
3206
3187
  try:
3207
3188
  mlrun.db.get_run_db(secrets=self._secrets).delete_function(
3208
3189
  name=function.metadata.name