polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. polyaxon/_auxiliaries/cleaner.py +8 -3
  2. polyaxon/_auxiliaries/init.py +7 -2
  3. polyaxon/_auxiliaries/notifier.py +8 -2
  4. polyaxon/_auxiliaries/sidecar.py +30 -2
  5. polyaxon/_cli/artifacts.py +96 -11
  6. polyaxon/_cli/components.py +96 -11
  7. polyaxon/_cli/config.py +118 -22
  8. polyaxon/_cli/dashboard.py +15 -2
  9. polyaxon/_cli/init.py +1 -1
  10. polyaxon/_cli/models.py +96 -11
  11. polyaxon/_cli/operations.py +267 -90
  12. polyaxon/_cli/project_versions.py +139 -6
  13. polyaxon/_cli/projects.py +23 -9
  14. polyaxon/_cli/run.py +37 -9
  15. polyaxon/_cli/services/agent.py +2 -2
  16. polyaxon/_cli/services/clean_artifacts.py +1 -1
  17. polyaxon/_cli/services/sidecar.py +8 -1
  18. polyaxon/_client/client.py +17 -0
  19. polyaxon/_client/mixin.py +39 -0
  20. polyaxon/_client/project.py +218 -23
  21. polyaxon/_client/run.py +131 -33
  22. polyaxon/_compiler/contexts/contexts.py +2 -2
  23. polyaxon/_compiler/contexts/ray_job.py +4 -2
  24. polyaxon/_compiler/resolver/agent.py +12 -2
  25. polyaxon/_compiler/resolver/runtime.py +2 -2
  26. polyaxon/_contexts/paths.py +4 -7
  27. polyaxon/_deploy/operators/compose.py +1 -27
  28. polyaxon/_deploy/schemas/deployment.py +4 -1
  29. polyaxon/_deploy/schemas/intervals.py +0 -7
  30. polyaxon/_deploy/schemas/proxy.py +1 -0
  31. polyaxon/_deploy/schemas/service.py +11 -1
  32. polyaxon/_docker/converter/base/base.py +8 -0
  33. polyaxon/_docker/executor.py +10 -4
  34. polyaxon/_env_vars/getters/owner_entity.py +4 -2
  35. polyaxon/_env_vars/getters/project.py +4 -2
  36. polyaxon/_env_vars/getters/run.py +5 -2
  37. polyaxon/_env_vars/keys.py +7 -1
  38. polyaxon/_flow/__init__.py +2 -0
  39. polyaxon/_flow/builds/__init__.py +19 -6
  40. polyaxon/_flow/component/base.py +1 -0
  41. polyaxon/_flow/component/component.py +14 -0
  42. polyaxon/_flow/environment/__init__.py +8 -8
  43. polyaxon/_flow/hooks/__init__.py +19 -6
  44. polyaxon/_flow/init/__init__.py +6 -6
  45. polyaxon/_flow/matrix/iterative.py +0 -1
  46. polyaxon/_flow/matrix/tuner.py +18 -6
  47. polyaxon/_flow/operations/operation.py +44 -17
  48. polyaxon/_flow/plugins/__init__.py +6 -0
  49. polyaxon/_flow/run/__init__.py +2 -2
  50. polyaxon/_flow/run/dag.py +2 -2
  51. polyaxon/_flow/run/dask/dask.py +0 -1
  52. polyaxon/_flow/run/dask/replica.py +3 -3
  53. polyaxon/_flow/run/enums.py +5 -0
  54. polyaxon/_flow/run/job.py +4 -4
  55. polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
  56. polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
  57. polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
  58. polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
  59. polyaxon/_flow/run/kubeflow/replica.py +4 -4
  60. polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
  61. polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
  62. polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
  63. polyaxon/_flow/run/ray/ray.py +2 -3
  64. polyaxon/_flow/run/ray/replica.py +3 -3
  65. polyaxon/_flow/run/service.py +4 -4
  66. polyaxon/_fs/fs.py +7 -2
  67. polyaxon/_fs/utils.py +3 -2
  68. polyaxon/_k8s/converter/base/base.py +2 -1
  69. polyaxon/_k8s/converter/base/main.py +1 -0
  70. polyaxon/_k8s/converter/base/sidecar.py +16 -1
  71. polyaxon/_k8s/converter/common/accelerators.py +7 -4
  72. polyaxon/_k8s/converter/converters/job.py +1 -1
  73. polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
  74. polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
  75. polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
  76. polyaxon/_k8s/converter/converters/ray_job.py +4 -2
  77. polyaxon/_k8s/custom_resources/dask_job.py +3 -0
  78. polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
  79. polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
  80. polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
  81. polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
  82. polyaxon/_k8s/custom_resources/ray_job.py +3 -0
  83. polyaxon/_k8s/custom_resources/setter.py +1 -1
  84. polyaxon/_k8s/executor/async_executor.py +2 -0
  85. polyaxon/_k8s/executor/base.py +23 -6
  86. polyaxon/_k8s/logging/async_monitor.py +150 -5
  87. polyaxon/_k8s/manager/async_manager.py +96 -23
  88. polyaxon/_k8s/manager/base.py +4 -0
  89. polyaxon/_k8s/manager/manager.py +282 -134
  90. polyaxon/_local_process/__init__.py +0 -0
  91. polyaxon/_local_process/agent.py +6 -0
  92. polyaxon/_local_process/converter/__init__.py +1 -0
  93. polyaxon/_local_process/converter/base/__init__.py +1 -0
  94. polyaxon/_local_process/converter/base/base.py +140 -0
  95. polyaxon/_local_process/converter/base/containers.py +69 -0
  96. polyaxon/_local_process/converter/base/env_vars.py +253 -0
  97. polyaxon/_local_process/converter/base/init.py +414 -0
  98. polyaxon/_local_process/converter/base/main.py +74 -0
  99. polyaxon/_local_process/converter/base/mounts.py +82 -0
  100. polyaxon/_local_process/converter/converters/__init__.py +8 -0
  101. polyaxon/_local_process/converter/converters/job.py +40 -0
  102. polyaxon/_local_process/converter/converters/service.py +41 -0
  103. polyaxon/_local_process/converter/mixins.py +38 -0
  104. polyaxon/_local_process/executor.py +132 -0
  105. polyaxon/_local_process/process_types.py +39 -0
  106. polyaxon/_managers/agent.py +2 -0
  107. polyaxon/_managers/home.py +2 -1
  108. polyaxon/_operations/tuner.py +1 -0
  109. polyaxon/_polyaxonfile/check.py +2 -0
  110. polyaxon/_polyaxonfile/manager/operations.py +3 -0
  111. polyaxon/_polyaxonfile/manager/workflows.py +2 -0
  112. polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
  113. polyaxon/_polyaxonfile/specs/operation.py +1 -0
  114. polyaxon/_polyaxonfile/specs/sections.py +3 -0
  115. polyaxon/_pql/manager.py +1 -1
  116. polyaxon/_runner/agent/async_agent.py +97 -21
  117. polyaxon/_runner/agent/base_agent.py +27 -9
  118. polyaxon/_runner/agent/client.py +15 -1
  119. polyaxon/_runner/agent/sync_agent.py +85 -20
  120. polyaxon/_runner/converter/converter.py +6 -2
  121. polyaxon/_runner/executor.py +13 -7
  122. polyaxon/_schemas/agent.py +27 -1
  123. polyaxon/_schemas/client.py +30 -3
  124. polyaxon/_schemas/installation.py +4 -3
  125. polyaxon/_schemas/lifecycle.py +10 -5
  126. polyaxon/_schemas/log_handler.py +2 -3
  127. polyaxon/_schemas/types/artifacts.py +3 -3
  128. polyaxon/_schemas/types/dockerfile.py +3 -3
  129. polyaxon/_schemas/types/file.py +3 -3
  130. polyaxon/_schemas/types/git.py +3 -3
  131. polyaxon/_schemas/types/tensorboard.py +3 -3
  132. polyaxon/_sdk/api/agents_v1_api.py +1076 -73
  133. polyaxon/_sdk/api/organizations_v1_api.py +371 -10
  134. polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
  135. polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
  136. polyaxon/_sdk/api/projects_v1_api.py +221 -44
  137. polyaxon/_sdk/api/runs_v1_api.py +917 -445
  138. polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
  139. polyaxon/_sdk/api/teams_v1_api.py +2827 -375
  140. polyaxon/_sdk/api/users_v1_api.py +231 -55
  141. polyaxon/_sdk/async_client/api_client.py +4 -0
  142. polyaxon/_sdk/schemas/__init__.py +10 -2
  143. polyaxon/_sdk/schemas/v1_agent.py +2 -1
  144. polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
  145. polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
  146. polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
  147. polyaxon/_sdk/schemas/v1_events_response.py +4 -0
  148. polyaxon/_sdk/schemas/v1_organization.py +1 -0
  149. polyaxon/_sdk/schemas/v1_preset.py +8 -0
  150. polyaxon/_sdk/schemas/v1_project.py +1 -0
  151. polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
  152. polyaxon/_sdk/schemas/v1_run.py +2 -2
  153. polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
  154. polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
  155. polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
  156. polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
  157. polyaxon/_sdk/schemas/v1_team.py +3 -0
  158. polyaxon/_sdk/schemas/v1_user.py +1 -2
  159. polyaxon/_sdk/schemas/v1_user_access.py +17 -0
  160. polyaxon/_services/values.py +1 -0
  161. polyaxon/_sidecar/container/__init__.py +39 -18
  162. polyaxon/_sidecar/container/monitors/__init__.py +1 -0
  163. polyaxon/_sidecar/container/monitors/logs.py +10 -13
  164. polyaxon/_sidecar/container/monitors/spec.py +24 -0
  165. polyaxon/_sidecar/ignore.py +0 -1
  166. polyaxon/_utils/fqn_utils.py +25 -2
  167. polyaxon/client.py +1 -1
  168. polyaxon/pkg.py +1 -1
  169. polyaxon/schemas.py +8 -1
  170. polyaxon/settings.py +6 -0
  171. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
  172. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
  173. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
  174. polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
  175. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
  176. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
  177. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
polyaxon/_client/run.py CHANGED
@@ -21,6 +21,7 @@ from clipped.utils.paths import (
21
21
  get_base_filename,
22
22
  get_dirs_under_path,
23
23
  get_files_in_path_context,
24
+ set_permissions,
24
25
  )
25
26
  from clipped.utils.query_params import (
26
27
  get_logs_params,
@@ -35,6 +36,7 @@ from polyaxon import settings
35
36
  from polyaxon._cli.errors import handle_cli_error
36
37
  from polyaxon._client.client import PolyaxonClient
37
38
  from polyaxon._client.decorators import client_handler, get_global_or_inline_config
39
+ from polyaxon._client.mixin import ClientMixin
38
40
  from polyaxon._client.store import PolyaxonStore
39
41
  from polyaxon._constants.metadata import META_COPY_ARTIFACTS, META_RECOMPILE
40
42
  from polyaxon._containers.names import MAIN_CONTAINER_NAMES
@@ -58,11 +60,17 @@ from polyaxon._schemas.lifecycle import (
58
60
  V1Statuses,
59
61
  )
60
62
  from polyaxon._schemas.types import V1ArtifactsType
63
+ from polyaxon._sdk.schemas import V1RunEdgeLineage, V1RunEdgesGraph
61
64
  from polyaxon._sdk.schemas.v1_operation_body import V1OperationBody
62
65
  from polyaxon._sdk.schemas.v1_project_version import V1ProjectVersion
63
66
  from polyaxon._sdk.schemas.v1_run import V1Run
64
67
  from polyaxon._sdk.schemas.v1_run_settings import V1RunSettings
65
- from polyaxon._utils.fqn_utils import get_entity_full_name, to_fqn_name
68
+ from polyaxon._sdk.schemas.v1_settings_catalog import V1SettingsCatalog
69
+ from polyaxon._utils.fqn_utils import (
70
+ get_entity_full_name,
71
+ split_owner_team_space,
72
+ to_fqn_name,
73
+ )
66
74
  from polyaxon._utils.urls_utils import get_proxy_run_url
67
75
  from polyaxon.api import K8S_V1_LOCATION, STREAMS_V1_LOCATION
68
76
  from polyaxon.exceptions import ApiException, PolyaxonClientException
@@ -79,7 +87,7 @@ if TYPE_CHECKING:
79
87
  from traceml.tracking.run import Run
80
88
 
81
89
 
82
- class RunClient:
90
+ class RunClient(ClientMixin):
83
91
  """RunClient is a client to communicate with Polyaxon runs endpoints.
84
92
 
85
93
  If no values are passed to this class,
@@ -145,7 +153,7 @@ class RunClient:
145
153
  return
146
154
 
147
155
  try:
148
- owner, project = get_project_or_local(
156
+ owner, _, project = get_project_or_local(
149
157
  get_entity_full_name(owner=owner, entity=project)
150
158
  )
151
159
  except PolyaxonClientException:
@@ -165,8 +173,10 @@ class RunClient:
165
173
  if error_message and not self._is_offline:
166
174
  raise PolyaxonClientException(error_message)
167
175
 
176
+ owner, team = split_owner_team_space(owner)
168
177
  self._client = client
169
178
  self._owner = owner
179
+ self._team = team
170
180
  self._project = project
171
181
  self._run_uuid = (
172
182
  get_run_or_local(run_uuid)
@@ -217,12 +227,23 @@ class RunClient:
217
227
  return client.config.no_op
218
228
  return settings.CLIENT_CONFIG.no_op
219
229
 
220
- @property
221
- def client(self):
222
- if self._client:
223
- return self._client
224
- self._client = PolyaxonClient()
225
- return self._client
230
+ def _reset_agent(self, agent: Optional[str] = None):
231
+ if agent:
232
+ agent = self.client.agents_v1.get_agent(self.owner, agent)
233
+ self.settings.agent = V1SettingsCatalog(
234
+ uuid=agent.uuid,
235
+ name=agent.name,
236
+ version=agent.version,
237
+ url=agent.hostname,
238
+ )
239
+ self.settings.namespace = agent.namespace
240
+ self.settings.artifacts_store = None
241
+
242
+ def _use_agent_host(self):
243
+ if self.settings.agent and self.settings.agent.url:
244
+ self.reset_client(
245
+ host=self.settings.agent.url, POLYAXON_HOST=self.settings.agent.url
246
+ )
226
247
 
227
248
  @property
228
249
  def store(self):
@@ -255,20 +276,6 @@ class RunClient:
255
276
  return self.settings.artifacts_store.name
256
277
  return None
257
278
 
258
- @property
259
- def owner(self) -> str:
260
- return self._owner or ""
261
-
262
- def set_owner(self, owner: str):
263
- self._owner = owner
264
-
265
- @property
266
- def project(self) -> str:
267
- return self._project or ""
268
-
269
- def set_project(self, project: str):
270
- self._project = project
271
-
272
279
  @property
273
280
  def run_uuid(self) -> str:
274
281
  return self._run_uuid
@@ -322,7 +329,7 @@ class RunClient:
322
329
  lineages = self.get_artifacts_lineage(limit=1000).results
323
330
  self._artifacts_lineage = {l.name: l for l in lineages}
324
331
  if load_metrics:
325
- self.get_metrics(self.metric_names, force)
332
+ self.get_metrics(self._metric_names, force=True)
326
333
 
327
334
  def _throttle_updates(self) -> bool:
328
335
  current_time = now().replace(microsecond=0)
@@ -500,6 +507,7 @@ class RunClient:
500
507
  matrix: Optional[Union[Dict, V1Matrix]] = None,
501
508
  presets: Optional[List[str]] = None,
502
509
  queue: Optional[str] = None,
510
+ namespace: Optional[str] = None,
503
511
  nocache: Optional[bool] = None,
504
512
  cache: Optional[Union[int, str, bool]] = None,
505
513
  approved: Optional[Union[int, str, bool]] = None,
@@ -529,6 +537,8 @@ class RunClient:
529
537
  [presets](/docs/core/scheduling-presets/).
530
538
  queue: str, optional, the name of the
531
539
  [queue](/docs/core/scheduling-strategies/queues/) to assign the run to.
540
+ namespace: str, optional, the namespace to create the run under, by default it will
541
+ use the agent's namespace.
532
542
  nocache: bool, optional, DEPRECATED Please use `cache='f'`
533
543
  simple flag to disable
534
544
  [cache check](/docs/automation/helpers/cache/).
@@ -554,6 +564,7 @@ class RunClient:
554
564
  matrix=matrix,
555
565
  presets=presets,
556
566
  queue=queue,
567
+ namespace=namespace,
557
568
  nocache=nocache,
558
569
  cache=cache,
559
570
  approved=approved,
@@ -575,6 +586,7 @@ class RunClient:
575
586
  matrix: Optional[Union[Dict, V1Matrix]] = None,
576
587
  presets: Optional[List[str]] = None,
577
588
  queue: Optional[str] = None,
589
+ namespace: Optional[str] = None,
578
590
  nocache: Optional[bool] = None,
579
591
  cache: Optional[Union[int, str, bool]] = None,
580
592
  approved: Optional[Union[int, str, bool]] = None,
@@ -603,6 +615,8 @@ class RunClient:
603
615
  [presets](/docs/core/scheduling-presets/).
604
616
  queue: str, optional, the name of the
605
617
  [queue](/docs/core/scheduling-strategies/queues/) to assign the run to.
618
+ namespace: str, optional, the namespace to create the run under, by default it will
619
+ use the agent's namespace.
606
620
  nocache: bool, optional, DEPRECATED Please use `cache='f'`
607
621
  simple flag to disable
608
622
  [cache check](/docs/automation/helpers/cache/).
@@ -628,6 +642,7 @@ class RunClient:
628
642
  matrix=matrix,
629
643
  presets=presets,
630
644
  queue=queue,
645
+ namespace=namespace,
631
646
  nocache=nocache,
632
647
  cache=cache,
633
648
  approved=approved,
@@ -649,6 +664,7 @@ class RunClient:
649
664
  matrix: Optional[Union[Dict, V1Matrix]] = None,
650
665
  presets: Optional[List[str]] = None,
651
666
  queue: Optional[str] = None,
667
+ namespace: Optional[str] = None,
652
668
  nocache: Optional[bool] = None,
653
669
  cache: Optional[Union[int, str, bool]] = None,
654
670
  approved: Optional[Union[int, str, bool]] = None,
@@ -676,6 +692,8 @@ class RunClient:
676
692
  [presets](/docs/core/scheduling-presets/).
677
693
  queue: str, optional, the name of the
678
694
  [queue](/docs/core/scheduling-strategies/queues/) to assign the run to.
695
+ namespace: str, optional, the namespace to create the run under, by default it will
696
+ use the agent's namespace.
679
697
  nocache: bool, optional, DEPRECATED Please use `cache='f'`
680
698
  simple flag to disable
681
699
  [cache check](/docs/automation/helpers/cache/).
@@ -701,6 +719,7 @@ class RunClient:
701
719
  matrix=matrix,
702
720
  presets=presets,
703
721
  queue=queue,
722
+ namespace=namespace,
704
723
  nocache=nocache,
705
724
  cache=cache,
706
725
  approved=approved,
@@ -885,6 +904,7 @@ class RunClient:
885
904
  """
886
905
  if not self.settings:
887
906
  self.refresh_data()
907
+ self._use_agent_host()
888
908
  params = get_logs_params(
889
909
  last_file=last_file, last_time=last_time, connection=self.artifacts_store
890
910
  )
@@ -908,6 +928,7 @@ class RunClient:
908
928
  def inspect(self):
909
929
  if not self.settings:
910
930
  self.refresh_data()
931
+ self._use_agent_host()
911
932
  params = get_streams_params(connection=self.artifacts_store, status=self.status)
912
933
  return self.client.runs_v1.inspect_run(
913
934
  self.namespace, self.owner, self.project, self.run_uuid, **params
@@ -981,6 +1002,10 @@ class RunClient:
981
1002
  if not container:
982
1003
  container = pod_containers[0]
983
1004
 
1005
+ if not self.settings:
1006
+ self.refresh_data()
1007
+ self._use_agent_host()
1008
+
984
1009
  url = get_proxy_run_url(
985
1010
  service=K8S_V1_LOCATION,
986
1011
  namespace=self.namespace,
@@ -1026,6 +1051,8 @@ class RunClient:
1026
1051
  """
1027
1052
  if not self.settings:
1028
1053
  self.refresh_data()
1054
+ self._use_agent_host()
1055
+
1029
1056
  params = get_streams_params(self.artifacts_store)
1030
1057
  return self.client.runs_v1.get_run_events(
1031
1058
  self.namespace,
@@ -1061,14 +1088,16 @@ class RunClient:
1061
1088
  """
1062
1089
  if not self.settings:
1063
1090
  self.refresh_data()
1091
+ self._use_agent_host()
1092
+
1064
1093
  params = get_streams_params(self.artifacts_store)
1065
1094
  return self.client.runs_v1.get_multi_run_events(
1066
1095
  self.namespace,
1067
1096
  self.owner,
1068
1097
  self.project,
1069
1098
  kind=kind,
1070
- names=names,
1071
- runs=runs,
1099
+ names=",".join(names),
1100
+ runs=",".join(runs),
1072
1101
  orient=orient,
1073
1102
  force=force,
1074
1103
  **params,
@@ -1297,6 +1326,7 @@ class RunClient:
1297
1326
  """
1298
1327
  if not self.settings:
1299
1328
  self.refresh_data()
1329
+ self._use_agent_host()
1300
1330
  params = get_streams_params(self.artifacts_store)
1301
1331
  return self.client.runs_v1.get_run_artifact(
1302
1332
  namespace=self.namespace,
@@ -1330,6 +1360,10 @@ class RunClient:
1330
1360
  if not self.run_uuid:
1331
1361
  return
1332
1362
 
1363
+ if not self.settings:
1364
+ self.refresh_data()
1365
+ self._use_agent_host()
1366
+
1333
1367
  lineage_path = lineage.path or ""
1334
1368
  summary = lineage.summary or {}
1335
1369
  is_event = summary.get("is_event")
@@ -1408,6 +1442,8 @@ class RunClient:
1408
1442
  """
1409
1443
  if not self.settings:
1410
1444
  self.refresh_data()
1445
+ self._use_agent_host()
1446
+
1411
1447
  url = get_proxy_run_url(
1412
1448
  service=STREAMS_V1_LOCATION,
1413
1449
  namespace=self.namespace,
@@ -1447,6 +1483,8 @@ class RunClient:
1447
1483
  """
1448
1484
  if not self.settings:
1449
1485
  self.refresh_data()
1486
+ self._use_agent_host()
1487
+
1450
1488
  url = get_proxy_run_url(
1451
1489
  service=STREAMS_V1_LOCATION,
1452
1490
  namespace=self.namespace,
@@ -1478,6 +1516,7 @@ class RunClient:
1478
1516
  untar: bool = False,
1479
1517
  overwrite: bool = True,
1480
1518
  show_progress: bool = True,
1519
+ agent: Optional[str] = None,
1481
1520
  ):
1482
1521
  """Uploads a single artifact to the run's artifacts store path.
1483
1522
 
@@ -1488,12 +1527,17 @@ class RunClient:
1488
1527
  it should be decompressed on the artifacts store.
1489
1528
  overwrite: bool, optional, if the file uploaded should overwrite any previous content.
1490
1529
  show_progress: bool, to show a progress bar.
1530
+ agent: str, optional, uuid reference of an agent to use.
1491
1531
 
1492
1532
  Returns:
1493
1533
  str
1494
1534
  """
1495
1535
  if not self.settings:
1496
1536
  self.refresh_data()
1537
+ if agent:
1538
+ self._reset_agent(agent)
1539
+
1540
+ self._use_agent_host()
1497
1541
 
1498
1542
  params = get_streams_params(connection=self.artifacts_store)
1499
1543
  url = get_proxy_run_url(
@@ -1523,6 +1567,7 @@ class RunClient:
1523
1567
  path: str = "",
1524
1568
  overwrite: bool = True,
1525
1569
  relative_to: Optional[str] = None,
1570
+ agent: Optional[str] = None,
1526
1571
  ):
1527
1572
  """Uploads a full directory to the run's artifacts store path.
1528
1573
 
@@ -1535,7 +1580,7 @@ class RunClient:
1535
1580
  overwrite: bool, optional, if the file uploaded should overwrite any previous content.
1536
1581
  relative_to: str, optional, if the path uploaded is not the current dir,
1537
1582
  and you want to cancel the relative path.
1538
-
1583
+ agent: str, optional, uuid reference of an agent to use.
1539
1584
  Returns:
1540
1585
  str.
1541
1586
  """
@@ -1555,6 +1600,7 @@ class RunClient:
1555
1600
  path=path or "",
1556
1601
  overwrite=overwrite,
1557
1602
  relative_to=relative_to,
1603
+ agent=agent,
1558
1604
  )
1559
1605
 
1560
1606
  @client_handler(check_no_op=True, check_offline=True)
@@ -1564,6 +1610,7 @@ class RunClient:
1564
1610
  path: str = "",
1565
1611
  overwrite: bool = True,
1566
1612
  relative_to: Optional[str] = None,
1613
+ agent: Optional[str] = None,
1567
1614
  ):
1568
1615
  """Uploads multiple artifacts to the run's artifacts store path.
1569
1616
 
@@ -1573,7 +1620,7 @@ class RunClient:
1573
1620
  overwrite: bool, optional, if the file uploaded should overwrite any previous content.
1574
1621
  relative_to: str, optional, if the path uploaded is not the current dir,
1575
1622
  and you want to cancel the relative path.
1576
-
1623
+ agent: str, optional, uuid reference of an agent to use.
1577
1624
  Returns:
1578
1625
  str.
1579
1626
  """
@@ -1583,6 +1630,10 @@ class RunClient:
1583
1630
 
1584
1631
  if not self.settings:
1585
1632
  self.refresh_data()
1633
+ if agent:
1634
+ self._reset_agent(agent)
1635
+
1636
+ self._use_agent_host()
1586
1637
 
1587
1638
  params = get_streams_params(connection=self.artifacts_store)
1588
1639
  url = get_proxy_run_url(
@@ -1613,6 +1664,8 @@ class RunClient:
1613
1664
  """
1614
1665
  if not self.settings:
1615
1666
  self.refresh_data()
1667
+ self._use_agent_host()
1668
+
1616
1669
  params = get_streams_params(connection=self.artifacts_store)
1617
1670
  self.client.runs_v1.delete_run_artifact(
1618
1671
  namespace=self.namespace,
@@ -1632,6 +1685,8 @@ class RunClient:
1632
1685
  """
1633
1686
  if not self.settings:
1634
1687
  self.refresh_data()
1688
+ self._use_agent_host()
1689
+
1635
1690
  params = get_streams_params(connection=self.artifacts_store)
1636
1691
  return self.client.runs_v1.delete_run_artifacts(
1637
1692
  namespace=self.namespace,
@@ -1654,6 +1709,8 @@ class RunClient:
1654
1709
  """
1655
1710
  if not self.settings:
1656
1711
  self.refresh_data()
1712
+ self._use_agent_host()
1713
+
1657
1714
  params = get_streams_params(connection=self.artifacts_store)
1658
1715
  return self.client.runs_v1.get_run_artifacts_tree(
1659
1716
  namespace=self.namespace,
@@ -1673,6 +1730,15 @@ class RunClient:
1673
1730
  self.run_uuid,
1674
1731
  )
1675
1732
 
1733
+ @client_handler(check_no_op=True, check_offline=True)
1734
+ def skip(self):
1735
+ """Skip the current run."""
1736
+ self.client.runs_v1.skip_run(
1737
+ self.owner,
1738
+ self.project,
1739
+ self.run_uuid,
1740
+ )
1741
+
1676
1742
  @client_handler(check_no_op=True, check_offline=True)
1677
1743
  def approve(self):
1678
1744
  """Approves the current run if it's pending upload or human approval."""
@@ -2095,9 +2161,9 @@ class RunClient:
2095
2161
  abspath = filepath if is_abs() else os.path.abspath(filepath)
2096
2162
 
2097
2163
  for_patterns = []
2098
- if getattr(self, "_artifacts_path"):
2164
+ if getattr(self, "_artifacts_path", None):
2099
2165
  for_patterns.append(getattr(self, "_artifacts_path"))
2100
- if getattr(self, "_store_path"):
2166
+ if getattr(self, "_store_path", None):
2101
2167
  for_patterns.append(getattr(self, "_store_path"))
2102
2168
  context_root = (
2103
2169
  ctx_paths.CONTEXT_OFFLINE_ROOT
@@ -2119,6 +2185,10 @@ class RunClient:
2119
2185
  if not self._has_meta_key("has_events"):
2120
2186
  self.log_meta(has_events=True)
2121
2187
 
2188
+ def _log_has_traces(self):
2189
+ if not self._has_meta_key("has_traces"):
2190
+ self.log_meta(has_traces=True, has_events=True)
2191
+
2122
2192
  def _log_has_metrics(self):
2123
2193
  data = {}
2124
2194
  if not self._has_meta_key("has_metrics"):
@@ -2597,8 +2667,8 @@ class RunClient:
2597
2667
  """
2598
2668
  params = get_query_params(limit=limit, offset=offset, query=query, sort=sort)
2599
2669
  query = params.get("query")
2600
- query = query + "&" if query else "?"
2601
- query += "pipeline={}".format(self.run_uuid)
2670
+ query = query + "," if query else ""
2671
+ query += "pipeline:{}".format(self.run_uuid)
2602
2672
  params["query"] = query
2603
2673
 
2604
2674
  return self.client.runs_v1.list_runs(self.owner, self.project, **params)
@@ -2646,6 +2716,21 @@ class RunClient:
2646
2716
  force=force,
2647
2717
  )
2648
2718
 
2719
+ @client_handler(check_no_op=True, check_offline=True)
2720
+ def set_run_edges_lineage(self, run_edges: List[V1RunEdgeLineage]):
2721
+ """
2722
+ Set run edges lineage.
2723
+
2724
+ Args:
2725
+ run_edges: List[V1RunEdgeLineage], list of run edges.
2726
+ """
2727
+ return self.client.runs_v1.set_run_edges_lineage(
2728
+ self.owner,
2729
+ self.project,
2730
+ self.run_uuid,
2731
+ body=V1RunEdgesGraph(edges=run_edges),
2732
+ )
2733
+
2649
2734
  @client_handler(check_no_op=True, check_offline=True)
2650
2735
  def promote_to_artifact_version(
2651
2736
  self,
@@ -2669,7 +2754,7 @@ class RunClient:
2669
2754
  description: str, optional, the version description.
2670
2755
  tags: str or List[str], optional.
2671
2756
  content: str or dict, optional, content/metadata (JSON object) of the version.
2672
- connection: str, optional, a uuid reference to a connection.
2757
+ connection: str, optional, uuid reference to a connection.
2673
2758
  artifacts: List[str], optional, list of artifacts to highlight(requires passing a run)
2674
2759
  force: bool, optional, to force push, i.e. update if exists.
2675
2760
 
@@ -2812,6 +2897,7 @@ class RunClient:
2812
2897
  config_file.write(
2813
2898
  orjson_dumps(self.client.sanitize_for_serialization(self.run_data))
2814
2899
  )
2900
+ set_permissions(run_path)
2815
2901
 
2816
2902
  if not self._artifacts_lineage:
2817
2903
  logger.debug("Persist offline run call did not find any lineage data. ")
@@ -2827,6 +2913,7 @@ class RunClient:
2827
2913
  ]
2828
2914
  )
2829
2915
  )
2916
+ set_permissions(lineages_path)
2830
2917
 
2831
2918
  @classmethod
2832
2919
  @client_handler(check_no_op=True)
@@ -2835,6 +2922,8 @@ class RunClient:
2835
2922
  path: str,
2836
2923
  run_client: Optional[Union["RunClient", "Run"]] = None,
2837
2924
  reset_project: bool = False,
2925
+ reset_uuid: bool = False,
2926
+ name: Optional[str] = None,
2838
2927
  raise_if_not_found: bool = False,
2839
2928
  ) -> Optional[Union["RunClient", "Run"]]:
2840
2929
  """Loads an offline run from a local path.
@@ -2851,6 +2940,8 @@ class RunClient:
2851
2940
  from the local run.
2852
2941
  raise_if_not_found: bool, optional, a flag to raise an error if the local path does not
2853
2942
  contain a persisted run.
2943
+ reset_uuid: bool, optional, a flag to reset the run's uuid.
2944
+ name: str, optional, a name to set for the run.
2854
2945
  """
2855
2946
  run_path = "{}/{}".format(path, ctx_paths.CONTEXT_LOCAL_RUN)
2856
2947
  if not os.path.isfile(run_path):
@@ -2865,6 +2956,10 @@ class RunClient:
2865
2956
  run_config = V1Run(**orjson_loads(config_str))
2866
2957
  owner = run_config.owner
2867
2958
  project = run_config.project
2959
+ if reset_uuid:
2960
+ run_config.uuid = uuid.uuid4().hex
2961
+ if name:
2962
+ run_config.name = name
2868
2963
  if run_client:
2869
2964
  if reset_project or not owner:
2870
2965
  owner = run_client.owner
@@ -2923,6 +3018,7 @@ class RunClient:
2923
3018
  path: str,
2924
3019
  upload_artifacts: bool = True,
2925
3020
  clean: bool = False,
3021
+ agent: Optional[str] = None,
2926
3022
  ):
2927
3023
  """Syncs an offline run to Polyaxon's API and artifacts store.
2928
3024
 
@@ -2931,6 +3027,7 @@ class RunClient:
2931
3027
  path: str, root path where the run's metadata & artifacts are stored.
2932
3028
  upload_artifacts: bool, optional, flag to trigger artifacts upload.
2933
3029
  clean: bool, optional, flag to clean local path after pushing the run.
3030
+ agent: str, optional, uuid reference of an agent to use.
2934
3031
  """
2935
3032
  # We ensure that the is_offline is False
2936
3033
  is_offline = self._is_offline
@@ -2963,6 +3060,7 @@ class RunClient:
2963
3060
  path="/",
2964
3061
  overwrite=True,
2965
3062
  relative_to=path,
3063
+ agent=agent,
2966
3064
  )
2967
3065
  logger.info(f"Offline artifacts for run {self.run_data.uuid} uploaded")
2968
3066
 
@@ -55,7 +55,7 @@ def resolve_globals_contexts(
55
55
  schedule_at: Optional[datetime] = None,
56
56
  started_at: Optional[datetime] = None,
57
57
  finished_at: Optional[datetime] = None,
58
- duration: Optional[int] = None,
58
+ duration: Optional[float] = None,
59
59
  plugins: Optional[V1Plugins] = None,
60
60
  artifacts_store: V1Connection = None,
61
61
  cloning_kind: V1CloningKind = None,
@@ -134,7 +134,7 @@ def resolve_contexts(
134
134
  schedule_at: Optional[datetime] = None,
135
135
  started_at: Optional[datetime] = None,
136
136
  finished_at: Optional[datetime] = None,
137
- duration: Optional[int] = None,
137
+ duration: Optional[float] = None,
138
138
  cloning_kind: V1CloningKind = None,
139
139
  original_uuid: Optional[str] = None,
140
140
  is_independent: bool = True,
@@ -33,7 +33,9 @@ class RayJobContextsManager(BaseContextsManager):
33
33
  connection_by_names=connection_by_names,
34
34
  )
35
35
 
36
- return {
36
+ data = {
37
37
  "head": _get_replica(job.head),
38
- "workers": {wn: _get_replica(job.workers[wn]) for wn in job.workers},
39
38
  }
39
+ if job.workers:
40
+ data["workers"] = {wn: _get_replica(job.workers[wn]) for wn in job.workers}
41
+ return data
@@ -49,8 +49,18 @@ class AgentResolver(BaseSchemaModel):
49
49
 
50
50
  self.polyaxon_sidecar = agent_config.sidecar or get_default_sidecar_container()
51
51
  self.polyaxon_init = agent_config.init or get_default_init_container()
52
- self.namespace = agent_config.namespace
53
- self.polyaxon_sidecar.monitor_logs = agent_config.is_replica
52
+ if compiled_operation.namespace:
53
+ namespaces = agent_config.additional_namespaces or []
54
+ namespaces.append(agent_config.namespace)
55
+ if compiled_operation.namespace not in namespaces:
56
+ raise PolyaxonCompilerError(
57
+ "The provided namespace `{}` is not managed by the agent.".format(
58
+ compiled_operation.namespace
59
+ )
60
+ )
61
+ self.namespace = compiled_operation.namespace
62
+ else:
63
+ self.namespace = agent_config.namespace
54
64
 
55
65
  def _resolve_run_connections(
56
66
  self, compiled_operation: V1CompiledOperation, agent_config: AgentConfig
@@ -174,7 +174,7 @@ class BaseResolver:
174
174
  def resolve_access(self):
175
175
  pass
176
176
 
177
- def resolve_connections(self):
177
+ def resolve_agent_environment(self):
178
178
  agent_env = AgentResolver.construct()
179
179
  agent_env.resolve(
180
180
  compiled_operation=self.compiled_operation, agent_config=self.agent_config
@@ -281,8 +281,8 @@ class BaseResolver:
281
281
  self.apply_operation_contexts()
282
282
  self.resolve_io()
283
283
  self.resolve_init_refs()
284
+ self.resolve_agent_environment()
284
285
  self.resolve_access()
285
- self.resolve_connections()
286
286
  self.resolve_artifacts_lineage()
287
287
  self.clean_init_refs()
288
288
  self.resolve_state()
@@ -10,7 +10,6 @@ from polyaxon._env_vars.keys import (
10
10
  ENV_KEYS_ARTIFACTS_ROOT,
11
11
  ENV_KEYS_CONTEXT_ROOT,
12
12
  ENV_KEYS_OFFLINE_ROOT,
13
- ENV_KEYS_SANDBOX_ROOT,
14
13
  )
15
14
 
16
15
  CONTEXT_RELATED_RUNS = "_related_runs"
@@ -19,7 +18,7 @@ CONTEXT_RELATED_RUNS = "_related_runs"
19
18
  CONTEXT_LOCAL_LINEAGES = "lineages.plx.json"
20
19
  CONTEXT_LOCAL_CONTENT = "content.plx.json"
21
20
  CONTEXT_LOCAL_README = "readme.plx.md"
22
- CONTEXT_LOCAL_POLYAXONFILE = "polyaxonfile.plx.md"
21
+ CONTEXT_LOCAL_POLYAXONFILE = "polyaxonfile.plx.json"
23
22
  CONTEXT_LOCAL_PROJECT = "project.plx.json"
24
23
  CONTEXT_LOCAL_RUN = "run.plx.json"
25
24
  CONTEXT_LOCAL_VERSION = "version.plx.json"
@@ -53,7 +52,6 @@ CONTEXT_TMP_RUNS_ROOT_FORMAT = os.environ.get(
53
52
  )
54
53
  CONTEXT_ARCHIVES_ROOT = os.environ.get(ENV_KEYS_ARCHIVES_ROOT, "/tmp/plx/archives")
55
54
  CONTEXT_ARTIFACTS_ROOT = os.environ.get(ENV_KEYS_ARTIFACTS_ROOT, "/tmp/plx/artifacts")
56
- CONTEXT_SANDBOX_ROOT = None
57
55
  CONTEXT_OFFLINE_ROOT = os.environ.get(ENV_KEYS_OFFLINE_ROOT, "/tmp/plx/offline")
58
56
  CONTEXT_OFFLINE_FORMAT = "{}/{{}}".format(CONTEXT_OFFLINE_ROOT)
59
57
  CONTEXT_ARTIFACTS_FORMAT = "{}/{{}}".format(CONTEXT_ARTIFACTS_ROOT)
@@ -84,15 +82,14 @@ def get_offline_path(
84
82
 
85
83
 
86
84
  def mount_sandbox(path: Optional[str] = None):
87
- global CONTEXT_SANDBOX_ROOT
88
85
  global CONTEXT_OFFLINE_ROOT
89
86
  global CONTEXT_ARTIFACTS_ROOT
90
87
  global CONTEXT_OFFLINE_FORMAT
91
88
  global CONTEXT_ARTIFACTS_FORMAT
92
89
 
93
90
  path = path or CONTEXT_OFFLINE_ROOT
94
- CONTEXT_SANDBOX_ROOT = os.environ.get(ENV_KEYS_SANDBOX_ROOT, path)
95
- CONTEXT_OFFLINE_ROOT = CONTEXT_SANDBOX_ROOT
96
- CONTEXT_ARTIFACTS_ROOT = CONTEXT_SANDBOX_ROOT
91
+ CONTEXT_OFFLINE_ROOT = path
92
+ CONTEXT_ARTIFACTS_ROOT = path
97
93
  CONTEXT_OFFLINE_FORMAT = "{}/{{}}".format(CONTEXT_OFFLINE_ROOT)
98
94
  CONTEXT_ARTIFACTS_FORMAT = "{}/{{}}".format(CONTEXT_ARTIFACTS_ROOT)
95
+ return path
@@ -5,7 +5,7 @@ from polyaxon._env_vars.keys import ENV_KEYS_LOG_LEVEL, ENV_KEYS_SECRET_KEY
5
5
 
6
6
 
7
7
  class ComposeOperator(CmdOperator):
8
- CMD = "docker-compose"
8
+ CMD = "docker compose"
9
9
 
10
10
  @classmethod
11
11
  def params(cls, args):
@@ -112,30 +112,6 @@ class ComposeOperator(CmdOperator):
112
112
  )
113
113
  return env
114
114
 
115
- @staticmethod
116
- def _generate_intervals(config):
117
- template = "{}={}"
118
- env = []
119
- if config.intervals:
120
- env.append(
121
- template.format(
122
- "POLYAXON_INTERVALS_RUNS_SCHEDULER", config.intervals.runs_scheduler
123
- )
124
- )
125
- env.append(
126
- template.format(
127
- "POLYAXON_INTERVALS_OPERATIONS_DEFAULT_RETRY_DELAY",
128
- config.intervals.operations_default_retry_delay,
129
- )
130
- )
131
- env.append(
132
- template.format(
133
- "POLYAXON_INTERVALS_OPERATIONS_MAX_RETRY_DELAY",
134
- config.intervals.operations_max_retry_delay,
135
- )
136
- )
137
- return env
138
-
139
115
  @staticmethod
140
116
  def _generate_email(config):
141
117
  template = "{}={}"
@@ -195,8 +171,6 @@ class ComposeOperator(CmdOperator):
195
171
  if config.allowed_hosts:
196
172
  env.append(template.format("POLYAXON_ALLOWED_HOSTS", config.allowed_hosts))
197
173
 
198
- env += cls._generate_intervals(config)
199
-
200
174
  if config.log_level:
201
175
  env.append(template.format(ENV_KEYS_LOG_LEVEL, config.log_level))
202
176
  env += cls._generate_postgres(config)