polyaxon 2.0.0rc49__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. polyaxon/_auxiliaries/cleaner.py +8 -3
  2. polyaxon/_auxiliaries/init.py +7 -2
  3. polyaxon/_auxiliaries/notifier.py +8 -2
  4. polyaxon/_auxiliaries/sidecar.py +30 -2
  5. polyaxon/_cli/artifacts.py +96 -11
  6. polyaxon/_cli/components.py +96 -11
  7. polyaxon/_cli/config.py +118 -22
  8. polyaxon/_cli/dashboard.py +15 -2
  9. polyaxon/_cli/init.py +1 -1
  10. polyaxon/_cli/models.py +96 -11
  11. polyaxon/_cli/operations.py +267 -90
  12. polyaxon/_cli/project_versions.py +139 -6
  13. polyaxon/_cli/projects.py +23 -9
  14. polyaxon/_cli/run.py +37 -9
  15. polyaxon/_cli/services/agent.py +2 -2
  16. polyaxon/_cli/services/clean_artifacts.py +1 -1
  17. polyaxon/_cli/services/sidecar.py +8 -1
  18. polyaxon/_client/client.py +17 -0
  19. polyaxon/_client/mixin.py +39 -0
  20. polyaxon/_client/project.py +218 -23
  21. polyaxon/_client/run.py +131 -33
  22. polyaxon/_compiler/contexts/contexts.py +2 -2
  23. polyaxon/_compiler/contexts/ray_job.py +4 -2
  24. polyaxon/_compiler/resolver/agent.py +12 -2
  25. polyaxon/_compiler/resolver/runtime.py +2 -2
  26. polyaxon/_contexts/paths.py +4 -7
  27. polyaxon/_deploy/operators/compose.py +1 -27
  28. polyaxon/_deploy/schemas/deployment.py +4 -1
  29. polyaxon/_deploy/schemas/intervals.py +0 -7
  30. polyaxon/_deploy/schemas/proxy.py +1 -0
  31. polyaxon/_deploy/schemas/service.py +11 -1
  32. polyaxon/_docker/converter/base/base.py +8 -0
  33. polyaxon/_docker/executor.py +10 -4
  34. polyaxon/_env_vars/getters/owner_entity.py +4 -2
  35. polyaxon/_env_vars/getters/project.py +4 -2
  36. polyaxon/_env_vars/getters/run.py +5 -2
  37. polyaxon/_env_vars/keys.py +7 -1
  38. polyaxon/_flow/__init__.py +2 -0
  39. polyaxon/_flow/builds/__init__.py +19 -6
  40. polyaxon/_flow/component/base.py +1 -0
  41. polyaxon/_flow/component/component.py +14 -0
  42. polyaxon/_flow/environment/__init__.py +8 -8
  43. polyaxon/_flow/hooks/__init__.py +19 -6
  44. polyaxon/_flow/init/__init__.py +6 -6
  45. polyaxon/_flow/matrix/iterative.py +0 -1
  46. polyaxon/_flow/matrix/tuner.py +18 -6
  47. polyaxon/_flow/operations/operation.py +44 -17
  48. polyaxon/_flow/plugins/__init__.py +6 -0
  49. polyaxon/_flow/run/__init__.py +2 -2
  50. polyaxon/_flow/run/dag.py +2 -2
  51. polyaxon/_flow/run/dask/dask.py +0 -1
  52. polyaxon/_flow/run/dask/replica.py +3 -3
  53. polyaxon/_flow/run/enums.py +5 -0
  54. polyaxon/_flow/run/job.py +4 -4
  55. polyaxon/_flow/run/kubeflow/mpi_job.py +1 -2
  56. polyaxon/_flow/run/kubeflow/mx_job.py +1 -2
  57. polyaxon/_flow/run/kubeflow/paddle_job.py +35 -4
  58. polyaxon/_flow/run/kubeflow/pytorch_job.py +51 -5
  59. polyaxon/_flow/run/kubeflow/replica.py +4 -4
  60. polyaxon/_flow/run/kubeflow/scheduling_policy.py +12 -0
  61. polyaxon/_flow/run/kubeflow/tf_job.py +3 -3
  62. polyaxon/_flow/run/kubeflow/xgboost_job.py +1 -2
  63. polyaxon/_flow/run/ray/ray.py +2 -3
  64. polyaxon/_flow/run/ray/replica.py +3 -3
  65. polyaxon/_flow/run/service.py +4 -4
  66. polyaxon/_fs/fs.py +7 -2
  67. polyaxon/_fs/utils.py +3 -2
  68. polyaxon/_k8s/converter/base/base.py +2 -1
  69. polyaxon/_k8s/converter/base/main.py +1 -0
  70. polyaxon/_k8s/converter/base/sidecar.py +16 -1
  71. polyaxon/_k8s/converter/common/accelerators.py +7 -4
  72. polyaxon/_k8s/converter/converters/job.py +1 -1
  73. polyaxon/_k8s/converter/converters/kubeflow/paddle_job.py +1 -0
  74. polyaxon/_k8s/converter/converters/kubeflow/pytroch_job.py +2 -0
  75. polyaxon/_k8s/converter/converters/kubeflow/tf_job.py +1 -0
  76. polyaxon/_k8s/converter/converters/ray_job.py +4 -2
  77. polyaxon/_k8s/custom_resources/dask_job.py +3 -0
  78. polyaxon/_k8s/custom_resources/kubeflow/common.py +4 -1
  79. polyaxon/_k8s/custom_resources/kubeflow/paddle_job.py +10 -1
  80. polyaxon/_k8s/custom_resources/kubeflow/pytorch_job.py +14 -1
  81. polyaxon/_k8s/custom_resources/kubeflow/tf_job.py +4 -0
  82. polyaxon/_k8s/custom_resources/ray_job.py +3 -0
  83. polyaxon/_k8s/custom_resources/setter.py +1 -1
  84. polyaxon/_k8s/executor/async_executor.py +2 -0
  85. polyaxon/_k8s/executor/base.py +23 -6
  86. polyaxon/_k8s/logging/async_monitor.py +150 -5
  87. polyaxon/_k8s/manager/async_manager.py +96 -23
  88. polyaxon/_k8s/manager/base.py +4 -0
  89. polyaxon/_k8s/manager/manager.py +282 -134
  90. polyaxon/_local_process/__init__.py +0 -0
  91. polyaxon/_local_process/agent.py +6 -0
  92. polyaxon/_local_process/converter/__init__.py +1 -0
  93. polyaxon/_local_process/converter/base/__init__.py +1 -0
  94. polyaxon/_local_process/converter/base/base.py +140 -0
  95. polyaxon/_local_process/converter/base/containers.py +69 -0
  96. polyaxon/_local_process/converter/base/env_vars.py +253 -0
  97. polyaxon/_local_process/converter/base/init.py +414 -0
  98. polyaxon/_local_process/converter/base/main.py +74 -0
  99. polyaxon/_local_process/converter/base/mounts.py +82 -0
  100. polyaxon/_local_process/converter/converters/__init__.py +8 -0
  101. polyaxon/_local_process/converter/converters/job.py +40 -0
  102. polyaxon/_local_process/converter/converters/service.py +41 -0
  103. polyaxon/_local_process/converter/mixins.py +38 -0
  104. polyaxon/_local_process/executor.py +132 -0
  105. polyaxon/_local_process/process_types.py +39 -0
  106. polyaxon/_managers/agent.py +2 -0
  107. polyaxon/_managers/home.py +2 -1
  108. polyaxon/_operations/tuner.py +1 -0
  109. polyaxon/_polyaxonfile/check.py +2 -0
  110. polyaxon/_polyaxonfile/manager/operations.py +3 -0
  111. polyaxon/_polyaxonfile/manager/workflows.py +2 -0
  112. polyaxon/_polyaxonfile/specs/compiled_operation.py +1 -0
  113. polyaxon/_polyaxonfile/specs/operation.py +1 -0
  114. polyaxon/_polyaxonfile/specs/sections.py +3 -0
  115. polyaxon/_pql/manager.py +1 -1
  116. polyaxon/_runner/agent/async_agent.py +97 -21
  117. polyaxon/_runner/agent/base_agent.py +27 -9
  118. polyaxon/_runner/agent/client.py +15 -1
  119. polyaxon/_runner/agent/sync_agent.py +85 -20
  120. polyaxon/_runner/converter/converter.py +6 -2
  121. polyaxon/_runner/executor.py +13 -7
  122. polyaxon/_schemas/agent.py +27 -1
  123. polyaxon/_schemas/client.py +30 -3
  124. polyaxon/_schemas/installation.py +4 -3
  125. polyaxon/_schemas/lifecycle.py +10 -5
  126. polyaxon/_schemas/log_handler.py +2 -3
  127. polyaxon/_schemas/types/artifacts.py +3 -3
  128. polyaxon/_schemas/types/dockerfile.py +3 -3
  129. polyaxon/_schemas/types/file.py +3 -3
  130. polyaxon/_schemas/types/git.py +3 -3
  131. polyaxon/_schemas/types/tensorboard.py +3 -3
  132. polyaxon/_sdk/api/agents_v1_api.py +1076 -73
  133. polyaxon/_sdk/api/organizations_v1_api.py +371 -10
  134. polyaxon/_sdk/api/project_dashboards_v1_api.py +12 -12
  135. polyaxon/_sdk/api/project_searches_v1_api.py +12 -12
  136. polyaxon/_sdk/api/projects_v1_api.py +221 -44
  137. polyaxon/_sdk/api/runs_v1_api.py +917 -445
  138. polyaxon/_sdk/api/service_accounts_v1_api.py +16 -16
  139. polyaxon/_sdk/api/teams_v1_api.py +2827 -375
  140. polyaxon/_sdk/api/users_v1_api.py +231 -55
  141. polyaxon/_sdk/async_client/api_client.py +4 -0
  142. polyaxon/_sdk/schemas/__init__.py +10 -2
  143. polyaxon/_sdk/schemas/v1_agent.py +2 -1
  144. polyaxon/_sdk/schemas/v1_agent_reconcile_body_request.py +14 -0
  145. polyaxon/_sdk/schemas/v1_artifact_tree.py +1 -1
  146. polyaxon/_sdk/schemas/v1_dashboard_spec.py +4 -0
  147. polyaxon/_sdk/schemas/v1_events_response.py +4 -0
  148. polyaxon/_sdk/schemas/v1_organization.py +1 -0
  149. polyaxon/_sdk/schemas/v1_preset.py +8 -0
  150. polyaxon/_sdk/schemas/v1_project.py +1 -0
  151. polyaxon/_sdk/schemas/v1_project_settings.py +4 -2
  152. polyaxon/_sdk/schemas/v1_run.py +2 -2
  153. polyaxon/_sdk/schemas/v1_run_edge_lineage.py +14 -0
  154. polyaxon/_sdk/schemas/v1_run_edges_graph.py +9 -0
  155. polyaxon/_sdk/schemas/v1_section_spec.py +7 -2
  156. polyaxon/_sdk/schemas/v1_settings_catalog.py +1 -0
  157. polyaxon/_sdk/schemas/v1_team.py +3 -0
  158. polyaxon/_sdk/schemas/v1_user.py +1 -2
  159. polyaxon/_sdk/schemas/v1_user_access.py +17 -0
  160. polyaxon/_services/values.py +1 -0
  161. polyaxon/_sidecar/container/__init__.py +39 -18
  162. polyaxon/_sidecar/container/monitors/__init__.py +1 -0
  163. polyaxon/_sidecar/container/monitors/logs.py +10 -13
  164. polyaxon/_sidecar/container/monitors/spec.py +24 -0
  165. polyaxon/_sidecar/ignore.py +0 -1
  166. polyaxon/_utils/fqn_utils.py +25 -2
  167. polyaxon/client.py +1 -1
  168. polyaxon/pkg.py +1 -1
  169. polyaxon/schemas.py +8 -1
  170. polyaxon/settings.py +6 -0
  171. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/METADATA +43 -43
  172. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/RECORD +176 -155
  173. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/WHEEL +1 -1
  174. polyaxon/_sdk/schemas/v1_project_user_access.py +0 -10
  175. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/LICENSE +0 -0
  176. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/entry_points.txt +0 -0
  177. {polyaxon-2.0.0rc49.dist-info → polyaxon-2.4.0rc1.dist-info}/top_level.txt +0 -0
@@ -6,6 +6,13 @@ from clipped.compact.pydantic import StrictStr
6
6
  from clipped.config.schema import BaseAllowSchemaModel
7
7
  from clipped.types.uuids import UUIDStr
8
8
 
9
+ from polyaxon._sdk.schemas.v1_settings_catalog import V1SettingsCatalog
10
+
11
+
12
+ class V1PresetSettings(BaseAllowSchemaModel):
13
+ projects: Optional[List[V1SettingsCatalog]]
14
+ runs: Optional[List[V1SettingsCatalog]]
15
+
9
16
 
10
17
  class V1Preset(BaseAllowSchemaModel):
11
18
  uuid: Optional[UUIDStr]
@@ -17,3 +24,4 @@ class V1Preset(BaseAllowSchemaModel):
17
24
  frozen: Optional[bool]
18
25
  live_state: Optional[int]
19
26
  content: Optional[StrictStr]
27
+ settings: Optional[V1PresetSettings]
@@ -22,6 +22,7 @@ class V1Project(BaseAllowSchemaModel):
22
22
  readme: Optional[StrictStr]
23
23
  excluded_features: Optional[List[StrictStr]]
24
24
  excluded_runtimes: Optional[List[StrictStr]]
25
+ archived_deletion_interval: Optional[int]
25
26
  settings: Optional[V1ProjectSettings]
26
27
  role: Optional[StrictStr]
27
28
  contributors: Optional[Dict[str, Any]]
@@ -3,7 +3,7 @@ from typing import List, Optional
3
3
  from clipped.compact.pydantic import StrictStr
4
4
  from clipped.config.schema import BaseAllowSchemaModel
5
5
 
6
- from polyaxon._sdk.schemas.v1_project_user_access import V1ProjectUserAccess
6
+ from polyaxon._sdk.schemas.v1_user_access import V1UserAccess
7
7
 
8
8
 
9
9
  class V1ProjectSettings(BaseAllowSchemaModel):
@@ -13,6 +13,8 @@ class V1ProjectSettings(BaseAllowSchemaModel):
13
13
  queue: Optional[StrictStr]
14
14
  queues: Optional[List[StrictStr]]
15
15
  agents: Optional[List[StrictStr]]
16
- user_accesses: Optional[List[V1ProjectUserAccess]]
16
+ namespaces: Optional[List[StrictStr]]
17
+ user_accesses: Optional[List[V1UserAccess]]
17
18
  teams: Optional[List[StrictStr]]
18
19
  projects: Optional[List[StrictStr]]
20
+ policy: Optional[StrictStr]
@@ -32,8 +32,8 @@ class V1Run(BaseAllowSchemaModel):
32
32
  updated_at: Optional[datetime.datetime]
33
33
  started_at: Optional[datetime.datetime]
34
34
  finished_at: Optional[datetime.datetime]
35
- wait_time: Optional[int]
36
- duration: Optional[int]
35
+ wait_time: Optional[float]
36
+ duration: Optional[float]
37
37
  managed_by: Optional[ManagedBy]
38
38
  is_managed: Optional[bool]
39
39
  is_approved: Optional[bool]
@@ -0,0 +1,14 @@
1
+ from typing import Any, Dict, Optional
2
+
3
+ from clipped.config.schema import BaseAllowSchemaModel
4
+ from clipped.types.uuids import UUIDStr
5
+
6
+
7
+ class V1RunEdgeLineage(BaseAllowSchemaModel):
8
+ """
9
+ V1RunEdgeLineage
10
+ """
11
+
12
+ uuid: Optional[UUIDStr]
13
+ is_upstream: Optional[bool]
14
+ values: Optional[Dict[str, Any]]
@@ -0,0 +1,9 @@
1
+ from typing import List, Optional
2
+
3
+ from clipped.config.schema import BaseAllowSchemaModel
4
+
5
+ from polyaxon._sdk.schemas.v1_run_edge_lineage import V1RunEdgeLineage
6
+
7
+
8
+ class V1RunEdgesGraph(BaseAllowSchemaModel):
9
+ edges: Optional[List[V1RunEdgeLineage]]
@@ -7,8 +7,13 @@ from clipped.config.schema import BaseAllowSchemaModel
7
7
  class V1SectionSpec(BaseAllowSchemaModel):
8
8
  name: Optional[StrictStr]
9
9
  is_minimized: Optional[bool]
10
+ is_frozen: Optional[str]
10
11
  columns: Optional[int]
11
12
  height: Optional[int]
13
+ xaxis: Optional[str]
14
+ smoothing: Optional[int]
15
+ ignore_outliers: Optional[bool]
16
+ sample_size: Optional[int]
12
17
  widgets: Optional[List[Dict[str, Any]]]
13
- page_index: Optional[int] = Field(alias="pageIndex")
14
- page_size: Optional[int] = Field(alias="pageSize")
18
+ page_index: Optional[int]
19
+ page_size: Optional[int]
@@ -9,3 +9,4 @@ class V1SettingsCatalog(BaseAllowSchemaModel):
9
9
  uuid: Optional[UUIDStr]
10
10
  name: Optional[StrictStr]
11
11
  version: Optional[StrictStr]
12
+ url: Optional[StrictStr]
@@ -11,10 +11,13 @@ from polyaxon._sdk.schemas.v1_team_settings import V1TeamSettings
11
11
 
12
12
  class V1Team(BaseAllowSchemaModel):
13
13
  uuid: Optional[UUIDStr]
14
+ owner: Optional[StrictStr]
14
15
  name: Optional[StrictStr]
15
16
  projects: Optional[List[StrictStr]]
16
17
  component_hubs: Optional[List[StrictStr]]
17
18
  model_registries: Optional[List[StrictStr]]
18
19
  settings: Optional[V1TeamSettings]
20
+ policy: Optional[StrictStr]
21
+ role: Optional[StrictStr]
19
22
  created_at: Optional[datetime.datetime]
20
23
  updated_at: Optional[datetime.datetime]
@@ -2,12 +2,11 @@ from typing import Optional
2
2
 
3
3
  from clipped.compact.pydantic import StrictStr
4
4
  from clipped.config.schema import BaseAllowSchemaModel
5
- from clipped.types.email import EmailStr
6
5
 
7
6
 
8
7
  class V1User(BaseAllowSchemaModel):
9
8
  username: Optional[StrictStr]
10
- email: Optional[EmailStr]
9
+ email: Optional[StrictStr]
11
10
  name: Optional[StrictStr]
12
11
  kind: Optional[StrictStr]
13
12
  theme: Optional[int]
@@ -0,0 +1,17 @@
1
+ from typing import Optional
2
+
3
+ from clipped.compact.pydantic import StrictStr
4
+ from clipped.config.schema import BaseAllowSchemaModel
5
+
6
+
7
+ class V1UserAccessData(BaseAllowSchemaModel):
8
+ username: Optional[StrictStr]
9
+ is_sa: Optional[bool]
10
+
11
+
12
+ class V1UserAccess(BaseAllowSchemaModel):
13
+ user: Optional[StrictStr]
14
+ user_data: Optional[V1UserAccessData]
15
+ queue: Optional[StrictStr]
16
+ preset: Optional[StrictStr]
17
+ namespace: Optional[StrictStr]
@@ -17,6 +17,7 @@ class PolyaxonServices(str, PEnum):
17
17
  GATEWAY = "gateway"
18
18
  STREAMS = "streams"
19
19
  SANDBOX = "sandbox"
20
+ VIEWER = "viewer"
20
21
  CLI = "cli"
21
22
  INITIALIZER = "initializer"
22
23
  INIT = "init"
@@ -15,8 +15,8 @@ from polyaxon._fs.fs import (
15
15
  from polyaxon._fs.watcher import FSWatcher
16
16
  from polyaxon._k8s.manager.async_manager import AsyncK8sManager
17
17
  from polyaxon._sidecar.container.intervals import get_sync_interval
18
- from polyaxon._sidecar.container.monitors import sync_artifacts, sync_logs
19
- from polyaxon._sidecar.ignore import CONTAINER_IGNORE_FOLDERS
18
+ from polyaxon._sidecar.container.monitors import sync_artifacts, sync_logs, sync_spec
19
+ from polyaxon._sidecar.ignore import IGNORE_FOLDERS
20
20
  from polyaxon.client import RunClient
21
21
  from polyaxon.exceptions import PolyaxonClientException, PolyaxonContainerException
22
22
  from polyaxon.logger import logger
@@ -29,10 +29,14 @@ async def start_sidecar(
29
29
  sync_interval: int,
30
30
  monitor_outputs: bool,
31
31
  monitor_logs: bool,
32
+ monitor_spec: bool,
32
33
  ):
33
34
  sync_interval = get_sync_interval(
34
35
  interval=sync_interval, sleep_interval=sleep_interval
35
36
  )
37
+ fs_refresh_interval = get_sync_interval(
38
+ interval=60 * 60 * 2, sleep_interval=sleep_interval
39
+ )
36
40
  try:
37
41
  pod_id = os.environ[ENV_KEYS_K8S_POD_ID]
38
42
  except KeyError as e:
@@ -60,46 +64,56 @@ async def start_sidecar(
60
64
  retry = 0
61
65
  is_running = True
62
66
  counter = 0
67
+ fs_refresh_counter = 0
63
68
  state = {
64
69
  "last_artifacts_check": None,
65
70
  "last_logs_check": None,
66
71
  }
67
72
 
68
73
  async def monitor():
74
+ if monitor_spec and pod.metadata.annotations:
75
+ await sync_spec(
76
+ k8s_manager=k8s_manager,
77
+ run_uuid=run_uuid,
78
+ run_kind=pod.metadata.annotations.get("operation.polyaxon.com/kind"),
79
+ )
69
80
  if monitor_logs:
70
- await sync_logs(
81
+ state["last_logs_check"] = await sync_logs(
71
82
  run_uuid=run_uuid,
72
83
  k8s_manager=k8s_manager,
73
84
  pod=pod,
74
- last_time=None,
75
- stream=True,
76
- is_running=is_running,
85
+ last_time=state["last_logs_check"],
86
+ stream=False,
77
87
  )
78
88
  if monitor_outputs:
79
- last_check = state["last_artifacts_check"]
80
- await sync_artifacts(
81
- fs=fs,
82
- fw=fw,
83
- store_path=connection.store_path,
84
- run_uuid=run_uuid,
85
- exclude=CONTAINER_IGNORE_FOLDERS,
86
- )
89
+ try:
90
+ await sync_artifacts(
91
+ fs=fs,
92
+ fw=fw,
93
+ store_path=connection.store_path,
94
+ run_uuid=run_uuid,
95
+ exclude=IGNORE_FOLDERS,
96
+ )
97
+ except Exception as e:
98
+ logger.debug(
99
+ "An error occurred while syncing artifacts, Exception %s" % repr(e)
100
+ )
87
101
  try:
88
102
  client.sync_events_summaries(
89
- last_check=last_check,
103
+ last_check=state["last_artifacts_check"],
90
104
  events_path=ctx_paths.CONTEXT_MOUNT_RUN_EVENTS_FORMAT.format(
91
105
  run_uuid
92
106
  ),
93
107
  )
94
108
  client.sync_system_events_summaries(
95
- last_check=last_check,
109
+ last_check=state["last_artifacts_check"],
96
110
  events_path=ctx_paths.CONTEXT_MOUNT_RUN_SYSTEM_RESOURCES_EVENTS_FORMAT.format(
97
111
  run_uuid
98
112
  ),
99
113
  )
100
114
  update_last_check = True
101
115
  except Exception as e:
102
- logger.info(
116
+ logger.debug(
103
117
  "An error occurred while syncing events summaries, "
104
118
  "Exception %s" % repr(e)
105
119
  )
@@ -124,7 +138,14 @@ async def start_sidecar(
124
138
  retry = 0
125
139
 
126
140
  counter += 1
127
- if counter == sync_interval:
141
+ fs_refresh_counter += 1
142
+
143
+ if fs_refresh_counter >= fs_refresh_interval:
144
+ fs_refresh_counter = 0
145
+ await close_fs(fs)
146
+ fs = await get_async_fs_from_connection(connection=connection)
147
+
148
+ if counter >= sync_interval:
128
149
  counter = 0
129
150
  try:
130
151
  await monitor()
@@ -1,2 +1,3 @@
1
1
  from polyaxon._sidecar.container.monitors.artifacts import sync_artifacts
2
2
  from polyaxon._sidecar.container.monitors.logs import sync_logs
3
+ from polyaxon._sidecar.container.monitors.spec import sync_spec
@@ -4,7 +4,7 @@ from typing import Optional
4
4
 
5
5
  import aiofiles
6
6
 
7
- from clipped.utils.paths import check_or_create_path, delete_path
7
+ from clipped.utils.paths import check_or_create_path
8
8
  from kubernetes_asyncio.client.models import V1Pod
9
9
 
10
10
  from polyaxon._contexts import paths as ctx_paths
@@ -19,26 +19,23 @@ async def sync_logs(
19
19
  pod: V1Pod,
20
20
  last_time: Optional[datetime.datetime],
21
21
  stream: bool = False,
22
- is_running: bool = True,
23
- ):
22
+ ) -> Optional[datetime.datetime]:
24
23
  path_from = ctx_paths.CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
25
- path_from = "{}/.tmpplxlogs".format(path_from)
24
+ path_from = "{}/plxlogs".format(path_from)
26
25
 
27
- if not is_running:
28
- delete_path(path_from)
29
- return
30
-
31
- logs, _ = await query_k8s_pod_logs(
26
+ logs, last_time = await query_k8s_pod_logs(
32
27
  k8s_manager=k8s_manager,
33
28
  pod=pod,
34
29
  last_time=last_time,
35
30
  stream=stream,
36
31
  )
37
32
  if not logs:
38
- return
33
+ return last_time
39
34
 
40
- path_from = "{}/{}.plx".format(path_from, pod.metadata.name)
35
+ path_from = "{}/{}.jsonl".format(path_from, pod.metadata.name)
41
36
  check_or_create_path(path_from, is_dir=False)
42
- async with aiofiles.open(path_from, "w") as filepath:
37
+ async with aiofiles.open(path_from, "a") as filepath:
43
38
  _logs = V1Logs.construct(logs=logs)
44
- await filepath.write("{}\n{}".format(_logs.get_csv_header(), _logs.to_csv()))
39
+ await filepath.write(_logs.get_jsonl_events())
40
+
41
+ return last_time
@@ -0,0 +1,24 @@
1
+ import aiofiles
2
+
3
+ from clipped.utils.json import orjson_dumps
4
+ from clipped.utils.paths import check_or_create_path
5
+
6
+ from polyaxon._contexts import paths as ctx_paths
7
+ from polyaxon._flow import V1RunKind
8
+ from polyaxon._k8s.logging.async_monitor import get_op_spec
9
+ from polyaxon._k8s.manager.async_manager import AsyncK8sManager
10
+
11
+
12
+ async def sync_spec(
13
+ run_uuid: str,
14
+ k8s_manager: AsyncK8sManager,
15
+ run_kind: V1RunKind,
16
+ ):
17
+ op_spec, _, _ = await get_op_spec(
18
+ k8s_manager=k8s_manager, run_uuid=run_uuid, run_kind=run_kind
19
+ )
20
+ path_from = ctx_paths.CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
21
+ path_from = "{}/outputs/spec.json".format(path_from)
22
+ check_or_create_path(path_from, is_dir=False)
23
+ async with aiofiles.open(path_from, "w") as filepath:
24
+ await filepath.write(orjson_dumps(op_spec))
@@ -1,2 +1 @@
1
1
  IGNORE_FOLDERS = [".git"]
2
- CONTAINER_IGNORE_FOLDERS = IGNORE_FOLDERS + ["plxlogs"]
@@ -9,15 +9,32 @@ from clipped.utils.paths import get_relative_path_to
9
9
  from polyaxon.exceptions import PolyaxonSchemaError
10
10
 
11
11
 
12
+ def get_owner_team_space(owner: str, team: Optional[str] = None) -> str:
13
+ if team:
14
+ return "{}/{}".format(owner, team)
15
+ return owner
16
+
17
+
18
+ def split_owner_team_space(owner: str) -> Tuple[str, Optional[str]]:
19
+ return owner.split("/") if owner and "/" in owner else (owner, None)
20
+
21
+
22
+ def _remove_team(owner: str) -> str:
23
+ return split_owner_team_space(owner)[0]
24
+
25
+
12
26
  def get_project_instance(owner: str, project: str) -> str:
27
+ owner = _remove_team(owner)
13
28
  return "{}.{}".format(owner, project)
14
29
 
15
30
 
16
31
  def get_run_instance(owner: str, project: str, run_uuid: str) -> str:
32
+ owner = _remove_team(owner)
17
33
  return "{}.{}.runs.{}".format(owner, project, run_uuid)
18
34
 
19
35
 
20
36
  def get_cleaner_instance(owner: str, project: str, run_uuid: str) -> str:
37
+ owner = _remove_team(owner)
21
38
  return "{}.{}.cleaners.{}".format(owner, project, run_uuid)
22
39
 
23
40
 
@@ -53,6 +70,7 @@ def to_fqn_name(name: str) -> str:
53
70
  def get_entity_full_name(
54
71
  owner: Optional[str] = None, entity: Optional[str] = None
55
72
  ) -> str:
73
+ owner = _remove_team(owner)
56
74
  if owner and entity:
57
75
  return "{}/{}".format(owner, entity)
58
76
  return entity
@@ -65,11 +83,14 @@ def get_entity_info(entity: str) -> Tuple[str, str]:
65
83
  )
66
84
 
67
85
  parts = entity.replace(".", "/").split("/")
68
- if len(parts) > 2:
86
+ if len(parts) > 3:
69
87
  raise PolyaxonSchemaError(
70
88
  "Received an invalid entity reference: `{}`".format(entity)
71
89
  )
72
- if len(parts) == 2:
90
+ if len(parts) == 3:
91
+ owner = "/".join(parts[:2])
92
+ entity_name = parts[2]
93
+ elif len(parts) == 2:
73
94
  owner, entity_name = parts
74
95
  else:
75
96
  owner = None
@@ -81,6 +102,8 @@ def get_entity_info(entity: str) -> Tuple[str, str]:
81
102
  def get_versioned_entity_full_name(
82
103
  owner: Optional[str], component: str, tag: Optional[str] = None
83
104
  ) -> str:
105
+ owner = _remove_team(owner)
106
+
84
107
  if tag:
85
108
  component = "{}:{}".format(component, tag)
86
109
  if owner:
polyaxon/client.py CHANGED
@@ -29,5 +29,5 @@ from polyaxon._sdk.api import (
29
29
  )
30
30
  from polyaxon._sdk.async_client.api_client import AsyncApiClient
31
31
  from polyaxon._sdk.configuration import Configuration
32
- from polyaxon._sdk.schemas import *
33
32
  from polyaxon._sdk.sync_client.api_client import ApiClient
33
+ from polyaxon.schemas import *
polyaxon/pkg.py CHANGED
@@ -1,5 +1,5 @@
1
1
  NAME = "polyaxon"
2
- VERSION = "2.0.0-rc49"
2
+ VERSION = "2.4.0-rc1"
3
3
  SCHEMA_VERSION = 1.1
4
4
  DESC = "Command Line Interface (CLI) and client to interact with Polyaxon API."
5
5
  URL = "https://github.com/polyaxon/polyaxon"
polyaxon/schemas.py CHANGED
@@ -1,5 +1,11 @@
1
1
  from clipped.config.patch_strategy import PatchStrategy as V1PatchStrategy
2
2
 
3
+ from polyaxon._auxiliaries import (
4
+ V1PolyaxonCleaner,
5
+ V1PolyaxonInitContainer,
6
+ V1PolyaxonNotifier,
7
+ V1PolyaxonSidecarContainer,
8
+ )
3
9
  from polyaxon._connections.kinds import V1ConnectionKind
4
10
  from polyaxon._connections.schemas import (
5
11
  V1BucketConnection,
@@ -195,6 +201,7 @@ from polyaxon._sdk.schemas import (
195
201
  V1ListTeamMembersResponse,
196
202
  V1ListTeamsResponse,
197
203
  V1ListTokenResponse,
204
+ V1MultiEventsResponse,
198
205
  V1OperationBody,
199
206
  V1Organization,
200
207
  V1OrganizationMember,
@@ -203,7 +210,6 @@ from polyaxon._sdk.schemas import (
203
210
  V1Preset,
204
211
  V1Project,
205
212
  V1ProjectSettings,
206
- V1ProjectUserAccess,
207
213
  V1ProjectVersion,
208
214
  V1Queue,
209
215
  V1Run,
@@ -223,6 +229,7 @@ from polyaxon._sdk.schemas import (
223
229
  V1Token,
224
230
  V1TrialStart,
225
231
  V1User,
232
+ V1UserAccess,
226
233
  V1UserEmail,
227
234
  V1UserSingup,
228
235
  V1Uuids,
polyaxon/settings.py CHANGED
@@ -38,6 +38,12 @@ AGENT_CONFIG: Optional[AgentConfig] = None
38
38
  PolyaxonServices.set_service_name()
39
39
 
40
40
 
41
+ def set_home_config(config: Optional[HomeConfig] = None):
42
+ global HOME_CONFIG
43
+
44
+ HOME_CONFIG = config or HomeConfigManager.get_config_from_env()
45
+
46
+
41
47
  def set_agent_config(config: Optional[AgentConfig] = None):
42
48
  from polyaxon._connections import CONNECTION_CONFIG
43
49
  from polyaxon._managers.agent import AgentConfigManager
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: polyaxon
3
- Version: 2.0.0rc49
3
+ Version: 2.4.0rc1
4
4
  Summary: Command Line Interface (CLI) and client to interact with Polyaxon API.
5
5
  Home-page: https://github.com/polyaxon/polyaxon
6
6
  Author: Polyaxon, Inc.
@@ -33,67 +33,67 @@ Classifier: Programming Language :: Python :: 3.11
33
33
  Requires-Python: >=3.8
34
34
  Description-Content-Type: text/markdown
35
35
  License-File: LICENSE
36
- Requires-Dist: click <9.0.0,>=7.1.1
37
- Requires-Dist: clipped ==0.7.*
38
- Requires-Dist: vents ==0.2.*
39
- Requires-Dist: Jinja2 <3.1.0,>=2.10.3
40
- Requires-Dist: kubernetes >=10.0.1
41
- Requires-Dist: python-dateutil >=2.7.3
42
- Requires-Dist: pytz >=2019.2
43
- Requires-Dist: PyYAML >=5.1
44
- Requires-Dist: orjson >=3.7
45
- Requires-Dist: psutil >=5.4.7
46
- Requires-Dist: requests >=2.20.1
47
- Requires-Dist: requests-toolbelt >=0.8.0
48
- Requires-Dist: rich >=12.0.0
49
- Requires-Dist: sentry-sdk >=1.2.0
50
- Requires-Dist: urllib3 >=1.25.6
51
- Requires-Dist: certifi >=2022.12.7
52
- Requires-Dist: pydantic >=1.10.2
53
- Requires-Dist: traceml ==1.1.0rc41
54
- Requires-Dist: hypertune ==1.1.0rc41
36
+ Requires-Dist: click<9.0.0,>=7.1.1
37
+ Requires-Dist: clipped==0.8.*
38
+ Requires-Dist: vents==0.4.*
39
+ Requires-Dist: Jinja2<3.1.5,>=2.10.3
40
+ Requires-Dist: kubernetes>=10.0.1
41
+ Requires-Dist: python-dateutil>=2.7.3
42
+ Requires-Dist: pytz>=2019.2
43
+ Requires-Dist: PyYAML>=5.1
44
+ Requires-Dist: orjson>=3.7
45
+ Requires-Dist: psutil>=5.4.7
46
+ Requires-Dist: requests>=2.20.1
47
+ Requires-Dist: requests-toolbelt>=0.8.0
48
+ Requires-Dist: rich>=12.0.0
49
+ Requires-Dist: sentry-sdk<2.6,>=1.2.0
50
+ Requires-Dist: urllib3>=1.25.6
51
+ Requires-Dist: certifi>=2022.12.7
52
+ Requires-Dist: pydantic>=1.10.2
53
+ Requires-Dist: traceml==1.1.*
54
+ Requires-Dist: hypertune==1.1.*
55
55
  Provides-Extra: azure
56
- Requires-Dist: adlfs ; extra == 'azure'
56
+ Requires-Dist: adlfs; extra == "azure"
57
57
  Provides-Extra: dev
58
- Requires-Dist: moto ==2.0.5 ; extra == 'dev'
58
+ Requires-Dist: moto==2.0.5; extra == "dev"
59
59
  Provides-Extra: docker
60
- Requires-Dist: docker ; extra == 'docker'
60
+ Requires-Dist: docker; extra == "docker"
61
61
  Provides-Extra: fs
62
- Requires-Dist: adlfs ==2023.9.0 ; extra == 'fs'
63
- Requires-Dist: fsspec ==2023.9.0 ; extra == 'fs'
64
- Requires-Dist: gcsfs ==2023.9.0 ; extra == 'fs'
65
- Requires-Dist: s3fs ==2023.9.0 ; extra == 'fs'
62
+ Requires-Dist: adlfs==2024.4.1; extra == "fs"
63
+ Requires-Dist: fsspec==2024.6.1; extra == "fs"
64
+ Requires-Dist: gcsfs==2024.6.1; extra == "fs"
65
+ Requires-Dist: s3fs==2024.6.1; extra == "fs"
66
66
  Provides-Extra: fsspec
67
- Requires-Dist: fsspec ; extra == 'fsspec'
67
+ Requires-Dist: fsspec; extra == "fsspec"
68
68
  Provides-Extra: gcs
69
- Requires-Dist: gcsfs ; extra == 'gcs'
69
+ Requires-Dist: gcsfs; extra == "gcs"
70
70
  Provides-Extra: git
71
- Requires-Dist: gitpython ; extra == 'git'
71
+ Requires-Dist: gitpython; extra == "git"
72
72
  Provides-Extra: init
73
- Requires-Dist: docker ; extra == 'init'
74
- Requires-Dist: GitPython <3.2.0 ; extra == 'init'
75
- Requires-Dist: aiofiles ==23.2.1 ; extra == 'init'
73
+ Requires-Dist: docker; extra == "init"
74
+ Requires-Dist: GitPython<3.2.0; extra == "init"
75
+ Requires-Dist: aiofiles==23.2.1; extra == "init"
76
76
  Provides-Extra: k8s_async
77
- Requires-Dist: kubernetes-asyncio ==25.11.0 ; extra == 'k8s_async'
78
- Requires-Dist: aiohttp >=3.0.0 ; extra == 'k8s_async'
79
- Requires-Dist: anyio ; extra == 'k8s_async'
77
+ Requires-Dist: kubernetes-asyncio>=28.2.1; extra == "k8s-async"
78
+ Requires-Dist: aiohttp>=3.0.0; extra == "k8s-async"
79
+ Requires-Dist: anyio; extra == "k8s-async"
80
80
  Provides-Extra: numpy
81
- Requires-Dist: numpy ; extra == 'numpy'
81
+ Requires-Dist: numpy; extra == "numpy"
82
82
  Provides-Extra: s3
83
- Requires-Dist: s3fs ; extra == 's3'
83
+ Requires-Dist: s3fs; extra == "s3"
84
84
  Provides-Extra: sandbox
85
- Requires-Dist: haupt[sandbox] ; extra == 'sandbox'
85
+ Requires-Dist: haupt[sandbox]; extra == "sandbox"
86
86
  Provides-Extra: sidecar
87
- Requires-Dist: aiofiles ==23.2.1 ; extra == 'sidecar'
88
- Requires-Dist: pandas ; extra == 'sidecar'
89
- Requires-Dist: anyio ; extra == 'sidecar'
87
+ Requires-Dist: aiofiles==23.2.1; extra == "sidecar"
88
+ Requires-Dist: pandas; extra == "sidecar"
89
+ Requires-Dist: anyio; extra == "sidecar"
90
90
 
91
91
  [![License: Apache 2](https://img.shields.io/badge/License-apache2-blue.svg?style=flat&longCache=true)](LICENSE)
92
92
  [![Polyaxon API](https://img.shields.io/docker/pulls/polyaxon/polyaxon-api)](https://hub.docker.com/r/polyaxon/polyaxon-api)
93
93
  [![Slack](https://img.shields.io/badge/Slack-1.5k%20members-blue.svg?style=flat&logo=slack&longCache=true)](https://polyaxon.com/slack/)
94
94
 
95
95
  [![Docs](https://img.shields.io/badge/docs-stable-brightgreen.svg?style=flat&longCache=true)](https://polyaxon.com/docs/)
96
- [![Release](https://img.shields.io/badge/release-v1.20.0-brightgreen.svg?longCache=true)](https://polyaxon.com/docs/releases/1-20/)
96
+ [![Release](https://img.shields.io/badge/release-v2.4.0-brightgreen.svg?longCache=true)](https://polyaxon.com/docs/releases/2-1/)
97
97
  [![GitHub](https://img.shields.io/badge/issue_tracker-github-blue?style=flat&logo=github&longCache=true)](https://github.com/polyaxon/polyaxon/issues)
98
98
  [![GitHub](https://img.shields.io/badge/roadmap-github-blue?style=flat&logo=github&longCache=true)](https://github.com/orgs/polyaxon/projects/5)
99
99