polyaxon 2.0.0rc28__tar.gz → 2.0.0rc29__tar.gz

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: polyaxon
-Version: 2.0.0rc28
+Version: 2.0.0rc29
 Summary: Command Line Interface (CLI) and client to interact with Polyaxon API.
 Home-page: https://github.com/polyaxon/polyaxon
 Author: Polyaxon, Inc.
@@ -182,7 +182,7 @@ Here are some examples of using distributed training:
  * [Distributed Pytorch](https://polyaxon.com/docs/experimentation/distributed/pytorch-jobs/)
  * [Distributed MPI](https://polyaxon.com/docs/experimentation/distributed/mpi-jobs/)
  * [Horovod](https://polyaxon.com/integrations/horovod/)
- * [Spark](https://polyaxon.com/docs/experimentation/distributed/spark-jobs/)
+ * [Ray](https://polyaxon.com/docs/experimentation/distributed/ray-jobs/)
  * [Dask](https://polyaxon.com/docs/experimentation/distributed/dask-jobs/)
 
 # Hyperparameters tuning

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/api.py

@@ -7,6 +7,8 @@ INTERNAL_V1 = "internal/{}".format(VERSION_V1)
 AUTH_REQUEST_V1 = "auth-request/{}".format(VERSION_V1)
 SERVICES_V1 = "services/{}".format(VERSION_V1)
 REWRITE_SERVICES_V1 = "rewrite-services/{}".format(VERSION_V1)
+MONITORS_V1 = "monitors/{}".format(VERSION_V1)
+REWRITE_MONITORS_V1 = "rewrite-monitors/{}".format(VERSION_V1)
 EXTERNAL_V1 = "external/{}".format(VERSION_V1)
 REWRITE_EXTERNAL_V1 = "rewrite-external/{}".format(VERSION_V1)
 K8S_V1 = "k8s/{}".format(VERSION_V1)
@@ -28,6 +30,8 @@ SSO_V1_LOCATION = "/" + SSO_V1 + "/"
 EVENTS_V1_LOCATION = "/" + EVENTS_V1 + "/"
 SERVICES_V1_LOCATION = "/" + SERVICES_V1 + "/"
 REWRITE_SERVICES_V1_LOCATION = "/" + REWRITE_SERVICES_V1 + "/"
+MONITORS_V1_LOCATION = "/" + MONITORS_V1 + "/"
+REWRITE_MONITORS_V1_LOCATION = "/" + REWRITE_MONITORS_V1 + "/"
 EXTERNAL_V1_LOCATION = "/" + EXTERNAL_V1 + "/"
 REWRITE_EXTERNAL_V1_LOCATION = "/" + REWRITE_EXTERNAL_V1 + "/"
 K8S_V1_LOCATION = "/" + K8S_V1 + "/"

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/auxiliaries/default_scheduling.py

@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-from pydantic import Field, StrictStr, validator
+from clipped.compact.pydantic import Field, StrictStr, validator
 
 from polyaxon.k8s import k8s_schemas, k8s_validation
 from polyaxon.schemas.base import BaseSchemaModel

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/auxiliaries/init.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional, Union
 
-from pydantic import Field, StrictStr, validator
+from clipped.compact.pydantic import Field, StrictStr, validator
 
 from polyaxon import pkg
 from polyaxon.containers.pull_policy import PullPolicy

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/auxiliaries/sidecar.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, Optional, Union
 
+from clipped.compact.pydantic import Field, StrictStr
 from clipped.types.ref_or_obj import BoolOrRef, IntOrRef, RefField
-from pydantic import Field, StrictStr
 
 from polyaxon import pkg
 from polyaxon.containers.pull_policy import PullPolicy

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/cli/operations.py

@@ -17,6 +17,7 @@ from clipped.utils.json import orjson_dumps
 from clipped.utils.lists import to_list
 from clipped.utils.responses import get_meta_response
 from clipped.utils.validation import validate_tags
+from clipped.utils.versions import compare_versions
 from urllib3.exceptions import HTTPError
 
 from polyaxon import settings
@@ -38,7 +39,7 @@ from polyaxon.cli.options import (
 )
 from polyaxon.cli.utils import handle_output
 from polyaxon.client import RunClient, get_run_logs
-from polyaxon.constants.metadata import META_IS_EXTERNAL, META_REWRITE_PATH
+from polyaxon.constants.metadata import META_IS_EXTERNAL, META_PORTS, META_REWRITE_PATH
 from polyaxon.contexts import paths as ctx_paths
 from polyaxon.env_vars.getters import get_project_or_local, get_project_run_or_local
 from polyaxon.exceptions import (
@@ -1963,9 +1964,24 @@ def service(ctx, project, uid, yes, external, url):
     service_endpoint = EXTERNAL_V1 if is_external else SERVICES_V1
     if rewrite_path:
         service_endpoint = REWRITE_EXTERNAL_V1 if is_external else REWRITE_SERVICES_V1
+
+    service_subpath = "{}/{}/{}/runs/{}/".format(
+        namespace, owner, project_name, run_uuid
+    )
+    port = 80
+    if client.settings and client.settings.agent:
+        version = client.settings.agent.version
+        if version and compare_versions(version, "2.0.0", ">="):
+            ports = client.run_data.meta_info.get(META_PORTS, [])
+            port = ports[0] if ports else 80
+        else:
+            port = None
+    if port:
+        service_subpath = "{}{}/".format(service_subpath, port)
+
     external_run_url = get_dashboard_url(
         base=service_endpoint,
-        subpath="{}/{}/{}/runs/{}/".format(namespace, owner, project_name, run_uuid),
+        subpath=service_subpath,
     )
 
     if url:

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/cli/services/docker.py

@@ -2,10 +2,10 @@ import sys
 
 import click
 
+from clipped.compact.pydantic import ValidationError
 from clipped.formatting import Printer
 from clipped.types.docker_image import validate_image
 from clipped.utils.paths import copy_file
-from pydantic import ValidationError
 
 from polyaxon.cli.check import check_polyaxonfile
 from polyaxon.cli.errors import handle_cli_error

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/cli/services/initializer.py

@@ -4,10 +4,10 @@ import sys
 
 import click
 
+from clipped.compact.pydantic import ValidationError
 from clipped.formatting import Printer
 from clipped.utils.lists import to_list
 from clipped.utils.paths import check_or_create_path, copy_file
-from pydantic import ValidationError
 
 from polyaxon.config.spec import ConfigSpec
 from polyaxon.connections import V1ConnectionKind

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/client/run.py

@@ -22,7 +22,11 @@ from clipped.utils.paths import (
     get_dirs_under_path,
     get_files_in_path_context,
 )
-from clipped.utils.query_params import get_logs_params, get_query_params
+from clipped.utils.query_params import (
+    get_logs_params,
+    get_query_params,
+    get_streams_params,
+)
 from clipped.utils.tz import now
 from clipped.utils.validation import validate_tags
 from urllib3.exceptions import HTTPError
@@ -896,8 +900,11 @@ class RunClient:
 
     @client_handler(check_no_op=True, check_offline=True)
     def inspect(self):
+        if not self.settings:
+            self.refresh_data()
+        params = get_streams_params(connection=self.artifacts_store, status=self.status)
         return self.client.runs_v1.inspect_run(
-            self.namespace, self.owner, self.project, self.run_uuid
+            self.namespace, self.owner, self.project, self.run_uuid, **params
         )
 
     @client_handler(check_no_op=True, check_offline=True)
@@ -1013,7 +1020,7 @@ class RunClient:
         """
         if not self.settings:
             self.refresh_data()
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
+        params = get_streams_params(self.artifacts_store)
         return self.client.runs_v1.get_run_events(
             self.namespace,
             self.owner,
@@ -1048,7 +1055,7 @@ class RunClient:
         """
         if not self.settings:
             self.refresh_data()
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
+        params = get_streams_params(self.artifacts_store)
         return self.client.runs_v1.get_multi_run_events(
             self.namespace,
             self.owner,
@@ -1137,7 +1144,7 @@ class RunClient:
         """
         if not self.settings:
             self.refresh_data()
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
+        params = get_streams_params(self.artifacts_store)
         return self.client.runs_v1.get_run_artifact(
             namespace=self.namespace,
             owner=self.owner,
@@ -1196,11 +1203,10 @@ class RunClient:
                     subpath="events/{}".format(lineage.kind),
                 )
                 url = absolute_uri(url=url, host=self.client.config.host)
-                params = {"names": lineage.name, "pkg_assets": True}
-                if force:
-                    params["force"] = True
-                if self.artifacts_store:
-                    params["connection"] = self.artifacts_store
+                params = get_streams_params(
+                    connection=self.artifacts_store, force=force
+                )
+                params.update({"names": lineage.name, "pkg_assets": True})
 
                 return self.store.download_file(
                     url=url,
@@ -1258,12 +1264,7 @@ class RunClient:
             subpath="artifact",
         )
         url = absolute_uri(url=url, host=self.client.config.host)
-        params = {}
-        if force:
-            params["force"] = True
-        if self.artifacts_store:
-            params["connection"] = self.artifacts_store
-
+        params = get_streams_params(connection=self.artifacts_store, force=force)
         return self.store.download_file(
             url=url, path=path, path_to=path_to, params=params
         )
@@ -1302,11 +1303,9 @@ class RunClient:
             subpath="artifacts",
         )
         url = absolute_uri(url=url, host=self.client.config.host)
-        params = {}
+        params = get_streams_params(connection=self.artifacts_store)
         if check_path:
             params["check_path"] = True
-        if self.artifacts_store:
-            params["connection"] = self.artifacts_store
 
         return self.store.download_file(
             url=url,
@@ -1343,8 +1342,7 @@ class RunClient:
         if not self.settings:
             self.refresh_data()
 
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
-
+        params = get_streams_params(connection=self.artifacts_store)
         url = get_proxy_run_url(
             service=STREAMS_V1_LOCATION,
             namespace=self.namespace,
@@ -1433,8 +1431,7 @@ class RunClient:
         if not self.settings:
             self.refresh_data()
 
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
-
+        params = get_streams_params(connection=self.artifacts_store)
         url = get_proxy_run_url(
             service=STREAMS_V1_LOCATION,
             namespace=self.namespace,
@@ -1463,7 +1460,7 @@ class RunClient:
         """
         if not self.settings:
             self.refresh_data()
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
+        params = get_streams_params(connection=self.artifacts_store)
         self.client.runs_v1.delete_run_artifact(
             namespace=self.namespace,
             owner=self.owner,
@@ -1482,7 +1479,7 @@ class RunClient:
         """
         if not self.settings:
             self.refresh_data()
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
+        params = get_streams_params(connection=self.artifacts_store)
         return self.client.runs_v1.delete_run_artifacts(
             namespace=self.namespace,
             owner=self.owner,
@@ -1504,7 +1501,7 @@ class RunClient:
         """
         if not self.settings:
             self.refresh_data()
-        params = {"connection": self.artifacts_store} if self.artifacts_store else {}
+        params = get_streams_params(connection=self.artifacts_store)
         return self.client.runs_v1.get_run_artifacts_tree(
             namespace=self.namespace,
             owner=self.owner,

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/compiler/contexts/contexts.py

@@ -15,7 +15,6 @@ from polyaxon.compiler.contexts.kubeflow import (
 )
 from polyaxon.compiler.contexts.ray_job import RayJobContextsManager
 from polyaxon.compiler.contexts.service import ServiceContextsManager
-from polyaxon.compiler.contexts.spark_job import SparkJobContextsManager
 from polyaxon.connections import V1Connection
 from polyaxon.contexts import keys as ctx_keys
 from polyaxon.contexts import paths as ctx_paths
@@ -39,7 +38,6 @@ CONTEXTS_MANAGERS = {
     V1RunKind.PADDLEJOB: PaddleJobContextsManager,
     V1RunKind.RAYJOB: RayJobContextsManager,
     V1RunKind.DASKJOB: DaskJobContextsManager,
-    V1RunKind.SPARKJOB: SparkJobContextsManager,
 }
 
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/compiler/contexts/service.py

@@ -44,13 +44,18 @@ class ServiceContextsManager(BaseContextsManager):
                     if compiled_operation.run.rewrite_path
                     else SERVICES_V1_LOCATION
                 )
-            base_url = get_proxy_run_url(
-                service=service,
-                namespace=namespace,
-                owner=owner_name,
-                project=project_name,
-                run_uuid=run_uuid,
-            )
-            contexts["globals"]["base_url"] = base_url
-
+            ports = compiled_operation.run.ports or [80]
+            base_urls = []
+            for port in ports:
+                base_url = get_proxy_run_url(
+                    service=service,
+                    namespace=namespace,
+                    owner=owner_name,
+                    project=project_name,
+                    run_uuid=run_uuid,
+                    port=port,
+                )
+                base_urls.append(base_url)
+            contexts["globals"]["base_url"] = base_urls[0]
+            contexts["globals"]["base_urls"] = base_urls
         return contexts

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/compiler/resolver/agent.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Optional
 
+from clipped.compact.pydantic import Field
 from clipped.utils.lists import to_list
-from pydantic import Field
 
 from polyaxon import settings
 from polyaxon.auxiliaries import (

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/compiler/resolver/runtime.py

@@ -25,7 +25,6 @@ class BaseResolver:
         V1RunKind.TUNER,
         V1RunKind.WATCHDOG,
         V1RunKind.DAG,
-        V1RunKind.SPARKJOB,
         V1RunKind.DASKJOB,
         V1RunKind.RAYJOB,
         V1RunKind.TUNER,

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/constants/metadata.py

@@ -3,6 +3,7 @@ META_ITERATION = "iteration"
 META_BRACKET_ITERATION = "bracket_iteration"
 META_REWRITE_PATH = "rewrite_path"
 META_IS_EXTERNAL = "is_external"
+META_PORTS = "ports"
 META_HAS_EARLY_STOPPING = "has_early_stopping"
 META_HAS_JOBS = "has_jobs"
 META_HAS_SERVICES = "has_services"

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/auth.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictStr
+from clipped.compact.pydantic import Field, StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/celery.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictInt
+from clipped.compact.pydantic import Field, StrictInt
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/deployment.py

@@ -1,8 +1,14 @@
 from typing import Dict, List, Optional, Union
 from typing_extensions import Literal
 
+from clipped.compact.pydantic import (
+    Field,
+    StrictInt,
+    StrictStr,
+    root_validator,
+    validator,
+)
 from clipped.formatting import Printer
-from pydantic import Field, StrictInt, StrictStr, root_validator, validator
 
 from polyaxon.auxiliaries import (
     V1PolyaxonCleaner,

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/email.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictInt, StrictStr
+from clipped.compact.pydantic import Field, StrictInt, StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/ingress.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional
 
-from pydantic import Extra, Field, StrictStr
+from clipped.compact.pydantic import Extra, Field, StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/intervals.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictInt
+from clipped.compact.pydantic import Field, StrictInt
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/operators.py

@@ -10,6 +10,5 @@ class OperatorsConfig(BaseSchemaModel):
     mpijob: Optional[bool]
     mxjob: Optional[bool]
     xgbjob: Optional[bool]
-    sparkjob: Optional[bool]
     daskjob: Optional[bool]
     rayjob: Optional[bool]

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/proxy.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictInt, StrictStr
+from clipped.compact.pydantic import Field, StrictInt, StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/root_user.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
+from clipped.compact.pydantic import StrictStr, constr
 from clipped.types.email import EmailStr
-from pydantic import StrictStr, constr
 
 from polyaxon.schemas.base import NAME_REGEX, BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/security_context.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
+from clipped.compact.pydantic import Field, StrictInt, StrictStr, root_validator
 from clipped.config.schema import skip_partial
-from pydantic import Field, StrictInt, StrictStr, root_validator
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/service.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional
 
-from pydantic import Extra, Field, StrictInt, StrictStr
+from clipped.compact.pydantic import Extra, Field, StrictInt, StrictStr
 
 from polyaxon.deploy.schemas.celery import CeleryConfig
 from polyaxon.deploy.schemas.service_types import ServiceTypes

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/ssl.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictStr
+from clipped.compact.pydantic import Field, StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/deploy/schemas/ui.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import Field, StrictStr
+from clipped.compact.pydantic import Field, StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/docker/docker_types.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional, Tuple, Union
 
-from pydantic import Field
+from clipped.compact.pydantic import Field
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/env_vars/keys.py

@@ -102,7 +102,6 @@ EV_KEYS_MXJOB_ENABLED = "POLYAXON_MXJOB_ENABLED"
 EV_KEYS_XGBOOST_JOB_ENABLED = "POLYAXON_XGBOOST_JOB_ENABLED"
 EV_KEYS_MPIJOB_ENABLED = "POLYAXON_MPIJOB_ENABLED"
 EV_KEYS_RAYJOB_ENABLED = "POLYAXON_RAYJOB_ENABLED"
-EV_KEYS_SPARKJOB_ENABLED = "POLYAXON_SPARKJOB_ENABLED"
 EV_KEYS_DASKJOB_ENABLED = "POLYAXON_DASKJOB_ENABLED"
 
 # Sandbox
@@ -128,7 +127,6 @@ EV_KEYS_PROXY_API_TARGET_PORT = "POLYAXON_PROXY_API_TARGET_PORT"
 EV_KEYS_PROXY_API_HOST = "POLYAXON_PROXY_API_HOST"
 EV_KEYS_PROXY_LOCAL_PORT = "POLYAXON_PROXY_LOCAL_PORT"
 EV_KEYS_PROXY_API_USE_RESOLVER = "POLYAXON_PROXY_API_USE_RESOLVER"
-EV_KEYS_PROXY_SERVICES_PORT = "POLYAXON_PROXY_SERVICES_PORT"
 EV_KEYS_PROXY_SSL_PATH = "POLYAXON_PROXY_SSL_PATH"
 EV_KEYS_PROXY_SSL_ENABLED = "POLYAXON_PROXY_SSL_ENABLED"
 EV_KEYS_PROXY_AUTH_ENABLED = "POLYAXON_PROXY_AUTH_ENABLED"

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/k8s/converter/converters/__init__.py

@@ -15,7 +15,6 @@ from polyaxon.k8s.converter.converters.kubeflow import (
 )
 from polyaxon.k8s.converter.converters.ray_job import RayJobConverter
 from polyaxon.k8s.converter.converters.service import ServiceConverter
-from polyaxon.k8s.converter.converters.spark_job import SparkJobConverter
 from polyaxon.polyflow import V1RunKind
 
 CONVERTERS = {
@@ -32,5 +31,4 @@ CONVERTERS = {
     V1RunKind.XGBJOB: XGBoostJobConverter,
     V1RunKind.RAYJOB: RayJobConverter,
     V1RunKind.DASKJOB: DaskJobConverter,
-    V1RunKind.SPARKJOB: SparkJobConverter,
 }

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/k8s/converter/converters/dask_job.py

@@ -53,6 +53,9 @@ class DaskJobConverter(DaskJobMixin, BaseConverter):
 
         return get_dask_job_custom_resource(
             namespace=self.namespace,
+            owner_name=self.owner_name,
+            project_name=self.project_name,
+            run_uuid=self.run_uuid,
             resource_name=self.resource_name,
             job=job_replica,
             worker=worker,

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/k8s/converter/mixins.py

@@ -98,12 +98,6 @@ class DaskJobMixin(BaseMixin):
     MAIN_CONTAINER_ID = MAIN_JOB_CONTAINER
 
 
-class SparkJobMixin(BaseMixin):
-    K8S_ANNOTATIONS_KIND = V1RunKind.SPARKJOB
-    K8S_LABELS_COMPONENT = "polyaxon-spark-jobs"
-    MAIN_CONTAINER_ID = MAIN_JOB_CONTAINER
-
-
 MIXIN_MAPPING: Dict = {
     V1RunKind.JOB: JobMixin,
     V1RunKind.NOTIFIER: NotifierMixin,
@@ -118,5 +112,4 @@ MIXIN_MAPPING: Dict = {
     V1RunKind.MPIJOB: MPIJobMixin,
     V1RunKind.RAYJOB: RayJobMixin,
     V1RunKind.DASKJOB: DaskJobMixin,
-    V1RunKind.SPARKJOB: SparkJobMixin,
 }

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/k8s/custom_resources/dask_job.py

@@ -1,5 +1,6 @@
 from typing import Dict, List, Optional
 
+from clipped.utils.lists import to_list
 from kubernetes import client
 
 from polyaxon.k8s import k8s_schemas
@@ -24,6 +25,7 @@ def get_dask_replicas_template(
     annotations: Dict[str, str],
     template_spec: Dict,
     args: Optional[List[str]] = None,
+    additional_args: Optional[List[str]] = None,
     ports: Optional[List[k8s_schemas.V1ContainerPort]] = None,
     readiness_probe: Optional[client.V1Probe] = None,
     liveness_probe: Optional[client.V1Probe] = None,
@@ -35,6 +37,10 @@ def get_dask_replicas_template(
         replica.main_container.ports = ports
     if args and replica.main_container.args is None:
         replica.main_container.args = args
+    if additional_args:
+        replica.main_container.args = (
+            to_list(replica.main_container.args, check_none=True) + additional_args
+        )
     if readiness_probe and replica.main_container.readiness_probe is None:
         replica.main_container.readiness_probe = readiness_probe
     if liveness_probe and replica.main_container.liveness_probe is None:
@@ -61,6 +67,9 @@ def get_dask_replicas_template(
 
 def get_dask_job_custom_resource(
     resource_name: str,
+    owner_name: str,
+    project_name: str,
+    run_uuid: str,
     namespace: str,
     job: Optional[ReplicaSpec],
     worker: Optional[ReplicaSpec],
@@ -105,6 +114,8 @@ def get_dask_job_custom_resource(
             )
         ],
     )
+    dash_prefix = f"/monitors/v1/{namespace}/{owner_name}/{project_name}/runs/{run_uuid}/plx-operation-{run_uuid}-scheduler/8787"
+    health_path = "{}/health".format(dash_prefix)
     get_dask_replicas_template(
         replica_name="Scheduler",
         replica=scheduler,
@@ -114,6 +125,7 @@ def get_dask_job_custom_resource(
         annotations=annotations,
         template_spec=template_spec,
         args=["dask-scheduler"],
+        additional_args=["--dashboard-prefix", dash_prefix],
         ports=[
             k8s_schemas.V1ContainerPort(
                 name="tcp-comm", container_port=8786, protocol="TCP"
@@ -123,12 +135,12 @@ def get_dask_job_custom_resource(
             ),
         ],
         readiness_probe=client.V1Probe(
-            http_get=client.V1HTTPGetAction(path="/health", port="http-dashboard"),
+            http_get=client.V1HTTPGetAction(path=health_path, port="http-dashboard"),
             initial_delay_seconds=5,
             period_seconds=5,
         ),
         liveness_probe=client.V1Probe(
-            http_get=client.V1HTTPGetAction(path="/health", port="http-dashboard"),
+            http_get=client.V1HTTPGetAction(path=health_path, port="http-dashboard"),
             initial_delay_seconds=15,
             period_seconds=15,
         ),

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/lifecycle.py

@@ -2,10 +2,10 @@ import datetime
 
 from typing import Any, Dict, List, Optional, Union
 
+from clipped.compact.pydantic import StrictStr
 from clipped.utils.dates import parse_datetime
 from clipped.utils.enums import PEnum
 from clipped.utils.tz import now
-from pydantic import StrictStr
 
 from polyaxon.schemas.base import BaseSchemaModel
 

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/pkg.py

@@ -1,5 +1,5 @@
 NAME = "polyaxon"
-VERSION = "2.0.0-rc28"
+VERSION = "2.0.0-rc29"
 SCHEMA_VERSION = 1.1
 DESC = "Command Line Interface (CLI) and client to interact with Polyaxon API."
 URL = "https://github.com/polyaxon/polyaxon"

{polyaxon-2.0.0rc28 → polyaxon-2.0.0rc29}/polyaxon/polyflow/__init__.py

@@ -91,10 +91,6 @@ from polyaxon.polyflow.run import (
     V1Runtime,
     V1SchedulingPolicy,
     V1Service,
-    V1SparkDeploy,
-    V1SparkJob,
-    V1SparkReplica,
-    V1SparkType,
     V1TFJob,
     V1TunerJob,
     V1XGBoostJob,