polyaxon 2.0.0rc27__py3-none-any.whl → 2.0.0rc29__py3-none-any.whl

{polyaxon-2.0.0rc27.dist-info → polyaxon-2.0.0rc29.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.0)
+Generator: bdist_wheel (0.41.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

polyaxon/compiler/contexts/spark_job.py

@@ -1,39 +0,0 @@
-import copy
-
-from typing import Dict, Optional
-
-from polyaxon.compiler.contexts.base import BaseContextsManager
-from polyaxon.connections import V1Connection
-from polyaxon.polyflow import V1CompiledOperation, V1SparkJob, V1SparkReplica
-
-
-class SparkJobContextsManager(BaseContextsManager):
-    @classmethod
-    def resolve(
-        cls,
-        namespace: str,
-        owner_name: str,
-        project_name: str,
-        run_uuid: str,
-        contexts: Dict,
-        compiled_operation: V1CompiledOperation,
-        connection_by_names: Dict[str, V1Connection],
-    ) -> Dict:
-        contexts["init"] = {}
-        contexts["connections"] = {}
-        job = compiled_operation.run  # type: V1SparkJob
-
-        def _get_replica(replica: Optional[V1SparkReplica]) -> Dict:
-            if not replica:
-                return contexts
-            return cls._resolver_replica(
-                contexts={"globals": copy.copy(contexts["globals"])},
-                init=replica.init,
-                connections=replica.connections,
-                connection_by_names=connection_by_names,
-            )
-
-        return {
-            "executor": _get_replica(job.executor),
-            "driver": _get_replica(job.driver),
-        }

polyaxon/k8s/converter/converters/spark_job.py

@@ -1,64 +0,0 @@
-from typing import Dict, Iterable, Optional
-
-from polyaxon import pkg
-from polyaxon.connections import V1Connection, V1ConnectionResource
-from polyaxon.k8s.converter.base import BaseConverter
-from polyaxon.k8s.converter.mixins import SparkJobMixin
-from polyaxon.k8s.custom_resources.spark_job import get_spark_job_custom_resource
-from polyaxon.k8s.replica import ReplicaSpec
-from polyaxon.polyflow import V1CompiledOperation, V1Plugins, V1SparkJob, V1SparkReplica
-
-
-class SparkJobConverter(SparkJobMixin, BaseConverter):
-    def get_resource(
-        self,
-        compiled_operation: V1CompiledOperation,
-        artifacts_store: V1Connection,
-        connection_by_names: Dict[str, V1Connection],
-        secrets: Optional[Iterable[V1ConnectionResource]],
-        config_maps: Optional[Iterable[V1ConnectionResource]],
-        default_sa: Optional[str] = None,
-        default_auth: bool = False,
-    ) -> Dict:
-        job = compiled_operation.run  # type: V1SparkJob
-
-        def _get_replica(replica: Optional[V1SparkReplica]) -> Optional[ReplicaSpec]:
-            if not replica:
-                return None
-            return self.get_replica_resource(
-                plugins=plugins,
-                environment=replica.environment,
-                volumes=job.volumes or [],
-                init=replica.init or [],
-                sidecars=replica.sidecars or [],
-                container=replica.container,
-                artifacts_store=artifacts_store,
-                connections=job.connections or [],
-                connection_by_names=connection_by_names,
-                secrets=secrets,
-                config_maps=config_maps,
-                kv_env_vars=kv_env_vars,
-                default_sa=default_sa,
-                num_replicas=replica.replicas,
-            )
-
-        kv_env_vars = compiled_operation.get_env_io()
-        plugins = V1Plugins.get_or_create(
-            config=compiled_operation.plugins, auth=default_auth
-        )
-        executor = _get_replica(job.executor)
-        driver = _get_replica(job.driver)
-        labels = self.get_labels(version=pkg.VERSION, labels={})
-
-        return get_spark_job_custom_resource(
-            namespace=self.namespace,
-            resource_name=self.resource_name,
-            executor=executor,
-            driver=driver,
-            termination=compiled_operation.termination,
-            collect_logs=plugins.collect_logs,
-            sync_statuses=plugins.sync_statuses,
-            notifications=plugins.notifications,
-            labels=labels,
-            annotations=self.annotations,
-        )

polyaxon/k8s/custom_resources/spark_job.py

@@ -1,97 +0,0 @@
-from typing import Dict, List, Optional
-
-from polyaxon.k8s.converter.pod.spec import get_pod_spec, get_pod_template_spec
-from polyaxon.k8s.custom_resources.operation import get_operation_custom_object
-from polyaxon.k8s.custom_resources.setter import (
-    set_collect_logs,
-    set_notify,
-    set_sync_statuses,
-    set_termination,
-)
-from polyaxon.k8s.replica import ReplicaSpec
-from polyaxon.polyflow import V1Notification, V1Termination
-
-
-def get_spark_replicas_template(
-    namespace: str,
-    resource_name: str,
-    replica_name: str,
-    replica: Optional[ReplicaSpec],
-    labels: Dict[str, str],
-    annotations: Dict[str, str],
-    template_spec: Dict,
-):
-    if not replica:
-        return
-
-    metadata, pod_spec = get_pod_spec(
-        namespace=namespace,
-        main_container=replica.main_container,
-        sidecar_containers=replica.sidecar_containers,
-        init_containers=replica.init_containers,
-        resource_name=resource_name,
-        volumes=replica.volumes,
-        environment=replica.environment,
-        labels=labels,
-        annotations=annotations,
-    )
-
-    template_spec[replica_name] = {
-        "replicas": replica.num_replicas,
-        "restartPolicy": pod_spec.restart_policy or "Never",
-        "template": get_pod_template_spec(metadata=metadata, pod_spec=pod_spec),
-    }
-
-
-def get_spark_job_custom_resource(
-    resource_name: str,
-    namespace: str,
-    executor: Optional[ReplicaSpec],
-    driver: Optional[ReplicaSpec],
-    termination: V1Termination,
-    collect_logs: bool,
-    sync_statuses: bool,
-    notifications: List[V1Notification],
-    labels: Dict[str, str],
-    annotations: Dict[str, str],
-) -> Dict:
-    template_spec = {}
-
-    get_spark_replicas_template(
-        replica_name="Executor",
-        replica=executor,
-        namespace=namespace,
-        resource_name=resource_name,
-        labels=labels,
-        annotations=annotations,
-        template_spec=template_spec,
-    )
-    get_spark_replicas_template(
-        replica_name="Driver",
-        replica=driver,
-        namespace=namespace,
-        resource_name=resource_name,
-        labels=labels,
-        annotations=annotations,
-        template_spec=template_spec,
-    )
-    template_spec = {"replicaSpecs": template_spec}
-    custom_object = {"sparkJobSpec": template_spec}
-    custom_object = set_termination(
-        custom_object=custom_object, termination=termination
-    )
-    custom_object = set_collect_logs(
-        custom_object=custom_object, collect_logs=collect_logs
-    )
-    custom_object = set_sync_statuses(
-        custom_object=custom_object, sync_statuses=sync_statuses
-    )
-    custom_object = set_notify(custom_object=custom_object, notifications=notifications)
-
-    return get_operation_custom_object(
-        namespace=namespace,
-        resource_name=resource_name,
-        labels=labels,
-        annotations=annotations,
-        custom_object=custom_object,
-    )

polyaxon/polyflow/run/spark/__init__.py

@@ -1,2 +0,0 @@
-from polyaxon.polyflow.run.spark.replica import V1SparkReplica
-from polyaxon.polyflow.run.spark.spark import V1SparkDeploy, V1SparkJob, V1SparkType

polyaxon/polyflow/run/spark/replica.py

@@ -1,165 +0,0 @@
-from typing import List, Optional, Union
-
-from clipped.types.ref_or_obj import IntOrRef, RefField
-from pydantic import validator
-
-from polyaxon.k8s import k8s_schemas, k8s_validation
-from polyaxon.polyflow.environment import V1Environment
-from polyaxon.polyflow.init import V1Init
-from polyaxon.schemas.base import BaseSchemaModel
-
-
-class V1SparkReplica(BaseSchemaModel):
-    """Spark replica is the specification for a Spark executor or driver.
-
-    Args:
-        replicas: str, int
-        environment: [V1Environment](/docs/core/specification/environment/), optional
-        init: List[[V1Init](/docs/core/specification/init/)], optional
-        sidecars: List[[sidecar containers](/docs/core/specification/sidecars/)], optional
-        container: [Kubernetes Container](https://kubernetes.io/docs/concepts/containers/)
-
-    ## YAML usage
-
-    ```yaml
-    >>> executor/driver:
-    >>>   replicas:
-    >>>   environment:
-    >>>   init:
-    >>>   sidecars:
-    >>>   container:
-    ```
-
-    ## Python usage
-
-    ```python
-    >>> from polyaxon.polyflow import V1Environment, V1Init, V1SparkReplica
-    >>> from polyaxon.k8s import k8s_schemas
-    >>> replica = V1SparkReplica(
-    >>>     replicas=2,
-    >>>     environment=V1Environment(...),
-    >>>     init=[V1Init(...)],
-    >>>     sidecars=[k8s_schemas.V1Container(...)],
-    >>>     container=k8s_schemas.V1Container(...),
-    >>> )
-    ```
-
-    ## Fields
-
-    ### replicas
-
-    The number of replica (executor/driver) instances.
-
-    ```yaml
-    >>> executor:
-    >>>   replicas: 2
-    ```
-
-    ### environment
-
-    Optional [environment section](/docs/core/specification/environment/),
-    it provides a way to inject pod related information into the replica (executor/driver).
-
-    ```yaml
-    >>> driver:
-    >>>   environment:
-    >>>     labels:
-    >>>        key1: "label1"
-    >>>        key2: "label2"
-    >>>      annotations:
-    >>>        key1: "value1"
-    >>>        key2: "value2"
-    >>>      nodeSelector:
-    >>>        node_label: node_value
-    >>>      ...
-    >>>  ...
-    ```
-
-    ### init
-
-    A list of [init handlers and containers](/docs/core/specification/init/)
-    to resolve for the replica (executor/driver).
-
-    <blockquote class="light">
-    If you are referencing a connection it must be configured.
-    All referenced connections will be checked:
-
-     * If they are accessible in the context of the project of this run
-
-     * If the user running the operation can have access to those connections
-    </blockquote>
-
-    ```yaml
-    >>> executor:
-    >>>   init:
-    >>>     - artifacts:
-    >>>         dirs: ["path/on/the/default/artifacts/store"]
-    >>>     - connection: gcs-large-datasets
-    >>>       artifacts:
-    >>>         dirs: ["data"]
-    >>>       container:
-    >>>         resources:
-    >>>           requests:
-    >>>             memory: "256Mi"
-    >>>             cpu: "500m"
-    >>>     - container:
-    >>>       name: myapp-container
-    >>>       image: busybox:1.28
-    >>>       command: ['sh', '-c', 'echo custom init container']
-    ```
-
-    ### sidecars
-
-
-    A list of [sidecar containers](/docs/core/specification/sidecars/)
-    that will be used as sidecars.
-
-    ```yaml
-    >>> driver:
-    >>>   sidecars:
-    >>>     - name: sidecar2
-    >>>       image: busybox:1.28
-    >>>       command: ['sh', '-c', 'echo sidecar2']
-    >>>     - name: sidecar1
-    >>>       image: busybox:1.28
-    >>>       command: ['sh', '-c', 'echo sidecar1']
-    >>>       resources:
-    >>>         requests:
-    >>>           memory: "128Mi"
-    >>>           cpu: "500m"
-    ```
-
-    ### container
-
-    The [main Kubernetes Container](https://kubernetes.io/docs/concepts/containers/)
-    that will run your experiment training or data processing
-    logic for the replica (executor/driver).
-
-    ```yaml
-    >>> executor:
-    >>>   init:
-    >>>     - connection: my-code-repo
-    >>>   container:
-    >>>     name: tensorflow:2.1
-    >>>     command: ["python", "/plx-context/artifacts/my-code-repo/model.py"]
-    ```
-    """
-
-    _IDENTIFIER = "replica"
-    _SWAGGER_FIELDS = ["sidecars", "container"]
-
-    replicas: Optional[IntOrRef]
-    environment: Optional[Union[V1Environment, RefField]]
-    init: Optional[Union[List[V1Init], RefField]]
-    sidecars: Optional[Union[List[k8s_schemas.V1Container], RefField]]
-    container: Optional[Union[k8s_schemas.V1Container, RefField]]
-
-    @validator("sidecars", always=True, pre=True)
-    def validate_helper_containers(cls, v):
-        if not v:
-            return v
-        return [k8s_validation.validate_k8s_container(vi) for vi in v]
-
-    @validator("container", always=True, pre=True)
-    def validate_container(cls, v):
-        return k8s_validation.validate_k8s_container(v)