esd-services-api-client 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

esd_services_api_client/_version.py

@@ -1 +1 @@
-__version__ = '1.0.1'
+__version__ = '1.0.3'

esd_services_api_client/beast/_connector.py

@@ -25,7 +25,11 @@ import backoff
 from adapta.utils import doze, session_with_retries
 from urllib3.exceptions import ProtocolError, HTTPError
 
-from esd_services_api_client.beast._models import JobRequest, BeastJobParams
+from esd_services_api_client.beast._models import (
+    JobRequest,
+    BeastJobParams,
+    SparkSubmissionConfiguration,
+)
 from esd_services_api_client.boxer import BoxerTokenAuth
 
 
@@ -172,6 +176,7 @@ class BeastConnector:
                 overwrite=job_params.overwrite_outputs,
                 extra_args=prepared_arguments,
                 client_tag=job_params.client_tag,
+                expected_parallelism=job_params.expected_parallelism,
             )
 
             (request_id, request_lifecycle) = self._submit(submit_request, job_name)
@@ -245,8 +250,25 @@ class BeastConnector:
                 overwrite=job_params.overwrite_outputs,
                 extra_args=prepared_arguments,
                 client_tag=job_params.client_tag,
+                expected_parallelism=job_params.expected_parallelism,
             )
 
             request_id, _ = self._submit(submit_request, job_name)
 
         return request_id
+
+    def get_configuration(
+        self, configuration_name: str
+    ) -> Optional[SparkSubmissionConfiguration]:
+        """
+          Returns a deployed SparkJob configuration.
+        :param configuration_name: Name of the configuration to find
+        :return: A SparkSubmissionConfiguration object, if found, or None
+        """
+        response = self.http.get(f"{self.base_url}/job/deployed/{configuration_name}")
+        if response.status_code == 404:
+            return None
+        if not response.ok:
+            response.raise_for_status()
+
+        return SparkSubmissionConfiguration.from_dict(response.json())

esd_services_api_client/beast/_models.py

@@ -18,7 +18,7 @@
 
 import os
 from dataclasses import dataclass, field
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Optional
 from warnings import warn
 
 from cryptography.fernet import Fernet
@@ -88,6 +88,7 @@ class JobRequest(DataClassJsonMixin):
     overwrite: bool
     extra_args: Dict[str, str]
     client_tag: str
+    expected_parallelism: Optional[int]
 
 
 class ArgumentValue:
@@ -182,3 +183,61 @@ class BeastJobParams:
         },
         default=False,
     )
+    expected_parallelism: Optional[int] = field(
+        metadata={
+            "description": "Expected number of executors for this job (overrides the deployed value)."
+        },
+        default=None,
+    )
+
+
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class RequestDebugMode(DataClassJsonMixin):
+    """
+    Debug mode config.
+    """
+
+    event_log_location: str
+    max_size_per_file: str
+
+
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class V1TypedLocalObjectReference(DataClassJsonMixin):
+    """
+    Reference to another SparkJob.
+    """
+
+    api_group: str
+    kind: str
+    name: str
+
+
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class SparkSubmissionConfiguration(DataClassJsonMixin):
+    """
+    Configuration CRD used by Beast to run Spark apps.
+    """
+
+    root_path: str
+    project_name: str
+    version: str
+    runnable: str
+    execution_group: str
+    expected_parallelism: Optional[int]
+    flexible_driver: Optional[bool]
+    additional_driver_node_tolerations: Dict[str, str]
+    max_runtime_hours: Optional[int]
+    debug_mode: RequestDebugMode
+    submission_mode: Optional[str]
+    extended_code_mount: Optional[bool]
+    submission_job_template: str
+    executor_spec_template: str
+    driver_job_retries: Optional[int]
+    default_arguments: Dict[str, str]
+    inputs: List[JobSocket]
+    outputs: List[JobSocket]
+    overwrite: bool
+    base_submission_configuration_ref: V1TypedLocalObjectReference

{esd_services_api_client-1.0.1.dist-info → esd_services_api_client-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esd-services-api-client
-Version: 1.0.1
+Version: 1.0.3
 Summary: Python clients for ESD services
 Home-page: https://github.com/SneaksAndData/esd-services-api-client
 License: Apache 2.0

{esd_services_api_client-1.0.1.dist-info → esd_services_api_client-1.0.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
 esd_services_api_client/__init__.py,sha256=rP0njtEgVSMm-sOVayVfcRUrrubl4lme7HI2zS678Lo,598
-esd_services_api_client/_version.py,sha256=3KXfAcA5rzzQXMtfwpHphgbJNZdA0XWaW96kdWVSZJw,22
+esd_services_api_client/_version.py,sha256=qKK4BMzdt84HiCorJPWqgEfa_DbJeuGx_bIPrDui2vc,22
 esd_services_api_client/beast/__init__.py,sha256=XU4thkgkY6ZvT4yyDpaqeiIg3yotfcyJEewpo8iica4,743
 esd_services_api_client/beast/_auth.py,sha256=V9m75EaFP8U_Ykf4DE5SWSSUrH8_UP_ugrmayJbwdmw,2666
-esd_services_api_client/beast/_connector.py,sha256=_Wkasgn_D-SDSBifpbDjiOuxiUl5Hzim5x7Vn8r_ysM,9015
-esd_services_api_client/beast/_models.py,sha256=3spwBrEsocHHfuxDXEYFKS2LEJvCIFQp-SbLOLObQvQ,5456
+esd_services_api_client/beast/_connector.py,sha256=DgoX2tU4S37LKKUPvyq_YhuP-e8dDsjp0dWfwNRuaMU,9828
+esd_services_api_client/beast/_models.py,sha256=hwvzqnw8H7N-6zDrdQftI7VUhZS_idHKnjPe9OXxIoA,6976
 esd_services_api_client/boxer/README.md,sha256=U8kXXtJFi1w0woQ4F_UOdDZiQl6-K7zFmQEUV4EktfM,2983
 esd_services_api_client/boxer/__init__.py,sha256=OYsWvdnLan0kmjUcH4I2-m1rbPeARKp5iqhp8uyudPk,780
 esd_services_api_client/boxer/_auth.py,sha256=vA7T9y0oZV2f17UWQ2or9CK8vAsNnHB10G5HNQe1l1I,7440
@@ -16,7 +16,7 @@ esd_services_api_client/crystal/__init__.py,sha256=afSGQRkDic0ECsJfgu3b291kX8CyU
 esd_services_api_client/crystal/_api_versions.py,sha256=2BMiQRS0D8IEpWCCys3dge5alVBRCZrOuCR1QAn8UIM,832
 esd_services_api_client/crystal/_connector.py,sha256=lWWKhVKvaFtLTsl6abtydFH4rsMoCATD_IMpVOARyl0,12150
 esd_services_api_client/crystal/_models.py,sha256=eRhGAl8LjglCyIFwf1bcFBhjbpSuRYucuF2LO388L2E,4025
-esd_services_api_client-1.0.1.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
-esd_services_api_client-1.0.1.dist-info/METADATA,sha256=-BOXKkPdl6wMoL06oTmzz4TG2MMIbJixTOiAK980Sio,1077
-esd_services_api_client-1.0.1.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-esd_services_api_client-1.0.1.dist-info/RECORD,,
+esd_services_api_client-1.0.3.dist-info/LICENSE,sha256=0gS6zXsPp8qZhzi1xaGCIYPzb_0e8on7HCeFJe8fOpw,10693
+esd_services_api_client-1.0.3.dist-info/METADATA,sha256=pPSDzzsXCLyypu2I6ZassMsyNcqP1CQapXkSk6ap22Q,1077
+esd_services_api_client-1.0.3.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+esd_services_api_client-1.0.3.dist-info/RECORD,,