runnable 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

extensions/job_executor/k8s.py

@@ -101,23 +101,40 @@ class HostPath(BaseModel):
     path: str
 
 
-class Volume(BaseModel):
+class HostPathVolume(BaseModel):
     name: str
     host_path: HostPath
 
 
-class TemplateSpec(BaseModel):
+class PVCClaim(BaseModel):
+    claim_name: str
+
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        from_attributes=True,
+    )
+
+
+class PVCVolume(BaseModel):
+    name: str
+    persistent_volume_claim: PVCClaim
+
+
+class K8sTemplateSpec(BaseModel):
     active_deadline_seconds: int = Field(default=60 * 60 * 2)  # 2 hours
     node_selector: Optional[dict[str, str]] = None
     tolerations: Optional[list[dict[str, str]]] = None
-    volumes: Optional[list[Volume]] = Field(default_factory=lambda: [])
+    volumes: Optional[list[HostPathVolume | PVCVolume]] = Field(
+        default_factory=lambda: []
+    )
     service_account_name: Optional[str] = "default"
     restart_policy: RestartPolicy = RestartPolicy.NEVER
     container: Container
 
 
-class Template(BaseModel):
-    spec: TemplateSpec
+class K8sTemplate(BaseModel):
+    spec: K8sTemplateSpec
     metadata: Optional[ObjectMetaData] = None
 
 
@@ -125,32 +142,25 @@ class Spec(BaseModel):
     active_deadline_seconds: Optional[int] = Field(default=60 * 60 * 2)  # 2 hours
     backoff_limit: int = 6
     selector: Optional[LabelSelector] = None
-    template: Template
+    template: K8sTemplate
     ttl_seconds_after_finished: Optional[int] = Field(default=60 * 60 * 24)  # 24 hours
 
 
-class K8sJobExecutor(GenericJobExecutor):
+class GenericK8sJobExecutor(GenericJobExecutor):
     service_name: str = "k8s-job"
     config_path: Optional[str] = None
     job_spec: Spec
     mock: bool = False
-
-    # The location the mount of .run_log_store is mounted to in minikube
-    # ensure that minikube mount $HOME/workspace/runnable/.run_log_store:/volume/run_logs is executed first
-    # $HOME/workspace/runnable/.catalog:/volume/catalog
-    # Ensure that the docker build is done with eval $(minikube docker-env)
-    mini_k8s_run_log_location: str = Field(default="/volume/run_logs/")
-    mini_k8s_catalog_location: str = Field(default="/volume/catalog/")
+    namespace: str = Field(default="default")
 
     _is_local: bool = PrivateAttr(default=False)
+    _volume_mounts: list[VolumeMount] = PrivateAttr(default_factory=lambda: [])
+    _volumes: list[HostPathVolume | PVCVolume] = PrivateAttr(default_factory=lambda: [])
 
     _container_log_location: str = PrivateAttr(default="/tmp/run_logs/")
     _container_catalog_location: str = PrivateAttr(default="/tmp/catalog/")
     _container_secrets_location: str = PrivateAttr(default="/tmp/dotenv")
 
-    _volumes: list[Volume] = []
-    _volume_mounts: list[VolumeMount] = []
-
     model_config = ConfigDict(
         alias_generator=to_camel,
         populate_by_name=True,
@@ -287,14 +297,17 @@ class K8sJobExecutor(GenericJobExecutor):
         )
 
         logger.info(f"Submitting job: {job.__dict__}")
+        if self.mock:
+            print(job.__dict__)
+            return
 
         try:
             k8s_batch = self._client.BatchV1Api()
             response = k8s_batch.create_namespaced_job(
                 body=job,
-                namespace="default",
                 _preload_content=False,
                 pretty=True,
+                namespace=self.namespace,
             )
             logger.debug(f"Kubernetes job response: {response}")
         except Exception as e:
@@ -302,6 +315,43 @@ class K8sJobExecutor(GenericJobExecutor):
             print(e)
             raise
 
+    def _create_volumes(self): ...
+
+    def _use_volumes(self):
+        match self._context.run_log_store.service_name:
+            case "file-system":
+                self._context.run_log_store.log_folder = self._container_log_location
+            case "chunked-fs":
+                self._context.run_log_store.log_folder = self._container_log_location
+
+        match self._context.catalog_handler.service_name:
+            case "file-system":
+                self._context.catalog_handler.catalog_location = (
+                    self._container_catalog_location
+                )
+
+
+class MiniK8sJobExecutor(GenericK8sJobExecutor):
+    service_name: str = "k8s-job"
+    config_path: Optional[str] = None
+    job_spec: Spec
+    mock: bool = False
+
+    # The location the mount of .run_log_store is mounted to in minikube
+    # ensure that minikube mount $HOME/workspace/runnable/.run_log_store:/volume/run_logs is executed first
+    # $HOME/workspace/runnable/.catalog:/volume/catalog
+    # Ensure that the docker build is done with eval $(minikube docker-env)
+    mini_k8s_run_log_location: str = Field(default="/volume/run_logs/")
+    mini_k8s_catalog_location: str = Field(default="/volume/catalog/")
+
+    _is_local: bool = PrivateAttr(default=False)
+
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        from_attributes=True,
+    )
+
     def _create_volumes(self):
         match self._context.run_log_store.service_name:
             case "file-system":
@@ -311,7 +361,7 @@ class K8sJobExecutor(GenericJobExecutor):
                     # You then are creating a volume that is mounted to /tmp/run_logs in the container
                     # You are then referring to it.
                     # https://stackoverflow.com/questions/57411456/minikube-mounted-host-folders-are-not-working
-                    Volume(
+                    HostPathVolume(
                         name="run-logs",
                         host_path=HostPath(path=self.mini_k8s_run_log_location),
                     )
@@ -323,7 +373,7 @@ class K8sJobExecutor(GenericJobExecutor):
                 )
             case "chunked-fs":
                 self._volumes.append(
-                    Volume(
+                    HostPathVolume(
                         name="run-logs",
                         host_path=HostPath(path=self.mini_k8s_run_log_location),
                     )
@@ -337,7 +387,7 @@ class K8sJobExecutor(GenericJobExecutor):
         match self._context.catalog_handler.service_name:
             case "file-system":
                 self._volumes.append(
-                    Volume(
+                    HostPathVolume(
                         name="catalog",
                         host_path=HostPath(path=self.mini_k8s_catalog_location),
                     )
@@ -348,15 +398,87 @@ class K8sJobExecutor(GenericJobExecutor):
                     )
                 )
 
-    def _use_volumes(self):
+
+class K8sJobExecutor(GenericK8sJobExecutor):
+    service_name: str = "k8s-job"
+    config_path: Optional[str] = None
+    job_spec: Spec
+    mock: bool = False
+    pvc_claim_name: str
+
+    # change the spec to pull image if not present
+    def model_post_init(self, __context):
+        self.job_spec.template.spec.container.image_pull_policy = ImagePullPolicy.ALWAYS
+
+    _is_local: bool = PrivateAttr(default=False)
+
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        from_attributes=True,
+    )
+
+    def execute_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        self._use_volumes()
+        self._set_up_run_log()
+
+        job_log = self._context.run_log_store.create_job_log()
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+
+        job_log = self._context.run_log_store.get_job_log(run_id=self._context.run_id)
+
+        attempt_log = job.execute_command(
+            attempt_number=self.step_attempt_number,
+            mock=self.mock,
+        )
+
+        job_log.status = attempt_log.status
+        job_log.attempts.append(attempt_log)
+
+        data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
+            catalog_settings=catalog_settings
+        )
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
+
+        job_log.add_data_catalogs(data_catalogs_put or [])
+
+        console.print("Summary of job")
+        console.print(job_log.get_summary())
+
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+
+    def _create_volumes(self):
+        self._volumes.append(
+            PVCVolume(
+                name=self.pvc_claim_name,
+                persistent_volume_claim=PVCClaim(claim_name=self.pvc_claim_name),
+            )
+        )
         match self._context.run_log_store.service_name:
             case "file-system":
-                self._context.run_log_store.log_folder = self._container_log_location
+                self._volume_mounts.append(
+                    VolumeMount(
+                        name=self.pvc_claim_name,
+                        mount_path=self._container_log_location,
+                    )
+                )
             case "chunked-fs":
-                self._context.run_log_store.log_folder = self._container_log_location
+                self._volume_mounts.append(
+                    VolumeMount(
+                        name=self.pvc_claim_name,
+                        mount_path=self._container_log_location,
+                    )
+                )
 
         match self._context.catalog_handler.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
-                    self._container_catalog_location
+                self._volume_mounts.append(
+                    VolumeMount(
+                        name=self.pvc_claim_name,
+                        mount_path=self._container_catalog_location,
+                    )
                 )

extensions/pipeline_executor/local_container.py

@@ -268,7 +268,6 @@ class LocalContainerExecutor(GenericPipelineExecutor):
                     f"Please provide a docker_image using executor_config of the step {node.name} or at global config"
                 )
 
-            # TODO: Should consider using getpass.getuser() when running the docker container? Volume permissions
             container = client.containers.create(
                 image=docker_image,
                 command=command,

runnable/__init__.py

@@ -1,5 +1,6 @@
 # ruff: noqa
 
+
 import logging
 import os
 from logging.config import dictConfig

runnable/catalog.py

@@ -10,8 +10,6 @@ from runnable.datastore import DataCatalog
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
-# TODO: Should ** be allowed as glob pattern as it can potentially copy everything to catalog
-
 
 def is_catalog_out_of_sync(
     catalog, synced_catalogs=Optional[List[DataCatalog]]
@@ -170,3 +168,4 @@ class DoNothingCatalog(BaseCatalog):
         Does nothing
         """
         logger.info("Using a do-nothing catalog, doing nothing while sync between runs")
+        logger.info("Using a do-nothing catalog, doing nothing while sync between runs")

runnable/entrypoints.py

@@ -16,9 +16,6 @@ from runnable.executor import BaseJobExecutor, BasePipelineExecutor
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-print("")  # removes the buffer print
-
-
 def get_default_configs() -> RunnableConfig:
     """
     User can provide extensions as part of their code base, runnable-config.yaml provides the place to put them.
@@ -128,11 +125,10 @@ def prepare_configurations(
             "job-executor", None
         )  # type: ignore
         if not job_executor_config:
-            executor_config = cast(
+            job_executor_config = cast(
                 ServiceConfig,
                 runnable_defaults.get("job-executor", defaults.DEFAULT_JOB_EXECUTOR),
             )
-
         assert job_executor_config, "Job executor is not provided"
         configured_executor = utils.get_provider_by_name_and_type(
             "job_executor", job_executor_config

runnable/executor.py

@@ -11,9 +11,9 @@ import runnable.context as context
 from runnable import defaults
 from runnable.datastore import DataCatalog, JobLog, StepLog
 from runnable.defaults import TypeMapVariable
-from runnable.graph import Graph
 
 if TYPE_CHECKING:  # pragma: no cover
+    from runnable.graph import Graph
     from runnable.nodes import BaseNode
     from runnable.tasks import BaseTaskType
 

runnable/parameters.py

@@ -15,8 +15,6 @@ from runnable.utils import remove_prefix
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
-# TODO: Revisit this, it might be a bit too complicated than required
-
 
 def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
     """
@@ -50,13 +48,6 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
     return parameters
 
 
-def serialize_parameter_as_str(value: Any) -> str:
-    if isinstance(value, BaseModel):
-        return json.dumps(value.model_dump())
-
-    return json.dumps(value)
-
-
 def filter_arguments_for_func(
     func: Callable[..., Any],
     params: Dict[str, Any],

runnable/utils.py

@@ -17,7 +17,7 @@ from ruamel.yaml import YAML
 from stevedore import driver
 
 import runnable.context as context
-from runnable import defaults, names
+from runnable import console, defaults, names
 from runnable.defaults import TypeMapVariable
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -176,7 +176,7 @@ def is_a_git_repo() -> bool:
         logger.info("Found the code to be git versioned")
         return True
     except BaseException:  # pylint: disable=W0702
-        logger.error("No git repo found, unsafe hash")
+        console.print("Not a git repo", style="bold red")
 
     return False
 
@@ -195,27 +195,7 @@ def get_current_code_commit() -> Union[str, None]:
         logger.info("Found the git commit to be: %s", label)
         return label
     except BaseException:  # pylint: disable=W0702
-        logger.exception("Error getting git hash")
-        raise
-
-
-def archive_git_tracked(name: str):
-    """Generate a git archive of the tracked files.
-
-    Args:
-        name (str): The name to give the archive
-
-    Raises:
-        Exception: If its not a git repo
-    """
-    command = f"git archive -v -o {name}.tar.gz --format=tar.gz HEAD"
-
-    if not is_a_git_repo():
-        raise Exception("Not a git repo")
-    try:
-        subprocess.check_output(command.split()).strip().decode("utf-8")
-    except BaseException:  # pylint: disable=W0702
-        logger.exception("Error archiving repo")
+        console.print("Not a git repo, error getting hash", style="bold red")
         raise
 
 
@@ -234,7 +214,7 @@ def is_git_clean() -> Tuple[bool, Union[None, str]]:
             return True, None
         return False, label
     except BaseException:  # pylint: disable=W0702
-        logger.exception("Error checking if the code is git clean")
+        console.print("Not a git repo, not clean", style="bold red")
 
     return False, None
 
@@ -253,7 +233,7 @@ def get_git_remote() -> Union[str, None]:
         logger.info("Found the git remote to be: %s", label)
         return label
     except BaseException:  # pylint: disable=W0702
-        logger.exception("Error getting git remote")
+        console.print("Not a git repo, no remote", style="bold red")
         raise
 
 

{runnable-0.18.0.dist-info → runnable-0.19.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.18.0
+Version: 0.19.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.18.0.dist-info → runnable-0.19.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ extensions/catalog/file_system.py,sha256=VZEUx4X-GDSM8rJ_2kiCOyw1eek3roN0CiSB8wd
 extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
 extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/job_executor/__init__.py,sha256=HINaPjBWz04Ni7GqhuDLi0lS0-gYzq52HcOioYueYJE,5513
-extensions/job_executor/k8s.py,sha256=BHICgJ_TT0JoGKPfX5GH_DpflKtIcSHp8I9i1ycSRZo,11516
+extensions/job_executor/k8s.py,sha256=huffLcDIQgTU-Qsz05CekrIwqFZLtFRGFlQAplwPRoE,15258
 extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
 extensions/job_executor/local.py,sha256=8ebu4TKo6FnFiUflil6fmE7Pk8eSoe1fHNwX8YwI1BQ,1865
 extensions/job_executor/local_container.py,sha256=7G2ARgoPwXbpNPgd5UDxWZqU2ABPIAk7bkNQkC4cNBQ,6585
@@ -18,7 +18,7 @@ extensions/pipeline_executor/__init__.py,sha256=YnKILiy-SxfnG3rYUoinjh1lfkuAF5QX
 extensions/pipeline_executor/argo.py,sha256=ClfuU_Of_2f5mvqVgY1QQwwJwXHB0LbzwNArG1x2Axc,44666
 extensions/pipeline_executor/argo_specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
 extensions/pipeline_executor/local.py,sha256=H8s6AdML_9_f-vdGG_6k0y9FbLqAqvA1S_7xMNyARzY,1946
-extensions/pipeline_executor/local_container.py,sha256=hqxLkxBOzk8P_PYnnZw5Ve9K0ztoyedv4kUVLkz8vVY,13967
+extensions/pipeline_executor/local_container.py,sha256=UCap8wCbHrtTN5acECBBkvcXkA3SXtrAOGW88JT7ofw,13853
 extensions/pipeline_executor/mocked.py,sha256=SuObJ6Myt7p8duW8sylIp1cYIAnFutsJW1avWaOUY3c,5798
 extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZQ5V_R7VYbZ-ido,291
 extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
@@ -33,26 +33,26 @@ extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 extensions/secrets/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/secrets/dotenv.py,sha256=FbYYd_pVuJuVuIDIvXbzKuSSQ9GPq7xJXTDbJMTQbhM,1583
 extensions/secrets/pyproject.toml,sha256=mLJNImNcBlbLKHh-0ugVWT9V83R4RibyyYDtBCSqVF4,282
-runnable/__init__.py,sha256=KqpLDTD1CfdEj2aDyEkSn2KW-_83qyrRrrWLc5lZVM4,624
-runnable/catalog.py,sha256=MiEmb-18liAKmgeMdDF41VVn0ZEAVLP8hR33oacQ1zs,4930
+runnable/__init__.py,sha256=fYkOrbsb-E1rGkrof7kOJ3KboTFH-HriGa-8npn4-50,625
+runnable/catalog.py,sha256=b9N40kTv1IBidzlWjkHcBGyYhq6qIDHZfBuFenzjsMI,4924
 runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
 runnable/context.py,sha256=by5uepmuCP0dmM9BmsliXihSes5QEFejwAsmekcqylE,1388
 runnable/datastore.py,sha256=9y5enzn6AXLHLdwvgkdjGPrBkVlrcjfbaAHsst-lJzg,32466
 runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
-runnable/entrypoints.py,sha256=67gPBiIIS4Kd9g6LdoGCraRJPda8K1i7Lp7XcD2iY5k,18913
+runnable/entrypoints.py,sha256=P958nFz5WAsgTwd9sW04Q30vtjweYpr3rPsHVY4gh2U,18876
 runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
-runnable/executor.py,sha256=Rafu9EECrNq1LBkJmS6KYCekchP5ufrR04mHWG-JzqQ,15543
+runnable/executor.py,sha256=ZPpfKwjDJnta03M2cWIINXcwke2ZDVc_QrIw7kwpHDQ,15547
 runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
 runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
 runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
-runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
+runnable/parameters.py,sha256=LyQb1d0SaFeI4PJ_yDYt9wArm9ThSPASWb36TwIdDUs,5213
 runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
 runnable/sdk.py,sha256=xN5F4XX8r5wCN131kgN2xG7MkNm0bSGJ3Ukw8prHYJ8,31444
 runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
 runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
-runnable/utils.py,sha256=wqyN7lMW56cBqyE59iDE6_i2HXPkvEUCQ-66UQnIwTA,19993
-runnable-0.18.0.dist-info/METADATA,sha256=lBeCXTFGeVWpbxnOj2pZY6hTSNyB2Q5OWx5kVAF252s,9945
-runnable-0.18.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.18.0.dist-info/entry_points.txt,sha256=I92DYldRrCb9HCsoum8GjC2UsQrWpuw2kawXTZpkIz4,1559
-runnable-0.18.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.18.0.dist-info/RECORD,,
+runnable/utils.py,sha256=Kwf54tHMVXYK7MCmvAi_FG08U_bHDKIQO-HDpM9X0QI,19500
+runnable-0.19.0.dist-info/METADATA,sha256=tgXvJ1oDrhC59zVORaSTyFKqqLDp785wKWu55NUwSOE,9945
+runnable-0.19.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.19.0.dist-info/entry_points.txt,sha256=seek5WVGvwYALm8lZ0TfPXwG5NaCeUKjU8urF8k3gvY,1621
+runnable-0.19.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.19.0.dist-info/RECORD,,

{runnable-0.18.0.dist-info → runnable-0.19.0.dist-info}/entry_points.txt

@@ -9,6 +9,7 @@ file-system = extensions.catalog.file_system:FileSystemCatalog
 k8s-job = extensions.job_executor.k8s:K8sJobExecutor
 local = extensions.job_executor.local:LocalJobExecutor
 local-container = extensions.job_executor.local_container:LocalContainerJobExecutor
+mini-k8s-job = extensions.job_executor.k8s:MiniK8sJobExecutor
 
 [nodes]
 dag = extensions.nodes.nodes:DagNode