datatailr 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

datatailr/scheduler/base.py

@@ -20,11 +20,18 @@ from typing import Callable, Optional, Tuple, Union
 
 from datatailr import ACL, Environment, User, dt__Job, is_dt_installed
 from datatailr.build.image import Image
+from datatailr.errors import BatchJobError
 from datatailr.logging import DatatailrLogger
 
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
 
 
+class RepoValidationError(BatchJobError):
+    def __init__(self, message: str):
+        super().__init__(message)
+        self.message = message
+
+
 class JobType(Enum):
     """
     Enum representing different types of DataTailr jobs.
@@ -220,7 +227,7 @@ class Job:
         if is_dt_installed():
             check_result = self.verify_repo_is_ready()
             if not check_result[0]:
-                return False, check_result[1]
+                raise RepoValidationError(check_result[1])
             logger.info(
                 f"Running job '{self.name}' in environment '{self.environment}' as '{self.run_as}'"
             )

datatailr/scheduler/batch.py

@@ -14,7 +14,7 @@ import contextvars
 import json
 import os
 from functools import reduce
-from typing import List, Optional, Sequence, Set, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
 from datatailr import Image
 from datatailr.errors import BatchJobError
@@ -31,7 +31,6 @@ from datatailr.scheduler.base import (
 from datatailr.scheduler.constants import DEFAULT_TASK_CPU, DEFAULT_TASK_MEMORY
 from datatailr.utils import is_dt_installed
 
-__BATCH_JOB_NAMES__ = {}
 __DAG_CONTEXT__: contextvars.ContextVar = contextvars.ContextVar("dag_context")
 logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
 
@@ -103,7 +102,7 @@ class BatchJob:
             raise MissingDagError()
         self.__id = dag.next_job_id
         self.dag = dag
-        __BATCH_JOB_NAMES__[self.name] = self.__id
+        self.dag.__BATCH_JOB_NAMES__[self.name] = self.__id
         self.dependencies = self.translate_dependencies()
         assert all(
             isinstance(dep, int) for dep in self.dependencies
@@ -116,10 +115,10 @@ class BatchJob:
 
         :param name: The alias name to set.
         """
-        if name in __BATCH_JOB_NAMES__:
+        if name in self.dag.__BATCH_JOB_NAMES__:
             raise DuplicateJobNameError(name)
-        assert __BATCH_JOB_NAMES__.pop(self.name) == self.__id
-        __BATCH_JOB_NAMES__[name] = self.__id
+        assert self.dag.__BATCH_JOB_NAMES__.pop(self.name) == self.__id
+        self.dag.__BATCH_JOB_NAMES__[name] = self.__id
         self.name = name
         return self
 
@@ -162,7 +161,10 @@ class BatchJob:
                 raise TypeError(f"Unsupported dependency type: {type(dep)}")
 
         return set(
-            [__BATCH_JOB_NAMES__[get_dependency_name(dep)] for dep in self.dependencies]
+            [
+                self.dag.__BATCH_JOB_NAMES__[get_dependency_name(dep)]
+                for dep in self.dependencies
+            ]
         )
 
     def __add_dependency__(self, other):
@@ -240,6 +242,7 @@ class Batch(Job):
         self.__jobs: List[BatchJob] = []
         self._auto_run = False
         self.__next_job_id = next_batch_job_id()
+        self.__BATCH_JOB_NAMES__: Dict[str, int] = {}
 
     @property
     def next_job_id(self):
@@ -285,11 +288,10 @@ class Batch(Job):
         return json.dumps(self.to_dict(), indent=4)
 
     def __repr__(self):
-        jobs_repr = ", ".join(repr(job) for job in self.__jobs)
         return (
             f"Batch(name={self.name}, environment={self.environment}, "
             f"run_as={self.run_as}, resources={self.resources}, "
-            f"acl={self.acl}, jobs=[{jobs_repr}])"
+            f"acl={self.acl}, {len(self.__jobs)} jobs)"
         )
 
     def set_autorun(self, auto_run):

{datatailr-0.1.2.dist-info → datatailr-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datatailr
-Version: 0.1.2
+Version: 0.1.4
 Summary: The datatailr package
 Author-email: Datatailr <pypi@datatailr.com>
 License-Expression: MIT

{datatailr-0.1.2.dist-info → datatailr-0.1.4.dist-info}/RECORD

@@ -14,16 +14,16 @@ datatailr/build/image.py,sha256=P2MiGxpzuZ6hOm9JubkLoOq-RdzKYnXbKJHiryIbJeA,3103
 datatailr/sbin/run_job.py,sha256=B3H3UI3zpll7zVm7lZLsfPtnRlA6jjk8s_D_w89pvC4,2175
 datatailr/scheduler/__init__.py,sha256=YtCnv9vuX-EPGr3WBXvXJIlLsZ94mW1dBzI6h7Yjcu0,1009
 datatailr/scheduler/arguments_cache.py,sha256=-JnAWXfHMSSkYJx8iYt9JgvdPgR-1Z8DcN5Kvcx9Amo,4574
-datatailr/scheduler/base.py,sha256=trsW3ETK0X00qoYYRHQrkfxoHEWhwgTrsJFVrhu6saY,7623
-datatailr/scheduler/batch.py,sha256=daxFRl5894Hb3cKEm_pubXchNwN8qjVuFi0G-zMj-kI,10712
+datatailr/scheduler/base.py,sha256=IRduJjhng4Uuv3Y9vDqUNTbs636AgH2ZISWRbIP6LqQ,7826
+datatailr/scheduler/batch.py,sha256=BlZq609thXM0skZiPZjGHLO9WozOq8T3SiErpFPU4Zg,10779
 datatailr/scheduler/batch_decorator.py,sha256=o6DAEODwsfYx9uYwjnaJZ9iUmqbFH2wMh-jz_L5bgNo,4598
 datatailr/scheduler/constants.py,sha256=ISG5uMnVPbGbjaaulU0xdmSggnd-DMr9ed0WTAZSUmU,604
 datatailr/scheduler/utils.py,sha256=YHtAc5sCwfgiClr5G6R3hfAjdlrFdnNW2l-3XwPZLXM,1070
-datatailr-0.1.2.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
+datatailr-0.1.4.dist-info/licenses/LICENSE,sha256=ikKP4_O-UD_b8FuNdKmbzTb6odd0JX085ZW_FAPN3VI,1066
 test_module/__init__.py,sha256=OF9XaL3RKdbWD5Ug4L-ufLqbykSt_rTK6gqZr4uBJ8g,576
 test_module/test_submodule.py,sha256=O07fFbzJEy5rf1vdlYzPvs8v0IxHFg-CaYMqaHkskbc,1294
-datatailr-0.1.2.dist-info/METADATA,sha256=yfNet0SvA-XFRMQa8YYFJRduScQFdB3j4wVEz7XU3oQ,774
-datatailr-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datatailr-0.1.2.dist-info/entry_points.txt,sha256=4lNE9VXvztJdIQsODI308v9FRzkVgmMu-VKGEceNxJs,59
-datatailr-0.1.2.dist-info/top_level.txt,sha256=UZOKaWS1kZGGwV7hP476-EcSUJBNspxVSSp9WqtORzk,22
-datatailr-0.1.2.dist-info/RECORD,,
+datatailr-0.1.4.dist-info/METADATA,sha256=8dyH2odt79utbgrPRP6--_aQRbWyv1ubfV6F4PSVpLM,774
+datatailr-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datatailr-0.1.4.dist-info/entry_points.txt,sha256=4lNE9VXvztJdIQsODI308v9FRzkVgmMu-VKGEceNxJs,59
+datatailr-0.1.4.dist-info/top_level.txt,sha256=UZOKaWS1kZGGwV7hP476-EcSUJBNspxVSSp9WqtORzk,22
+datatailr-0.1.4.dist-info/RECORD,,