datatailr 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datatailr might be problematic. Click here for more details.
- {datatailr-0.1.2 → datatailr-0.1.3}/PKG-INFO +1 -1
- {datatailr-0.1.2 → datatailr-0.1.3}/pyproject.toml +1 -1
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/batch.py +10 -7
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr.egg-info/PKG-INFO +1 -1
- {datatailr-0.1.2 → datatailr-0.1.3}/LICENSE +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/README.md +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/setup.cfg +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/setup.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/__init__.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/acl.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/blob.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/build/__init__.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/build/image.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/dt_json.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/errors.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/group.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/logging.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/__init__.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/arguments_cache.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/base.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/batch_decorator.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/constants.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/scheduler/utils.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/user.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/utils.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/version.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr/wrapper.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr.egg-info/SOURCES.txt +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr.egg-info/dependency_links.txt +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr.egg-info/entry_points.txt +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr.egg-info/requires.txt +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/datatailr.egg-info/top_level.txt +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/sbin/run_job.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/test_module/__init__.py +0 -0
- {datatailr-0.1.2 → datatailr-0.1.3}/src/test_module/test_submodule.py +0 -0
|
@@ -14,7 +14,7 @@ import contextvars
|
|
|
14
14
|
import json
|
|
15
15
|
import os
|
|
16
16
|
from functools import reduce
|
|
17
|
-
from typing import List, Optional, Sequence, Set, Tuple, Union
|
|
17
|
+
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
|
|
18
18
|
|
|
19
19
|
from datatailr import Image
|
|
20
20
|
from datatailr.errors import BatchJobError
|
|
@@ -31,7 +31,6 @@ from datatailr.scheduler.base import (
|
|
|
31
31
|
from datatailr.scheduler.constants import DEFAULT_TASK_CPU, DEFAULT_TASK_MEMORY
|
|
32
32
|
from datatailr.utils import is_dt_installed
|
|
33
33
|
|
|
34
|
-
__BATCH_JOB_NAMES__ = {}
|
|
35
34
|
__DAG_CONTEXT__: contextvars.ContextVar = contextvars.ContextVar("dag_context")
|
|
36
35
|
logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
|
|
37
36
|
|
|
@@ -103,7 +102,7 @@ class BatchJob:
|
|
|
103
102
|
raise MissingDagError()
|
|
104
103
|
self.__id = dag.next_job_id
|
|
105
104
|
self.dag = dag
|
|
106
|
-
__BATCH_JOB_NAMES__[self.name] = self.__id
|
|
105
|
+
self.dag.__BATCH_JOB_NAMES__[self.name] = self.__id
|
|
107
106
|
self.dependencies = self.translate_dependencies()
|
|
108
107
|
assert all(
|
|
109
108
|
isinstance(dep, int) for dep in self.dependencies
|
|
@@ -116,10 +115,10 @@ class BatchJob:
|
|
|
116
115
|
|
|
117
116
|
:param name: The alias name to set.
|
|
118
117
|
"""
|
|
119
|
-
if name in __BATCH_JOB_NAMES__:
|
|
118
|
+
if name in self.dag.__BATCH_JOB_NAMES__:
|
|
120
119
|
raise DuplicateJobNameError(name)
|
|
121
|
-
assert __BATCH_JOB_NAMES__.pop(self.name) == self.__id
|
|
122
|
-
__BATCH_JOB_NAMES__[name] = self.__id
|
|
120
|
+
assert self.dag.__BATCH_JOB_NAMES__.pop(self.name) == self.__id
|
|
121
|
+
self.dag.__BATCH_JOB_NAMES__[name] = self.__id
|
|
123
122
|
self.name = name
|
|
124
123
|
return self
|
|
125
124
|
|
|
@@ -162,7 +161,10 @@ class BatchJob:
|
|
|
162
161
|
raise TypeError(f"Unsupported dependency type: {type(dep)}")
|
|
163
162
|
|
|
164
163
|
return set(
|
|
165
|
-
[
|
|
164
|
+
[
|
|
165
|
+
self.dag.__BATCH_JOB_NAMES__[get_dependency_name(dep)]
|
|
166
|
+
for dep in self.dependencies
|
|
167
|
+
]
|
|
166
168
|
)
|
|
167
169
|
|
|
168
170
|
def __add_dependency__(self, other):
|
|
@@ -240,6 +242,7 @@ class Batch(Job):
|
|
|
240
242
|
self.__jobs: List[BatchJob] = []
|
|
241
243
|
self._auto_run = False
|
|
242
244
|
self.__next_job_id = next_batch_job_id()
|
|
245
|
+
self.__BATCH_JOB_NAMES__: Dict[str, int] = {}
|
|
243
246
|
|
|
244
247
|
@property
|
|
245
248
|
def next_job_id(self):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|