pipeloom-databricks 0.6.0__tar.gz → 0.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/PKG-INFO +1 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/pyproject.toml +1 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/__init__.py +6 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/sdk.py +8 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/finalize_states.py +2 -2
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/jobs_model.py +2 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/render.py +2 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/repair.py +6 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/task.py +39 -8
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/fake.py +9 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_folder.json +7 -4
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_folder.yml +4 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_tag.json +9 -5
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_tag.yml +5 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/job_cluster.json +11 -6
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/job_cluster.yml +6 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/serverless.json +15 -8
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/serverless.yml +8 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/submit.json +14 -7
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_finalize_states.py +6 -3
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_repair.py +11 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_sdk_client.py +4 -1
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_task.py +23 -4
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/.gitignore +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/LICENSE +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/NOTICE +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/README.md +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/__init__.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/api.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/bundle.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/cli.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/connect.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/deploy.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/grouping.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/keys.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/options.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/py.typed +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/run.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/secrets.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/serialize.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/spec.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/task_values.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/__init__.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/jobsim.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/unity.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/upload.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/conftest.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_bundle.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_connect.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_deploy.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_deploy_cli.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_grouping.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_jobsim.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_keys.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_options.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_render.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_run.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_run_cli.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_secrets.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_spec_import.py +0 -0
- {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_unity.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pipeloom-databricks
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: Databricks renderer for pipeloom: a compiled plan as Jobs settings JSON and bundle YAML.
|
|
5
5
|
Project-URL: Repository, https://github.com/emanuel-luis/pipeloom
|
|
6
6
|
Project-URL: Documentation, https://emanuel-luis.github.io/pipeloom/
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pipeloom-databricks"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.1"
|
|
8
8
|
description = "Databricks renderer for pipeloom: a compiled plan as Jobs settings JSON and bundle YAML."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/__init__.py
RENAMED
|
@@ -90,7 +90,12 @@ class DatabricksClient(Protocol):
|
|
|
90
90
|
def get_run(self, run_id: int) -> RunInfo: ...
|
|
91
91
|
def list_runs(self, job_id: int, *, active_only: bool = False) -> Iterator[RunInfo]: ...
|
|
92
92
|
def repair_run(
|
|
93
|
-
self,
|
|
93
|
+
self,
|
|
94
|
+
run_id: int,
|
|
95
|
+
*,
|
|
96
|
+
rerun_tasks: Sequence[str],
|
|
97
|
+
latest_repair_id: int | None,
|
|
98
|
+
job_parameters: Mapping[str, str] | None = None,
|
|
94
99
|
) -> int: ...
|
|
95
100
|
def current_user(self) -> str: ...
|
|
96
101
|
def mkdirs(self, path: str) -> None: ...
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/sdk.py
RENAMED
|
@@ -220,7 +220,12 @@ class SdkClient:
|
|
|
220
220
|
return
|
|
221
221
|
|
|
222
222
|
def repair_run(
|
|
223
|
-
self,
|
|
223
|
+
self,
|
|
224
|
+
run_id: int,
|
|
225
|
+
*,
|
|
226
|
+
rerun_tasks: Sequence[str],
|
|
227
|
+
latest_repair_id: int | None,
|
|
228
|
+
job_parameters: Mapping[str, str] | None = None,
|
|
224
229
|
) -> int:
|
|
225
230
|
body: dict[str, Any] = {
|
|
226
231
|
"run_id": run_id,
|
|
@@ -229,6 +234,8 @@ class SdkClient:
|
|
|
229
234
|
}
|
|
230
235
|
if latest_repair_id is not None:
|
|
231
236
|
body["latest_repair_id"] = latest_repair_id
|
|
237
|
+
if job_parameters:
|
|
238
|
+
body["job_parameters"] = dict(job_parameters)
|
|
232
239
|
return int(self._call("POST", f"{JOBS}/runs/repair", body=body)["repair_id"])
|
|
233
240
|
|
|
234
241
|
def current_user(self) -> str:
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/finalize_states.py
RENAMED
|
@@ -24,7 +24,7 @@ def unit_states(
|
|
|
24
24
|
) -> dict[str, ExternalUnitState]:
|
|
25
25
|
"""The external state of every unit that has no attempt file.
|
|
26
26
|
|
|
27
|
-
`key_map` is unit id -> task key. EXCLUDED
|
|
27
|
+
`key_map` is unit id -> task key. EXCLUDED and DISABLED are `excluded`; UPSTREAM_FAILED and
|
|
28
28
|
UPSTREAM_CANCELED are `skipped` (upstream_failed); CANCELED and TIMEDOUT are `cancelled`; FAILED
|
|
29
29
|
is `failed` with PLD011 and the termination message. Any other state (success, still running,
|
|
30
30
|
unknown task) gives no entry, so core finalize counts the unit as not run.
|
|
@@ -40,7 +40,7 @@ def unit_states(
|
|
|
40
40
|
"databricks_task_run_id": str(task.run_id),
|
|
41
41
|
}
|
|
42
42
|
match task.result_state:
|
|
43
|
-
case "EXCLUDED":
|
|
43
|
+
case "EXCLUDED" | "DISABLED": # a task outside `only` ends DISABLED on serverless
|
|
44
44
|
out[unit] = ExternalUnitState("excluded")
|
|
45
45
|
case "UPSTREAM_FAILED" | "UPSTREAM_CANCELED":
|
|
46
46
|
out[unit] = ExternalUnitState("skipped", orchestrator_ids=ids)
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/jobs_model.py
RENAMED
|
@@ -66,6 +66,8 @@ class Task:
|
|
|
66
66
|
max_retries: int | None
|
|
67
67
|
min_retry_interval_millis: int | None
|
|
68
68
|
timeout_seconds: int | None
|
|
69
|
+
disable_auto_optimization: bool | None = None
|
|
70
|
+
"""True on serverless: auto-optimization retries a failed task on top of `max_retries`."""
|
|
69
71
|
|
|
70
72
|
|
|
71
73
|
@dataclass(frozen=True)
|
|
@@ -311,6 +311,7 @@ def _assemble(
|
|
|
311
311
|
if all(timeouts)
|
|
312
312
|
else None
|
|
313
313
|
),
|
|
314
|
+
disable_auto_optimization=True if env_key else None,
|
|
314
315
|
)
|
|
315
316
|
)
|
|
316
317
|
waits = {p for u in units for p in u.parents}
|
|
@@ -329,6 +330,7 @@ def _assemble(
|
|
|
329
330
|
max_retries=1,
|
|
330
331
|
min_retry_interval_millis=None,
|
|
331
332
|
timeout_seconds=FINALIZE_TIMEOUT,
|
|
333
|
+
disable_auto_optimization=True if env_key else None,
|
|
332
334
|
)
|
|
333
335
|
)
|
|
334
336
|
return _Parts(opts, units, tuple(tasks), tuple(environments), tuple(clusters))
|
|
@@ -61,6 +61,11 @@ def repair(
|
|
|
61
61
|
if not rerun:
|
|
62
62
|
return None
|
|
63
63
|
return client.repair_run(
|
|
64
|
-
job_run_id,
|
|
64
|
+
job_run_id,
|
|
65
|
+
rerun_tasks=rerun,
|
|
66
|
+
latest_repair_id=run.latest_repair_id,
|
|
67
|
+
# Without them the repair resolves {{job.parameters.*}} to the defaults, so an empty
|
|
68
|
+
# pipeloom_run_id would become dbx-<job run id> and start a new pipeloom run.
|
|
69
|
+
job_parameters=run.job_parameters,
|
|
65
70
|
)
|
|
66
71
|
return None
|
|
@@ -230,6 +230,30 @@ def _default_client() -> DatabricksClient:
|
|
|
230
230
|
return connect(profile=None, env=os.environ)
|
|
231
231
|
|
|
232
232
|
|
|
233
|
+
def _run_start(ctx: click.Context, given: Mapping[str, str]) -> str:
|
|
234
|
+
"""The start of the job run from the Jobs API, else `--start-ms`.
|
|
235
|
+
|
|
236
|
+
`{{job.start_time.timestamp_ms}}` is the start of the latest repair, not of the run, so a
|
|
237
|
+
repaired task would pass a start the run did not record (PLX108). Best effort: on any client
|
|
238
|
+
error one warning goes to stderr and `--start-ms` is used. A one-time run has no job run id
|
|
239
|
+
and keeps `--start-ms`, which the renderer writes literally.
|
|
240
|
+
"""
|
|
241
|
+
fallback = started_at(given["start_ms"])
|
|
242
|
+
job_run_id = given["job_run_id"]
|
|
243
|
+
if not job_run_id:
|
|
244
|
+
return fallback
|
|
245
|
+
factory = ctx.obj.client_factory if isinstance(ctx.obj, _Context) else None
|
|
246
|
+
try:
|
|
247
|
+
start_ms = (factory or _default_client)().get_run(int(job_run_id)).start_time_ms
|
|
248
|
+
except Exception as exc: # noqa: BLE001 - the flag is a usable fallback
|
|
249
|
+
click.echo(
|
|
250
|
+
f"warning: run start not read, using --start-ms: {type(exc).__name__}: {exc}",
|
|
251
|
+
err=True,
|
|
252
|
+
)
|
|
253
|
+
return fallback
|
|
254
|
+
return fallback if start_ms is None else started_at(str(start_ms))
|
|
255
|
+
|
|
256
|
+
|
|
233
257
|
def _states_file(
|
|
234
258
|
ctx: click.Context,
|
|
235
259
|
opts: Mapping[str, str],
|
|
@@ -336,7 +360,7 @@ def unit_command(
|
|
|
336
360
|
named.update({"unit": unit_id or "", "group": group or ""})
|
|
337
361
|
check_resolved(named)
|
|
338
362
|
run_id = resolve_run_id(given["run_id"], given["job_run_id"])
|
|
339
|
-
start =
|
|
363
|
+
start = _run_start(ctx, given)
|
|
340
364
|
refresh = _full_refresh(given["full_refresh"])
|
|
341
365
|
members = (
|
|
342
366
|
(unit_id,)
|
|
@@ -377,7 +401,7 @@ def finalize_command(ctx: click.Context, /, **given: str) -> None:
|
|
|
377
401
|
"""
|
|
378
402
|
check_resolved({k.replace("_", "-"): v for k, v in given.items()})
|
|
379
403
|
run_id = resolve_run_id(given["run_id"], given["job_run_id"])
|
|
380
|
-
start =
|
|
404
|
+
start = _run_start(ctx, given)
|
|
381
405
|
plan, opts = _deploy(given["manifest"], given["pipeline"])
|
|
382
406
|
report = Report()
|
|
383
407
|
units = pipeline_units(plan, opts, given["pipeline"], report)
|
|
@@ -409,14 +433,21 @@ def main(
|
|
|
409
433
|
task_values: TaskValues | None = None,
|
|
410
434
|
client_factory: Callable[[], DatabricksClient] | None = None,
|
|
411
435
|
) -> None:
|
|
412
|
-
"""The entry point. `argv` defaults to `sys.argv[1:]
|
|
436
|
+
"""The entry point. `argv` defaults to `sys.argv[1:]`.
|
|
437
|
+
|
|
438
|
+
Returns on success and raises `SystemExit` with a non-zero code on failure. Serverless runs
|
|
439
|
+
the wheel entry point under IPython, which reports even `SystemExit(0)` as a failed task.
|
|
413
440
|
|
|
414
441
|
`task_values` replaces the platform's task values and `client_factory` the connection that
|
|
415
442
|
finalize reads task states through (a test passes a recorder and a fake client).
|
|
416
443
|
"""
|
|
417
444
|
given = task_values is not None or client_factory is not None
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
445
|
+
try:
|
|
446
|
+
cli.main(
|
|
447
|
+
args=None if argv is None else list(argv),
|
|
448
|
+
prog_name="pipeloom_databricks_task",
|
|
449
|
+
obj=_Context(task_values, client_factory) if given else None,
|
|
450
|
+
)
|
|
451
|
+
except SystemExit as stop:
|
|
452
|
+
if stop.code not in (None, 0):
|
|
453
|
+
raise
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/fake.py
RENAMED
|
@@ -38,6 +38,8 @@ class FakeDatabricks:
|
|
|
38
38
|
self.runs: dict[int, RunInfo] = {}
|
|
39
39
|
self.run_tokens: dict[str, int] = {}
|
|
40
40
|
self.repairs: list[tuple[int, tuple[str, ...], int | None]] = []
|
|
41
|
+
self.repair_parameters: list[dict[str, str]] = []
|
|
42
|
+
"""The job parameters each repair passed, in order."""
|
|
41
43
|
self.starts: list[tuple[int, dict[str, str], tuple[str, ...] | None, str | None]] = []
|
|
42
44
|
"""Every `run_now`: (job_id, job_parameters, only, idempotency_token)."""
|
|
43
45
|
self.submissions: list[tuple[dict[str, JsonValue], str | None]] = []
|
|
@@ -177,7 +179,12 @@ class FakeDatabricks:
|
|
|
177
179
|
yield run
|
|
178
180
|
|
|
179
181
|
def repair_run(
|
|
180
|
-
self,
|
|
182
|
+
self,
|
|
183
|
+
run_id: int,
|
|
184
|
+
*,
|
|
185
|
+
rerun_tasks: Sequence[str],
|
|
186
|
+
latest_repair_id: int | None,
|
|
187
|
+
job_parameters: Mapping[str, str] | None = None,
|
|
181
188
|
) -> int:
|
|
182
189
|
self._enter("repair_run", run_id)
|
|
183
190
|
if run_id not in self.runs:
|
|
@@ -188,6 +195,7 @@ class FakeDatabricks:
|
|
|
188
195
|
)
|
|
189
196
|
repair_id = self._id()
|
|
190
197
|
self.repairs.append((run_id, tuple(rerun_tasks), latest_repair_id))
|
|
198
|
+
self.repair_parameters.append(dict(job_parameters or {}))
|
|
191
199
|
if self.stale_polls:
|
|
192
200
|
before = replace(self.runs[run_id], latest_repair_id=repair_id)
|
|
193
201
|
self._stale[run_id] = (before, self.stale_polls)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tags": {
|
|
4
4
|
"pipeloom_job": "shop.main.ci",
|
|
5
5
|
"pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
|
|
6
|
-
"pipeloom_settings_hash": "
|
|
6
|
+
"pipeloom_settings_hash": "c34a68f8140da68d14267702fc8854f834d1c2f0521d307b82007c7dc517f404",
|
|
7
7
|
"pipeloom_version": "1.2.3"
|
|
8
8
|
},
|
|
9
9
|
"parameters": [
|
|
@@ -61,7 +61,8 @@
|
|
|
61
61
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
62
62
|
]
|
|
63
63
|
},
|
|
64
|
-
"max_retries": 0
|
|
64
|
+
"max_retries": 0,
|
|
65
|
+
"disable_auto_optimization": true
|
|
65
66
|
},
|
|
66
67
|
{
|
|
67
68
|
"task_key": "dir_models_staging",
|
|
@@ -86,7 +87,8 @@
|
|
|
86
87
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
87
88
|
]
|
|
88
89
|
},
|
|
89
|
-
"max_retries": 0
|
|
90
|
+
"max_retries": 0,
|
|
91
|
+
"disable_auto_optimization": true
|
|
90
92
|
},
|
|
91
93
|
{
|
|
92
94
|
"task_key": "pipeloom__finalize",
|
|
@@ -115,7 +117,8 @@
|
|
|
115
117
|
]
|
|
116
118
|
},
|
|
117
119
|
"max_retries": 1,
|
|
118
|
-
"timeout_seconds": 1800
|
|
120
|
+
"timeout_seconds": 1800,
|
|
121
|
+
"disable_auto_optimization": true
|
|
119
122
|
}
|
|
120
123
|
]
|
|
121
124
|
}
|
|
@@ -6,7 +6,7 @@ resources:
|
|
|
6
6
|
tags:
|
|
7
7
|
pipeloom_job: shop.main.ci
|
|
8
8
|
pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
|
|
9
|
-
pipeloom_settings_hash:
|
|
9
|
+
pipeloom_settings_hash: c34a68f8140da68d14267702fc8854f834d1c2f0521d307b82007c7dc517f404
|
|
10
10
|
pipeloom_version: 1.2.3
|
|
11
11
|
parameters:
|
|
12
12
|
- name: pipeloom_run_id
|
|
@@ -47,6 +47,7 @@ resources:
|
|
|
47
47
|
- --task-run-id={{task.run_id}}
|
|
48
48
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
49
49
|
max_retries: 0
|
|
50
|
+
disable_auto_optimization: true
|
|
50
51
|
- task_key: dir_models_staging
|
|
51
52
|
environment_key: pipeloom_default
|
|
52
53
|
python_wheel_task:
|
|
@@ -68,6 +69,7 @@ resources:
|
|
|
68
69
|
- --task-run-id={{task.run_id}}
|
|
69
70
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
70
71
|
max_retries: 0
|
|
72
|
+
disable_auto_optimization: true
|
|
71
73
|
- task_key: pipeloom__finalize
|
|
72
74
|
depends_on:
|
|
73
75
|
- task_key: dir_models_marts
|
|
@@ -90,3 +92,4 @@ resources:
|
|
|
90
92
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
91
93
|
max_retries: 1
|
|
92
94
|
timeout_seconds: 1800
|
|
95
|
+
disable_auto_optimization: true
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tags": {
|
|
4
4
|
"pipeloom_job": "shop.main.ci",
|
|
5
5
|
"pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
|
|
6
|
-
"pipeloom_settings_hash": "
|
|
6
|
+
"pipeloom_settings_hash": "1200befe7dce232bdc277d7dfc500b77e5fceecdbf9e926b398df89ab487a216",
|
|
7
7
|
"pipeloom_version": "1.2.3"
|
|
8
8
|
},
|
|
9
9
|
"parameters": [
|
|
@@ -61,7 +61,8 @@
|
|
|
61
61
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
62
62
|
]
|
|
63
63
|
},
|
|
64
|
-
"max_retries": 0
|
|
64
|
+
"max_retries": 0,
|
|
65
|
+
"disable_auto_optimization": true
|
|
65
66
|
},
|
|
66
67
|
{
|
|
67
68
|
"task_key": "tag_staging",
|
|
@@ -86,7 +87,8 @@
|
|
|
86
87
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
87
88
|
]
|
|
88
89
|
},
|
|
89
|
-
"max_retries": 0
|
|
90
|
+
"max_retries": 0,
|
|
91
|
+
"disable_auto_optimization": true
|
|
90
92
|
},
|
|
91
93
|
{
|
|
92
94
|
"task_key": "tag_untagged",
|
|
@@ -116,7 +118,8 @@
|
|
|
116
118
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
117
119
|
]
|
|
118
120
|
},
|
|
119
|
-
"max_retries": 0
|
|
121
|
+
"max_retries": 0,
|
|
122
|
+
"disable_auto_optimization": true
|
|
120
123
|
},
|
|
121
124
|
{
|
|
122
125
|
"task_key": "pipeloom__finalize",
|
|
@@ -145,7 +148,8 @@
|
|
|
145
148
|
]
|
|
146
149
|
},
|
|
147
150
|
"max_retries": 1,
|
|
148
|
-
"timeout_seconds": 1800
|
|
151
|
+
"timeout_seconds": 1800,
|
|
152
|
+
"disable_auto_optimization": true
|
|
149
153
|
}
|
|
150
154
|
]
|
|
151
155
|
}
|
|
@@ -6,7 +6,7 @@ resources:
|
|
|
6
6
|
tags:
|
|
7
7
|
pipeloom_job: shop.main.ci
|
|
8
8
|
pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
|
|
9
|
-
pipeloom_settings_hash:
|
|
9
|
+
pipeloom_settings_hash: 1200befe7dce232bdc277d7dfc500b77e5fceecdbf9e926b398df89ab487a216
|
|
10
10
|
pipeloom_version: 1.2.3
|
|
11
11
|
parameters:
|
|
12
12
|
- name: pipeloom_run_id
|
|
@@ -47,6 +47,7 @@ resources:
|
|
|
47
47
|
- --task-run-id={{task.run_id}}
|
|
48
48
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
49
49
|
max_retries: 0
|
|
50
|
+
disable_auto_optimization: true
|
|
50
51
|
- task_key: tag_staging
|
|
51
52
|
environment_key: pipeloom_default
|
|
52
53
|
python_wheel_task:
|
|
@@ -68,6 +69,7 @@ resources:
|
|
|
68
69
|
- --task-run-id={{task.run_id}}
|
|
69
70
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
70
71
|
max_retries: 0
|
|
72
|
+
disable_auto_optimization: true
|
|
71
73
|
- task_key: tag_untagged
|
|
72
74
|
depends_on:
|
|
73
75
|
- task_key: tag_mart
|
|
@@ -91,6 +93,7 @@ resources:
|
|
|
91
93
|
- --task-run-id={{task.run_id}}
|
|
92
94
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
93
95
|
max_retries: 0
|
|
96
|
+
disable_auto_optimization: true
|
|
94
97
|
- task_key: pipeloom__finalize
|
|
95
98
|
depends_on:
|
|
96
99
|
- task_key: tag_untagged
|
|
@@ -113,3 +116,4 @@ resources:
|
|
|
113
116
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
114
117
|
max_retries: 1
|
|
115
118
|
timeout_seconds: 1800
|
|
119
|
+
disable_auto_optimization: true
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tags": {
|
|
4
4
|
"pipeloom_job": "shop.main.ci",
|
|
5
5
|
"pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
|
|
6
|
-
"pipeloom_settings_hash": "
|
|
6
|
+
"pipeloom_settings_hash": "c5bb6cb107c5cf386d3a0a134aefb8f16268e925290dba8f88a58e5b74868962",
|
|
7
7
|
"pipeloom_version": "1.2.3",
|
|
8
8
|
"team": "data"
|
|
9
9
|
},
|
|
@@ -165,7 +165,8 @@
|
|
|
165
165
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
166
166
|
]
|
|
167
167
|
},
|
|
168
|
-
"max_retries": 0
|
|
168
|
+
"max_retries": 0,
|
|
169
|
+
"disable_auto_optimization": true
|
|
169
170
|
},
|
|
170
171
|
{
|
|
171
172
|
"task_key": "stg_customers",
|
|
@@ -190,7 +191,8 @@
|
|
|
190
191
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
191
192
|
]
|
|
192
193
|
},
|
|
193
|
-
"max_retries": 0
|
|
194
|
+
"max_retries": 0,
|
|
195
|
+
"disable_auto_optimization": true
|
|
194
196
|
},
|
|
195
197
|
{
|
|
196
198
|
"task_key": "stg_orders",
|
|
@@ -215,7 +217,8 @@
|
|
|
215
217
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
216
218
|
]
|
|
217
219
|
},
|
|
218
|
-
"max_retries": 0
|
|
220
|
+
"max_retries": 0,
|
|
221
|
+
"disable_auto_optimization": true
|
|
219
222
|
},
|
|
220
223
|
{
|
|
221
224
|
"task_key": "stg_payments",
|
|
@@ -240,7 +243,8 @@
|
|
|
240
243
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
241
244
|
]
|
|
242
245
|
},
|
|
243
|
-
"max_retries": 0
|
|
246
|
+
"max_retries": 0,
|
|
247
|
+
"disable_auto_optimization": true
|
|
244
248
|
},
|
|
245
249
|
{
|
|
246
250
|
"task_key": "pipeloom__finalize",
|
|
@@ -272,7 +276,8 @@
|
|
|
272
276
|
]
|
|
273
277
|
},
|
|
274
278
|
"max_retries": 1,
|
|
275
|
-
"timeout_seconds": 1800
|
|
279
|
+
"timeout_seconds": 1800,
|
|
280
|
+
"disable_auto_optimization": true
|
|
276
281
|
}
|
|
277
282
|
],
|
|
278
283
|
"schedule": {
|
|
@@ -6,7 +6,7 @@ resources:
|
|
|
6
6
|
tags:
|
|
7
7
|
pipeloom_job: shop.main.ci
|
|
8
8
|
pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
|
|
9
|
-
pipeloom_settings_hash:
|
|
9
|
+
pipeloom_settings_hash: c5bb6cb107c5cf386d3a0a134aefb8f16268e925290dba8f88a58e5b74868962
|
|
10
10
|
pipeloom_version: 1.2.3
|
|
11
11
|
team: data
|
|
12
12
|
parameters:
|
|
@@ -113,6 +113,7 @@ resources:
|
|
|
113
113
|
- --task-run-id={{task.run_id}}
|
|
114
114
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
115
115
|
max_retries: 0
|
|
116
|
+
disable_auto_optimization: true
|
|
116
117
|
- task_key: stg_customers
|
|
117
118
|
environment_key: pipeloom_default
|
|
118
119
|
python_wheel_task:
|
|
@@ -134,6 +135,7 @@ resources:
|
|
|
134
135
|
- --task-run-id={{task.run_id}}
|
|
135
136
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
136
137
|
max_retries: 0
|
|
138
|
+
disable_auto_optimization: true
|
|
137
139
|
- task_key: stg_orders
|
|
138
140
|
environment_key: pipeloom_default
|
|
139
141
|
python_wheel_task:
|
|
@@ -155,6 +157,7 @@ resources:
|
|
|
155
157
|
- --task-run-id={{task.run_id}}
|
|
156
158
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
157
159
|
max_retries: 0
|
|
160
|
+
disable_auto_optimization: true
|
|
158
161
|
- task_key: stg_payments
|
|
159
162
|
environment_key: pipeloom_default
|
|
160
163
|
python_wheel_task:
|
|
@@ -176,6 +179,7 @@ resources:
|
|
|
176
179
|
- --task-run-id={{task.run_id}}
|
|
177
180
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
178
181
|
max_retries: 0
|
|
182
|
+
disable_auto_optimization: true
|
|
179
183
|
- task_key: pipeloom__finalize
|
|
180
184
|
depends_on:
|
|
181
185
|
- task_key: dim_customers
|
|
@@ -199,6 +203,7 @@ resources:
|
|
|
199
203
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
200
204
|
max_retries: 1
|
|
201
205
|
timeout_seconds: 1800
|
|
206
|
+
disable_auto_optimization: true
|
|
202
207
|
schedule:
|
|
203
208
|
quartz_cron_expression: 0 0 5 * * ?
|
|
204
209
|
timezone_id: UTC
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tags": {
|
|
4
4
|
"pipeloom_job": "shop.main.ci",
|
|
5
5
|
"pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
|
|
6
|
-
"pipeloom_settings_hash": "
|
|
6
|
+
"pipeloom_settings_hash": "da6b2e5934482e56accfa64dcb70de7be7d491607292f0939cf3b8fbf2aae71c",
|
|
7
7
|
"pipeloom_version": "1.2.3"
|
|
8
8
|
},
|
|
9
9
|
"parameters": [
|
|
@@ -64,7 +64,8 @@
|
|
|
64
64
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
65
65
|
]
|
|
66
66
|
},
|
|
67
|
-
"max_retries": 0
|
|
67
|
+
"max_retries": 0,
|
|
68
|
+
"disable_auto_optimization": true
|
|
68
69
|
},
|
|
69
70
|
{
|
|
70
71
|
"task_key": "fct_orders",
|
|
@@ -100,7 +101,8 @@
|
|
|
100
101
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
101
102
|
]
|
|
102
103
|
},
|
|
103
|
-
"max_retries": 0
|
|
104
|
+
"max_retries": 0,
|
|
105
|
+
"disable_auto_optimization": true
|
|
104
106
|
},
|
|
105
107
|
{
|
|
106
108
|
"task_key": "rpt_revenue",
|
|
@@ -130,7 +132,8 @@
|
|
|
130
132
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
131
133
|
]
|
|
132
134
|
},
|
|
133
|
-
"max_retries": 0
|
|
135
|
+
"max_retries": 0,
|
|
136
|
+
"disable_auto_optimization": true
|
|
134
137
|
},
|
|
135
138
|
{
|
|
136
139
|
"task_key": "stg_customers",
|
|
@@ -155,7 +158,8 @@
|
|
|
155
158
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
156
159
|
]
|
|
157
160
|
},
|
|
158
|
-
"max_retries": 0
|
|
161
|
+
"max_retries": 0,
|
|
162
|
+
"disable_auto_optimization": true
|
|
159
163
|
},
|
|
160
164
|
{
|
|
161
165
|
"task_key": "stg_orders",
|
|
@@ -180,7 +184,8 @@
|
|
|
180
184
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
181
185
|
]
|
|
182
186
|
},
|
|
183
|
-
"max_retries": 0
|
|
187
|
+
"max_retries": 0,
|
|
188
|
+
"disable_auto_optimization": true
|
|
184
189
|
},
|
|
185
190
|
{
|
|
186
191
|
"task_key": "stg_payments",
|
|
@@ -205,7 +210,8 @@
|
|
|
205
210
|
"--start-ms={{job.start_time.timestamp_ms}}"
|
|
206
211
|
]
|
|
207
212
|
},
|
|
208
|
-
"max_retries": 0
|
|
213
|
+
"max_retries": 0,
|
|
214
|
+
"disable_auto_optimization": true
|
|
209
215
|
},
|
|
210
216
|
{
|
|
211
217
|
"task_key": "pipeloom__finalize",
|
|
@@ -237,7 +243,8 @@
|
|
|
237
243
|
]
|
|
238
244
|
},
|
|
239
245
|
"max_retries": 1,
|
|
240
|
-
"timeout_seconds": 1800
|
|
246
|
+
"timeout_seconds": 1800,
|
|
247
|
+
"disable_auto_optimization": true
|
|
241
248
|
}
|
|
242
249
|
]
|
|
243
250
|
}
|
|
@@ -6,7 +6,7 @@ resources:
|
|
|
6
6
|
tags:
|
|
7
7
|
pipeloom_job: shop.main.ci
|
|
8
8
|
pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
|
|
9
|
-
pipeloom_settings_hash:
|
|
9
|
+
pipeloom_settings_hash: da6b2e5934482e56accfa64dcb70de7be7d491607292f0939cf3b8fbf2aae71c
|
|
10
10
|
pipeloom_version: 1.2.3
|
|
11
11
|
parameters:
|
|
12
12
|
- name: pipeloom_run_id
|
|
@@ -48,6 +48,7 @@ resources:
|
|
|
48
48
|
- --task-run-id={{task.run_id}}
|
|
49
49
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
50
50
|
max_retries: 0
|
|
51
|
+
disable_auto_optimization: true
|
|
51
52
|
- task_key: fct_orders
|
|
52
53
|
depends_on:
|
|
53
54
|
- task_key: stg_customers
|
|
@@ -73,6 +74,7 @@ resources:
|
|
|
73
74
|
- --task-run-id={{task.run_id}}
|
|
74
75
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
75
76
|
max_retries: 0
|
|
77
|
+
disable_auto_optimization: true
|
|
76
78
|
- task_key: rpt_revenue
|
|
77
79
|
depends_on:
|
|
78
80
|
- task_key: fct_orders
|
|
@@ -96,6 +98,7 @@ resources:
|
|
|
96
98
|
- --task-run-id={{task.run_id}}
|
|
97
99
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
98
100
|
max_retries: 0
|
|
101
|
+
disable_auto_optimization: true
|
|
99
102
|
- task_key: stg_customers
|
|
100
103
|
environment_key: pipeloom_default
|
|
101
104
|
python_wheel_task:
|
|
@@ -117,6 +120,7 @@ resources:
|
|
|
117
120
|
- --task-run-id={{task.run_id}}
|
|
118
121
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
119
122
|
max_retries: 0
|
|
123
|
+
disable_auto_optimization: true
|
|
120
124
|
- task_key: stg_orders
|
|
121
125
|
environment_key: pipeloom_default
|
|
122
126
|
python_wheel_task:
|
|
@@ -138,6 +142,7 @@ resources:
|
|
|
138
142
|
- --task-run-id={{task.run_id}}
|
|
139
143
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
140
144
|
max_retries: 0
|
|
145
|
+
disable_auto_optimization: true
|
|
141
146
|
- task_key: stg_payments
|
|
142
147
|
environment_key: pipeloom_default
|
|
143
148
|
python_wheel_task:
|
|
@@ -159,6 +164,7 @@ resources:
|
|
|
159
164
|
- --task-run-id={{task.run_id}}
|
|
160
165
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
161
166
|
max_retries: 0
|
|
167
|
+
disable_auto_optimization: true
|
|
162
168
|
- task_key: pipeloom__finalize
|
|
163
169
|
depends_on:
|
|
164
170
|
- task_key: dim_customers
|
|
@@ -182,3 +188,4 @@ resources:
|
|
|
182
188
|
- --start-ms={{job.start_time.timestamp_ms}}
|
|
183
189
|
max_retries: 1
|
|
184
190
|
timeout_seconds: 1800
|
|
191
|
+
disable_auto_optimization: true
|
|
@@ -30,7 +30,8 @@
|
|
|
30
30
|
"--start-ms=1700000000000"
|
|
31
31
|
]
|
|
32
32
|
},
|
|
33
|
-
"max_retries": 0
|
|
33
|
+
"max_retries": 0,
|
|
34
|
+
"disable_auto_optimization": true
|
|
34
35
|
},
|
|
35
36
|
{
|
|
36
37
|
"task_key": "fct_orders",
|
|
@@ -64,7 +65,8 @@
|
|
|
64
65
|
"--start-ms=1700000000000"
|
|
65
66
|
]
|
|
66
67
|
},
|
|
67
|
-
"max_retries": 0
|
|
68
|
+
"max_retries": 0,
|
|
69
|
+
"disable_auto_optimization": true
|
|
68
70
|
},
|
|
69
71
|
{
|
|
70
72
|
"task_key": "rpt_revenue",
|
|
@@ -92,7 +94,8 @@
|
|
|
92
94
|
"--start-ms=1700000000000"
|
|
93
95
|
]
|
|
94
96
|
},
|
|
95
|
-
"max_retries": 0
|
|
97
|
+
"max_retries": 0,
|
|
98
|
+
"disable_auto_optimization": true
|
|
96
99
|
},
|
|
97
100
|
{
|
|
98
101
|
"task_key": "stg_customers",
|
|
@@ -115,7 +118,8 @@
|
|
|
115
118
|
"--start-ms=1700000000000"
|
|
116
119
|
]
|
|
117
120
|
},
|
|
118
|
-
"max_retries": 0
|
|
121
|
+
"max_retries": 0,
|
|
122
|
+
"disable_auto_optimization": true
|
|
119
123
|
},
|
|
120
124
|
{
|
|
121
125
|
"task_key": "stg_orders",
|
|
@@ -138,7 +142,8 @@
|
|
|
138
142
|
"--start-ms=1700000000000"
|
|
139
143
|
]
|
|
140
144
|
},
|
|
141
|
-
"max_retries": 0
|
|
145
|
+
"max_retries": 0,
|
|
146
|
+
"disable_auto_optimization": true
|
|
142
147
|
},
|
|
143
148
|
{
|
|
144
149
|
"task_key": "stg_payments",
|
|
@@ -161,7 +166,8 @@
|
|
|
161
166
|
"--start-ms=1700000000000"
|
|
162
167
|
]
|
|
163
168
|
},
|
|
164
|
-
"max_retries": 0
|
|
169
|
+
"max_retries": 0,
|
|
170
|
+
"disable_auto_optimization": true
|
|
165
171
|
},
|
|
166
172
|
{
|
|
167
173
|
"task_key": "pipeloom__finalize",
|
|
@@ -191,7 +197,8 @@
|
|
|
191
197
|
]
|
|
192
198
|
},
|
|
193
199
|
"max_retries": 1,
|
|
194
|
-
"timeout_seconds": 1800
|
|
200
|
+
"timeout_seconds": 1800,
|
|
201
|
+
"disable_auto_optimization": true
|
|
195
202
|
}
|
|
196
203
|
],
|
|
197
204
|
"environments": [
|
|
@@ -40,10 +40,12 @@ def finalize_argv(s: Sim, fake: FakeDatabricks, *, job_run_id: str, run_id: str)
|
|
|
40
40
|
f"--job-run-id={job_run_id}",
|
|
41
41
|
"--start-ms=1700000000000",
|
|
42
42
|
]
|
|
43
|
-
|
|
43
|
+
try:
|
|
44
44
|
main(argv, client_factory=lambda: fake)
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
except SystemExit as stop:
|
|
46
|
+
assert isinstance(stop.code, int)
|
|
47
|
+
return stop.code
|
|
48
|
+
return 0
|
|
47
49
|
|
|
48
50
|
|
|
49
51
|
def run_of(*tasks: TaskRunInfo) -> RunInfo:
|
|
@@ -57,6 +59,7 @@ KEYS = {"a": "ta", "b": "tb", "c": "tc"}
|
|
|
57
59
|
("result", "expected"),
|
|
58
60
|
[
|
|
59
61
|
("EXCLUDED", ExternalUnitState("excluded")),
|
|
62
|
+
("DISABLED", ExternalUnitState("excluded")),
|
|
60
63
|
("UPSTREAM_FAILED", "skipped"),
|
|
61
64
|
("UPSTREAM_CANCELED", "skipped"),
|
|
62
65
|
("CANCELED", "cancelled"),
|
|
@@ -47,6 +47,17 @@ def test_the_rerun_set_is_every_unsuccessful_task_plus_finalize() -> None:
|
|
|
47
47
|
assert "SUCCESS" not in REPAIRABLE and "EXCLUDED" not in REPAIRABLE
|
|
48
48
|
|
|
49
49
|
|
|
50
|
+
def test_a_repair_passes_the_run_job_parameters_again() -> None:
|
|
51
|
+
# without them the repair resolves {{job.parameters.pipeloom_run_id}} to its empty default
|
|
52
|
+
fake = FakeDatabricks()
|
|
53
|
+
params = {"pipeloom_run_id": "nightly.1", "full_refresh": "false"}
|
|
54
|
+
fake.set_run(
|
|
55
|
+
50, job_id=1, result_state="FAILED", tasks=(t("a", "FAILED"),), job_parameters=params
|
|
56
|
+
)
|
|
57
|
+
repair(fake, 50, Report())
|
|
58
|
+
assert fake.repair_parameters == [params]
|
|
59
|
+
|
|
60
|
+
|
|
50
61
|
def test_finalize_joins_the_set_when_a_unit_failed_even_if_it_succeeded() -> None:
|
|
51
62
|
fake = FakeDatabricks()
|
|
52
63
|
finished(fake, t("a", "FAILED"), t(FINALIZE_KEY, "SUCCESS"))
|
|
@@ -227,8 +227,11 @@ def test_repair_run_body() -> None:
|
|
|
227
227
|
"rerun_tasks": ["a", "b"],
|
|
228
228
|
"rerun_dependent_tasks": True,
|
|
229
229
|
}
|
|
230
|
-
sdk.repair_run(
|
|
230
|
+
sdk.repair_run(
|
|
231
|
+
11, rerun_tasks=["a"], latest_repair_id=31, job_parameters={"pipeloom_run_id": "r"}
|
|
232
|
+
)
|
|
231
233
|
assert api.calls[1]["body"]["latest_repair_id"] == 31
|
|
234
|
+
assert api.calls[1]["body"]["job_parameters"] == {"pipeloom_run_id": "r"}
|
|
232
235
|
|
|
233
236
|
|
|
234
237
|
def test_workspace_calls() -> None:
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import json
|
|
6
6
|
import sys
|
|
7
7
|
from collections.abc import Callable
|
|
8
|
+
from dataclasses import replace
|
|
8
9
|
from types import ModuleType
|
|
9
10
|
from typing import Any
|
|
10
11
|
|
|
@@ -15,6 +16,7 @@ from pydantic import JsonValue
|
|
|
15
16
|
|
|
16
17
|
import pipeloom_databricks
|
|
17
18
|
from pipeloom_databricks import task
|
|
19
|
+
from pipeloom_databricks.client import ApiError
|
|
18
20
|
from pipeloom_databricks.task import (
|
|
19
21
|
check_resolved,
|
|
20
22
|
cli,
|
|
@@ -28,6 +30,7 @@ from pipeloom_databricks.task_values import (
|
|
|
28
30
|
runtime_task_values,
|
|
29
31
|
summary,
|
|
30
32
|
)
|
|
33
|
+
from pipeloom_databricks.testing.fake import FakeDatabricks
|
|
31
34
|
|
|
32
35
|
Sim = Any
|
|
33
36
|
HASH = "0" * 64
|
|
@@ -160,7 +163,7 @@ def test_task_values_are_published_and_never_change_the_exit_code(
|
|
|
160
163
|
assert "task value not set: RuntimeError" in r.stderr
|
|
161
164
|
|
|
162
165
|
|
|
163
|
-
def
|
|
166
|
+
def test_main_exits_only_on_failure_and_the_lazy_entry_reads_argv(
|
|
164
167
|
sim: Callable[..., Sim], monkeypatch: pytest.MonkeyPatch
|
|
165
168
|
) -> None:
|
|
166
169
|
s = sim()
|
|
@@ -168,9 +171,25 @@ def test_main_ends_in_system_exit_and_the_lazy_entry_reads_argv(
|
|
|
168
171
|
task.main(unit_args(s, unit="c", **{"task-key": "c"}))
|
|
169
172
|
assert stop.value.code == 1
|
|
170
173
|
monkeypatch.setattr(sys, "argv", ["pipeloom_databricks_task", *unit_args(s)])
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
+
# success returns: serverless (IPython) reports even SystemExit(0) as a failed task
|
|
175
|
+
pipeloom_databricks.pipeloom_databricks_task()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_the_run_start_comes_from_the_jobs_api_not_the_repair_start(
|
|
179
|
+
capsys: pytest.CaptureFixture[str],
|
|
180
|
+
) -> None:
|
|
181
|
+
# {{job.start_time}} is the start of the latest repair; the run keeps its own start
|
|
182
|
+
fake = FakeDatabricks()
|
|
183
|
+
fake.set_run(5, job_id=1, result_state="FAILED")
|
|
184
|
+
fake.runs[5] = replace(fake.runs[5], start_time_ms=1700000000000)
|
|
185
|
+
ctx = click.Context(task.cli, obj=task._Context(None, lambda: fake))
|
|
186
|
+
repair_start = {"start_ms": "1700000090000", "job_run_id": "5"}
|
|
187
|
+
assert task._run_start(ctx, repair_start) == started_at("1700000000000")
|
|
188
|
+
# a one-time run has no job run id: the literal start stays
|
|
189
|
+
assert task._run_start(ctx, {**repair_start, "job_run_id": ""}) == started_at("1700000090000")
|
|
190
|
+
fake.fail["get_run"] = ApiError(500, "unavailable")
|
|
191
|
+
assert task._run_start(ctx, repair_start) == started_at("1700000090000")
|
|
192
|
+
assert "warning: run start not read, using --start-ms" in capsys.readouterr().err
|
|
174
193
|
|
|
175
194
|
|
|
176
195
|
# -- helpers -------------------------------------------------------------------------------------
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/connect.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/serialize.py
RENAMED
|
File without changes
|
|
File without changes
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/task_values.py
RENAMED
|
File without changes
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/__init__.py
RENAMED
|
File without changes
|
{pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/jobsim.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|