pipeloom-databricks 0.6.0__tar.gz → 0.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/PKG-INFO +1 -1
  2. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/pyproject.toml +1 -1
  3. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/__init__.py +6 -1
  4. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/sdk.py +8 -1
  5. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/finalize_states.py +2 -2
  6. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/jobs_model.py +2 -0
  7. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/render.py +2 -0
  8. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/repair.py +6 -1
  9. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/task.py +39 -8
  10. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/fake.py +9 -1
  11. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_folder.json +7 -4
  12. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_folder.yml +4 -1
  13. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_tag.json +9 -5
  14. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/by_tag.yml +5 -1
  15. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/job_cluster.json +11 -6
  16. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/job_cluster.yml +6 -1
  17. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/serverless.json +15 -8
  18. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/serverless.yml +8 -1
  19. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/goldens/submit.json +14 -7
  20. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_finalize_states.py +6 -3
  21. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_repair.py +11 -0
  22. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_sdk_client.py +4 -1
  23. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_task.py +23 -4
  24. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/.gitignore +0 -0
  25. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/LICENSE +0 -0
  26. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/NOTICE +0 -0
  27. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/README.md +0 -0
  28. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/__init__.py +0 -0
  29. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/api.py +0 -0
  30. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/bundle.py +0 -0
  31. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/cli.py +0 -0
  32. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/client/connect.py +0 -0
  33. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/deploy.py +0 -0
  34. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/grouping.py +0 -0
  35. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/keys.py +0 -0
  36. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/options.py +0 -0
  37. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/py.typed +0 -0
  38. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/run.py +0 -0
  39. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/secrets.py +0 -0
  40. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/serialize.py +0 -0
  41. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/spec.py +0 -0
  42. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/task_values.py +0 -0
  43. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/__init__.py +0 -0
  44. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/testing/jobsim.py +0 -0
  45. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/unity.py +0 -0
  46. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/src/pipeloom_databricks/upload.py +0 -0
  47. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/conftest.py +0 -0
  48. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_bundle.py +0 -0
  49. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_connect.py +0 -0
  50. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_deploy.py +0 -0
  51. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_deploy_cli.py +0 -0
  52. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_grouping.py +0 -0
  53. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_jobsim.py +0 -0
  54. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_keys.py +0 -0
  55. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_options.py +0 -0
  56. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_render.py +0 -0
  57. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_run.py +0 -0
  58. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_run_cli.py +0 -0
  59. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_secrets.py +0 -0
  60. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_spec_import.py +0 -0
  61. {pipeloom_databricks-0.6.0 → pipeloom_databricks-0.6.1}/tests/test_unity.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pipeloom-databricks
3
- Version: 0.6.0
3
+ Version: 0.6.1
4
4
  Summary: Databricks renderer for pipeloom: a compiled plan as Jobs settings JSON and bundle YAML.
5
5
  Project-URL: Repository, https://github.com/emanuel-luis/pipeloom
6
6
  Project-URL: Documentation, https://emanuel-luis.github.io/pipeloom/
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "pipeloom-databricks"
7
- version = "0.6.0"
7
+ version = "0.6.1"
8
8
  description = "Databricks renderer for pipeloom: a compiled plan as Jobs settings JSON and bundle YAML."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -90,7 +90,12 @@ class DatabricksClient(Protocol):
90
90
  def get_run(self, run_id: int) -> RunInfo: ...
91
91
  def list_runs(self, job_id: int, *, active_only: bool = False) -> Iterator[RunInfo]: ...
92
92
  def repair_run(
93
- self, run_id: int, *, rerun_tasks: Sequence[str], latest_repair_id: int | None
93
+ self,
94
+ run_id: int,
95
+ *,
96
+ rerun_tasks: Sequence[str],
97
+ latest_repair_id: int | None,
98
+ job_parameters: Mapping[str, str] | None = None,
94
99
  ) -> int: ...
95
100
  def current_user(self) -> str: ...
96
101
  def mkdirs(self, path: str) -> None: ...
@@ -220,7 +220,12 @@ class SdkClient:
220
220
  return
221
221
 
222
222
  def repair_run(
223
- self, run_id: int, *, rerun_tasks: Sequence[str], latest_repair_id: int | None
223
+ self,
224
+ run_id: int,
225
+ *,
226
+ rerun_tasks: Sequence[str],
227
+ latest_repair_id: int | None,
228
+ job_parameters: Mapping[str, str] | None = None,
224
229
  ) -> int:
225
230
  body: dict[str, Any] = {
226
231
  "run_id": run_id,
@@ -229,6 +234,8 @@ class SdkClient:
229
234
  }
230
235
  if latest_repair_id is not None:
231
236
  body["latest_repair_id"] = latest_repair_id
237
+ if job_parameters:
238
+ body["job_parameters"] = dict(job_parameters)
232
239
  return int(self._call("POST", f"{JOBS}/runs/repair", body=body)["repair_id"])
233
240
 
234
241
  def current_user(self) -> str:
@@ -24,7 +24,7 @@ def unit_states(
24
24
  ) -> dict[str, ExternalUnitState]:
25
25
  """The external state of every unit that has no attempt file.
26
26
 
27
- `key_map` is unit id -> task key. EXCLUDED is `excluded`; UPSTREAM_FAILED and
27
+ `key_map` is unit id -> task key. EXCLUDED and DISABLED are `excluded`; UPSTREAM_FAILED and
28
28
  UPSTREAM_CANCELED are `skipped` (upstream_failed); CANCELED and TIMEDOUT are `cancelled`; FAILED
29
29
  is `failed` with PLD011 and the termination message. Any other state (success, still running,
30
30
  unknown task) gives no entry, so core finalize counts the unit as not run.
@@ -40,7 +40,7 @@ def unit_states(
40
40
  "databricks_task_run_id": str(task.run_id),
41
41
  }
42
42
  match task.result_state:
43
- case "EXCLUDED":
43
+ case "EXCLUDED" | "DISABLED": # a task outside `only` ends DISABLED on serverless
44
44
  out[unit] = ExternalUnitState("excluded")
45
45
  case "UPSTREAM_FAILED" | "UPSTREAM_CANCELED":
46
46
  out[unit] = ExternalUnitState("skipped", orchestrator_ids=ids)
@@ -66,6 +66,8 @@ class Task:
66
66
  max_retries: int | None
67
67
  min_retry_interval_millis: int | None
68
68
  timeout_seconds: int | None
69
+ disable_auto_optimization: bool | None = None
70
+ """True on serverless: auto-optimization retries a failed task on top of `max_retries`."""
69
71
 
70
72
 
71
73
  @dataclass(frozen=True)
@@ -311,6 +311,7 @@ def _assemble(
311
311
  if all(timeouts)
312
312
  else None
313
313
  ),
314
+ disable_auto_optimization=True if env_key else None,
314
315
  )
315
316
  )
316
317
  waits = {p for u in units for p in u.parents}
@@ -329,6 +330,7 @@ def _assemble(
329
330
  max_retries=1,
330
331
  min_retry_interval_millis=None,
331
332
  timeout_seconds=FINALIZE_TIMEOUT,
333
+ disable_auto_optimization=True if env_key else None,
332
334
  )
333
335
  )
334
336
  return _Parts(opts, units, tuple(tasks), tuple(environments), tuple(clusters))
@@ -61,6 +61,11 @@ def repair(
61
61
  if not rerun:
62
62
  return None
63
63
  return client.repair_run(
64
- job_run_id, rerun_tasks=rerun, latest_repair_id=run.latest_repair_id
64
+ job_run_id,
65
+ rerun_tasks=rerun,
66
+ latest_repair_id=run.latest_repair_id,
67
+ # Without them the repair resolves {{job.parameters.*}} to the defaults, so an empty
68
+ # pipeloom_run_id would become dbx-<job run id> and start a new pipeloom run.
69
+ job_parameters=run.job_parameters,
65
70
  )
66
71
  return None
@@ -230,6 +230,30 @@ def _default_client() -> DatabricksClient:
230
230
  return connect(profile=None, env=os.environ)
231
231
 
232
232
 
233
+ def _run_start(ctx: click.Context, given: Mapping[str, str]) -> str:
234
+ """The start of the job run from the Jobs API, else `--start-ms`.
235
+
236
+ `{{job.start_time.timestamp_ms}}` is the start of the latest repair, not of the run, so a
237
+ repaired task would pass a start the run did not record (PLX108). Best effort: on any client
238
+ error one warning goes to stderr and `--start-ms` is used. A one-time run has no job run id
239
+ and keeps `--start-ms`, which the renderer writes literally.
240
+ """
241
+ fallback = started_at(given["start_ms"])
242
+ job_run_id = given["job_run_id"]
243
+ if not job_run_id:
244
+ return fallback
245
+ factory = ctx.obj.client_factory if isinstance(ctx.obj, _Context) else None
246
+ try:
247
+ start_ms = (factory or _default_client)().get_run(int(job_run_id)).start_time_ms
248
+ except Exception as exc: # noqa: BLE001 - the flag is a usable fallback
249
+ click.echo(
250
+ f"warning: run start not read, using --start-ms: {type(exc).__name__}: {exc}",
251
+ err=True,
252
+ )
253
+ return fallback
254
+ return fallback if start_ms is None else started_at(str(start_ms))
255
+
256
+
233
257
  def _states_file(
234
258
  ctx: click.Context,
235
259
  opts: Mapping[str, str],
@@ -336,7 +360,7 @@ def unit_command(
336
360
  named.update({"unit": unit_id or "", "group": group or ""})
337
361
  check_resolved(named)
338
362
  run_id = resolve_run_id(given["run_id"], given["job_run_id"])
339
- start = started_at(given["start_ms"])
363
+ start = _run_start(ctx, given)
340
364
  refresh = _full_refresh(given["full_refresh"])
341
365
  members = (
342
366
  (unit_id,)
@@ -377,7 +401,7 @@ def finalize_command(ctx: click.Context, /, **given: str) -> None:
377
401
  """
378
402
  check_resolved({k.replace("_", "-"): v for k, v in given.items()})
379
403
  run_id = resolve_run_id(given["run_id"], given["job_run_id"])
380
- start = started_at(given["start_ms"])
404
+ start = _run_start(ctx, given)
381
405
  plan, opts = _deploy(given["manifest"], given["pipeline"])
382
406
  report = Report()
383
407
  units = pipeline_units(plan, opts, given["pipeline"], report)
@@ -409,14 +433,21 @@ def main(
409
433
  task_values: TaskValues | None = None,
410
434
  client_factory: Callable[[], DatabricksClient] | None = None,
411
435
  ) -> None:
412
- """The entry point. `argv` defaults to `sys.argv[1:]`; always ends in `SystemExit`.
436
+ """The entry point. `argv` defaults to `sys.argv[1:]`.
437
+
438
+ Returns on success and raises `SystemExit` with a non-zero code on failure. Serverless runs
439
+ the wheel entry point under IPython, which reports even `SystemExit(0)` as a failed task.
413
440
 
414
441
  `task_values` replaces the platform's task values and `client_factory` the connection that
415
442
  finalize reads task states through (a test passes a recorder and a fake client).
416
443
  """
417
444
  given = task_values is not None or client_factory is not None
418
- cli.main(
419
- args=None if argv is None else list(argv),
420
- prog_name="pipeloom_databricks_task",
421
- obj=_Context(task_values, client_factory) if given else None,
422
- )
445
+ try:
446
+ cli.main(
447
+ args=None if argv is None else list(argv),
448
+ prog_name="pipeloom_databricks_task",
449
+ obj=_Context(task_values, client_factory) if given else None,
450
+ )
451
+ except SystemExit as stop:
452
+ if stop.code not in (None, 0):
453
+ raise
@@ -38,6 +38,8 @@ class FakeDatabricks:
38
38
  self.runs: dict[int, RunInfo] = {}
39
39
  self.run_tokens: dict[str, int] = {}
40
40
  self.repairs: list[tuple[int, tuple[str, ...], int | None]] = []
41
+ self.repair_parameters: list[dict[str, str]] = []
42
+ """The job parameters each repair passed, in order."""
41
43
  self.starts: list[tuple[int, dict[str, str], tuple[str, ...] | None, str | None]] = []
42
44
  """Every `run_now`: (job_id, job_parameters, only, idempotency_token)."""
43
45
  self.submissions: list[tuple[dict[str, JsonValue], str | None]] = []
@@ -177,7 +179,12 @@ class FakeDatabricks:
177
179
  yield run
178
180
 
179
181
  def repair_run(
180
- self, run_id: int, *, rerun_tasks: Sequence[str], latest_repair_id: int | None
182
+ self,
183
+ run_id: int,
184
+ *,
185
+ rerun_tasks: Sequence[str],
186
+ latest_repair_id: int | None,
187
+ job_parameters: Mapping[str, str] | None = None,
181
188
  ) -> int:
182
189
  self._enter("repair_run", run_id)
183
190
  if run_id not in self.runs:
@@ -188,6 +195,7 @@ class FakeDatabricks:
188
195
  )
189
196
  repair_id = self._id()
190
197
  self.repairs.append((run_id, tuple(rerun_tasks), latest_repair_id))
198
+ self.repair_parameters.append(dict(job_parameters or {}))
191
199
  if self.stale_polls:
192
200
  before = replace(self.runs[run_id], latest_repair_id=repair_id)
193
201
  self._stale[run_id] = (before, self.stale_polls)
@@ -3,7 +3,7 @@
3
3
  "tags": {
4
4
  "pipeloom_job": "shop.main.ci",
5
5
  "pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
6
- "pipeloom_settings_hash": "2623a393d369b3c3f076850239bf729fe127d726ec0f30ade67894b47e4472d8",
6
+ "pipeloom_settings_hash": "c34a68f8140da68d14267702fc8854f834d1c2f0521d307b82007c7dc517f404",
7
7
  "pipeloom_version": "1.2.3"
8
8
  },
9
9
  "parameters": [
@@ -61,7 +61,8 @@
61
61
  "--start-ms={{job.start_time.timestamp_ms}}"
62
62
  ]
63
63
  },
64
- "max_retries": 0
64
+ "max_retries": 0,
65
+ "disable_auto_optimization": true
65
66
  },
66
67
  {
67
68
  "task_key": "dir_models_staging",
@@ -86,7 +87,8 @@
86
87
  "--start-ms={{job.start_time.timestamp_ms}}"
87
88
  ]
88
89
  },
89
- "max_retries": 0
90
+ "max_retries": 0,
91
+ "disable_auto_optimization": true
90
92
  },
91
93
  {
92
94
  "task_key": "pipeloom__finalize",
@@ -115,7 +117,8 @@
115
117
  ]
116
118
  },
117
119
  "max_retries": 1,
118
- "timeout_seconds": 1800
120
+ "timeout_seconds": 1800,
121
+ "disable_auto_optimization": true
119
122
  }
120
123
  ]
121
124
  }
@@ -6,7 +6,7 @@ resources:
6
6
  tags:
7
7
  pipeloom_job: shop.main.ci
8
8
  pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
9
- pipeloom_settings_hash: 2623a393d369b3c3f076850239bf729fe127d726ec0f30ade67894b47e4472d8
9
+ pipeloom_settings_hash: c34a68f8140da68d14267702fc8854f834d1c2f0521d307b82007c7dc517f404
10
10
  pipeloom_version: 1.2.3
11
11
  parameters:
12
12
  - name: pipeloom_run_id
@@ -47,6 +47,7 @@ resources:
47
47
  - --task-run-id={{task.run_id}}
48
48
  - --start-ms={{job.start_time.timestamp_ms}}
49
49
  max_retries: 0
50
+ disable_auto_optimization: true
50
51
  - task_key: dir_models_staging
51
52
  environment_key: pipeloom_default
52
53
  python_wheel_task:
@@ -68,6 +69,7 @@ resources:
68
69
  - --task-run-id={{task.run_id}}
69
70
  - --start-ms={{job.start_time.timestamp_ms}}
70
71
  max_retries: 0
72
+ disable_auto_optimization: true
71
73
  - task_key: pipeloom__finalize
72
74
  depends_on:
73
75
  - task_key: dir_models_marts
@@ -90,3 +92,4 @@ resources:
90
92
  - --start-ms={{job.start_time.timestamp_ms}}
91
93
  max_retries: 1
92
94
  timeout_seconds: 1800
95
+ disable_auto_optimization: true
@@ -3,7 +3,7 @@
3
3
  "tags": {
4
4
  "pipeloom_job": "shop.main.ci",
5
5
  "pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
6
- "pipeloom_settings_hash": "f5a10d5fdb1f1d09c10bca06580b2babc243f86e5fc10dfab550d58de85f5b6e",
6
+ "pipeloom_settings_hash": "1200befe7dce232bdc277d7dfc500b77e5fceecdbf9e926b398df89ab487a216",
7
7
  "pipeloom_version": "1.2.3"
8
8
  },
9
9
  "parameters": [
@@ -61,7 +61,8 @@
61
61
  "--start-ms={{job.start_time.timestamp_ms}}"
62
62
  ]
63
63
  },
64
- "max_retries": 0
64
+ "max_retries": 0,
65
+ "disable_auto_optimization": true
65
66
  },
66
67
  {
67
68
  "task_key": "tag_staging",
@@ -86,7 +87,8 @@
86
87
  "--start-ms={{job.start_time.timestamp_ms}}"
87
88
  ]
88
89
  },
89
- "max_retries": 0
90
+ "max_retries": 0,
91
+ "disable_auto_optimization": true
90
92
  },
91
93
  {
92
94
  "task_key": "tag_untagged",
@@ -116,7 +118,8 @@
116
118
  "--start-ms={{job.start_time.timestamp_ms}}"
117
119
  ]
118
120
  },
119
- "max_retries": 0
121
+ "max_retries": 0,
122
+ "disable_auto_optimization": true
120
123
  },
121
124
  {
122
125
  "task_key": "pipeloom__finalize",
@@ -145,7 +148,8 @@
145
148
  ]
146
149
  },
147
150
  "max_retries": 1,
148
- "timeout_seconds": 1800
151
+ "timeout_seconds": 1800,
152
+ "disable_auto_optimization": true
149
153
  }
150
154
  ]
151
155
  }
@@ -6,7 +6,7 @@ resources:
6
6
  tags:
7
7
  pipeloom_job: shop.main.ci
8
8
  pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
9
- pipeloom_settings_hash: f5a10d5fdb1f1d09c10bca06580b2babc243f86e5fc10dfab550d58de85f5b6e
9
+ pipeloom_settings_hash: 1200befe7dce232bdc277d7dfc500b77e5fceecdbf9e926b398df89ab487a216
10
10
  pipeloom_version: 1.2.3
11
11
  parameters:
12
12
  - name: pipeloom_run_id
@@ -47,6 +47,7 @@ resources:
47
47
  - --task-run-id={{task.run_id}}
48
48
  - --start-ms={{job.start_time.timestamp_ms}}
49
49
  max_retries: 0
50
+ disable_auto_optimization: true
50
51
  - task_key: tag_staging
51
52
  environment_key: pipeloom_default
52
53
  python_wheel_task:
@@ -68,6 +69,7 @@ resources:
68
69
  - --task-run-id={{task.run_id}}
69
70
  - --start-ms={{job.start_time.timestamp_ms}}
70
71
  max_retries: 0
72
+ disable_auto_optimization: true
71
73
  - task_key: tag_untagged
72
74
  depends_on:
73
75
  - task_key: tag_mart
@@ -91,6 +93,7 @@ resources:
91
93
  - --task-run-id={{task.run_id}}
92
94
  - --start-ms={{job.start_time.timestamp_ms}}
93
95
  max_retries: 0
96
+ disable_auto_optimization: true
94
97
  - task_key: pipeloom__finalize
95
98
  depends_on:
96
99
  - task_key: tag_untagged
@@ -113,3 +116,4 @@ resources:
113
116
  - --start-ms={{job.start_time.timestamp_ms}}
114
117
  max_retries: 1
115
118
  timeout_seconds: 1800
119
+ disable_auto_optimization: true
@@ -3,7 +3,7 @@
3
3
  "tags": {
4
4
  "pipeloom_job": "shop.main.ci",
5
5
  "pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
6
- "pipeloom_settings_hash": "1e3fd002d6840fe1e17c7a502df544df71c74646b059dac268984f2d6b82dce0",
6
+ "pipeloom_settings_hash": "c5bb6cb107c5cf386d3a0a134aefb8f16268e925290dba8f88a58e5b74868962",
7
7
  "pipeloom_version": "1.2.3",
8
8
  "team": "data"
9
9
  },
@@ -165,7 +165,8 @@
165
165
  "--start-ms={{job.start_time.timestamp_ms}}"
166
166
  ]
167
167
  },
168
- "max_retries": 0
168
+ "max_retries": 0,
169
+ "disable_auto_optimization": true
169
170
  },
170
171
  {
171
172
  "task_key": "stg_customers",
@@ -190,7 +191,8 @@
190
191
  "--start-ms={{job.start_time.timestamp_ms}}"
191
192
  ]
192
193
  },
193
- "max_retries": 0
194
+ "max_retries": 0,
195
+ "disable_auto_optimization": true
194
196
  },
195
197
  {
196
198
  "task_key": "stg_orders",
@@ -215,7 +217,8 @@
215
217
  "--start-ms={{job.start_time.timestamp_ms}}"
216
218
  ]
217
219
  },
218
- "max_retries": 0
220
+ "max_retries": 0,
221
+ "disable_auto_optimization": true
219
222
  },
220
223
  {
221
224
  "task_key": "stg_payments",
@@ -240,7 +243,8 @@
240
243
  "--start-ms={{job.start_time.timestamp_ms}}"
241
244
  ]
242
245
  },
243
- "max_retries": 0
246
+ "max_retries": 0,
247
+ "disable_auto_optimization": true
244
248
  },
245
249
  {
246
250
  "task_key": "pipeloom__finalize",
@@ -272,7 +276,8 @@
272
276
  ]
273
277
  },
274
278
  "max_retries": 1,
275
- "timeout_seconds": 1800
279
+ "timeout_seconds": 1800,
280
+ "disable_auto_optimization": true
276
281
  }
277
282
  ],
278
283
  "schedule": {
@@ -6,7 +6,7 @@ resources:
6
6
  tags:
7
7
  pipeloom_job: shop.main.ci
8
8
  pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
9
- pipeloom_settings_hash: 1e3fd002d6840fe1e17c7a502df544df71c74646b059dac268984f2d6b82dce0
9
+ pipeloom_settings_hash: c5bb6cb107c5cf386d3a0a134aefb8f16268e925290dba8f88a58e5b74868962
10
10
  pipeloom_version: 1.2.3
11
11
  team: data
12
12
  parameters:
@@ -113,6 +113,7 @@ resources:
113
113
  - --task-run-id={{task.run_id}}
114
114
  - --start-ms={{job.start_time.timestamp_ms}}
115
115
  max_retries: 0
116
+ disable_auto_optimization: true
116
117
  - task_key: stg_customers
117
118
  environment_key: pipeloom_default
118
119
  python_wheel_task:
@@ -134,6 +135,7 @@ resources:
134
135
  - --task-run-id={{task.run_id}}
135
136
  - --start-ms={{job.start_time.timestamp_ms}}
136
137
  max_retries: 0
138
+ disable_auto_optimization: true
137
139
  - task_key: stg_orders
138
140
  environment_key: pipeloom_default
139
141
  python_wheel_task:
@@ -155,6 +157,7 @@ resources:
155
157
  - --task-run-id={{task.run_id}}
156
158
  - --start-ms={{job.start_time.timestamp_ms}}
157
159
  max_retries: 0
160
+ disable_auto_optimization: true
158
161
  - task_key: stg_payments
159
162
  environment_key: pipeloom_default
160
163
  python_wheel_task:
@@ -176,6 +179,7 @@ resources:
176
179
  - --task-run-id={{task.run_id}}
177
180
  - --start-ms={{job.start_time.timestamp_ms}}
178
181
  max_retries: 0
182
+ disable_auto_optimization: true
179
183
  - task_key: pipeloom__finalize
180
184
  depends_on:
181
185
  - task_key: dim_customers
@@ -199,6 +203,7 @@ resources:
199
203
  - --start-ms={{job.start_time.timestamp_ms}}
200
204
  max_retries: 1
201
205
  timeout_seconds: 1800
206
+ disable_auto_optimization: true
202
207
  schedule:
203
208
  quartz_cron_expression: 0 0 5 * * ?
204
209
  timezone_id: UTC
@@ -3,7 +3,7 @@
3
3
  "tags": {
4
4
  "pipeloom_job": "shop.main.ci",
5
5
  "pipeloom_plan_hash": "a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac",
6
- "pipeloom_settings_hash": "13e35887eb13be6dd7b3931cec6808b3c7c800d5a0955dbc62e9300198a1f711",
6
+ "pipeloom_settings_hash": "da6b2e5934482e56accfa64dcb70de7be7d491607292f0939cf3b8fbf2aae71c",
7
7
  "pipeloom_version": "1.2.3"
8
8
  },
9
9
  "parameters": [
@@ -64,7 +64,8 @@
64
64
  "--start-ms={{job.start_time.timestamp_ms}}"
65
65
  ]
66
66
  },
67
- "max_retries": 0
67
+ "max_retries": 0,
68
+ "disable_auto_optimization": true
68
69
  },
69
70
  {
70
71
  "task_key": "fct_orders",
@@ -100,7 +101,8 @@
100
101
  "--start-ms={{job.start_time.timestamp_ms}}"
101
102
  ]
102
103
  },
103
- "max_retries": 0
104
+ "max_retries": 0,
105
+ "disable_auto_optimization": true
104
106
  },
105
107
  {
106
108
  "task_key": "rpt_revenue",
@@ -130,7 +132,8 @@
130
132
  "--start-ms={{job.start_time.timestamp_ms}}"
131
133
  ]
132
134
  },
133
- "max_retries": 0
135
+ "max_retries": 0,
136
+ "disable_auto_optimization": true
134
137
  },
135
138
  {
136
139
  "task_key": "stg_customers",
@@ -155,7 +158,8 @@
155
158
  "--start-ms={{job.start_time.timestamp_ms}}"
156
159
  ]
157
160
  },
158
- "max_retries": 0
161
+ "max_retries": 0,
162
+ "disable_auto_optimization": true
159
163
  },
160
164
  {
161
165
  "task_key": "stg_orders",
@@ -180,7 +184,8 @@
180
184
  "--start-ms={{job.start_time.timestamp_ms}}"
181
185
  ]
182
186
  },
183
- "max_retries": 0
187
+ "max_retries": 0,
188
+ "disable_auto_optimization": true
184
189
  },
185
190
  {
186
191
  "task_key": "stg_payments",
@@ -205,7 +210,8 @@
205
210
  "--start-ms={{job.start_time.timestamp_ms}}"
206
211
  ]
207
212
  },
208
- "max_retries": 0
213
+ "max_retries": 0,
214
+ "disable_auto_optimization": true
209
215
  },
210
216
  {
211
217
  "task_key": "pipeloom__finalize",
@@ -237,7 +243,8 @@
237
243
  ]
238
244
  },
239
245
  "max_retries": 1,
240
- "timeout_seconds": 1800
246
+ "timeout_seconds": 1800,
247
+ "disable_auto_optimization": true
241
248
  }
242
249
  ]
243
250
  }
@@ -6,7 +6,7 @@ resources:
6
6
  tags:
7
7
  pipeloom_job: shop.main.ci
8
8
  pipeloom_plan_hash: a2501b77df2ad29619b953019adbd873b9cbe132e6d1b5b2a52f0da6a45ecbac
9
- pipeloom_settings_hash: 13e35887eb13be6dd7b3931cec6808b3c7c800d5a0955dbc62e9300198a1f711
9
+ pipeloom_settings_hash: da6b2e5934482e56accfa64dcb70de7be7d491607292f0939cf3b8fbf2aae71c
10
10
  pipeloom_version: 1.2.3
11
11
  parameters:
12
12
  - name: pipeloom_run_id
@@ -48,6 +48,7 @@ resources:
48
48
  - --task-run-id={{task.run_id}}
49
49
  - --start-ms={{job.start_time.timestamp_ms}}
50
50
  max_retries: 0
51
+ disable_auto_optimization: true
51
52
  - task_key: fct_orders
52
53
  depends_on:
53
54
  - task_key: stg_customers
@@ -73,6 +74,7 @@ resources:
73
74
  - --task-run-id={{task.run_id}}
74
75
  - --start-ms={{job.start_time.timestamp_ms}}
75
76
  max_retries: 0
77
+ disable_auto_optimization: true
76
78
  - task_key: rpt_revenue
77
79
  depends_on:
78
80
  - task_key: fct_orders
@@ -96,6 +98,7 @@ resources:
96
98
  - --task-run-id={{task.run_id}}
97
99
  - --start-ms={{job.start_time.timestamp_ms}}
98
100
  max_retries: 0
101
+ disable_auto_optimization: true
99
102
  - task_key: stg_customers
100
103
  environment_key: pipeloom_default
101
104
  python_wheel_task:
@@ -117,6 +120,7 @@ resources:
117
120
  - --task-run-id={{task.run_id}}
118
121
  - --start-ms={{job.start_time.timestamp_ms}}
119
122
  max_retries: 0
123
+ disable_auto_optimization: true
120
124
  - task_key: stg_orders
121
125
  environment_key: pipeloom_default
122
126
  python_wheel_task:
@@ -138,6 +142,7 @@ resources:
138
142
  - --task-run-id={{task.run_id}}
139
143
  - --start-ms={{job.start_time.timestamp_ms}}
140
144
  max_retries: 0
145
+ disable_auto_optimization: true
141
146
  - task_key: stg_payments
142
147
  environment_key: pipeloom_default
143
148
  python_wheel_task:
@@ -159,6 +164,7 @@ resources:
159
164
  - --task-run-id={{task.run_id}}
160
165
  - --start-ms={{job.start_time.timestamp_ms}}
161
166
  max_retries: 0
167
+ disable_auto_optimization: true
162
168
  - task_key: pipeloom__finalize
163
169
  depends_on:
164
170
  - task_key: dim_customers
@@ -182,3 +188,4 @@ resources:
182
188
  - --start-ms={{job.start_time.timestamp_ms}}
183
189
  max_retries: 1
184
190
  timeout_seconds: 1800
191
+ disable_auto_optimization: true
@@ -30,7 +30,8 @@
30
30
  "--start-ms=1700000000000"
31
31
  ]
32
32
  },
33
- "max_retries": 0
33
+ "max_retries": 0,
34
+ "disable_auto_optimization": true
34
35
  },
35
36
  {
36
37
  "task_key": "fct_orders",
@@ -64,7 +65,8 @@
64
65
  "--start-ms=1700000000000"
65
66
  ]
66
67
  },
67
- "max_retries": 0
68
+ "max_retries": 0,
69
+ "disable_auto_optimization": true
68
70
  },
69
71
  {
70
72
  "task_key": "rpt_revenue",
@@ -92,7 +94,8 @@
92
94
  "--start-ms=1700000000000"
93
95
  ]
94
96
  },
95
- "max_retries": 0
97
+ "max_retries": 0,
98
+ "disable_auto_optimization": true
96
99
  },
97
100
  {
98
101
  "task_key": "stg_customers",
@@ -115,7 +118,8 @@
115
118
  "--start-ms=1700000000000"
116
119
  ]
117
120
  },
118
- "max_retries": 0
121
+ "max_retries": 0,
122
+ "disable_auto_optimization": true
119
123
  },
120
124
  {
121
125
  "task_key": "stg_orders",
@@ -138,7 +142,8 @@
138
142
  "--start-ms=1700000000000"
139
143
  ]
140
144
  },
141
- "max_retries": 0
145
+ "max_retries": 0,
146
+ "disable_auto_optimization": true
142
147
  },
143
148
  {
144
149
  "task_key": "stg_payments",
@@ -161,7 +166,8 @@
161
166
  "--start-ms=1700000000000"
162
167
  ]
163
168
  },
164
- "max_retries": 0
169
+ "max_retries": 0,
170
+ "disable_auto_optimization": true
165
171
  },
166
172
  {
167
173
  "task_key": "pipeloom__finalize",
@@ -191,7 +197,8 @@
191
197
  ]
192
198
  },
193
199
  "max_retries": 1,
194
- "timeout_seconds": 1800
200
+ "timeout_seconds": 1800,
201
+ "disable_auto_optimization": true
195
202
  }
196
203
  ],
197
204
  "environments": [
@@ -40,10 +40,12 @@ def finalize_argv(s: Sim, fake: FakeDatabricks, *, job_run_id: str, run_id: str)
40
40
  f"--job-run-id={job_run_id}",
41
41
  "--start-ms=1700000000000",
42
42
  ]
43
- with pytest.raises(SystemExit) as stop:
43
+ try:
44
44
  main(argv, client_factory=lambda: fake)
45
- assert isinstance(stop.value.code, int)
46
- return stop.value.code
45
+ except SystemExit as stop:
46
+ assert isinstance(stop.code, int)
47
+ return stop.code
48
+ return 0
47
49
 
48
50
 
49
51
  def run_of(*tasks: TaskRunInfo) -> RunInfo:
@@ -57,6 +59,7 @@ KEYS = {"a": "ta", "b": "tb", "c": "tc"}
57
59
  ("result", "expected"),
58
60
  [
59
61
  ("EXCLUDED", ExternalUnitState("excluded")),
62
+ ("DISABLED", ExternalUnitState("excluded")),
60
63
  ("UPSTREAM_FAILED", "skipped"),
61
64
  ("UPSTREAM_CANCELED", "skipped"),
62
65
  ("CANCELED", "cancelled"),
@@ -47,6 +47,17 @@ def test_the_rerun_set_is_every_unsuccessful_task_plus_finalize() -> None:
47
47
  assert "SUCCESS" not in REPAIRABLE and "EXCLUDED" not in REPAIRABLE
48
48
 
49
49
 
50
+ def test_a_repair_passes_the_run_job_parameters_again() -> None:
51
+ # without them the repair resolves {{job.parameters.pipeloom_run_id}} to its empty default
52
+ fake = FakeDatabricks()
53
+ params = {"pipeloom_run_id": "nightly.1", "full_refresh": "false"}
54
+ fake.set_run(
55
+ 50, job_id=1, result_state="FAILED", tasks=(t("a", "FAILED"),), job_parameters=params
56
+ )
57
+ repair(fake, 50, Report())
58
+ assert fake.repair_parameters == [params]
59
+
60
+
50
61
  def test_finalize_joins_the_set_when_a_unit_failed_even_if_it_succeeded() -> None:
51
62
  fake = FakeDatabricks()
52
63
  finished(fake, t("a", "FAILED"), t(FINALIZE_KEY, "SUCCESS"))
@@ -227,8 +227,11 @@ def test_repair_run_body() -> None:
227
227
  "rerun_tasks": ["a", "b"],
228
228
  "rerun_dependent_tasks": True,
229
229
  }
230
- sdk.repair_run(11, rerun_tasks=["a"], latest_repair_id=31)
230
+ sdk.repair_run(
231
+ 11, rerun_tasks=["a"], latest_repair_id=31, job_parameters={"pipeloom_run_id": "r"}
232
+ )
231
233
  assert api.calls[1]["body"]["latest_repair_id"] == 31
234
+ assert api.calls[1]["body"]["job_parameters"] == {"pipeloom_run_id": "r"}
232
235
 
233
236
 
234
237
  def test_workspace_calls() -> None:
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import json
6
6
  import sys
7
7
  from collections.abc import Callable
8
+ from dataclasses import replace
8
9
  from types import ModuleType
9
10
  from typing import Any
10
11
 
@@ -15,6 +16,7 @@ from pydantic import JsonValue
15
16
 
16
17
  import pipeloom_databricks
17
18
  from pipeloom_databricks import task
19
+ from pipeloom_databricks.client import ApiError
18
20
  from pipeloom_databricks.task import (
19
21
  check_resolved,
20
22
  cli,
@@ -28,6 +30,7 @@ from pipeloom_databricks.task_values import (
28
30
  runtime_task_values,
29
31
  summary,
30
32
  )
33
+ from pipeloom_databricks.testing.fake import FakeDatabricks
31
34
 
32
35
  Sim = Any
33
36
  HASH = "0" * 64
@@ -160,7 +163,7 @@ def test_task_values_are_published_and_never_change_the_exit_code(
160
163
  assert "task value not set: RuntimeError" in r.stderr
161
164
 
162
165
 
163
- def test_main_ends_in_system_exit_and_the_lazy_entry_reads_argv(
166
+ def test_main_exits_only_on_failure_and_the_lazy_entry_reads_argv(
164
167
  sim: Callable[..., Sim], monkeypatch: pytest.MonkeyPatch
165
168
  ) -> None:
166
169
  s = sim()
@@ -168,9 +171,25 @@ def test_main_ends_in_system_exit_and_the_lazy_entry_reads_argv(
168
171
  task.main(unit_args(s, unit="c", **{"task-key": "c"}))
169
172
  assert stop.value.code == 1
170
173
  monkeypatch.setattr(sys, "argv", ["pipeloom_databricks_task", *unit_args(s)])
171
- with pytest.raises(SystemExit) as ok:
172
- pipeloom_databricks.pipeloom_databricks_task()
173
- assert ok.value.code == 0
174
+ # success returns: serverless (IPython) reports even SystemExit(0) as a failed task
175
+ pipeloom_databricks.pipeloom_databricks_task()
176
+
177
+
178
+ def test_the_run_start_comes_from_the_jobs_api_not_the_repair_start(
179
+ capsys: pytest.CaptureFixture[str],
180
+ ) -> None:
181
+ # {{job.start_time}} is the start of the latest repair; the run keeps its own start
182
+ fake = FakeDatabricks()
183
+ fake.set_run(5, job_id=1, result_state="FAILED")
184
+ fake.runs[5] = replace(fake.runs[5], start_time_ms=1700000000000)
185
+ ctx = click.Context(task.cli, obj=task._Context(None, lambda: fake))
186
+ repair_start = {"start_ms": "1700000090000", "job_run_id": "5"}
187
+ assert task._run_start(ctx, repair_start) == started_at("1700000000000")
188
+ # a one-time run has no job run id: the literal start stays
189
+ assert task._run_start(ctx, {**repair_start, "job_run_id": ""}) == started_at("1700000090000")
190
+ fake.fail["get_run"] = ApiError(500, "unavailable")
191
+ assert task._run_start(ctx, repair_start) == started_at("1700000090000")
192
+ assert "warning: run start not read, using --start-ms" in capsys.readouterr().err
174
193
 
175
194
 
176
195
  # -- helpers -------------------------------------------------------------------------------------