sparkforensics-operator 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/.github/workflows/ci.yml +30 -0
  2. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/AGENTS.md +2 -2
  3. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/ARCHITECTURE.md +62 -20
  4. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/CHANGELOG.md +35 -1
  5. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/CLAUDE.md +2 -2
  6. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/CONTRIBUTING.md +25 -2
  7. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/PKG-INFO +10 -7
  8. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/README.md +9 -6
  9. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/configuration.md +196 -61
  10. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/troubleshooting.md +86 -25
  11. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/pyproject.toml +1 -1
  12. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/callback.py +28 -7
  13. sparkforensics_operator-0.4.0/src/sparkforensics_operator/exceptions.py +29 -0
  14. sparkforensics_operator-0.4.0/src/sparkforensics_operator/hooks/analyze/_cli.py +220 -0
  15. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/_remote_job.py +38 -4
  16. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/base.py +16 -36
  17. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/ssh.py +94 -14
  18. sparkforensics_operator-0.4.0/src/sparkforensics_operator/hooks/analyze/subprocess.py +89 -0
  19. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/notify.py +25 -4
  20. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/operator.py +173 -34
  21. sparkforensics_operator-0.4.0/src/sparkforensics_operator/report.py +207 -0
  22. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/sinks.py +5 -1
  23. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/summary.py +12 -2
  24. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_base.py +17 -25
  25. sparkforensics_operator-0.4.0/tests/hooks/analyze/test_cli.py +394 -0
  26. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_ssh.py +66 -3
  27. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_ssh_deferrable.py +151 -0
  28. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_subprocess.py +131 -6
  29. sparkforensics_operator-0.4.0/tests/integration/__init__.py +0 -0
  30. sparkforensics_operator-0.4.0/tests/integration/conftest.py +70 -0
  31. sparkforensics_operator-0.4.0/tests/integration/test_real_cli.py +101 -0
  32. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_callback.py +3 -0
  33. sparkforensics_operator-0.4.0/tests/test_notify.py +92 -0
  34. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_operator.py +346 -0
  35. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_operator_deferrable.py +59 -1
  36. sparkforensics_operator-0.4.0/tests/test_report.py +392 -0
  37. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_sinks.py +25 -0
  38. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_summary.py +79 -0
  39. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_templating.py +47 -0
  40. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tox.ini +2 -0
  41. sparkforensics_operator-0.3.0/src/sparkforensics_operator/exceptions.py +0 -10
  42. sparkforensics_operator-0.3.0/src/sparkforensics_operator/hooks/analyze/_cli.py +0 -101
  43. sparkforensics_operator-0.3.0/src/sparkforensics_operator/hooks/analyze/subprocess.py +0 -63
  44. sparkforensics_operator-0.3.0/src/sparkforensics_operator/report.py +0 -129
  45. sparkforensics_operator-0.3.0/tests/hooks/analyze/test_cli.py +0 -49
  46. sparkforensics_operator-0.3.0/tests/test_notify.py +0 -42
  47. sparkforensics_operator-0.3.0/tests/test_report.py +0 -180
  48. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/.github/workflows/publish.yml +0 -0
  49. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/.gitignore +0 -0
  50. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/LICENSE +0 -0
  51. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/constraints-airflow-2.6-floor.txt +0 -0
  52. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/constraints-deferrable-floor.txt +0 -0
  53. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/coverage-no-ssh.ini +0 -0
  54. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/getting-started.md +0 -0
  55. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/index.md +0 -0
  56. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/__init__.py +0 -0
  57. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/_compat.py +0 -0
  58. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/get_provider_info.py +0 -0
  59. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/__init__.py +0 -0
  60. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/_templating.py +0 -0
  61. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/__init__.py +0 -0
  62. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/__init__.py +0 -0
  63. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_dest_root.py +0 -0
  64. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_history_server_args.py +0 -0
  65. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_path_template.py +0 -0
  66. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_rolling_log.py +0 -0
  67. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/base.py +0 -0
  68. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/filesystem.py +0 -0
  69. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/history_server.py +0 -0
  70. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/history_server_app.py +0 -0
  71. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/remote_path.py +0 -0
  72. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/sftp.py +0 -0
  73. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/tunnel.py +0 -0
  74. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/xcom.py +0 -0
  75. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/links.py +0 -0
  76. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/log_ref.py +0 -0
  77. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/__init__.py +0 -0
  78. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/_render.py +0 -0
  79. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/README.md +0 -0
  80. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/comparison.json +0 -0
  81. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/comparison.stderr +0 -0
  82. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/incomplete-run.json +0 -0
  83. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/incomplete-run.stderr +0 -0
  84. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/report.json +0 -0
  85. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/report.stderr +0 -0
  86. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/__init__.py +0 -0
  87. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/__init__.py +0 -0
  88. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/_local_ssh.py +0 -0
  89. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_deferrable_parity.py +0 -0
  90. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/__init__.py +0 -0
  91. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_base.py +0 -0
  92. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_filesystem.py +0 -0
  93. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_history_server.py +0 -0
  94. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_history_server_app.py +0 -0
  95. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_remote_path.py +0 -0
  96. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_sftp.py +0 -0
  97. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_tunnel.py +0 -0
  98. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_xcom.py +0 -0
  99. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_compat.py +0 -0
  100. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_dagbag_integration.py +0 -0
  101. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_exceptions.py +0 -0
  102. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_links.py +0 -0
  103. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_log_ref.py +0 -0
  104. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_package_exports.py +0 -0
  105. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_provider_info.py +0 -0
  106. {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_serialization.py +0 -0
@@ -5,6 +5,10 @@ on:
5
5
  branches: [main]
6
6
  pull_request:
7
7
 
8
+ concurrency:
9
+ group: ${{ github.workflow }}-${{ github.ref }}
10
+ cancel-in-progress: true
11
+
8
12
  jobs:
9
13
  test:
10
14
  runs-on: ubuntu-latest
@@ -40,6 +44,32 @@ jobs:
40
44
  path-to-lcov: coverage-${{ matrix.tox-env }}.lcov
41
45
  parallel: true
42
46
 
47
+ # The rest of the suite simulates the CLI. This job runs the real one on
48
+ # public corpus logs, so a change in its exit codes or flag rules fails
49
+ # here. SPARKFORENSICS_REQUIRE_CLI turns a missing CLI or an unreachable
50
+ # corpus into a failure instead of a skip.
51
+ cli-integration:
52
+ runs-on: ubuntu-latest
53
+ strategy:
54
+ fail-fast: false
55
+ matrix:
56
+ # The release the operator is tested against (TESTED_CLI_VERSION in
57
+ # report.py), and the newest one.
58
+ cli: ["0.12.0", "latest"]
59
+ env:
60
+ SPARKFORENSICS_REQUIRE_CLI: "1"
61
+ steps:
62
+ - uses: actions/checkout@v4
63
+ - uses: actions/setup-python@v5
64
+ with:
65
+ python-version: "3.12"
66
+ - uses: actions/setup-node@v4
67
+ with:
68
+ node-version: "22"
69
+ - run: npm install -g sparkforensics-cli@${{ matrix.cli }}
70
+ - run: pip install tox
71
+ - run: tox -e py312-airflow3 -- tests/integration -v
72
+
43
73
  finish:
44
74
  needs: test
45
75
  runs-on: ubuntu-latest
@@ -15,8 +15,8 @@ SSH analysis as the SSH provider's detached remote job (`ARCHITECTURE.md`'s
15
15
 
16
16
  ## Commands
17
17
  - `pip install -e ".[test,s3,ssh]"`, install for local dev.
18
- - `pytest`, run the test suite (mocks everything: no live Spark/Airflow/Node
19
- needed for any single test).
18
+ - `pytest`, run the test suite (mocks everything except `tests/integration`,
19
+ which runs the real CLI on public logs and skips without it).
20
20
  - `tox -e py311-airflow2` / `tox -e py311-airflow3`, run the suite against a
21
21
  specific Airflow major version.
22
22
 
@@ -34,11 +34,13 @@ downloaded) fails fast instead of analyzing the wrong file.
34
34
 
35
35
  A run can also be compared against an earlier one: `baseline_log_source`
36
36
  is a second `LogSourceHook`, located and cleaned up next to `log_source`,
37
- and the backend passes its reference to the CLI as `--baseline`. The CLI
38
- reads a baseline only from a path, never from a History Server, so each
39
- backend declares the kinds it can pass that way
40
- (`AnalyzeHook.supported_baseline_refs`: `LocalEventLog` on the worker,
41
- `RemoteEventLog` on the SSH host) and `analyze()` or the deferrable
37
+ and the backend passes its reference to the CLI as `--baseline` (an event
38
+ log path) or `--baseline-shs-base-url`/`--baseline-app-id`/
39
+ `--baseline-attempt-id` (a History Server application, which the CLI
40
+ fetches itself, like the run under analysis). Each backend declares the
41
+ kinds it can pass (`AnalyzeHook.supported_baseline_refs`: `LocalEventLog`
42
+ and `HistoryServerApp` on the worker, `RemoteEventLog` on its own host and
43
+ `HistoryServerApp` on the SSH host), and `analyze()` or the deferrable
42
44
  submit step rejects any other before running anything. The comparison
43
45
  itself, and the budgets on it, are the CLI's. Which run is the baseline
44
46
  is the DAG author's choice, made through that log source: the operator
@@ -125,12 +127,31 @@ breach that raises still gets a notification out first.
125
127
  execution" below.
126
128
  - `hooks/analyze/_cli.py`, the CLI contract every backend shares: argument
127
129
  building for each reference kind (a positional path, or
128
- `--shs-base-url`/`--app-id`/`--attempt-id`), `--baseline`, threshold flags, exit-code
129
- handling (0/1/3 usable, 2 and anything else raise), and building the
130
- `Report` from the JSON report and stderr threshold lines. Backends only
131
- differ in how they run the command and collect its output.
130
+ `--shs-base-url`/`--app-id`/`--attempt-id`), `--baseline`, threshold flags
131
+ (a dict of per-metric budgets becomes one `--regression-budget` or
132
+ `--stage-regression-budget <metric>:<pct>` each), the comparison tuning
133
+ flags (`--normalize-path` and `--ignore-conf` once per pattern,
134
+ `--noise-band-pct`), the `--redact` switch, the Markdown run (below),
135
+ exit-code handling, and building the `Report` from the JSON report and
136
+ stderr threshold lines. Exit codes 0, 1 and 3 carry a report. Every other
137
+ code raises a `CliError` with its own message: 2 (usage error), 4
138
+ (unreadable event log or failed History Server fetch), 5 (unreadable
139
+ baseline), 6 (internal error), anything else "unexpected". The CLI's
140
+ stderr separates a History Server failure a retry can clear
141
+ (`HistoryServerFetchError`) from one it cannot (`CliConfigurationError`,
142
+ an `AirflowFailException`, so Airflow does not retry it). Backends differ
143
+ only in how they run the command and collect its output.
132
144
  - `hooks/analyze/subprocess.py`, `SubprocessAnalyzeHook`, runs the CLI on
133
- the worker with `--out <tmpfile>`.
145
+ the worker with `--out <tmpfile>`. The CLI writes one report format per
146
+ run, so for the Markdown report (`thresholds["markdown"]`, set by
147
+ `markdown_dest`) the backend runs it a second time with `--format md` and
148
+ no budget flags, and sets `Report.markdown`
149
+ (`AnalyzeHook.supports_markdown`). The first run's report and thresholds
150
+ decide the outcome; the second run only adds the text. `SSHAnalyzeHook`
151
+ does the same on the host: a second synchronous run or, in a deferred
152
+ job, a second command in the same remote job. Its outputs (report, stderr,
153
+ exit code) go to their own files, and the job's exit code stays the
154
+ first run's.
134
155
  - `hooks/analyze/ssh.py`, `SSHAnalyzeHook`, runs the CLI on the SSH host
135
156
  over `SSHHook` (imported lazily, `ssh` extra) and reads the report from
136
157
  stdout, so nothing is written on that host (see "Killing a task" below
@@ -156,17 +177,35 @@ breach that raises still gets a notification out first.
156
177
  which reports the CLI's pid, and the command that stops it.
157
178
  - `report.py`, `Report`/`ThresholdResult` dataclasses, plus parsing of the
158
179
  CLI's JSON report (`--out` file or stdout) and its stderr threshold
159
- lines. No
180
+ lines. Several regression budgets print as `max-regression` lines
181
+ (`max-stage-regression` for the stage budgets), and
182
+ `parse_threshold_results()` keeps every line it reads, so a name can have
183
+ several results. It refuses a `schemaVersion` outside
184
+ `MIN_SCHEMA_VERSION` to `MAX_SCHEMA_VERSION` (an
185
+ `UnsupportedReportSchema` that `build_report()` turns into a
186
+ `CliConfigurationError`), reads the CLI's version from the report's
187
+ `generator`, and checks it against the release the operator is tested
188
+ against (`TESTED_CLI_VERSION`). No
160
189
  threshold-evaluation *logic* lives here: sparkforensics's own CLI decides
161
190
  pass/violation/inconclusive, this module only parses what it printed, so
162
191
  it can never drift from what the CLI actually enforces.
163
192
  - `exceptions.py`, `ThresholdBreached`, an `AirflowFailException` so a
164
- breach fails the task without consuming its retries.
165
- - `sinks.py`, persists the report JSON (local path, `file://`, `s3://`).
193
+ breach fails the task without consuming its retries, and the CLI
194
+ errors: `CliError` (a CLI exit code with no report, carrying
195
+ `exit_code`), its `HistoryServerFetchError` (retried) and
196
+ `CliConfigurationError` (an `AirflowFailException`, not retried).
197
+ - `sinks.py`, persists the report JSON (local path, `file://`, `s3://`),
198
+ and, through `persist_text()`, the Markdown report. The JSON is the
199
+ CLI's whole report (`Report.raw`) plus the operator's `thresholdResults`
200
+ and the comparison section, so a field the CLI adds reaches the file
201
+ without a change here. `operator._report_json()` builds it; a `Report`
202
+ built by hand (no `raw`) persists the sections the operator reads.
166
203
  - `notify.py`, the `Notifier` base class: subclasses implement `_send()`
167
204
  for whatever messaging/paging system they target, and `notify()` makes
168
- that best-effort, never raising. No concrete notifier ships in this
169
- package.
205
+ that best-effort, never raising. `_default_message()` writes the
206
+ one-line summary (application, verdict, run time, finding counts,
207
+ skipped lines, destination), leaving out what the report lacks. No
208
+ concrete notifier ships in this package.
170
209
  - `summary.py`, the summary XCom (`SUMMARY_XCOM_KEY`,
171
210
  `build_summary()`) and the report URL (`render_report_url()`,
172
211
  `validate_report_url_template()`). See "Summary XCom and report URL"
@@ -271,9 +310,9 @@ reboot is left alone.
271
310
 
272
311
  Failure paths. A trigger error event without an exit code makes
273
312
  `collect()` stop and remove the job before raising. A deferral that times
274
- out (the backend's `defer_timeout`, `timeout` plus 120 seconds counted
275
- from submission, or the task's
276
- `execution_timeout`) resumes with `next_method="__fail__"`, which never
313
+ out (the backend's `defer_timeout`, `timeout` per CLI run in the job, so
314
+ twice with `markdown_dest`, plus 120 seconds counted from submission, or
315
+ the task's `execution_timeout`) resumes with `next_method="__fail__"`, which never
277
316
  reaches `execute_complete()`; `resume_execution()` is overridden to call
278
317
  `backend.abandon(context, job)` first, with the job from XCom. Airflow 2
279
318
  takes a different path when `execution_timeout` ran out during the
@@ -374,8 +413,11 @@ Before notifying and before any threshold breach raises,
374
413
  `handle_report()` pushes a summary under the `sparkforensics_summary`
375
414
  XCom key: the schema version, the destination, the report URL, the
376
415
  impact-band counts, whether the run violated a threshold, the breached
377
- and inconclusive threshold names, and the CLI exit code. A run compared
378
- against a baseline adds a `comparison` entry: the direction the CLI
416
+ and inconclusive threshold names, the CLI exit code, the CLI version
417
+ that wrote the report, and the run's verdict title, application name,
418
+ wall-clock time and skipped line count (`Report.verdict_title`,
419
+ `app_name`, `run_time_ms` and `skipped_lines`, read from the CLI's
420
+ summary). A run compared against a baseline adds a `comparison` entry: the direction the CLI
379
421
  computed for one metric (`regression_metric`, wall-clock by default) as
380
422
  the verdict, the comparison's confidence and reason, and that metric's
381
423
  values and change in percent, taken from the CLI's comparison section.
@@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.4.0] - 2026-11-04
11
+
12
+ ### Upgrade notes
13
+
14
+ - `sparkforensics-cli` 0.10.2 counts every failed task attempt in `max_failed_task_rate_pct`, so a job with retried tasks can fail where it passed.
15
+ - With 0.10.2, a log with skipped lines gives exit 3 and an inconclusive `parse-incomplete` result. The task does not fail; see `skipped_lines` in the summary XCom.
16
+ - The operator is tested against `sparkforensics-cli` 0.12.0.
17
+ - With CLI 0.12.0 and newer, `findings` lists critical and warning findings only. The persisted report, `report.findings` and the `findings` XCom omit info findings; `notesHidden` in the report counts them. The summary's `impact_band_counts` is unchanged.
18
+
19
+ ### Added
20
+
21
+ - `markdown_dest` saves the CLI's Markdown report (`--format md`) next to the JSON report. The analysis runs twice.
22
+ - `redact=True` passes `--redact`.
23
+ - `normalize_path`, `ignore_conf` and `noise_band_pct` tune the baseline comparison.
24
+ - `regression_budgets`, `stage_regression_budgets` and `stage_qualities` set several regression budgets at once.
25
+ - A Spark History Server application can be the baseline on `SubprocessAnalyzeHook` and `SSHAnalyzeHook`.
26
+ - The summary XCom has `verdict`, `app_name`, `run_time_ms`, `skipped_lines` and `cli_version`. The default notification names the application, the verdict and the run time.
27
+ - A warning is logged when the CLI is older than the tested version.
28
+ - `CliError`, `CliConfigurationError` and `HistoryServerFetchError` in `sparkforensics_operator.exceptions`.
29
+
30
+ ### Changed
31
+
32
+ - The operator reads the JSON output shape of `sparkforensics-cli` 0.12.0 (`{schemaVersion, run, comparison, budgets}`) and still reads the older shapes.
33
+ - Each CLI exit code without a report (2, 4, 5, 6) has its own error message. The task still fails.
34
+ - Airflow does not retry usage errors, and retries History Server failures that can clear.
35
+ - A report with a newer `schemaVersion` than the operator reads fails the task.
36
+ - The saved report is the CLI's whole JSON report. `Report.raw` holds it. It includes `effectiveConf`, where the CLI masks secret-looking keys and values; see [Report destinations](docs/configuration.md#report-destinations).
37
+ - `Report.threshold_results` keeps every result of a threshold the CLI reports on several lines.
38
+
39
+ ### Fixed
40
+
41
+ - Thresholds such as `0.00001` or `1e16` reach the CLI as plain decimals, not exponent form, which the CLI rejects.
42
+
10
43
  ## [0.3.0] - 2026-09-28
11
44
 
12
45
  ### Added
@@ -188,7 +221,8 @@ Initial release.
188
221
  - Generic `Notifier` interface for threshold-breach alerting.
189
222
  - PyPI trusted-publishing GitHub Actions workflow (OIDC, no stored token).
190
223
 
191
- [Unreleased]: https://github.com/shuffle-works/sparkforensics-operator/compare/v0.3.0...HEAD
224
+ [Unreleased]: https://github.com/shuffle-works/sparkforensics-operator/compare/v0.4.0...HEAD
225
+ [0.4.0]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.4.0
192
226
  [0.3.0]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.3.0
193
227
  [0.2.0]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.2.0
194
228
  [0.1.1]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.1.1
@@ -13,8 +13,8 @@ that backends consume; see `ARCHITECTURE.md`.
13
13
 
14
14
  ## Commands
15
15
  - `pip install -e ".[test,s3,ssh]"`, install for local dev.
16
- - `pytest`, run the test suite (mocks everything: no live Spark/Airflow/Node
17
- needed for any single test).
16
+ - `pytest`, run the test suite (mocks everything except `tests/integration`,
17
+ which runs the real CLI on public logs and skips without it).
18
18
  - `tox -e py311-airflow2` / `tox -e py311-airflow3`, run the suite against a
19
19
  specific Airflow major version.
20
20
 
@@ -15,13 +15,36 @@ pip install -e ".[test,s3,ssh]"
15
15
  pytest
16
16
  ```
17
17
 
18
- The suite needs no live Spark, Airflow or Node.js. The one real process is
19
- a shell: the deferred-mode tests run the SSH provider's real shell wrapper against a local `sh` standing in
18
+ The suite needs no live Spark or Airflow, and no Node.js apart from
19
+ `tests/integration` (below). The one real process is a shell: the deferred-mode tests run the SSH provider's real shell wrapper against a local `sh` standing in
20
20
  for the SSH host (`tests/hooks/analyze/_local_ssh.py`), so they need
21
21
  `bash`, `setsid` and coreutils `timeout` on the machine running the tests.
22
22
  The kill and sweep tests also run with procps hidden
23
23
  (`LocalHost.hide_procps()`), as on slim images.
24
24
 
25
+ ### Tests against the real CLI
26
+
27
+ The other tests simulate `sparkforensics-analyze`, so they cannot catch the
28
+ CLI changing an exit code or a flag's rules. `tests/integration` runs the
29
+ real CLI on public logs from the
30
+ [spark-event-corpus-data](https://github.com/shuffle-works/spark-event-corpus-data)
31
+ repository (fetched at a pinned tag), through `SubprocessAnalyzeHook`. It
32
+ covers a clean log, a violation, an unreadable log, an unreadable baseline
33
+ and budgets that print in exponent form.
34
+
35
+ ```bash
36
+ npm install -g sparkforensics-cli
37
+ pytest tests/integration
38
+ ```
39
+
40
+ They need Node.js 18+, the `sparkforensics-cli` package on `PATH` (or
41
+ its binary at `SPARKFORENSICS_ANALYZE_BIN`) and network access to GitHub, and
42
+ skip when the CLI or the network is missing. With `SPARKFORENSICS_REQUIRE_CLI=1`
43
+ a missing CLI or an unreachable corpus fails the run instead, as in CI. The
44
+ `cli-integration` job in `.github/workflows/ci.yml` runs them against the
45
+ release the operator is tested against (`TESTED_CLI_VERSION` in
46
+ `report.py`) and the newest one.
47
+
25
48
  ## The tox Airflow matrix
26
49
 
27
50
  `tox.ini` runs the suite against each supported Airflow major version:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sparkforensics-operator
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Airflow operator that runs sparkforensics analysis on a Spark job's event log and acts on the result.
5
5
  Project-URL: Homepage, https://github.com/shuffle-works/sparkforensics-operator
6
6
  Project-URL: Repository, https://github.com/shuffle-works/sparkforensics-operator
@@ -73,12 +73,15 @@ runs as part of the DAG, right where the Spark task just ran.
73
73
  the same verdict.
74
74
  - Compare a run against an earlier one. `baseline_log_source` takes any
75
75
  log source that resolves to an event log path the analysis can read
76
- (`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull),
77
- and `max_regression_pct` fails the DAG when runtime, or the metric
78
- `regression_metric` names, regressed past that budget.
79
- `fail_on_introduced` fails it when the run gained new findings. The
80
- comparison's verdict, confidence and measured regression land in the
81
- summary XCom.
76
+ (`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull) or
77
+ to a Spark History Server application. `max_regression_pct` fails the
78
+ DAG when runtime, or the metric `regression_metric` names, regressed past
79
+ that budget. `regression_budgets` budgets several metrics at once, and
80
+ `stage_regression_budgets` the paired stages. `fail_on_introduced` fails
81
+ it when the run gained new findings. The comparison's verdict,
82
+ confidence and measured regression land in the summary XCom.
83
+ - `markdown_dest` also persists the CLI's Markdown report, for a notifier
84
+ to post or a person to read.
82
85
  - Reports persist to a local path, `file://`, or `s3://`, and a clickable
83
86
  "SparkForensics report" link shows up on the task in the Airflow UI. Set
84
87
  `report_url_template` (an S3 console URL, say) to make it open in a
@@ -38,12 +38,15 @@ runs as part of the DAG, right where the Spark task just ran.
38
38
  the same verdict.
39
39
  - Compare a run against an earlier one. `baseline_log_source` takes any
40
40
  log source that resolves to an event log path the analysis can read
41
- (`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull),
42
- and `max_regression_pct` fails the DAG when runtime, or the metric
43
- `regression_metric` names, regressed past that budget.
44
- `fail_on_introduced` fails it when the run gained new findings. The
45
- comparison's verdict, confidence and measured regression land in the
46
- summary XCom.
41
+ (`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull) or
42
+ to a Spark History Server application. `max_regression_pct` fails the
43
+ DAG when runtime, or the metric `regression_metric` names, regressed past
44
+ that budget. `regression_budgets` budgets several metrics at once, and
45
+ `stage_regression_budgets` the paired stages. `fail_on_introduced` fails
46
+ it when the run gained new findings. The comparison's verdict,
47
+ confidence and measured regression land in the summary XCom.
48
+ - `markdown_dest` also persists the CLI's Markdown report, for a notifier
49
+ to post or a person to read.
47
50
  - Reports persist to a local path, `file://`, or `s3://`, and a clickable
48
51
  "SparkForensics report" link shows up on the task in the Airflow UI. Set
49
52
  `report_url_template` (an S3 console URL, say) to make it open in a