sparkforensics-operator 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/.github/workflows/ci.yml +30 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/AGENTS.md +2 -2
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/ARCHITECTURE.md +62 -20
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/CHANGELOG.md +35 -1
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/CLAUDE.md +2 -2
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/CONTRIBUTING.md +25 -2
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/PKG-INFO +10 -7
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/README.md +9 -6
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/configuration.md +196 -61
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/troubleshooting.md +86 -25
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/pyproject.toml +1 -1
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/callback.py +28 -7
- sparkforensics_operator-0.4.0/src/sparkforensics_operator/exceptions.py +29 -0
- sparkforensics_operator-0.4.0/src/sparkforensics_operator/hooks/analyze/_cli.py +220 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/_remote_job.py +38 -4
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/base.py +16 -36
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/ssh.py +94 -14
- sparkforensics_operator-0.4.0/src/sparkforensics_operator/hooks/analyze/subprocess.py +89 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/notify.py +25 -4
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/operator.py +173 -34
- sparkforensics_operator-0.4.0/src/sparkforensics_operator/report.py +207 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/sinks.py +5 -1
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/summary.py +12 -2
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_base.py +17 -25
- sparkforensics_operator-0.4.0/tests/hooks/analyze/test_cli.py +394 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_ssh.py +66 -3
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_ssh_deferrable.py +151 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_subprocess.py +131 -6
- sparkforensics_operator-0.4.0/tests/integration/__init__.py +0 -0
- sparkforensics_operator-0.4.0/tests/integration/conftest.py +70 -0
- sparkforensics_operator-0.4.0/tests/integration/test_real_cli.py +101 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_callback.py +3 -0
- sparkforensics_operator-0.4.0/tests/test_notify.py +92 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_operator.py +346 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_operator_deferrable.py +59 -1
- sparkforensics_operator-0.4.0/tests/test_report.py +392 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_sinks.py +25 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_summary.py +79 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_templating.py +47 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tox.ini +2 -0
- sparkforensics_operator-0.3.0/src/sparkforensics_operator/exceptions.py +0 -10
- sparkforensics_operator-0.3.0/src/sparkforensics_operator/hooks/analyze/_cli.py +0 -101
- sparkforensics_operator-0.3.0/src/sparkforensics_operator/hooks/analyze/subprocess.py +0 -63
- sparkforensics_operator-0.3.0/src/sparkforensics_operator/report.py +0 -129
- sparkforensics_operator-0.3.0/tests/hooks/analyze/test_cli.py +0 -49
- sparkforensics_operator-0.3.0/tests/test_notify.py +0 -42
- sparkforensics_operator-0.3.0/tests/test_report.py +0 -180
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/.github/workflows/publish.yml +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/.gitignore +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/LICENSE +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/constraints-airflow-2.6-floor.txt +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/constraints-deferrable-floor.txt +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/coverage-no-ssh.ini +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/getting-started.md +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/docs/index.md +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/_compat.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/get_provider_info.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/_templating.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/analyze/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_dest_root.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_history_server_args.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_path_template.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/_rolling_log.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/base.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/filesystem.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/history_server.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/history_server_app.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/remote_path.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/sftp.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/tunnel.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/hooks/log_source/xcom.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/links.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/src/sparkforensics_operator/log_ref.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/_render.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/README.md +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/comparison.json +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/comparison.stderr +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/incomplete-run.json +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/incomplete-run.stderr +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/report.json +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/fixtures/sparkforensics_cli_0_4_0/report.stderr +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/_local_ssh.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/analyze/test_deferrable_parity.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/__init__.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_base.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_filesystem.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_history_server.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_history_server_app.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_remote_path.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_sftp.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_tunnel.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/hooks/log_source/test_xcom.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_compat.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_dagbag_integration.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_exceptions.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_links.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_log_ref.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_package_exports.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_provider_info.py +0 -0
- {sparkforensics_operator-0.3.0 → sparkforensics_operator-0.4.0}/tests/test_serialization.py +0 -0
|
@@ -5,6 +5,10 @@ on:
|
|
|
5
5
|
branches: [main]
|
|
6
6
|
pull_request:
|
|
7
7
|
|
|
8
|
+
concurrency:
|
|
9
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
10
|
+
cancel-in-progress: true
|
|
11
|
+
|
|
8
12
|
jobs:
|
|
9
13
|
test:
|
|
10
14
|
runs-on: ubuntu-latest
|
|
@@ -40,6 +44,32 @@ jobs:
|
|
|
40
44
|
path-to-lcov: coverage-${{ matrix.tox-env }}.lcov
|
|
41
45
|
parallel: true
|
|
42
46
|
|
|
47
|
+
# The rest of the suite simulates the CLI. This job runs the real one on
|
|
48
|
+
# public corpus logs, so a change in its exit codes or flag rules fails
|
|
49
|
+
# here. SPARKFORENSICS_REQUIRE_CLI turns a missing CLI or an unreachable
|
|
50
|
+
# corpus into a failure instead of a skip.
|
|
51
|
+
cli-integration:
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
strategy:
|
|
54
|
+
fail-fast: false
|
|
55
|
+
matrix:
|
|
56
|
+
# The release the operator is tested against (TESTED_CLI_VERSION in
|
|
57
|
+
# report.py), and the newest one.
|
|
58
|
+
cli: ["0.12.0", "latest"]
|
|
59
|
+
env:
|
|
60
|
+
SPARKFORENSICS_REQUIRE_CLI: "1"
|
|
61
|
+
steps:
|
|
62
|
+
- uses: actions/checkout@v4
|
|
63
|
+
- uses: actions/setup-python@v5
|
|
64
|
+
with:
|
|
65
|
+
python-version: "3.12"
|
|
66
|
+
- uses: actions/setup-node@v4
|
|
67
|
+
with:
|
|
68
|
+
node-version: "22"
|
|
69
|
+
- run: npm install -g sparkforensics-cli@${{ matrix.cli }}
|
|
70
|
+
- run: pip install tox
|
|
71
|
+
- run: tox -e py312-airflow3 -- tests/integration -v
|
|
72
|
+
|
|
43
73
|
finish:
|
|
44
74
|
needs: test
|
|
45
75
|
runs-on: ubuntu-latest
|
|
@@ -15,8 +15,8 @@ SSH analysis as the SSH provider's detached remote job (`ARCHITECTURE.md`'s
|
|
|
15
15
|
|
|
16
16
|
## Commands
|
|
17
17
|
- `pip install -e ".[test,s3,ssh]"`, install for local dev.
|
|
18
|
-
- `pytest`, run the test suite (mocks everything
|
|
19
|
-
|
|
18
|
+
- `pytest`, run the test suite (mocks everything except `tests/integration`,
|
|
19
|
+
which runs the real CLI on public logs and skips without it).
|
|
20
20
|
- `tox -e py311-airflow2` / `tox -e py311-airflow3`, run the suite against a
|
|
21
21
|
specific Airflow major version.
|
|
22
22
|
|
|
@@ -34,11 +34,13 @@ downloaded) fails fast instead of analyzing the wrong file.
|
|
|
34
34
|
|
|
35
35
|
A run can also be compared against an earlier one: `baseline_log_source`
|
|
36
36
|
is a second `LogSourceHook`, located and cleaned up next to `log_source`,
|
|
37
|
-
and the backend passes its reference to the CLI as `--baseline
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
37
|
+
and the backend passes its reference to the CLI as `--baseline` (an event
|
|
38
|
+
log path) or `--baseline-shs-base-url`/`--baseline-app-id`/
|
|
39
|
+
`--baseline-attempt-id` (a History Server application, which the CLI
|
|
40
|
+
fetches itself, like the run under analysis). Each backend declares the
|
|
41
|
+
kinds it can pass (`AnalyzeHook.supported_baseline_refs`: `LocalEventLog`
|
|
42
|
+
and `HistoryServerApp` on the worker, `RemoteEventLog` on its own host and
|
|
43
|
+
`HistoryServerApp` on the SSH host), and `analyze()` or the deferrable
|
|
42
44
|
submit step rejects any other before running anything. The comparison
|
|
43
45
|
itself, and the budgets on it, are the CLI's. Which run is the baseline
|
|
44
46
|
is the DAG author's choice, made through that log source: the operator
|
|
@@ -125,12 +127,31 @@ breach that raises still gets a notification out first.
|
|
|
125
127
|
execution" below.
|
|
126
128
|
- `hooks/analyze/_cli.py`, the CLI contract every backend shares: argument
|
|
127
129
|
building for each reference kind (a positional path, or
|
|
128
|
-
`--shs-base-url`/`--app-id`/`--attempt-id`), `--baseline`, threshold flags
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
`--shs-base-url`/`--app-id`/`--attempt-id`), `--baseline`, threshold flags
|
|
131
|
+
(a dict of per-metric budgets becomes one `--regression-budget` or
|
|
132
|
+
`--stage-regression-budget <metric>:<pct>` each), the comparison tuning
|
|
133
|
+
flags (`--normalize-path` and `--ignore-conf` once per pattern,
|
|
134
|
+
`--noise-band-pct`), the `--redact` switch, the Markdown run (below),
|
|
135
|
+
exit-code handling, and building the `Report` from the JSON report and
|
|
136
|
+
stderr threshold lines. Exit codes 0, 1 and 3 carry a report. Every other
|
|
137
|
+
code raises a `CliError` with its own message: 2 (usage error), 4
|
|
138
|
+
(unreadable event log or failed History Server fetch), 5 (unreadable
|
|
139
|
+
baseline), 6 (internal error), anything else "unexpected". The CLI's
|
|
140
|
+
stderr separates a History Server failure a retry can clear
|
|
141
|
+
(`HistoryServerFetchError`) from one it cannot (`CliConfigurationError`,
|
|
142
|
+
an `AirflowFailException`, so Airflow does not retry it). Backends differ
|
|
143
|
+
only in how they run the command and collect its output.
|
|
132
144
|
- `hooks/analyze/subprocess.py`, `SubprocessAnalyzeHook`, runs the CLI on
|
|
133
|
-
the worker with `--out <tmpfile>`.
|
|
145
|
+
the worker with `--out <tmpfile>`. The CLI writes one report format per
|
|
146
|
+
run, so for the Markdown report (`thresholds["markdown"]`, set by
|
|
147
|
+
`markdown_dest`) the backend runs it a second time with `--format md` and
|
|
148
|
+
no budget flags, and sets `Report.markdown`
|
|
149
|
+
(`AnalyzeHook.supports_markdown`). The first run's report and thresholds
|
|
150
|
+
decide the outcome; the second run only adds the text. `SSHAnalyzeHook`
|
|
151
|
+
does the same on the host: a second synchronous run or, in a deferred
|
|
152
|
+
job, a second command in the same remote job. Its outputs (report, stderr,
|
|
153
|
+
exit code) go to their own files, and the job's exit code stays the
|
|
154
|
+
first run's.
|
|
134
155
|
- `hooks/analyze/ssh.py`, `SSHAnalyzeHook`, runs the CLI on the SSH host
|
|
135
156
|
over `SSHHook` (imported lazily, `ssh` extra) and reads the report from
|
|
136
157
|
stdout, so nothing is written on that host (see "Killing a task" below
|
|
@@ -156,17 +177,35 @@ breach that raises still gets a notification out first.
|
|
|
156
177
|
which reports the CLI's pid, and the command that stops it.
|
|
157
178
|
- `report.py`, `Report`/`ThresholdResult` dataclasses, plus parsing of the
|
|
158
179
|
CLI's JSON report (`--out` file or stdout) and its stderr threshold
|
|
159
|
-
lines.
|
|
180
|
+
lines. Several regression budgets print as `max-regression` lines
|
|
181
|
+
(`max-stage-regression` for the stage budgets), and
|
|
182
|
+
`parse_threshold_results()` keeps every line it reads, so a name can have
|
|
183
|
+
several results. It refuses a `schemaVersion` outside
|
|
184
|
+
`MIN_SCHEMA_VERSION` to `MAX_SCHEMA_VERSION` (an
|
|
185
|
+
`UnsupportedReportSchema` that `build_report()` turns into a
|
|
186
|
+
`CliConfigurationError`), reads the CLI's version from the report's
|
|
187
|
+
`generator`, and checks it against the release the operator is tested
|
|
188
|
+
against (`TESTED_CLI_VERSION`). No
|
|
160
189
|
threshold-evaluation *logic* lives here: sparkforensics's own CLI decides
|
|
161
190
|
pass/violation/inconclusive, this module only parses what it printed, so
|
|
162
191
|
it can never drift from what the CLI actually enforces.
|
|
163
192
|
- `exceptions.py`, `ThresholdBreached`, an `AirflowFailException` so a
|
|
164
|
-
breach fails the task without consuming its retries
|
|
165
|
-
|
|
193
|
+
breach fails the task without consuming its retries, and the CLI
|
|
194
|
+
errors: `CliError` (a CLI exit code with no report, carrying
|
|
195
|
+
`exit_code`), its `HistoryServerFetchError` (retried) and
|
|
196
|
+
`CliConfigurationError` (an `AirflowFailException`, not retried).
|
|
197
|
+
- `sinks.py`, persists the report JSON (local path, `file://`, `s3://`),
|
|
198
|
+
and, through `persist_text()`, the Markdown report. The JSON is the
|
|
199
|
+
CLI's whole report (`Report.raw`) plus the operator's `thresholdResults`
|
|
200
|
+
and the comparison section, so a field the CLI adds reaches the file
|
|
201
|
+
without a change here. `operator._report_json()` builds it; a `Report`
|
|
202
|
+
built by hand (no `raw`) persists the sections the operator reads.
|
|
166
203
|
- `notify.py`, the `Notifier` base class: subclasses implement `_send()`
|
|
167
204
|
for whatever messaging/paging system they target, and `notify()` makes
|
|
168
|
-
that best-effort, never raising.
|
|
169
|
-
|
|
205
|
+
that best-effort, never raising. `_default_message()` writes the
|
|
206
|
+
one-line summary (application, verdict, run time, finding counts,
|
|
207
|
+
skipped lines, destination), leaving out what the report lacks. No
|
|
208
|
+
concrete notifier ships in this package.
|
|
170
209
|
- `summary.py`, the summary XCom (`SUMMARY_XCOM_KEY`,
|
|
171
210
|
`build_summary()`) and the report URL (`render_report_url()`,
|
|
172
211
|
`validate_report_url_template()`). See "Summary XCom and report URL"
|
|
@@ -271,9 +310,9 @@ reboot is left alone.
|
|
|
271
310
|
|
|
272
311
|
Failure paths. A trigger error event without an exit code makes
|
|
273
312
|
`collect()` stop and remove the job before raising. A deferral that times
|
|
274
|
-
out (the backend's `defer_timeout`, `timeout`
|
|
275
|
-
from submission, or
|
|
276
|
-
`execution_timeout`) resumes with `next_method="__fail__"`, which never
|
|
313
|
+
out (the backend's `defer_timeout`, `timeout` per CLI run in the job, so
|
|
314
|
+
twice with `markdown_dest`, plus 120 seconds counted from submission, or
|
|
315
|
+
the task's `execution_timeout`) resumes with `next_method="__fail__"`, which never
|
|
277
316
|
reaches `execute_complete()`; `resume_execution()` is overridden to call
|
|
278
317
|
`backend.abandon(context, job)` first, with the job from XCom. Airflow 2
|
|
279
318
|
takes a different path when `execution_timeout` ran out during the
|
|
@@ -374,8 +413,11 @@ Before notifying and before any threshold breach raises,
|
|
|
374
413
|
`handle_report()` pushes a summary under the `sparkforensics_summary`
|
|
375
414
|
XCom key: the schema version, the destination, the report URL, the
|
|
376
415
|
impact-band counts, whether the run violated a threshold, the breached
|
|
377
|
-
and inconclusive threshold names,
|
|
378
|
-
|
|
416
|
+
and inconclusive threshold names, the CLI exit code, the CLI version
|
|
417
|
+
that wrote the report, and the run's verdict title, application name,
|
|
418
|
+
wall-clock time and skipped line count (`Report.verdict_title`,
|
|
419
|
+
`app_name`, `run_time_ms` and `skipped_lines`, read from the CLI's
|
|
420
|
+
summary). A run compared against a baseline adds a `comparison` entry: the direction the CLI
|
|
379
421
|
computed for one metric (`regression_metric`, wall-clock by default) as
|
|
380
422
|
the verdict, the comparison's confidence and reason, and that metric's
|
|
381
423
|
values and change in percent, taken from the CLI's comparison section.
|
|
@@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.4.0] - 2026-11-04
|
|
11
|
+
|
|
12
|
+
### Upgrade notes
|
|
13
|
+
|
|
14
|
+
- `sparkforensics-cli` 0.10.2 counts every failed task attempt in `max_failed_task_rate_pct`, so a job with retried tasks can fail where it passed.
|
|
15
|
+
- With 0.10.2, a log with skipped lines gives exit 3 and an inconclusive `parse-incomplete` result. The task does not fail; see `skipped_lines` in the summary XCom.
|
|
16
|
+
- The operator is tested against `sparkforensics-cli` 0.12.0.
|
|
17
|
+
- With CLI 0.12.0 and newer, `findings` lists critical and warning findings only. The persisted report, `report.findings` and the `findings` XCom omit info findings; `notesHidden` in the report counts them. The summary's `impact_band_counts` is unchanged.
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
|
|
21
|
+
- `markdown_dest` saves the CLI's Markdown report (`--format md`) next to the JSON report. The analysis runs twice.
|
|
22
|
+
- `redact=True` passes `--redact`.
|
|
23
|
+
- `normalize_path`, `ignore_conf` and `noise_band_pct` tune the baseline comparison.
|
|
24
|
+
- `regression_budgets`, `stage_regression_budgets` and `stage_qualities` set several regression budgets at once.
|
|
25
|
+
- A Spark History Server application can be the baseline on `SubprocessAnalyzeHook` and `SSHAnalyzeHook`.
|
|
26
|
+
- The summary XCom has `verdict`, `app_name`, `run_time_ms`, `skipped_lines` and `cli_version`. The default notification names the application, the verdict and the run time.
|
|
27
|
+
- A warning is logged when the CLI is older than the tested version.
|
|
28
|
+
- `CliError`, `CliConfigurationError` and `HistoryServerFetchError` in `sparkforensics_operator.exceptions`.
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- The operator reads the JSON output shape of `sparkforensics-cli` 0.12.0 (`{schemaVersion, run, comparison, budgets}`) and still reads the older shapes.
|
|
33
|
+
- Each CLI exit code without a report (2, 4, 5, 6) has its own error message. The task still fails.
|
|
34
|
+
- Airflow does not retry usage errors, and retries History Server failures that can clear.
|
|
35
|
+
- A report with a newer `schemaVersion` than the operator reads fails the task.
|
|
36
|
+
- The saved report is the CLI's whole JSON report. `Report.raw` holds it. It includes `effectiveConf`, where the CLI masks secret-looking keys and values; see [Report destinations](docs/configuration.md#report-destinations).
|
|
37
|
+
- `Report.threshold_results` keeps every result of a threshold the CLI reports on several lines.
|
|
38
|
+
|
|
39
|
+
### Fixed
|
|
40
|
+
|
|
41
|
+
- Thresholds such as `0.00001` or `1e16` reach the CLI as plain decimals, not exponent form, which the CLI rejects.
|
|
42
|
+
|
|
10
43
|
## [0.3.0] - 2026-09-28
|
|
11
44
|
|
|
12
45
|
### Added
|
|
@@ -188,7 +221,8 @@ Initial release.
|
|
|
188
221
|
- Generic `Notifier` interface for threshold-breach alerting.
|
|
189
222
|
- PyPI trusted-publishing GitHub Actions workflow (OIDC, no stored token).
|
|
190
223
|
|
|
191
|
-
[Unreleased]: https://github.com/shuffle-works/sparkforensics-operator/compare/v0.
|
|
224
|
+
[Unreleased]: https://github.com/shuffle-works/sparkforensics-operator/compare/v0.4.0...HEAD
|
|
225
|
+
[0.4.0]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.4.0
|
|
192
226
|
[0.3.0]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.3.0
|
|
193
227
|
[0.2.0]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.2.0
|
|
194
228
|
[0.1.1]: https://github.com/shuffle-works/sparkforensics-operator/releases/tag/v0.1.1
|
|
@@ -13,8 +13,8 @@ that backends consume; see `ARCHITECTURE.md`.
|
|
|
13
13
|
|
|
14
14
|
## Commands
|
|
15
15
|
- `pip install -e ".[test,s3,ssh]"`, install for local dev.
|
|
16
|
-
- `pytest`, run the test suite (mocks everything
|
|
17
|
-
|
|
16
|
+
- `pytest`, run the test suite (mocks everything except `tests/integration`,
|
|
17
|
+
which runs the real CLI on public logs and skips without it).
|
|
18
18
|
- `tox -e py311-airflow2` / `tox -e py311-airflow3`, run the suite against a
|
|
19
19
|
specific Airflow major version.
|
|
20
20
|
|
|
@@ -15,13 +15,36 @@ pip install -e ".[test,s3,ssh]"
|
|
|
15
15
|
pytest
|
|
16
16
|
```
|
|
17
17
|
|
|
18
|
-
The suite needs no live Spark
|
|
19
|
-
a shell: the deferred-mode tests run the SSH provider's real shell wrapper against a local `sh` standing in
|
|
18
|
+
The suite needs no live Spark or Airflow, and no Node.js apart from
|
|
19
|
+
`tests/integration` (below). The one real process is a shell: the deferred-mode tests run the SSH provider's real shell wrapper against a local `sh` standing in
|
|
20
20
|
for the SSH host (`tests/hooks/analyze/_local_ssh.py`), so they need
|
|
21
21
|
`bash`, `setsid` and coreutils `timeout` on the machine running the tests.
|
|
22
22
|
The kill and sweep tests also run with procps hidden
|
|
23
23
|
(`LocalHost.hide_procps()`), as on slim images.
|
|
24
24
|
|
|
25
|
+
### Tests against the real CLI
|
|
26
|
+
|
|
27
|
+
The other tests simulate `sparkforensics-analyze`, so they cannot catch the
|
|
28
|
+
CLI changing an exit code or a flag's rules. `tests/integration` runs the
|
|
29
|
+
real CLI on public logs from the
|
|
30
|
+
[spark-event-corpus-data](https://github.com/shuffle-works/spark-event-corpus-data)
|
|
31
|
+
repository (fetched at a pinned tag), through `SubprocessAnalyzeHook`. It
|
|
32
|
+
covers a clean log, a violation, an unreadable log, an unreadable baseline
|
|
33
|
+
and budgets that print in exponent form.
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
npm install -g sparkforensics-cli
|
|
37
|
+
pytest tests/integration
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
They need Node.js 18+, the `sparkforensics-cli` package on `PATH` (or
|
|
41
|
+
its binary at `SPARKFORENSICS_ANALYZE_BIN`) and network access to GitHub, and
|
|
42
|
+
skip when the CLI or the network is missing. With `SPARKFORENSICS_REQUIRE_CLI=1`
|
|
43
|
+
a missing CLI or an unreachable corpus fails the run instead, as in CI. The
|
|
44
|
+
`cli-integration` job in `.github/workflows/ci.yml` runs them against the
|
|
45
|
+
release the operator is tested against (`TESTED_CLI_VERSION` in
|
|
46
|
+
`report.py`) and the newest one.
|
|
47
|
+
|
|
25
48
|
## The tox Airflow matrix
|
|
26
49
|
|
|
27
50
|
`tox.ini` runs the suite against each supported Airflow major version:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sparkforensics-operator
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Airflow operator that runs sparkforensics analysis on a Spark job's event log and acts on the result.
|
|
5
5
|
Project-URL: Homepage, https://github.com/shuffle-works/sparkforensics-operator
|
|
6
6
|
Project-URL: Repository, https://github.com/shuffle-works/sparkforensics-operator
|
|
@@ -73,12 +73,15 @@ runs as part of the DAG, right where the Spark task just ran.
|
|
|
73
73
|
the same verdict.
|
|
74
74
|
- Compare a run against an earlier one. `baseline_log_source` takes any
|
|
75
75
|
log source that resolves to an event log path the analysis can read
|
|
76
|
-
(`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull)
|
|
77
|
-
|
|
78
|
-
`regression_metric` names, regressed past
|
|
79
|
-
`
|
|
80
|
-
|
|
81
|
-
|
|
76
|
+
(`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull) or
|
|
77
|
+
to a Spark History Server application. `max_regression_pct` fails the
|
|
78
|
+
DAG when runtime, or the metric `regression_metric` names, regressed past
|
|
79
|
+
that budget. `regression_budgets` budgets several metrics at once, and
|
|
80
|
+
`stage_regression_budgets` the paired stages. `fail_on_introduced` fails
|
|
81
|
+
it when the run gained new findings. The comparison's verdict,
|
|
82
|
+
confidence and measured regression land in the summary XCom.
|
|
83
|
+
- `markdown_dest` also persists the CLI's Markdown report, for a notifier
|
|
84
|
+
to post or a person to read.
|
|
82
85
|
- Reports persist to a local path, `file://`, or `s3://`, and a clickable
|
|
83
86
|
"SparkForensics report" link shows up on the task in the Airflow UI. Set
|
|
84
87
|
`report_url_template` (an S3 console URL, say) to make it open in a
|
|
@@ -38,12 +38,15 @@ runs as part of the DAG, right where the Spark task just ran.
|
|
|
38
38
|
the same verdict.
|
|
39
39
|
- Compare a run against an earlier one. `baseline_log_source` takes any
|
|
40
40
|
log source that resolves to an event log path the analysis can read
|
|
41
|
-
(`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull)
|
|
42
|
-
|
|
43
|
-
`regression_metric` names, regressed past
|
|
44
|
-
`
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
(`path_template="/spark-logs/{{ prev_ds }}/eventlog"`, an XCom pull) or
|
|
42
|
+
to a Spark History Server application. `max_regression_pct` fails the
|
|
43
|
+
DAG when runtime, or the metric `regression_metric` names, regressed past
|
|
44
|
+
that budget. `regression_budgets` budgets several metrics at once, and
|
|
45
|
+
`stage_regression_budgets` the paired stages. `fail_on_introduced` fails
|
|
46
|
+
it when the run gained new findings. The comparison's verdict,
|
|
47
|
+
confidence and measured regression land in the summary XCom.
|
|
48
|
+
- `markdown_dest` also persists the CLI's Markdown report, for a notifier
|
|
49
|
+
to post or a person to read.
|
|
47
50
|
- Reports persist to a local path, `file://`, or `s3://`, and a clickable
|
|
48
51
|
"SparkForensics report" link shows up on the task in the Airflow UI. Set
|
|
49
52
|
`report_url_template` (an S3 console URL, say) to make it open in a
|