bulk-post 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bulk_post-0.2.0/src/bulk_post.egg-info → bulk_post-0.3.0}/PKG-INFO +96 -14
- {bulk_post-0.2.0 → bulk_post-0.3.0}/README.md +95 -13
- {bulk_post-0.2.0 → bulk_post-0.3.0}/pyproject.toml +1 -1
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/__init__.py +16 -1
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/cli.py +140 -25
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/csvio.py +6 -6
- bulk_post-0.3.0/src/bulk_post/retry.py +245 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/runner.py +158 -29
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/state.py +1 -1
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/variables.py +3 -3
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/workflow.py +73 -23
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/workflow_runner.py +90 -20
- {bulk_post-0.2.0 → bulk_post-0.3.0/src/bulk_post.egg-info}/PKG-INFO +96 -14
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post.egg-info/SOURCES.txt +3 -1
- {bulk_post-0.2.0 → bulk_post-0.3.0}/tests/test_bulk_post.py +379 -55
- bulk_post-0.3.0/tests/test_retry.py +935 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/LICENSE +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/setup.cfg +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/__main__.py +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/auth.py +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/http.py +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/templating.py +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post/terminal.py +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post.egg-info/dependency_links.txt +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post.egg-info/entry_points.txt +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post.egg-info/requires.txt +0 -0
- {bulk_post-0.2.0 → bulk_post-0.3.0}/src/bulk_post.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bulk-post
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A near-stdlib Python CLI that fires templated HTTP requests driven by CSV data, one request (or a multi-step workflow) per row.
|
|
5
5
|
Author-email: Valerii Kirichenko <true.monte.kristo@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -31,7 +31,7 @@ Dynamic: license-file
|
|
|
31
31
|
[](https://www.python.org/downloads/)
|
|
32
32
|
[](https://github.com/true-monte-kristo/bulk-post/security/advisories/new)
|
|
33
33
|
|
|
34
|
-
A near-stdlib Python CLI that fires templated HTTP requests driven by CSV data. You define the request — URL, method, body, headers — with `{{placeholder}}` slots, and each CSV row supplies the values that fill them: one request per row, or a multi-step request workflow per row in `--workflow` mode. Supports bearer or basic auth (default: no auth) with automatic 401 re-prompt, a live terminal UI with pause/resume, parallel execution, and a
|
|
34
|
+
A near-stdlib Python CLI that fires templated HTTP requests driven by CSV data. You define the request — URL, method, body, headers — with `{{placeholder}}` slots, and each CSV row supplies the values that fill them: one request per row, or a multi-step request workflow per row in `--workflow` mode. Supports bearer or basic auth (default: no auth) with automatic 401 re-prompt, a live terminal UI with pause/resume, parallel execution, and a redrive file for failed rows. Third-party dependencies: PyYAML and jsonpath-ng — both always installed, but lazily imported (PyYAML only on the `--workflow` code path, jsonpath-ng only on the workflow-variables code path).
|
|
35
35
|
|
|
36
36
|
## Requirements
|
|
37
37
|
|
|
@@ -137,20 +137,26 @@ bulk-post -u "https://api.example.com/items/{{id}}" -c items.csv -o 47
|
|
|
137
137
|
| `--delay` | `-d` | `0` | Milliseconds to wait between requests |
|
|
138
138
|
| `--offset` | `-o` | `0` | Skip the first N data rows (useful for resuming after a failure) |
|
|
139
139
|
| `--timeout` | `-T` | `30` | Per-request timeout in seconds |
|
|
140
|
-
| `--
|
|
140
|
+
| `--redrive-file` | `-r` | `<stem>_failed.csv` | Where to write rows that failed; auto-named from the CSV path if omitted |
|
|
141
141
|
| `--verbose` | `-v` | false | Print URL, request/response headers (Authorization masked), body, status, and timing for every row |
|
|
142
142
|
| `--header` | `-H` | — | Add a custom request header in `Name: value` format; repeatable. Values support `{{col}}` placeholders |
|
|
143
143
|
| `--parallel` | `-p` | false | Process rows concurrently using multiple threads; `--delay` is ignored in this mode |
|
|
144
144
|
| `--concurrency-level` | `-n` | CPU count | Number of worker threads; only used with `--parallel` |
|
|
145
145
|
| `--debug` | `-D` | false | Print worker thread name on each row log line and show a live debug bar with queue depth, active thread count, and ok/fail counters; only meaningful with `--parallel` |
|
|
146
146
|
| `--workflow` | `-w` | — | Path to a workflow YAML file; mutually exclusive with `--url` |
|
|
147
|
+
| `--retry-on` | `-R` | — (off) | Comma-separated HTTP status codes to retry on (e.g. `503,429`); enables retries |
|
|
148
|
+
| `--retry-backoff` | `-B` | `fixed` | `fixed` or `exponential`; requires `--retry-on` |
|
|
149
|
+
| `--max-retries` | `-M` | `5` | Retries after the initial attempt (5 ⇒ up to 6 requests); requires `--retry-on` |
|
|
150
|
+
| `--retry-delay` | `-y` | `200` | Milliseconds; fixed delay, or initial delay for exponential; requires `--retry-on` |
|
|
151
|
+
| `--multiplier` | `-x` | `1.5` | Exponential backoff multiplier; requires `--retry-backoff exponential` |
|
|
152
|
+
| `--max-retry-delay` | `-Y` | `30000` | Millisecond hard cap on any single retry wait; requires `--retry-on` |
|
|
147
153
|
| `--version` | `-V` | — | Print version and exit |
|
|
148
154
|
|
|
149
155
|
## CSV format
|
|
150
156
|
|
|
151
157
|
The CSV must have a header row. Column names are used as placeholder names in `--url`, `--body`, and `--header` values. Every `{{placeholder}}` in the URL, body, or header values must match a column name; the script exits with an error if any are missing.
|
|
152
158
|
|
|
153
|
-
The input delimiter is detected automatically (comma, semicolon, tab, or pipe), falling back to comma when it can't be determined. The failed-rows
|
|
159
|
+
The input delimiter is detected automatically (comma, semicolon, tab, or pipe), falling back to comma when it can't be determined. The failed-rows redrive CSV is written with the same delimiter as the input. There is no delimiter flag.
|
|
154
160
|
|
|
155
161
|
```csv
|
|
156
162
|
id,reason
|
|
@@ -176,6 +182,33 @@ Pass `--auth-type basic` (or `-a basic`). Credentials (`user:pass`) are resolved
|
|
|
176
182
|
|
|
177
183
|
The default when `--auth-type` is omitted (or pass `--auth-type none` / `-a none` explicitly). No `Authorization` header is sent.
|
|
178
184
|
|
|
185
|
+
## Retries
|
|
186
|
+
|
|
187
|
+
Retries are disabled by default. Pass `--retry-on` with a comma-separated list of HTTP status codes to enable them:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
bulk-post -u "https://api.example.com/{{id}}" -c rows.csv -R 503,429 -B exponential
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Only the exact listed status codes trigger a retry — network errors and timeouts do not. If the server returns a `Retry-After` or `X-Retry-After` header, the wait is extended to at least that value before the next attempt. All waits are capped by `--max-retry-delay`. Each retry prints a short `[RETRY]` notice; `--verbose` adds the elapsed time and body snippet of the failed attempt.
|
|
194
|
+
|
|
195
|
+
**Flags:**
|
|
196
|
+
|
|
197
|
+
| Flag | Short | Default | Description |
|
|
198
|
+
|------|-------|---------|-------------|
|
|
199
|
+
| `--retry-on` | `-R` | — | Comma-separated status codes (e.g. `503,429`); required to enable retries |
|
|
200
|
+
| `--retry-backoff` | `-B` | `fixed` | `fixed` or `exponential` |
|
|
201
|
+
| `--max-retries` | `-M` | `5` | Retries after the first attempt (5 ⇒ up to 6 total requests) |
|
|
202
|
+
| `--retry-delay` | `-y` | `200` | ms; fixed wait between retries, or starting delay for exponential |
|
|
203
|
+
| `--multiplier` | `-x` | `1.5` | Growth factor for exponential backoff |
|
|
204
|
+
| `--max-retry-delay` | `-Y` | `30000` | ms hard cap on any single retry wait |
|
|
205
|
+
|
|
206
|
+
**Notes:**
|
|
207
|
+
|
|
208
|
+
- `401` cannot be listed in `--retry-on` when `--auth-type` is `bearer` or `basic` — the 401 auth-refresh flow owns that status code.
|
|
209
|
+
- Retry sleeps are interruptible: `/pause` freezes the countdown, `/exit` abandons the wait and routes the row to the redrive file.
|
|
210
|
+
- In workflow mode, retry flags are not available; use `retry_policy:` in the workflow YAML instead (see [Workflow mode](#workflow-mode)).
|
|
211
|
+
|
|
179
212
|
## Terminal UI
|
|
180
213
|
|
|
181
214
|
When running in an interactive terminal, a live bottom bar shows:
|
|
@@ -193,11 +226,11 @@ Available commands:
|
|
|
193
226
|
|
|
194
227
|
In non-TTY mode (piped input, CI, test environments) the bottom bar is skipped and no interactive commands are available.
|
|
195
228
|
|
|
196
|
-
##
|
|
229
|
+
## Redrive file
|
|
197
230
|
|
|
198
|
-
Rows that fail (network error, non-2xx response, or substitution error) are written to the
|
|
231
|
+
Rows that fail (network error, non-2xx response, or substitution error) are written to the redrive file. By default this is `<csv-stem>_failed.csv` next to the input file. Re-run with `-c <stem>_failed.csv` to redrive only those rows.
|
|
199
232
|
|
|
200
|
-
If no rows fail, the
|
|
233
|
+
If no rows fail, the redrive file is deleted automatically.
|
|
201
234
|
|
|
202
235
|
## Workflow mode
|
|
203
236
|
|
|
@@ -209,6 +242,7 @@ Each CSV row fires all steps in document order. Steps within a row are always se
|
|
|
209
242
|
|
|
210
243
|
```yaml
|
|
211
244
|
workflow:
|
|
245
|
+
persist_context: false # optional; default false — see "Redrive and resume" below
|
|
212
246
|
description: Optional human-readable description # skipped at runtime
|
|
213
247
|
|
|
214
248
|
groupA: # logical grouping for shared auth
|
|
@@ -249,12 +283,60 @@ Key rules:
|
|
|
249
283
|
|
|
250
284
|
- **Execution order** — steps fire in the order they appear in the document (top to bottom across all groups).
|
|
251
285
|
- **Group auth** — all steps in a group inherit the group's auth unless they declare their own.
|
|
252
|
-
- **`on_error`** — `stop` (default) halts remaining steps for that row and writes it to the
|
|
286
|
+
- **`on_error`** — `stop` (default) halts remaining steps for that row and writes it to the redrive file; `continue` logs the failure, writes the row, and proceeds to the next step.
|
|
253
287
|
- **Placeholders** — `{{col}}` in `url`, `body`, and header values is replaced with the matching CSV column value, same as in single-URL mode.
|
|
254
288
|
|
|
255
|
-
###
|
|
289
|
+
### Workflow retries
|
|
290
|
+
|
|
291
|
+
Each group and each endpoint may carry an optional `retry_policy:` block. An endpoint-level `retry_policy` **replaces** the group's entirely — there is no merging. `retry_policy: none` on an endpoint disables any inherited group policy. `on_error` evaluates only the final post-retries result.
|
|
292
|
+
|
|
293
|
+
```yaml
|
|
294
|
+
workflow:
|
|
295
|
+
groupA:
|
|
296
|
+
auth:
|
|
297
|
+
type: bearer
|
|
298
|
+
retry_policy: # group-level policy, inherited by all endpoints in the group
|
|
299
|
+
retry_on: 503,429 # required: comma-separated string or YAML list
|
|
300
|
+
backoff: exponential # optional: fixed (default) or exponential
|
|
301
|
+
multiplier: 1.5 # optional: exponential only; defaults to 1.5
|
|
302
|
+
delay: 200 # optional: fixed delay / initial delay in ms; defaults to 200
|
|
303
|
+
max_retries: 5 # optional: retries after the initial attempt; defaults to 5
|
|
304
|
+
max_delay: 30000 # optional: hard cap on any single retry wait in ms; defaults to 30000
|
|
305
|
+
endpoints:
|
|
306
|
+
- call-api:
|
|
307
|
+
url: https://api.example.com/{{id}}
|
|
308
|
+
method: POST
|
|
309
|
+
|
|
310
|
+
groupB:
|
|
311
|
+
endpoints:
|
|
312
|
+
- call-other:
|
|
313
|
+
url: https://other.example.com/{{id}}
|
|
314
|
+
method: DELETE
|
|
315
|
+
retry_policy: # endpoint-level policy replaces any group policy entirely
|
|
316
|
+
retry_on: 503,429
|
|
317
|
+
backoff: fixed
|
|
318
|
+
delay: 200
|
|
319
|
+
max_retries: 5
|
|
320
|
+
|
|
321
|
+
groupC:
|
|
322
|
+
retry_policy: # group has a policy …
|
|
323
|
+
retry_on: 503
|
|
324
|
+
endpoints:
|
|
325
|
+
- no-retry-step:
|
|
326
|
+
retry_policy: none # … but this endpoint opts out
|
|
327
|
+
url: https://public.example.com/{{id}}
|
|
328
|
+
method: GET
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
### Redrive and resume
|
|
332
|
+
|
|
333
|
+
Mid-workflow resume is opt-in via a `persist_context: true` key at the top of the `workflow:` mapping (default `false`).
|
|
334
|
+
|
|
335
|
+
- **`persist_context: false` (default):** failed rows are written to the redrive file with only the original input columns. Re-running always starts each row from step 1. If the input CSV contains `_bulk_post_step` or `_bulk_post_var/…` context columns (e.g. from a previous run with `persist_context: true`), they are ignored, with a startup warning printed to stderr.
|
|
336
|
+
|
|
337
|
+
- **`persist_context: true`:** when a step fails, the row is written to the redrive file with an extra column `_bulk_post_step` set to the path of the first failed step (e.g. `groupA/step-name`) plus any persisted variable columns. Re-running with that redrive CSV skips all steps before the failed one, resuming mid-workflow automatically.
|
|
256
338
|
|
|
257
|
-
|
|
339
|
+
> **Security note:** `persist_context: true` writes response-derived data (potentially sensitive) to disk in plaintext. Do not share or commit redrive CSVs produced with this option enabled.
|
|
258
340
|
|
|
259
341
|
### Workflow variables
|
|
260
342
|
|
|
@@ -291,15 +373,15 @@ workflow:
|
|
|
291
373
|
- Names must start with `$` (e.g. `$id`) and are referenced as `{{$id}}` in URL, headers, and body.
|
|
292
374
|
- `source` is written as `.workflow.<group>.<endpoint>` (or just `<group>/<endpoint>`). It must refer to an endpoint that runs before the current step — forward and self references are rejected at startup.
|
|
293
375
|
- `jsonPath` uses full JSONPath syntax (powered by [`jsonpath-ng`](https://pypi.org/project/jsonpath-ng/)). Only the first match is used. A match that is an object or array (non-scalar) fails the step.
|
|
294
|
-
- `nullable` defaults to `true`. When `false`, a null value or no-match fails the step (row written to
|
|
376
|
+
- `nullable` defaults to `true`. When `false`, a null value or no-match fails the step (row written to redrive file); when `true`, it resolves to an empty string.
|
|
295
377
|
- Variable values are scoped to a single CSV row and are never shared across rows.
|
|
296
378
|
- All variable declarations are validated at startup — undefined references, bad names, unreachable sources, and invalid JSONPath expressions all cause an immediate exit with a clear error.
|
|
297
379
|
|
|
298
|
-
**Resume/
|
|
380
|
+
**Resume/redrive with variables:**
|
|
299
381
|
|
|
300
|
-
When
|
|
382
|
+
Requires `persist_context: true` at the top of the `workflow:` mapping. When enabled, on row failure any resolved variable values are persisted into reserved redrive-CSV columns named `_bulk_post_var/<source_path>/<name>`. Re-running the redrive CSV skips completed steps and reads these persisted values for variables whose source step was skipped. When `persist_context` is `false` (the default), variable values are not persisted and re-runs always start from step 1.
|
|
301
383
|
|
|
302
|
-
> **Security note:**
|
|
384
|
+
> **Security note:** redrive CSVs may contain response-derived data (potentially sensitive) in plaintext. Do not share or commit redrive CSVs that were produced from workflows using variables.
|
|
303
385
|
|
|
304
386
|
### Example
|
|
305
387
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://www.python.org/downloads/)
|
|
6
6
|
[](https://github.com/true-monte-kristo/bulk-post/security/advisories/new)
|
|
7
7
|
|
|
8
|
-
A near-stdlib Python CLI that fires templated HTTP requests driven by CSV data. You define the request — URL, method, body, headers — with `{{placeholder}}` slots, and each CSV row supplies the values that fill them: one request per row, or a multi-step request workflow per row in `--workflow` mode. Supports bearer or basic auth (default: no auth) with automatic 401 re-prompt, a live terminal UI with pause/resume, parallel execution, and a
|
|
8
|
+
A near-stdlib Python CLI that fires templated HTTP requests driven by CSV data. You define the request — URL, method, body, headers — with `{{placeholder}}` slots, and each CSV row supplies the values that fill them: one request per row, or a multi-step request workflow per row in `--workflow` mode. Supports bearer or basic auth (default: no auth) with automatic 401 re-prompt, a live terminal UI with pause/resume, parallel execution, and a redrive file for failed rows. Third-party dependencies: PyYAML and jsonpath-ng — both always installed, but lazily imported (PyYAML only on the `--workflow` code path, jsonpath-ng only on the workflow-variables code path).
|
|
9
9
|
|
|
10
10
|
## Requirements
|
|
11
11
|
|
|
@@ -111,20 +111,26 @@ bulk-post -u "https://api.example.com/items/{{id}}" -c items.csv -o 47
|
|
|
111
111
|
| `--delay` | `-d` | `0` | Milliseconds to wait between requests |
|
|
112
112
|
| `--offset` | `-o` | `0` | Skip the first N data rows (useful for resuming after a failure) |
|
|
113
113
|
| `--timeout` | `-T` | `30` | Per-request timeout in seconds |
|
|
114
|
-
| `--
|
|
114
|
+
| `--redrive-file` | `-r` | `<stem>_failed.csv` | Where to write rows that failed; auto-named from the CSV path if omitted |
|
|
115
115
|
| `--verbose` | `-v` | false | Print URL, request/response headers (Authorization masked), body, status, and timing for every row |
|
|
116
116
|
| `--header` | `-H` | — | Add a custom request header in `Name: value` format; repeatable. Values support `{{col}}` placeholders |
|
|
117
117
|
| `--parallel` | `-p` | false | Process rows concurrently using multiple threads; `--delay` is ignored in this mode |
|
|
118
118
|
| `--concurrency-level` | `-n` | CPU count | Number of worker threads; only used with `--parallel` |
|
|
119
119
|
| `--debug` | `-D` | false | Print worker thread name on each row log line and show a live debug bar with queue depth, active thread count, and ok/fail counters; only meaningful with `--parallel` |
|
|
120
120
|
| `--workflow` | `-w` | — | Path to a workflow YAML file; mutually exclusive with `--url` |
|
|
121
|
+
| `--retry-on` | `-R` | — (off) | Comma-separated HTTP status codes to retry on (e.g. `503,429`); enables retries |
|
|
122
|
+
| `--retry-backoff` | `-B` | `fixed` | `fixed` or `exponential`; requires `--retry-on` |
|
|
123
|
+
| `--max-retries` | `-M` | `5` | Retries after the initial attempt (5 ⇒ up to 6 requests); requires `--retry-on` |
|
|
124
|
+
| `--retry-delay` | `-y` | `200` | Milliseconds; fixed delay, or initial delay for exponential; requires `--retry-on` |
|
|
125
|
+
| `--multiplier` | `-x` | `1.5` | Exponential backoff multiplier; requires `--retry-backoff exponential` |
|
|
126
|
+
| `--max-retry-delay` | `-Y` | `30000` | Millisecond hard cap on any single retry wait; requires `--retry-on` |
|
|
121
127
|
| `--version` | `-V` | — | Print version and exit |
|
|
122
128
|
|
|
123
129
|
## CSV format
|
|
124
130
|
|
|
125
131
|
The CSV must have a header row. Column names are used as placeholder names in `--url`, `--body`, and `--header` values. Every `{{placeholder}}` in the URL, body, or header values must match a column name; the script exits with an error if any are missing.
|
|
126
132
|
|
|
127
|
-
The input delimiter is detected automatically (comma, semicolon, tab, or pipe), falling back to comma when it can't be determined. The failed-rows
|
|
133
|
+
The input delimiter is detected automatically (comma, semicolon, tab, or pipe), falling back to comma when it can't be determined. The failed-rows redrive CSV is written with the same delimiter as the input. There is no delimiter flag.
|
|
128
134
|
|
|
129
135
|
```csv
|
|
130
136
|
id,reason
|
|
@@ -150,6 +156,33 @@ Pass `--auth-type basic` (or `-a basic`). Credentials (`user:pass`) are resolved
|
|
|
150
156
|
|
|
151
157
|
The default when `--auth-type` is omitted (or pass `--auth-type none` / `-a none` explicitly). No `Authorization` header is sent.
|
|
152
158
|
|
|
159
|
+
## Retries
|
|
160
|
+
|
|
161
|
+
Retries are disabled by default. Pass `--retry-on` with a comma-separated list of HTTP status codes to enable them:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
bulk-post -u "https://api.example.com/{{id}}" -c rows.csv -R 503,429 -B exponential
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Only the exact listed status codes trigger a retry — network errors and timeouts do not. If the server returns a `Retry-After` or `X-Retry-After` header, the wait is extended to at least that value before the next attempt. All waits are capped by `--max-retry-delay`. Each retry prints a short `[RETRY]` notice; `--verbose` adds the elapsed time and body snippet of the failed attempt.
|
|
168
|
+
|
|
169
|
+
**Flags:**
|
|
170
|
+
|
|
171
|
+
| Flag | Short | Default | Description |
|
|
172
|
+
|------|-------|---------|-------------|
|
|
173
|
+
| `--retry-on` | `-R` | — | Comma-separated status codes (e.g. `503,429`); required to enable retries |
|
|
174
|
+
| `--retry-backoff` | `-B` | `fixed` | `fixed` or `exponential` |
|
|
175
|
+
| `--max-retries` | `-M` | `5` | Retries after the first attempt (5 ⇒ up to 6 total requests) |
|
|
176
|
+
| `--retry-delay` | `-y` | `200` | ms; fixed wait between retries, or starting delay for exponential |
|
|
177
|
+
| `--multiplier` | `-x` | `1.5` | Growth factor for exponential backoff |
|
|
178
|
+
| `--max-retry-delay` | `-Y` | `30000` | ms hard cap on any single retry wait |
|
|
179
|
+
|
|
180
|
+
**Notes:**
|
|
181
|
+
|
|
182
|
+
- `401` cannot be listed in `--retry-on` when `--auth-type` is `bearer` or `basic` — the 401 auth-refresh flow owns that status code.
|
|
183
|
+
- Retry sleeps are interruptible: `/pause` freezes the countdown, `/exit` abandons the wait and routes the row to the redrive file.
|
|
184
|
+
- In workflow mode, retry flags are not available; use `retry_policy:` in the workflow YAML instead (see [Workflow mode](#workflow-mode)).
|
|
185
|
+
|
|
153
186
|
## Terminal UI
|
|
154
187
|
|
|
155
188
|
When running in an interactive terminal, a live bottom bar shows:
|
|
@@ -167,11 +200,11 @@ Available commands:
|
|
|
167
200
|
|
|
168
201
|
In non-TTY mode (piped input, CI, test environments) the bottom bar is skipped and no interactive commands are available.
|
|
169
202
|
|
|
170
|
-
##
|
|
203
|
+
## Redrive file
|
|
171
204
|
|
|
172
|
-
Rows that fail (network error, non-2xx response, or substitution error) are written to the
|
|
205
|
+
Rows that fail (network error, non-2xx response, or substitution error) are written to the redrive file. By default this is `<csv-stem>_failed.csv` next to the input file. Re-run with `-c <stem>_failed.csv` to redrive only those rows.
|
|
173
206
|
|
|
174
|
-
If no rows fail, the
|
|
207
|
+
If no rows fail, the redrive file is deleted automatically.
|
|
175
208
|
|
|
176
209
|
## Workflow mode
|
|
177
210
|
|
|
@@ -183,6 +216,7 @@ Each CSV row fires all steps in document order. Steps within a row are always se
|
|
|
183
216
|
|
|
184
217
|
```yaml
|
|
185
218
|
workflow:
|
|
219
|
+
persist_context: false # optional; default false — see "Redrive and resume" below
|
|
186
220
|
description: Optional human-readable description # skipped at runtime
|
|
187
221
|
|
|
188
222
|
groupA: # logical grouping for shared auth
|
|
@@ -223,12 +257,60 @@ Key rules:
|
|
|
223
257
|
|
|
224
258
|
- **Execution order** — steps fire in the order they appear in the document (top to bottom across all groups).
|
|
225
259
|
- **Group auth** — all steps in a group inherit the group's auth unless they declare their own.
|
|
226
|
-
- **`on_error`** — `stop` (default) halts remaining steps for that row and writes it to the
|
|
260
|
+
- **`on_error`** — `stop` (default) halts remaining steps for that row and writes it to the redrive file; `continue` logs the failure, writes the row, and proceeds to the next step.
|
|
227
261
|
- **Placeholders** — `{{col}}` in `url`, `body`, and header values is replaced with the matching CSV column value, same as in single-URL mode.
|
|
228
262
|
|
|
229
|
-
###
|
|
263
|
+
### Workflow retries
|
|
264
|
+
|
|
265
|
+
Each group and each endpoint may carry an optional `retry_policy:` block. An endpoint-level `retry_policy` **replaces** the group's entirely — there is no merging. `retry_policy: none` on an endpoint disables any inherited group policy. `on_error` evaluates only the final post-retries result.
|
|
266
|
+
|
|
267
|
+
```yaml
|
|
268
|
+
workflow:
|
|
269
|
+
groupA:
|
|
270
|
+
auth:
|
|
271
|
+
type: bearer
|
|
272
|
+
retry_policy: # group-level policy, inherited by all endpoints in the group
|
|
273
|
+
retry_on: 503,429 # required: comma-separated string or YAML list
|
|
274
|
+
backoff: exponential # optional: fixed (default) or exponential
|
|
275
|
+
multiplier: 1.5 # optional: exponential only; defaults to 1.5
|
|
276
|
+
delay: 200 # optional: fixed delay / initial delay in ms; defaults to 200
|
|
277
|
+
max_retries: 5 # optional: retries after the initial attempt; defaults to 5
|
|
278
|
+
max_delay: 30000 # optional: hard cap on any single retry wait in ms; defaults to 30000
|
|
279
|
+
endpoints:
|
|
280
|
+
- call-api:
|
|
281
|
+
url: https://api.example.com/{{id}}
|
|
282
|
+
method: POST
|
|
283
|
+
|
|
284
|
+
groupB:
|
|
285
|
+
endpoints:
|
|
286
|
+
- call-other:
|
|
287
|
+
url: https://other.example.com/{{id}}
|
|
288
|
+
method: DELETE
|
|
289
|
+
retry_policy: # endpoint-level policy replaces any group policy entirely
|
|
290
|
+
retry_on: 503,429
|
|
291
|
+
backoff: fixed
|
|
292
|
+
delay: 200
|
|
293
|
+
max_retries: 5
|
|
294
|
+
|
|
295
|
+
groupC:
|
|
296
|
+
retry_policy: # group has a policy …
|
|
297
|
+
retry_on: 503
|
|
298
|
+
endpoints:
|
|
299
|
+
- no-retry-step:
|
|
300
|
+
retry_policy: none # … but this endpoint opts out
|
|
301
|
+
url: https://public.example.com/{{id}}
|
|
302
|
+
method: GET
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
### Redrive and resume
|
|
306
|
+
|
|
307
|
+
Mid-workflow resume is opt-in via a `persist_context: true` key at the top of the `workflow:` mapping (default `false`).
|
|
308
|
+
|
|
309
|
+
- **`persist_context: false` (default):** failed rows are written to the redrive file with only the original input columns. Re-running always starts each row from step 1. If the input CSV contains `_bulk_post_step` or `_bulk_post_var/…` context columns (e.g. from a previous run with `persist_context: true`), they are ignored, with a startup warning printed to stderr.
|
|
310
|
+
|
|
311
|
+
- **`persist_context: true`:** when a step fails, the row is written to the redrive file with an extra column `_bulk_post_step` set to the path of the first failed step (e.g. `groupA/step-name`) plus any persisted variable columns. Re-running with that redrive CSV skips all steps before the failed one, resuming mid-workflow automatically.
|
|
230
312
|
|
|
231
|
-
|
|
313
|
+
> **Security note:** `persist_context: true` writes response-derived data (potentially sensitive) to disk in plaintext. Do not share or commit redrive CSVs produced with this option enabled.
|
|
232
314
|
|
|
233
315
|
### Workflow variables
|
|
234
316
|
|
|
@@ -265,15 +347,15 @@ workflow:
|
|
|
265
347
|
- Names must start with `$` (e.g. `$id`) and are referenced as `{{$id}}` in URL, headers, and body.
|
|
266
348
|
- `source` is written as `.workflow.<group>.<endpoint>` (or just `<group>/<endpoint>`). It must refer to an endpoint that runs before the current step — forward and self references are rejected at startup.
|
|
267
349
|
- `jsonPath` uses full JSONPath syntax (powered by [`jsonpath-ng`](https://pypi.org/project/jsonpath-ng/)). Only the first match is used. A match that is an object or array (non-scalar) fails the step.
|
|
268
|
-
- `nullable` defaults to `true`. When `false`, a null value or no-match fails the step (row written to
|
|
350
|
+
- `nullable` defaults to `true`. When `false`, a null value or no-match fails the step (row written to redrive file); when `true`, it resolves to an empty string.
|
|
269
351
|
- Variable values are scoped to a single CSV row and are never shared across rows.
|
|
270
352
|
- All variable declarations are validated at startup — undefined references, bad names, unreachable sources, and invalid JSONPath expressions all cause an immediate exit with a clear error.
|
|
271
353
|
|
|
272
|
-
**Resume/
|
|
354
|
+
**Resume/redrive with variables:**
|
|
273
355
|
|
|
274
|
-
When
|
|
356
|
+
Requires `persist_context: true` at the top of the `workflow:` mapping. When enabled, on row failure any resolved variable values are persisted into reserved redrive-CSV columns named `_bulk_post_var/<source_path>/<name>`. Re-running the redrive CSV skips completed steps and reads these persisted values for variables whose source step was skipped. When `persist_context` is `false` (the default), variable values are not persisted and re-runs always start from step 1.
|
|
275
357
|
|
|
276
|
-
> **Security note:**
|
|
358
|
+
> **Security note:** redrive CSVs may contain response-derived data (potentially sensitive) in plaintext. Do not share or commit redrive CSVs that were produced from workflows using variables.
|
|
277
359
|
|
|
278
360
|
### Example
|
|
279
361
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "bulk-post"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "A near-stdlib Python CLI that fires templated HTTP requests driven by CSV data, one request (or a multi-step workflow) per row."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -58,7 +58,7 @@ from .csvio import (
|
|
|
58
58
|
_open_log_file as _open_log_file,
|
|
59
59
|
)
|
|
60
60
|
from .csvio import (
|
|
61
|
-
|
|
61
|
+
_open_redrive_writer as _open_redrive_writer,
|
|
62
62
|
)
|
|
63
63
|
from .csvio import (
|
|
64
64
|
_skip_rows as _skip_rows,
|
|
@@ -74,6 +74,18 @@ from .csvio import (
|
|
|
74
74
|
)
|
|
75
75
|
from .http import _mask_headers as _mask_headers
|
|
76
76
|
from .http import http_request as http_request
|
|
77
|
+
from .retry import (
|
|
78
|
+
RetryPolicy as RetryPolicy,
|
|
79
|
+
)
|
|
80
|
+
from .retry import (
|
|
81
|
+
compute_delay as compute_delay,
|
|
82
|
+
)
|
|
83
|
+
from .retry import (
|
|
84
|
+
parse_retry_policy as parse_retry_policy,
|
|
85
|
+
)
|
|
86
|
+
from .retry import (
|
|
87
|
+
request_with_retry as request_with_retry,
|
|
88
|
+
)
|
|
77
89
|
from .runner import (
|
|
78
90
|
_fire as _fire,
|
|
79
91
|
)
|
|
@@ -212,6 +224,9 @@ from .workflow import _WORKFLOW_STEP_COL as _WORKFLOW_STEP_COL
|
|
|
212
224
|
from .workflow import (
|
|
213
225
|
WorkflowStep as WorkflowStep,
|
|
214
226
|
)
|
|
227
|
+
from .workflow import (
|
|
228
|
+
WorkflowSteps as WorkflowSteps,
|
|
229
|
+
)
|
|
215
230
|
from .workflow import (
|
|
216
231
|
_fire_workflow_step as _fire_workflow_step,
|
|
217
232
|
)
|