gulp-cli 1.1.9__tar.gz → 1.1.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gulp_cli-1.1.9/src/gulp_cli.egg-info → gulp_cli-1.1.10}/PKG-INFO +1 -1
- gulp_cli-1.1.10/docs/stdout-markers.md +125 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/_version.py +3 -3
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/ingest.py +163 -39
- {gulp_cli-1.1.9 → gulp_cli-1.1.10/src/gulp_cli.egg-info}/PKG-INFO +1 -1
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli.egg-info/SOURCES.txt +1 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli.egg-info/scm_file_list.json +1 -0
- gulp_cli-1.1.10/src/gulp_cli.egg-info/scm_version.json +8 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/tests/test_ingest_wait_log.py +87 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/tests/test_zip_create.py +26 -0
- gulp_cli-1.1.9/src/gulp_cli.egg-info/scm_version.json +0 -8
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/.github/workflows/portable-bundles.yml +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/.github/workflows/python-package.yml +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/.gitignore +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/GULP_DUAL_LICENSE.txt +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/LICENSE.AGPL-3.0.txt +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/LICENSE.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/README.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/command-reference.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/examples.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/extensions.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/getting-started.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/portable.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/resource-management.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/docs/troubleshooting-cli.md +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/gulp-cli.spec +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/portable/launch-linux.sh +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/portable/launch-macos.sh +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/portable/launch-windows.bat +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/pyproject.toml +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/setup.cfg +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/__init__.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/__main__.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/cli.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/client.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/__init__.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/acl.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/auth.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/collab.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/context.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/db.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/enhance_map.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/enrich.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/glyph.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/mapping.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/operations.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/plugin.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/query.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/source.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/stats.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/storage.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/user_group.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/users.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/commands/utility.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/config.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/extension/__init__.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/extension/query_sigma_zip.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/extension/story.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/extension_helpers.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/extensions.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/output.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/utils.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli/version.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli.egg-info/dependency_links.txt +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli.egg-info/entry_points.txt +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli.egg-info/requires.txt +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/src/gulp_cli.egg-info/top_level.txt +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/tests/test_enrich_live.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/tests/test_ingest_paths_file.py +0 -0
- {gulp_cli-1.1.9 → gulp_cli-1.1.10}/tests/test_version.py +0 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Stdout Markers Cheat Sheet
|
|
2
|
+
|
|
3
|
+
Marker lines are emitted as one unwrapped stdout line:
|
|
4
|
+
|
|
5
|
+
```text
|
|
6
|
+
GULP_MARKER: [MARKER_NAME] | {"json": "payload"}
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Parser rule: match lines starting with `GULP_MARKER: `, split once on ` | `, then JSON-decode the right side.
|
|
10
|
+
|
|
11
|
+
## Ingest `--wait-log`
|
|
12
|
+
|
|
13
|
+
These markers are emitted by `ingest file --wait-log`, `ingest file-to-source --wait-log`, and `ingest raw --wait-log`.
|
|
14
|
+
|
|
15
|
+
### `[MARKER_INGEST_SOURCE_DONE_RECEIVED]`
|
|
16
|
+
|
|
17
|
+
Emitted when the backend sends `INGEST_SOURCE_DONE`.
|
|
18
|
+
|
|
19
|
+
Payload:
|
|
20
|
+
- `req_id`: backend request id.
|
|
21
|
+
- `file`: display label for the file/request.
|
|
22
|
+
- `status`: status from the source-done packet or current state.
|
|
23
|
+
- `ingested`: accumulated ingested record count.
|
|
24
|
+
- `skipped`: accumulated skipped record count.
|
|
25
|
+
- `failed`: accumulated failed record count.
|
|
26
|
+
- `errors`: extracted backend errors, usually an array of strings.
|
|
27
|
+
|
|
28
|
+
### `[MARKER_ONGOING_STATS_RECEIVED]`
|
|
29
|
+
|
|
30
|
+
Emitted for `STATS_CREATE` or `STATS_UPDATE` while status is not `done`, `failed`, or `canceled`.
|
|
31
|
+
|
|
32
|
+
Payload:
|
|
33
|
+
- `req_id`: backend request id.
|
|
34
|
+
- `file`: display label for the file/request.
|
|
35
|
+
- `event`: websocket event name, usually `STATS_CREATE` or `STATS_UPDATE`.
|
|
36
|
+
- `status`: current stats status.
|
|
37
|
+
- `ingested`: current authoritative ingested count.
|
|
38
|
+
- `skipped`: current authoritative skipped count.
|
|
39
|
+
- `failed`: current authoritative failed count.
|
|
40
|
+
- `pct`: ingest percentage when provided by the backend, otherwise `null`.
|
|
41
|
+
- `errors`: extracted backend errors, usually an array of strings.
|
|
42
|
+
|
|
43
|
+
### `[MARKER_DONE_STATS_RECEIVED]`
|
|
44
|
+
|
|
45
|
+
Emitted for `STATS_CREATE` or `STATS_UPDATE` when status is `done`.
|
|
46
|
+
|
|
47
|
+
Payload: same fields as `[MARKER_ONGOING_STATS_RECEIVED]`.
|
|
48
|
+
|
|
49
|
+
### `[MARKER_FAILED_STATS_RECEIVED]`
|
|
50
|
+
|
|
51
|
+
Emitted for `STATS_CREATE` or `STATS_UPDATE` when status is `failed` or `canceled`.
|
|
52
|
+
|
|
53
|
+
Also emitted after a backend websocket `ERROR` packet.
|
|
54
|
+
|
|
55
|
+
Payload:
|
|
56
|
+
- From stats packets: same fields as `[MARKER_ONGOING_STATS_RECEIVED]`.
|
|
57
|
+
- From `ERROR` packets: `req_id`, `file`, `event`, `status`, `ingested`, `skipped`, `failed`, `errors`.
|
|
58
|
+
|
|
59
|
+
### `[MARKER_BACKEND_EXCEPTION_REPORTED]`
|
|
60
|
+
|
|
61
|
+
Emitted when the backend sends websocket `ERROR`.
|
|
62
|
+
|
|
63
|
+
Payload:
|
|
64
|
+
- `req_id`: backend request id.
|
|
65
|
+
- `file`: display label for the file/request.
|
|
66
|
+
- `event`: websocket event name, `ERROR`.
|
|
67
|
+
- `status`: always set to `failed` by the CLI for this request.
|
|
68
|
+
- `errors`: extracted backend errors, usually an array of strings.
|
|
69
|
+
|
|
70
|
+
### `[MARKER_INGESTION_FINISHED]`
|
|
71
|
+
|
|
72
|
+
Emitted once after wait-log completion, before the final CLI result JSON/table output.
|
|
73
|
+
|
|
74
|
+
Payload:
|
|
75
|
+
- `requests_total`: number of request results.
|
|
76
|
+
- `requests_done`: count with status `done`.
|
|
77
|
+
- `requests_failed`: count with status `failed`.
|
|
78
|
+
- `requests_canceled`: count with status `canceled`.
|
|
79
|
+
- `requests_timeout`: count with status `timeout`.
|
|
80
|
+
- `ingested`: accumulated ingested record count.
|
|
81
|
+
- `skipped`: accumulated skipped record count.
|
|
82
|
+
- `failed`: accumulated failed record count.
|
|
83
|
+
|
|
84
|
+
## `ingest zip-create`
|
|
85
|
+
|
|
86
|
+
These markers are emitted by local ZIP creation.
|
|
87
|
+
|
|
88
|
+
### `[MARKER_FILE_ADDED_TO_ZIP]`
|
|
89
|
+
|
|
90
|
+
Emitted after each ZIP entry is written.
|
|
91
|
+
|
|
92
|
+
Payload:
|
|
93
|
+
- `zip`: requested output ZIP path.
|
|
94
|
+
- `entry`: ZIP entry name.
|
|
95
|
+
- `source`: source filesystem path for the entry.
|
|
96
|
+
- `file_size`: uncompressed entry size in bytes.
|
|
97
|
+
- `compress_size`: compressed entry size in bytes.
|
|
98
|
+
- `current`: 1-based count of entries added so far.
|
|
99
|
+
- `total`: total entries planned for the ZIP.
|
|
100
|
+
- `percent`: integer progress percentage.
|
|
101
|
+
|
|
102
|
+
### `[MARKER_ZIP_CREATED_SUCCESSFULLY]`
|
|
103
|
+
|
|
104
|
+
Emitted once after the ZIP or multipart ZIP volumes are published.
|
|
105
|
+
|
|
106
|
+
Payload:
|
|
107
|
+
- `zip_files`: array of created archive paths.
|
|
108
|
+
- `files_archived`: number of source files archived. Directory entries are not counted here.
|
|
109
|
+
- `entries_archived`: number of ZIP entries archived, including directory entries.
|
|
110
|
+
|
|
111
|
+
### `[MARKER_ZIP_CREATE_ERROR]`
|
|
112
|
+
|
|
113
|
+
Emitted when ZIP creation hits a validation, scan, write, or publish error.
|
|
114
|
+
|
|
115
|
+
Payload variants:
|
|
116
|
+
- Scan/write skip errors: `path`, `zip`, `error`.
|
|
117
|
+
- Top-level `zip-create` command errors: `zip`, `error`.
|
|
118
|
+
|
|
119
|
+
### `[MARKER_OTHER_ERROR]`
|
|
120
|
+
|
|
121
|
+
Emitted with top-level `zip-create` command exceptions after `[MARKER_ZIP_CREATE_ERROR]`.
|
|
122
|
+
|
|
123
|
+
Payload:
|
|
124
|
+
- `command`: currently `zip-create`.
|
|
125
|
+
- `error`: exception text.
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1, 1,
|
|
21
|
+
__version__ = version = '1.1.10'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 1, 10)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g015adf27b'
|
|
@@ -46,6 +46,31 @@ _TERMINAL_STATUSES = {"done", "failed", "canceled"}
|
|
|
46
46
|
_MAX_ZIP_PART_SIZE_BYTES = 4 * 1024**3 - 1
|
|
47
47
|
|
|
48
48
|
|
|
49
|
+
def _print_marker(marker: str, **payload: Any) -> None:
|
|
50
|
+
console.print(
|
|
51
|
+
f"GULP_MARKER: {marker} | {json.dumps(payload, sort_keys=True, default=str)}",
|
|
52
|
+
markup=False,
|
|
53
|
+
highlight=False,
|
|
54
|
+
soft_wrap=True,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _print_ingestion_finished_marker(results: list[dict[str, Any]]) -> None:
|
|
59
|
+
_print_marker(
|
|
60
|
+
"[MARKER_INGESTION_FINISHED]",
|
|
61
|
+
requests_total=len(results),
|
|
62
|
+
requests_done=sum(1 for item in results if item.get("status") == "done"),
|
|
63
|
+
requests_failed=sum(1 for item in results if item.get("status") == "failed"),
|
|
64
|
+
requests_canceled=sum(
|
|
65
|
+
1 for item in results if item.get("status") == "canceled"
|
|
66
|
+
),
|
|
67
|
+
requests_timeout=sum(1 for item in results if item.get("status") == "timeout"),
|
|
68
|
+
ingested=sum(int(item.get("ingested") or 0) for item in results),
|
|
69
|
+
skipped=sum(int(item.get("skipped") or 0) for item in results),
|
|
70
|
+
failed=sum(int(item.get("failed") or 0) for item in results),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
49
74
|
def _plugin_params_request_compression(plugin_params: dict[str, Any] | None) -> bool:
|
|
50
75
|
"""Return True when plugin parameters request core decompression."""
|
|
51
76
|
return bool(plugin_params and plugin_params.get("compressed"))
|
|
@@ -405,10 +430,22 @@ def _build_zip_from_sources(
|
|
|
405
430
|
)
|
|
406
431
|
)
|
|
407
432
|
except Exception as exc:
|
|
433
|
+
_print_marker(
|
|
434
|
+
"[MARKER_ZIP_CREATE_ERROR]",
|
|
435
|
+
path=str(current_path),
|
|
436
|
+
zip=str(output_zip),
|
|
437
|
+
error=str(exc),
|
|
438
|
+
)
|
|
408
439
|
print_error(
|
|
409
440
|
f"Skipping {current_path} while creating ZIP {output_zip}: {exc}"
|
|
410
441
|
)
|
|
411
442
|
except Exception as exc:
|
|
443
|
+
_print_marker(
|
|
444
|
+
"[MARKER_ZIP_CREATE_ERROR]",
|
|
445
|
+
path=str(source_path),
|
|
446
|
+
zip=str(output_zip),
|
|
447
|
+
error=str(exc),
|
|
448
|
+
)
|
|
412
449
|
print_error(
|
|
413
450
|
f"Skipping {source_path} while creating ZIP {output_zip}: {exc}"
|
|
414
451
|
)
|
|
@@ -461,7 +498,24 @@ def _build_zip_from_sources(
|
|
|
461
498
|
f"({info.file_size} bytes -> {info.compress_size} bytes, "
|
|
462
499
|
f"{entries_added}/{total_entries}, {percent}%)"
|
|
463
500
|
)
|
|
501
|
+
_print_marker(
|
|
502
|
+
"[MARKER_FILE_ADDED_TO_ZIP]",
|
|
503
|
+
zip=str(output_zip),
|
|
504
|
+
entry=info.filename,
|
|
505
|
+
source=str(source_path),
|
|
506
|
+
file_size=info.file_size,
|
|
507
|
+
compress_size=info.compress_size,
|
|
508
|
+
current=entries_added,
|
|
509
|
+
total=total_entries,
|
|
510
|
+
percent=percent,
|
|
511
|
+
)
|
|
464
512
|
except Exception as exc:
|
|
513
|
+
_print_marker(
|
|
514
|
+
"[MARKER_ZIP_CREATE_ERROR]",
|
|
515
|
+
path=str(source_path),
|
|
516
|
+
zip=str(output_zip),
|
|
517
|
+
error=str(exc),
|
|
518
|
+
)
|
|
465
519
|
print_error(
|
|
466
520
|
f"Skipping {source_path} while creating ZIP {output_zip}: {exc}"
|
|
467
521
|
)
|
|
@@ -615,6 +669,12 @@ def _build_zip_from_sources(
|
|
|
615
669
|
console.print(
|
|
616
670
|
f"[green]Created[/] {len(created_archives)} multipart ZIP volume(s)"
|
|
617
671
|
)
|
|
672
|
+
_print_marker(
|
|
673
|
+
"[MARKER_ZIP_CREATED_SUCCESSFULLY]",
|
|
674
|
+
zip_files=[str(path) for path in created_archives],
|
|
675
|
+
files_archived=archived_count,
|
|
676
|
+
entries_archived=len(archived_entries),
|
|
677
|
+
)
|
|
618
678
|
return archived_count, archived_entries, created_archives
|
|
619
679
|
else:
|
|
620
680
|
temp_archive_path = temp_dir / output_zip.name
|
|
@@ -633,6 +693,12 @@ def _build_zip_from_sources(
|
|
|
633
693
|
created_archives = [output_zip]
|
|
634
694
|
|
|
635
695
|
console.print(f"[green]Created[/] {len(created_archives)} archive part(s)")
|
|
696
|
+
_print_marker(
|
|
697
|
+
"[MARKER_ZIP_CREATED_SUCCESSFULLY]",
|
|
698
|
+
zip_files=[str(path) for path in created_archives],
|
|
699
|
+
files_archived=archived_count,
|
|
700
|
+
entries_archived=len(archived_entries),
|
|
701
|
+
)
|
|
636
702
|
return archived_count, archived_entries, created_archives
|
|
637
703
|
|
|
638
704
|
|
|
@@ -1069,6 +1135,16 @@ class _IngestWsTracker:
|
|
|
1069
1135
|
if terminal_ev is not None:
|
|
1070
1136
|
terminal_ev.set()
|
|
1071
1137
|
if self._log_updates:
|
|
1138
|
+
_print_marker(
|
|
1139
|
+
"[MARKER_INGEST_SOURCE_DONE_RECEIVED]",
|
|
1140
|
+
req_id=req_id,
|
|
1141
|
+
file=self._request_label(req_id),
|
|
1142
|
+
status=source_status,
|
|
1143
|
+
ingested=state.get("ingested", 0),
|
|
1144
|
+
skipped=state.get("skipped", 0),
|
|
1145
|
+
failed=state.get("failed", 0),
|
|
1146
|
+
errors=_extract_errors(obj),
|
|
1147
|
+
)
|
|
1072
1148
|
console.print(
|
|
1073
1149
|
_format_per_file_log_line(
|
|
1074
1150
|
{
|
|
@@ -1110,6 +1186,24 @@ class _IngestWsTracker:
|
|
|
1110
1186
|
if terminal_ev is not None:
|
|
1111
1187
|
terminal_ev.set()
|
|
1112
1188
|
if self._log_updates:
|
|
1189
|
+
if status == "done":
|
|
1190
|
+
marker = "[MARKER_DONE_STATS_RECEIVED]"
|
|
1191
|
+
elif status in {"failed", "canceled"}:
|
|
1192
|
+
marker = "[MARKER_FAILED_STATS_RECEIVED]"
|
|
1193
|
+
else:
|
|
1194
|
+
marker = "[MARKER_ONGOING_STATS_RECEIVED]"
|
|
1195
|
+
_print_marker(
|
|
1196
|
+
marker,
|
|
1197
|
+
req_id=req_id,
|
|
1198
|
+
file=self._request_label(req_id),
|
|
1199
|
+
event=msg.type,
|
|
1200
|
+
status=state["status"],
|
|
1201
|
+
ingested=state["ingested"],
|
|
1202
|
+
skipped=state["skipped"],
|
|
1203
|
+
failed=state["failed"],
|
|
1204
|
+
pct=obj.get("ingest_percentage"),
|
|
1205
|
+
errors=_extract_errors(obj),
|
|
1206
|
+
)
|
|
1113
1207
|
console.print(
|
|
1114
1208
|
_format_per_file_log_line(
|
|
1115
1209
|
{
|
|
@@ -1134,6 +1228,25 @@ class _IngestWsTracker:
|
|
|
1134
1228
|
if terminal_ev is not None:
|
|
1135
1229
|
terminal_ev.set()
|
|
1136
1230
|
if self._log_updates:
|
|
1231
|
+
_print_marker(
|
|
1232
|
+
"[MARKER_BACKEND_EXCEPTION_REPORTED]",
|
|
1233
|
+
req_id=req_id,
|
|
1234
|
+
file=self._request_label(req_id),
|
|
1235
|
+
event=msg.type,
|
|
1236
|
+
status=state["status"],
|
|
1237
|
+
errors=_extract_errors(obj),
|
|
1238
|
+
)
|
|
1239
|
+
_print_marker(
|
|
1240
|
+
"[MARKER_FAILED_STATS_RECEIVED]",
|
|
1241
|
+
req_id=req_id,
|
|
1242
|
+
file=self._request_label(req_id),
|
|
1243
|
+
event=msg.type,
|
|
1244
|
+
status=state["status"],
|
|
1245
|
+
ingested=state["ingested"],
|
|
1246
|
+
skipped=state["skipped"],
|
|
1247
|
+
failed=state["failed"],
|
|
1248
|
+
errors=_extract_errors(obj),
|
|
1249
|
+
)
|
|
1137
1250
|
console.print(
|
|
1138
1251
|
_format_per_file_log_line(
|
|
1139
1252
|
{
|
|
@@ -1643,6 +1756,8 @@ def ingest_file(
|
|
|
1643
1756
|
for fp, rid in file_req_map.items()
|
|
1644
1757
|
]
|
|
1645
1758
|
|
|
1759
|
+
if wait_log:
|
|
1760
|
+
_print_ingestion_finished_marker(results)
|
|
1646
1761
|
print_result(results)
|
|
1647
1762
|
|
|
1648
1763
|
asyncio.run(_run())
|
|
@@ -1949,6 +2064,8 @@ def ingest_file_to_source(
|
|
|
1949
2064
|
for fp, rid in file_req_map.items()
|
|
1950
2065
|
]
|
|
1951
2066
|
|
|
2067
|
+
if wait_log:
|
|
2068
|
+
_print_ingestion_finished_marker(results)
|
|
1952
2069
|
print_result(results)
|
|
1953
2070
|
|
|
1954
2071
|
asyncio.run(_run())
|
|
@@ -1987,48 +2104,53 @@ def ingest_zip_create(
|
|
|
1987
2104
|
) -> None:
|
|
1988
2105
|
"""Create a ZIP archive from files, directories, or glob patterns."""
|
|
1989
2106
|
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
f"Output ZIP already exists: {output_path}. Omit --no-overwrite to replace it"
|
|
2001
|
-
)
|
|
2002
|
-
if split_size_bytes and split_size_bytes > 0 and not overwrite:
|
|
2003
|
-
for archive_path in output_path.parent.glob(f"{output_path.name}.*"):
|
|
2004
|
-
if not archive_path.name.removeprefix(f"{output_path.name}.").isdigit():
|
|
2005
|
-
continue
|
|
2107
|
+
try:
|
|
2108
|
+
overwrite = not no_overwrite
|
|
2109
|
+
preserve_path = not no_preserve_path
|
|
2110
|
+
output_path = _resolve_path(output_zip)
|
|
2111
|
+
source_paths = _expand_source_patterns(path_patterns or [], paths_file)
|
|
2112
|
+
split_size_bytes = _parse_zip_split_size(split_size)
|
|
2113
|
+
if split_size_bytes is not None and split_size_bytes < 0:
|
|
2114
|
+
raise typer.BadParameter("Split size must be zero or greater")
|
|
2115
|
+
|
|
2116
|
+
if output_path.exists() and not overwrite:
|
|
2006
2117
|
raise typer.BadParameter(
|
|
2007
|
-
f"Output ZIP already exists: {
|
|
2118
|
+
f"Output ZIP already exists: {output_path}. Omit --no-overwrite to replace it"
|
|
2008
2119
|
)
|
|
2120
|
+
if split_size_bytes and split_size_bytes > 0 and not overwrite:
|
|
2121
|
+
for archive_path in output_path.parent.glob(f"{output_path.name}.*"):
|
|
2122
|
+
if not archive_path.name.removeprefix(f"{output_path.name}.").isdigit():
|
|
2123
|
+
continue
|
|
2124
|
+
raise typer.BadParameter(
|
|
2125
|
+
f"Output ZIP already exists: {archive_path}. Omit --no-overwrite to replace it"
|
|
2126
|
+
)
|
|
2009
2127
|
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2128
|
+
archived_count, archived_entries, created_archives = _build_zip_from_sources(
|
|
2129
|
+
output_path,
|
|
2130
|
+
source_paths,
|
|
2131
|
+
preserve_path=preserve_path,
|
|
2132
|
+
split_size_bytes=split_size_bytes,
|
|
2133
|
+
)
|
|
2134
|
+
print_result(
|
|
2135
|
+
{
|
|
2136
|
+
"zip_files": [str(path) for path in created_archives],
|
|
2137
|
+
"sources": [str(path) for path, _ in source_paths],
|
|
2138
|
+
"files_archived": archived_count,
|
|
2139
|
+
"entries_archived": len(archived_entries),
|
|
2140
|
+
"entries": archived_entries,
|
|
2141
|
+
"preserve_path": preserve_path,
|
|
2142
|
+
"split_size": split_size_bytes,
|
|
2143
|
+
},
|
|
2144
|
+
formatter=lambda data: console.print(
|
|
2145
|
+
f"Created ZIP archive(s) {', '.join(data['zip_files'])} from {len(data['sources'])} source"
|
|
2146
|
+
f"{'s' if len(data['sources']) != 1 else ''} with {data['files_archived']} file(s), "
|
|
2147
|
+
f"{data['entries_archived']} ZIP entr{'y' if data['entries_archived'] == 1 else 'ies'}."
|
|
2148
|
+
),
|
|
2149
|
+
)
|
|
2150
|
+
except Exception as exc:
|
|
2151
|
+
_print_marker("[MARKER_ZIP_CREATE_ERROR]", zip=output_zip, error=str(exc))
|
|
2152
|
+
_print_marker("[MARKER_OTHER_ERROR]", command="zip-create", error=str(exc))
|
|
2153
|
+
raise
|
|
2032
2154
|
|
|
2033
2155
|
|
|
2034
2156
|
@app.command("raw")
|
|
@@ -2176,6 +2298,8 @@ def ingest_raw(
|
|
|
2176
2298
|
waited = await tracker.wait_for_terminal(
|
|
2177
2299
|
request_label, resolved_req_id, wait_timeout
|
|
2178
2300
|
)
|
|
2301
|
+
if wait_log:
|
|
2302
|
+
_print_ingestion_finished_marker([waited])
|
|
2179
2303
|
print_result(waited)
|
|
2180
2304
|
else:
|
|
2181
2305
|
timed_out = await _wait_for_stats_create(client, [resolved_req_id])
|
|
@@ -7,6 +7,7 @@ import pytest
|
|
|
7
7
|
|
|
8
8
|
from gulp_cli.commands.ingest import (
|
|
9
9
|
_IngestWsTracker,
|
|
10
|
+
_print_ingestion_finished_marker,
|
|
10
11
|
ingest_file,
|
|
11
12
|
ingest_file_to_source,
|
|
12
13
|
ingest_raw,
|
|
@@ -76,6 +77,7 @@ def test_ingest_ws_tracker_logs_updates_to_stdout(capsys) -> None:
|
|
|
76
77
|
assert "req12345" in out
|
|
77
78
|
assert "ingested=12" in out
|
|
78
79
|
assert "skipped=1" in out
|
|
80
|
+
assert "GULP_MARKER: [MARKER_ONGOING_STATS_RECEIVED]" in out
|
|
79
81
|
|
|
80
82
|
|
|
81
83
|
def test_ingest_ws_tracker_marks_failures_obviously(capsys) -> None:
|
|
@@ -106,6 +108,91 @@ def test_ingest_ws_tracker_marks_failures_obviously(capsys) -> None:
|
|
|
106
108
|
assert "req12345" in out
|
|
107
109
|
assert "sample.txt" in out
|
|
108
110
|
assert "backend rejected file" in out
|
|
111
|
+
assert "GULP_MARKER: [MARKER_BACKEND_EXCEPTION_REPORTED]" in out
|
|
112
|
+
assert "GULP_MARKER: [MARKER_FAILED_STATS_RECEIVED]" in out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_ingest_ws_tracker_marks_source_done(capsys) -> None:
|
|
116
|
+
class _FakeWs:
|
|
117
|
+
def on_message(self, *_args, **_kwargs) -> None:
|
|
118
|
+
pass
|
|
119
|
+
|
|
120
|
+
def off_message(self, *_args, **_kwargs) -> None:
|
|
121
|
+
pass
|
|
122
|
+
|
|
123
|
+
tracker = _IngestWsTracker(
|
|
124
|
+
_FakeWs(),
|
|
125
|
+
log_updates=True,
|
|
126
|
+
request_label_getter=lambda _req_id: "/tmp/sample.txt",
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
tracker._on_message( # noqa: SLF001
|
|
130
|
+
WSMessage(
|
|
131
|
+
type=WSMessageType.INGEST_SOURCE_DONE.value,
|
|
132
|
+
req_id="req12345",
|
|
133
|
+
timestamp_msec=0,
|
|
134
|
+
data={"obj": {"status": "done", "records_ingested": 3}},
|
|
135
|
+
)
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
out = capsys.readouterr().out
|
|
139
|
+
assert "GULP_MARKER: [MARKER_INGEST_SOURCE_DONE_RECEIVED]" in out
|
|
140
|
+
assert '"ingested": 3' in out
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@pytest.mark.parametrize(
|
|
144
|
+
("status", "marker"),
|
|
145
|
+
[
|
|
146
|
+
("done", "GULP_MARKER: [MARKER_DONE_STATS_RECEIVED]"),
|
|
147
|
+
("failed", "GULP_MARKER: [MARKER_FAILED_STATS_RECEIVED]"),
|
|
148
|
+
],
|
|
149
|
+
)
|
|
150
|
+
def test_ingest_ws_tracker_marks_terminal_stats(
|
|
151
|
+
status: str, marker: str, capsys
|
|
152
|
+
) -> None:
|
|
153
|
+
class _FakeWs:
|
|
154
|
+
def on_message(self, *_args, **_kwargs) -> None:
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
def off_message(self, *_args, **_kwargs) -> None:
|
|
158
|
+
pass
|
|
159
|
+
|
|
160
|
+
tracker = _IngestWsTracker(
|
|
161
|
+
_FakeWs(),
|
|
162
|
+
log_updates=True,
|
|
163
|
+
request_label_getter=lambda _req_id: "/tmp/sample.txt",
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
tracker._on_message( # noqa: SLF001
|
|
167
|
+
WSMessage(
|
|
168
|
+
type=WSMessageType.STATS_UPDATE.value,
|
|
169
|
+
req_id="req12345",
|
|
170
|
+
timestamp_msec=0,
|
|
171
|
+
data={"obj": {"status": status}},
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
assert marker in capsys.readouterr().out
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_ingestion_finished_marker(capsys) -> None:
|
|
179
|
+
_print_ingestion_finished_marker(
|
|
180
|
+
[
|
|
181
|
+
{"status": "done", "ingested": 10, "skipped": 2, "failed": 1},
|
|
182
|
+
{"status": "failed", "ingested": 3, "skipped": 0, "failed": 4},
|
|
183
|
+
{"status": "timeout"},
|
|
184
|
+
]
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
out = capsys.readouterr().out
|
|
188
|
+
assert "GULP_MARKER: [MARKER_INGESTION_FINISHED]" in out
|
|
189
|
+
assert '"requests_total": 3' in out
|
|
190
|
+
assert '"requests_done": 1' in out
|
|
191
|
+
assert '"requests_failed": 1' in out
|
|
192
|
+
assert '"requests_timeout": 1' in out
|
|
193
|
+
assert '"ingested": 13' in out
|
|
194
|
+
assert '"skipped": 2' in out
|
|
195
|
+
assert '"failed": 5' in out
|
|
109
196
|
|
|
110
197
|
|
|
111
198
|
@pytest.mark.asyncio
|
|
@@ -158,6 +158,8 @@ def test_build_zip_from_sources_prints_file_progress(
|
|
|
158
158
|
assert "1/1" in captured
|
|
159
159
|
assert "100%" in captured
|
|
160
160
|
assert "bytes ->" in captured
|
|
161
|
+
assert "GULP_MARKER: [MARKER_FILE_ADDED_TO_ZIP]" in captured
|
|
162
|
+
assert "GULP_MARKER: [MARKER_ZIP_CREATED_SUCCESSFULLY]" in captured
|
|
161
163
|
|
|
162
164
|
|
|
163
165
|
def test_build_zip_from_sources_prints_directory_entries(
|
|
@@ -182,11 +184,35 @@ def test_build_zip_from_sources_prints_directory_entries(
|
|
|
182
184
|
assert "samples/nested/" in captured
|
|
183
185
|
assert "samples/nested/sample.txt" in captured
|
|
184
186
|
assert "2/2" in captured
|
|
187
|
+
assert captured.count("GULP_MARKER: [MARKER_FILE_ADDED_TO_ZIP]") == 2
|
|
185
188
|
|
|
186
189
|
with zipfile.ZipFile(created_archives[0]) as archive:
|
|
187
190
|
assert archive.namelist() == archived_entries
|
|
188
191
|
|
|
189
192
|
|
|
193
|
+
def test_ingest_zip_create_marks_errors(
|
|
194
|
+
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
|
195
|
+
) -> None:
|
|
196
|
+
source_file = tmp_path / "sample.txt"
|
|
197
|
+
source_file.write_text("payload", encoding="utf-8")
|
|
198
|
+
output_zip = tmp_path / "out.zip"
|
|
199
|
+
output_zip.write_text("existing", encoding="utf-8")
|
|
200
|
+
|
|
201
|
+
with pytest.raises(typer.BadParameter):
|
|
202
|
+
ingest_zip_create(
|
|
203
|
+
str(output_zip),
|
|
204
|
+
[str(source_file)],
|
|
205
|
+
None,
|
|
206
|
+
True,
|
|
207
|
+
False,
|
|
208
|
+
None,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
captured = capsys.readouterr().out
|
|
212
|
+
assert "GULP_MARKER: [MARKER_ZIP_CREATE_ERROR]" in captured
|
|
213
|
+
assert "GULP_MARKER: [MARKER_OTHER_ERROR]" in captured
|
|
214
|
+
|
|
215
|
+
|
|
190
216
|
def test_build_zip_from_sources_uses_config_dir_for_temp_files(
|
|
191
217
|
tmp_path: Path, monkeypatch
|
|
192
218
|
) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|