gulp-cli 1.1.7__tar.gz → 1.1.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. {gulp_cli-1.1.7/src/gulp_cli.egg-info → gulp_cli-1.1.9}/PKG-INFO +2 -2
  2. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/README.md +1 -1
  3. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/command-reference.md +6 -60
  4. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/examples.md +5 -10
  5. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/getting-started.md +3 -3
  6. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/resource-management.md +0 -52
  7. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/_version.py +3 -3
  8. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/ingest.py +117 -185
  9. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/plugin.py +0 -68
  10. {gulp_cli-1.1.7 → gulp_cli-1.1.9/src/gulp_cli.egg-info}/PKG-INFO +2 -2
  11. gulp_cli-1.1.9/src/gulp_cli.egg-info/scm_version.json +8 -0
  12. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_ingest_paths_file.py +37 -0
  13. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_ingest_wait_log.py +0 -11
  14. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_zip_create.py +15 -27
  15. gulp_cli-1.1.7/src/gulp_cli.egg-info/scm_version.json +0 -8
  16. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/.github/workflows/portable-bundles.yml +0 -0
  17. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/.github/workflows/python-package.yml +0 -0
  18. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/.gitignore +0 -0
  19. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/GULP_DUAL_LICENSE.txt +0 -0
  20. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/LICENSE.AGPL-3.0.txt +0 -0
  21. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/LICENSE.md +0 -0
  22. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/extensions.md +0 -0
  23. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/portable.md +0 -0
  24. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/troubleshooting-cli.md +0 -0
  25. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/gulp-cli.spec +0 -0
  26. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/portable/launch-linux.sh +0 -0
  27. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/portable/launch-macos.sh +0 -0
  28. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/portable/launch-windows.bat +0 -0
  29. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/pyproject.toml +0 -0
  30. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/setup.cfg +0 -0
  31. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/__init__.py +0 -0
  32. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/__main__.py +0 -0
  33. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/cli.py +0 -0
  34. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/client.py +0 -0
  35. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/__init__.py +0 -0
  36. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/acl.py +0 -0
  37. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/auth.py +0 -0
  38. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/collab.py +0 -0
  39. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/context.py +0 -0
  40. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/db.py +0 -0
  41. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/enhance_map.py +0 -0
  42. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/enrich.py +0 -0
  43. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/glyph.py +0 -0
  44. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/mapping.py +0 -0
  45. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/operations.py +0 -0
  46. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/query.py +0 -0
  47. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/source.py +0 -0
  48. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/stats.py +0 -0
  49. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/storage.py +0 -0
  50. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/user_group.py +0 -0
  51. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/users.py +0 -0
  52. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/utility.py +0 -0
  53. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/config.py +0 -0
  54. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension/__init__.py +0 -0
  55. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension/query_sigma_zip.py +0 -0
  56. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension/story.py +0 -0
  57. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension_helpers.py +0 -0
  58. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extensions.py +0 -0
  59. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/output.py +0 -0
  60. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/utils.py +0 -0
  61. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/version.py +0 -0
  62. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/SOURCES.txt +0 -0
  63. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/dependency_links.txt +0 -0
  64. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/entry_points.txt +0 -0
  65. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/requires.txt +0 -0
  66. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/scm_file_list.json +0 -0
  67. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/top_level.txt +0 -0
  68. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_enrich_live.py +0 -0
  69. {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gulp-cli
3
- Version: 1.1.7
3
+ Version: 1.1.9
4
4
  Summary: Command-line client for gULP
5
5
  Author-email: Mentat <info@mentat.is>
6
6
  Requires-Python: >=3.12
@@ -22,7 +22,7 @@ Dynamic: license-file
22
22
  ## ✨ What can you do?
23
23
 
24
24
  - 🔐 **Authentication** — secure login with token persistence
25
- - 📥 **Ingestion** — ingest files (single/batch/wildcard), zip archives, with concurrent uploads
25
+ - 📥 **Ingestion** — ingest files (single/batch/wildcard), optionally compressed before upload, with concurrent uploads
26
26
  - 🔍 **Querying** — raw OpenSearch queries, Sigma rules, external plugins
27
27
  - 🏷️ **Enrichment** — enrich documents, tag/untag, update fields
28
28
  - 👥 **User Management** — create users, manage permissions (admin only)
@@ -5,7 +5,7 @@
5
5
  ## ✨ What can you do?
6
6
 
7
7
  - 🔐 **Authentication** — secure login with token persistence
8
- - 📥 **Ingestion** — ingest files (single/batch/wildcard), zip archives, with concurrent uploads
8
+ - 📥 **Ingestion** — ingest files (single/batch/wildcard), optionally compressed before upload, with concurrent uploads
9
9
  - 🔍 **Querying** — raw OpenSearch queries, Sigma rules, external plugins
10
10
  - 🏷️ **Enrichment** — enrich documents, tag/untag, update fields
11
11
  - 👥 **User Management** — create users, manage permissions (admin only)
@@ -461,7 +461,7 @@ gulp-cli ingest file OPERATION_ID PLUGIN [FILE...] [OPTIONS]
461
461
  **Options:**
462
462
 
463
463
  - `--context TEXT` — Context for ingestion, specify a name to create a new context if it doesn't exist, or an existing `context_id` (default: `sdk_context`)
464
- - `--plugin-params TEXT` — JSON string with plugin parameters
464
+ - `--plugin-params TEXT` — JSON string with plugin parameters. When it includes `"compressed": true`, gulp-cli bzip2-compresses each input file before upload and the backend decompresses it before plugin ingestion.
465
465
  - `--flt TEXT` — JSON object for `GulpIngestionFilter`
466
466
  - `--paths-file PATH` — Text file with one file or glob pattern per line. When set, positional `FILE` arguments are optional and ignored if also provided.
467
467
  - `--reset-operation` — Delete and recreate the operation before ingest starts
@@ -493,6 +493,9 @@ gulp-cli ingest file my_op win_evtx --paths-file /tmp/file_paths.txt
493
493
  # With plugin parameters
494
494
  gulp-cli ingest file my_op csv file.csv --plugin-params '{"delimiter":";","encoding":"utf-8"}'
495
495
 
496
+ # Compress before upload and let the backend decompress before ingestion
497
+ gulp-cli ingest file my_op win_evtx /path/to/System.evtx --plugin-params '{"compressed": true}'
498
+
496
499
  # Preview mode (no ingestion persisted)
497
500
  gulp-cli ingest file my_op win_evtx /path/to/System.evtx --preview
498
501
 
@@ -521,7 +524,7 @@ gulp-cli ingest file-to-source SOURCE_ID [FILE...] [OPTIONS]
521
524
  **Options:**
522
525
 
523
526
  - `--plugin TEXT` — Override the plugin associated with the source (requires `--plugin-params`)
524
- - `--plugin-params TEXT` — JSON object for plugin_params (overrides source defaults; required when `--plugin` is set, `{}` is allowed)
527
+ - `--plugin-params TEXT` — JSON object for plugin_params (overrides source defaults; required when `--plugin` is set, `{}` is allowed). When it includes `"compressed": true`, gulp-cli bzip2-compresses each input file before upload and the backend decompresses it before plugin ingestion.
525
528
  - `--flt TEXT` — JSON object for GulpIngestionFilter
526
529
  - `--paths-file PATH` — Text file with one file or glob pattern per line. When set, positional `FILE` arguments are optional and ignored if also provided.
527
530
  - `--wait` — Wait for completion with progress
@@ -536,38 +539,7 @@ gulp-cli ingest file-to-source source123 /new/System.evtx --wait
536
539
  gulp-cli ingest file-to-source source123 '/new/*.evtx'
537
540
  gulp-cli ingest file-to-source source123 --paths-file /tmp/file_paths.txt
538
541
  gulp-cli ingest file-to-source source123 /new/events.json --plugin json --plugin-params '{"mapping_parameters": {}}'
539
- ```
540
-
541
- ---
542
-
543
- #### `zip`
544
-
545
- Ingest from a ZIP archive.
546
-
547
- > TO BE DEPRECATED...
548
-
549
- ```bash
550
- gulp-cli ingest zip OPERATION_ID ZIP_FILE [OPTIONS]
551
- ```
552
-
553
- **Arguments:**
554
-
555
- - `OPERATION_ID` — Target operation
556
- - `ZIP_FILE` — Path to ZIP file containing a `metadata.json` in the root, describing the content as specified in gulp's `ingest_zip` docs.
557
-
558
- **Options:**
559
-
560
- - `--context TEXT` — Context for ingestion, specify a name to create a new context if it doesn't exist, or an existing `context_id` (default: `sdk_context`)
561
- - `--flt TEXT` — JSON object for `GulpIngestionFilter`
562
- - `--create-operation` — Create operation automatically when it does not exist
563
- - `--wait` — Wait for completion
564
- - `--timeout INTEGER` — Timeout in seconds for `--wait` mode
565
-
566
- **Examples:**
567
-
568
- ```bash
569
- gulp-cli ingest zip my_op /data/evidence.zip --wait
570
- gulp-cli ingest zip my_op /data/evidence.zip --create-operation
542
+ gulp-cli ingest file-to-source source123 /new/System.evtx --plugin win_evtx --plugin-params '{"compressed": true}'
571
543
  ```
572
544
 
573
545
  ---
@@ -2330,32 +2302,6 @@ gulp-cli plugin list-ui [OPTIONS]
2330
2302
 
2331
2303
  ---
2332
2304
 
2333
- #### `plugin upload`
2334
-
2335
- Upload a new plugin.
2336
-
2337
- ```bash
2338
- gulp-cli plugin upload PLUGIN_FILE [OPTIONS]
2339
- ```
2340
-
2341
- #### `plugin download`
2342
-
2343
- Download a plugin.
2344
-
2345
- ```bash
2346
- gulp-cli plugin download FILENAME OUTPUT_PATH [OPTIONS]
2347
- ```
2348
-
2349
- #### `plugin delete`
2350
-
2351
- Delete a plugin.
2352
-
2353
- ```bash
2354
- gulp-cli plugin delete FILENAME [OPTIONS]
2355
- ```
2356
-
2357
- ---
2358
-
2359
2305
  #### `mapping list`
2360
2306
 
2361
2307
  List all mapping files.
@@ -8,7 +8,7 @@
8
8
  - [CSV Data Ingestion with Custom Parameters](#csv-data-ingestion-with-custom-parameters)
9
9
  - [JSON Logs Ingestion](#json-logs-ingestion)
10
10
  - [Add More Evidence to Existing Source](#add-more-evidence-to-existing-source)
11
- - [ZIP Archive Ingestion](#zip-archive-ingestion)
11
+ - [Compressed File Ingestion](#compressed-file-ingestion)
12
12
  - [Create ZIP Archive from Paths and Masks](#create-zip-archive-from-paths-and-masks)
13
13
  - [Raw Payload Ingestion](#raw-payload-ingestion)
14
14
  - [Request Stats Monitoring Workflows](#request-stats-monitoring-workflows)
@@ -205,14 +205,12 @@ gulp-cli ingest file-to-source data-2026-001 /additional/evidence.evtx --wait
205
205
  gulp-cli ingest file-to-source data-2026-001 '/additional/*.evtx' --wait
206
206
  ```
207
207
 
208
- ### ZIP Archive Ingestion
208
+ ### Compressed File Ingestion
209
209
 
210
210
  ```bash
211
- # Ingest prepared ZIP evidence archive
212
- gulp-cli ingest zip incident-001 /evidence/evidence.zip --wait
213
-
214
- # Auto-create operation when missing
215
- gulp-cli ingest zip incident-001 /evidence/evidence.zip --create-operation
211
+ # Compress files with bzip2 before upload; the backend decompresses before ingestion
212
+ gulp-cli ingest file incident-001 win_evtx /evidence/System.evtx \
213
+ --plugin-params '{"compressed": true}' --wait
216
214
  ```
217
215
 
218
216
  ### Create ZIP Archive from Paths and Masks
@@ -233,9 +231,6 @@ $CASE_ROOT/linux/**/*.log
233
231
  EOF
234
232
 
235
233
  gulp-cli ingest zip-create '$CASE_ROOT/evidence.zip' --paths-file /tmp/evidence_paths.txt --overwrite
236
-
237
- # Then ingest the generated ZIP
238
- gulp-cli ingest zip incident-001 '$CASE_ROOT/evidence.zip' --wait
239
234
  ```
240
235
 
241
236
  ### Raw Payload Ingestion
@@ -207,11 +207,11 @@ This shows a real-time progress bar while documents are being ingested. Per-file
207
207
  gulp-cli ingest file-to-source existing-source-id /gulp/samples/win_evtx/Security.evtx --wait
208
208
  ```
209
209
 
210
- ### Ingest ZIP / Raw Data
210
+ ### Compressed / Raw Data
211
211
 
212
212
  ```bash
213
- # ZIP archive ingestion
214
- gulp-cli ingest zip my_investigation /path/to/evidence.zip --wait
213
+ # Compress locally with bzip2 before upload; the backend decompresses before ingestion
214
+ gulp-cli ingest file my_investigation win_evtx /path/to/System.evtx --plugin-params '{"compressed": true}' --wait
215
215
 
216
216
  # Raw payload ingestion from a file
217
217
  gulp-cli ingest raw my_investigation --data-file /tmp/raw_chunk.json --last --wait
@@ -196,58 +196,6 @@ List available UI plugins:
196
196
  gulp plugin list-ui
197
197
  ```
198
198
 
199
- ### Upload Plugin
200
-
201
- Upload a custom plugin:
202
-
203
- ```bash
204
- # Upload as default plugin
205
- gulp plugin upload /path/to/my_plugin.py
206
-
207
- # Upload as extension plugin
208
- gulp plugin upload /path/to/my_extension.py --type extension
209
-
210
- # Upload as UI plugin
211
- gulp plugin upload /path/to/my_ui_plugin.py --type ui
212
-
213
- # Fail if already exists
214
- gulp plugin upload /path/to/my_plugin.py --fail-if-exists
215
- ```
216
-
217
- Response includes the file path where the plugin was saved on the server.
218
-
219
- ### Download Plugin
220
-
221
- Download a plugin from the server:
222
-
223
- ```bash
224
- # Download default plugin
225
- gulp plugin download win_evtx.py /local/path/win_evtx.py
226
-
227
- # Download extension plugin
228
- gulp plugin download my_extension.py /path/to/save --type extension
229
-
230
- # Download UI plugin
231
- gulp plugin download my_ui.py /path/to/save --type ui
232
- ```
233
-
234
- ### Delete Plugin
235
-
236
- Delete a plugin from the server:
237
-
238
- ```bash
239
- # Delete default plugin
240
- gulp plugin delete my_plugin.py
241
-
242
- # Delete extension plugin
243
- gulp plugin delete my_extension.py --type extension
244
-
245
- # Delete UI plugin
246
- gulp plugin delete my_ui.py --type ui
247
- ```
248
-
249
- ---
250
-
251
199
  ## Mapping File Management
252
200
 
253
201
  Mapping files define data transformations and field mappings for ingestion plugins.
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '1.1.7'
22
- __version_tuple__ = version_tuple = (1, 1, 7)
21
+ __version__ = version = '1.1.9'
22
+ __version_tuple__ = version_tuple = (1, 1, 9)
23
23
 
24
- __commit_id__ = commit_id = 'g5f188bb2b'
24
+ __commit_id__ = commit_id = 'g487f18224'
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import asyncio
4
+ import bz2
4
5
  import json
5
6
  import os
6
7
  import re
@@ -45,6 +46,43 @@ _TERMINAL_STATUSES = {"done", "failed", "canceled"}
45
46
  _MAX_ZIP_PART_SIZE_BYTES = 4 * 1024**3 - 1
46
47
 
47
48
 
49
+ def _plugin_params_request_compression(plugin_params: dict[str, Any] | None) -> bool:
50
+ """Return True when plugin parameters request core decompression."""
51
+ return bool(plugin_params and plugin_params.get("compressed"))
52
+
53
+
54
+ def _bz2_compress_file_for_ingestion_sync(file_path: str) -> str:
55
+ """Compress a file into a temporary bzip2 file for upload."""
56
+ source_path = Path(file_path)
57
+ fd, temp_path = tempfile.mkstemp(
58
+ prefix=f"{source_path.name}.",
59
+ suffix=".bz2",
60
+ )
61
+ os.close(fd)
62
+
63
+ try:
64
+ with open(source_path, "rb") as src, bz2.open(temp_path, "wb") as dst:
65
+ shutil.copyfileobj(src, dst)
66
+ except Exception:
67
+ try:
68
+ Path(temp_path).unlink()
69
+ except FileNotFoundError:
70
+ pass
71
+ raise
72
+
73
+ return temp_path
74
+
75
+
76
+ async def _maybe_bz2_compress_file_for_ingestion(
77
+ file_path: str, plugin_params: dict[str, Any] | None
78
+ ) -> tuple[str, str | None]:
79
+ """Return an upload path and optional temporary path to delete after upload."""
80
+ if not _plugin_params_request_compression(plugin_params):
81
+ return file_path, None
82
+ temp_path = await asyncio.to_thread(_bz2_compress_file_for_ingestion_sync, file_path)
83
+ return temp_path, temp_path
84
+
85
+
48
86
  def _default_ingest_batch_size() -> int:
49
87
  return max(1, (os.cpu_count() or 1) * 2)
50
88
 
@@ -280,6 +318,29 @@ def _resolve_glob_base(pattern_path: Path) -> Path:
280
318
  return base_dir.resolve()
281
319
 
282
320
 
321
+ def _safe_archive_name(path: str) -> str:
322
+ """Return a ZIP entry path without roots or dot-directory components."""
323
+ return "/".join(
324
+ _zip_path_part(part)
325
+ for part in path.replace("\\", "/").split("/")
326
+ if part not in ("", ".", "..")
327
+ )
328
+
329
+
330
+ def _zip_path_part(part: str) -> str:
331
+ if re.fullmatch(r"[A-Za-z]:", part):
332
+ return part[0]
333
+ return part
334
+
335
+
336
+ def _preserved_archive_name(source_path: Path) -> str:
337
+ try:
338
+ absolute_path = source_path.resolve()
339
+ except OSError:
340
+ absolute_path = Path(os.path.abspath(source_path))
341
+ return _safe_archive_name(str(absolute_path))
342
+
343
+
283
344
  def _build_zip_from_sources(
284
345
  output_zip: Path,
285
346
  sources: list[tuple[Path, Path]],
@@ -297,21 +358,9 @@ def _build_zip_from_sources(
297
358
  total_entries = 0
298
359
  entries_added = 0
299
360
 
300
- def _safe_archive_name(path: str) -> str:
301
- """Return a ZIP entry path without dot-directory components."""
302
- return "/".join(
303
- part
304
- for part in path.replace("\\", "/").split("/")
305
- if part not in ("", ".", "..")
306
- )
307
-
308
361
  def _archive_name_for_path(source_path: Path, base_dir: Path) -> str:
309
362
  if preserve_path:
310
- try:
311
- rel_path = os.path.relpath(source_path, Path.cwd())
312
- except ValueError:
313
- rel_path = source_path.name
314
- return _safe_archive_name(rel_path)
363
+ return _preserved_archive_name(source_path)
315
364
  try:
316
365
  rel_path = source_path.relative_to(base_dir)
317
366
  except ValueError:
@@ -1361,17 +1410,24 @@ def ingest_file(
1361
1410
  if preview:
1362
1411
  previews: list[dict[str, Any]] = []
1363
1412
  for file_path in unique_files:
1364
- data = await client.ingest.preview(
1365
- operation_id=operation_id,
1366
- plugin_name=plugin,
1367
- file_path=file_path,
1368
- params={
1369
- "context_name": context_name,
1370
- "plugin_params": params["plugin_params"],
1371
- "flt": params["flt"],
1372
- "original_file_path": str(Path(file_path).resolve()),
1373
- },
1413
+ upload_path, temp_path = await _maybe_bz2_compress_file_for_ingestion(
1414
+ file_path, params["plugin_params"]
1374
1415
  )
1416
+ try:
1417
+ data = await client.ingest.preview(
1418
+ operation_id=operation_id,
1419
+ plugin_name=plugin,
1420
+ file_path=upload_path,
1421
+ params={
1422
+ "context_name": context_name,
1423
+ "plugin_params": params["plugin_params"],
1424
+ "flt": params["flt"],
1425
+ "original_file_path": str(Path(file_path).resolve()),
1426
+ },
1427
+ )
1428
+ finally:
1429
+ if temp_path:
1430
+ Path(temp_path).unlink(missing_ok=True)
1375
1431
  previews.append({"file": file_path, "preview": data})
1376
1432
  print_result(previews)
1377
1433
  return
@@ -1397,18 +1453,25 @@ def ingest_file(
1397
1453
  """Submit ingestion for one file; returns (file_path, req_id, submission)."""
1398
1454
  req_id = str(uuid.uuid4())
1399
1455
  req_to_file[req_id] = file_path
1400
- result = await client.ingest.file(
1401
- operation_id=operation_id,
1402
- plugin_name=plugin,
1403
- file_path=file_path,
1404
- context_name=context_name,
1405
- params={
1406
- **params,
1407
- "original_file_path": str(Path(file_path).resolve()),
1408
- "req_id": req_id,
1409
- },
1410
- wait=False, # we handle waiting ourselves below
1456
+ upload_path, temp_path = await _maybe_bz2_compress_file_for_ingestion(
1457
+ file_path, params["plugin_params"]
1411
1458
  )
1459
+ try:
1460
+ result = await client.ingest.file(
1461
+ operation_id=operation_id,
1462
+ plugin_name=plugin,
1463
+ file_path=upload_path,
1464
+ context_name=context_name,
1465
+ params={
1466
+ **params,
1467
+ "original_file_path": str(Path(file_path).resolve()),
1468
+ "req_id": req_id,
1469
+ },
1470
+ wait=False, # we handle waiting ourselves below
1471
+ )
1472
+ finally:
1473
+ if temp_path:
1474
+ Path(temp_path).unlink(missing_ok=True)
1412
1475
  if hasattr(result, "model_dump"):
1413
1476
  submission = result.model_dump(exclude_none=True)
1414
1477
  else:
@@ -1694,19 +1757,27 @@ def ingest_file_to_source(
1694
1757
  async def _fire_one(file_path: str) -> tuple[str, str, dict[str, Any]]:
1695
1758
  req_id = str(uuid.uuid4())
1696
1759
  req_to_file[req_id] = file_path
1697
- result = await client.ingest.file_to_source(
1698
- source_id=source_id,
1699
- file_path=file_path,
1700
- plugin=plugin,
1701
- plugin_params=(
1702
- params["plugin_params"]
1703
- if plugin is not None
1704
- else params["plugin_params"] or None
1705
- ),
1706
- flt=params["flt"] or None,
1707
- req_id=req_id,
1708
- wait=False,
1760
+ upload_path, temp_path = await _maybe_bz2_compress_file_for_ingestion(
1761
+ file_path, params["plugin_params"]
1709
1762
  )
1763
+ try:
1764
+ result = await client.ingest.file_to_source(
1765
+ source_id=source_id,
1766
+ file_path=upload_path,
1767
+ plugin=plugin,
1768
+ plugin_params=(
1769
+ params["plugin_params"]
1770
+ if plugin is not None
1771
+ else params["plugin_params"] or None
1772
+ ),
1773
+ flt=params["flt"] or None,
1774
+ req_id=req_id,
1775
+ original_file_path=str(Path(file_path).resolve()),
1776
+ wait=False,
1777
+ )
1778
+ finally:
1779
+ if temp_path:
1780
+ Path(temp_path).unlink(missing_ok=True)
1710
1781
  if hasattr(result, "model_dump"):
1711
1782
  submission = result.model_dump(exclude_none=True)
1712
1783
  else:
@@ -1883,145 +1954,6 @@ def ingest_file_to_source(
1883
1954
  asyncio.run(_run())
1884
1955
 
1885
1956
 
1886
- @app.command("zip")
1887
- def ingest_zip(
1888
- operation_id: str,
1889
- zip_file: str = typer.Argument(
1890
- ...,
1891
- help="Path to a ZIP file which must contain a `metadata.json` in the root, describing the content as specified in gulp's `ingest_zip` docs.",
1892
- ),
1893
- context_name: str = typer.Option(
1894
- "sdk_context",
1895
- "--context",
1896
- help=(
1897
- "Context for ingestion, specify a name to create a new context if it "
1898
- "doesn't exist, or an existing `context_id`."
1899
- ),
1900
- ),
1901
- flt: str | None = typer.Option(
1902
- None, "--flt", help="JSON object for GulpIngestionFilter"
1903
- ),
1904
- reset_operation: bool = typer.Option(
1905
- False,
1906
- "--reset-operation",
1907
- help="Delete and recreate the operation before ingest starts",
1908
- ),
1909
- create_operation_if_missing: bool = typer.Option(
1910
- False,
1911
- "--create-operation",
1912
- help="Create operation automatically when it does not exist",
1913
- ),
1914
- wait: bool = typer.Option(False, "--wait", help="Wait for ingestion completion"),
1915
- wait_log: bool = typer.Option(
1916
- False,
1917
- "--wait-log",
1918
- help="Wait for ingestion completion and print websocket updates as they arrive.",
1919
- ),
1920
- wait_timeout: int = typer.Option(
1921
- 300,
1922
- "--timeout",
1923
- help="Seconds to wait for completion (only used with --wait/--wait-log)",
1924
- ),
1925
- ) -> None:
1926
- """Ingest a ZIP archive into an operation."""
1927
-
1928
- async def _run() -> None:
1929
- if reset_operation and create_operation_if_missing:
1930
- raise typer.BadParameter(
1931
- "--reset-operation and --create-operation are mutually exclusive"
1932
- )
1933
- if wait and wait_log:
1934
- raise typer.BadParameter("--wait and --wait-log are mutually exclusive")
1935
- wait_mode = wait or wait_log
1936
-
1937
- async with get_client() as client:
1938
- await client.ensure_websocket()
1939
-
1940
- async def _fetch_request_status(req_id: str) -> dict[str, Any] | None:
1941
- try:
1942
- resp = await client._request(
1943
- "GET", "/request_get_by_id", params={"obj_id": req_id}
1944
- )
1945
- except Exception:
1946
- return None
1947
- data = resp.get("data")
1948
- return data if isinstance(data, dict) else None
1949
-
1950
- if reset_operation:
1951
- await client.operations.delete(operation_id, force=True)
1952
- await client.operations.create(name=operation_id)
1953
- if not get_runtime_verbose():
1954
- print_warning(
1955
- f"Operation {operation_id} reset (deleted and recreated)."
1956
- )
1957
- else:
1958
- await _ensure_operation_exists(
1959
- client,
1960
- operation_id,
1961
- create_if_missing=create_operation_if_missing,
1962
- )
1963
-
1964
- params = {
1965
- "context_name": context_name,
1966
- "flt": parse_json_option(flt, field_name="flt") or {},
1967
- }
1968
- req_to_label: dict[str, str] = {}
1969
- tracker = await _IngestWsTracker.create(
1970
- client,
1971
- source_done_is_terminal=True,
1972
- log_updates=wait_log,
1973
- request_label_getter=lambda req_id: req_to_label.get(req_id, zip_file),
1974
- request_status_fetcher=_fetch_request_status,
1975
- )
1976
- result = await client.ingest.zip(
1977
- operation_id=operation_id,
1978
- plugin_name="zip",
1979
- zipfile_path=zip_file,
1980
- params=params,
1981
- wait=False,
1982
- )
1983
-
1984
- if hasattr(result, "model_dump"):
1985
- submission = result.model_dump(exclude_none=True)
1986
- else:
1987
- submission = {"req_id": getattr(result, "req_id", None)}
1988
-
1989
- req_id = str(submission.get("req_id") or "")
1990
- if not req_id:
1991
- # Fallback: if backend did not return req_id, surface submission as-is.
1992
- print_result(submission)
1993
- return
1994
-
1995
- req_to_label[req_id] = zip_file
1996
- file_req_map: dict[str, str] = {zip_file: req_id}
1997
-
1998
- if wait_mode:
1999
- waited = await tracker.wait_for_terminal(zip_file, req_id, wait_timeout)
2000
- print_result(waited)
2001
- else:
2002
- timed_out = await _wait_for_stats_create(client, [req_id])
2003
- if timed_out and not get_runtime_verbose():
2004
- print_warning(
2005
- f"Request did not receive STATS_CREATE within {_WS_CONFIRM_TIMEOUT_SEC:.0f}s - backend may still be processing: {req_id}"
2006
- )
2007
-
2008
- if get_runtime_verbose():
2009
- print_result(
2010
- {
2011
- "file": zip_file,
2012
- "req_id": req_id,
2013
- "submitted": submission,
2014
- "ws_confirmed": req_id not in timed_out,
2015
- }
2016
- )
2017
- else:
2018
- print_result(
2019
- {"file": zip_file, "req_id": req_id, "status": "pending"}
2020
- )
2021
-
2022
- asyncio.run(_run())
2023
-
2024
-
2025
1957
  @app.command("zip-create")
2026
1958
  def ingest_zip_create(
2027
1959
  output_zip: str = typer.Argument(
@@ -4,7 +4,6 @@ Plugin management commands for Gulp CLI.
4
4
  from __future__ import annotations
5
5
 
6
6
  import asyncio
7
- from pathlib import Path
8
7
  from typing import Any, Literal
9
8
 
10
9
  import typer
@@ -81,70 +80,3 @@ def plugin_list_ui(
81
80
  )
82
81
 
83
82
  asyncio.run(_run())
84
-
85
-
86
- @app.command("upload")
87
- def plugin_upload(
88
- file_path: str = typer.Argument(..., help="Path to the plugin .py file"),
89
- plugin_type: Literal["default", "extension", "ui"] = typer.Option(
90
- "default", "--type", help="Plugin type category"
91
- ),
92
- fail_if_exists: bool = typer.Option(
93
- False, "--fail-if-exists", help="Fail if plugin file already exists"
94
- ),
95
- ) -> None:
96
- """Upload a plugin file."""
97
- async def _run() -> None:
98
- # Check file exists
99
- p = Path(file_path)
100
- if not p.exists():
101
- typer.echo(f"Error: File not found: {file_path}", err=True)
102
- raise typer.Exit(1)
103
-
104
- async with get_client() as client:
105
- result = await client.plugins.upload(
106
- file_path,
107
- plugin_type=plugin_type,
108
- fail_if_exists=fail_if_exists,
109
- )
110
- print_result(result)
111
-
112
- asyncio.run(_run())
113
-
114
-
115
- @app.command("delete")
116
- def plugin_delete(
117
- filename: str = typer.Argument(..., help="Plugin filename to delete, e.g., 'my_plugin.py'"),
118
- plugin_type: Literal["default", "extension", "ui"] = typer.Option(
119
- "default", "--type", help="Plugin type category"
120
- ),
121
- ) -> None:
122
- """Delete a plugin file."""
123
- async def _run() -> None:
124
- async with get_client() as client:
125
- result = await client.plugins.delete(filename, plugin_type=plugin_type)
126
- print_result(result)
127
-
128
- asyncio.run(_run())
129
-
130
-
131
- @app.command("download")
132
- def plugin_download(
133
- filename: str = typer.Argument(..., help="Plugin filename to download, e.g., 'my_plugin.py'"),
134
- output_path: str = typer.Argument(..., help="Local path to save the plugin"),
135
- plugin_type: Literal["default", "extension", "ui"] = typer.Option(
136
- "default", "--type", help="Plugin type category"
137
- ),
138
- ) -> None:
139
- """Download a plugin file."""
140
- async def _run() -> None:
141
- async with get_client() as client:
142
- result = await client.plugins.download(
143
- filename,
144
- output_path,
145
- plugin_type=plugin_type,
146
- )
147
- data = {"file": result}
148
- print_result(data)
149
-
150
- asyncio.run(_run())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gulp-cli
3
- Version: 1.1.7
3
+ Version: 1.1.9
4
4
  Summary: Command-line client for gULP
5
5
  Author-email: Mentat <info@mentat.is>
6
6
  Requires-Python: >=3.12
@@ -22,7 +22,7 @@ Dynamic: license-file
22
22
  ## ✨ What can you do?
23
23
 
24
24
  - 🔐 **Authentication** — secure login with token persistence
25
- - 📥 **Ingestion** — ingest files (single/batch/wildcard), zip archives, with concurrent uploads
25
+ - 📥 **Ingestion** — ingest files (single/batch/wildcard), optionally compressed before upload, with concurrent uploads
26
26
  - 🔍 **Querying** — raw OpenSearch queries, Sigma rules, external plugins
27
27
  - 🏷️ **Enrichment** — enrich documents, tag/untag, update fields
28
28
  - 👥 **User Management** — create users, manage permissions (admin only)
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "1.1.9",
3
+ "distance": 0,
4
+ "node": "g487f18224426c2f3bfbaa588969bc1d5e4588997",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-06-24"
8
+ }
@@ -1,6 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import inspect
4
+ import bz2
5
+ import asyncio
4
6
  from pathlib import Path
5
7
 
6
8
  import pytest
@@ -8,6 +10,8 @@ import typer
8
10
 
9
11
  from gulp_cli.commands.ingest import (
10
12
  _expand_source_patterns,
13
+ _maybe_bz2_compress_file_for_ingestion,
14
+ _plugin_params_request_compression,
11
15
  ingest_file_to_source,
12
16
  ingest_file,
13
17
  )
@@ -76,3 +80,36 @@ def test_expand_source_patterns_rejects_binary_paths_file(tmp_path: Path) -> Non
76
80
 
77
81
  with pytest.raises(typer.BadParameter, match="must be UTF-8 text"):
78
82
  _expand_source_patterns([], str(paths_file))
83
+
84
+
85
+ def test_compressed_plugin_params_create_bz2_upload_file(tmp_path: Path) -> None:
86
+ source = tmp_path / "sample.evtx"
87
+ payload = b"event log bytes"
88
+ source.write_bytes(payload)
89
+
90
+ assert _plugin_params_request_compression({"compressed": True})
91
+ upload_path, temp_path = asyncio.run(
92
+ _maybe_bz2_compress_file_for_ingestion(
93
+ str(source), {"compressed": True}
94
+ )
95
+ )
96
+ try:
97
+ assert temp_path == upload_path
98
+ assert upload_path.endswith(".bz2")
99
+ with bz2.open(upload_path, "rb") as f:
100
+ assert f.read() == payload
101
+ finally:
102
+ Path(upload_path).unlink(missing_ok=True)
103
+
104
+
105
+ def test_uncompressed_plugin_params_keep_original_upload_file(tmp_path: Path) -> None:
106
+ source = tmp_path / "sample.evtx"
107
+ source.write_bytes(b"event log bytes")
108
+
109
+ assert not _plugin_params_request_compression({})
110
+ upload_path, temp_path = asyncio.run(
111
+ _maybe_bz2_compress_file_for_ingestion(str(source), {})
112
+ )
113
+
114
+ assert upload_path == str(source)
115
+ assert temp_path is None
@@ -10,7 +10,6 @@ from gulp_cli.commands.ingest import (
10
10
  ingest_file,
11
11
  ingest_file_to_source,
12
12
  ingest_raw,
13
- ingest_zip,
14
13
  )
15
14
  from gulp_sdk.websocket import WSMessage, WSMessageType
16
15
 
@@ -34,16 +33,6 @@ def test_ingest_file_to_source_exposes_plugin_option() -> None:
34
33
  assert plugin_default.param_decls == ("--plugin",)
35
34
 
36
35
 
37
- def test_ingest_zip_exposes_wait_log_option() -> None:
38
- wait_log_default = inspect.signature(ingest_zip).parameters["wait_log"].default
39
- assert wait_log_default.default is False
40
-
41
-
42
- def test_ingest_zip_exposes_context_option() -> None:
43
- context_default = inspect.signature(ingest_zip).parameters["context_name"].default
44
- assert context_default.param_decls == ("--context",)
45
-
46
-
47
36
  def test_ingest_raw_exposes_wait_log_option() -> None:
48
37
  wait_log_default = inspect.signature(ingest_raw).parameters["wait_log"].default
49
38
  assert wait_log_default.default is False
@@ -12,6 +12,7 @@ import typer
12
12
  from gulp_cli.commands.ingest import (
13
13
  _build_zip_from_sources,
14
14
  _expand_source_patterns,
15
+ _safe_archive_name,
15
16
  ingest_zip_create,
16
17
  )
17
18
  from gulp_cli.config import set_runtime_config_dir
@@ -87,40 +88,27 @@ def test_build_zip_from_sources_preserves_path_when_requested(
87
88
  )
88
89
 
89
90
  assert archived_count == 1
90
- assert archived_entries == ["samples/nested/data.json"]
91
+ assert archived_entries == [_safe_archive_name(str(source_file.resolve()))]
91
92
  assert len(created_archives) == 1
92
93
 
93
94
  with zipfile.ZipFile(created_archives[0]) as archive:
94
- assert archive.namelist() == ["samples/nested/data.json"]
95
+ assert archive.namelist() == archived_entries
95
96
 
96
97
 
97
98
  @pytest.mark.parametrize(
98
- "rel_path",
99
- ["./samples/nested/data.json", ".\\samples\\nested\\data.json"],
99
+ ("path", "expected"),
100
+ [
101
+ ("./samples/nested/data.json", "samples/nested/data.json"),
102
+ (".\\samples\\nested\\data.json", "samples/nested/data.json"),
103
+ ("../../../samples/nested/data.json", "samples/nested/data.json"),
104
+ ("C:\\samples\\nested\\data.json", "C/samples/nested/data.json"),
105
+ ("\\\\server\\share\\samples\\nested\\data.json", "server/share/samples/nested/data.json"),
106
+ ],
100
107
  )
101
- def test_build_zip_from_sources_strips_current_dir_paths_when_preserving(
102
- tmp_path: Path, monkeypatch, rel_path: str
108
+ def test_safe_archive_name_strips_roots_and_dot_paths(
109
+ path: str, expected: str
103
110
  ) -> None:
104
- source_file = tmp_path / "data.json"
105
- source_file.write_text('{"value": 1}', encoding="utf-8")
106
- monkeypatch.setattr(
107
- "gulp_cli.commands.ingest.os.path.relpath",
108
- lambda _source_path, _cwd: rel_path,
109
- )
110
-
111
- output_zip = tmp_path / "out.zip"
112
- archived_count, archived_entries, created_archives = _build_zip_from_sources(
113
- output_zip,
114
- [(source_file, tmp_path)],
115
- preserve_path=True,
116
- )
117
-
118
- expected = "samples/nested/data.json"
119
- assert archived_count == 1
120
- assert archived_entries == [expected]
121
-
122
- with zipfile.ZipFile(created_archives[0]) as archive:
123
- assert archive.namelist() == [expected]
111
+ assert _safe_archive_name(path) == expected
124
112
 
125
113
 
126
114
  def test_build_zip_from_sources_strips_parent_paths_when_preserving(
@@ -141,7 +129,7 @@ def test_build_zip_from_sources_strips_parent_paths_when_preserving(
141
129
  preserve_path=True,
142
130
  )
143
131
 
144
- expected = "samples/win_evtx/2-system-Microsoft-Windows-LiveId%4Operational.evtx"
132
+ expected = _safe_archive_name(str(source_file.resolve()))
145
133
  assert archived_count == 1
146
134
  assert archived_entries == [expected]
147
135
 
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "1.1.7",
3
- "distance": 0,
4
- "node": "g5f188bb2b8862aa3c0a6ca011bef91125feed6a5",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-06-23"
8
- }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes