gulp-cli 1.1.7__tar.gz → 1.1.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gulp_cli-1.1.7/src/gulp_cli.egg-info → gulp_cli-1.1.9}/PKG-INFO +2 -2
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/README.md +1 -1
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/command-reference.md +6 -60
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/examples.md +5 -10
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/getting-started.md +3 -3
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/resource-management.md +0 -52
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/_version.py +3 -3
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/ingest.py +117 -185
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/plugin.py +0 -68
- {gulp_cli-1.1.7 → gulp_cli-1.1.9/src/gulp_cli.egg-info}/PKG-INFO +2 -2
- gulp_cli-1.1.9/src/gulp_cli.egg-info/scm_version.json +8 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_ingest_paths_file.py +37 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_ingest_wait_log.py +0 -11
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_zip_create.py +15 -27
- gulp_cli-1.1.7/src/gulp_cli.egg-info/scm_version.json +0 -8
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/.github/workflows/portable-bundles.yml +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/.github/workflows/python-package.yml +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/.gitignore +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/GULP_DUAL_LICENSE.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/LICENSE.AGPL-3.0.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/LICENSE.md +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/extensions.md +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/portable.md +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/docs/troubleshooting-cli.md +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/gulp-cli.spec +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/portable/launch-linux.sh +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/portable/launch-macos.sh +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/portable/launch-windows.bat +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/pyproject.toml +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/setup.cfg +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/__init__.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/__main__.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/cli.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/client.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/__init__.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/acl.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/auth.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/collab.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/context.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/db.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/enhance_map.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/enrich.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/glyph.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/mapping.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/operations.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/query.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/source.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/stats.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/storage.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/user_group.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/users.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/commands/utility.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/config.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension/__init__.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension/query_sigma_zip.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension/story.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extension_helpers.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/extensions.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/output.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/utils.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli/version.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/SOURCES.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/dependency_links.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/entry_points.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/requires.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/scm_file_list.json +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/src/gulp_cli.egg-info/top_level.txt +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_enrich_live.py +0 -0
- {gulp_cli-1.1.7 → gulp_cli-1.1.9}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gulp-cli
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.9
|
|
4
4
|
Summary: Command-line client for gULP
|
|
5
5
|
Author-email: Mentat <info@mentat.is>
|
|
6
6
|
Requires-Python: >=3.12
|
|
@@ -22,7 +22,7 @@ Dynamic: license-file
|
|
|
22
22
|
## ✨ What can you do?
|
|
23
23
|
|
|
24
24
|
- 🔐 **Authentication** — secure login with token persistence
|
|
25
|
-
- 📥 **Ingestion** — ingest files (single/batch/wildcard),
|
|
25
|
+
- 📥 **Ingestion** — ingest files (single/batch/wildcard), optionally compressed before upload, with concurrent uploads
|
|
26
26
|
- 🔍 **Querying** — raw OpenSearch queries, Sigma rules, external plugins
|
|
27
27
|
- 🏷️ **Enrichment** — enrich documents, tag/untag, update fields
|
|
28
28
|
- 👥 **User Management** — create users, manage permissions (admin only)
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
## ✨ What can you do?
|
|
6
6
|
|
|
7
7
|
- 🔐 **Authentication** — secure login with token persistence
|
|
8
|
-
- 📥 **Ingestion** — ingest files (single/batch/wildcard),
|
|
8
|
+
- 📥 **Ingestion** — ingest files (single/batch/wildcard), optionally compressed before upload, with concurrent uploads
|
|
9
9
|
- 🔍 **Querying** — raw OpenSearch queries, Sigma rules, external plugins
|
|
10
10
|
- 🏷️ **Enrichment** — enrich documents, tag/untag, update fields
|
|
11
11
|
- 👥 **User Management** — create users, manage permissions (admin only)
|
|
@@ -461,7 +461,7 @@ gulp-cli ingest file OPERATION_ID PLUGIN [FILE...] [OPTIONS]
|
|
|
461
461
|
**Options:**
|
|
462
462
|
|
|
463
463
|
- `--context TEXT` — Context for ingestion, specify a name to create a new context if it doesn't exist, or an existing `context_id` (default: `sdk_context`)
|
|
464
|
-
- `--plugin-params TEXT` — JSON string with plugin parameters
|
|
464
|
+
- `--plugin-params TEXT` — JSON string with plugin parameters. When it includes `"compressed": true`, gulp-cli bzip2-compresses each input file before upload and the backend decompresses it before plugin ingestion.
|
|
465
465
|
- `--flt TEXT` — JSON object for `GulpIngestionFilter`
|
|
466
466
|
- `--paths-file PATH` — Text file with one file or glob pattern per line. When set, positional `FILE` arguments are optional and ignored if also provided.
|
|
467
467
|
- `--reset-operation` — Delete and recreate the operation before ingest starts
|
|
@@ -493,6 +493,9 @@ gulp-cli ingest file my_op win_evtx --paths-file /tmp/file_paths.txt
|
|
|
493
493
|
# With plugin parameters
|
|
494
494
|
gulp-cli ingest file my_op csv file.csv --plugin-params '{"delimiter":";","encoding":"utf-8"}'
|
|
495
495
|
|
|
496
|
+
# Compress before upload and let the backend decompress before ingestion
|
|
497
|
+
gulp-cli ingest file my_op win_evtx /path/to/System.evtx --plugin-params '{"compressed": true}'
|
|
498
|
+
|
|
496
499
|
# Preview mode (no ingestion persisted)
|
|
497
500
|
gulp-cli ingest file my_op win_evtx /path/to/System.evtx --preview
|
|
498
501
|
|
|
@@ -521,7 +524,7 @@ gulp-cli ingest file-to-source SOURCE_ID [FILE...] [OPTIONS]
|
|
|
521
524
|
**Options:**
|
|
522
525
|
|
|
523
526
|
- `--plugin TEXT` — Override the plugin associated with the source (requires `--plugin-params`)
|
|
524
|
-
- `--plugin-params TEXT` — JSON object for plugin_params (overrides source defaults; required when `--plugin` is set, `{}` is allowed)
|
|
527
|
+
- `--plugin-params TEXT` — JSON object for plugin_params (overrides source defaults; required when `--plugin` is set, `{}` is allowed). When it includes `"compressed": true`, gulp-cli bzip2-compresses each input file before upload and the backend decompresses it before plugin ingestion.
|
|
525
528
|
- `--flt TEXT` — JSON object for GulpIngestionFilter
|
|
526
529
|
- `--paths-file PATH` — Text file with one file or glob pattern per line. When set, positional `FILE` arguments are optional and ignored if also provided.
|
|
527
530
|
- `--wait` — Wait for completion with progress
|
|
@@ -536,38 +539,7 @@ gulp-cli ingest file-to-source source123 /new/System.evtx --wait
|
|
|
536
539
|
gulp-cli ingest file-to-source source123 '/new/*.evtx'
|
|
537
540
|
gulp-cli ingest file-to-source source123 --paths-file /tmp/file_paths.txt
|
|
538
541
|
gulp-cli ingest file-to-source source123 /new/events.json --plugin json --plugin-params '{"mapping_parameters": {}}'
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
---
|
|
542
|
-
|
|
543
|
-
#### `zip`
|
|
544
|
-
|
|
545
|
-
Ingest from a ZIP archive.
|
|
546
|
-
|
|
547
|
-
> TO BE DEPRECATED...
|
|
548
|
-
|
|
549
|
-
```bash
|
|
550
|
-
gulp-cli ingest zip OPERATION_ID ZIP_FILE [OPTIONS]
|
|
551
|
-
```
|
|
552
|
-
|
|
553
|
-
**Arguments:**
|
|
554
|
-
|
|
555
|
-
- `OPERATION_ID` — Target operation
|
|
556
|
-
- `ZIP_FILE` — Path to ZIP file containing a `metadata.json` in the root, describing the content as specified in gulp's `ingest_zip` docs.
|
|
557
|
-
|
|
558
|
-
**Options:**
|
|
559
|
-
|
|
560
|
-
- `--context TEXT` — Context for ingestion, specify a name to create a new context if it doesn't exist, or an existing `context_id` (default: `sdk_context`)
|
|
561
|
-
- `--flt TEXT` — JSON object for `GulpIngestionFilter`
|
|
562
|
-
- `--create-operation` — Create operation automatically when it does not exist
|
|
563
|
-
- `--wait` — Wait for completion
|
|
564
|
-
- `--timeout INTEGER` — Timeout in seconds for `--wait` mode
|
|
565
|
-
|
|
566
|
-
**Examples:**
|
|
567
|
-
|
|
568
|
-
```bash
|
|
569
|
-
gulp-cli ingest zip my_op /data/evidence.zip --wait
|
|
570
|
-
gulp-cli ingest zip my_op /data/evidence.zip --create-operation
|
|
542
|
+
gulp-cli ingest file-to-source source123 /new/System.evtx --plugin win_evtx --plugin-params '{"compressed": true}'
|
|
571
543
|
```
|
|
572
544
|
|
|
573
545
|
---
|
|
@@ -2330,32 +2302,6 @@ gulp-cli plugin list-ui [OPTIONS]
|
|
|
2330
2302
|
|
|
2331
2303
|
---
|
|
2332
2304
|
|
|
2333
|
-
#### `plugin upload`
|
|
2334
|
-
|
|
2335
|
-
Upload a new plugin.
|
|
2336
|
-
|
|
2337
|
-
```bash
|
|
2338
|
-
gulp-cli plugin upload PLUGIN_FILE [OPTIONS]
|
|
2339
|
-
```
|
|
2340
|
-
|
|
2341
|
-
#### `plugin download`
|
|
2342
|
-
|
|
2343
|
-
Download a plugin.
|
|
2344
|
-
|
|
2345
|
-
```bash
|
|
2346
|
-
gulp-cli plugin download FILENAME OUTPUT_PATH [OPTIONS]
|
|
2347
|
-
```
|
|
2348
|
-
|
|
2349
|
-
#### `plugin delete`
|
|
2350
|
-
|
|
2351
|
-
Delete a plugin.
|
|
2352
|
-
|
|
2353
|
-
```bash
|
|
2354
|
-
gulp-cli plugin delete FILENAME [OPTIONS]
|
|
2355
|
-
```
|
|
2356
|
-
|
|
2357
|
-
---
|
|
2358
|
-
|
|
2359
2305
|
#### `mapping list`
|
|
2360
2306
|
|
|
2361
2307
|
List all mapping files.
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
- [CSV Data Ingestion with Custom Parameters](#csv-data-ingestion-with-custom-parameters)
|
|
9
9
|
- [JSON Logs Ingestion](#json-logs-ingestion)
|
|
10
10
|
- [Add More Evidence to Existing Source](#add-more-evidence-to-existing-source)
|
|
11
|
-
- [
|
|
11
|
+
- [Compressed File Ingestion](#compressed-file-ingestion)
|
|
12
12
|
- [Create ZIP Archive from Paths and Masks](#create-zip-archive-from-paths-and-masks)
|
|
13
13
|
- [Raw Payload Ingestion](#raw-payload-ingestion)
|
|
14
14
|
- [Request Stats Monitoring Workflows](#request-stats-monitoring-workflows)
|
|
@@ -205,14 +205,12 @@ gulp-cli ingest file-to-source data-2026-001 /additional/evidence.evtx --wait
|
|
|
205
205
|
gulp-cli ingest file-to-source data-2026-001 '/additional/*.evtx' --wait
|
|
206
206
|
```
|
|
207
207
|
|
|
208
|
-
###
|
|
208
|
+
### Compressed File Ingestion
|
|
209
209
|
|
|
210
210
|
```bash
|
|
211
|
-
#
|
|
212
|
-
gulp-cli ingest
|
|
213
|
-
|
|
214
|
-
# Auto-create operation when missing
|
|
215
|
-
gulp-cli ingest zip incident-001 /evidence/evidence.zip --create-operation
|
|
211
|
+
# Compress files with bzip2 before upload; the backend decompresses before ingestion
|
|
212
|
+
gulp-cli ingest file incident-001 win_evtx /evidence/System.evtx \
|
|
213
|
+
--plugin-params '{"compressed": true}' --wait
|
|
216
214
|
```
|
|
217
215
|
|
|
218
216
|
### Create ZIP Archive from Paths and Masks
|
|
@@ -233,9 +231,6 @@ $CASE_ROOT/linux/**/*.log
|
|
|
233
231
|
EOF
|
|
234
232
|
|
|
235
233
|
gulp-cli ingest zip-create '$CASE_ROOT/evidence.zip' --paths-file /tmp/evidence_paths.txt --overwrite
|
|
236
|
-
|
|
237
|
-
# Then ingest the generated ZIP
|
|
238
|
-
gulp-cli ingest zip incident-001 '$CASE_ROOT/evidence.zip' --wait
|
|
239
234
|
```
|
|
240
235
|
|
|
241
236
|
### Raw Payload Ingestion
|
|
@@ -207,11 +207,11 @@ This shows a real-time progress bar while documents are being ingested. Per-file
|
|
|
207
207
|
gulp-cli ingest file-to-source existing-source-id /gulp/samples/win_evtx/Security.evtx --wait
|
|
208
208
|
```
|
|
209
209
|
|
|
210
|
-
###
|
|
210
|
+
### Compressed / Raw Data
|
|
211
211
|
|
|
212
212
|
```bash
|
|
213
|
-
#
|
|
214
|
-
gulp-cli ingest
|
|
213
|
+
# Compress locally with bzip2 before upload; the backend decompresses before ingestion
|
|
214
|
+
gulp-cli ingest file my_investigation win_evtx /path/to/System.evtx --plugin-params '{"compressed": true}' --wait
|
|
215
215
|
|
|
216
216
|
# Raw payload ingestion from a file
|
|
217
217
|
gulp-cli ingest raw my_investigation --data-file /tmp/raw_chunk.json --last --wait
|
|
@@ -196,58 +196,6 @@ List available UI plugins:
|
|
|
196
196
|
gulp plugin list-ui
|
|
197
197
|
```
|
|
198
198
|
|
|
199
|
-
### Upload Plugin
|
|
200
|
-
|
|
201
|
-
Upload a custom plugin:
|
|
202
|
-
|
|
203
|
-
```bash
|
|
204
|
-
# Upload as default plugin
|
|
205
|
-
gulp plugin upload /path/to/my_plugin.py
|
|
206
|
-
|
|
207
|
-
# Upload as extension plugin
|
|
208
|
-
gulp plugin upload /path/to/my_extension.py --type extension
|
|
209
|
-
|
|
210
|
-
# Upload as UI plugin
|
|
211
|
-
gulp plugin upload /path/to/my_ui_plugin.py --type ui
|
|
212
|
-
|
|
213
|
-
# Fail if already exists
|
|
214
|
-
gulp plugin upload /path/to/my_plugin.py --fail-if-exists
|
|
215
|
-
```
|
|
216
|
-
|
|
217
|
-
Response includes the file path where the plugin was saved on the server.
|
|
218
|
-
|
|
219
|
-
### Download Plugin
|
|
220
|
-
|
|
221
|
-
Download a plugin from the server:
|
|
222
|
-
|
|
223
|
-
```bash
|
|
224
|
-
# Download default plugin
|
|
225
|
-
gulp plugin download win_evtx.py /local/path/win_evtx.py
|
|
226
|
-
|
|
227
|
-
# Download extension plugin
|
|
228
|
-
gulp plugin download my_extension.py /path/to/save --type extension
|
|
229
|
-
|
|
230
|
-
# Download UI plugin
|
|
231
|
-
gulp plugin download my_ui.py /path/to/save --type ui
|
|
232
|
-
```
|
|
233
|
-
|
|
234
|
-
### Delete Plugin
|
|
235
|
-
|
|
236
|
-
Delete a plugin from the server:
|
|
237
|
-
|
|
238
|
-
```bash
|
|
239
|
-
# Delete default plugin
|
|
240
|
-
gulp plugin delete my_plugin.py
|
|
241
|
-
|
|
242
|
-
# Delete extension plugin
|
|
243
|
-
gulp plugin delete my_extension.py --type extension
|
|
244
|
-
|
|
245
|
-
# Delete UI plugin
|
|
246
|
-
gulp plugin delete my_ui.py --type ui
|
|
247
|
-
```
|
|
248
|
-
|
|
249
|
-
---
|
|
250
|
-
|
|
251
199
|
## Mapping File Management
|
|
252
200
|
|
|
253
201
|
Mapping files define data transformations and field mappings for ingestion plugins.
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.1.
|
|
22
|
-
__version_tuple__ = version_tuple = (1, 1,
|
|
21
|
+
__version__ = version = '1.1.9'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 1, 9)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g487f18224'
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import bz2
|
|
4
5
|
import json
|
|
5
6
|
import os
|
|
6
7
|
import re
|
|
@@ -45,6 +46,43 @@ _TERMINAL_STATUSES = {"done", "failed", "canceled"}
|
|
|
45
46
|
_MAX_ZIP_PART_SIZE_BYTES = 4 * 1024**3 - 1
|
|
46
47
|
|
|
47
48
|
|
|
49
|
+
def _plugin_params_request_compression(plugin_params: dict[str, Any] | None) -> bool:
|
|
50
|
+
"""Return True when plugin parameters request core decompression."""
|
|
51
|
+
return bool(plugin_params and plugin_params.get("compressed"))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _bz2_compress_file_for_ingestion_sync(file_path: str) -> str:
|
|
55
|
+
"""Compress a file into a temporary bzip2 file for upload."""
|
|
56
|
+
source_path = Path(file_path)
|
|
57
|
+
fd, temp_path = tempfile.mkstemp(
|
|
58
|
+
prefix=f"{source_path.name}.",
|
|
59
|
+
suffix=".bz2",
|
|
60
|
+
)
|
|
61
|
+
os.close(fd)
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
with open(source_path, "rb") as src, bz2.open(temp_path, "wb") as dst:
|
|
65
|
+
shutil.copyfileobj(src, dst)
|
|
66
|
+
except Exception:
|
|
67
|
+
try:
|
|
68
|
+
Path(temp_path).unlink()
|
|
69
|
+
except FileNotFoundError:
|
|
70
|
+
pass
|
|
71
|
+
raise
|
|
72
|
+
|
|
73
|
+
return temp_path
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
async def _maybe_bz2_compress_file_for_ingestion(
|
|
77
|
+
file_path: str, plugin_params: dict[str, Any] | None
|
|
78
|
+
) -> tuple[str, str | None]:
|
|
79
|
+
"""Return an upload path and optional temporary path to delete after upload."""
|
|
80
|
+
if not _plugin_params_request_compression(plugin_params):
|
|
81
|
+
return file_path, None
|
|
82
|
+
temp_path = await asyncio.to_thread(_bz2_compress_file_for_ingestion_sync, file_path)
|
|
83
|
+
return temp_path, temp_path
|
|
84
|
+
|
|
85
|
+
|
|
48
86
|
def _default_ingest_batch_size() -> int:
|
|
49
87
|
return max(1, (os.cpu_count() or 1) * 2)
|
|
50
88
|
|
|
@@ -280,6 +318,29 @@ def _resolve_glob_base(pattern_path: Path) -> Path:
|
|
|
280
318
|
return base_dir.resolve()
|
|
281
319
|
|
|
282
320
|
|
|
321
|
+
def _safe_archive_name(path: str) -> str:
|
|
322
|
+
"""Return a ZIP entry path without roots or dot-directory components."""
|
|
323
|
+
return "/".join(
|
|
324
|
+
_zip_path_part(part)
|
|
325
|
+
for part in path.replace("\\", "/").split("/")
|
|
326
|
+
if part not in ("", ".", "..")
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _zip_path_part(part: str) -> str:
|
|
331
|
+
if re.fullmatch(r"[A-Za-z]:", part):
|
|
332
|
+
return part[0]
|
|
333
|
+
return part
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _preserved_archive_name(source_path: Path) -> str:
|
|
337
|
+
try:
|
|
338
|
+
absolute_path = source_path.resolve()
|
|
339
|
+
except OSError:
|
|
340
|
+
absolute_path = Path(os.path.abspath(source_path))
|
|
341
|
+
return _safe_archive_name(str(absolute_path))
|
|
342
|
+
|
|
343
|
+
|
|
283
344
|
def _build_zip_from_sources(
|
|
284
345
|
output_zip: Path,
|
|
285
346
|
sources: list[tuple[Path, Path]],
|
|
@@ -297,21 +358,9 @@ def _build_zip_from_sources(
|
|
|
297
358
|
total_entries = 0
|
|
298
359
|
entries_added = 0
|
|
299
360
|
|
|
300
|
-
def _safe_archive_name(path: str) -> str:
|
|
301
|
-
"""Return a ZIP entry path without dot-directory components."""
|
|
302
|
-
return "/".join(
|
|
303
|
-
part
|
|
304
|
-
for part in path.replace("\\", "/").split("/")
|
|
305
|
-
if part not in ("", ".", "..")
|
|
306
|
-
)
|
|
307
|
-
|
|
308
361
|
def _archive_name_for_path(source_path: Path, base_dir: Path) -> str:
|
|
309
362
|
if preserve_path:
|
|
310
|
-
|
|
311
|
-
rel_path = os.path.relpath(source_path, Path.cwd())
|
|
312
|
-
except ValueError:
|
|
313
|
-
rel_path = source_path.name
|
|
314
|
-
return _safe_archive_name(rel_path)
|
|
363
|
+
return _preserved_archive_name(source_path)
|
|
315
364
|
try:
|
|
316
365
|
rel_path = source_path.relative_to(base_dir)
|
|
317
366
|
except ValueError:
|
|
@@ -1361,17 +1410,24 @@ def ingest_file(
|
|
|
1361
1410
|
if preview:
|
|
1362
1411
|
previews: list[dict[str, Any]] = []
|
|
1363
1412
|
for file_path in unique_files:
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
plugin_name=plugin,
|
|
1367
|
-
file_path=file_path,
|
|
1368
|
-
params={
|
|
1369
|
-
"context_name": context_name,
|
|
1370
|
-
"plugin_params": params["plugin_params"],
|
|
1371
|
-
"flt": params["flt"],
|
|
1372
|
-
"original_file_path": str(Path(file_path).resolve()),
|
|
1373
|
-
},
|
|
1413
|
+
upload_path, temp_path = await _maybe_bz2_compress_file_for_ingestion(
|
|
1414
|
+
file_path, params["plugin_params"]
|
|
1374
1415
|
)
|
|
1416
|
+
try:
|
|
1417
|
+
data = await client.ingest.preview(
|
|
1418
|
+
operation_id=operation_id,
|
|
1419
|
+
plugin_name=plugin,
|
|
1420
|
+
file_path=upload_path,
|
|
1421
|
+
params={
|
|
1422
|
+
"context_name": context_name,
|
|
1423
|
+
"plugin_params": params["plugin_params"],
|
|
1424
|
+
"flt": params["flt"],
|
|
1425
|
+
"original_file_path": str(Path(file_path).resolve()),
|
|
1426
|
+
},
|
|
1427
|
+
)
|
|
1428
|
+
finally:
|
|
1429
|
+
if temp_path:
|
|
1430
|
+
Path(temp_path).unlink(missing_ok=True)
|
|
1375
1431
|
previews.append({"file": file_path, "preview": data})
|
|
1376
1432
|
print_result(previews)
|
|
1377
1433
|
return
|
|
@@ -1397,18 +1453,25 @@ def ingest_file(
|
|
|
1397
1453
|
"""Submit ingestion for one file; returns (file_path, req_id, submission)."""
|
|
1398
1454
|
req_id = str(uuid.uuid4())
|
|
1399
1455
|
req_to_file[req_id] = file_path
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
plugin_name=plugin,
|
|
1403
|
-
file_path=file_path,
|
|
1404
|
-
context_name=context_name,
|
|
1405
|
-
params={
|
|
1406
|
-
**params,
|
|
1407
|
-
"original_file_path": str(Path(file_path).resolve()),
|
|
1408
|
-
"req_id": req_id,
|
|
1409
|
-
},
|
|
1410
|
-
wait=False, # we handle waiting ourselves below
|
|
1456
|
+
upload_path, temp_path = await _maybe_bz2_compress_file_for_ingestion(
|
|
1457
|
+
file_path, params["plugin_params"]
|
|
1411
1458
|
)
|
|
1459
|
+
try:
|
|
1460
|
+
result = await client.ingest.file(
|
|
1461
|
+
operation_id=operation_id,
|
|
1462
|
+
plugin_name=plugin,
|
|
1463
|
+
file_path=upload_path,
|
|
1464
|
+
context_name=context_name,
|
|
1465
|
+
params={
|
|
1466
|
+
**params,
|
|
1467
|
+
"original_file_path": str(Path(file_path).resolve()),
|
|
1468
|
+
"req_id": req_id,
|
|
1469
|
+
},
|
|
1470
|
+
wait=False, # we handle waiting ourselves below
|
|
1471
|
+
)
|
|
1472
|
+
finally:
|
|
1473
|
+
if temp_path:
|
|
1474
|
+
Path(temp_path).unlink(missing_ok=True)
|
|
1412
1475
|
if hasattr(result, "model_dump"):
|
|
1413
1476
|
submission = result.model_dump(exclude_none=True)
|
|
1414
1477
|
else:
|
|
@@ -1694,19 +1757,27 @@ def ingest_file_to_source(
|
|
|
1694
1757
|
async def _fire_one(file_path: str) -> tuple[str, str, dict[str, Any]]:
|
|
1695
1758
|
req_id = str(uuid.uuid4())
|
|
1696
1759
|
req_to_file[req_id] = file_path
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
file_path=file_path,
|
|
1700
|
-
plugin=plugin,
|
|
1701
|
-
plugin_params=(
|
|
1702
|
-
params["plugin_params"]
|
|
1703
|
-
if plugin is not None
|
|
1704
|
-
else params["plugin_params"] or None
|
|
1705
|
-
),
|
|
1706
|
-
flt=params["flt"] or None,
|
|
1707
|
-
req_id=req_id,
|
|
1708
|
-
wait=False,
|
|
1760
|
+
upload_path, temp_path = await _maybe_bz2_compress_file_for_ingestion(
|
|
1761
|
+
file_path, params["plugin_params"]
|
|
1709
1762
|
)
|
|
1763
|
+
try:
|
|
1764
|
+
result = await client.ingest.file_to_source(
|
|
1765
|
+
source_id=source_id,
|
|
1766
|
+
file_path=upload_path,
|
|
1767
|
+
plugin=plugin,
|
|
1768
|
+
plugin_params=(
|
|
1769
|
+
params["plugin_params"]
|
|
1770
|
+
if plugin is not None
|
|
1771
|
+
else params["plugin_params"] or None
|
|
1772
|
+
),
|
|
1773
|
+
flt=params["flt"] or None,
|
|
1774
|
+
req_id=req_id,
|
|
1775
|
+
original_file_path=str(Path(file_path).resolve()),
|
|
1776
|
+
wait=False,
|
|
1777
|
+
)
|
|
1778
|
+
finally:
|
|
1779
|
+
if temp_path:
|
|
1780
|
+
Path(temp_path).unlink(missing_ok=True)
|
|
1710
1781
|
if hasattr(result, "model_dump"):
|
|
1711
1782
|
submission = result.model_dump(exclude_none=True)
|
|
1712
1783
|
else:
|
|
@@ -1883,145 +1954,6 @@ def ingest_file_to_source(
|
|
|
1883
1954
|
asyncio.run(_run())
|
|
1884
1955
|
|
|
1885
1956
|
|
|
1886
|
-
@app.command("zip")
|
|
1887
|
-
def ingest_zip(
|
|
1888
|
-
operation_id: str,
|
|
1889
|
-
zip_file: str = typer.Argument(
|
|
1890
|
-
...,
|
|
1891
|
-
help="Path to a ZIP file which must contain a `metadata.json` in the root, describing the content as specified in gulp's `ingest_zip` docs.",
|
|
1892
|
-
),
|
|
1893
|
-
context_name: str = typer.Option(
|
|
1894
|
-
"sdk_context",
|
|
1895
|
-
"--context",
|
|
1896
|
-
help=(
|
|
1897
|
-
"Context for ingestion, specify a name to create a new context if it "
|
|
1898
|
-
"doesn't exist, or an existing `context_id`."
|
|
1899
|
-
),
|
|
1900
|
-
),
|
|
1901
|
-
flt: str | None = typer.Option(
|
|
1902
|
-
None, "--flt", help="JSON object for GulpIngestionFilter"
|
|
1903
|
-
),
|
|
1904
|
-
reset_operation: bool = typer.Option(
|
|
1905
|
-
False,
|
|
1906
|
-
"--reset-operation",
|
|
1907
|
-
help="Delete and recreate the operation before ingest starts",
|
|
1908
|
-
),
|
|
1909
|
-
create_operation_if_missing: bool = typer.Option(
|
|
1910
|
-
False,
|
|
1911
|
-
"--create-operation",
|
|
1912
|
-
help="Create operation automatically when it does not exist",
|
|
1913
|
-
),
|
|
1914
|
-
wait: bool = typer.Option(False, "--wait", help="Wait for ingestion completion"),
|
|
1915
|
-
wait_log: bool = typer.Option(
|
|
1916
|
-
False,
|
|
1917
|
-
"--wait-log",
|
|
1918
|
-
help="Wait for ingestion completion and print websocket updates as they arrive.",
|
|
1919
|
-
),
|
|
1920
|
-
wait_timeout: int = typer.Option(
|
|
1921
|
-
300,
|
|
1922
|
-
"--timeout",
|
|
1923
|
-
help="Seconds to wait for completion (only used with --wait/--wait-log)",
|
|
1924
|
-
),
|
|
1925
|
-
) -> None:
|
|
1926
|
-
"""Ingest a ZIP archive into an operation."""
|
|
1927
|
-
|
|
1928
|
-
async def _run() -> None:
|
|
1929
|
-
if reset_operation and create_operation_if_missing:
|
|
1930
|
-
raise typer.BadParameter(
|
|
1931
|
-
"--reset-operation and --create-operation are mutually exclusive"
|
|
1932
|
-
)
|
|
1933
|
-
if wait and wait_log:
|
|
1934
|
-
raise typer.BadParameter("--wait and --wait-log are mutually exclusive")
|
|
1935
|
-
wait_mode = wait or wait_log
|
|
1936
|
-
|
|
1937
|
-
async with get_client() as client:
|
|
1938
|
-
await client.ensure_websocket()
|
|
1939
|
-
|
|
1940
|
-
async def _fetch_request_status(req_id: str) -> dict[str, Any] | None:
|
|
1941
|
-
try:
|
|
1942
|
-
resp = await client._request(
|
|
1943
|
-
"GET", "/request_get_by_id", params={"obj_id": req_id}
|
|
1944
|
-
)
|
|
1945
|
-
except Exception:
|
|
1946
|
-
return None
|
|
1947
|
-
data = resp.get("data")
|
|
1948
|
-
return data if isinstance(data, dict) else None
|
|
1949
|
-
|
|
1950
|
-
if reset_operation:
|
|
1951
|
-
await client.operations.delete(operation_id, force=True)
|
|
1952
|
-
await client.operations.create(name=operation_id)
|
|
1953
|
-
if not get_runtime_verbose():
|
|
1954
|
-
print_warning(
|
|
1955
|
-
f"Operation {operation_id} reset (deleted and recreated)."
|
|
1956
|
-
)
|
|
1957
|
-
else:
|
|
1958
|
-
await _ensure_operation_exists(
|
|
1959
|
-
client,
|
|
1960
|
-
operation_id,
|
|
1961
|
-
create_if_missing=create_operation_if_missing,
|
|
1962
|
-
)
|
|
1963
|
-
|
|
1964
|
-
params = {
|
|
1965
|
-
"context_name": context_name,
|
|
1966
|
-
"flt": parse_json_option(flt, field_name="flt") or {},
|
|
1967
|
-
}
|
|
1968
|
-
req_to_label: dict[str, str] = {}
|
|
1969
|
-
tracker = await _IngestWsTracker.create(
|
|
1970
|
-
client,
|
|
1971
|
-
source_done_is_terminal=True,
|
|
1972
|
-
log_updates=wait_log,
|
|
1973
|
-
request_label_getter=lambda req_id: req_to_label.get(req_id, zip_file),
|
|
1974
|
-
request_status_fetcher=_fetch_request_status,
|
|
1975
|
-
)
|
|
1976
|
-
result = await client.ingest.zip(
|
|
1977
|
-
operation_id=operation_id,
|
|
1978
|
-
plugin_name="zip",
|
|
1979
|
-
zipfile_path=zip_file,
|
|
1980
|
-
params=params,
|
|
1981
|
-
wait=False,
|
|
1982
|
-
)
|
|
1983
|
-
|
|
1984
|
-
if hasattr(result, "model_dump"):
|
|
1985
|
-
submission = result.model_dump(exclude_none=True)
|
|
1986
|
-
else:
|
|
1987
|
-
submission = {"req_id": getattr(result, "req_id", None)}
|
|
1988
|
-
|
|
1989
|
-
req_id = str(submission.get("req_id") or "")
|
|
1990
|
-
if not req_id:
|
|
1991
|
-
# Fallback: if backend did not return req_id, surface submission as-is.
|
|
1992
|
-
print_result(submission)
|
|
1993
|
-
return
|
|
1994
|
-
|
|
1995
|
-
req_to_label[req_id] = zip_file
|
|
1996
|
-
file_req_map: dict[str, str] = {zip_file: req_id}
|
|
1997
|
-
|
|
1998
|
-
if wait_mode:
|
|
1999
|
-
waited = await tracker.wait_for_terminal(zip_file, req_id, wait_timeout)
|
|
2000
|
-
print_result(waited)
|
|
2001
|
-
else:
|
|
2002
|
-
timed_out = await _wait_for_stats_create(client, [req_id])
|
|
2003
|
-
if timed_out and not get_runtime_verbose():
|
|
2004
|
-
print_warning(
|
|
2005
|
-
f"Request did not receive STATS_CREATE within {_WS_CONFIRM_TIMEOUT_SEC:.0f}s - backend may still be processing: {req_id}"
|
|
2006
|
-
)
|
|
2007
|
-
|
|
2008
|
-
if get_runtime_verbose():
|
|
2009
|
-
print_result(
|
|
2010
|
-
{
|
|
2011
|
-
"file": zip_file,
|
|
2012
|
-
"req_id": req_id,
|
|
2013
|
-
"submitted": submission,
|
|
2014
|
-
"ws_confirmed": req_id not in timed_out,
|
|
2015
|
-
}
|
|
2016
|
-
)
|
|
2017
|
-
else:
|
|
2018
|
-
print_result(
|
|
2019
|
-
{"file": zip_file, "req_id": req_id, "status": "pending"}
|
|
2020
|
-
)
|
|
2021
|
-
|
|
2022
|
-
asyncio.run(_run())
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
1957
|
@app.command("zip-create")
|
|
2026
1958
|
def ingest_zip_create(
|
|
2027
1959
|
output_zip: str = typer.Argument(
|
|
@@ -4,7 +4,6 @@ Plugin management commands for Gulp CLI.
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import asyncio
|
|
7
|
-
from pathlib import Path
|
|
8
7
|
from typing import Any, Literal
|
|
9
8
|
|
|
10
9
|
import typer
|
|
@@ -81,70 +80,3 @@ def plugin_list_ui(
|
|
|
81
80
|
)
|
|
82
81
|
|
|
83
82
|
asyncio.run(_run())
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
@app.command("upload")
|
|
87
|
-
def plugin_upload(
|
|
88
|
-
file_path: str = typer.Argument(..., help="Path to the plugin .py file"),
|
|
89
|
-
plugin_type: Literal["default", "extension", "ui"] = typer.Option(
|
|
90
|
-
"default", "--type", help="Plugin type category"
|
|
91
|
-
),
|
|
92
|
-
fail_if_exists: bool = typer.Option(
|
|
93
|
-
False, "--fail-if-exists", help="Fail if plugin file already exists"
|
|
94
|
-
),
|
|
95
|
-
) -> None:
|
|
96
|
-
"""Upload a plugin file."""
|
|
97
|
-
async def _run() -> None:
|
|
98
|
-
# Check file exists
|
|
99
|
-
p = Path(file_path)
|
|
100
|
-
if not p.exists():
|
|
101
|
-
typer.echo(f"Error: File not found: {file_path}", err=True)
|
|
102
|
-
raise typer.Exit(1)
|
|
103
|
-
|
|
104
|
-
async with get_client() as client:
|
|
105
|
-
result = await client.plugins.upload(
|
|
106
|
-
file_path,
|
|
107
|
-
plugin_type=plugin_type,
|
|
108
|
-
fail_if_exists=fail_if_exists,
|
|
109
|
-
)
|
|
110
|
-
print_result(result)
|
|
111
|
-
|
|
112
|
-
asyncio.run(_run())
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
@app.command("delete")
|
|
116
|
-
def plugin_delete(
|
|
117
|
-
filename: str = typer.Argument(..., help="Plugin filename to delete, e.g., 'my_plugin.py'"),
|
|
118
|
-
plugin_type: Literal["default", "extension", "ui"] = typer.Option(
|
|
119
|
-
"default", "--type", help="Plugin type category"
|
|
120
|
-
),
|
|
121
|
-
) -> None:
|
|
122
|
-
"""Delete a plugin file."""
|
|
123
|
-
async def _run() -> None:
|
|
124
|
-
async with get_client() as client:
|
|
125
|
-
result = await client.plugins.delete(filename, plugin_type=plugin_type)
|
|
126
|
-
print_result(result)
|
|
127
|
-
|
|
128
|
-
asyncio.run(_run())
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
@app.command("download")
|
|
132
|
-
def plugin_download(
|
|
133
|
-
filename: str = typer.Argument(..., help="Plugin filename to download, e.g., 'my_plugin.py'"),
|
|
134
|
-
output_path: str = typer.Argument(..., help="Local path to save the plugin"),
|
|
135
|
-
plugin_type: Literal["default", "extension", "ui"] = typer.Option(
|
|
136
|
-
"default", "--type", help="Plugin type category"
|
|
137
|
-
),
|
|
138
|
-
) -> None:
|
|
139
|
-
"""Download a plugin file."""
|
|
140
|
-
async def _run() -> None:
|
|
141
|
-
async with get_client() as client:
|
|
142
|
-
result = await client.plugins.download(
|
|
143
|
-
filename,
|
|
144
|
-
output_path,
|
|
145
|
-
plugin_type=plugin_type,
|
|
146
|
-
)
|
|
147
|
-
data = {"file": result}
|
|
148
|
-
print_result(data)
|
|
149
|
-
|
|
150
|
-
asyncio.run(_run())
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gulp-cli
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.9
|
|
4
4
|
Summary: Command-line client for gULP
|
|
5
5
|
Author-email: Mentat <info@mentat.is>
|
|
6
6
|
Requires-Python: >=3.12
|
|
@@ -22,7 +22,7 @@ Dynamic: license-file
|
|
|
22
22
|
## ✨ What can you do?
|
|
23
23
|
|
|
24
24
|
- 🔐 **Authentication** — secure login with token persistence
|
|
25
|
-
- 📥 **Ingestion** — ingest files (single/batch/wildcard),
|
|
25
|
+
- 📥 **Ingestion** — ingest files (single/batch/wildcard), optionally compressed before upload, with concurrent uploads
|
|
26
26
|
- 🔍 **Querying** — raw OpenSearch queries, Sigma rules, external plugins
|
|
27
27
|
- 🏷️ **Enrichment** — enrich documents, tag/untag, update fields
|
|
28
28
|
- 👥 **User Management** — create users, manage permissions (admin only)
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
|
+
import bz2
|
|
5
|
+
import asyncio
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
|
|
6
8
|
import pytest
|
|
@@ -8,6 +10,8 @@ import typer
|
|
|
8
10
|
|
|
9
11
|
from gulp_cli.commands.ingest import (
|
|
10
12
|
_expand_source_patterns,
|
|
13
|
+
_maybe_bz2_compress_file_for_ingestion,
|
|
14
|
+
_plugin_params_request_compression,
|
|
11
15
|
ingest_file_to_source,
|
|
12
16
|
ingest_file,
|
|
13
17
|
)
|
|
@@ -76,3 +80,36 @@ def test_expand_source_patterns_rejects_binary_paths_file(tmp_path: Path) -> Non
|
|
|
76
80
|
|
|
77
81
|
with pytest.raises(typer.BadParameter, match="must be UTF-8 text"):
|
|
78
82
|
_expand_source_patterns([], str(paths_file))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_compressed_plugin_params_create_bz2_upload_file(tmp_path: Path) -> None:
|
|
86
|
+
source = tmp_path / "sample.evtx"
|
|
87
|
+
payload = b"event log bytes"
|
|
88
|
+
source.write_bytes(payload)
|
|
89
|
+
|
|
90
|
+
assert _plugin_params_request_compression({"compressed": True})
|
|
91
|
+
upload_path, temp_path = asyncio.run(
|
|
92
|
+
_maybe_bz2_compress_file_for_ingestion(
|
|
93
|
+
str(source), {"compressed": True}
|
|
94
|
+
)
|
|
95
|
+
)
|
|
96
|
+
try:
|
|
97
|
+
assert temp_path == upload_path
|
|
98
|
+
assert upload_path.endswith(".bz2")
|
|
99
|
+
with bz2.open(upload_path, "rb") as f:
|
|
100
|
+
assert f.read() == payload
|
|
101
|
+
finally:
|
|
102
|
+
Path(upload_path).unlink(missing_ok=True)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_uncompressed_plugin_params_keep_original_upload_file(tmp_path: Path) -> None:
|
|
106
|
+
source = tmp_path / "sample.evtx"
|
|
107
|
+
source.write_bytes(b"event log bytes")
|
|
108
|
+
|
|
109
|
+
assert not _plugin_params_request_compression({})
|
|
110
|
+
upload_path, temp_path = asyncio.run(
|
|
111
|
+
_maybe_bz2_compress_file_for_ingestion(str(source), {})
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
assert upload_path == str(source)
|
|
115
|
+
assert temp_path is None
|
|
@@ -10,7 +10,6 @@ from gulp_cli.commands.ingest import (
|
|
|
10
10
|
ingest_file,
|
|
11
11
|
ingest_file_to_source,
|
|
12
12
|
ingest_raw,
|
|
13
|
-
ingest_zip,
|
|
14
13
|
)
|
|
15
14
|
from gulp_sdk.websocket import WSMessage, WSMessageType
|
|
16
15
|
|
|
@@ -34,16 +33,6 @@ def test_ingest_file_to_source_exposes_plugin_option() -> None:
|
|
|
34
33
|
assert plugin_default.param_decls == ("--plugin",)
|
|
35
34
|
|
|
36
35
|
|
|
37
|
-
def test_ingest_zip_exposes_wait_log_option() -> None:
|
|
38
|
-
wait_log_default = inspect.signature(ingest_zip).parameters["wait_log"].default
|
|
39
|
-
assert wait_log_default.default is False
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def test_ingest_zip_exposes_context_option() -> None:
|
|
43
|
-
context_default = inspect.signature(ingest_zip).parameters["context_name"].default
|
|
44
|
-
assert context_default.param_decls == ("--context",)
|
|
45
|
-
|
|
46
|
-
|
|
47
36
|
def test_ingest_raw_exposes_wait_log_option() -> None:
|
|
48
37
|
wait_log_default = inspect.signature(ingest_raw).parameters["wait_log"].default
|
|
49
38
|
assert wait_log_default.default is False
|
|
@@ -12,6 +12,7 @@ import typer
|
|
|
12
12
|
from gulp_cli.commands.ingest import (
|
|
13
13
|
_build_zip_from_sources,
|
|
14
14
|
_expand_source_patterns,
|
|
15
|
+
_safe_archive_name,
|
|
15
16
|
ingest_zip_create,
|
|
16
17
|
)
|
|
17
18
|
from gulp_cli.config import set_runtime_config_dir
|
|
@@ -87,40 +88,27 @@ def test_build_zip_from_sources_preserves_path_when_requested(
|
|
|
87
88
|
)
|
|
88
89
|
|
|
89
90
|
assert archived_count == 1
|
|
90
|
-
assert archived_entries == [
|
|
91
|
+
assert archived_entries == [_safe_archive_name(str(source_file.resolve()))]
|
|
91
92
|
assert len(created_archives) == 1
|
|
92
93
|
|
|
93
94
|
with zipfile.ZipFile(created_archives[0]) as archive:
|
|
94
|
-
assert archive.namelist() ==
|
|
95
|
+
assert archive.namelist() == archived_entries
|
|
95
96
|
|
|
96
97
|
|
|
97
98
|
@pytest.mark.parametrize(
|
|
98
|
-
"
|
|
99
|
-
[
|
|
99
|
+
("path", "expected"),
|
|
100
|
+
[
|
|
101
|
+
("./samples/nested/data.json", "samples/nested/data.json"),
|
|
102
|
+
(".\\samples\\nested\\data.json", "samples/nested/data.json"),
|
|
103
|
+
("../../../samples/nested/data.json", "samples/nested/data.json"),
|
|
104
|
+
("C:\\samples\\nested\\data.json", "C/samples/nested/data.json"),
|
|
105
|
+
("\\\\server\\share\\samples\\nested\\data.json", "server/share/samples/nested/data.json"),
|
|
106
|
+
],
|
|
100
107
|
)
|
|
101
|
-
def
|
|
102
|
-
|
|
108
|
+
def test_safe_archive_name_strips_roots_and_dot_paths(
|
|
109
|
+
path: str, expected: str
|
|
103
110
|
) -> None:
|
|
104
|
-
|
|
105
|
-
source_file.write_text('{"value": 1}', encoding="utf-8")
|
|
106
|
-
monkeypatch.setattr(
|
|
107
|
-
"gulp_cli.commands.ingest.os.path.relpath",
|
|
108
|
-
lambda _source_path, _cwd: rel_path,
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
output_zip = tmp_path / "out.zip"
|
|
112
|
-
archived_count, archived_entries, created_archives = _build_zip_from_sources(
|
|
113
|
-
output_zip,
|
|
114
|
-
[(source_file, tmp_path)],
|
|
115
|
-
preserve_path=True,
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
expected = "samples/nested/data.json"
|
|
119
|
-
assert archived_count == 1
|
|
120
|
-
assert archived_entries == [expected]
|
|
121
|
-
|
|
122
|
-
with zipfile.ZipFile(created_archives[0]) as archive:
|
|
123
|
-
assert archive.namelist() == [expected]
|
|
111
|
+
assert _safe_archive_name(path) == expected
|
|
124
112
|
|
|
125
113
|
|
|
126
114
|
def test_build_zip_from_sources_strips_parent_paths_when_preserving(
|
|
@@ -141,7 +129,7 @@ def test_build_zip_from_sources_strips_parent_paths_when_preserving(
|
|
|
141
129
|
preserve_path=True,
|
|
142
130
|
)
|
|
143
131
|
|
|
144
|
-
expected =
|
|
132
|
+
expected = _safe_archive_name(str(source_file.resolve()))
|
|
145
133
|
assert archived_count == 1
|
|
146
134
|
assert archived_entries == [expected]
|
|
147
135
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|