gulp-cli 1.1.10__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gulp_cli-1.1.10/src/gulp_cli.egg-info → gulp_cli-2.0.0}/PKG-INFO +2 -2
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/examples.md +3 -2
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/pyproject.toml +1 -1
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/_version.py +3 -3
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/collab.py +23 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/ingest.py +2 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/query.py +2 -2
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/stats.py +3 -5
- {gulp_cli-1.1.10 → gulp_cli-2.0.0/src/gulp_cli.egg-info}/PKG-INFO +2 -2
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli.egg-info/SOURCES.txt +2 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli.egg-info/requires.txt +1 -1
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli.egg-info/scm_file_list.json +2 -0
- gulp_cli-2.0.0/src/gulp_cli.egg-info/scm_version.json +8 -0
- gulp_cli-2.0.0/tests/test_collab_count.py +34 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/tests/test_enrich_live.py +2 -2
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/tests/test_ingest_paths_file.py +71 -0
- gulp_cli-2.0.0/tests/test_ingest_raw_wait.py +52 -0
- gulp_cli-1.1.10/src/gulp_cli.egg-info/scm_version.json +0 -8
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/.github/workflows/portable-bundles.yml +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/.github/workflows/python-package.yml +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/.gitignore +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/GULP_DUAL_LICENSE.txt +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/LICENSE.AGPL-3.0.txt +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/LICENSE.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/README.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/command-reference.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/extensions.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/getting-started.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/portable.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/resource-management.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/stdout-markers.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/docs/troubleshooting-cli.md +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/gulp-cli.spec +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/portable/launch-linux.sh +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/portable/launch-macos.sh +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/portable/launch-windows.bat +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/setup.cfg +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/__init__.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/__main__.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/cli.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/client.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/__init__.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/acl.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/auth.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/context.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/db.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/enhance_map.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/enrich.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/glyph.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/mapping.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/operations.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/plugin.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/source.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/storage.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/user_group.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/users.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/commands/utility.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/config.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/extension/__init__.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/extension/query_sigma_zip.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/extension/story.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/extension_helpers.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/extensions.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/output.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/utils.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli/version.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli.egg-info/dependency_links.txt +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli.egg-info/entry_points.txt +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/src/gulp_cli.egg-info/top_level.txt +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/tests/test_ingest_wait_log.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/tests/test_version.py +0 -0
- {gulp_cli-1.1.10 → gulp_cli-2.0.0}/tests/test_zip_create.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gulp-cli
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Command-line client for gULP
|
|
5
5
|
Author-email: Mentat <info@mentat.is>
|
|
6
6
|
Requires-Python: >=3.12
|
|
@@ -10,7 +10,7 @@ License-File: LICENSE.md
|
|
|
10
10
|
Requires-Dist: typer==0.23.1
|
|
11
11
|
Requires-Dist: rich==15.0.0
|
|
12
12
|
Requires-Dist: click==8.3.3
|
|
13
|
-
Requires-Dist: gulp-sdk
|
|
13
|
+
Requires-Dist: gulp-sdk>=2.0.0
|
|
14
14
|
Provides-Extra: portable
|
|
15
15
|
Requires-Dist: pyinstaller<7,>=6.11; extra == "portable"
|
|
16
16
|
Dynamic: license-file
|
|
@@ -38,6 +38,8 @@
|
|
|
38
38
|
- [Search by Field Value](#search-by-field-value)
|
|
39
39
|
- [Complex Queries (Range, Boolean)](#complex-queries-range-boolean)
|
|
40
40
|
- [Filter by Source](#filter-by-source)
|
|
41
|
+
- [Query External Data Source](#query-external-data-source)
|
|
42
|
+
- [Export Query Results](#export-query-results)
|
|
41
43
|
- [Aggregation, Document Lookup, and History](#aggregation-document-lookup-and-history)
|
|
42
44
|
- [Sigma Rule Queries](#sigma-rule-queries)
|
|
43
45
|
- [Run Single Sigma Rule](#run-single-sigma-rule)
|
|
@@ -522,6 +524,7 @@ gulp-cli query gulp incident-001 --flt '{"tags":["suspicious"]}' --preview
|
|
|
522
524
|
|
|
523
525
|
# Paginated results via q_options overrides
|
|
524
526
|
gulp-cli query gulp incident-001 --flt '{"tags":["suspicious"]}' --limit 200 --offset 400
|
|
527
|
+
```
|
|
525
528
|
|
|
526
529
|
### Query External Data Source
|
|
527
530
|
|
|
@@ -533,8 +536,6 @@ gulp-cli query external incident-001 \
|
|
|
533
536
|
--preview --limit 100 --offset 0
|
|
534
537
|
```
|
|
535
538
|
|
|
536
|
-
```
|
|
537
|
-
|
|
538
539
|
### Export Query Results
|
|
539
540
|
|
|
540
541
|
```bash
|
|
@@ -14,7 +14,7 @@ authors = [{ name = "Mentat", email = "info@mentat.is" }]
|
|
|
14
14
|
description = "Command-line client for gULP"
|
|
15
15
|
readme = "README.md"
|
|
16
16
|
requires-python = ">=3.12"
|
|
17
|
-
dependencies = ["typer==0.23.1", "rich==15.0.0", "click==8.3.3", "gulp-sdk"]
|
|
17
|
+
dependencies = ["typer==0.23.1", "rich==15.0.0", "click==8.3.3", "gulp-sdk>=2.0.0"]
|
|
18
18
|
|
|
19
19
|
[project.optional-dependencies]
|
|
20
20
|
portable = ["pyinstaller>=6.11,<7"]
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '
|
|
22
|
-
__version_tuple__ = version_tuple = (
|
|
21
|
+
__version__ = version = '2.0.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 0, 0)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g457370818'
|
|
@@ -119,6 +119,29 @@ async def _bulk_delete(
|
|
|
119
119
|
print_result(deleted)
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
@app.command("count")
|
|
123
|
+
def object_count(
|
|
124
|
+
obj_type: str = typer.Argument(..., help="Collab object type, e.g. note"),
|
|
125
|
+
operation_id: str | None = typer.Option(
|
|
126
|
+
None, "--operation-id", help="Optional operation ID scope"
|
|
127
|
+
),
|
|
128
|
+
flt: str | None = typer.Option(
|
|
129
|
+
None, "--flt", help="GulpCollabFilter JSON object"
|
|
130
|
+
),
|
|
131
|
+
) -> None:
|
|
132
|
+
async def _run() -> None:
|
|
133
|
+
flt_parsed = parse_json_option(flt, field_name="flt") or {}
|
|
134
|
+
async with get_client() as client:
|
|
135
|
+
counted = await client.plugins.object_count(
|
|
136
|
+
obj_type=obj_type,
|
|
137
|
+
flt=flt_parsed,
|
|
138
|
+
operation_id=operation_id,
|
|
139
|
+
)
|
|
140
|
+
print_result(counted)
|
|
141
|
+
|
|
142
|
+
asyncio.run(_run())
|
|
143
|
+
|
|
144
|
+
|
|
122
145
|
@note_app.command("create")
|
|
123
146
|
def note_create(
|
|
124
147
|
operation_id: str,
|
|
@@ -76,14 +76,14 @@ def query_raw_paginate(
|
|
|
76
76
|
@app.command("raw")
|
|
77
77
|
def query_raw(
|
|
78
78
|
operation_id: str,
|
|
79
|
-
q: str = typer.Option(..., "--q", help="JSON object
|
|
79
|
+
q: str = typer.Option(..., "--q", help="JSON object with an OpenSearch DSL query"),
|
|
80
80
|
q_options: str | None = typer.Option(None, "--q-options", help="JSON object for GulpQueryParameters"),
|
|
81
81
|
preview: bool = typer.Option(False, "--preview", help="Enable preview_mode in q_options (synchronous limited result set)"),
|
|
82
82
|
limit: int | None = typer.Option(None, "--limit", min=1, help="Set q_options.limit"),
|
|
83
83
|
wait: bool = typer.Option(False, "--wait"),
|
|
84
84
|
) -> None:
|
|
85
85
|
async def _run() -> None:
|
|
86
|
-
q_parsed =
|
|
86
|
+
q_parsed = parse_json_option(q, field_name="q")
|
|
87
87
|
if not q_parsed:
|
|
88
88
|
raise typer.BadParameter("--q is required")
|
|
89
89
|
options = parse_json_option(q_options, field_name="q-options")
|
|
@@ -413,21 +413,19 @@ def _significant_data(
|
|
|
413
413
|
failed = data.get("records_failed")
|
|
414
414
|
return f"ing={ingested or 0} skip={skipped or 0} fail={failed or 0}"
|
|
415
415
|
|
|
416
|
-
# query / ext_query: focus on
|
|
416
|
+
# query / ext_query: focus on progress
|
|
417
417
|
if req_type in {"query", "ext_query"}:
|
|
418
|
-
group = _first_non_empty(data, ["q_group", "group", "name"])
|
|
419
418
|
num_q = data.get("num_queries")
|
|
420
419
|
done_q = data.get("completed_queries")
|
|
421
420
|
failed_q = data.get("failed_queries")
|
|
422
421
|
hits = data.get("total_hits")
|
|
423
|
-
prefix = f"group={group}" if group else "query"
|
|
424
422
|
if num_q is not None and done_q is not None:
|
|
425
423
|
return (
|
|
426
|
-
f"
|
|
424
|
+
f"query ({done_q}/{num_q}, failed={failed_q or 0}) "
|
|
427
425
|
f"hits={hits if hits is not None else '-'}"
|
|
428
426
|
)
|
|
429
427
|
if hits is not None:
|
|
430
|
-
return f"
|
|
428
|
+
return f"query hits={hits}"
|
|
431
429
|
return prefix
|
|
432
430
|
|
|
433
431
|
# rebase / enrich: focus on updated docs counts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gulp-cli
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Command-line client for gULP
|
|
5
5
|
Author-email: Mentat <info@mentat.is>
|
|
6
6
|
Requires-Python: >=3.12
|
|
@@ -10,7 +10,7 @@ License-File: LICENSE.md
|
|
|
10
10
|
Requires-Dist: typer==0.23.1
|
|
11
11
|
Requires-Dist: rich==15.0.0
|
|
12
12
|
Requires-Dist: click==8.3.3
|
|
13
|
-
Requires-Dist: gulp-sdk
|
|
13
|
+
Requires-Dist: gulp-sdk>=2.0.0
|
|
14
14
|
Provides-Extra: portable
|
|
15
15
|
Requires-Dist: pyinstaller<7,>=6.11; extra == "portable"
|
|
16
16
|
Dynamic: license-file
|
|
@@ -60,8 +60,10 @@ src/gulp_cli/commands/utility.py
|
|
|
60
60
|
src/gulp_cli/extension/__init__.py
|
|
61
61
|
src/gulp_cli/extension/query_sigma_zip.py
|
|
62
62
|
src/gulp_cli/extension/story.py
|
|
63
|
+
tests/test_collab_count.py
|
|
63
64
|
tests/test_enrich_live.py
|
|
64
65
|
tests/test_ingest_paths_file.py
|
|
66
|
+
tests/test_ingest_raw_wait.py
|
|
65
67
|
tests/test_ingest_wait_log.py
|
|
66
68
|
tests/test_version.py
|
|
67
69
|
tests/test_zip_create.py
|
|
@@ -55,7 +55,9 @@
|
|
|
55
55
|
"tests/test_version.py",
|
|
56
56
|
"tests/test_enrich_live.py",
|
|
57
57
|
"tests/test_zip_create.py",
|
|
58
|
+
"tests/test_ingest_raw_wait.py",
|
|
58
59
|
"tests/test_ingest_wait_log.py",
|
|
60
|
+
"tests/test_collab_count.py",
|
|
59
61
|
".github/workflows/portable-bundles.yml",
|
|
60
62
|
".github/workflows/python-package.yml"
|
|
61
63
|
]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_collab_count_command_uses_sdk(monkeypatch) -> None:
|
|
5
|
+
import gulp_cli.commands.collab as collab
|
|
6
|
+
|
|
7
|
+
seen: dict[str, object] = {}
|
|
8
|
+
|
|
9
|
+
class _FakePlugins:
|
|
10
|
+
async def object_count(self, **kwargs) -> dict[str, int]:
|
|
11
|
+
seen.update(kwargs)
|
|
12
|
+
return {"count": 7}
|
|
13
|
+
|
|
14
|
+
class _FakeClient:
|
|
15
|
+
plugins = _FakePlugins()
|
|
16
|
+
|
|
17
|
+
class _FakeGetClient:
|
|
18
|
+
async def __aenter__(self):
|
|
19
|
+
return _FakeClient()
|
|
20
|
+
|
|
21
|
+
async def __aexit__(self, exc_type, exc, tb):
|
|
22
|
+
return None
|
|
23
|
+
|
|
24
|
+
monkeypatch.setattr(collab, "get_client", lambda: _FakeGetClient())
|
|
25
|
+
monkeypatch.setattr(collab, "print_result", lambda data: seen.setdefault("data", data))
|
|
26
|
+
|
|
27
|
+
collab.object_count("note", operation_id="op-a", flt='{"tags":["auto"]}')
|
|
28
|
+
|
|
29
|
+
assert seen == {
|
|
30
|
+
"obj_type": "note",
|
|
31
|
+
"flt": {"tags": ["auto"]},
|
|
32
|
+
"operation_id": "op-a",
|
|
33
|
+
"data": {"count": 7},
|
|
34
|
+
}
|
|
@@ -48,7 +48,7 @@ async def _setup_sample_docs() -> tuple[str, str]:
|
|
|
48
48
|
|
|
49
49
|
preview = await client.queries.query_raw(
|
|
50
50
|
operation_id=operation.id,
|
|
51
|
-
q=
|
|
51
|
+
q={"query": {"match_all": {}}},
|
|
52
52
|
q_options={"preview_mode": True, "limit": 1, "name": "cli_enrich_preview"},
|
|
53
53
|
)
|
|
54
54
|
docs = (preview.get("data") or {}).get("docs") or []
|
|
@@ -75,7 +75,7 @@ async def _get_first_doc_id(operation_id: str, config_dir: Path) -> str:
|
|
|
75
75
|
async with GulpClient(url, token=token) as client:
|
|
76
76
|
preview = await client.queries.query_raw(
|
|
77
77
|
operation_id=operation_id,
|
|
78
|
-
q=
|
|
78
|
+
q={"query": {"match_all": {}}},
|
|
79
79
|
q_options={"preview_mode": True, "limit": 1, "name": "cli_enrich_preview"},
|
|
80
80
|
)
|
|
81
81
|
docs = (preview.get("data") or {}).get("docs") or []
|
|
@@ -4,10 +4,12 @@ import inspect
|
|
|
4
4
|
import bz2
|
|
5
5
|
import asyncio
|
|
6
6
|
from pathlib import Path
|
|
7
|
+
from types import SimpleNamespace
|
|
7
8
|
|
|
8
9
|
import pytest
|
|
9
10
|
import typer
|
|
10
11
|
|
|
12
|
+
from gulp_cli.commands import ingest as ingest_module
|
|
11
13
|
from gulp_cli.commands.ingest import (
|
|
12
14
|
_expand_source_patterns,
|
|
13
15
|
_maybe_bz2_compress_file_for_ingestion,
|
|
@@ -17,6 +19,75 @@ from gulp_cli.commands.ingest import (
|
|
|
17
19
|
)
|
|
18
20
|
|
|
19
21
|
|
|
22
|
+
def test_ingest_file_assigns_a_distinct_req_id_to_each_file(
|
|
23
|
+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
24
|
+
) -> None:
|
|
25
|
+
files = [tmp_path / "one.log", tmp_path / "two.log"]
|
|
26
|
+
for path in files:
|
|
27
|
+
path.write_text("event", encoding="utf-8")
|
|
28
|
+
|
|
29
|
+
req_ids: list[str] = []
|
|
30
|
+
|
|
31
|
+
class _Ingest:
|
|
32
|
+
async def file(self, **kwargs):
|
|
33
|
+
req_id = kwargs["params"]["req_id"]
|
|
34
|
+
req_ids.append(req_id)
|
|
35
|
+
return SimpleNamespace(req_id=req_id, status="pending")
|
|
36
|
+
|
|
37
|
+
class _Client:
|
|
38
|
+
ingest = _Ingest()
|
|
39
|
+
|
|
40
|
+
async def ensure_websocket(self) -> None:
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
class _ClientContext:
|
|
44
|
+
async def __aenter__(self):
|
|
45
|
+
return _Client()
|
|
46
|
+
|
|
47
|
+
async def __aexit__(self, exc_type, exc, tb):
|
|
48
|
+
return False
|
|
49
|
+
|
|
50
|
+
class _Tracker:
|
|
51
|
+
@classmethod
|
|
52
|
+
async def create(cls, *_args, **_kwargs):
|
|
53
|
+
return cls()
|
|
54
|
+
|
|
55
|
+
async def wait_for_confirm(self, _req_id: str, _timeout: float) -> bool:
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
def close(self) -> None:
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
async def _operation_exists(*_args, **_kwargs) -> None:
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
monkeypatch.setattr(ingest_module, "get_client", lambda: _ClientContext())
|
|
65
|
+
monkeypatch.setattr(ingest_module, "_IngestWsTracker", _Tracker)
|
|
66
|
+
monkeypatch.setattr(ingest_module, "_ensure_operation_exists", _operation_exists)
|
|
67
|
+
monkeypatch.setattr(ingest_module, "print_result", lambda _result: None)
|
|
68
|
+
|
|
69
|
+
ingest_file(
|
|
70
|
+
operation_id="op",
|
|
71
|
+
plugin="raw",
|
|
72
|
+
file_patterns=[str(path) for path in files],
|
|
73
|
+
paths_file=None,
|
|
74
|
+
context_name="ctx",
|
|
75
|
+
plugin_params=None,
|
|
76
|
+
flt=None,
|
|
77
|
+
reset_operation=False,
|
|
78
|
+
create_operation_if_missing=False,
|
|
79
|
+
preview=False,
|
|
80
|
+
wait=False,
|
|
81
|
+
wait_log=False,
|
|
82
|
+
wait_timeout=300,
|
|
83
|
+
show_per_file_progress=False,
|
|
84
|
+
batch_size=2,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
assert len(req_ids) == len(files)
|
|
88
|
+
assert len(set(req_ids)) == len(files)
|
|
89
|
+
|
|
90
|
+
|
|
20
91
|
def test_ingest_file_paths_file_option_defaults_to_unset() -> None:
|
|
21
92
|
paths_file_default = inspect.signature(ingest_file).parameters["paths_file"].default
|
|
22
93
|
assert paths_file_default.default is None
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Raw-ingest worker wait coverage."""
|
|
2
|
+
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from types import SimpleNamespace
|
|
5
|
+
from unittest.mock import AsyncMock
|
|
6
|
+
|
|
7
|
+
from gulp_cli.commands import ingest as ingest_module
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_ingest_raw_passes_wait_and_timeout_to_sdk(monkeypatch) -> None:
|
|
11
|
+
raw = AsyncMock(return_value=SimpleNamespace(req_id="req-1"))
|
|
12
|
+
client = SimpleNamespace(
|
|
13
|
+
ingest=SimpleNamespace(raw=raw),
|
|
14
|
+
ensure_websocket=AsyncMock(),
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
@asynccontextmanager
|
|
18
|
+
async def _client_context():
|
|
19
|
+
yield client
|
|
20
|
+
|
|
21
|
+
tracker = SimpleNamespace(
|
|
22
|
+
wait_for_terminal=AsyncMock(return_value={"status": "done"})
|
|
23
|
+
)
|
|
24
|
+
monkeypatch.setattr(ingest_module, "get_client", _client_context)
|
|
25
|
+
monkeypatch.setattr(
|
|
26
|
+
ingest_module,
|
|
27
|
+
"_IngestWsTracker",
|
|
28
|
+
SimpleNamespace(create=AsyncMock(return_value=tracker)),
|
|
29
|
+
)
|
|
30
|
+
monkeypatch.setattr(ingest_module, "_ensure_operation_exists", AsyncMock())
|
|
31
|
+
monkeypatch.setattr(ingest_module, "print_result", lambda _result: None)
|
|
32
|
+
|
|
33
|
+
ingest_module.ingest_raw(
|
|
34
|
+
operation_id="op-1",
|
|
35
|
+
data="[]",
|
|
36
|
+
data_file=None,
|
|
37
|
+
plugin="raw",
|
|
38
|
+
plugin_params=None,
|
|
39
|
+
flt=None,
|
|
40
|
+
req_id=None,
|
|
41
|
+
last=True,
|
|
42
|
+
reset_operation=False,
|
|
43
|
+
create_operation_if_missing=False,
|
|
44
|
+
wait=True,
|
|
45
|
+
wait_log=False,
|
|
46
|
+
wait_timeout=321,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
kwargs = raw.await_args.kwargs
|
|
50
|
+
assert kwargs["wait_for_worker"] is True
|
|
51
|
+
assert kwargs["timeout"] == 321
|
|
52
|
+
assert kwargs["wait"] is False
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|