datalake-scripts 3.0.0rc7__tar.gz → 3.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/PKG-INFO +1 -1
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/bulk_search_task.py +8 -1
- datalake_scripts-3.0.1/datalake/common/user_agent.py +26 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/bulk_search.py +15 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/endpoint.py +2 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/cli.py +4 -2
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/PKG-INFO +1 -1
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/SOURCES.txt +4 -0
- datalake_scripts-3.0.1/tests/common/test_user_agent.py +36 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_bulk_search.py +137 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/README.md +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/atom.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/atom_type.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/config.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/logger.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/output.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/throttler.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/token_manager.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/utils.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/common/warn.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/config/endpoints.json +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/datalake.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/comments.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/my_account.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/sightings.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/sources.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/tags.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/miscellaneous/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/miscellaneous/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/common/base_script.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/output_builder.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/utils.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_comments.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_tags.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/bulk_lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/edit_score.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_atom_values.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_my_user_info.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats_from_query_hash.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/lookup_threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/search_watch.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/dependency_links.txt +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/entry_points.txt +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/requires.txt +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/top_level.txt +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/pyproject.toml +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/setup.cfg +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/setup.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/common/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/common/fixture.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/common/test_output.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/common/test_utils.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/conftest.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_advanced_search.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_atom_type.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_auth_via_tokens.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_filtered_threat_entity.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_my_account.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_search_watch.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_sightings.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_tags.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/scripts/__init__.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/scripts/test_add_new_threats.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/scripts/test_base_scripts.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/scripts/test_cli.py +0 -0
- {datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/scripts/test_csv_builder.py +0 -0
|
@@ -108,8 +108,11 @@ class BulkSearchTask:
|
|
|
108
108
|
raise BulkSearchFailedError(self.state)
|
|
109
109
|
time_passed = datetime.datetime.now(datetime.timezone.utc) - start
|
|
110
110
|
if time_passed.total_seconds() > timeout:
|
|
111
|
+
try:
|
|
112
|
+
self.cancel()
|
|
113
|
+
except Exception:
|
|
114
|
+
pass
|
|
111
115
|
raise TimeoutError()
|
|
112
|
-
|
|
113
116
|
await asyncio.sleep(self.REQUEST_INTERVAL)
|
|
114
117
|
self.update()
|
|
115
118
|
return self.download(output=output, stream=stream)
|
|
@@ -134,3 +137,7 @@ class BulkSearchTask:
|
|
|
134
137
|
"""Query the API to refresh the tasks attributes"""
|
|
135
138
|
updated_bs = self._endpoint.get_task(self.uuid)
|
|
136
139
|
self.__dict__.update(updated_bs.__dict__) # Avoid to return a new object
|
|
140
|
+
|
|
141
|
+
def cancel(self):
|
|
142
|
+
"""Cancel this bulk search task server-side."""
|
|
143
|
+
self._endpoint.cancel(self.uuid)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""User-Agent string builder for outbound Endpoint HTTP calls."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
# this weird code block is to maintain python 3.7 compatibility
|
|
6
|
+
try:
|
|
7
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
8
|
+
except ImportError: # Python < 3.8
|
|
9
|
+
from importlib_metadata import PackageNotFoundError, version
|
|
10
|
+
|
|
11
|
+
USER_AGENT_INTEGRATION_ENV = "OCD_DTL_USER_AGENT_INTEGRATION"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _get_version() -> str:
|
|
15
|
+
try:
|
|
16
|
+
return version("datalake_scripts")
|
|
17
|
+
except PackageNotFoundError:
|
|
18
|
+
return "unknown"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def build_user_agent() -> str:
|
|
22
|
+
base = f"datalake-python-sdk/{_get_version()}"
|
|
23
|
+
integration = (os.getenv(USER_AGENT_INTEGRATION_ENV) or "").strip()
|
|
24
|
+
if integration:
|
|
25
|
+
return f"{integration} {base}"
|
|
26
|
+
return base
|
|
@@ -54,6 +54,21 @@ class BulkSearch(Endpoint):
|
|
|
54
54
|
bs_as_json = results[0]
|
|
55
55
|
return BulkSearchTask(endpoint=self, **bs_as_json)
|
|
56
56
|
|
|
57
|
+
def cancel(self, task_uuid):
|
|
58
|
+
"""
|
|
59
|
+
Cancel the bulk search task with the given uuid.
|
|
60
|
+
|
|
61
|
+
Raises BulkSearchNotFound if the task does not exist.
|
|
62
|
+
"""
|
|
63
|
+
url = self._build_url_for_endpoint("bulk-search-task")
|
|
64
|
+
url = url.format(task_uuid=task_uuid)
|
|
65
|
+
try:
|
|
66
|
+
self.datalake_requests(url, "delete", headers=self._get_headers())
|
|
67
|
+
except ValueError as e:
|
|
68
|
+
if str(e).startswith("404"):
|
|
69
|
+
raise BulkSearchNotFound from e
|
|
70
|
+
raise
|
|
71
|
+
|
|
57
72
|
@output_supported(
|
|
58
73
|
{
|
|
59
74
|
Output.JSON,
|
|
@@ -14,6 +14,7 @@ from requests.adapters import HTTPAdapter, Retry
|
|
|
14
14
|
from datalake.common.output import Output
|
|
15
15
|
from datalake.common.throttler import throttle
|
|
16
16
|
from datalake.common.token_manager import TokenManager
|
|
17
|
+
from datalake.common.user_agent import build_user_agent
|
|
17
18
|
from datalake.common.utils import get_error_message
|
|
18
19
|
|
|
19
20
|
OCD_DTL_QUOTA_TIME = int(os.getenv("OCD_DTL_QUOTA_TIME", 1))
|
|
@@ -43,6 +44,7 @@ class Endpoint:
|
|
|
43
44
|
self.proxies = proxies
|
|
44
45
|
self.verify = verify
|
|
45
46
|
self.session = requests.Session()
|
|
47
|
+
self.session.headers.update({"User-Agent": build_user_agent()})
|
|
46
48
|
|
|
47
49
|
# Configure HTTP retry policy
|
|
48
50
|
retry_policy = Retry(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import argparse
|
|
3
|
-
import sys
|
|
3
|
+
import sys, os
|
|
4
4
|
from datalake_scripts.scripts import (
|
|
5
5
|
add_threats,
|
|
6
6
|
get_threats,
|
|
@@ -21,7 +21,7 @@ from datalake_scripts.scripts import (
|
|
|
21
21
|
|
|
22
22
|
class Cli:
|
|
23
23
|
CLI_NAME = "ocd-dtl"
|
|
24
|
-
VERSION = "3.0.
|
|
24
|
+
VERSION = "3.0.1"
|
|
25
25
|
|
|
26
26
|
def __init__(self):
|
|
27
27
|
parser = argparse.ArgumentParser(
|
|
@@ -74,6 +74,8 @@ class Cli:
|
|
|
74
74
|
parser.print_help()
|
|
75
75
|
exit(1)
|
|
76
76
|
|
|
77
|
+
os.environ["OCD_DTL_USER_AGENT_INTEGRATION"] = f"{self.CLI_NAME}/{self.VERSION}"
|
|
78
|
+
|
|
77
79
|
# Call the subcommand method
|
|
78
80
|
args.func(command_arguments)
|
|
79
81
|
|
|
@@ -13,6 +13,7 @@ setup.py
|
|
|
13
13
|
./datalake/common/output.py
|
|
14
14
|
./datalake/common/throttler.py
|
|
15
15
|
./datalake/common/token_manager.py
|
|
16
|
+
./datalake/common/user_agent.py
|
|
16
17
|
./datalake/common/utils.py
|
|
17
18
|
./datalake/common/warn.py
|
|
18
19
|
./datalake/config/endpoints.json
|
|
@@ -56,6 +57,7 @@ setup.py
|
|
|
56
57
|
./tests/common/__init__.py
|
|
57
58
|
./tests/common/fixture.py
|
|
58
59
|
./tests/common/test_output.py
|
|
60
|
+
./tests/common/test_user_agent.py
|
|
59
61
|
./tests/common/test_utils.py
|
|
60
62
|
./tests/lib/__init__.py
|
|
61
63
|
./tests/lib/test_advanced_search.py
|
|
@@ -84,6 +86,7 @@ datalake/common/logger.py
|
|
|
84
86
|
datalake/common/output.py
|
|
85
87
|
datalake/common/throttler.py
|
|
86
88
|
datalake/common/token_manager.py
|
|
89
|
+
datalake/common/user_agent.py
|
|
87
90
|
datalake/common/utils.py
|
|
88
91
|
datalake/common/warn.py
|
|
89
92
|
datalake/config/endpoints.json
|
|
@@ -133,6 +136,7 @@ tests/conftest.py
|
|
|
133
136
|
tests/common/__init__.py
|
|
134
137
|
tests/common/fixture.py
|
|
135
138
|
tests/common/test_output.py
|
|
139
|
+
tests/common/test_user_agent.py
|
|
136
140
|
tests/common/test_utils.py
|
|
137
141
|
tests/lib/__init__.py
|
|
138
142
|
tests/lib/test_advanced_search.py
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from datalake.common import user_agent as user_agent_mod
|
|
4
|
+
from datalake.common.user_agent import (
|
|
5
|
+
USER_AGENT_INTEGRATION_ENV,
|
|
6
|
+
build_user_agent,
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture
|
|
11
|
+
def fixed_version(monkeypatch):
|
|
12
|
+
monkeypatch.setattr(user_agent_mod, "_get_version", lambda: "9.9.9")
|
|
13
|
+
return "9.9.9"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_build_user_agent_default(monkeypatch, fixed_version):
|
|
17
|
+
monkeypatch.delenv(USER_AGENT_INTEGRATION_ENV, raising=False)
|
|
18
|
+
assert build_user_agent() == f"datalake-python-sdk/{fixed_version}"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_build_user_agent_with_integration(monkeypatch, fixed_version):
|
|
22
|
+
monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, "my-soar/2.1")
|
|
23
|
+
assert build_user_agent() == f"my-soar/2.1 datalake-python-sdk/{fixed_version}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.mark.parametrize("value", ["", " "])
|
|
27
|
+
def test_build_user_agent_ignores_empty_or_whitespace(
|
|
28
|
+
monkeypatch, fixed_version, value
|
|
29
|
+
):
|
|
30
|
+
monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, value)
|
|
31
|
+
assert build_user_agent() == f"datalake-python-sdk/{fixed_version}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_build_user_agent_strips_surrounding_whitespace(monkeypatch, fixed_version):
|
|
35
|
+
monkeypatch.setenv(USER_AGENT_INTEGRATION_ENV, " foo/1.0 ")
|
|
36
|
+
assert build_user_agent() == f"foo/1.0 datalake-python-sdk/{fixed_version}"
|
|
@@ -418,11 +418,26 @@ async def test_bulk_search_task_download_async_timeout(
|
|
|
418
418
|
response_context.add(
|
|
419
419
|
responses.POST, bs_status_url, json=bs_update_json, status=200
|
|
420
420
|
)
|
|
421
|
+
bs_cancel_url = (
|
|
422
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
423
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
424
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
425
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
426
|
+
)
|
|
427
|
+
)
|
|
428
|
+
response_context.add(responses.DELETE, bs_cancel_url, status=204)
|
|
421
429
|
|
|
422
430
|
task = bulk_search_task.download_async(timeout=0.2)
|
|
423
431
|
with pytest.raises(TimeoutError):
|
|
424
432
|
await task
|
|
425
433
|
|
|
434
|
+
cancel_calls = [
|
|
435
|
+
c
|
|
436
|
+
for c in response_context.calls
|
|
437
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
438
|
+
]
|
|
439
|
+
assert len(cancel_calls) == 1, "cancel() must be called on timeout"
|
|
440
|
+
|
|
426
441
|
|
|
427
442
|
@responses.activate
|
|
428
443
|
def test_bulk_search_task_download_sync_timeout(bulk_search_task: BulkSearchTask):
|
|
@@ -437,10 +452,25 @@ def test_bulk_search_task_download_sync_timeout(bulk_search_task: BulkSearchTask
|
|
|
437
452
|
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
|
|
438
453
|
)
|
|
439
454
|
responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
|
|
455
|
+
bs_cancel_url = (
|
|
456
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
457
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
458
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
459
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
460
|
+
)
|
|
461
|
+
)
|
|
462
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
440
463
|
|
|
441
464
|
with pytest.raises(TimeoutError):
|
|
442
465
|
bulk_search_task.download_sync(timeout=0.2)
|
|
443
466
|
|
|
467
|
+
cancel_calls = [
|
|
468
|
+
c
|
|
469
|
+
for c in responses.calls
|
|
470
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
471
|
+
]
|
|
472
|
+
assert len(cancel_calls) == 1, "cancel() must be called on timeout"
|
|
473
|
+
|
|
444
474
|
|
|
445
475
|
@responses.activate
|
|
446
476
|
def test_bulk_search_task_download_sync_failed(bulk_search_task: BulkSearchTask):
|
|
@@ -453,7 +483,114 @@ def test_bulk_search_task_download_sync_failed(bulk_search_task: BulkSearchTask)
|
|
|
453
483
|
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
|
|
454
484
|
)
|
|
455
485
|
responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
|
|
486
|
+
bs_cancel_url = (
|
|
487
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
488
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
489
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
490
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
491
|
+
)
|
|
492
|
+
)
|
|
493
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
456
494
|
|
|
457
495
|
with pytest.raises(BulkSearchFailedError) as err:
|
|
458
496
|
bulk_search_task.download_sync()
|
|
459
497
|
assert err.value.failed_state == BulkSearchTaskState.CANCELLED
|
|
498
|
+
|
|
499
|
+
cancel_calls = [
|
|
500
|
+
c
|
|
501
|
+
for c in responses.calls
|
|
502
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
503
|
+
]
|
|
504
|
+
assert len(cancel_calls) == 0, "cancel() must not be called on failed state"
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
@pytest.mark.parametrize(
|
|
508
|
+
"failed_state",
|
|
509
|
+
[
|
|
510
|
+
"CANCELLED",
|
|
511
|
+
"FAILED_TIMEOUT",
|
|
512
|
+
"FAILED_ERROR",
|
|
513
|
+
"FAILED_QUOTA_EXCEEDED",
|
|
514
|
+
"FAILED_TOO_MANY_RESULTS",
|
|
515
|
+
],
|
|
516
|
+
)
|
|
517
|
+
@responses.activate
|
|
518
|
+
def test_bulk_search_task_download_sync_cancel_on_any_failed_state(
|
|
519
|
+
bulk_search_task: BulkSearchTask, failed_state
|
|
520
|
+
):
|
|
521
|
+
bulk_search_task.state = BulkSearchTaskState.IN_PROGRESS
|
|
522
|
+
bs_update_json = copy.deepcopy(bs_status_json)
|
|
523
|
+
bs_update_json["results"][0]["state"] = failed_state
|
|
524
|
+
bs_status_url = (
|
|
525
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
526
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
527
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-tasks"]
|
|
528
|
+
)
|
|
529
|
+
responses.add(responses.POST, bs_status_url, json=bs_update_json, status=200)
|
|
530
|
+
bs_cancel_url = (
|
|
531
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
532
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
533
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
534
|
+
"{task_uuid}", bulk_search_task.uuid
|
|
535
|
+
)
|
|
536
|
+
)
|
|
537
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
538
|
+
|
|
539
|
+
with pytest.raises(BulkSearchFailedError) as err:
|
|
540
|
+
bulk_search_task.download_sync()
|
|
541
|
+
assert err.value.failed_state == BulkSearchTaskState[failed_state]
|
|
542
|
+
|
|
543
|
+
cancel_calls = [
|
|
544
|
+
c
|
|
545
|
+
for c in responses.calls
|
|
546
|
+
if c.request.method == "DELETE" and c.request.url == bs_cancel_url
|
|
547
|
+
]
|
|
548
|
+
assert (
|
|
549
|
+
len(cancel_calls) == 0
|
|
550
|
+
), f"cancel() must not be called for state {failed_state}"
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
@responses.activate
|
|
554
|
+
def test_bulk_search_task_cancel(bulk_search_task: BulkSearchTask):
|
|
555
|
+
task_uuid = bulk_search_task.uuid
|
|
556
|
+
bs_cancel_url = (
|
|
557
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
558
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
559
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
560
|
+
"{task_uuid}", task_uuid
|
|
561
|
+
)
|
|
562
|
+
)
|
|
563
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
564
|
+
|
|
565
|
+
assert bulk_search_task.cancel() is None
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
@responses.activate
|
|
569
|
+
def test_bulk_search_task_cancel_not_found(bulk_search_task: BulkSearchTask):
|
|
570
|
+
task_uuid = bulk_search_task.uuid
|
|
571
|
+
bs_cancel_url = (
|
|
572
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
573
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
574
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
575
|
+
"{task_uuid}", task_uuid
|
|
576
|
+
)
|
|
577
|
+
)
|
|
578
|
+
responses.add(responses.DELETE, bs_cancel_url, json={}, status=404)
|
|
579
|
+
|
|
580
|
+
with pytest.raises(BulkSearchNotFound):
|
|
581
|
+
bulk_search_task.cancel()
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
@responses.activate
|
|
585
|
+
def test_bulk_search_endpoint_cancel(datalake: Datalake):
|
|
586
|
+
task_uuid = "d9c00380-2784-4386-9bc3-aff35cfeeb41"
|
|
587
|
+
bs_cancel_url = (
|
|
588
|
+
TestData.TEST_CONFIG["main"][TestData.TEST_ENV]
|
|
589
|
+
+ TestData.TEST_CONFIG["api_version"]
|
|
590
|
+
+ TestData.TEST_CONFIG["endpoints"]["bulk-search-task"].replace(
|
|
591
|
+
"{task_uuid}", task_uuid
|
|
592
|
+
)
|
|
593
|
+
)
|
|
594
|
+
responses.add(responses.DELETE, bs_cancel_url, status=204)
|
|
595
|
+
|
|
596
|
+
assert datalake.BulkSearch.cancel(task_uuid) is None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake/endpoints/filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/helper_scripts/utils.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_comments.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/add_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/advanced_search.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/bulk_lookup_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_atom_values.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_my_user_info.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_query_hash.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/get_threats.py
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/lookup_threats.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts/scripts/search_watch.py
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/datalake_scripts.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{datalake_scripts-3.0.0rc7 → datalake_scripts-3.0.1}/tests/lib/test_filtered_threat_entity.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|