azul-client 9.0.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azul_client/__init__.py +4 -0
- azul_client/api/__init__.py +74 -0
- azul_client/api/base_api.py +163 -0
- azul_client/api/binaries_data.py +513 -0
- azul_client/api/binaries_meta.py +510 -0
- azul_client/api/features.py +175 -0
- azul_client/api/plugins.py +49 -0
- azul_client/api/purge.py +71 -0
- azul_client/api/security.py +29 -0
- azul_client/api/sources.py +51 -0
- azul_client/api/statistics.py +23 -0
- azul_client/api/users.py +29 -0
- azul_client/client.py +510 -0
- azul_client/config.py +116 -0
- azul_client/exceptions.py +30 -0
- azul_client/oidc/__init__.py +5 -0
- azul_client/oidc/callback.py +73 -0
- azul_client/oidc/oidc.py +215 -0
- azul_client-9.0.24.dist-info/METADATA +102 -0
- azul_client-9.0.24.dist-info/RECORD +23 -0
- azul_client-9.0.24.dist-info/WHEEL +5 -0
- azul_client-9.0.24.dist-info/entry_points.txt +2 -0
- azul_client-9.0.24.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
"""Module for interacting with binary endpoints."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from http import HTTPMethod
|
|
6
|
+
from typing import Callable, Generator
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
import pendulum
|
|
10
|
+
from azul_bedrock import models_network as azm
|
|
11
|
+
from azul_bedrock import models_restapi
|
|
12
|
+
from pydantic import TypeAdapter
|
|
13
|
+
|
|
14
|
+
from azul_client import config
|
|
15
|
+
from azul_client.api.base_api import BaseApiHandler
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
autocomplete_type_adapter = TypeAdapter(models_restapi.AutocompleteContext)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class FindOptions:
|
|
23
|
+
"""Store all of the options available for creating a term query.
|
|
24
|
+
|
|
25
|
+
Note list values assume an OR relationship when looking for matches.
|
|
26
|
+
Note list values assume an AND relationship when excluding matches.
|
|
27
|
+
|
|
28
|
+
Note dict values always assume an AND relationship between members of dict.
|
|
29
|
+
|
|
30
|
+
Note when specifying string values you can provide a wildcard '*' character.
|
|
31
|
+
Typically only useful at the end, rather than the start or middle e.g when looking for al file type image/bmp:
|
|
32
|
+
works:
|
|
33
|
+
file_format:"image*"
|
|
34
|
+
doesn't work:
|
|
35
|
+
file_format:"ima*e"
|
|
36
|
+
file_format:"*bmp"
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
_query: str = ""
|
|
40
|
+
|
|
41
|
+
# ------------------------- Source options
|
|
42
|
+
# Name(id) of the source(s) to look for entities within.
|
|
43
|
+
sources: list[str] | str | None = None
|
|
44
|
+
# Name(id) of the source(s) to exclude when looking for entities.
|
|
45
|
+
source_excludes: list[str] | str | None = None
|
|
46
|
+
|
|
47
|
+
# Includes only entities that have the depths provided.
|
|
48
|
+
source_depth: list[int] | int | None = None
|
|
49
|
+
# Exclude entities that have the depths provided.
|
|
50
|
+
source_depth_exclude: list[int] | int | None = None
|
|
51
|
+
# Includes only entities that have the depth greater than the value provided.
|
|
52
|
+
source_depth_greater: int | None = None
|
|
53
|
+
# Includes only entities that have the depth less than the value provided.
|
|
54
|
+
source_depth_less: int | None = None
|
|
55
|
+
|
|
56
|
+
# Includes only entities that have been sourced by the username.
|
|
57
|
+
source_username: str | None = None
|
|
58
|
+
# Includes only entities that have ALL the provided reference fields.
|
|
59
|
+
source_reference: dict[str, str] | None = None # Key value pairs of reference fields.
|
|
60
|
+
|
|
61
|
+
# ------------------------- Time options
|
|
62
|
+
# Include only entities that are newer than or equal to the provided timestamp (to nearest second)
|
|
63
|
+
source_timestamp_newer_or_equal: pendulum.DateTime | None = None
|
|
64
|
+
# Include only entities that are older than or equal to the provided timestamp (to nearest second)
|
|
65
|
+
source_timestamp_older_or_equal: pendulum.DateTime | None = None
|
|
66
|
+
# Include only entities that are newer than the provided timestamp (to nearest second)
|
|
67
|
+
source_timestamp_newer: pendulum.DateTime | None = None
|
|
68
|
+
# Include only entities that are older than the provided timestamp (to nearest second)
|
|
69
|
+
source_timestamp_older: pendulum.DateTime | None = None
|
|
70
|
+
|
|
71
|
+
# ------------------------- Plugin/Author filtering
|
|
72
|
+
# Include entities that have valid results from the provided plugin (case sensitive).
|
|
73
|
+
plugin_name: str | None = None
|
|
74
|
+
# Include entities that have valid results from the provided plugin version (case sensitive).
|
|
75
|
+
plugin_version: str | None = None
|
|
76
|
+
|
|
77
|
+
# ------------------------- Feature filtering
|
|
78
|
+
# Include entities that have any of the feature keys.
|
|
79
|
+
has_feature_keys: list[str] | str | None = None
|
|
80
|
+
# Include entities that have any of the feature values.
|
|
81
|
+
has_feature_values: list[str] | str | None = None
|
|
82
|
+
|
|
83
|
+
# ------------------------- Binary Info
|
|
84
|
+
# Include entities that are greater (in bytes) than the provided value.
|
|
85
|
+
greater_than_size_bytes: int | None = None
|
|
86
|
+
# Include entities that are less than (in bytes) than the provided value.
|
|
87
|
+
less_than_size_bytes: int | None = None
|
|
88
|
+
|
|
89
|
+
# Include entities that have the specified file type.
|
|
90
|
+
file_formats_legacy: str | list[str] | None = None
|
|
91
|
+
# Exclude entities that have the specified file type.
|
|
92
|
+
file_formats_legacy_exclude: str | list[str] | None = None
|
|
93
|
+
# Include entities that have the specified file type (AL type).
|
|
94
|
+
file_formats: str | list[str] | None = None
|
|
95
|
+
# Exclude entities that have the specified file type (AL type).
|
|
96
|
+
file_formats_exclude: str | list[str] | None = None
|
|
97
|
+
|
|
98
|
+
# ------------------------- Tags
|
|
99
|
+
# Find all binaries that have any of the provided tags.
|
|
100
|
+
binary_tags: str | list[str] | None = None
|
|
101
|
+
# Final all binaries that have features with the provided tags.
|
|
102
|
+
feature_tags: str | list[str] | None = None
|
|
103
|
+
|
|
104
|
+
# --- End of options ---
|
|
105
|
+
|
|
106
|
+
def _add(self, value: str):
|
|
107
|
+
"""Append a new value to the query."""
|
|
108
|
+
if not self._query:
|
|
109
|
+
self._query = value
|
|
110
|
+
else:
|
|
111
|
+
self._query += " " + value
|
|
112
|
+
|
|
113
|
+
def _add_date(self, search_key: str, value: pendulum.DateTime | None):
|
|
114
|
+
"""Add a date as an integer to the query."""
|
|
115
|
+
if value is None:
|
|
116
|
+
return
|
|
117
|
+
# Timestamp is just milliseconds since epoch, Format docs - https://pendulum.eustace.io/docs/#string-formatting
|
|
118
|
+
self._add(search_key % (value.format("x")))
|
|
119
|
+
|
|
120
|
+
def _add_if_not_none(self, search_key: str, value: str | int | None):
|
|
121
|
+
"""Add a value to the internal query if the provided value isn't none."""
|
|
122
|
+
if value is None:
|
|
123
|
+
return
|
|
124
|
+
self._add(search_key % (value))
|
|
125
|
+
|
|
126
|
+
def _add_list(self, search_key: str, value: str | list[str] | int | list[int] | None, negation: bool = False):
|
|
127
|
+
"""Add a list of values to the internal query if the provided value isn't none.
|
|
128
|
+
|
|
129
|
+
Negation is used to switch between 'AND'ing and 'OR'ing members of the list.
|
|
130
|
+
"""
|
|
131
|
+
if value is None or (isinstance(value, list) and len(value) == 0):
|
|
132
|
+
return
|
|
133
|
+
|
|
134
|
+
# Convert to a list if required.
|
|
135
|
+
if isinstance(value, str) or isinstance(value, int):
|
|
136
|
+
value = [value]
|
|
137
|
+
|
|
138
|
+
for i, val in enumerate(value):
|
|
139
|
+
if i > 0:
|
|
140
|
+
prefix = "OR "
|
|
141
|
+
if negation:
|
|
142
|
+
prefix = "AND "
|
|
143
|
+
self._add(prefix + search_key % (val))
|
|
144
|
+
else:
|
|
145
|
+
self._add(search_key % (val))
|
|
146
|
+
|
|
147
|
+
def _add_key_value(self, search_key: str, value: dict[str, str] | None):
|
|
148
|
+
"""Add a number key value pair queries."""
|
|
149
|
+
if value is None or len(value) == 0:
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
for k, v in value.items():
|
|
153
|
+
self._add(search_key % (k, v))
|
|
154
|
+
|
|
155
|
+
def to_query(self) -> str:
|
|
156
|
+
"""Convert the find options into a term query."""
|
|
157
|
+
self._query = ""
|
|
158
|
+
# Source options
|
|
159
|
+
self._add_list('source.name:"%s"', self.sources)
|
|
160
|
+
self._add_list('!source.name:"%s"', self.source_excludes, negation=True)
|
|
161
|
+
|
|
162
|
+
self._add_list("depth:%s", self.source_depth)
|
|
163
|
+
self._add_list("!depth:%s", self.source_depth_exclude, negation=True)
|
|
164
|
+
self._add_if_not_none("depth:>%s", self.source_depth_greater)
|
|
165
|
+
self._add_if_not_none("depth:<%s", self.source_depth_less)
|
|
166
|
+
|
|
167
|
+
self._add_if_not_none('source.encoded_references.key_value:"user.%s"', self.source_username)
|
|
168
|
+
self._add_key_value('source.encoded_references.key_value:"%s.%s"', self.source_reference)
|
|
169
|
+
|
|
170
|
+
# Time options
|
|
171
|
+
self._add_date("source.timestamp:>=%s", self.source_timestamp_newer_or_equal)
|
|
172
|
+
self._add_date("source.timestamp:<=%s", self.source_timestamp_older_or_equal)
|
|
173
|
+
self._add_date("source.timestamp:>%s", self.source_timestamp_newer)
|
|
174
|
+
self._add_date("source.timestamp:<%s", self.source_timestamp_older)
|
|
175
|
+
|
|
176
|
+
# Author options
|
|
177
|
+
self._add_if_not_none('author.name:"%s"', self.plugin_name)
|
|
178
|
+
self._add_if_not_none('author.version:"%s"', self.plugin_version)
|
|
179
|
+
|
|
180
|
+
# Features
|
|
181
|
+
self._add_list('features.name:"%s"', self.has_feature_keys)
|
|
182
|
+
self._add_list('features.value:"%s"', self.has_feature_values)
|
|
183
|
+
|
|
184
|
+
# Binary Info
|
|
185
|
+
self._add_if_not_none("size:>%s", self.greater_than_size_bytes)
|
|
186
|
+
self._add_if_not_none("size:<%s", self.less_than_size_bytes)
|
|
187
|
+
|
|
188
|
+
self._add_list('file_format_legacy:"%s"', self.file_formats_legacy)
|
|
189
|
+
self._add_list('!file_format_legacy:"%s"', self.file_formats_legacy_exclude, negation=True)
|
|
190
|
+
|
|
191
|
+
self._add_list('file_format:"%s"', self.file_formats)
|
|
192
|
+
self._add_list('!file_format:"%s"', self.file_formats_exclude, negation=True)
|
|
193
|
+
|
|
194
|
+
# Tags
|
|
195
|
+
self._add_list('binary.tag:"%s"', self.binary_tags)
|
|
196
|
+
self._add_list('feature.tag:"%s"', self.feature_tags)
|
|
197
|
+
|
|
198
|
+
return self._query
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class BinariesMeta(BaseApiHandler):
|
|
202
|
+
"""Interact with binary endpoints."""
|
|
203
|
+
|
|
204
|
+
SHA256_regex = r"^[a-fA-F0-9]{64}$"
|
|
205
|
+
upload_download_timeout = 120
|
|
206
|
+
|
|
207
|
+
def __init__(self, cfg: config.Config, get_client: Callable[[], httpx.Client]) -> None:
|
|
208
|
+
"""Init."""
|
|
209
|
+
super().__init__(cfg, get_client)
|
|
210
|
+
|
|
211
|
+
def check_meta(self, sha256: str) -> bool:
|
|
212
|
+
"""Check metadata exists for hash."""
|
|
213
|
+
return self._generic_head_request(self.cfg.azul_url + f"/api/v0/binaries/{sha256}")
|
|
214
|
+
|
|
215
|
+
def get_meta(
|
|
216
|
+
self,
|
|
217
|
+
sha256: str,
|
|
218
|
+
*,
|
|
219
|
+
details: list[models_restapi.BinaryMetadataDetail] | None = None,
|
|
220
|
+
author: str | None = None,
|
|
221
|
+
bucket_size: int = 100,
|
|
222
|
+
) -> models_restapi.BinaryMetadata:
|
|
223
|
+
"""Get metadata for hash.
|
|
224
|
+
|
|
225
|
+
:param str sha256: sha256 of the binary to get metadata for.
|
|
226
|
+
:param bool detail: Set to True to get detailed information about the binary including
|
|
227
|
+
children, features, streams, parents etc.
|
|
228
|
+
:param int bucket_size: Edit bucket size to get data if a query overflows the current bucket count
|
|
229
|
+
(Buckets this affects are Features, Info, Streams(data) and Instances(Authors)).
|
|
230
|
+
"""
|
|
231
|
+
params = {"detail": details, "author": author, "bucket_size": bucket_size}
|
|
232
|
+
for k in list(params.keys()):
|
|
233
|
+
if params[k] is None:
|
|
234
|
+
params.pop(k)
|
|
235
|
+
return self._request_with_pydantic_model_response(
|
|
236
|
+
method=HTTPMethod.GET,
|
|
237
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}",
|
|
238
|
+
response_model=models_restapi.BinaryMetadata,
|
|
239
|
+
params=params,
|
|
240
|
+
get_data_only=True,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
def _base_find(
|
|
244
|
+
self,
|
|
245
|
+
*,
|
|
246
|
+
term: str | None = None,
|
|
247
|
+
max_entities: int | None = None,
|
|
248
|
+
count_entities: bool | None = None,
|
|
249
|
+
hashes: list[str] | None = None,
|
|
250
|
+
sort_prop: models_restapi.FindBinariesSortEnum | None = None,
|
|
251
|
+
sort_asc: bool | None = None,
|
|
252
|
+
) -> models_restapi.EntityFind:
|
|
253
|
+
params = dict(
|
|
254
|
+
term=term,
|
|
255
|
+
max_entities=max_entities,
|
|
256
|
+
count_entities=count_entities,
|
|
257
|
+
sort=sort_prop,
|
|
258
|
+
sort_asc=sort_asc,
|
|
259
|
+
)
|
|
260
|
+
params = self.filter_none_values(params)
|
|
261
|
+
|
|
262
|
+
if not hashes:
|
|
263
|
+
hashes = []
|
|
264
|
+
return self._request_with_pydantic_model_response(
|
|
265
|
+
method=HTTPMethod.POST,
|
|
266
|
+
url=self.cfg.azul_url + "/api/v0/binaries",
|
|
267
|
+
params=params,
|
|
268
|
+
json={"hashes": hashes},
|
|
269
|
+
response_model=models_restapi.EntityFind,
|
|
270
|
+
get_data_only=True,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
def find(
|
|
274
|
+
self,
|
|
275
|
+
term: str | None,
|
|
276
|
+
*,
|
|
277
|
+
max_entities: int | None = None,
|
|
278
|
+
count_entities: bool | None = None,
|
|
279
|
+
sort_prop: models_restapi.FindBinariesSortEnum | None = None,
|
|
280
|
+
sort_asc: bool | None = None,
|
|
281
|
+
) -> models_restapi.EntityFind:
|
|
282
|
+
"""Find binaries matching the term query, limiting to the max_entities."""
|
|
283
|
+
return self._base_find(
|
|
284
|
+
term=term, max_entities=max_entities, count_entities=count_entities, sort_prop=sort_prop, sort_asc=sort_asc
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
def find_simple(
|
|
288
|
+
self,
|
|
289
|
+
find_options: FindOptions,
|
|
290
|
+
*,
|
|
291
|
+
max_entities: int | None = None,
|
|
292
|
+
count_entities: bool | None = None,
|
|
293
|
+
sort_prop: models_restapi.FindBinariesSortEnum | None = None,
|
|
294
|
+
sort_asc: bool | None = None,
|
|
295
|
+
) -> models_restapi.EntityFind:
|
|
296
|
+
"""Find binaries matching the term query, limiting to the max_entities."""
|
|
297
|
+
return self._base_find(
|
|
298
|
+
term=find_options.to_query(),
|
|
299
|
+
max_entities=max_entities,
|
|
300
|
+
count_entities=count_entities,
|
|
301
|
+
sort_prop=sort_prop,
|
|
302
|
+
sort_asc=sort_asc,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
def find_hashes(self, hashes: list[str]) -> models_restapi.EntityFind:
|
|
306
|
+
"""Check if a list of hashes are in Azul and returns their basic summary information."""
|
|
307
|
+
return self._base_find(hashes=hashes)
|
|
308
|
+
|
|
309
|
+
@dataclass
|
|
310
|
+
class FindAll:
|
|
311
|
+
"""Result of a find_all query, with iterator to look at each binary."""
|
|
312
|
+
|
|
313
|
+
approx_total: int = 0
|
|
314
|
+
iter: Generator[models_restapi.EntityFindSimpleItem, None, None] = None
|
|
315
|
+
|
|
316
|
+
def __iter__(self):
|
|
317
|
+
"""Iterator over the default iterator of iter."""
|
|
318
|
+
return self.iter
|
|
319
|
+
|
|
320
|
+
def find_all(
|
|
321
|
+
self,
|
|
322
|
+
find_options: FindOptions,
|
|
323
|
+
*,
|
|
324
|
+
max_binaries: int = 0,
|
|
325
|
+
request_binaries: int = 5000,
|
|
326
|
+
) -> FindAll:
|
|
327
|
+
"""Find all binaries matching the term query, returned via a generator."""
|
|
328
|
+
if max_binaries and max_binaries < request_binaries:
|
|
329
|
+
request_binaries = max_binaries
|
|
330
|
+
|
|
331
|
+
params = dict(
|
|
332
|
+
term=find_options.to_query(),
|
|
333
|
+
numodels_restapi=request_binaries,
|
|
334
|
+
)
|
|
335
|
+
params = self.filter_none_values(params)
|
|
336
|
+
resp: models_restapi.EntityFindSimple = self._request_with_pydantic_model_response(
|
|
337
|
+
method=HTTPMethod.POST,
|
|
338
|
+
url=self.cfg.azul_url + "/api/v0/binaries/all",
|
|
339
|
+
params=params,
|
|
340
|
+
json={"after": None},
|
|
341
|
+
response_model=models_restapi.EntityFindSimple,
|
|
342
|
+
get_data_only=True,
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
def _iterate_binaries(params: dict, resp: models_restapi.EntityFindSimple):
|
|
346
|
+
"""Iterate over the found binaries."""
|
|
347
|
+
after = resp.after
|
|
348
|
+
found = 0
|
|
349
|
+
while True:
|
|
350
|
+
after = resp.after
|
|
351
|
+
if len(resp.items) == 0 or not after:
|
|
352
|
+
return
|
|
353
|
+
for item in resp.items:
|
|
354
|
+
yield item
|
|
355
|
+
found += 1
|
|
356
|
+
if max_binaries and found >= max_binaries:
|
|
357
|
+
# quit even if we have more than requested that we can supply
|
|
358
|
+
return
|
|
359
|
+
resp: models_restapi.EntityFindSimple = self._request_with_pydantic_model_response(
|
|
360
|
+
method=HTTPMethod.POST,
|
|
361
|
+
url=self.cfg.azul_url + "/api/v0/binaries/all",
|
|
362
|
+
params=params,
|
|
363
|
+
json={"after": after},
|
|
364
|
+
response_model=models_restapi.EntityFindSimple,
|
|
365
|
+
get_data_only=True,
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
return BinariesMeta.FindAll(
|
|
369
|
+
approx_total=resp.total if resp.total else 0,
|
|
370
|
+
iter=_iterate_binaries(params, resp) if len(resp.items) > 0 else iter([]),
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
def get_model(self) -> models_restapi.EntityModel:
|
|
374
|
+
"""Return the model for the binaries."""
|
|
375
|
+
return self._request_with_pydantic_model_response(
|
|
376
|
+
method=HTTPMethod.GET,
|
|
377
|
+
url=self.cfg.azul_url + "/api/v0/binaries/model",
|
|
378
|
+
response_model=models_restapi.EntityModel,
|
|
379
|
+
get_data_only=True,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
def find_autocomplete(self, term: str) -> models_restapi.AutocompleteContext:
|
|
383
|
+
"""Looks for potential auto-completes for a search term and returns them.
|
|
384
|
+
|
|
385
|
+
:param str term: Term to try and auto-complete.
|
|
386
|
+
"""
|
|
387
|
+
# NOTE - offset is auto calculated as it's used for cursor position which makes no sense
|
|
388
|
+
# from a client API perspective.
|
|
389
|
+
|
|
390
|
+
return self._request_with_pydantic_model_response(
|
|
391
|
+
method=HTTPMethod.GET,
|
|
392
|
+
url=self.cfg.azul_url + "/api/v0/binaries/autocomplete",
|
|
393
|
+
response_model=autocomplete_type_adapter,
|
|
394
|
+
params={"term": term, "offset": len(term) - 1},
|
|
395
|
+
get_data_only=True,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
def get_has_newer_metadata(self, sha256: str, timestamp: str) -> models_restapi.BinaryDocuments:
|
|
399
|
+
"""Check if a binary has data newer than the provided timestamp in ISO8601 format."""
|
|
400
|
+
return self._request_with_pydantic_model_response(
|
|
401
|
+
method=HTTPMethod.GET,
|
|
402
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/new",
|
|
403
|
+
response_model=models_restapi.BinaryDocuments,
|
|
404
|
+
params={"timestamp": timestamp},
|
|
405
|
+
get_data_only=True,
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
def get_similar_ssdeep_entities(self, ssdeep: str, *, max_matches: int = 20) -> models_restapi.SimilarFuzzyMatch:
|
|
409
|
+
"""Return id and similarity score of entities with a similar ssdeep fuzzyhash."""
|
|
410
|
+
if not ssdeep:
|
|
411
|
+
raise ValueError("ssdeep must be set to something.")
|
|
412
|
+
return self._request_with_pydantic_model_response(
|
|
413
|
+
method=HTTPMethod.GET,
|
|
414
|
+
url=self.cfg.azul_url + "/api/v0/binaries/similar/ssdeep",
|
|
415
|
+
response_model=models_restapi.SimilarFuzzyMatch,
|
|
416
|
+
params={"ssdeep": ssdeep, "max_matches": max_matches},
|
|
417
|
+
get_data_only=True,
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
def get_similar_tlsh_entities(self, tlsh: str, *, max_matches: int = 20) -> models_restapi.SimilarFuzzyMatch:
|
|
421
|
+
"""Return id and similarity score of entities with a similar tlsh fuzzyhash."""
|
|
422
|
+
if not tlsh:
|
|
423
|
+
raise ValueError("tlsh must be set to something.")
|
|
424
|
+
return self._request_with_pydantic_model_response(
|
|
425
|
+
method=HTTPMethod.GET,
|
|
426
|
+
url=self.cfg.azul_url + "/api/v0/binaries/similar/tlsh",
|
|
427
|
+
response_model=models_restapi.SimilarFuzzyMatch,
|
|
428
|
+
params={"tlsh": tlsh, "max_matches": max_matches},
|
|
429
|
+
get_data_only=True,
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
def get_similar_entities(self, sha256: str, *, recalculate: bool = False) -> models_restapi.SimilarMatch:
|
|
433
|
+
"""Return information about similar entities."""
|
|
434
|
+
return self._request_with_pydantic_model_response(
|
|
435
|
+
method=HTTPMethod.GET,
|
|
436
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/similar",
|
|
437
|
+
response_model=models_restapi.SimilarMatch,
|
|
438
|
+
params={"recalculate": recalculate},
|
|
439
|
+
get_data_only=True,
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
def get_nearby_entities(
|
|
443
|
+
self,
|
|
444
|
+
sha256: str,
|
|
445
|
+
*,
|
|
446
|
+
include_cousins: models_restapi.IncludeCousinsEnum = models_restapi.IncludeCousinsEnum.Standard,
|
|
447
|
+
) -> models_restapi.ReadNearby:
|
|
448
|
+
"""Get information about nearby entities (used to build relational tree graph)."""
|
|
449
|
+
return self._request_with_pydantic_model_response(
|
|
450
|
+
method=HTTPMethod.GET,
|
|
451
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/nearby",
|
|
452
|
+
response_model=models_restapi.ReadNearby,
|
|
453
|
+
params={"include_cousins": include_cousins.value},
|
|
454
|
+
get_data_only=True,
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
def get_binary_tags(self, sha256: str) -> models_restapi.ReadAllEntityTags:
|
|
458
|
+
"""Return all tags for an binary."""
|
|
459
|
+
return self._request_with_pydantic_model_response(
|
|
460
|
+
method=HTTPMethod.GET,
|
|
461
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/tags",
|
|
462
|
+
response_model=models_restapi.ReadAllEntityTags,
|
|
463
|
+
get_data_only=True,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
def create_tag_on_binary(self, sha256: str, tag: str, security: str) -> None:
|
|
467
|
+
"""Attach a tag to the provided binaries sha256."""
|
|
468
|
+
return self._request(
|
|
469
|
+
method=HTTPMethod.POST,
|
|
470
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/tags/{tag}",
|
|
471
|
+
json={"security": security},
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
def delete_tag_on_binary(self, sha256: str, tag: str) -> models_restapi.AnnotationUpdated:
|
|
475
|
+
"""Delete the specified tag from the specified binary."""
|
|
476
|
+
return self._request_with_pydantic_model_response(
|
|
477
|
+
method=HTTPMethod.DELETE,
|
|
478
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/tags/{tag}",
|
|
479
|
+
response_model=models_restapi.AnnotationUpdated,
|
|
480
|
+
get_data_only=True,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
def get_binary_status(self, sha256: str) -> models_restapi.Status:
|
|
484
|
+
"""Get the plugin statuses for a binary."""
|
|
485
|
+
return self._request_with_pydantic_model_response(
|
|
486
|
+
method=HTTPMethod.GET,
|
|
487
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/statuses",
|
|
488
|
+
response_model=models_restapi.Status,
|
|
489
|
+
get_data_only=True,
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
def get_binary_documents(
|
|
493
|
+
self, sha256: str, *, action: azm.BinaryAction | None = None, size: int = 1000
|
|
494
|
+
) -> models_restapi.OpensearchDocuments:
|
|
495
|
+
"""Get opensearch documents for a binary.
|
|
496
|
+
|
|
497
|
+
:param str sha256: The sha256 of the binary to look for documents for.
|
|
498
|
+
:param BinaryAction action: The action to get events for.
|
|
499
|
+
:param int size: Maximum number of events that will be returned.
|
|
500
|
+
"""
|
|
501
|
+
params = {"event_type": action, "size": size}
|
|
502
|
+
params = self.filter_none_values(params)
|
|
503
|
+
|
|
504
|
+
return self._request_with_pydantic_model_response(
|
|
505
|
+
method=HTTPMethod.GET,
|
|
506
|
+
url=self.cfg.azul_url + f"/api/v0/binaries/{sha256}/events",
|
|
507
|
+
response_model=models_restapi.OpensearchDocuments,
|
|
508
|
+
params=params,
|
|
509
|
+
get_data_only=True,
|
|
510
|
+
)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Mappings for the Azul features API."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from http import HTTPMethod
|
|
5
|
+
|
|
6
|
+
from azul_bedrock import models_restapi
|
|
7
|
+
from pydantic import TypeAdapter
|
|
8
|
+
|
|
9
|
+
from azul_client.api.base_api import BaseApiHandler
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
feature_type_count_adapter = TypeAdapter(dict[str, models_restapi.FeatureMulticountRet])
|
|
14
|
+
entities_in_features_count_adapter = TypeAdapter(dict[str, dict[str, models_restapi.ValueCountRet]])
|
|
15
|
+
entities_in_featurevalueparts_count_adapter = TypeAdapter(dict[str, dict[str, models_restapi.ValuePartCountRet]])
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Features(BaseApiHandler):
|
|
19
|
+
"""API for counting, and requesting features and their values from Azul."""
|
|
20
|
+
|
|
21
|
+
def count_unique_values_in_feature(
|
|
22
|
+
self, items: list[str], *, skip_count: bool = False, author: str = "", author_version: str = ""
|
|
23
|
+
) -> dict[str, models_restapi.FeatureMulticountRet]:
|
|
24
|
+
"""Count number of unique values for provided feature(s)."""
|
|
25
|
+
return self._request_with_pydantic_model_response(
|
|
26
|
+
url=self.cfg.azul_url + "/api/v0/features/values/counts",
|
|
27
|
+
method=HTTPMethod.POST,
|
|
28
|
+
response_model=feature_type_count_adapter,
|
|
29
|
+
params={
|
|
30
|
+
"skip_count": skip_count,
|
|
31
|
+
"author": author,
|
|
32
|
+
"author_version": author_version,
|
|
33
|
+
},
|
|
34
|
+
json={"items": items},
|
|
35
|
+
get_data_only=True,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
def count_unique_entities_in_features(
|
|
39
|
+
self, items: list[str], *, skip_count: bool = False, author: str = "", author_version: str = ""
|
|
40
|
+
) -> dict[str, models_restapi.FeatureMulticountRet]:
|
|
41
|
+
"""Count number of unique entities for provided feature(s)."""
|
|
42
|
+
return self._request_with_pydantic_model_response(
|
|
43
|
+
url=self.cfg.azul_url + "/api/v0/features/entities/counts",
|
|
44
|
+
method=HTTPMethod.POST,
|
|
45
|
+
response_model=feature_type_count_adapter,
|
|
46
|
+
params={
|
|
47
|
+
"skip_count": skip_count,
|
|
48
|
+
"author": author,
|
|
49
|
+
"author_version": author_version,
|
|
50
|
+
},
|
|
51
|
+
json={"items": items},
|
|
52
|
+
get_data_only=True,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def count_unique_entities_in_featurevalues(
|
|
56
|
+
self,
|
|
57
|
+
items: list[models_restapi.ValueCountItem],
|
|
58
|
+
*,
|
|
59
|
+
skip_count: bool = False,
|
|
60
|
+
author: str = "",
|
|
61
|
+
author_version: str = "",
|
|
62
|
+
) -> dict[str, dict[str, models_restapi.ValueCountRet]]:
|
|
63
|
+
"""Count unique entities for multiple feature values."""
|
|
64
|
+
return self._request_with_pydantic_model_response(
|
|
65
|
+
url=self.cfg.azul_url + "/api/v0/features/values/entities/counts",
|
|
66
|
+
method=HTTPMethod.POST,
|
|
67
|
+
response_model=entities_in_features_count_adapter,
|
|
68
|
+
params={
|
|
69
|
+
"skip_count": skip_count,
|
|
70
|
+
"author": author,
|
|
71
|
+
"author_version": author_version,
|
|
72
|
+
},
|
|
73
|
+
json={"items": [x.model_dump() for x in items]},
|
|
74
|
+
get_data_only=True,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def count_unique_entities_in_featurevalueparts(
|
|
78
|
+
self,
|
|
79
|
+
items: list[models_restapi.ValuePartCountItem],
|
|
80
|
+
*,
|
|
81
|
+
skip_count: bool = False,
|
|
82
|
+
author: str = "",
|
|
83
|
+
author_version: str = "",
|
|
84
|
+
) -> dict[str, dict[str, models_restapi.ValuePartCountRet]]:
|
|
85
|
+
"""Count unique entities for multiple value parts."""
|
|
86
|
+
return self._request_with_pydantic_model_response(
|
|
87
|
+
url=self.cfg.azul_url + "/api/v0/features/values/parts/entities/counts",
|
|
88
|
+
method=HTTPMethod.POST,
|
|
89
|
+
response_model=entities_in_featurevalueparts_count_adapter,
|
|
90
|
+
params={
|
|
91
|
+
"skip_count": skip_count,
|
|
92
|
+
"author": author,
|
|
93
|
+
"author_version": author_version,
|
|
94
|
+
},
|
|
95
|
+
json={"items": [x.model_dump() for x in items]},
|
|
96
|
+
get_data_only=True,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
def get_all_feature_value_tags(self) -> models_restapi.ReadFeatureValueTags:
|
|
100
|
+
"""Get a list and count of all feature value tags."""
|
|
101
|
+
return self._request_with_pydantic_model_response(
|
|
102
|
+
url=self.cfg.azul_url + "/api/v0/features/all/tags",
|
|
103
|
+
method=HTTPMethod.GET,
|
|
104
|
+
response_model=models_restapi.ReadFeatureValueTags,
|
|
105
|
+
get_data_only=True,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def get_feature_values_in_tag(self, tag: str) -> models_restapi.ReadFeatureTagValues:
|
|
109
|
+
"""Get feature values that are tagged with the provided tag."""
|
|
110
|
+
return self._request_with_pydantic_model_response(
|
|
111
|
+
url=self.cfg.azul_url + f"/api/v0/features/tags/{tag}",
|
|
112
|
+
method=HTTPMethod.GET,
|
|
113
|
+
response_model=models_restapi.ReadFeatureTagValues,
|
|
114
|
+
get_data_only=True,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def create_feature_value_tag(self, tag: str, feature: str, value: str, security: str) -> bool:
|
|
118
|
+
"""Create a feature value tag."""
|
|
119
|
+
return self._request(
|
|
120
|
+
method=HTTPMethod.POST,
|
|
121
|
+
url=self.cfg.azul_url + f"/api/v0/features/tags/{tag}",
|
|
122
|
+
params={"feature": feature, "value": value},
|
|
123
|
+
json={"security": security},
|
|
124
|
+
).json()
|
|
125
|
+
|
|
126
|
+
def delete_feature_value_tag(self, tag: str, feature: str, value: str) -> bool:
|
|
127
|
+
"""Delete a feature value tag."""
|
|
128
|
+
return self._request(
|
|
129
|
+
method=HTTPMethod.DELETE,
|
|
130
|
+
url=self.cfg.azul_url + f"/api/v0/features/tags/{tag}",
|
|
131
|
+
params={"feature": feature, "value": value},
|
|
132
|
+
).json()
|
|
133
|
+
|
|
134
|
+
def find_features(self, *, author: str = "", author_version: str = "") -> models_restapi.Features:
|
|
135
|
+
"""Find all features optionally selecting a specific author and author_version to search for."""
|
|
136
|
+
return self._request_with_pydantic_model_response(
|
|
137
|
+
url=self.cfg.azul_url + "/api/v0/features",
|
|
138
|
+
method=HTTPMethod.GET,
|
|
139
|
+
response_model=models_restapi.Features,
|
|
140
|
+
params={"author": author, "author_version": author_version},
|
|
141
|
+
get_data_only=True,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def find_values_in_feature(
|
|
145
|
+
self,
|
|
146
|
+
feature: str,
|
|
147
|
+
*,
|
|
148
|
+
term: str = "",
|
|
149
|
+
sort_asc: bool = True,
|
|
150
|
+
case_insensitive: bool = False,
|
|
151
|
+
author: str = "",
|
|
152
|
+
author_version: str = "",
|
|
153
|
+
num_values: int = 500,
|
|
154
|
+
after: str = "",
|
|
155
|
+
) -> models_restapi.ReadFeatureValues:
|
|
156
|
+
"""Find all the values associated with a feature.
|
|
157
|
+
|
|
158
|
+
Note - After should only ever be set when you want to paginate and you get the after value from the previous
|
|
159
|
+
query response.
|
|
160
|
+
"""
|
|
161
|
+
return self._request_with_pydantic_model_response(
|
|
162
|
+
url=self.cfg.azul_url + f"/api/v0/features/feature/{feature}",
|
|
163
|
+
method=HTTPMethod.POST,
|
|
164
|
+
response_model=models_restapi.ReadFeatureValues,
|
|
165
|
+
params={
|
|
166
|
+
"term": term,
|
|
167
|
+
"sort_asc": sort_asc,
|
|
168
|
+
"case_insensitive": case_insensitive,
|
|
169
|
+
"author": author,
|
|
170
|
+
"author_version": author_version,
|
|
171
|
+
"num_values": num_values,
|
|
172
|
+
},
|
|
173
|
+
json={"after": after},
|
|
174
|
+
get_data_only=True,
|
|
175
|
+
)
|