pyoaev 1.18.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/conf.py +65 -0
- pyoaev/__init__.py +26 -0
- pyoaev/_version.py +6 -0
- pyoaev/apis/__init__.py +20 -0
- pyoaev/apis/attack_pattern.py +28 -0
- pyoaev/apis/collector.py +29 -0
- pyoaev/apis/cve.py +18 -0
- pyoaev/apis/document.py +29 -0
- pyoaev/apis/endpoint.py +38 -0
- pyoaev/apis/inject.py +29 -0
- pyoaev/apis/inject_expectation/__init__.py +1 -0
- pyoaev/apis/inject_expectation/inject_expectation.py +118 -0
- pyoaev/apis/inject_expectation/model/__init__.py +7 -0
- pyoaev/apis/inject_expectation/model/expectation.py +173 -0
- pyoaev/apis/inject_expectation_trace.py +36 -0
- pyoaev/apis/injector.py +26 -0
- pyoaev/apis/injector_contract.py +56 -0
- pyoaev/apis/inputs/__init__.py +0 -0
- pyoaev/apis/inputs/search.py +72 -0
- pyoaev/apis/kill_chain_phase.py +22 -0
- pyoaev/apis/me.py +17 -0
- pyoaev/apis/organization.py +11 -0
- pyoaev/apis/payload.py +27 -0
- pyoaev/apis/security_platform.py +33 -0
- pyoaev/apis/tag.py +19 -0
- pyoaev/apis/team.py +25 -0
- pyoaev/apis/user.py +31 -0
- pyoaev/backends/__init__.py +14 -0
- pyoaev/backends/backend.py +136 -0
- pyoaev/backends/protocol.py +32 -0
- pyoaev/base.py +320 -0
- pyoaev/client.py +596 -0
- pyoaev/configuration/__init__.py +3 -0
- pyoaev/configuration/configuration.py +188 -0
- pyoaev/configuration/sources.py +44 -0
- pyoaev/contracts/__init__.py +5 -0
- pyoaev/contracts/contract_builder.py +44 -0
- pyoaev/contracts/contract_config.py +292 -0
- pyoaev/contracts/contract_utils.py +22 -0
- pyoaev/contracts/variable_helper.py +124 -0
- pyoaev/daemons/__init__.py +4 -0
- pyoaev/daemons/base_daemon.py +131 -0
- pyoaev/daemons/collector_daemon.py +91 -0
- pyoaev/exceptions.py +219 -0
- pyoaev/helpers.py +451 -0
- pyoaev/mixins.py +242 -0
- pyoaev/signatures/__init__.py +0 -0
- pyoaev/signatures/signature_match.py +12 -0
- pyoaev/signatures/signature_type.py +51 -0
- pyoaev/signatures/types.py +17 -0
- pyoaev/utils.py +211 -0
- pyoaev-1.18.20.dist-info/METADATA +134 -0
- pyoaev-1.18.20.dist-info/RECORD +72 -0
- pyoaev-1.18.20.dist-info/WHEEL +5 -0
- pyoaev-1.18.20.dist-info/licenses/LICENSE +201 -0
- pyoaev-1.18.20.dist-info/top_level.txt +4 -0
- scripts/release.py +127 -0
- test/__init__.py +0 -0
- test/apis/__init__.py +0 -0
- test/apis/expectation/__init__.py +0 -0
- test/apis/expectation/test_expectation.py +338 -0
- test/apis/injector_contract/__init__.py +0 -0
- test/apis/injector_contract/test_injector_contract.py +58 -0
- test/configuration/__init__.py +0 -0
- test/configuration/test_configuration.py +257 -0
- test/configuration/test_sources.py +69 -0
- test/daemons/__init__.py +0 -0
- test/daemons/test_base_daemon.py +109 -0
- test/daemons/test_collector_daemon.py +39 -0
- test/signatures/__init__.py +0 -0
- test/signatures/test_signature_match.py +25 -0
- test/signatures/test_signature_type.py +57 -0
pyoaev/helpers.py
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import os.path
|
|
5
|
+
import re
|
|
6
|
+
import sched
|
|
7
|
+
import ssl
|
|
8
|
+
import tempfile
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
import traceback
|
|
12
|
+
from typing import Callable, Dict, List
|
|
13
|
+
|
|
14
|
+
import pika
|
|
15
|
+
from thefuzz import fuzz
|
|
16
|
+
|
|
17
|
+
from pyoaev import OpenAEV, utils
|
|
18
|
+
from pyoaev.configuration import Configuration
|
|
19
|
+
from pyoaev.daemons import CollectorDaemon
|
|
20
|
+
from pyoaev.exceptions import ConfigurationError
|
|
21
|
+
|
|
22
|
+
TRUTHY: List[str] = ["yes", "true", "True"]
|
|
23
|
+
FALSY: List[str] = ["no", "false", "False"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# As cert must be written in files to be loaded in ssl context
|
|
27
|
+
# Creates a temporary file in the most secure manner possible
|
|
28
|
+
def data_to_temp_file(data):
|
|
29
|
+
# The file is readable and writable only by the creating user ID.
|
|
30
|
+
# If the operating system uses permission bits to indicate whether a
|
|
31
|
+
# file is executable, the file is executable by no one. The file
|
|
32
|
+
# descriptor is not inherited by children of this process.
|
|
33
|
+
file_descriptor, file_path = tempfile.mkstemp()
|
|
34
|
+
with os.fdopen(file_descriptor, "w") as open_file:
|
|
35
|
+
open_file.write(data)
|
|
36
|
+
open_file.close()
|
|
37
|
+
return file_path
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def is_memory_certificate(certificate):
|
|
41
|
+
return certificate.startswith("-----BEGIN")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def ssl_cert_chain(ssl_context, cert_data, key_data, passphrase):
|
|
45
|
+
if cert_data is None:
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
cert_file_path = None
|
|
49
|
+
key_file_path = None
|
|
50
|
+
|
|
51
|
+
# Cert loading
|
|
52
|
+
if cert_data is not None and is_memory_certificate(cert_data):
|
|
53
|
+
cert_file_path = data_to_temp_file(cert_data)
|
|
54
|
+
cert = cert_file_path if cert_file_path is not None else cert_data
|
|
55
|
+
|
|
56
|
+
# Key loading
|
|
57
|
+
if key_data is not None and is_memory_certificate(key_data):
|
|
58
|
+
key_file_path = data_to_temp_file(key_data)
|
|
59
|
+
key = key_file_path if key_file_path is not None else key_data
|
|
60
|
+
|
|
61
|
+
# Load cert
|
|
62
|
+
ssl_context.load_cert_chain(cert, key, passphrase)
|
|
63
|
+
# Remove temp files
|
|
64
|
+
if cert_file_path is not None:
|
|
65
|
+
os.unlink(cert_file_path)
|
|
66
|
+
if key_file_path is not None:
|
|
67
|
+
os.unlink(key_file_path)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def ssl_verify_locations(ssl_context, certdata):
|
|
71
|
+
if certdata is None:
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
if is_memory_certificate(certdata):
|
|
75
|
+
ssl_context.load_verify_locations(cadata=certdata)
|
|
76
|
+
else:
|
|
77
|
+
ssl_context.load_verify_locations(cafile=certdata)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def create_mq_ssl_context(config) -> ssl.SSLContext:
|
|
81
|
+
config_obj = Configuration(
|
|
82
|
+
config_hints={
|
|
83
|
+
"MQ_USE_SSL_CA": {
|
|
84
|
+
"env": "MQ_USE_SSL_CA",
|
|
85
|
+
"file_path": ["mq", "use_ssl_ca"],
|
|
86
|
+
},
|
|
87
|
+
"MQ_USE_SSL_CERT": {
|
|
88
|
+
"env": "MQ_USE_SSL_CERT",
|
|
89
|
+
"file_path": ["mq", "use_ssl_cert"],
|
|
90
|
+
},
|
|
91
|
+
"MQ_USE_SSL_KEY": {
|
|
92
|
+
"env": "MQ_USE_SSL_KEY",
|
|
93
|
+
"file_path": ["mq", "use_ssl_key"],
|
|
94
|
+
},
|
|
95
|
+
"MQ_USE_SSL_REJECT_UNAUTHORIZED": {
|
|
96
|
+
"env": "MQ_USE_SSL_REJECT_UNAUTHORIZED",
|
|
97
|
+
"file_path": ["mq", "use_ssl_reject_unauthorized"],
|
|
98
|
+
"is_number": False,
|
|
99
|
+
"default": False,
|
|
100
|
+
},
|
|
101
|
+
"MQ_USE_SSL_PASSPHRASE": {
|
|
102
|
+
"env": "MQ_USE_SSL_PASSPHRASE",
|
|
103
|
+
"file_path": ["mq", "use_ssl_passphrase"],
|
|
104
|
+
},
|
|
105
|
+
},
|
|
106
|
+
config_values=config,
|
|
107
|
+
)
|
|
108
|
+
use_ssl_ca = config_obj.get("MQ_USE_SSL_CA")
|
|
109
|
+
use_ssl_cert = config_obj.get("MQ_USE_SSL_CERT")
|
|
110
|
+
use_ssl_key = config_obj.get("MQ_USE_SSL_KEY")
|
|
111
|
+
use_ssl_reject_unauthorized = config_obj.get("MQ_USE_SSL_REJECT_UNAUTHORIZED")
|
|
112
|
+
use_ssl_passphrase = config_obj.get("MQ_USE_SSL_PASSPHRASE")
|
|
113
|
+
ssl_context = ssl.create_default_context()
|
|
114
|
+
# If no rejection allowed, use private function to generate unverified context
|
|
115
|
+
if not use_ssl_reject_unauthorized:
|
|
116
|
+
# noinspection PyUnresolvedReferences,PyProtectedMember
|
|
117
|
+
ssl_context = ssl._create_unverified_context()
|
|
118
|
+
ssl_verify_locations(ssl_context, use_ssl_ca)
|
|
119
|
+
# Thanks to https://bugs.python.org/issue16487 is not possible today to easily use memory pem
|
|
120
|
+
# in SSL context. We need to write it to a temporary file before
|
|
121
|
+
ssl_cert_chain(ssl_context, use_ssl_cert, use_ssl_key, use_ssl_passphrase)
|
|
122
|
+
return ssl_context
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ListenQueue(threading.Thread):
|
|
126
|
+
def __init__(
|
|
127
|
+
self,
|
|
128
|
+
config: Dict,
|
|
129
|
+
injector_config,
|
|
130
|
+
logger,
|
|
131
|
+
callback,
|
|
132
|
+
) -> None:
|
|
133
|
+
threading.Thread.__init__(self)
|
|
134
|
+
self.pika_credentials = None
|
|
135
|
+
self.pika_parameters = None
|
|
136
|
+
self.pika_connection = None
|
|
137
|
+
self.channel = None
|
|
138
|
+
|
|
139
|
+
self.callback = callback
|
|
140
|
+
self.config = config
|
|
141
|
+
self.logger = logger
|
|
142
|
+
self.host = injector_config.connection["host"]
|
|
143
|
+
self.vhost = injector_config.connection["vhost"]
|
|
144
|
+
self.use_ssl = injector_config.connection["use_ssl"]
|
|
145
|
+
self.port = injector_config.connection["port"]
|
|
146
|
+
self.user = injector_config.connection["user"]
|
|
147
|
+
self.password = injector_config.connection["pass"]
|
|
148
|
+
self.queue_name = injector_config.listen
|
|
149
|
+
self.exit_event = threading.Event()
|
|
150
|
+
self.thread = None
|
|
151
|
+
|
|
152
|
+
# noinspection PyUnusedLocal
|
|
153
|
+
def _process_message(self, channel, method, properties, body) -> None:
|
|
154
|
+
"""process a message from the rabbit queue
|
|
155
|
+
|
|
156
|
+
:param channel: channel instance
|
|
157
|
+
:type channel: callable
|
|
158
|
+
:param method: message methods
|
|
159
|
+
:type method: callable
|
|
160
|
+
:param properties: unused
|
|
161
|
+
:type properties: str
|
|
162
|
+
:param body: message body (data)
|
|
163
|
+
:type body: str or bytes or bytearray
|
|
164
|
+
"""
|
|
165
|
+
json_data = json.loads(body)
|
|
166
|
+
# Message should be ack before processing as we don't own the processing
|
|
167
|
+
# Not ACK the message here may lead to infinite re-deliver if the connector is broken
|
|
168
|
+
# Also ACK, will not have any impact on the blocking aspect of the following functions
|
|
169
|
+
channel.basic_ack(delivery_tag=method.delivery_tag)
|
|
170
|
+
self.thread = threading.Thread(target=self._data_handler, args=[json_data])
|
|
171
|
+
self.thread.start()
|
|
172
|
+
|
|
173
|
+
def _data_handler(self, json_data) -> None:
|
|
174
|
+
self.callback(json_data)
|
|
175
|
+
|
|
176
|
+
def run(self) -> None:
|
|
177
|
+
self.logger.info("Starting ListenQueue thread")
|
|
178
|
+
while not self.exit_event.is_set():
|
|
179
|
+
try:
|
|
180
|
+
self.logger.info("ListenQueue connecting to RabbitMQ.")
|
|
181
|
+
# Connect the broker
|
|
182
|
+
self.pika_credentials = pika.PlainCredentials(self.user, self.password)
|
|
183
|
+
self.pika_parameters = pika.ConnectionParameters(
|
|
184
|
+
host=self.host,
|
|
185
|
+
port=self.port,
|
|
186
|
+
virtual_host=self.vhost,
|
|
187
|
+
credentials=self.pika_credentials,
|
|
188
|
+
ssl_options=(
|
|
189
|
+
pika.SSLOptions(create_mq_ssl_context(self.config), self.host)
|
|
190
|
+
if self.use_ssl
|
|
191
|
+
else None
|
|
192
|
+
),
|
|
193
|
+
)
|
|
194
|
+
self.pika_connection = pika.BlockingConnection(self.pika_parameters)
|
|
195
|
+
self.channel = self.pika_connection.channel()
|
|
196
|
+
try:
|
|
197
|
+
# confirm_delivery is only for cluster mode rabbitMQ
|
|
198
|
+
# when not in cluster mode this line raise an exception
|
|
199
|
+
self.channel.confirm_delivery()
|
|
200
|
+
except Exception as err: # pylint: disable=broad-except
|
|
201
|
+
self.logger.error(str(err))
|
|
202
|
+
self.channel.basic_qos(prefetch_count=1)
|
|
203
|
+
assert self.channel is not None
|
|
204
|
+
self.channel.basic_consume(
|
|
205
|
+
queue=self.queue_name, on_message_callback=self._process_message
|
|
206
|
+
)
|
|
207
|
+
self.channel.start_consuming()
|
|
208
|
+
except Exception: # pylint: disable=broad-except
|
|
209
|
+
try:
|
|
210
|
+
self.pika_connection.close()
|
|
211
|
+
except Exception as errInException:
|
|
212
|
+
self.logger.error(str(errInException))
|
|
213
|
+
traceback.print_exc()
|
|
214
|
+
# Wait some time and then retry ListenQueue again.
|
|
215
|
+
time.sleep(10)
|
|
216
|
+
|
|
217
|
+
def stop(self):
|
|
218
|
+
self.logger.info("Preparing ListenQueue for clean shutdown")
|
|
219
|
+
self.exit_event.set()
|
|
220
|
+
self.pika_connection.close()
|
|
221
|
+
if self.thread:
|
|
222
|
+
self.thread.join()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class PingAlive(utils.PingAlive):
|
|
226
|
+
pass
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
### DEPRECATED
|
|
230
|
+
class OpenAEVConfigHelper:
|
|
231
|
+
def __init__(self, base_path, variables: Dict):
|
|
232
|
+
self.__config_obj = Configuration(
|
|
233
|
+
config_hints=variables,
|
|
234
|
+
config_file_path=os.path.join(
|
|
235
|
+
os.path.dirname(os.path.abspath(base_path)), "config.yml"
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
def get_conf(self, variable, is_number=None, default=None, required=None):
|
|
240
|
+
result = None
|
|
241
|
+
try:
|
|
242
|
+
result = (
|
|
243
|
+
self.__config_obj.get(variable)
|
|
244
|
+
if (self.__config_obj.get(variable) is not None)
|
|
245
|
+
else default
|
|
246
|
+
)
|
|
247
|
+
except ConfigurationError:
|
|
248
|
+
result = default
|
|
249
|
+
finally:
|
|
250
|
+
if result is None and default is None and required:
|
|
251
|
+
raise ValueError(
|
|
252
|
+
f"Could not find required key {variable} with no available default."
|
|
253
|
+
)
|
|
254
|
+
return result
|
|
255
|
+
|
|
256
|
+
def to_configuration(self):
|
|
257
|
+
return self.__config_obj
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
### DEPRECATED
|
|
261
|
+
class OpenAEVCollectorHelper:
|
|
262
|
+
def __init__(
|
|
263
|
+
self,
|
|
264
|
+
config: OpenAEVConfigHelper,
|
|
265
|
+
icon,
|
|
266
|
+
collector_type=None,
|
|
267
|
+
security_platform_type=None,
|
|
268
|
+
connect_run_and_terminate: bool = False,
|
|
269
|
+
) -> None:
|
|
270
|
+
config_obj = config.to_configuration()
|
|
271
|
+
# ensure the icon path is set in config
|
|
272
|
+
config_obj.set("collector_icon_filepath", icon)
|
|
273
|
+
# override the platform in config if passed this way
|
|
274
|
+
if security_platform_type is not None:
|
|
275
|
+
config_obj.set("collector_platform", security_platform_type)
|
|
276
|
+
|
|
277
|
+
self.__daemon = CollectorDaemon(
|
|
278
|
+
configuration=config_obj,
|
|
279
|
+
callback=None,
|
|
280
|
+
collector_type=collector_type,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
self.__daemon.logger.warning(
|
|
284
|
+
f"DEPRECATED: this collector should be migrated to use {CollectorDaemon}."
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
self.logger_class = utils.logger(
|
|
288
|
+
config.get_conf("collector_log_level", default="error").upper(),
|
|
289
|
+
config.get_conf("collector_json_logging", default=True),
|
|
290
|
+
)
|
|
291
|
+
self.collector_logger = self.logger_class(config.get_conf("collector_name"))
|
|
292
|
+
self.api = self.__daemon.api
|
|
293
|
+
self.config_helper = config
|
|
294
|
+
self.config = {
|
|
295
|
+
"collector_id": config_obj.get("collector_id"),
|
|
296
|
+
"collector_name": config_obj.get("collector_name"),
|
|
297
|
+
"collector_type": collector_type,
|
|
298
|
+
"collector_period": config_obj.get("collector_period"),
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
def schedule(self, message_callback, delay):
|
|
302
|
+
# backwards compatibility: when older style call sets delay
|
|
303
|
+
# and no config exists,
|
|
304
|
+
if self.__daemon._configuration.get("collector_period") is None:
|
|
305
|
+
self.__daemon._configuration.set("collector_period", delay)
|
|
306
|
+
self.__daemon.set_callback(message_callback)
|
|
307
|
+
self.__daemon.start()
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
class OpenAEVInjectorHelper:
|
|
311
|
+
def __init__(self, config: OpenAEVConfigHelper, icon) -> None:
|
|
312
|
+
self.api = OpenAEV(
|
|
313
|
+
url=config.get_conf("openaev_url"),
|
|
314
|
+
token=config.get_conf("openaev_token"),
|
|
315
|
+
)
|
|
316
|
+
# Get the mq configuration from api
|
|
317
|
+
self.config = {
|
|
318
|
+
"injector_id": config.get_conf("injector_id"),
|
|
319
|
+
"injector_name": config.get_conf("injector_name"),
|
|
320
|
+
"injector_type": config.get_conf("injector_type"),
|
|
321
|
+
"injector_contracts": config.get_conf("injector_contracts"),
|
|
322
|
+
"injector_custom_contracts": config.get_conf(
|
|
323
|
+
"injector_custom_contracts", default=False
|
|
324
|
+
),
|
|
325
|
+
"injector_category": config.get_conf("injector_category", default=None),
|
|
326
|
+
"injector_executor_commands": config.get_conf(
|
|
327
|
+
"injector_executor_commands", default=None
|
|
328
|
+
),
|
|
329
|
+
"injector_executor_clear_commands": config.get_conf(
|
|
330
|
+
"injector_executor_clear_commands", default=None
|
|
331
|
+
),
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
self.logger_class = utils.logger(
|
|
335
|
+
config.get_conf("injector_log_level", default="error").upper(),
|
|
336
|
+
config.get_conf("injector_json_logging", default=True),
|
|
337
|
+
)
|
|
338
|
+
self.injector_logger = self.logger_class(config.get_conf("injector_name"))
|
|
339
|
+
|
|
340
|
+
icon_name = config.get_conf("injector_type") + ".png"
|
|
341
|
+
injector_icon = (icon_name, icon, "image/png")
|
|
342
|
+
self.injector_config = self.api.injector.create(self.config, injector_icon)
|
|
343
|
+
self.connect_run_and_terminate = False
|
|
344
|
+
self.scheduler = sched.scheduler(time.time, time.sleep)
|
|
345
|
+
# Start ping thread
|
|
346
|
+
if not self.connect_run_and_terminate:
|
|
347
|
+
self.ping = PingAlive(
|
|
348
|
+
self.api, self.config, self.injector_logger, "injector"
|
|
349
|
+
)
|
|
350
|
+
self.ping.start()
|
|
351
|
+
self.listen_queue = None
|
|
352
|
+
|
|
353
|
+
def listen(self, message_callback: Callable[[Dict], None]) -> None:
|
|
354
|
+
self.listen_queue = ListenQueue(
|
|
355
|
+
self.config, self.injector_config, self.injector_logger, message_callback
|
|
356
|
+
)
|
|
357
|
+
self.listen_queue.start()
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
class OpenAEVDetectionHelper:
|
|
361
|
+
def __init__(self, logger, relevant_signatures_types) -> None:
|
|
362
|
+
self.logger = logger
|
|
363
|
+
self.relevant_signatures_types = relevant_signatures_types
|
|
364
|
+
|
|
365
|
+
def match_alert_element_fuzzy(self, signature_value, alert_values, fuzzy_scoring):
|
|
366
|
+
for alert_value in alert_values:
|
|
367
|
+
self.logger.info(
|
|
368
|
+
"Comparing alert value (" + alert_value + ", " + signature_value + ")"
|
|
369
|
+
)
|
|
370
|
+
ratio = fuzz.ratio(alert_value, signature_value)
|
|
371
|
+
if ratio > fuzzy_scoring:
|
|
372
|
+
self.logger.info("MATCHING! (score: " + str(ratio) + ")")
|
|
373
|
+
return True
|
|
374
|
+
return False
|
|
375
|
+
|
|
376
|
+
def match_alert_elements(self, signatures, alert_data):
|
|
377
|
+
return self._match_alert_elements_original(
|
|
378
|
+
signatures, alert_data
|
|
379
|
+
) or self._match_alert_elements_for_command_line(signatures, alert_data)
|
|
380
|
+
|
|
381
|
+
def _match_alert_elements_original(self, signatures, alert_data):
|
|
382
|
+
# Example for alert_data
|
|
383
|
+
# {"process_name": {"list": ["xx", "yy"], "fuzzy": 90}}
|
|
384
|
+
relevant_signatures = [
|
|
385
|
+
s for s in signatures if s["type"] in self.relevant_signatures_types
|
|
386
|
+
]
|
|
387
|
+
|
|
388
|
+
# Matching logics
|
|
389
|
+
signatures_number = len(relevant_signatures)
|
|
390
|
+
matching_number = 0
|
|
391
|
+
for signature in relevant_signatures:
|
|
392
|
+
alert_data_for_signature = alert_data[signature["type"]]
|
|
393
|
+
signature_result = False
|
|
394
|
+
if alert_data_for_signature["type"] == "fuzzy":
|
|
395
|
+
signature_result = self.match_alert_element_fuzzy(
|
|
396
|
+
signature["value"],
|
|
397
|
+
alert_data_for_signature["data"],
|
|
398
|
+
alert_data_for_signature["score"],
|
|
399
|
+
)
|
|
400
|
+
elif alert_data_for_signature["type"] == "simple":
|
|
401
|
+
signature_result = signature["value"] in str(
|
|
402
|
+
alert_data_for_signature["data"]
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
if signature_result:
|
|
406
|
+
matching_number = matching_number + 1
|
|
407
|
+
|
|
408
|
+
if signatures_number == matching_number:
|
|
409
|
+
return True
|
|
410
|
+
return False
|
|
411
|
+
|
|
412
|
+
def _match_alert_elements_for_command_line(self, signatures, alert_data):
|
|
413
|
+
command_line_signatures = [
|
|
414
|
+
signature
|
|
415
|
+
for signature in signatures
|
|
416
|
+
if signature.get("type") == "command_line"
|
|
417
|
+
]
|
|
418
|
+
if len(command_line_signatures) == 0:
|
|
419
|
+
return False
|
|
420
|
+
key_types = ["command_line", "process_name", "file_name"]
|
|
421
|
+
alert_datas = [alert_data.get(key) for key in key_types if key in alert_data]
|
|
422
|
+
for signature in command_line_signatures:
|
|
423
|
+
signature_result = False
|
|
424
|
+
signature_value = self._decode_value(signature["value"]).strip().lower()
|
|
425
|
+
for alert_data in alert_datas:
|
|
426
|
+
trimmed_lowered_datas = [s.strip().lower() for s in alert_data["data"]]
|
|
427
|
+
signature_result = any(
|
|
428
|
+
data in signature_value for data in trimmed_lowered_datas
|
|
429
|
+
)
|
|
430
|
+
if signature_result:
|
|
431
|
+
return True
|
|
432
|
+
return False
|
|
433
|
+
|
|
434
|
+
def _decode_value(self, signature_value):
|
|
435
|
+
if _is_base64_encoded(signature_value):
|
|
436
|
+
try:
|
|
437
|
+
decoded_bytes = base64.b64decode(signature_value)
|
|
438
|
+
decoded_str = decoded_bytes.decode("utf-8")
|
|
439
|
+
return decoded_str
|
|
440
|
+
except Exception as e:
|
|
441
|
+
self.logger.error(str(e))
|
|
442
|
+
else:
|
|
443
|
+
return signature_value
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _is_base64_encoded(str_maybe_base64):
|
|
447
|
+
# Check if the length is a multiple of 4 and matches the Base64 character set
|
|
448
|
+
base64_pattern = re.compile(r"^[A-Za-z0-9+/]*={0,2}$")
|
|
449
|
+
return len(str_maybe_base64) % 4 == 0 and bool(
|
|
450
|
+
base64_pattern.match(str_maybe_base64)
|
|
451
|
+
)
|
pyoaev/mixins.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import enum
|
|
2
|
+
from typing import (
|
|
3
|
+
TYPE_CHECKING,
|
|
4
|
+
Any,
|
|
5
|
+
Callable,
|
|
6
|
+
Dict,
|
|
7
|
+
List,
|
|
8
|
+
Optional,
|
|
9
|
+
Tuple,
|
|
10
|
+
Type,
|
|
11
|
+
Union,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
import pyoaev
|
|
17
|
+
from pyoaev import base
|
|
18
|
+
from pyoaev import exceptions as exc
|
|
19
|
+
from pyoaev import utils
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"GetMixin",
|
|
23
|
+
"GetWithoutIdMixin",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
# When running mypy we use these as the base classes
|
|
28
|
+
_RestManagerBase = base.RESTManager
|
|
29
|
+
_RestObjectBase = base.RESTObject
|
|
30
|
+
else:
|
|
31
|
+
_RestManagerBase = object
|
|
32
|
+
_RestObjectBase = object
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class HeadMixin(_RestManagerBase):
|
|
36
|
+
@exc.on_http_error(exc.OpenAEVHeadError)
|
|
37
|
+
def head(
|
|
38
|
+
self, id: Optional[Union[str, int]] = None, **kwargs: Any
|
|
39
|
+
) -> "requests.structures.CaseInsensitiveDict[Any]":
|
|
40
|
+
if TYPE_CHECKING:
|
|
41
|
+
assert self.path is not None
|
|
42
|
+
|
|
43
|
+
path = self.path
|
|
44
|
+
if id is not None:
|
|
45
|
+
path = f"{path}/{utils.EncodedId(id)}"
|
|
46
|
+
|
|
47
|
+
return self.openaev.http_head(path, **kwargs)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class GetMixin(HeadMixin, _RestManagerBase):
|
|
51
|
+
_computed_path: Optional[str]
|
|
52
|
+
_from_parent_attrs: Dict[str, Any]
|
|
53
|
+
_obj_cls: Optional[Type[base.RESTObject]]
|
|
54
|
+
_optional_get_attrs: Tuple[str, ...] = ()
|
|
55
|
+
_parent: Optional[base.RESTObject]
|
|
56
|
+
_parent_attrs: Dict[str, Any]
|
|
57
|
+
_path: Optional[str]
|
|
58
|
+
openaev: pyoaev.OpenAEV
|
|
59
|
+
|
|
60
|
+
@exc.on_http_error(exc.OpenAEVGetError)
|
|
61
|
+
def get(self, id: Union[str, int], **kwargs: Any) -> base.RESTObject:
|
|
62
|
+
if isinstance(id, str):
|
|
63
|
+
id = utils.EncodedId(id)
|
|
64
|
+
path = f"{self.path}/{id}"
|
|
65
|
+
if TYPE_CHECKING:
|
|
66
|
+
assert self._obj_cls is not None
|
|
67
|
+
server_data = self.openaev.http_get(path, **kwargs)
|
|
68
|
+
if TYPE_CHECKING:
|
|
69
|
+
assert not isinstance(server_data, requests.Response)
|
|
70
|
+
return self._obj_cls(self, server_data)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class GetWithoutIdMixin(HeadMixin, _RestManagerBase):
|
|
74
|
+
_computed_path: Optional[str]
|
|
75
|
+
_from_parent_attrs: Dict[str, Any]
|
|
76
|
+
_obj_cls: Optional[Type[base.RESTObject]]
|
|
77
|
+
_optional_get_attrs: Tuple[str, ...] = ()
|
|
78
|
+
_parent: Optional[base.RESTObject]
|
|
79
|
+
_parent_attrs: Dict[str, Any]
|
|
80
|
+
_path: Optional[str]
|
|
81
|
+
openaev: pyoaev.OpenAEV
|
|
82
|
+
|
|
83
|
+
@exc.on_http_error(exc.OpenAEVGetError)
|
|
84
|
+
def get(self, **kwargs: Any) -> base.RESTObject:
|
|
85
|
+
if TYPE_CHECKING:
|
|
86
|
+
assert self.path is not None
|
|
87
|
+
server_data = self.openaev.http_get(self.path, **kwargs)
|
|
88
|
+
if TYPE_CHECKING:
|
|
89
|
+
assert not isinstance(server_data, requests.Response)
|
|
90
|
+
assert self._obj_cls is not None
|
|
91
|
+
return self._obj_cls(self, server_data)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class ListMixin(HeadMixin, _RestManagerBase):
|
|
95
|
+
_computed_path: Optional[str]
|
|
96
|
+
_from_parent_attrs: Dict[str, Any]
|
|
97
|
+
_list_filters: Tuple[str, ...] = ()
|
|
98
|
+
_obj_cls: Optional[Type[base.RESTObject]]
|
|
99
|
+
_parent: Optional[base.RESTObject]
|
|
100
|
+
_parent_attrs: Dict[str, Any]
|
|
101
|
+
_path: Optional[str]
|
|
102
|
+
openaev: pyoaev.OpenAEV
|
|
103
|
+
|
|
104
|
+
@exc.on_http_error(exc.OpenAEVListError)
|
|
105
|
+
def list(self, **kwargs: Any) -> Union[base.RESTObjectList, List[base.RESTObject]]:
|
|
106
|
+
|
|
107
|
+
if self.openaev.per_page:
|
|
108
|
+
kwargs.setdefault("per_page", self.openaev.per_page)
|
|
109
|
+
|
|
110
|
+
# global keyset pagination
|
|
111
|
+
if self.openaev.pagination:
|
|
112
|
+
kwargs.setdefault("pagination", self.openaev.pagination)
|
|
113
|
+
|
|
114
|
+
if self.openaev.order_by:
|
|
115
|
+
kwargs.setdefault("order_by", self.openaev.order_by)
|
|
116
|
+
|
|
117
|
+
# Allow to overwrite the path, handy for custom listings
|
|
118
|
+
path = kwargs.pop("path", self.path)
|
|
119
|
+
|
|
120
|
+
if TYPE_CHECKING:
|
|
121
|
+
assert self._obj_cls is not None
|
|
122
|
+
obj = self.openaev.http_list(path, **kwargs)
|
|
123
|
+
if isinstance(obj, list):
|
|
124
|
+
return [self._obj_cls(self, item, created_from_list=True) for item in obj]
|
|
125
|
+
return base.RESTObjectList(self, self._obj_cls, obj)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@enum.unique
|
|
129
|
+
class UpdateMethod(enum.IntEnum):
|
|
130
|
+
PUT = 1
|
|
131
|
+
POST = 2
|
|
132
|
+
PATCH = 3
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class UpdateMixin(_RestManagerBase):
|
|
136
|
+
_computed_path: Optional[str]
|
|
137
|
+
_from_parent_attrs: Dict[str, Any]
|
|
138
|
+
_obj_cls: Optional[Type[base.RESTObject]]
|
|
139
|
+
_parent: Optional[base.RESTObject]
|
|
140
|
+
_parent_attrs: Dict[str, Any]
|
|
141
|
+
_path: Optional[str]
|
|
142
|
+
_update_method: UpdateMethod = UpdateMethod.PUT
|
|
143
|
+
openaev: pyoaev.OpenAEV
|
|
144
|
+
|
|
145
|
+
def _get_update_method(
|
|
146
|
+
self,
|
|
147
|
+
) -> Callable[..., Union[Dict[str, Any], requests.Response]]:
|
|
148
|
+
"""Return the HTTP method to use.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
http_put (default) or http_post
|
|
152
|
+
"""
|
|
153
|
+
if self._update_method is UpdateMethod.POST:
|
|
154
|
+
http_method = self.openaev.http_post
|
|
155
|
+
elif self._update_method is UpdateMethod.PATCH:
|
|
156
|
+
# only patch uses required kwargs, so our types are a bit misaligned
|
|
157
|
+
http_method = self.openaev.http_patch # type: ignore[assignment]
|
|
158
|
+
else:
|
|
159
|
+
http_method = self.openaev.http_put
|
|
160
|
+
return http_method
|
|
161
|
+
|
|
162
|
+
@exc.on_http_error(exc.OpenAEVUpdateError)
|
|
163
|
+
def update(
|
|
164
|
+
self,
|
|
165
|
+
id: Optional[Union[str, int]] = None,
|
|
166
|
+
new_data: Optional[Dict[str, Any]] = None,
|
|
167
|
+
**kwargs: Any,
|
|
168
|
+
) -> Dict[str, Any]:
|
|
169
|
+
new_data = new_data or {}
|
|
170
|
+
|
|
171
|
+
if id is None:
|
|
172
|
+
path = self.path
|
|
173
|
+
else:
|
|
174
|
+
path = f"{self.path}/{utils.EncodedId(id)}"
|
|
175
|
+
|
|
176
|
+
# excludes = []
|
|
177
|
+
# if self._obj_cls is not None and self._obj_cls._id_attr is not None:
|
|
178
|
+
# excludes = [self._obj_cls._id_attr]
|
|
179
|
+
# self._update_attrs.validate_attrs(data=new_data, excludes=excludes)
|
|
180
|
+
http_method = self._get_update_method()
|
|
181
|
+
result = http_method(path, post_data=new_data, **kwargs)
|
|
182
|
+
if TYPE_CHECKING:
|
|
183
|
+
assert not isinstance(result, requests.Response)
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class CreateMixin(_RestManagerBase):
|
|
188
|
+
_computed_path: Optional[str]
|
|
189
|
+
_from_parent_attrs: Dict[str, Any]
|
|
190
|
+
_obj_cls: Optional[Type[base.RESTObject]]
|
|
191
|
+
_parent: Optional[base.RESTObject]
|
|
192
|
+
_parent_attrs: Dict[str, Any]
|
|
193
|
+
_path: Optional[str]
|
|
194
|
+
openaev: pyoaev.OpenAEV
|
|
195
|
+
|
|
196
|
+
@exc.on_http_error(exc.OpenAEVCreateError)
|
|
197
|
+
def create(
|
|
198
|
+
self, data: Optional[Dict[str, Any]] = None, icon: tuple = None, **kwargs: Any
|
|
199
|
+
) -> base.RESTObject:
|
|
200
|
+
if data is None:
|
|
201
|
+
data = {}
|
|
202
|
+
self._create_attrs.validate_attrs(data=data)
|
|
203
|
+
# Handle specific URL for creation
|
|
204
|
+
path = kwargs.pop("path", self.path)
|
|
205
|
+
|
|
206
|
+
if icon:
|
|
207
|
+
files = {"icon": icon}
|
|
208
|
+
elif icon is False:
|
|
209
|
+
files = {}
|
|
210
|
+
else:
|
|
211
|
+
files = None
|
|
212
|
+
server_data = self.openaev.http_post(
|
|
213
|
+
path, post_data=data, files=files, **kwargs
|
|
214
|
+
)
|
|
215
|
+
if TYPE_CHECKING:
|
|
216
|
+
assert not isinstance(server_data, requests.Response)
|
|
217
|
+
assert self._obj_cls is not None
|
|
218
|
+
return self._obj_cls(self, server_data)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class DeleteMixin(_RestManagerBase):
|
|
222
|
+
_computed_path: Optional[str]
|
|
223
|
+
_from_parent_attrs: Dict[str, Any]
|
|
224
|
+
_obj_cls: Optional[Type[base.RESTObject]]
|
|
225
|
+
_parent: Optional[base.RESTObject]
|
|
226
|
+
_parent_attrs: Dict[str, Any]
|
|
227
|
+
_path: Optional[str]
|
|
228
|
+
openaev: pyoaev.OpenAEV
|
|
229
|
+
|
|
230
|
+
@exc.on_http_error(exc.OpenAEVCreateError)
|
|
231
|
+
def delete(
|
|
232
|
+
self, id: Optional[Union[str, int]] = None, **kwargs: Any
|
|
233
|
+
) -> requests.Response:
|
|
234
|
+
if id is None:
|
|
235
|
+
path = self.path
|
|
236
|
+
else:
|
|
237
|
+
path = f"{self.path}/{utils.EncodedId(id)}"
|
|
238
|
+
|
|
239
|
+
result = self.openaev.http_delete(path, **kwargs)
|
|
240
|
+
if TYPE_CHECKING:
|
|
241
|
+
assert isinstance(result, requests.Response)
|
|
242
|
+
return result
|
|
File without changes
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from pyoaev.exceptions import OpenAEVError
|
|
2
|
+
from pyoaev.signatures.types import MatchTypes
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class SignatureMatch:
|
|
6
|
+
def __init__(self, match_type: MatchTypes, match_score: int | None):
|
|
7
|
+
if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE:
|
|
8
|
+
raise OpenAEVError(
|
|
9
|
+
f"Match type {match_type} requires score to be set, found score = {match_score}"
|
|
10
|
+
)
|
|
11
|
+
self.match_type = match_type
|
|
12
|
+
self.match_score = match_score
|