cyvest 0.1.0__py3-none-any.whl → 5.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cyvest/__init__.py +48 -38
- cyvest/cli.py +487 -0
- cyvest/compare.py +318 -0
- cyvest/cyvest.py +1431 -0
- cyvest/investigation.py +1682 -0
- cyvest/io_rich.py +1153 -0
- cyvest/io_schema.py +35 -0
- cyvest/io_serialization.py +465 -0
- cyvest/io_visualization.py +358 -0
- cyvest/keys.py +237 -0
- cyvest/level_score_rules.py +78 -0
- cyvest/levels.py +175 -0
- cyvest/model.py +595 -0
- cyvest/model_enums.py +69 -0
- cyvest/model_schema.py +164 -0
- cyvest/proxies.py +595 -0
- cyvest/score.py +473 -0
- cyvest/shared.py +508 -0
- cyvest/stats.py +291 -0
- cyvest/ulid.py +36 -0
- cyvest-5.1.3.dist-info/METADATA +632 -0
- cyvest-5.1.3.dist-info/RECORD +24 -0
- {cyvest-0.1.0.dist-info → cyvest-5.1.3.dist-info}/WHEEL +1 -2
- cyvest-5.1.3.dist-info/entry_points.txt +3 -0
- cyvest/builder.py +0 -182
- cyvest/check_tree.py +0 -117
- cyvest/models.py +0 -785
- cyvest/observable_registry.py +0 -69
- cyvest/report_render.py +0 -306
- cyvest/report_serialization.py +0 -237
- cyvest/visitors.py +0 -332
- cyvest-0.1.0.dist-info/METADATA +0 -110
- cyvest-0.1.0.dist-info/RECORD +0 -13
- cyvest-0.1.0.dist-info/licenses/LICENSE +0 -21
- cyvest-0.1.0.dist-info/top_level.txt +0 -1
cyvest/models.py
DELETED
|
@@ -1,785 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import enum
|
|
4
|
-
from abc import ABC, abstractmethod
|
|
5
|
-
from collections.abc import Callable, Sequence
|
|
6
|
-
from queue import Queue
|
|
7
|
-
from typing import TYPE_CHECKING, Any
|
|
8
|
-
|
|
9
|
-
if TYPE_CHECKING:
|
|
10
|
-
from .visitors import Visitor
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class Scope(enum.Enum):
|
|
14
|
-
PARSER = 0
|
|
15
|
-
FULL = 1
|
|
16
|
-
HEADER = 2
|
|
17
|
-
MIME_HEADER = 3
|
|
18
|
-
BODY = 4
|
|
19
|
-
ATTACHMENT = 5
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class ObsCategory(enum.Enum):
|
|
23
|
-
NETWORK = "network"
|
|
24
|
-
FILE = "file"
|
|
25
|
-
EMAIL = "email"
|
|
26
|
-
HOST = "host"
|
|
27
|
-
PROCESS = "process"
|
|
28
|
-
REGISTRY = "registry"
|
|
29
|
-
OTHER = "other"
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
class ObsType(enum.Enum):
|
|
33
|
-
def __new__(cls, display_name: str, category: ObsCategory):
|
|
34
|
-
value = len(cls.__members__) + 1
|
|
35
|
-
obj = object.__new__(cls)
|
|
36
|
-
obj._value_ = value
|
|
37
|
-
obj.display_name = display_name
|
|
38
|
-
obj.category = category
|
|
39
|
-
return obj
|
|
40
|
-
|
|
41
|
-
EMAIL_FROM = ("Email From", ObsCategory.EMAIL)
|
|
42
|
-
URL = ("URL", ObsCategory.NETWORK)
|
|
43
|
-
DOMAIN = ("Domain", ObsCategory.NETWORK)
|
|
44
|
-
IP = ("IPv4", ObsCategory.NETWORK)
|
|
45
|
-
IPV6 = ("IPv6", ObsCategory.NETWORK)
|
|
46
|
-
SHA256 = ("SHA256", ObsCategory.FILE)
|
|
47
|
-
MD5 = ("MD5", ObsCategory.FILE)
|
|
48
|
-
FILE = ("File", ObsCategory.FILE)
|
|
49
|
-
SERVER = ("Server", ObsCategory.HOST)
|
|
50
|
-
ANALYZED_MAIL = ("Analyzed Mail", ObsCategory.EMAIL)
|
|
51
|
-
BODY = ("Body", ObsCategory.EMAIL)
|
|
52
|
-
EMAIL = ("Email", ObsCategory.EMAIL)
|
|
53
|
-
PROCESS = ("Process", ObsCategory.PROCESS)
|
|
54
|
-
COMMAND_LINE = ("Command Line", ObsCategory.PROCESS)
|
|
55
|
-
REGISTRY_KEY = ("Registry Key", ObsCategory.REGISTRY)
|
|
56
|
-
CERTIFICATE = ("Certificate", ObsCategory.NETWORK)
|
|
57
|
-
HOSTNAME = ("Hostname", ObsCategory.HOST)
|
|
58
|
-
SERVICE = ("Service", ObsCategory.HOST)
|
|
59
|
-
GEOLOCATION = ("Geolocation", ObsCategory.NETWORK)
|
|
60
|
-
MAC = ("MAC Address", ObsCategory.NETWORK)
|
|
61
|
-
PHONE_NUMBER = ("Phone Number", ObsCategory.OTHER)
|
|
62
|
-
AWS_ACCOUNT = ("AWS Account", ObsCategory.OTHER)
|
|
63
|
-
|
|
64
|
-
@classmethod
|
|
65
|
-
def from_string(cls, value: str, default: ObsType | None = None) -> ObsType | None:
|
|
66
|
-
normalized = value.strip().replace("-", "_").replace(" ", "_").upper()
|
|
67
|
-
return cls.__members__.get(normalized, default)
|
|
68
|
-
|
|
69
|
-
def to_string(self) -> str:
|
|
70
|
-
return self.display_name
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
class Level(enum.IntEnum):
|
|
74
|
-
NONE = 0 # No classification applied.
|
|
75
|
-
TRUSTED = 1 # Explicitly trusted by the system itself.
|
|
76
|
-
INFO = 2
|
|
77
|
-
SAFE = 3 # Whitelisted by an external system.
|
|
78
|
-
NOTABLE = 4
|
|
79
|
-
SUSPICIOUS = 5
|
|
80
|
-
MALICIOUS = 6
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
MAP_LEVEL_DATA = {
|
|
84
|
-
Level.NONE.name: {
|
|
85
|
-
"stdout_color": "bold",
|
|
86
|
-
"global_name": "NONE",
|
|
87
|
-
"css": "background-color: white; color: black;",
|
|
88
|
-
"global_css": "background-color: white; color: black;",
|
|
89
|
-
},
|
|
90
|
-
Level.TRUSTED.name: {
|
|
91
|
-
"stdout_color": "green",
|
|
92
|
-
"global_name": "INFO",
|
|
93
|
-
"css": "background-color: green; color: white;",
|
|
94
|
-
"global_css": "background-color: white; color: black;",
|
|
95
|
-
},
|
|
96
|
-
Level.INFO.name: {
|
|
97
|
-
"stdout_color": "blue",
|
|
98
|
-
"global_name": "INFO",
|
|
99
|
-
"css": "background-color: white; color: black;",
|
|
100
|
-
"global_css": "background-color: white; color: black;",
|
|
101
|
-
},
|
|
102
|
-
Level.SAFE.name: {
|
|
103
|
-
"stdout_color": "green",
|
|
104
|
-
"global_name": "INFO",
|
|
105
|
-
"css": "background-color: green; color: white;",
|
|
106
|
-
"global_css": "background-color: white; color: black;",
|
|
107
|
-
},
|
|
108
|
-
Level.NOTABLE.name: {
|
|
109
|
-
"stdout_color": "yellow",
|
|
110
|
-
"global_name": "INFO",
|
|
111
|
-
"css": "background-color: #B58B00; color: white;",
|
|
112
|
-
"global_css": "background-color: white; color: black;",
|
|
113
|
-
},
|
|
114
|
-
Level.SUSPICIOUS.name: {
|
|
115
|
-
"stdout_color": "yellow",
|
|
116
|
-
"global_name": "SUSPICIOUS",
|
|
117
|
-
"css": "background-color: orange; color: white;",
|
|
118
|
-
"global_css": "background-color: orange; color: white;",
|
|
119
|
-
},
|
|
120
|
-
Level.MALICIOUS.name: {
|
|
121
|
-
"stdout_color": "red",
|
|
122
|
-
"global_name": "MALICIOUS",
|
|
123
|
-
"css": "background-color: red; color: white;",
|
|
124
|
-
"global_css": "background-color: red; color: white;",
|
|
125
|
-
},
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
def update_full_key(
|
|
130
|
-
local: dict[str, Model],
|
|
131
|
-
remote: dict[str, Model],
|
|
132
|
-
on_add: Callable[[Model], bool],
|
|
133
|
-
on_update: Callable[[Model, Model], bool],
|
|
134
|
-
) -> None:
|
|
135
|
-
updated = []
|
|
136
|
-
for key, local_obs in list(local.items()):
|
|
137
|
-
remote_obs = remote.get(key)
|
|
138
|
-
if remote_obs and remote_obs is not local_obs:
|
|
139
|
-
on_update(local_obs, remote_obs)
|
|
140
|
-
updated.append(key)
|
|
141
|
-
to_add = list(set(remote.keys()) - set(updated))
|
|
142
|
-
for remote_obs_full_key in to_add:
|
|
143
|
-
remote_obs = remote[remote_obs_full_key]
|
|
144
|
-
on_add(remote_obs)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def update_all_parents_score(
|
|
148
|
-
node: Observable,
|
|
149
|
-
scored_level_model: ScoredLevelModel,
|
|
150
|
-
seen: set[int],
|
|
151
|
-
) -> None:
|
|
152
|
-
# Update onset
|
|
153
|
-
for p in node.observables_parents.values():
|
|
154
|
-
p.update_score(scored_level_model)
|
|
155
|
-
if id(p) not in seen:
|
|
156
|
-
seen.add(id(p))
|
|
157
|
-
update_all_parents_score(p, scored_level_model, seen)
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def get_color_level(level: Level | str) -> str:
|
|
161
|
-
current_level = Level[level] if isinstance(level, str) else level
|
|
162
|
-
return MAP_LEVEL_DATA.get(current_level.name, {}).get("stdout_color")
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
def get_color_score(score: float) -> str:
|
|
166
|
-
if score <= 0.0:
|
|
167
|
-
return "bold"
|
|
168
|
-
if score < 5.0:
|
|
169
|
-
return "yellow"
|
|
170
|
-
return "red"
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def get_level_from_score(score: float) -> Level | None:
|
|
174
|
-
# Score-derived promotion starts at TRUSTED; Level.NONE is reserved for explicit assignments.
|
|
175
|
-
if score < 0.0:
|
|
176
|
-
return Level.TRUSTED
|
|
177
|
-
if score == 0.0:
|
|
178
|
-
return Level.INFO
|
|
179
|
-
if score < 3.0:
|
|
180
|
-
return Level.NOTABLE
|
|
181
|
-
if score < 5.0:
|
|
182
|
-
return Level.SUSPICIOUS
|
|
183
|
-
if score >= 5.0:
|
|
184
|
-
return Level.MALICIOUS
|
|
185
|
-
return None
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
def combine_score_level(
|
|
189
|
-
current_score: float,
|
|
190
|
-
current_level: Level,
|
|
191
|
-
candidate_score: float,
|
|
192
|
-
candidate_level: Level | None,
|
|
193
|
-
) -> tuple[float, Level]:
|
|
194
|
-
"""Return the consolidated score/level when a new datum is applied."""
|
|
195
|
-
|
|
196
|
-
new_score = max(current_score, candidate_score)
|
|
197
|
-
best_level = current_level
|
|
198
|
-
if candidate_level and candidate_level > best_level:
|
|
199
|
-
best_level = candidate_level
|
|
200
|
-
inferred = get_level_from_score(new_score)
|
|
201
|
-
if inferred and inferred > best_level:
|
|
202
|
-
best_level = inferred
|
|
203
|
-
return new_score, best_level
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
class Model(ABC):
|
|
207
|
-
def __init__(self) -> None:
|
|
208
|
-
super().__init__()
|
|
209
|
-
self.generated_by: set[str] = set()
|
|
210
|
-
|
|
211
|
-
@property
|
|
212
|
-
@abstractmethod
|
|
213
|
-
def full_key(self) -> str:
|
|
214
|
-
"""Method that accept a visitor"""
|
|
215
|
-
raise NotImplementedError("missing property full_key")
|
|
216
|
-
|
|
217
|
-
@abstractmethod
|
|
218
|
-
def accept(self, visitor: Visitor) -> Model:
|
|
219
|
-
"""Method that accept a visitor"""
|
|
220
|
-
raise NotImplementedError("missing method visit_observable")
|
|
221
|
-
|
|
222
|
-
def update(self, model: Model) -> None:
|
|
223
|
-
self.generated_by.update(model.generated_by)
|
|
224
|
-
|
|
225
|
-
def add_generated_by(self, name: str) -> None:
|
|
226
|
-
self.generated_by.add(name)
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
class ScoredLevelModel(Model):
|
|
230
|
-
def __init__(
|
|
231
|
-
self,
|
|
232
|
-
score: float,
|
|
233
|
-
level: Level | None,
|
|
234
|
-
details: dict[str, Any] | None = None,
|
|
235
|
-
) -> None:
|
|
236
|
-
if details is None:
|
|
237
|
-
details = {}
|
|
238
|
-
super().__init__()
|
|
239
|
-
self._score: float = score
|
|
240
|
-
level_from_score = get_level_from_score(self._score)
|
|
241
|
-
self.level: Level = level if level is not None else level_from_score or Level.INFO
|
|
242
|
-
self.details: dict[str, Any] = details if details else {}
|
|
243
|
-
|
|
244
|
-
@property
|
|
245
|
-
def score(self) -> float:
|
|
246
|
-
return self._score
|
|
247
|
-
|
|
248
|
-
@score.setter
|
|
249
|
-
def score(self, score: float) -> None:
|
|
250
|
-
level = get_level_from_score(score)
|
|
251
|
-
# If current level is Safe and new level is greater then Safe,
|
|
252
|
-
# The current level should be the new level
|
|
253
|
-
if level is not None:
|
|
254
|
-
if self.level != Level.SAFE or level > Level.SAFE:
|
|
255
|
-
self.level = level
|
|
256
|
-
self._score = score
|
|
257
|
-
|
|
258
|
-
def update(self, model: ScoredLevelModel) -> None:
|
|
259
|
-
# Update Model
|
|
260
|
-
super().update(model)
|
|
261
|
-
self.details.update(model.details)
|
|
262
|
-
self.update_score(model)
|
|
263
|
-
|
|
264
|
-
def handle_safe(self, model: ScoredLevelModel, is_merge: bool) -> None:
|
|
265
|
-
# If the model used is Safe, and current level is lower then Safe,
|
|
266
|
-
# The current level can be Safe, except for rc & obs reconciliation
|
|
267
|
-
if model.level == Level.SAFE and self.level < Level.SAFE and is_merge is False:
|
|
268
|
-
self.level = Level.SAFE
|
|
269
|
-
|
|
270
|
-
def update_score(self, model: ScoredLevelModel, is_merge: bool = False) -> None:
|
|
271
|
-
new_score, new_level = combine_score_level(self.score, self.level, model.score, model.level)
|
|
272
|
-
self._score = new_score
|
|
273
|
-
self.level = new_level
|
|
274
|
-
self.handle_safe(model, is_merge)
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
class ThreatIntel(ScoredLevelModel):
|
|
278
|
-
def __init__(
|
|
279
|
-
self,
|
|
280
|
-
name: str,
|
|
281
|
-
display_name: str,
|
|
282
|
-
obs_value: str,
|
|
283
|
-
obs_type: ObsType,
|
|
284
|
-
score: float,
|
|
285
|
-
level: Level | None = None,
|
|
286
|
-
comment: str | None = None,
|
|
287
|
-
extra: dict[str, Any] | None = None,
|
|
288
|
-
taxonomies: list[dict[str, Any]] | None = None,
|
|
289
|
-
) -> None:
|
|
290
|
-
super().__init__(score, level)
|
|
291
|
-
self.name = name
|
|
292
|
-
self.display_name = display_name
|
|
293
|
-
self.obs_type = obs_type
|
|
294
|
-
self.obs_value = obs_value
|
|
295
|
-
self.comment = comment
|
|
296
|
-
self.extra = extra
|
|
297
|
-
self.taxonomies = taxonomies
|
|
298
|
-
|
|
299
|
-
def __repr__(self) -> str:
|
|
300
|
-
color_score = get_color_score(self.score)
|
|
301
|
-
color_level = get_color_level(self.level)
|
|
302
|
-
full_str = (
|
|
303
|
-
f"{self.name} -> [{color_score}]{self.score}[/{color_score}] {color_level}{self.level.name}[/{color_level}]"
|
|
304
|
-
)
|
|
305
|
-
return full_str
|
|
306
|
-
|
|
307
|
-
@property
|
|
308
|
-
def full_key(self) -> str:
|
|
309
|
-
full_key = f"{self.name}.{self.obs_type.name}.{self.obs_value}"
|
|
310
|
-
return full_key
|
|
311
|
-
|
|
312
|
-
def accept(self, visitor: Visitor) -> ThreatIntel:
|
|
313
|
-
return visitor.visit_threat_intel(self)
|
|
314
|
-
|
|
315
|
-
def update(self, threat_intel: ThreatIntel) -> None:
|
|
316
|
-
if threat_intel.full_key != self.full_key:
|
|
317
|
-
raise Exception(f"Obs impossible to update. Mismatch key: {threat_intel.full_key} {self.full_key}")
|
|
318
|
-
# Update ScoreModel
|
|
319
|
-
super().update(threat_intel)
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
class Observable(ScoredLevelModel):
|
|
323
|
-
def __init__(self, obs_type: ObsType, obs_value: str) -> None:
|
|
324
|
-
super().__init__(0.0, Level.INFO)
|
|
325
|
-
self.obs_type = obs_type
|
|
326
|
-
self.obs_value = obs_value
|
|
327
|
-
self.threat_intels: dict[str, ThreatIntel] = {}
|
|
328
|
-
self.whitelisted = False
|
|
329
|
-
self.observables_children: dict[str, Observable] = {}
|
|
330
|
-
self.observables_parents: dict[str, Observable] = {}
|
|
331
|
-
|
|
332
|
-
def __repr__(self) -> str:
|
|
333
|
-
color_score = get_color_score(self.score)
|
|
334
|
-
color_level = get_color_level(self.level)
|
|
335
|
-
full_str = f"{self.obs_type.name}.{self.obs_value}"
|
|
336
|
-
more_detail = ""
|
|
337
|
-
if self.whitelisted:
|
|
338
|
-
more_detail = " [green]WHITELISTED[/green]"
|
|
339
|
-
full = f"{full_str} -> [{color_score}]{self.score}[/{color_score}] [{color_level}]{self.level.name}[/{color_level}]{more_detail}" # noqa
|
|
340
|
-
return full
|
|
341
|
-
|
|
342
|
-
@property
|
|
343
|
-
def full_key(self) -> str:
|
|
344
|
-
full_key = f"{self.obs_type.name}.{self.obs_value}"
|
|
345
|
-
return full_key
|
|
346
|
-
|
|
347
|
-
def _accept(self, visitor: Visitor, seen: set[int]) -> Observable:
|
|
348
|
-
ref = visitor.visit_observable(self)
|
|
349
|
-
seen.add(id(ref))
|
|
350
|
-
# Visit threat intel
|
|
351
|
-
to_update = {}
|
|
352
|
-
for ti in ref.threat_intels.values():
|
|
353
|
-
ref_ti = ti.accept(visitor)
|
|
354
|
-
if ref_ti is not ti:
|
|
355
|
-
to_update[ti.full_key] = ref_ti
|
|
356
|
-
ref.threat_intels.update(to_update)
|
|
357
|
-
# Visit parent
|
|
358
|
-
to_update = {}
|
|
359
|
-
for parent in list(ref.observables_parents.values()):
|
|
360
|
-
if id(parent) not in seen:
|
|
361
|
-
seen.add(id(parent))
|
|
362
|
-
ref_parent = parent._accept(visitor, seen)
|
|
363
|
-
if ref_parent is not parent:
|
|
364
|
-
to_update[parent.full_key] = ref_parent
|
|
365
|
-
update_all_parents_score(ref, ref_parent, set([id(ref)]))
|
|
366
|
-
ref.observables_parents.update(to_update)
|
|
367
|
-
# Visit children
|
|
368
|
-
to_update = {}
|
|
369
|
-
for child in list(ref.observables_children.values()):
|
|
370
|
-
if id(child) not in seen:
|
|
371
|
-
seen.add(id(child))
|
|
372
|
-
ref_child = child._accept(visitor, seen)
|
|
373
|
-
if ref_child is not child:
|
|
374
|
-
to_update[child.full_key] = ref_child
|
|
375
|
-
ref.update_score(ref_child)
|
|
376
|
-
update_all_parents_score(ref, ref_child, set([id(ref)]))
|
|
377
|
-
ref.observables_children.update(to_update)
|
|
378
|
-
return ref
|
|
379
|
-
|
|
380
|
-
def accept(self, visitor: Visitor) -> Observable:
|
|
381
|
-
return self._accept(visitor, {id(self)})
|
|
382
|
-
|
|
383
|
-
def _update(self, observable: Observable, seen: set[str]) -> None:
|
|
384
|
-
# Lambda functions
|
|
385
|
-
def on_update(local_obj: Observable, update_obj: Observable) -> bool:
|
|
386
|
-
if update_obj.full_key not in seen:
|
|
387
|
-
local_obj._update(update_obj, seen)
|
|
388
|
-
return True
|
|
389
|
-
return False
|
|
390
|
-
|
|
391
|
-
def on_add_children(new_obj: Observable) -> bool:
|
|
392
|
-
seen.add(new_obj.full_key)
|
|
393
|
-
self.add_observable_children(new_obj)
|
|
394
|
-
return True
|
|
395
|
-
|
|
396
|
-
def on_add_parent(new_obj: Observable) -> bool:
|
|
397
|
-
seen.add(new_obj.full_key)
|
|
398
|
-
self.add_observable_parent(new_obj)
|
|
399
|
-
return True
|
|
400
|
-
|
|
401
|
-
# don't update if object is same
|
|
402
|
-
if self is observable:
|
|
403
|
-
return
|
|
404
|
-
# Update model
|
|
405
|
-
self.threat_intels.update(observable.threat_intels)
|
|
406
|
-
# Update ScoreModel
|
|
407
|
-
super().update(observable)
|
|
408
|
-
# Update parent score
|
|
409
|
-
update_all_parents_score(self, observable, {id(self)})
|
|
410
|
-
# Update links & nodes
|
|
411
|
-
update_full_key(self.observables_children, observable.observables_children, on_add_children, on_update)
|
|
412
|
-
update_full_key(self.observables_parents, observable.observables_parents, on_add_parent, on_update)
|
|
413
|
-
|
|
414
|
-
def update(self, observable: Observable) -> None:
|
|
415
|
-
if observable.full_key != self.full_key:
|
|
416
|
-
raise Exception(f"Obs impossible to update. Mismatch key: {observable.full_key} {self.full_key}")
|
|
417
|
-
# Update links
|
|
418
|
-
self._update(observable, {self.full_key})
|
|
419
|
-
|
|
420
|
-
def add_threat_intel(self, threat_intel: ThreatIntel) -> None:
|
|
421
|
-
name = threat_intel.name
|
|
422
|
-
self.threat_intels[name] = threat_intel
|
|
423
|
-
# Update score
|
|
424
|
-
self.update_score(threat_intel)
|
|
425
|
-
update_all_parents_score(self, threat_intel, {id(self)})
|
|
426
|
-
|
|
427
|
-
def attach_intel(
|
|
428
|
-
self,
|
|
429
|
-
*,
|
|
430
|
-
name: str,
|
|
431
|
-
score: float,
|
|
432
|
-
level: Level,
|
|
433
|
-
display_name: str | None = None,
|
|
434
|
-
comment: str | None = None,
|
|
435
|
-
extra: dict[str, Any] | None = None,
|
|
436
|
-
taxonomies: list[dict[str, Any]] | None = None,
|
|
437
|
-
) -> ThreatIntel:
|
|
438
|
-
intel = ThreatIntel(
|
|
439
|
-
name=name,
|
|
440
|
-
display_name=display_name or name,
|
|
441
|
-
obs_value=self.obs_value,
|
|
442
|
-
obs_type=self.obs_type,
|
|
443
|
-
score=score,
|
|
444
|
-
level=level,
|
|
445
|
-
comment=comment,
|
|
446
|
-
extra=extra,
|
|
447
|
-
taxonomies=taxonomies,
|
|
448
|
-
)
|
|
449
|
-
self.add_threat_intel(intel)
|
|
450
|
-
return intel
|
|
451
|
-
|
|
452
|
-
def add_observable_parent(self, observable: Observable) -> Observable:
|
|
453
|
-
full_key = observable.full_key
|
|
454
|
-
exist_obs = self.observables_parents.get(full_key)
|
|
455
|
-
if exist_obs:
|
|
456
|
-
exist_obs.update(observable)
|
|
457
|
-
else:
|
|
458
|
-
exist_obs = observable
|
|
459
|
-
self.observables_parents[full_key] = observable
|
|
460
|
-
observable.observables_children[self.full_key] = self
|
|
461
|
-
# Update score
|
|
462
|
-
update_all_parents_score(self, observable, {id(self)})
|
|
463
|
-
return exist_obs
|
|
464
|
-
|
|
465
|
-
def add_observable_children(self, observable: Observable) -> Observable:
|
|
466
|
-
full_key = observable.full_key
|
|
467
|
-
exist_obs = self.observables_children.get(full_key)
|
|
468
|
-
if exist_obs:
|
|
469
|
-
exist_obs.update(observable)
|
|
470
|
-
else:
|
|
471
|
-
exist_obs = observable
|
|
472
|
-
self.observables_children[full_key] = observable
|
|
473
|
-
observable.observables_parents[self.full_key] = self
|
|
474
|
-
# Update score
|
|
475
|
-
self.update_score(observable)
|
|
476
|
-
update_all_parents_score(self, observable, {id(self)})
|
|
477
|
-
return exist_obs
|
|
478
|
-
|
|
479
|
-
def add_generated_by(self, name: str) -> None:
|
|
480
|
-
super().add_generated_by(name)
|
|
481
|
-
queue = Queue()
|
|
482
|
-
queue.put_nowait(self)
|
|
483
|
-
seen = set()
|
|
484
|
-
while not queue.empty():
|
|
485
|
-
curr_obs = queue.get_nowait()
|
|
486
|
-
if id(curr_obs) not in seen:
|
|
487
|
-
super(Observable, curr_obs).add_generated_by(name)
|
|
488
|
-
seen.add(id(curr_obs))
|
|
489
|
-
for child in curr_obs.observables_children.values():
|
|
490
|
-
queue.put_nowait(child)
|
|
491
|
-
for parent in curr_obs.observables_parents.values():
|
|
492
|
-
queue.put_nowait(parent)
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
class ContainableSLM(ScoredLevelModel):
|
|
496
|
-
def __init__(
|
|
497
|
-
self,
|
|
498
|
-
path: str,
|
|
499
|
-
scope: Scope | None = None,
|
|
500
|
-
identifier: str | None = None,
|
|
501
|
-
description: str | None = None,
|
|
502
|
-
score: float = 0.0,
|
|
503
|
-
level: Level | None = None,
|
|
504
|
-
details: dict | None = None,
|
|
505
|
-
) -> None:
|
|
506
|
-
if details is None:
|
|
507
|
-
details = {}
|
|
508
|
-
super().__init__(score, level, details)
|
|
509
|
-
self.path = path
|
|
510
|
-
self.scope = scope
|
|
511
|
-
self.description = description
|
|
512
|
-
self.parent: Container | None = None
|
|
513
|
-
self._identifier = identifier.replace("#", "-") if identifier else None
|
|
514
|
-
|
|
515
|
-
@property
|
|
516
|
-
def identifier(self) -> str | None:
|
|
517
|
-
return self._identifier
|
|
518
|
-
|
|
519
|
-
@identifier.setter
|
|
520
|
-
def identifier(self, value: str | None) -> None:
|
|
521
|
-
self._identifier = value.replace("#", "-") if value else None
|
|
522
|
-
|
|
523
|
-
@property
|
|
524
|
-
def local_key(self) -> str:
|
|
525
|
-
ident = f"#{self.identifier}" if self.identifier else ""
|
|
526
|
-
return f"{self.path}{ident}"
|
|
527
|
-
|
|
528
|
-
@property
|
|
529
|
-
def full_key(self) -> str:
|
|
530
|
-
if self.parent is not None:
|
|
531
|
-
base = f"{self.parent.full_key}.{self.path}"
|
|
532
|
-
else:
|
|
533
|
-
if self.scope is None:
|
|
534
|
-
raise Exception(f"No scope is set for {self.path}")
|
|
535
|
-
base = f"{self.scope.name}.{self.path}"
|
|
536
|
-
if self.identifier:
|
|
537
|
-
base = f"{base}#{self.identifier}#"
|
|
538
|
-
return base
|
|
539
|
-
|
|
540
|
-
@property
|
|
541
|
-
def short_key(self) -> str:
|
|
542
|
-
ident = f"#{self.identifier}" if self.identifier else ""
|
|
543
|
-
return f"{self.path}{ident}"
|
|
544
|
-
|
|
545
|
-
def set_parent(self, container: Container | None) -> None:
|
|
546
|
-
self.parent = container
|
|
547
|
-
if container and self.scope is None:
|
|
548
|
-
self.scope = container.scope
|
|
549
|
-
|
|
550
|
-
def update_metadata(self, other: ContainableSLM) -> None:
|
|
551
|
-
if other.description:
|
|
552
|
-
self.description = other.description
|
|
553
|
-
self.details.update(other.details)
|
|
554
|
-
if other.scope is not None:
|
|
555
|
-
self.scope = other.scope
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
class ResultCheck(ContainableSLM):
|
|
559
|
-
def __init__(
|
|
560
|
-
self,
|
|
561
|
-
path: str,
|
|
562
|
-
scope: Scope | None = None,
|
|
563
|
-
identifier: str | None = None,
|
|
564
|
-
description: str | None = None,
|
|
565
|
-
score: float = 0.0,
|
|
566
|
-
level: Level | None = None,
|
|
567
|
-
details: dict | None = None,
|
|
568
|
-
) -> None:
|
|
569
|
-
super().__init__(
|
|
570
|
-
path, scope=scope, identifier=identifier, description=description, score=score, level=level, details=details
|
|
571
|
-
)
|
|
572
|
-
self.observables: dict[str, Observable] = {}
|
|
573
|
-
|
|
574
|
-
def __repr__(self) -> str:
|
|
575
|
-
color_score = get_color_score(self.score)
|
|
576
|
-
color_level = get_color_level(self.level)
|
|
577
|
-
full_key = self.full_key
|
|
578
|
-
return (
|
|
579
|
-
f"{full_key} -> [{color_score}]{self.score}[/{color_score}] {color_level}{self.level.name}[/{color_level}]"
|
|
580
|
-
)
|
|
581
|
-
|
|
582
|
-
def accept(self, visitor: Visitor) -> ResultCheck:
|
|
583
|
-
ref_rc: ResultCheck = visitor.visit_result_check(self)
|
|
584
|
-
normalized: dict[str, Observable] = {}
|
|
585
|
-
for obs in self.observables.values():
|
|
586
|
-
ref_obs = obs.accept(visitor)
|
|
587
|
-
normalized[ref_obs.full_key] = ref_obs
|
|
588
|
-
ref_rc.observables.update(normalized)
|
|
589
|
-
if normalized:
|
|
590
|
-
for observable in normalized.values():
|
|
591
|
-
ref_rc.update_score(observable)
|
|
592
|
-
return ref_rc
|
|
593
|
-
|
|
594
|
-
def merge_from(self, other: ResultCheck) -> ResultCheck:
|
|
595
|
-
self.update_metadata(other)
|
|
596
|
-
for obs in other.observables.values():
|
|
597
|
-
existing = self.observables.get(obs.full_key)
|
|
598
|
-
if existing:
|
|
599
|
-
existing.update(obs)
|
|
600
|
-
merged = existing
|
|
601
|
-
else:
|
|
602
|
-
merged = obs
|
|
603
|
-
self.observables[obs.full_key] = merged
|
|
604
|
-
self.update_score(merged)
|
|
605
|
-
self.update_score(other)
|
|
606
|
-
return self
|
|
607
|
-
|
|
608
|
-
def add_observable(self, observable: Observable) -> Observable:
|
|
609
|
-
self.observables[observable.full_key] = observable
|
|
610
|
-
self.update_score(observable)
|
|
611
|
-
return observable
|
|
612
|
-
|
|
613
|
-
@classmethod
|
|
614
|
-
def create(
|
|
615
|
-
cls,
|
|
616
|
-
path: str,
|
|
617
|
-
*,
|
|
618
|
-
scope: Scope,
|
|
619
|
-
identifier: str | None = None,
|
|
620
|
-
description: str | None = None,
|
|
621
|
-
level: Level = Level.INFO,
|
|
622
|
-
score: float = 0.0,
|
|
623
|
-
details: dict | None = None,
|
|
624
|
-
) -> ResultCheck:
|
|
625
|
-
return cls(
|
|
626
|
-
path,
|
|
627
|
-
scope=scope,
|
|
628
|
-
identifier=identifier,
|
|
629
|
-
description=description,
|
|
630
|
-
score=score,
|
|
631
|
-
level=level,
|
|
632
|
-
details=details,
|
|
633
|
-
)
|
|
634
|
-
|
|
635
|
-
def add_observable_chain(self, chain: Sequence[dict[str, Any]]) -> Observable:
|
|
636
|
-
if not chain:
|
|
637
|
-
raise ValueError("Observable chain cannot be empty")
|
|
638
|
-
parent: Observable | None = None
|
|
639
|
-
root: Observable | None = None
|
|
640
|
-
for entry in chain:
|
|
641
|
-
obs_type = entry.get("obs_type")
|
|
642
|
-
value = entry.get("value")
|
|
643
|
-
if obs_type is None or value is None:
|
|
644
|
-
raise ValueError("Each chain entry must include 'obs_type' and 'value'")
|
|
645
|
-
observable = Observable(obs_type, value)
|
|
646
|
-
intel_data = entry.get("intel")
|
|
647
|
-
if intel_data:
|
|
648
|
-
observable.attach_intel(**intel_data)
|
|
649
|
-
if parent is None:
|
|
650
|
-
root = self.add_observable(observable)
|
|
651
|
-
else:
|
|
652
|
-
parent.add_observable_children(observable)
|
|
653
|
-
parent = observable
|
|
654
|
-
assert root is not None
|
|
655
|
-
return root
|
|
656
|
-
|
|
657
|
-
def add_threat_intel(self, threat_intel: ThreatIntel) -> Observable:
|
|
658
|
-
observable = self.add_observable(Observable(threat_intel.obs_type, threat_intel.obs_value))
|
|
659
|
-
observable.add_threat_intel(threat_intel)
|
|
660
|
-
self.update_score(threat_intel)
|
|
661
|
-
return observable
|
|
662
|
-
|
|
663
|
-
def add_generated_by(self, name: str) -> None:
|
|
664
|
-
super().add_generated_by(name)
|
|
665
|
-
for observable in self.observables.values():
|
|
666
|
-
observable.add_generated_by(name)
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
class Container(ContainableSLM):
|
|
670
|
-
def __init__(
|
|
671
|
-
self,
|
|
672
|
-
path: str,
|
|
673
|
-
scope: Scope | None = None,
|
|
674
|
-
identifier: str | None = None,
|
|
675
|
-
description: str | None = None,
|
|
676
|
-
score: float = 0.0,
|
|
677
|
-
level: Level | None = None,
|
|
678
|
-
details: dict | None = None,
|
|
679
|
-
) -> None:
|
|
680
|
-
super().__init__(
|
|
681
|
-
path,
|
|
682
|
-
scope=scope,
|
|
683
|
-
identifier=identifier,
|
|
684
|
-
description=description,
|
|
685
|
-
score=score,
|
|
686
|
-
level=level,
|
|
687
|
-
details=details,
|
|
688
|
-
)
|
|
689
|
-
self.children: list[ContainableSLM] = []
|
|
690
|
-
self._child_keys: set[str] = set()
|
|
691
|
-
|
|
692
|
-
def __repr__(self) -> str:
|
|
693
|
-
color_score = get_color_score(self.score)
|
|
694
|
-
color_level = get_color_level(self.level)
|
|
695
|
-
return (
|
|
696
|
-
f"{self.full_key} -> [{color_score}]{self.score}[/{color_score}] "
|
|
697
|
-
f"{color_level}{self.level.name}[/{color_level}] (children={len(self.children)})"
|
|
698
|
-
)
|
|
699
|
-
|
|
700
|
-
@property
|
|
701
|
-
def nb_checks(self) -> int:
|
|
702
|
-
count = 0
|
|
703
|
-
for child in self.children:
|
|
704
|
-
if isinstance(child, Container):
|
|
705
|
-
count += child.nb_checks
|
|
706
|
-
else:
|
|
707
|
-
count += 1
|
|
708
|
-
return count
|
|
709
|
-
|
|
710
|
-
def attach_child(self, node: ContainableSLM) -> ContainableSLM:
|
|
711
|
-
node.set_parent(self)
|
|
712
|
-
key = node.local_key
|
|
713
|
-
if key in self._child_keys:
|
|
714
|
-
for existing in self.children:
|
|
715
|
-
if existing.local_key == key:
|
|
716
|
-
if isinstance(existing, Container) and isinstance(node, Container):
|
|
717
|
-
existing.merge_from(node)
|
|
718
|
-
return existing
|
|
719
|
-
if isinstance(existing, ResultCheck) and isinstance(node, ResultCheck):
|
|
720
|
-
existing.merge_from(node)
|
|
721
|
-
return existing
|
|
722
|
-
return existing
|
|
723
|
-
self.children.append(node)
|
|
724
|
-
self._child_keys.add(key)
|
|
725
|
-
return node
|
|
726
|
-
|
|
727
|
-
def merge_from(self, other: Container) -> Container:
|
|
728
|
-
self.update_metadata(other)
|
|
729
|
-
for child in other.children:
|
|
730
|
-
self.attach_child(child)
|
|
731
|
-
self.recompute()
|
|
732
|
-
return self
|
|
733
|
-
|
|
734
|
-
def contain(self, node: ResultCheck | Container) -> ResultCheck | Container:
|
|
735
|
-
if not isinstance(node, (ResultCheck, Container)):
|
|
736
|
-
raise TypeError("Containers can only hold ResultCheck or Container instances")
|
|
737
|
-
if self.scope is not None and node.scope is not None and node.scope != self.scope:
|
|
738
|
-
raise ValueError(f"Scope doesn't match: {node.scope} with container {self.scope}")
|
|
739
|
-
node.set_parent(self)
|
|
740
|
-
attached = self.attach_child(node)
|
|
741
|
-
return attached
|
|
742
|
-
|
|
743
|
-
def recompute(self) -> None:
|
|
744
|
-
total = 0.0
|
|
745
|
-
highest = self.level or Level.INFO
|
|
746
|
-
for child in self.children:
|
|
747
|
-
if isinstance(child, Container):
|
|
748
|
-
child.recompute()
|
|
749
|
-
total += child.score
|
|
750
|
-
if child.level > highest:
|
|
751
|
-
highest = child.level
|
|
752
|
-
self._score = total
|
|
753
|
-
self.level = highest
|
|
754
|
-
|
|
755
|
-
def accept(self, visitor: Visitor) -> Container:
|
|
756
|
-
ref_container: Container = visitor.visit_container(self)
|
|
757
|
-
for child in self.children:
|
|
758
|
-
child.accept(visitor)
|
|
759
|
-
return ref_container
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
class Enrichment(Model):
|
|
763
|
-
def __init__(self, ref_struct: dict, key: str, data: Any) -> None:
|
|
764
|
-
super().__init__()
|
|
765
|
-
self.ref_struct = ref_struct
|
|
766
|
-
self.key = key
|
|
767
|
-
self.data = data
|
|
768
|
-
|
|
769
|
-
def __repr__(self) -> str:
|
|
770
|
-
ref = id(self.ref_struct)
|
|
771
|
-
full = f"ENRICHMENT.{ref}.{self.key}"
|
|
772
|
-
return full
|
|
773
|
-
|
|
774
|
-
@property
|
|
775
|
-
def full_key(self) -> str:
|
|
776
|
-
full = f"ENRICHMENT.{self.scope.name}.{self.path}"
|
|
777
|
-
return full
|
|
778
|
-
|
|
779
|
-
def accept(self, visitor):
|
|
780
|
-
visitor.visit_enrichment(self)
|
|
781
|
-
|
|
782
|
-
def update(self, enrichment: Enrichment):
|
|
783
|
-
self.ref_struct = enrichment.ref_struct
|
|
784
|
-
self.key = enrichment.key
|
|
785
|
-
self.data = enrichment.data
|