psengine 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psengine/__init__.py +22 -0
- psengine/_sdk_id.py +16 -0
- psengine/_version.py +14 -0
- psengine/analyst_notes/__init__.py +32 -0
- psengine/analyst_notes/constants.py +15 -0
- psengine/analyst_notes/errors.py +42 -0
- psengine/analyst_notes/helpers.py +90 -0
- psengine/analyst_notes/models.py +219 -0
- psengine/analyst_notes/note.py +149 -0
- psengine/analyst_notes/note_mgr.py +400 -0
- psengine/base_http_client.py +285 -0
- psengine/classic_alerts/__init__.py +24 -0
- psengine/classic_alerts/classic_alert.py +275 -0
- psengine/classic_alerts/classic_alert_mgr.py +507 -0
- psengine/classic_alerts/constants.py +31 -0
- psengine/classic_alerts/errors.py +38 -0
- psengine/classic_alerts/helpers.py +87 -0
- psengine/classic_alerts/markdown/__init__.py +13 -0
- psengine/classic_alerts/markdown/markdown.py +359 -0
- psengine/classic_alerts/models.py +141 -0
- psengine/collective_insights/__init__.py +29 -0
- psengine/collective_insights/collective_insights.py +164 -0
- psengine/collective_insights/constants.py +44 -0
- psengine/collective_insights/errors.py +18 -0
- psengine/collective_insights/insight.py +89 -0
- psengine/collective_insights/models.py +81 -0
- psengine/common_models.py +89 -0
- psengine/config/__init__.py +15 -0
- psengine/config/config.py +284 -0
- psengine/config/errors.py +18 -0
- psengine/constants.py +63 -0
- psengine/detection/__init__.py +17 -0
- psengine/detection/detection_mgr.py +135 -0
- psengine/detection/detection_rule.py +85 -0
- psengine/detection/errors.py +26 -0
- psengine/detection/helpers.py +56 -0
- psengine/detection/models.py +47 -0
- psengine/endpoints.py +98 -0
- psengine/enrich/__init__.py +28 -0
- psengine/enrich/constants.py +73 -0
- psengine/enrich/errors.py +26 -0
- psengine/enrich/lookup.py +299 -0
- psengine/enrich/lookup_mgr.py +341 -0
- psengine/enrich/models/__init__.py +13 -0
- psengine/enrich/models/base_enriched_entity.py +43 -0
- psengine/enrich/models/lookup.py +271 -0
- psengine/enrich/models/soar.py +138 -0
- psengine/enrich/soar.py +89 -0
- psengine/enrich/soar_mgr.py +176 -0
- psengine/entity_lists/__init__.py +16 -0
- psengine/entity_lists/constants.py +19 -0
- psengine/entity_lists/entity_list.py +435 -0
- psengine/entity_lists/entity_list_mgr.py +185 -0
- psengine/entity_lists/errors.py +26 -0
- psengine/entity_lists/models.py +87 -0
- psengine/entity_match/__init__.py +16 -0
- psengine/entity_match/entity_match.py +90 -0
- psengine/entity_match/entity_match_mgr.py +235 -0
- psengine/entity_match/errors.py +18 -0
- psengine/entity_match/models.py +22 -0
- psengine/errors.py +41 -0
- psengine/helpers/__init__.py +23 -0
- psengine/helpers/helpers.py +471 -0
- psengine/logger/__init__.py +15 -0
- psengine/logger/constants.py +39 -0
- psengine/logger/errors.py +18 -0
- psengine/logger/rf_logger.py +148 -0
- psengine/markdown/__init__.py +21 -0
- psengine/markdown/markdown.py +169 -0
- psengine/markdown/models.py +22 -0
- psengine/playbook_alerts/__init__.py +34 -0
- psengine/playbook_alerts/constants.py +35 -0
- psengine/playbook_alerts/errors.py +35 -0
- psengine/playbook_alerts/helpers.py +80 -0
- psengine/playbook_alerts/mappings.py +44 -0
- psengine/playbook_alerts/markdown/__init__.py +13 -0
- psengine/playbook_alerts/markdown/markdown.py +98 -0
- psengine/playbook_alerts/markdown/markdown_code_repo.py +64 -0
- psengine/playbook_alerts/markdown/markdown_domain_abuse.py +118 -0
- psengine/playbook_alerts/markdown/markdown_identity_exposure.py +158 -0
- psengine/playbook_alerts/models/__init__.py +36 -0
- psengine/playbook_alerts/models/common_models.py +18 -0
- psengine/playbook_alerts/models/panel_log.py +329 -0
- psengine/playbook_alerts/models/panel_status.py +70 -0
- psengine/playbook_alerts/models/pba_code_repo_leak.py +52 -0
- psengine/playbook_alerts/models/pba_cyber_vulnerability.py +53 -0
- psengine/playbook_alerts/models/pba_domain_abuse.py +139 -0
- psengine/playbook_alerts/models/pba_identity_exposures.py +93 -0
- psengine/playbook_alerts/models/pba_third_party_risk.py +103 -0
- psengine/playbook_alerts/models/search_endpoint.py +68 -0
- psengine/playbook_alerts/pa_category.py +37 -0
- psengine/playbook_alerts/playbook_alert_mgr.py +593 -0
- psengine/playbook_alerts/playbook_alerts.py +393 -0
- psengine/rf_client.py +430 -0
- psengine/risklists/__init__.py +17 -0
- psengine/risklists/constants.py +15 -0
- psengine/risklists/errors.py +20 -0
- psengine/risklists/models.py +65 -0
- psengine/risklists/risklist_mgr.py +156 -0
- psengine/stix2/__init__.py +21 -0
- psengine/stix2/base_stix_entity.py +62 -0
- psengine/stix2/complex_entity.py +372 -0
- psengine/stix2/constants.py +81 -0
- psengine/stix2/enriched_indicator.py +261 -0
- psengine/stix2/errors.py +22 -0
- psengine/stix2/helpers.py +68 -0
- psengine/stix2/rf_bundle.py +240 -0
- psengine/stix2/simple_entity.py +145 -0
- psengine/stix2/util.py +53 -0
- psengine-2.0.4.dist-info/METADATA +189 -0
- psengine-2.0.4.dist-info/RECORD +115 -0
- psengine-2.0.4.dist-info/WHEEL +5 -0
- psengine-2.0.4.dist-info/entry_points.txt +2 -0
- psengine-2.0.4.dist-info/licenses/LICENSE +21 -0
- psengine-2.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from itertools import chain
|
|
16
|
+
|
|
17
|
+
from markdown_strings import blockquote, bold, esc_format, link
|
|
18
|
+
|
|
19
|
+
from ...constants import TIMESTAMP_STR, TRUNCATE_COMMENT
|
|
20
|
+
from ...markdown import (
|
|
21
|
+
MarkdownMaker,
|
|
22
|
+
clean_text,
|
|
23
|
+
escape_pipe_characters,
|
|
24
|
+
html_textarea,
|
|
25
|
+
table_from_rows,
|
|
26
|
+
)
|
|
27
|
+
from ..constants import MARKDOWN_ENTITY_TYPES_TO_DEFANG
|
|
28
|
+
from ..errors import AlertMarkdownError
|
|
29
|
+
|
|
30
|
+
TRIGGERED_BY_HTML = (
|
|
31
|
+
'<details>\n<summary>Triggered By (Click to expand)\n</summary>\n- {}\n</details>\n'
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _clean_title(alert_title: str) -> str:
|
|
36
|
+
"""Utility function to remove the new references count from the legacy alert title.
|
|
37
|
+
|
|
38
|
+
INPUT: "Leaked Credentials Monitoring - 1 reference"
|
|
39
|
+
OUTPUT: "Leaked Credentials Monitoring"
|
|
40
|
+
"""
|
|
41
|
+
alert_title = alert_title.strip()
|
|
42
|
+
expression = re.compile(r'(\-\s\d+(\+?)\sreference(s?))')
|
|
43
|
+
|
|
44
|
+
return re.sub(expression, '', alert_title).strip()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _owner_org_markdown(classic_alert) -> list[str]:
|
|
48
|
+
results = []
|
|
49
|
+
details = classic_alert.owner_organisation_details
|
|
50
|
+
|
|
51
|
+
if not details:
|
|
52
|
+
return []
|
|
53
|
+
|
|
54
|
+
if details.enterprise_name:
|
|
55
|
+
results.append(f"{bold('Enterprise:')} {details.enterprise_name} ")
|
|
56
|
+
if details.owner_name:
|
|
57
|
+
results.append(f"{bold('Owner:')} {details.owner_name} ")
|
|
58
|
+
|
|
59
|
+
orgs = [[org.organisation_id, org.organisation_name] for org in details.organisations]
|
|
60
|
+
orgs.insert(0, ['Organisation ID', 'Organisation Name'])
|
|
61
|
+
results.append(table_from_rows(orgs))
|
|
62
|
+
|
|
63
|
+
return results
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _process_hit_fragment(
|
|
67
|
+
hit, include_triggered_by: bool, html_tags: bool, classic_alert
|
|
68
|
+
) -> tuple[str, str]:
|
|
69
|
+
content = []
|
|
70
|
+
authors = ', '.join(author.name for author in hit.document.authors)
|
|
71
|
+
|
|
72
|
+
title_line = f' From {hit.document.source.name}'
|
|
73
|
+
if authors:
|
|
74
|
+
content.append(f"{bold('Author(s):')} {authors}\n")
|
|
75
|
+
|
|
76
|
+
if hit.document.title and hit.fragment:
|
|
77
|
+
first_half_title = hit.document.title[: (len(hit.document.title) // 2)]
|
|
78
|
+
if not hit.fragment.lower().startswith(first_half_title.lower()):
|
|
79
|
+
content.append(f"{bold('Title:')} {clean_text(hit.document.title)}\n")
|
|
80
|
+
elif hit.document.title and not hit.fragment:
|
|
81
|
+
content.append(f"{bold('Title:')} {clean_text(hit.document.title)}\n")
|
|
82
|
+
|
|
83
|
+
if hit.document.url:
|
|
84
|
+
content.append(f"{bold('URL:')} {hit.document.url}\n")
|
|
85
|
+
|
|
86
|
+
if hit.fragment:
|
|
87
|
+
fragment = (
|
|
88
|
+
html_textarea(clean_text(hit.fragment)) if html_tags else clean_text(hit.fragment)
|
|
89
|
+
)
|
|
90
|
+
content.append(f'{blockquote(fragment)}\n')
|
|
91
|
+
else:
|
|
92
|
+
content.append(
|
|
93
|
+
f"_Reference text is missing, check the Recorded Future {link('Portal', str(classic_alert.url.portal))} for more information._\n" # noqa: E501
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if include_triggered_by:
|
|
97
|
+
triggered_by = classic_alert.triggered_by_from_hit(hit)
|
|
98
|
+
if triggered_by:
|
|
99
|
+
triggered_by = '\n+ '.join(triggered_by).replace('->', '→')
|
|
100
|
+
if html_tags:
|
|
101
|
+
triggered_by = TRIGGERED_BY_HTML.format(triggered_by)
|
|
102
|
+
content.append(triggered_by)
|
|
103
|
+
else:
|
|
104
|
+
content.append(f"{bold('Triggered By:')}\n+ {triggered_by}\n")
|
|
105
|
+
|
|
106
|
+
return title_line, content
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _process_entities(entities, hit) -> list[list[str]]:
|
|
110
|
+
if entities is None:
|
|
111
|
+
if not any(entity.description for entity in hit.entities):
|
|
112
|
+
entities = [[entity.name, entity.type_] for entity in hit.entities]
|
|
113
|
+
entity_headers = ['Entity', 'Type']
|
|
114
|
+
else:
|
|
115
|
+
entities = [
|
|
116
|
+
[entity.name, entity.type_, entity.description or ''] for entity in hit.entities
|
|
117
|
+
]
|
|
118
|
+
entity_headers = ['Entity', 'Type', 'Description']
|
|
119
|
+
entities.insert(0, entity_headers)
|
|
120
|
+
else:
|
|
121
|
+
for entity in hit.entities:
|
|
122
|
+
entities.append([entity.name, entity.type_, entity.description or ''])
|
|
123
|
+
|
|
124
|
+
return entities
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _hits_markdown(
|
|
128
|
+
classic_alert,
|
|
129
|
+
hits,
|
|
130
|
+
include_fragment_entities: bool = True,
|
|
131
|
+
include_triggered_by: bool = True,
|
|
132
|
+
html_tags: bool = False,
|
|
133
|
+
) -> list:
|
|
134
|
+
sections = []
|
|
135
|
+
for idx, hit in enumerate(hits):
|
|
136
|
+
section = {
|
|
137
|
+
'title': f'{idx + 1}.',
|
|
138
|
+
'content': [],
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
title_line, fragment_content = _process_hit_fragment(
|
|
142
|
+
hit, include_triggered_by, html_tags, classic_alert
|
|
143
|
+
)
|
|
144
|
+
section['title'] += title_line
|
|
145
|
+
section['content'].extend(fragment_content)
|
|
146
|
+
|
|
147
|
+
entities = None
|
|
148
|
+
if hit.primary_entity and include_fragment_entities:
|
|
149
|
+
description = 'This is the primary entity for this reference'
|
|
150
|
+
if hit.primary_entity.description:
|
|
151
|
+
description += f'.\n{hit.primary_entity.description}'
|
|
152
|
+
entities = [[hit.primary_entity.name, hit.primary_entity.type_, description]]
|
|
153
|
+
entity_headers = ['Entity', 'Type', 'Description']
|
|
154
|
+
entities.insert(0, entity_headers)
|
|
155
|
+
|
|
156
|
+
if hit.entities and include_fragment_entities:
|
|
157
|
+
entities = _process_entities(entities, hit)
|
|
158
|
+
|
|
159
|
+
if entities:
|
|
160
|
+
section['content'].append(table_from_rows(entities))
|
|
161
|
+
|
|
162
|
+
if idx < len(hits) - 1:
|
|
163
|
+
section['content'].append('\n---\n')
|
|
164
|
+
|
|
165
|
+
sections.append(section)
|
|
166
|
+
|
|
167
|
+
return sections
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _enriched_entities_markdown(classic_alert) -> list:
|
|
171
|
+
results = []
|
|
172
|
+
for entity in classic_alert.enriched_entities:
|
|
173
|
+
if not entity.evidence:
|
|
174
|
+
continue
|
|
175
|
+
|
|
176
|
+
criticality = entity.criticality
|
|
177
|
+
contents = [
|
|
178
|
+
f"{bold('Risk Score:')} {criticality.score}",
|
|
179
|
+
f"{bold('Criticality:')} {criticality.name}",
|
|
180
|
+
f"{bold('Triggered:')} {criticality.triggered.strftime(TIMESTAMP_STR)}",
|
|
181
|
+
f"{bold('Last Triggered:')} {criticality.last_triggered.strftime(TIMESTAMP_STR)} \n\n",
|
|
182
|
+
]
|
|
183
|
+
|
|
184
|
+
evidences = []
|
|
185
|
+
for evidence in sorted(entity.evidence, key=lambda x: x.criticality, reverse=True):
|
|
186
|
+
evidence_result = [
|
|
187
|
+
evidence.criticality,
|
|
188
|
+
evidence.rule,
|
|
189
|
+
escape_pipe_characters(evidence.evidence_string),
|
|
190
|
+
evidence.timestamp.strftime(TIMESTAMP_STR),
|
|
191
|
+
]
|
|
192
|
+
evidences.append(evidence_result)
|
|
193
|
+
evidences.insert(0, ['Rule Criticality', 'Rule', 'Evidence', 'Timestamp'])
|
|
194
|
+
contents.append(table_from_rows(evidences))
|
|
195
|
+
results.append((entity.entity.name, contents))
|
|
196
|
+
|
|
197
|
+
return results
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _target_entities_markdown(classic_alert, html_tags: bool = False) -> list:
|
|
201
|
+
results = []
|
|
202
|
+
for entity in classic_alert.enriched_entities:
|
|
203
|
+
result = {'title': f'Target {entity.entity.name}'}
|
|
204
|
+
result['content'] = _hits_markdown(
|
|
205
|
+
classic_alert,
|
|
206
|
+
hits=entity.references,
|
|
207
|
+
include_fragment_entities=False,
|
|
208
|
+
html_tags=html_tags,
|
|
209
|
+
)
|
|
210
|
+
if len(result['content']):
|
|
211
|
+
results.append(result)
|
|
212
|
+
|
|
213
|
+
return results
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _create_summary_section(ca) -> None:
|
|
217
|
+
summary_content = [
|
|
218
|
+
f"{bold('ID:')} {ca.id_} ",
|
|
219
|
+
f"{bold('Triggered:')} {ca.log.triggered.strftime(TIMESTAMP_STR)} ",
|
|
220
|
+
f"{bold('Alerting Rule:')} {ca.rule.name} ",
|
|
221
|
+
f"{link('API', str(ca.url.api))} | {link('Portal', str(ca.url.portal))}",
|
|
222
|
+
]
|
|
223
|
+
return summary_content
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _get_entities_to_defang(classic_alert) -> set:
|
|
227
|
+
"""Return a set of IOC entities to defang from the classic_alert hits."""
|
|
228
|
+
if not classic_alert.hits:
|
|
229
|
+
return set()
|
|
230
|
+
|
|
231
|
+
raw_entities = {
|
|
232
|
+
entity.name
|
|
233
|
+
for entity in chain.from_iterable(h.entities for h in classic_alert.hits)
|
|
234
|
+
if entity.type_ in MARKDOWN_ENTITY_TYPES_TO_DEFANG
|
|
235
|
+
}
|
|
236
|
+
defanged_entities = raw_entities.union({esc_format(ent, esc=True) for ent in raw_entities})
|
|
237
|
+
return defanged_entities
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _add_summary_section(md_maker: MarkdownMaker, classic_alert) -> None:
|
|
241
|
+
"""Adds the 'Summary' section to the markdown builder."""
|
|
242
|
+
md_maker.add_title(_clean_title(classic_alert.title))
|
|
243
|
+
md_maker.add_section('Summary', _create_summary_section(classic_alert))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _add_owner_org_section(md_maker: MarkdownMaker, classic_alert, owner_org: bool) -> None:
|
|
247
|
+
"""Adds 'Owner Organisation Details' section if owner_org is True and details are present."""
|
|
248
|
+
if owner_org and classic_alert.owner_organisation_details:
|
|
249
|
+
md_maker.add_section('Owner Organisation Details', _owner_org_markdown(classic_alert))
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _add_ai_insights_section(md_maker: MarkdownMaker, classic_alert, ai_insights: bool) -> None:
|
|
253
|
+
"""Adds the 'AI Insights' sections if ai_insights is True and data is present."""
|
|
254
|
+
if ai_insights and classic_alert.ai_insights:
|
|
255
|
+
if classic_alert.ai_insights.text:
|
|
256
|
+
md_maker.add_section('AI Insights', [classic_alert.ai_insights.text])
|
|
257
|
+
if classic_alert.ai_insights.comment:
|
|
258
|
+
md_maker.add_section(
|
|
259
|
+
'AI Insights', [f"{bold('Comment:')} {classic_alert.ai_insights.comment}"]
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _add_enriched_entities_sections(
|
|
264
|
+
md_maker: MarkdownMaker, classic_alert, html_tags: bool
|
|
265
|
+
) -> None:
|
|
266
|
+
"""Adds sections related to enriched entities (evidence and references)."""
|
|
267
|
+
if any(x.evidence for x in classic_alert.enriched_entities):
|
|
268
|
+
for entity, contents in _enriched_entities_markdown(classic_alert):
|
|
269
|
+
md_maker.add_section(entity, contents)
|
|
270
|
+
|
|
271
|
+
if any(x.references for x in classic_alert.enriched_entities):
|
|
272
|
+
md_maker.add_section(
|
|
273
|
+
'Target Entities',
|
|
274
|
+
_target_entities_markdown(classic_alert, html_tags),
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _add_hits_section_if_no_enriched_entities(
|
|
279
|
+
md_maker: MarkdownMaker,
|
|
280
|
+
classic_alert,
|
|
281
|
+
fragment_entities: bool,
|
|
282
|
+
triggered_by: bool,
|
|
283
|
+
html_tags: bool,
|
|
284
|
+
) -> None:
|
|
285
|
+
"""If there are no enriched entities, add a 'References' section from alert hits."""
|
|
286
|
+
if classic_alert.hits:
|
|
287
|
+
md_maker.add_section(
|
|
288
|
+
'References',
|
|
289
|
+
_hits_markdown(
|
|
290
|
+
classic_alert,
|
|
291
|
+
hits=classic_alert.hits,
|
|
292
|
+
include_fragment_entities=fragment_entities,
|
|
293
|
+
include_triggered_by=triggered_by,
|
|
294
|
+
html_tags=html_tags,
|
|
295
|
+
),
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _markdown_alert(
|
|
300
|
+
classic_alert,
|
|
301
|
+
owner_org: bool = False,
|
|
302
|
+
ai_insights: bool = True,
|
|
303
|
+
fragment_entities: bool = True,
|
|
304
|
+
triggered_by: bool = True,
|
|
305
|
+
html_tags: bool = False,
|
|
306
|
+
character_limit: int = None,
|
|
307
|
+
defang_iocs: bool = False,
|
|
308
|
+
) -> str:
|
|
309
|
+
"""Returns a markdown string representation of the ``ClassicAlert`` instance.
|
|
310
|
+
|
|
311
|
+
This function works on ``ClassicAlert`` instances returned by ``ClassicAlertMgr.fetch()``,
|
|
312
|
+
if you are passing the result of ``ClassicAlertMgr.search()`` make sure the ``search`` method
|
|
313
|
+
has been called with all the fields. Keep in mind that this will make the ``search`` slower.
|
|
314
|
+
|
|
315
|
+
Args:
|
|
316
|
+
classic_alert (ClassicAlert): ClassicAlert instance to create markdown from.
|
|
317
|
+
owner_org (bool, optional): Include owner org details. Defaults to False.
|
|
318
|
+
ai_insights (bool, optional): Include AI insights. Defaults to True.
|
|
319
|
+
fragment_entities (bool, optional): Include fragment entities. Defaults to True.
|
|
320
|
+
triggered_by (bool, optional): Include triggered by. Defaults to True.
|
|
321
|
+
html_tags (bool, optional): Include HTML tags in the markdown. Defaults to False.
|
|
322
|
+
character_limit (int, optional): Character limit for the markdown. Defaults to None.
|
|
323
|
+
defang_iocs (bool, optional): Defang IOCs in hits. Defaults to False.
|
|
324
|
+
|
|
325
|
+
Raises:
|
|
326
|
+
AlertMarkdownError: If fields are not available.
|
|
327
|
+
|
|
328
|
+
Returns:
|
|
329
|
+
str: Markdown representation of the alert.
|
|
330
|
+
"""
|
|
331
|
+
try:
|
|
332
|
+
entities_to_defang = _get_entities_to_defang(classic_alert)
|
|
333
|
+
|
|
334
|
+
md_maker = MarkdownMaker(
|
|
335
|
+
TRUNCATE_COMMENT.format(type_='alert', url=str(classic_alert.url.portal)),
|
|
336
|
+
character_limit=character_limit,
|
|
337
|
+
defang_iocs=defang_iocs,
|
|
338
|
+
iocs_to_defang=entities_to_defang,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
_add_summary_section(md_maker, classic_alert)
|
|
342
|
+
_add_owner_org_section(md_maker, classic_alert, owner_org)
|
|
343
|
+
_add_ai_insights_section(md_maker, classic_alert, ai_insights)
|
|
344
|
+
|
|
345
|
+
if classic_alert.enriched_entities:
|
|
346
|
+
_add_enriched_entities_sections(md_maker, classic_alert, html_tags)
|
|
347
|
+
else:
|
|
348
|
+
_add_hits_section_if_no_enriched_entities(
|
|
349
|
+
md_maker, classic_alert, fragment_entities, triggered_by, html_tags
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
return md_maker.format_output()
|
|
353
|
+
|
|
354
|
+
except AttributeError as ae:
|
|
355
|
+
message = (
|
|
356
|
+
f'Unable to create markdown for {classic_alert.id_}. '
|
|
357
|
+
f'Request all CA fields if you are working with search results. Error: {ae}'
|
|
358
|
+
)
|
|
359
|
+
raise AlertMarkdownError(message=message) from ae
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
from pydantic import Field, HttpUrl
|
|
18
|
+
|
|
19
|
+
from ..common_models import IdName, IdNameType, IdNameTypeDescription, RFBaseModel
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AlertReview(RFBaseModel):
|
|
23
|
+
assignee: Optional[str] = None
|
|
24
|
+
note: Optional[str] = None
|
|
25
|
+
status_in_portal: str
|
|
26
|
+
status: Optional[str] = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Organisation(RFBaseModel):
|
|
30
|
+
organisation_id: str
|
|
31
|
+
organisation_name: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class OwnerOrganisationDetails(RFBaseModel):
|
|
35
|
+
organisations: Optional[list[Organisation]] = []
|
|
36
|
+
enterprise_id: Optional[str] = None
|
|
37
|
+
enterprise_name: Optional[str] = None
|
|
38
|
+
owner_id: Optional[str] = None
|
|
39
|
+
owner_name: Optional[str] = None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class AlertURL(RFBaseModel):
|
|
43
|
+
api: HttpUrl
|
|
44
|
+
portal: HttpUrl
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class AlertAnalystNote(RFBaseModel):
|
|
48
|
+
id_: str = Field(alias='id')
|
|
49
|
+
url: AlertURL
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class PortalURL(RFBaseModel):
|
|
53
|
+
portal: HttpUrl
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class AlertDeprecation(RFBaseModel):
|
|
57
|
+
use_case_deprecation: Optional[str] = None
|
|
58
|
+
name: str
|
|
59
|
+
id_: str = Field(alias='id')
|
|
60
|
+
url: PortalURL
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class AlertDocument(RFBaseModel):
|
|
64
|
+
source: Optional[IdNameType] = None
|
|
65
|
+
title: Optional[str] = None
|
|
66
|
+
url: Optional[str] = None
|
|
67
|
+
authors: list[IdNameType]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class AlertAiInsight(RFBaseModel):
|
|
71
|
+
comment: Optional[str] = None
|
|
72
|
+
text: Optional[str] = None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class AlertLog(RFBaseModel):
|
|
76
|
+
note_author: Optional[str] = None
|
|
77
|
+
note_date: Optional[datetime] = None
|
|
78
|
+
status_date: Optional[datetime] = None
|
|
79
|
+
triggered: datetime
|
|
80
|
+
status_change_by: Optional[str] = None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class AlertSummary(RFBaseModel):
|
|
84
|
+
id_: str = Field(alias='id')
|
|
85
|
+
title: str
|
|
86
|
+
triggered: datetime
|
|
87
|
+
url: HttpUrl
|
|
88
|
+
type_: str = Field(alias='type')
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class AlertCounts(RFBaseModel):
|
|
92
|
+
returned: int
|
|
93
|
+
total: int
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class NotificationSettings(RFBaseModel):
|
|
97
|
+
email_subscribers: list[IdName]
|
|
98
|
+
mobile_subsribers: Optional[list[IdName]] = None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Evidence(RFBaseModel):
|
|
102
|
+
timestamp: datetime
|
|
103
|
+
mitigation_string: str
|
|
104
|
+
criticality_label: str
|
|
105
|
+
rule: str
|
|
106
|
+
evidence_string: str
|
|
107
|
+
criticality: int
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class EntityCriticality(RFBaseModel):
|
|
111
|
+
name: str
|
|
112
|
+
score: Optional[int] = None
|
|
113
|
+
last_triggered: datetime
|
|
114
|
+
triggered: datetime
|
|
115
|
+
level: int
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class ClassicAlertHit(RFBaseModel):
|
|
119
|
+
"""Validate data received from ``/v3/alerts/hits``, ``/v3/alert/search``, ``/v3/alert/{id}``."""
|
|
120
|
+
|
|
121
|
+
entities: list[IdNameTypeDescription]
|
|
122
|
+
document: AlertDocument
|
|
123
|
+
fragment: Optional[str] = None
|
|
124
|
+
id_: str = Field(alias='id')
|
|
125
|
+
language: Optional[str] = None
|
|
126
|
+
primary_entity: Optional[IdNameTypeDescription] = None
|
|
127
|
+
analyst_note: Optional[AlertAnalystNote] = None
|
|
128
|
+
alert_id: Optional[str] = None
|
|
129
|
+
index: Optional[int] = None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class EnrichedEntity(RFBaseModel):
|
|
133
|
+
evidence: list[Evidence]
|
|
134
|
+
references: list[ClassicAlertHit]
|
|
135
|
+
criticality: EntityCriticality
|
|
136
|
+
entity: IdNameType
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class TriggeredBy(RFBaseModel):
|
|
140
|
+
reference_id: str
|
|
141
|
+
triggered_by_strings: Optional[list[str]] = None
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from .collective_insights import CollectiveInsights
|
|
15
|
+
from .constants import (
|
|
16
|
+
DETECTION_SUB_TYPE_SIGMA,
|
|
17
|
+
DETECTION_SUB_TYPE_SNORT,
|
|
18
|
+
DETECTION_SUB_TYPE_YARA,
|
|
19
|
+
DETECTION_TYPE_CORRELATION,
|
|
20
|
+
DETECTION_TYPE_PLAYBOOK,
|
|
21
|
+
DETECTION_TYPE_RULE,
|
|
22
|
+
ENTITY_DOMAIN,
|
|
23
|
+
ENTITY_HASH,
|
|
24
|
+
ENTITY_IP,
|
|
25
|
+
ENTITY_URL,
|
|
26
|
+
ENTITY_VULNERABILITY,
|
|
27
|
+
)
|
|
28
|
+
from .errors import CollectiveInsightsError
|
|
29
|
+
from .insight import Insight, InsightsIn, InsightsOut
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
from pydantic import validate_call
|
|
19
|
+
|
|
20
|
+
from ..endpoints import EP_COLLECTIVE_INSIGHTS_DETECTIONS
|
|
21
|
+
from ..helpers import connection_exceptions, debug_call
|
|
22
|
+
from ..rf_client import RFClient
|
|
23
|
+
from .constants import SUMMARY_DEFAULT
|
|
24
|
+
from .errors import CollectiveInsightsError
|
|
25
|
+
from .insight import Insight, InsightsIn, InsightsOut
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CollectiveInsights:
|
|
29
|
+
"""Class for interacting with the Recorded Future Collective Insights API."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, rf_token: str = None):
|
|
32
|
+
"""Initializes the CollectiveInsights object.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
rf_token (str, optional): Recorded Future API token. Defaults to None
|
|
36
|
+
"""
|
|
37
|
+
self.log = logging.getLogger(__name__)
|
|
38
|
+
self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient()
|
|
39
|
+
|
|
40
|
+
@validate_call
|
|
41
|
+
@debug_call
|
|
42
|
+
def create(
|
|
43
|
+
self,
|
|
44
|
+
ioc_value: str,
|
|
45
|
+
ioc_type: str,
|
|
46
|
+
timestamp: str,
|
|
47
|
+
detection_type: str,
|
|
48
|
+
detection_sub_type: str = None,
|
|
49
|
+
detection_id: str = None,
|
|
50
|
+
detection_name: str = None,
|
|
51
|
+
ioc_field: str = None,
|
|
52
|
+
ioc_source_type: str = None,
|
|
53
|
+
incident_id: str = None,
|
|
54
|
+
incident_name: str = None,
|
|
55
|
+
incident_type: str = None,
|
|
56
|
+
mitre_codes: Union[list[str], str] = None,
|
|
57
|
+
malwares: Union[list[str], str] = None,
|
|
58
|
+
**kwargs,
|
|
59
|
+
) -> Insight:
|
|
60
|
+
"""Create a new Insight object.
|
|
61
|
+
|
|
62
|
+
Raises:
|
|
63
|
+
ValidationError if any supplied parameter is of incorrect type.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
Insight object.
|
|
67
|
+
"""
|
|
68
|
+
malwares = malwares if isinstance(malwares, list) else [malwares] if malwares else None
|
|
69
|
+
mitre_codes = (
|
|
70
|
+
mitre_codes if isinstance(mitre_codes, list) else [mitre_codes] if mitre_codes else None
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
incident = {'id': incident_id, 'type': incident_type, 'name': incident_name}
|
|
74
|
+
detection = {
|
|
75
|
+
'id': detection_id,
|
|
76
|
+
'name': detection_name,
|
|
77
|
+
'type': detection_type,
|
|
78
|
+
'sub_type': detection_sub_type,
|
|
79
|
+
}
|
|
80
|
+
ioc = {
|
|
81
|
+
'type': ioc_type,
|
|
82
|
+
'value': ioc_value,
|
|
83
|
+
'source_type': ioc_source_type,
|
|
84
|
+
'field': ioc_field,
|
|
85
|
+
}
|
|
86
|
+
data = {
|
|
87
|
+
'timestamp': timestamp,
|
|
88
|
+
'ioc': ioc,
|
|
89
|
+
'incident': incident,
|
|
90
|
+
'detection': detection,
|
|
91
|
+
'mitre_codes': mitre_codes,
|
|
92
|
+
'malwares': malwares,
|
|
93
|
+
}
|
|
94
|
+
data['incident'] = (
|
|
95
|
+
None
|
|
96
|
+
if isinstance(data['incident'], dict)
|
|
97
|
+
and all(sub_v is None for sub_v in data['incident'].values())
|
|
98
|
+
else data['incident']
|
|
99
|
+
)
|
|
100
|
+
if kwargs:
|
|
101
|
+
data.update(kwargs)
|
|
102
|
+
|
|
103
|
+
return Insight.model_validate(data)
|
|
104
|
+
|
|
105
|
+
@validate_call
|
|
106
|
+
@debug_call
|
|
107
|
+
@connection_exceptions(ignore_status_code=[], exception_to_raise=CollectiveInsightsError)
|
|
108
|
+
def submit(
|
|
109
|
+
self,
|
|
110
|
+
insight: Union[Insight, list[Insight]],
|
|
111
|
+
debug: bool = True,
|
|
112
|
+
organization_ids: list = None,
|
|
113
|
+
) -> InsightsIn:
|
|
114
|
+
"""Submit a detection or insight to Recorded Future Collective Insights API.
|
|
115
|
+
|
|
116
|
+
Endpoint:
|
|
117
|
+
``collective-insights/detections``
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
insight (list[Insight] or Insight): A detection/insight
|
|
121
|
+
debug (bool, optional): Determines if submission will show in SecOPS dashboard.
|
|
122
|
+
organization_ids (list, optional): Org ID. Defaults to None.
|
|
123
|
+
|
|
124
|
+
Raises:
|
|
125
|
+
CollectiveInsightsError: if connection error occurs.
|
|
126
|
+
ValidationError if any supplied parameter is of incorrect type.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
InsightsIn: response from Recorded Future API
|
|
130
|
+
"""
|
|
131
|
+
if not insight:
|
|
132
|
+
raise ValueError('Insight cannot be empty')
|
|
133
|
+
|
|
134
|
+
insight = insight if isinstance(insight, list) else [insight]
|
|
135
|
+
|
|
136
|
+
ci_data = self._prepare_ci_request(insight, debug, organization_ids)
|
|
137
|
+
response = self.rf_client.request(
|
|
138
|
+
'post',
|
|
139
|
+
url=EP_COLLECTIVE_INSIGHTS_DETECTIONS,
|
|
140
|
+
data=ci_data.json(),
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
return InsightsIn.model_validate(response.json())
|
|
144
|
+
|
|
145
|
+
def _prepare_ci_request(
|
|
146
|
+
self,
|
|
147
|
+
insight: list[Insight],
|
|
148
|
+
debug: bool = True,
|
|
149
|
+
organization_ids: list = None,
|
|
150
|
+
) -> InsightsOut:
|
|
151
|
+
params = {'options': {}}
|
|
152
|
+
|
|
153
|
+
params['data'] = [ins.json() for ins in insight]
|
|
154
|
+
|
|
155
|
+
if organization_ids is not None and len(organization_ids):
|
|
156
|
+
params['organization_ids'] = organization_ids
|
|
157
|
+
params['options']['debug'] = debug
|
|
158
|
+
|
|
159
|
+
# We always have summary of the submission
|
|
160
|
+
params['options']['summary'] = SUMMARY_DEFAULT
|
|
161
|
+
|
|
162
|
+
self.log.debug(f'Params for submission: \n{json.dumps(params, indent=2)}')
|
|
163
|
+
|
|
164
|
+
return InsightsOut.model_validate(params)
|