psengine 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psengine/__init__.py +22 -0
- psengine/_sdk_id.py +16 -0
- psengine/_version.py +14 -0
- psengine/analyst_notes/__init__.py +32 -0
- psengine/analyst_notes/constants.py +15 -0
- psengine/analyst_notes/errors.py +42 -0
- psengine/analyst_notes/helpers.py +90 -0
- psengine/analyst_notes/models.py +219 -0
- psengine/analyst_notes/note.py +149 -0
- psengine/analyst_notes/note_mgr.py +400 -0
- psengine/base_http_client.py +285 -0
- psengine/classic_alerts/__init__.py +24 -0
- psengine/classic_alerts/classic_alert.py +275 -0
- psengine/classic_alerts/classic_alert_mgr.py +507 -0
- psengine/classic_alerts/constants.py +31 -0
- psengine/classic_alerts/errors.py +38 -0
- psengine/classic_alerts/helpers.py +87 -0
- psengine/classic_alerts/markdown/__init__.py +13 -0
- psengine/classic_alerts/markdown/markdown.py +359 -0
- psengine/classic_alerts/models.py +141 -0
- psengine/collective_insights/__init__.py +29 -0
- psengine/collective_insights/collective_insights.py +164 -0
- psengine/collective_insights/constants.py +44 -0
- psengine/collective_insights/errors.py +18 -0
- psengine/collective_insights/insight.py +89 -0
- psengine/collective_insights/models.py +81 -0
- psengine/common_models.py +89 -0
- psengine/config/__init__.py +15 -0
- psengine/config/config.py +284 -0
- psengine/config/errors.py +18 -0
- psengine/constants.py +63 -0
- psengine/detection/__init__.py +17 -0
- psengine/detection/detection_mgr.py +135 -0
- psengine/detection/detection_rule.py +85 -0
- psengine/detection/errors.py +26 -0
- psengine/detection/helpers.py +56 -0
- psengine/detection/models.py +47 -0
- psengine/endpoints.py +98 -0
- psengine/enrich/__init__.py +28 -0
- psengine/enrich/constants.py +73 -0
- psengine/enrich/errors.py +26 -0
- psengine/enrich/lookup.py +299 -0
- psengine/enrich/lookup_mgr.py +341 -0
- psengine/enrich/models/__init__.py +13 -0
- psengine/enrich/models/base_enriched_entity.py +43 -0
- psengine/enrich/models/lookup.py +271 -0
- psengine/enrich/models/soar.py +138 -0
- psengine/enrich/soar.py +89 -0
- psengine/enrich/soar_mgr.py +176 -0
- psengine/entity_lists/__init__.py +16 -0
- psengine/entity_lists/constants.py +19 -0
- psengine/entity_lists/entity_list.py +435 -0
- psengine/entity_lists/entity_list_mgr.py +185 -0
- psengine/entity_lists/errors.py +26 -0
- psengine/entity_lists/models.py +87 -0
- psengine/entity_match/__init__.py +16 -0
- psengine/entity_match/entity_match.py +90 -0
- psengine/entity_match/entity_match_mgr.py +235 -0
- psengine/entity_match/errors.py +18 -0
- psengine/entity_match/models.py +22 -0
- psengine/errors.py +41 -0
- psengine/helpers/__init__.py +23 -0
- psengine/helpers/helpers.py +471 -0
- psengine/logger/__init__.py +15 -0
- psengine/logger/constants.py +39 -0
- psengine/logger/errors.py +18 -0
- psengine/logger/rf_logger.py +148 -0
- psengine/markdown/__init__.py +21 -0
- psengine/markdown/markdown.py +169 -0
- psengine/markdown/models.py +22 -0
- psengine/playbook_alerts/__init__.py +34 -0
- psengine/playbook_alerts/constants.py +35 -0
- psengine/playbook_alerts/errors.py +35 -0
- psengine/playbook_alerts/helpers.py +80 -0
- psengine/playbook_alerts/mappings.py +44 -0
- psengine/playbook_alerts/markdown/__init__.py +13 -0
- psengine/playbook_alerts/markdown/markdown.py +98 -0
- psengine/playbook_alerts/markdown/markdown_code_repo.py +64 -0
- psengine/playbook_alerts/markdown/markdown_domain_abuse.py +118 -0
- psengine/playbook_alerts/markdown/markdown_identity_exposure.py +158 -0
- psengine/playbook_alerts/models/__init__.py +36 -0
- psengine/playbook_alerts/models/common_models.py +18 -0
- psengine/playbook_alerts/models/panel_log.py +329 -0
- psengine/playbook_alerts/models/panel_status.py +70 -0
- psengine/playbook_alerts/models/pba_code_repo_leak.py +52 -0
- psengine/playbook_alerts/models/pba_cyber_vulnerability.py +53 -0
- psengine/playbook_alerts/models/pba_domain_abuse.py +139 -0
- psengine/playbook_alerts/models/pba_identity_exposures.py +93 -0
- psengine/playbook_alerts/models/pba_third_party_risk.py +103 -0
- psengine/playbook_alerts/models/search_endpoint.py +68 -0
- psengine/playbook_alerts/pa_category.py +37 -0
- psengine/playbook_alerts/playbook_alert_mgr.py +593 -0
- psengine/playbook_alerts/playbook_alerts.py +393 -0
- psengine/rf_client.py +430 -0
- psengine/risklists/__init__.py +17 -0
- psengine/risklists/constants.py +15 -0
- psengine/risklists/errors.py +20 -0
- psengine/risklists/models.py +65 -0
- psengine/risklists/risklist_mgr.py +156 -0
- psengine/stix2/__init__.py +21 -0
- psengine/stix2/base_stix_entity.py +62 -0
- psengine/stix2/complex_entity.py +372 -0
- psengine/stix2/constants.py +81 -0
- psengine/stix2/enriched_indicator.py +261 -0
- psengine/stix2/errors.py +22 -0
- psengine/stix2/helpers.py +68 -0
- psengine/stix2/rf_bundle.py +240 -0
- psengine/stix2/simple_entity.py +145 -0
- psengine/stix2/util.py +53 -0
- psengine-2.0.4.dist-info/METADATA +189 -0
- psengine-2.0.4.dist-info/RECORD +115 -0
- psengine-2.0.4.dist-info/WHEEL +5 -0
- psengine-2.0.4.dist-info/entry_points.txt +2 -0
- psengine-2.0.4.dist-info/licenses/LICENSE +21 -0
- psengine-2.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
from functools import total_ordering
|
|
14
|
+
from typing import Optional, Union
|
|
15
|
+
|
|
16
|
+
from pydantic import Field
|
|
17
|
+
|
|
18
|
+
from ..common_models import IdNameType, IdNameTypeDescription, RFBaseModel
|
|
19
|
+
from ..constants import TIMESTAMP_STR
|
|
20
|
+
from .models.base_enriched_entity import BaseEnrichedEntity
|
|
21
|
+
from .models.lookup import (
|
|
22
|
+
CVSS,
|
|
23
|
+
CVSSV3,
|
|
24
|
+
CVSSRating,
|
|
25
|
+
DnsPortCert,
|
|
26
|
+
EnterpriseList,
|
|
27
|
+
EntityRisk,
|
|
28
|
+
IPLocation,
|
|
29
|
+
LinkedMalware,
|
|
30
|
+
Links,
|
|
31
|
+
NvdReference,
|
|
32
|
+
RawRisk,
|
|
33
|
+
RiskMapping,
|
|
34
|
+
RiskyCIDRPIP,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class EnrichedIP(BaseEnrichedEntity):
|
|
39
|
+
"""IP Enriched by ``/v2/ip/{ip}`` endpoint. Inherit behaviours from ``BaseEnrichedEntity``."""
|
|
40
|
+
|
|
41
|
+
risk: Optional[EntityRisk] = None
|
|
42
|
+
links: Optional[Links] = None
|
|
43
|
+
enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
|
|
44
|
+
threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
|
|
45
|
+
risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
|
|
46
|
+
dns_port_cert: Optional[DnsPortCert] = Field(alias='dnsPortCert', default=None)
|
|
47
|
+
location: Optional[IPLocation] = None
|
|
48
|
+
risky_cidr_ips: Optional[list[RiskyCIDRPIP]] = Field(alias='riskyCIDRIPs', default=None)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class EnrichedDomain(BaseEnrichedEntity):
|
|
52
|
+
"""Domain Enriched by ``/v2/domain/{domain}`` endpoint.
|
|
53
|
+
Inherit behaviours from ``BaseEnrichedEntity``.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
risk: Optional[EntityRisk] = None
|
|
57
|
+
links: Optional[Links] = None
|
|
58
|
+
enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
|
|
59
|
+
threat_lists: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
|
|
60
|
+
risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class EnrichedURL(BaseEnrichedEntity):
|
|
64
|
+
"""URL Enriched by ``/v2/url/{url}`` endpoint.
|
|
65
|
+
Inherit behaviours from ``BaseEnrichedEntity``.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
risk: Optional[EntityRisk] = None
|
|
69
|
+
links: Optional[Links] = None
|
|
70
|
+
enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
|
|
71
|
+
risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class EnrichedHash(BaseEnrichedEntity):
|
|
75
|
+
"""Hash Enriched by ``/v2/hash/{hash}`` endpoint.
|
|
76
|
+
Inherit behaviours from ``BaseEnrichedEntity``.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
risk: Optional[EntityRisk] = None
|
|
80
|
+
links: Optional[Links] = None
|
|
81
|
+
enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
|
|
82
|
+
threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
|
|
83
|
+
risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
|
|
84
|
+
hash_algorithm: Optional[str] = Field(alias='hashAlgorithm', default=None)
|
|
85
|
+
file_hashes: Optional[list[str]] = Field(alias='fileHashes', default=None)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class EnrichedVulnerability(BaseEnrichedEntity):
|
|
89
|
+
"""Vulnerability Enriched by ``/v2/vulnerability/{cve}`` endpoint.
|
|
90
|
+
Inherit behaviours from ``BaseEnrichedEntity``.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
risk: Optional[EntityRisk] = None
|
|
94
|
+
links: Optional[Links] = None
|
|
95
|
+
enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
|
|
96
|
+
threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
|
|
97
|
+
risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
|
|
98
|
+
common_names: Optional[list[str]] = Field(alias='commonNames', default=None)
|
|
99
|
+
lifecycle_stage: Optional[str] = Field(alias='lifecycleStage', default=None)
|
|
100
|
+
linked_malware: Optional[LinkedMalware] = Field(alias='linkedMalware', default=None)
|
|
101
|
+
cpe: Optional[list[str]] = None
|
|
102
|
+
cpe_22_uri: Optional[list[str]] = Field(alias='cpe22uri', default=None)
|
|
103
|
+
cvss: Optional[CVSS] = None
|
|
104
|
+
cvss_ratings: list[CVSSRating] = Field(alias='cvssRatings', default=None)
|
|
105
|
+
cvssv3: Optional[CVSSV3] = None
|
|
106
|
+
nvd_description: Optional[str] = Field(alias='nvdDescription', default=None)
|
|
107
|
+
nvd_references: Optional[list[NvdReference]] = Field(alias='nvdReferences', default=None)
|
|
108
|
+
raw_risk: Optional[list[RawRisk]] = Field(alias='rawrisk', default=None)
|
|
109
|
+
related_links: Optional[list[str]] = Field(alias='relatedLinks', default=None)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class EnrichedMalware(BaseEnrichedEntity):
|
|
113
|
+
"""Malware Enriched by ``/v2/malware/{id}`` endpoint.
|
|
114
|
+
Inherit behaviours from ``BaseEnrichedEntity``.
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
links: Optional[Links] = None
|
|
118
|
+
categories: Optional[list[IdNameType]] = None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class EnrichedCompany(BaseEnrichedEntity):
|
|
122
|
+
"""Company Enriched by ``/v2/company/{id}`` and ``/v2/company/by_domain/{domain}`` endpoint.
|
|
123
|
+
Inherit behaviours from ``BaseEnrichedEntity``.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
risk: Optional[EntityRisk] = None
|
|
127
|
+
curated: Optional[bool] = None
|
|
128
|
+
threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
|
|
129
|
+
risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
_EnrichmentObjectType = Union[
|
|
133
|
+
EnrichedCompany,
|
|
134
|
+
EnrichedDomain,
|
|
135
|
+
EnrichedIP,
|
|
136
|
+
EnrichedHash,
|
|
137
|
+
EnrichedMalware,
|
|
138
|
+
EnrichedURL,
|
|
139
|
+
EnrichedVulnerability,
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@total_ordering
|
|
144
|
+
class EnrichmentData(RFBaseModel):
|
|
145
|
+
"""Model for the custom return of lookup of IOCs.
|
|
146
|
+
|
|
147
|
+
Methods:
|
|
148
|
+
__hash__:
|
|
149
|
+
Returns a hash value based on the content's attributes.
|
|
150
|
+
|
|
151
|
+
- If ``content`` is an instance of ``EnrichedMalware``:
|
|
152
|
+
The hash is calculated using the entity ``id_`` and the last seen timestamp.
|
|
153
|
+
- Else:
|
|
154
|
+
The hash includes the entity ``id_``, risk score, and the last seen timestamp.
|
|
155
|
+
|
|
156
|
+
__eq__:
|
|
157
|
+
Checks equality between two ``EnrichmentData`` instances based on their ``content``.
|
|
158
|
+
|
|
159
|
+
- If ``content`` is an instance of ``EnrichedMalware``:
|
|
160
|
+
Equality is determined by comparing the entity name and the last seen timestamp.
|
|
161
|
+
- Else:
|
|
162
|
+
Equality is determined by comparing the entity name, last seen timestamp and
|
|
163
|
+
risk score.
|
|
164
|
+
|
|
165
|
+
__gt__:
|
|
166
|
+
Defines a greater-than comparison between ``EnrichmentData`` instances based on their
|
|
167
|
+
``content``.
|
|
168
|
+
|
|
169
|
+
- If ``content`` is an instance of ``EnrichedMalware``:
|
|
170
|
+
Comparison is based on the last seen timestamp and entity name.
|
|
171
|
+
- Else:
|
|
172
|
+
Comparison is based on the last seen timestamp, entity name, and risk score.
|
|
173
|
+
|
|
174
|
+
__str__ and __repr__:
|
|
175
|
+
Returns a string representation of the ``EnrichmentData`` instance.
|
|
176
|
+
__repr__ is used for printing elements properly within a list.
|
|
177
|
+
|
|
178
|
+
- If ``content`` is an instance of ``EnrichedMalware``:
|
|
179
|
+
Includes class name, entity name, and last seen timestamp.
|
|
180
|
+
|
|
181
|
+
.. code-block:: python
|
|
182
|
+
|
|
183
|
+
>>> print(enrichment_data)
|
|
184
|
+
EnrichedMalware: exampleMalware, Last Seen: 2024-05-21 01:30:00PM
|
|
185
|
+
|
|
186
|
+
- Else:
|
|
187
|
+
Includes class name, entity name, risk score, and last seen timestamp.
|
|
188
|
+
|
|
189
|
+
.. code-block:: python
|
|
190
|
+
|
|
191
|
+
>>> print(enrichment_data)
|
|
192
|
+
EnrichedIP: 1.1.1.1, Risk Score: 85, Last Seen: 2024-05-21 01:30:00PM
|
|
193
|
+
|
|
194
|
+
Total Ordering:
|
|
195
|
+
The ordering of ``EnrichmentData`` instances is determined by the ``content``'s last
|
|
196
|
+
seen timestamp.
|
|
197
|
+
|
|
198
|
+
- If ``content`` is an instance of ``EnrichedMalware``:
|
|
199
|
+
If two instances have the same last seen timestamp, their entity name is used as a
|
|
200
|
+
secondary criterion.
|
|
201
|
+
|
|
202
|
+
- Else:
|
|
203
|
+
If two instances have the same last seen timestamp, their entity name and risk score are
|
|
204
|
+
used as secondary criteria.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
entity: str
|
|
208
|
+
entity_type: Optional[str]
|
|
209
|
+
is_enriched: bool
|
|
210
|
+
content: Union[str, _EnrichmentObjectType]
|
|
211
|
+
|
|
212
|
+
def __hash__(self):
|
|
213
|
+
if isinstance(self.content, EnrichedMalware):
|
|
214
|
+
return hash(
|
|
215
|
+
(
|
|
216
|
+
self.content.entity.id_,
|
|
217
|
+
self.content.timestamps.last_seen,
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
elif isinstance(self.content, str):
|
|
221
|
+
return hash(self.entity)
|
|
222
|
+
|
|
223
|
+
return hash(
|
|
224
|
+
(
|
|
225
|
+
self.content.entity.id_,
|
|
226
|
+
self.content.risk.score,
|
|
227
|
+
self.content.timestamps.last_seen,
|
|
228
|
+
)
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
def __eq__(self, other: 'EnrichmentData'):
|
|
232
|
+
if isinstance(self.content, EnrichedMalware):
|
|
233
|
+
return (
|
|
234
|
+
self.content.entity.name,
|
|
235
|
+
self.content.timestamps.last_seen,
|
|
236
|
+
) == (
|
|
237
|
+
other.content.entity.name,
|
|
238
|
+
other.content.timestamps.last_seen,
|
|
239
|
+
)
|
|
240
|
+
elif isinstance(self.content, str):
|
|
241
|
+
return self.entity == other.entity
|
|
242
|
+
return (
|
|
243
|
+
self.content.risk.score,
|
|
244
|
+
self.content.entity.name,
|
|
245
|
+
self.content.timestamps.last_seen,
|
|
246
|
+
) == (
|
|
247
|
+
other.content.risk.score,
|
|
248
|
+
other.content.entity.name,
|
|
249
|
+
other.content.timestamps.last_seen,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def __gt__(self, other: 'EnrichmentData'):
|
|
253
|
+
if isinstance(self.content, EnrichedMalware):
|
|
254
|
+
return (
|
|
255
|
+
self.content.timestamps.last_seen,
|
|
256
|
+
self.content.entity.name,
|
|
257
|
+
) > (
|
|
258
|
+
other.content.timestamps.last_seen,
|
|
259
|
+
other.content.entity.name,
|
|
260
|
+
)
|
|
261
|
+
elif isinstance(self.content, str):
|
|
262
|
+
return self.entity > other.entity
|
|
263
|
+
return (
|
|
264
|
+
self.content.risk.score,
|
|
265
|
+
self.content.timestamps.last_seen,
|
|
266
|
+
self.content.entity.name,
|
|
267
|
+
) > (
|
|
268
|
+
other.content.risk.score,
|
|
269
|
+
other.content.timestamps.last_seen,
|
|
270
|
+
other.content.entity.name,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
def __str__(self):
|
|
274
|
+
if isinstance(self.content, EnrichedMalware):
|
|
275
|
+
return (
|
|
276
|
+
f'{self.content.__class__.__name__}: {self.content.entity.name}, '
|
|
277
|
+
f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
|
|
278
|
+
)
|
|
279
|
+
elif isinstance(self.content, str):
|
|
280
|
+
return f'{self.entity}: {self.content}'
|
|
281
|
+
return (
|
|
282
|
+
f'{self.content.__class__.__name__}: {self.content.entity.name}, '
|
|
283
|
+
f'Risk Score: {self.content.risk.score}, '
|
|
284
|
+
f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
def __repr__(self):
|
|
288
|
+
if isinstance(self.content, EnrichedMalware):
|
|
289
|
+
return (
|
|
290
|
+
f'{self.content.__class__.__name__}: {self.content.entity.name}, '
|
|
291
|
+
f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
|
|
292
|
+
)
|
|
293
|
+
elif isinstance(self.content, str):
|
|
294
|
+
return f'{self.entity}: {self.content}'
|
|
295
|
+
return (
|
|
296
|
+
f'{self.content.__class__.__name__}: {self.content.entity.name}, '
|
|
297
|
+
f'Risk Score: {self.content.risk.score}, '
|
|
298
|
+
f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
|
|
299
|
+
)
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
from typing import Optional
|
|
16
|
+
from urllib.parse import quote
|
|
17
|
+
|
|
18
|
+
from pydantic import validate_call
|
|
19
|
+
|
|
20
|
+
from ..endpoints import CONNECT_API_BASE_URL
|
|
21
|
+
from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call
|
|
22
|
+
from ..rf_client import RFClient
|
|
23
|
+
from .constants import (
|
|
24
|
+
ALLOWED_ENTITIES,
|
|
25
|
+
ENTITY_FIELDS,
|
|
26
|
+
IOC_TO_MODEL,
|
|
27
|
+
MALWARE_FIELDS,
|
|
28
|
+
MESSAGE_404,
|
|
29
|
+
TYPE_MAPPING,
|
|
30
|
+
)
|
|
31
|
+
from .errors import EnrichmentLookupError
|
|
32
|
+
from .lookup import EnrichmentData
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LookupMgr:
|
|
36
|
+
"""Enrichment of a single or a group of Entities."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, rf_token: str = None):
|
|
39
|
+
"""Initializes the LookupMgr object.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
rf_token (str, optional): Recorded Future API token. Defaults to None
|
|
43
|
+
"""
|
|
44
|
+
self.log = logging.getLogger(__name__)
|
|
45
|
+
self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient()
|
|
46
|
+
|
|
47
|
+
@validate_call
|
|
48
|
+
@debug_call
|
|
49
|
+
def lookup(
|
|
50
|
+
self,
|
|
51
|
+
entity: str,
|
|
52
|
+
entity_type: ALLOWED_ENTITIES,
|
|
53
|
+
fields: Optional[list[str]] = None,
|
|
54
|
+
) -> EnrichmentData:
|
|
55
|
+
"""Perform lookup of an entity based on its id or name.
|
|
56
|
+
``entity`` can contain both a Recorded Future ID or a entity name. See example below.
|
|
57
|
+
|
|
58
|
+
The entity_type must always be specified.
|
|
59
|
+
The allowed values are:
|
|
60
|
+
|
|
61
|
+
- ``company``,
|
|
62
|
+
- ``Company``,
|
|
63
|
+
- ``company_by_domain``,
|
|
64
|
+
- ``CyberVulnerability``,
|
|
65
|
+
- ``domain``,
|
|
66
|
+
- ``hash``,
|
|
67
|
+
- ``Hash``,
|
|
68
|
+
- ``InternetDomainName``,
|
|
69
|
+
- ``ip``,
|
|
70
|
+
- ``IpAddress``,
|
|
71
|
+
- ``malware``,
|
|
72
|
+
- ``Malware``,
|
|
73
|
+
- ``Organization``,
|
|
74
|
+
- ``url``,
|
|
75
|
+
- ``URL``,
|
|
76
|
+
- ``vulnerability``,
|
|
77
|
+
|
|
78
|
+
If ``fields`` parameter is specified, it will be added to the mandatory fields, which
|
|
79
|
+
are:
|
|
80
|
+
|
|
81
|
+
- for every entity except malware: ``['entity', 'risk', 'timestamps']``
|
|
82
|
+
- for malware: ``['entity', 'timestamps']``
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
Examples:
|
|
86
|
+
Performing a lookup using different ``entity`` type:
|
|
87
|
+
|
|
88
|
+
.. code-block:: python
|
|
89
|
+
:linenos:
|
|
90
|
+
|
|
91
|
+
mgr.lookup('idn:google.com', 'domain')
|
|
92
|
+
mgr.lookup('google.com', 'domain')
|
|
93
|
+
mgr.lookup('A_BCDE', 'company')
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
Performing a lookup of a ``company_by_domain`` specifying additional fields:
|
|
97
|
+
|
|
98
|
+
.. code-block:: python
|
|
99
|
+
:linenos:
|
|
100
|
+
|
|
101
|
+
mgr.lookup(
|
|
102
|
+
'recordedfuture.com',
|
|
103
|
+
entity_type='company_by_domain',
|
|
104
|
+
fields=['curated']
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
The output is always ``EnrichmentData`` object with a format like below:
|
|
108
|
+
If a 404 is received:
|
|
109
|
+
|
|
110
|
+
.. code-block:: python
|
|
111
|
+
|
|
112
|
+
{
|
|
113
|
+
'entity': entity,
|
|
114
|
+
'entity_type': entity_type,
|
|
115
|
+
'is_enriched': False,
|
|
116
|
+
'content': '404 received. Nothing known on this entity',
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
If a 200 is received:
|
|
120
|
+
|
|
121
|
+
.. code-block:: python
|
|
122
|
+
|
|
123
|
+
{
|
|
124
|
+
'entity': entity,
|
|
125
|
+
'entity_type': entity_type,
|
|
126
|
+
'is_enriched': True,
|
|
127
|
+
'content': the enriched data model
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
To write an enriched object to file:
|
|
131
|
+
|
|
132
|
+
.. code-block:: python
|
|
133
|
+
:linenos:
|
|
134
|
+
|
|
135
|
+
from pathlib import Path
|
|
136
|
+
from json import dump
|
|
137
|
+
from psengine.enrich import LookupMgr
|
|
138
|
+
|
|
139
|
+
mgr = LookupMgr()
|
|
140
|
+
OUTPUT_DIR = Path('your' / 'path')
|
|
141
|
+
OUTPUT_DIR.mkdir(exists_ok=True)
|
|
142
|
+
ip = mgr.lookup('1.1.1.1', 'ip')
|
|
143
|
+
(OUTPUT_DIR / f'{ip.entity}.json').write_text(dumps(ip.json(), indent=2))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
Endpoint:
|
|
147
|
+
``v2/{entity_type}/{entity}``
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
entity (str): Name or RFID of the entity.
|
|
151
|
+
entity_type (str): Type of the entity.
|
|
152
|
+
fields (List[str], optional): Fields for the entity to enrich.
|
|
153
|
+
EnrichmentData: An object containing the entity details.
|
|
154
|
+
|
|
155
|
+
Raises:
|
|
156
|
+
ValidationError If a field does not match the type hint.
|
|
157
|
+
EnrichmentLookupError: if a lookup terminates with a non 200 or 404 return code.
|
|
158
|
+
"""
|
|
159
|
+
default_fields = MALWARE_FIELDS if entity_type.lower() == 'malware' else ENTITY_FIELDS
|
|
160
|
+
fields = fields if fields else default_fields
|
|
161
|
+
fields = self._merge_fields(fields, default_fields)
|
|
162
|
+
|
|
163
|
+
return self._lookup(entity, entity_type, fields)
|
|
164
|
+
|
|
165
|
+
@validate_call
|
|
166
|
+
@debug_call
|
|
167
|
+
def lookup_bulk(
|
|
168
|
+
self,
|
|
169
|
+
entity: list[str],
|
|
170
|
+
entity_type: ALLOWED_ENTITIES,
|
|
171
|
+
fields: list[str] = ENTITY_FIELDS,
|
|
172
|
+
max_workers: Optional[int] = 0,
|
|
173
|
+
) -> list[EnrichmentData]:
|
|
174
|
+
"""Perform lookup of multiple entities based on ids or name.
|
|
175
|
+
``entity`` can contain both a Recorded Future ID or a entity name.
|
|
176
|
+
The entities must be of the same ``entity_type``. See example below.
|
|
177
|
+
|
|
178
|
+
The ``entity_type`` must always be specified.
|
|
179
|
+
The allowed values are:
|
|
180
|
+
|
|
181
|
+
- ``company``,
|
|
182
|
+
- ``Company``,
|
|
183
|
+
- ``company_by_domain``,
|
|
184
|
+
- ``CyberVulnerability``,
|
|
185
|
+
- ``domain``,
|
|
186
|
+
- ``hash``,
|
|
187
|
+
- ``Hash``,
|
|
188
|
+
- ``InternetDomainName``,
|
|
189
|
+
- ``ip``,
|
|
190
|
+
- ``IpAddress``,
|
|
191
|
+
- ``malware``,
|
|
192
|
+
- ``Malware``,
|
|
193
|
+
- ``Organization``,
|
|
194
|
+
- ``url``,
|
|
195
|
+
- ``URL``,
|
|
196
|
+
- ``vulnerability``,
|
|
197
|
+
|
|
198
|
+
If ``fields`` parameter is specified, it will be added to the mandatory fields, which
|
|
199
|
+
are:
|
|
200
|
+
|
|
201
|
+
- for every entity except malware: ``['entity', 'risk', 'timestamps']``
|
|
202
|
+
- for malware: ``['entity', 'timestamps']``
|
|
203
|
+
|
|
204
|
+
Examples:
|
|
205
|
+
Multiple IOC types enrichment:
|
|
206
|
+
|
|
207
|
+
.. code-block:: python
|
|
208
|
+
:linenos:
|
|
209
|
+
|
|
210
|
+
data = {
|
|
211
|
+
'IpAddress': ['1.1.1.1', '2.2.2.2'],
|
|
212
|
+
'InternetDomainName': [ 'google.com', 'facebook.com']
|
|
213
|
+
}
|
|
214
|
+
results = []
|
|
215
|
+
for entity_type, entities in data.items():
|
|
216
|
+
results.extend(mgr.lookup_bulk(entities, entity_type)
|
|
217
|
+
|
|
218
|
+
To write an enriched object to file:
|
|
219
|
+
|
|
220
|
+
.. code-block:: python
|
|
221
|
+
:linenos:
|
|
222
|
+
|
|
223
|
+
from pathlib import Path
|
|
224
|
+
from json import dump
|
|
225
|
+
from psengine.enrich import LookupMgr
|
|
226
|
+
|
|
227
|
+
mgr = LookupMgr()
|
|
228
|
+
OUTPUT_DIR = Path('your' / 'path')
|
|
229
|
+
OUTPUT_DIR.mkdir(exists_ok=True)
|
|
230
|
+
data = mgr.lookup_bulk(['1.1.1.1', '8.8.8.8'], 'ip')
|
|
231
|
+
for ip in data:
|
|
232
|
+
(OUTPUT_DIR / f'{ip.entity}.json').write_text(dumps(ip.json(), indent=2))
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
The output is always a list of ``EnrichmentData`` object with a format like below:
|
|
236
|
+
If a 404 is received:
|
|
237
|
+
|
|
238
|
+
.. code-block:: python
|
|
239
|
+
|
|
240
|
+
{
|
|
241
|
+
'entity': entity,
|
|
242
|
+
'entity_type': entity_type,
|
|
243
|
+
'is_enriched': False,
|
|
244
|
+
'content': '404 received. Nothing known on this entity',
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
If a 200 is received:
|
|
248
|
+
|
|
249
|
+
.. code-block:: python
|
|
250
|
+
|
|
251
|
+
{
|
|
252
|
+
'entity': entity,
|
|
253
|
+
'entity_type': entity_type,
|
|
254
|
+
'is_enriched': True,
|
|
255
|
+
'content': the enriched data model
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
Multithreaded examples:
|
|
259
|
+
|
|
260
|
+
.. code-block:: python
|
|
261
|
+
|
|
262
|
+
mgr.lookup_bulk(['google.com', 'facebook.com'], 'domain', max_workers=10)
|
|
263
|
+
|
|
264
|
+
Endpoint:
|
|
265
|
+
``v2/{entity_type}/{entity}``
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
entity (List[str]): list of names or RFIDs.
|
|
270
|
+
entity_type (List[str]): Type of the entities.
|
|
271
|
+
fields (List[str], optional): Fields for the entities to enrich.
|
|
272
|
+
max_workers (int, optional): number of workers to multithread requests.
|
|
273
|
+
|
|
274
|
+
Returns:
|
|
275
|
+
List[EnrichmentData]: A list of object containing the entity details.
|
|
276
|
+
|
|
277
|
+
Raises:
|
|
278
|
+
ValidationError If a field does not match the type hint.
|
|
279
|
+
EnrichmentLookupError: if a lookup terminates with a non 200 or 404 return code.
|
|
280
|
+
"""
|
|
281
|
+
default_fields = MALWARE_FIELDS if entity_type.lower() == 'malware' else ENTITY_FIELDS
|
|
282
|
+
fields = fields if fields else default_fields
|
|
283
|
+
fields = self._merge_fields(fields, default_fields)
|
|
284
|
+
if max_workers:
|
|
285
|
+
res = MultiThreadingHelper.multithread_it(
|
|
286
|
+
max_workers,
|
|
287
|
+
self._lookup,
|
|
288
|
+
iterator=entity,
|
|
289
|
+
entity_type=entity_type,
|
|
290
|
+
fields=fields,
|
|
291
|
+
)
|
|
292
|
+
else:
|
|
293
|
+
res = [self._lookup(entity, entity_type, fields) for entity in entity]
|
|
294
|
+
|
|
295
|
+
return res
|
|
296
|
+
|
|
297
|
+
def _lookup(
|
|
298
|
+
self,
|
|
299
|
+
entity: str,
|
|
300
|
+
entity_type: str,
|
|
301
|
+
fields: list,
|
|
302
|
+
):
|
|
303
|
+
entity_type = TYPE_MAPPING.get(entity_type, entity_type)
|
|
304
|
+
|
|
305
|
+
enriched = self._fetch_data(
|
|
306
|
+
entity=entity,
|
|
307
|
+
entity_type=entity_type,
|
|
308
|
+
fields=fields,
|
|
309
|
+
)
|
|
310
|
+
if not enriched:
|
|
311
|
+
enriched = EnrichmentData(
|
|
312
|
+
entity=entity,
|
|
313
|
+
entity_type=entity_type,
|
|
314
|
+
is_enriched=False,
|
|
315
|
+
content=MESSAGE_404,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
return enriched
|
|
319
|
+
|
|
320
|
+
@connection_exceptions(ignore_status_code=[404], exception_to_raise=EnrichmentLookupError)
|
|
321
|
+
@debug_call
|
|
322
|
+
def _fetch_data(self, entity: str, entity_type: str, fields: list) -> EnrichmentData:
|
|
323
|
+
"""Perform the actual lookup. If a 404 is returned, return None."""
|
|
324
|
+
encoded_entity = quote(entity, safe='.')
|
|
325
|
+
entity_type = 'company/by_domain' if entity_type == 'company_by_domain' else entity_type
|
|
326
|
+
|
|
327
|
+
url = f'{CONNECT_API_BASE_URL}/{entity_type}/{encoded_entity}'
|
|
328
|
+
|
|
329
|
+
params = {}
|
|
330
|
+
params['fields'] = ','.join(fields)
|
|
331
|
+
|
|
332
|
+
response = self.rf_client.request('get', url, params=params).json()
|
|
333
|
+
return EnrichmentData(
|
|
334
|
+
entity=entity,
|
|
335
|
+
entity_type=entity_type,
|
|
336
|
+
is_enriched=True,
|
|
337
|
+
content=IOC_TO_MODEL[entity_type].model_validate(response['data']),
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
def _merge_fields(self, fields: list[str], default_fields: list[str]) -> list[str]:
|
|
341
|
+
return list(set(fields).union(set(default_fields)))
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from pydantic import Field
|
|
17
|
+
|
|
18
|
+
from ...analyst_notes.note import AnalystNote
|
|
19
|
+
from ...common_models import IdNameTypeDescription, RFBaseModel
|
|
20
|
+
from ..models.lookup import (
|
|
21
|
+
AIInsights,
|
|
22
|
+
Metric,
|
|
23
|
+
ReferenceCount,
|
|
24
|
+
RelatedEntities,
|
|
25
|
+
Sighting,
|
|
26
|
+
Timestamps,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class BaseEnrichedEntity(RFBaseModel):
|
|
31
|
+
"""Base Model for Enrichment.
|
|
32
|
+
This model is intended to be inherited and should not be used on its own.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
ai_insights: Optional[AIInsights] = Field(alias='aiInsights', default=None)
|
|
36
|
+
analyst_notes: Optional[list[AnalystNote]] = Field(alias='analystNotes', default=[])
|
|
37
|
+
counts: Optional[list[ReferenceCount]] = []
|
|
38
|
+
entity: Optional[IdNameTypeDescription] = None
|
|
39
|
+
intel_card: Optional[str] = Field(alias='intelCard', default=None)
|
|
40
|
+
metrics: Optional[list[Metric]] = []
|
|
41
|
+
related_entities: Optional[list[RelatedEntities]] = Field(alias='relatedEntities', default=[])
|
|
42
|
+
sightings: Optional[list[Sighting]] = []
|
|
43
|
+
timestamps: Optional[Timestamps] = None
|