psengine 2.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. psengine/__init__.py +22 -0
  2. psengine/_sdk_id.py +16 -0
  3. psengine/_version.py +14 -0
  4. psengine/analyst_notes/__init__.py +32 -0
  5. psengine/analyst_notes/constants.py +15 -0
  6. psengine/analyst_notes/errors.py +42 -0
  7. psengine/analyst_notes/helpers.py +90 -0
  8. psengine/analyst_notes/models.py +219 -0
  9. psengine/analyst_notes/note.py +149 -0
  10. psengine/analyst_notes/note_mgr.py +400 -0
  11. psengine/base_http_client.py +285 -0
  12. psengine/classic_alerts/__init__.py +24 -0
  13. psengine/classic_alerts/classic_alert.py +275 -0
  14. psengine/classic_alerts/classic_alert_mgr.py +507 -0
  15. psengine/classic_alerts/constants.py +31 -0
  16. psengine/classic_alerts/errors.py +38 -0
  17. psengine/classic_alerts/helpers.py +87 -0
  18. psengine/classic_alerts/markdown/__init__.py +13 -0
  19. psengine/classic_alerts/markdown/markdown.py +359 -0
  20. psengine/classic_alerts/models.py +141 -0
  21. psengine/collective_insights/__init__.py +29 -0
  22. psengine/collective_insights/collective_insights.py +164 -0
  23. psengine/collective_insights/constants.py +44 -0
  24. psengine/collective_insights/errors.py +18 -0
  25. psengine/collective_insights/insight.py +89 -0
  26. psengine/collective_insights/models.py +81 -0
  27. psengine/common_models.py +89 -0
  28. psengine/config/__init__.py +15 -0
  29. psengine/config/config.py +284 -0
  30. psengine/config/errors.py +18 -0
  31. psengine/constants.py +63 -0
  32. psengine/detection/__init__.py +17 -0
  33. psengine/detection/detection_mgr.py +135 -0
  34. psengine/detection/detection_rule.py +85 -0
  35. psengine/detection/errors.py +26 -0
  36. psengine/detection/helpers.py +56 -0
  37. psengine/detection/models.py +47 -0
  38. psengine/endpoints.py +98 -0
  39. psengine/enrich/__init__.py +28 -0
  40. psengine/enrich/constants.py +73 -0
  41. psengine/enrich/errors.py +26 -0
  42. psengine/enrich/lookup.py +299 -0
  43. psengine/enrich/lookup_mgr.py +341 -0
  44. psengine/enrich/models/__init__.py +13 -0
  45. psengine/enrich/models/base_enriched_entity.py +43 -0
  46. psengine/enrich/models/lookup.py +271 -0
  47. psengine/enrich/models/soar.py +138 -0
  48. psengine/enrich/soar.py +89 -0
  49. psengine/enrich/soar_mgr.py +176 -0
  50. psengine/entity_lists/__init__.py +16 -0
  51. psengine/entity_lists/constants.py +19 -0
  52. psengine/entity_lists/entity_list.py +435 -0
  53. psengine/entity_lists/entity_list_mgr.py +185 -0
  54. psengine/entity_lists/errors.py +26 -0
  55. psengine/entity_lists/models.py +87 -0
  56. psengine/entity_match/__init__.py +16 -0
  57. psengine/entity_match/entity_match.py +90 -0
  58. psengine/entity_match/entity_match_mgr.py +235 -0
  59. psengine/entity_match/errors.py +18 -0
  60. psengine/entity_match/models.py +22 -0
  61. psengine/errors.py +41 -0
  62. psengine/helpers/__init__.py +23 -0
  63. psengine/helpers/helpers.py +471 -0
  64. psengine/logger/__init__.py +15 -0
  65. psengine/logger/constants.py +39 -0
  66. psengine/logger/errors.py +18 -0
  67. psengine/logger/rf_logger.py +148 -0
  68. psengine/markdown/__init__.py +21 -0
  69. psengine/markdown/markdown.py +169 -0
  70. psengine/markdown/models.py +22 -0
  71. psengine/playbook_alerts/__init__.py +34 -0
  72. psengine/playbook_alerts/constants.py +35 -0
  73. psengine/playbook_alerts/errors.py +35 -0
  74. psengine/playbook_alerts/helpers.py +80 -0
  75. psengine/playbook_alerts/mappings.py +44 -0
  76. psengine/playbook_alerts/markdown/__init__.py +13 -0
  77. psengine/playbook_alerts/markdown/markdown.py +98 -0
  78. psengine/playbook_alerts/markdown/markdown_code_repo.py +64 -0
  79. psengine/playbook_alerts/markdown/markdown_domain_abuse.py +118 -0
  80. psengine/playbook_alerts/markdown/markdown_identity_exposure.py +158 -0
  81. psengine/playbook_alerts/models/__init__.py +36 -0
  82. psengine/playbook_alerts/models/common_models.py +18 -0
  83. psengine/playbook_alerts/models/panel_log.py +329 -0
  84. psengine/playbook_alerts/models/panel_status.py +70 -0
  85. psengine/playbook_alerts/models/pba_code_repo_leak.py +52 -0
  86. psengine/playbook_alerts/models/pba_cyber_vulnerability.py +53 -0
  87. psengine/playbook_alerts/models/pba_domain_abuse.py +139 -0
  88. psengine/playbook_alerts/models/pba_identity_exposures.py +93 -0
  89. psengine/playbook_alerts/models/pba_third_party_risk.py +103 -0
  90. psengine/playbook_alerts/models/search_endpoint.py +68 -0
  91. psengine/playbook_alerts/pa_category.py +37 -0
  92. psengine/playbook_alerts/playbook_alert_mgr.py +593 -0
  93. psengine/playbook_alerts/playbook_alerts.py +393 -0
  94. psengine/rf_client.py +430 -0
  95. psengine/risklists/__init__.py +17 -0
  96. psengine/risklists/constants.py +15 -0
  97. psengine/risklists/errors.py +20 -0
  98. psengine/risklists/models.py +65 -0
  99. psengine/risklists/risklist_mgr.py +156 -0
  100. psengine/stix2/__init__.py +21 -0
  101. psengine/stix2/base_stix_entity.py +62 -0
  102. psengine/stix2/complex_entity.py +372 -0
  103. psengine/stix2/constants.py +81 -0
  104. psengine/stix2/enriched_indicator.py +261 -0
  105. psengine/stix2/errors.py +22 -0
  106. psengine/stix2/helpers.py +68 -0
  107. psengine/stix2/rf_bundle.py +240 -0
  108. psengine/stix2/simple_entity.py +145 -0
  109. psengine/stix2/util.py +53 -0
  110. psengine-2.0.4.dist-info/METADATA +189 -0
  111. psengine-2.0.4.dist-info/RECORD +115 -0
  112. psengine-2.0.4.dist-info/WHEEL +5 -0
  113. psengine-2.0.4.dist-info/entry_points.txt +2 -0
  114. psengine-2.0.4.dist-info/licenses/LICENSE +21 -0
  115. psengine-2.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,299 @@
1
+ ##################################### TERMS OF USE ###########################################
2
+ # The following code is provided for demonstration purpose only, and should not be used #
3
+ # without independent verification. Recorded Future makes no representations or warranties, #
4
+ # express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
5
+ # information it may retrieve, and provides it both strictly “as-is” and without assuming #
6
+ # responsibility for any information it may retrieve. Recorded Future shall not be liable #
7
+ # for, and you assume all risk of using, the foregoing. By using this code, Customer #
8
+ # represents that it is solely responsible for having all necessary licenses, permissions, #
9
+ # rights, and/or consents to connect to third party APIs, and that it is solely responsible #
10
+ # for having all necessary licenses, permissions, rights, and/or consents to any data #
11
+ # accessed from any third party API. #
12
+ ##############################################################################################
13
+ from functools import total_ordering
14
+ from typing import Optional, Union
15
+
16
+ from pydantic import Field
17
+
18
+ from ..common_models import IdNameType, IdNameTypeDescription, RFBaseModel
19
+ from ..constants import TIMESTAMP_STR
20
+ from .models.base_enriched_entity import BaseEnrichedEntity
21
+ from .models.lookup import (
22
+ CVSS,
23
+ CVSSV3,
24
+ CVSSRating,
25
+ DnsPortCert,
26
+ EnterpriseList,
27
+ EntityRisk,
28
+ IPLocation,
29
+ LinkedMalware,
30
+ Links,
31
+ NvdReference,
32
+ RawRisk,
33
+ RiskMapping,
34
+ RiskyCIDRPIP,
35
+ )
36
+
37
+
38
+ class EnrichedIP(BaseEnrichedEntity):
39
+ """IP Enriched by ``/v2/ip/{ip}`` endpoint. Inherit behaviours from ``BaseEnrichedEntity``."""
40
+
41
+ risk: Optional[EntityRisk] = None
42
+ links: Optional[Links] = None
43
+ enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
44
+ threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
45
+ risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
46
+ dns_port_cert: Optional[DnsPortCert] = Field(alias='dnsPortCert', default=None)
47
+ location: Optional[IPLocation] = None
48
+ risky_cidr_ips: Optional[list[RiskyCIDRPIP]] = Field(alias='riskyCIDRIPs', default=None)
49
+
50
+
51
+ class EnrichedDomain(BaseEnrichedEntity):
52
+ """Domain Enriched by ``/v2/domain/{domain}`` endpoint.
53
+ Inherit behaviours from ``BaseEnrichedEntity``.
54
+ """
55
+
56
+ risk: Optional[EntityRisk] = None
57
+ links: Optional[Links] = None
58
+ enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
59
+ threat_lists: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
60
+ risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
61
+
62
+
63
+ class EnrichedURL(BaseEnrichedEntity):
64
+ """URL Enriched by ``/v2/url/{url}`` endpoint.
65
+ Inherit behaviours from ``BaseEnrichedEntity``.
66
+ """
67
+
68
+ risk: Optional[EntityRisk] = None
69
+ links: Optional[Links] = None
70
+ enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
71
+ risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
72
+
73
+
74
+ class EnrichedHash(BaseEnrichedEntity):
75
+ """Hash Enriched by ``/v2/hash/{hash}`` endpoint.
76
+ Inherit behaviours from ``BaseEnrichedEntity``.
77
+ """
78
+
79
+ risk: Optional[EntityRisk] = None
80
+ links: Optional[Links] = None
81
+ enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
82
+ threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
83
+ risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
84
+ hash_algorithm: Optional[str] = Field(alias='hashAlgorithm', default=None)
85
+ file_hashes: Optional[list[str]] = Field(alias='fileHashes', default=None)
86
+
87
+
88
+ class EnrichedVulnerability(BaseEnrichedEntity):
89
+ """Vulnerability Enriched by ``/v2/vulnerability/{cve}`` endpoint.
90
+ Inherit behaviours from ``BaseEnrichedEntity``.
91
+ """
92
+
93
+ risk: Optional[EntityRisk] = None
94
+ links: Optional[Links] = None
95
+ enterprise_lists: Optional[list[EnterpriseList]] = Field(alias='enterpriseLists', default=None)
96
+ threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
97
+ risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
98
+ common_names: Optional[list[str]] = Field(alias='commonNames', default=None)
99
+ lifecycle_stage: Optional[str] = Field(alias='lifecycleStage', default=None)
100
+ linked_malware: Optional[LinkedMalware] = Field(alias='linkedMalware', default=None)
101
+ cpe: Optional[list[str]] = None
102
+ cpe_22_uri: Optional[list[str]] = Field(alias='cpe22uri', default=None)
103
+ cvss: Optional[CVSS] = None
104
+ cvss_ratings: list[CVSSRating] = Field(alias='cvssRatings', default=None)
105
+ cvssv3: Optional[CVSSV3] = None
106
+ nvd_description: Optional[str] = Field(alias='nvdDescription', default=None)
107
+ nvd_references: Optional[list[NvdReference]] = Field(alias='nvdReferences', default=None)
108
+ raw_risk: Optional[list[RawRisk]] = Field(alias='rawrisk', default=None)
109
+ related_links: Optional[list[str]] = Field(alias='relatedLinks', default=None)
110
+
111
+
112
+ class EnrichedMalware(BaseEnrichedEntity):
113
+ """Malware Enriched by ``/v2/malware/{id}`` endpoint.
114
+ Inherit behaviours from ``BaseEnrichedEntity``.
115
+ """
116
+
117
+ links: Optional[Links] = None
118
+ categories: Optional[list[IdNameType]] = None
119
+
120
+
121
+ class EnrichedCompany(BaseEnrichedEntity):
122
+ """Company Enriched by ``/v2/company/{id}`` and ``/v2/company/by_domain/{domain}`` endpoint.
123
+ Inherit behaviours from ``BaseEnrichedEntity``.
124
+ """
125
+
126
+ risk: Optional[EntityRisk] = None
127
+ curated: Optional[bool] = None
128
+ threat_list: Optional[list[IdNameTypeDescription]] = Field(alias='threatLists', default=None)
129
+ risk_mapping: Optional[list[RiskMapping]] = Field(alias='riskMapping', default=None)
130
+
131
+
132
+ _EnrichmentObjectType = Union[
133
+ EnrichedCompany,
134
+ EnrichedDomain,
135
+ EnrichedIP,
136
+ EnrichedHash,
137
+ EnrichedMalware,
138
+ EnrichedURL,
139
+ EnrichedVulnerability,
140
+ ]
141
+
142
+
143
+ @total_ordering
144
+ class EnrichmentData(RFBaseModel):
145
+ """Model for the custom return of lookup of IOCs.
146
+
147
+ Methods:
148
+ __hash__:
149
+ Returns a hash value based on the content's attributes.
150
+
151
+ - If ``content`` is an instance of ``EnrichedMalware``:
152
+ The hash is calculated using the entity ``id_`` and the last seen timestamp.
153
+ - Else:
154
+ The hash includes the entity ``id_``, risk score, and the last seen timestamp.
155
+
156
+ __eq__:
157
+ Checks equality between two ``EnrichmentData`` instances based on their ``content``.
158
+
159
+ - If ``content`` is an instance of ``EnrichedMalware``:
160
+ Equality is determined by comparing the entity name and the last seen timestamp.
161
+ - Else:
162
+ Equality is determined by comparing the entity name, last seen timestamp and
163
+ risk score.
164
+
165
+ __gt__:
166
+ Defines a greater-than comparison between ``EnrichmentData`` instances based on their
167
+ ``content``.
168
+
169
+ - If ``content`` is an instance of ``EnrichedMalware``:
170
+ Comparison is based on the last seen timestamp and entity name.
171
+ - Else:
172
+ Comparison is based on the last seen timestamp, entity name, and risk score.
173
+
174
+ __str__ and __repr__:
175
+ Returns a string representation of the ``EnrichmentData`` instance.
176
+ __repr__ is used for printing elements properly within a list.
177
+
178
+ - If ``content`` is an instance of ``EnrichedMalware``:
179
+ Includes class name, entity name, and last seen timestamp.
180
+
181
+ .. code-block:: python
182
+
183
+ >>> print(enrichment_data)
184
+ EnrichedMalware: exampleMalware, Last Seen: 2024-05-21 01:30:00PM
185
+
186
+ - Else:
187
+ Includes class name, entity name, risk score, and last seen timestamp.
188
+
189
+ .. code-block:: python
190
+
191
+ >>> print(enrichment_data)
192
+ EnrichedIP: 1.1.1.1, Risk Score: 85, Last Seen: 2024-05-21 01:30:00PM
193
+
194
+ Total Ordering:
195
+ The ordering of ``EnrichmentData`` instances is determined by the ``content``'s last
196
+ seen timestamp.
197
+
198
+ - If ``content`` is an instance of ``EnrichedMalware``:
199
+ If two instances have the same last seen timestamp, their entity name is used as a
200
+ secondary criterion.
201
+
202
+ - Else:
203
+ If two instances have the same last seen timestamp, their entity name and risk score are
204
+ used as secondary criteria.
205
+ """
206
+
207
+ entity: str
208
+ entity_type: Optional[str]
209
+ is_enriched: bool
210
+ content: Union[str, _EnrichmentObjectType]
211
+
212
+ def __hash__(self):
213
+ if isinstance(self.content, EnrichedMalware):
214
+ return hash(
215
+ (
216
+ self.content.entity.id_,
217
+ self.content.timestamps.last_seen,
218
+ )
219
+ )
220
+ elif isinstance(self.content, str):
221
+ return hash(self.entity)
222
+
223
+ return hash(
224
+ (
225
+ self.content.entity.id_,
226
+ self.content.risk.score,
227
+ self.content.timestamps.last_seen,
228
+ )
229
+ )
230
+
231
+ def __eq__(self, other: 'EnrichmentData'):
232
+ if isinstance(self.content, EnrichedMalware):
233
+ return (
234
+ self.content.entity.name,
235
+ self.content.timestamps.last_seen,
236
+ ) == (
237
+ other.content.entity.name,
238
+ other.content.timestamps.last_seen,
239
+ )
240
+ elif isinstance(self.content, str):
241
+ return self.entity == other.entity
242
+ return (
243
+ self.content.risk.score,
244
+ self.content.entity.name,
245
+ self.content.timestamps.last_seen,
246
+ ) == (
247
+ other.content.risk.score,
248
+ other.content.entity.name,
249
+ other.content.timestamps.last_seen,
250
+ )
251
+
252
+ def __gt__(self, other: 'EnrichmentData'):
253
+ if isinstance(self.content, EnrichedMalware):
254
+ return (
255
+ self.content.timestamps.last_seen,
256
+ self.content.entity.name,
257
+ ) > (
258
+ other.content.timestamps.last_seen,
259
+ other.content.entity.name,
260
+ )
261
+ elif isinstance(self.content, str):
262
+ return self.entity > other.entity
263
+ return (
264
+ self.content.risk.score,
265
+ self.content.timestamps.last_seen,
266
+ self.content.entity.name,
267
+ ) > (
268
+ other.content.risk.score,
269
+ other.content.timestamps.last_seen,
270
+ other.content.entity.name,
271
+ )
272
+
273
+ def __str__(self):
274
+ if isinstance(self.content, EnrichedMalware):
275
+ return (
276
+ f'{self.content.__class__.__name__}: {self.content.entity.name}, '
277
+ f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
278
+ )
279
+ elif isinstance(self.content, str):
280
+ return f'{self.entity}: {self.content}'
281
+ return (
282
+ f'{self.content.__class__.__name__}: {self.content.entity.name}, '
283
+ f'Risk Score: {self.content.risk.score}, '
284
+ f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
285
+ )
286
+
287
+ def __repr__(self):
288
+ if isinstance(self.content, EnrichedMalware):
289
+ return (
290
+ f'{self.content.__class__.__name__}: {self.content.entity.name}, '
291
+ f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
292
+ )
293
+ elif isinstance(self.content, str):
294
+ return f'{self.entity}: {self.content}'
295
+ return (
296
+ f'{self.content.__class__.__name__}: {self.content.entity.name}, '
297
+ f'Risk Score: {self.content.risk.score}, '
298
+ f'Last Seen: {self.content.timestamps.last_seen.strftime(TIMESTAMP_STR)}'
299
+ )
@@ -0,0 +1,341 @@
1
+ ##################################### TERMS OF USE ###########################################
2
+ # The following code is provided for demonstration purpose only, and should not be used #
3
+ # without independent verification. Recorded Future makes no representations or warranties, #
4
+ # express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
5
+ # information it may retrieve, and provides it both strictly “as-is” and without assuming #
6
+ # responsibility for any information it may retrieve. Recorded Future shall not be liable #
7
+ # for, and you assume all risk of using, the foregoing. By using this code, Customer #
8
+ # represents that it is solely responsible for having all necessary licenses, permissions, #
9
+ # rights, and/or consents to connect to third party APIs, and that it is solely responsible #
10
+ # for having all necessary licenses, permissions, rights, and/or consents to any data #
11
+ # accessed from any third party API. #
12
+ ##############################################################################################
13
+
14
+ import logging
15
+ from typing import Optional
16
+ from urllib.parse import quote
17
+
18
+ from pydantic import validate_call
19
+
20
+ from ..endpoints import CONNECT_API_BASE_URL
21
+ from ..helpers import MultiThreadingHelper, connection_exceptions, debug_call
22
+ from ..rf_client import RFClient
23
+ from .constants import (
24
+ ALLOWED_ENTITIES,
25
+ ENTITY_FIELDS,
26
+ IOC_TO_MODEL,
27
+ MALWARE_FIELDS,
28
+ MESSAGE_404,
29
+ TYPE_MAPPING,
30
+ )
31
+ from .errors import EnrichmentLookupError
32
+ from .lookup import EnrichmentData
33
+
34
+
35
+ class LookupMgr:
36
+ """Enrichment of a single or a group of Entities."""
37
+
38
+ def __init__(self, rf_token: str = None):
39
+ """Initializes the LookupMgr object.
40
+
41
+ Args:
42
+ rf_token (str, optional): Recorded Future API token. Defaults to None
43
+ """
44
+ self.log = logging.getLogger(__name__)
45
+ self.rf_client = RFClient(api_token=rf_token) if rf_token else RFClient()
46
+
47
+ @validate_call
48
+ @debug_call
49
+ def lookup(
50
+ self,
51
+ entity: str,
52
+ entity_type: ALLOWED_ENTITIES,
53
+ fields: Optional[list[str]] = None,
54
+ ) -> EnrichmentData:
55
+ """Perform lookup of an entity based on its id or name.
56
+ ``entity`` can contain both a Recorded Future ID or a entity name. See example below.
57
+
58
+ The entity_type must always be specified.
59
+ The allowed values are:
60
+
61
+ - ``company``,
62
+ - ``Company``,
63
+ - ``company_by_domain``,
64
+ - ``CyberVulnerability``,
65
+ - ``domain``,
66
+ - ``hash``,
67
+ - ``Hash``,
68
+ - ``InternetDomainName``,
69
+ - ``ip``,
70
+ - ``IpAddress``,
71
+ - ``malware``,
72
+ - ``Malware``,
73
+ - ``Organization``,
74
+ - ``url``,
75
+ - ``URL``,
76
+ - ``vulnerability``,
77
+
78
+ If ``fields`` parameter is specified, it will be added to the mandatory fields, which
79
+ are:
80
+
81
+ - for every entity except malware: ``['entity', 'risk', 'timestamps']``
82
+ - for malware: ``['entity', 'timestamps']``
83
+
84
+
85
+ Examples:
86
+ Performing a lookup using different ``entity`` type:
87
+
88
+ .. code-block:: python
89
+ :linenos:
90
+
91
+ mgr.lookup('idn:google.com', 'domain')
92
+ mgr.lookup('google.com', 'domain')
93
+ mgr.lookup('A_BCDE', 'company')
94
+
95
+
96
+ Performing a lookup of a ``company_by_domain`` specifying additional fields:
97
+
98
+ .. code-block:: python
99
+ :linenos:
100
+
101
+ mgr.lookup(
102
+ 'recordedfuture.com',
103
+ entity_type='company_by_domain',
104
+ fields=['curated']
105
+ )
106
+
107
+ The output is always ``EnrichmentData`` object with a format like below:
108
+ If a 404 is received:
109
+
110
+ .. code-block:: python
111
+
112
+ {
113
+ 'entity': entity,
114
+ 'entity_type': entity_type,
115
+ 'is_enriched': False,
116
+ 'content': '404 received. Nothing known on this entity',
117
+ }
118
+
119
+ If a 200 is received:
120
+
121
+ .. code-block:: python
122
+
123
+ {
124
+ 'entity': entity,
125
+ 'entity_type': entity_type,
126
+ 'is_enriched': True,
127
+ 'content': the enriched data model
128
+ }
129
+
130
+ To write an enriched object to file:
131
+
132
+ .. code-block:: python
133
+ :linenos:
134
+
135
+ from pathlib import Path
136
+ from json import dump
137
+ from psengine.enrich import LookupMgr
138
+
139
+ mgr = LookupMgr()
140
+ OUTPUT_DIR = Path('your' / 'path')
141
+ OUTPUT_DIR.mkdir(exists_ok=True)
142
+ ip = mgr.lookup('1.1.1.1', 'ip')
143
+ (OUTPUT_DIR / f'{ip.entity}.json').write_text(dumps(ip.json(), indent=2))
144
+
145
+
146
+ Endpoint:
147
+ ``v2/{entity_type}/{entity}``
148
+
149
+ Args:
150
+ entity (str): Name or RFID of the entity.
151
+ entity_type (str): Type of the entity.
152
+ fields (List[str], optional): Fields for the entity to enrich.
153
+ EnrichmentData: An object containing the entity details.
154
+
155
+ Raises:
156
+ ValidationError If a field does not match the type hint.
157
+ EnrichmentLookupError: if a lookup terminates with a non 200 or 404 return code.
158
+ """
159
+ default_fields = MALWARE_FIELDS if entity_type.lower() == 'malware' else ENTITY_FIELDS
160
+ fields = fields if fields else default_fields
161
+ fields = self._merge_fields(fields, default_fields)
162
+
163
+ return self._lookup(entity, entity_type, fields)
164
+
165
+ @validate_call
166
+ @debug_call
167
+ def lookup_bulk(
168
+ self,
169
+ entity: list[str],
170
+ entity_type: ALLOWED_ENTITIES,
171
+ fields: list[str] = ENTITY_FIELDS,
172
+ max_workers: Optional[int] = 0,
173
+ ) -> list[EnrichmentData]:
174
+ """Perform lookup of multiple entities based on ids or name.
175
+ ``entity`` can contain both a Recorded Future ID or a entity name.
176
+ The entities must be of the same ``entity_type``. See example below.
177
+
178
+ The ``entity_type`` must always be specified.
179
+ The allowed values are:
180
+
181
+ - ``company``,
182
+ - ``Company``,
183
+ - ``company_by_domain``,
184
+ - ``CyberVulnerability``,
185
+ - ``domain``,
186
+ - ``hash``,
187
+ - ``Hash``,
188
+ - ``InternetDomainName``,
189
+ - ``ip``,
190
+ - ``IpAddress``,
191
+ - ``malware``,
192
+ - ``Malware``,
193
+ - ``Organization``,
194
+ - ``url``,
195
+ - ``URL``,
196
+ - ``vulnerability``,
197
+
198
+ If ``fields`` parameter is specified, it will be added to the mandatory fields, which
199
+ are:
200
+
201
+ - for every entity except malware: ``['entity', 'risk', 'timestamps']``
202
+ - for malware: ``['entity', 'timestamps']``
203
+
204
+ Examples:
205
+ Multiple IOC types enrichment:
206
+
207
+ .. code-block:: python
208
+ :linenos:
209
+
210
+ data = {
211
+ 'IpAddress': ['1.1.1.1', '2.2.2.2'],
212
+ 'InternetDomainName': [ 'google.com', 'facebook.com']
213
+ }
214
+ results = []
215
+ for entity_type, entities in data.items():
216
+ results.extend(mgr.lookup_bulk(entities, entity_type)
217
+
218
+ To write an enriched object to file:
219
+
220
+ .. code-block:: python
221
+ :linenos:
222
+
223
+ from pathlib import Path
224
+ from json import dump
225
+ from psengine.enrich import LookupMgr
226
+
227
+ mgr = LookupMgr()
228
+ OUTPUT_DIR = Path('your' / 'path')
229
+ OUTPUT_DIR.mkdir(exists_ok=True)
230
+ data = mgr.lookup_bulk(['1.1.1.1', '8.8.8.8'], 'ip')
231
+ for ip in data:
232
+ (OUTPUT_DIR / f'{ip.entity}.json').write_text(dumps(ip.json(), indent=2))
233
+
234
+
235
+ The output is always a list of ``EnrichmentData`` object with a format like below:
236
+ If a 404 is received:
237
+
238
+ .. code-block:: python
239
+
240
+ {
241
+ 'entity': entity,
242
+ 'entity_type': entity_type,
243
+ 'is_enriched': False,
244
+ 'content': '404 received. Nothing known on this entity',
245
+ }
246
+
247
+ If a 200 is received:
248
+
249
+ .. code-block:: python
250
+
251
+ {
252
+ 'entity': entity,
253
+ 'entity_type': entity_type,
254
+ 'is_enriched': True,
255
+ 'content': the enriched data model
256
+ }
257
+
258
+ Multithreaded examples:
259
+
260
+ .. code-block:: python
261
+
262
+ mgr.lookup_bulk(['google.com', 'facebook.com'], 'domain', max_workers=10)
263
+
264
+ Endpoint:
265
+ ``v2/{entity_type}/{entity}``
266
+
267
+
268
+ Args:
269
+ entity (List[str]): list of names or RFIDs.
270
+ entity_type (List[str]): Type of the entities.
271
+ fields (List[str], optional): Fields for the entities to enrich.
272
+ max_workers (int, optional): number of workers to multithread requests.
273
+
274
+ Returns:
275
+ List[EnrichmentData]: A list of object containing the entity details.
276
+
277
+ Raises:
278
+ ValidationError If a field does not match the type hint.
279
+ EnrichmentLookupError: if a lookup terminates with a non 200 or 404 return code.
280
+ """
281
+ default_fields = MALWARE_FIELDS if entity_type.lower() == 'malware' else ENTITY_FIELDS
282
+ fields = fields if fields else default_fields
283
+ fields = self._merge_fields(fields, default_fields)
284
+ if max_workers:
285
+ res = MultiThreadingHelper.multithread_it(
286
+ max_workers,
287
+ self._lookup,
288
+ iterator=entity,
289
+ entity_type=entity_type,
290
+ fields=fields,
291
+ )
292
+ else:
293
+ res = [self._lookup(entity, entity_type, fields) for entity in entity]
294
+
295
+ return res
296
+
297
+ def _lookup(
298
+ self,
299
+ entity: str,
300
+ entity_type: str,
301
+ fields: list,
302
+ ):
303
+ entity_type = TYPE_MAPPING.get(entity_type, entity_type)
304
+
305
+ enriched = self._fetch_data(
306
+ entity=entity,
307
+ entity_type=entity_type,
308
+ fields=fields,
309
+ )
310
+ if not enriched:
311
+ enriched = EnrichmentData(
312
+ entity=entity,
313
+ entity_type=entity_type,
314
+ is_enriched=False,
315
+ content=MESSAGE_404,
316
+ )
317
+
318
+ return enriched
319
+
320
+ @connection_exceptions(ignore_status_code=[404], exception_to_raise=EnrichmentLookupError)
321
+ @debug_call
322
+ def _fetch_data(self, entity: str, entity_type: str, fields: list) -> EnrichmentData:
323
+ """Perform the actual lookup. If a 404 is returned, return None."""
324
+ encoded_entity = quote(entity, safe='.')
325
+ entity_type = 'company/by_domain' if entity_type == 'company_by_domain' else entity_type
326
+
327
+ url = f'{CONNECT_API_BASE_URL}/{entity_type}/{encoded_entity}'
328
+
329
+ params = {}
330
+ params['fields'] = ','.join(fields)
331
+
332
+ response = self.rf_client.request('get', url, params=params).json()
333
+ return EnrichmentData(
334
+ entity=entity,
335
+ entity_type=entity_type,
336
+ is_enriched=True,
337
+ content=IOC_TO_MODEL[entity_type].model_validate(response['data']),
338
+ )
339
+
340
+ def _merge_fields(self, fields: list[str], default_fields: list[str]) -> list[str]:
341
+ return list(set(fields).union(set(default_fields)))
@@ -0,0 +1,13 @@
1
+ ##################################### TERMS OF USE ###########################################
2
+ # The following code is provided for demonstration purpose only, and should not be used #
3
+ # without independent verification. Recorded Future makes no representations or warranties, #
4
+ # express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
5
+ # information it may retrieve, and provides it both strictly “as-is” and without assuming #
6
+ # responsibility for any information it may retrieve. Recorded Future shall not be liable #
7
+ # for, and you assume all risk of using, the foregoing. By using this code, Customer #
8
+ # represents that it is solely responsible for having all necessary licenses, permissions, #
9
+ # rights, and/or consents to connect to third party APIs, and that it is solely responsible #
10
+ # for having all necessary licenses, permissions, rights, and/or consents to any data #
11
+ # accessed from any third party API. #
12
+ ##############################################################################################
13
+
@@ -0,0 +1,43 @@
1
+ ##################################### TERMS OF USE ###########################################
2
+ # The following code is provided for demonstration purpose only, and should not be used #
3
+ # without independent verification. Recorded Future makes no representations or warranties, #
4
+ # express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
5
+ # information it may retrieve, and provides it both strictly “as-is” and without assuming #
6
+ # responsibility for any information it may retrieve. Recorded Future shall not be liable #
7
+ # for, and you assume all risk of using, the foregoing. By using this code, Customer #
8
+ # represents that it is solely responsible for having all necessary licenses, permissions, #
9
+ # rights, and/or consents to connect to third party APIs, and that it is solely responsible #
10
+ # for having all necessary licenses, permissions, rights, and/or consents to any data #
11
+ # accessed from any third party API. #
12
+ ##############################################################################################
13
+
14
+ from typing import Optional
15
+
16
+ from pydantic import Field
17
+
18
+ from ...analyst_notes.note import AnalystNote
19
+ from ...common_models import IdNameTypeDescription, RFBaseModel
20
+ from ..models.lookup import (
21
+ AIInsights,
22
+ Metric,
23
+ ReferenceCount,
24
+ RelatedEntities,
25
+ Sighting,
26
+ Timestamps,
27
+ )
28
+
29
+
30
+ class BaseEnrichedEntity(RFBaseModel):
31
+ """Base Model for Enrichment.
32
+ This model is intended to be inherited and should not be used on its own.
33
+ """
34
+
35
+ ai_insights: Optional[AIInsights] = Field(alias='aiInsights', default=None)
36
+ analyst_notes: Optional[list[AnalystNote]] = Field(alias='analystNotes', default=[])
37
+ counts: Optional[list[ReferenceCount]] = []
38
+ entity: Optional[IdNameTypeDescription] = None
39
+ intel_card: Optional[str] = Field(alias='intelCard', default=None)
40
+ metrics: Optional[list[Metric]] = []
41
+ related_entities: Optional[list[RelatedEntities]] = Field(alias='relatedEntities', default=[])
42
+ sightings: Optional[list[Sighting]] = []
43
+ timestamps: Optional[Timestamps] = None