psengine 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psengine/__init__.py +22 -0
- psengine/_sdk_id.py +16 -0
- psengine/_version.py +14 -0
- psengine/analyst_notes/__init__.py +32 -0
- psengine/analyst_notes/constants.py +15 -0
- psengine/analyst_notes/errors.py +42 -0
- psengine/analyst_notes/helpers.py +90 -0
- psengine/analyst_notes/models.py +219 -0
- psengine/analyst_notes/note.py +149 -0
- psengine/analyst_notes/note_mgr.py +400 -0
- psengine/base_http_client.py +285 -0
- psengine/classic_alerts/__init__.py +24 -0
- psengine/classic_alerts/classic_alert.py +275 -0
- psengine/classic_alerts/classic_alert_mgr.py +507 -0
- psengine/classic_alerts/constants.py +31 -0
- psengine/classic_alerts/errors.py +38 -0
- psengine/classic_alerts/helpers.py +87 -0
- psengine/classic_alerts/markdown/__init__.py +13 -0
- psengine/classic_alerts/markdown/markdown.py +359 -0
- psengine/classic_alerts/models.py +141 -0
- psengine/collective_insights/__init__.py +29 -0
- psengine/collective_insights/collective_insights.py +164 -0
- psengine/collective_insights/constants.py +44 -0
- psengine/collective_insights/errors.py +18 -0
- psengine/collective_insights/insight.py +89 -0
- psengine/collective_insights/models.py +81 -0
- psengine/common_models.py +89 -0
- psengine/config/__init__.py +15 -0
- psengine/config/config.py +284 -0
- psengine/config/errors.py +18 -0
- psengine/constants.py +63 -0
- psengine/detection/__init__.py +17 -0
- psengine/detection/detection_mgr.py +135 -0
- psengine/detection/detection_rule.py +85 -0
- psengine/detection/errors.py +26 -0
- psengine/detection/helpers.py +56 -0
- psengine/detection/models.py +47 -0
- psengine/endpoints.py +98 -0
- psengine/enrich/__init__.py +28 -0
- psengine/enrich/constants.py +73 -0
- psengine/enrich/errors.py +26 -0
- psengine/enrich/lookup.py +299 -0
- psengine/enrich/lookup_mgr.py +341 -0
- psengine/enrich/models/__init__.py +13 -0
- psengine/enrich/models/base_enriched_entity.py +43 -0
- psengine/enrich/models/lookup.py +271 -0
- psengine/enrich/models/soar.py +138 -0
- psengine/enrich/soar.py +89 -0
- psengine/enrich/soar_mgr.py +176 -0
- psengine/entity_lists/__init__.py +16 -0
- psengine/entity_lists/constants.py +19 -0
- psengine/entity_lists/entity_list.py +435 -0
- psengine/entity_lists/entity_list_mgr.py +185 -0
- psengine/entity_lists/errors.py +26 -0
- psengine/entity_lists/models.py +87 -0
- psengine/entity_match/__init__.py +16 -0
- psengine/entity_match/entity_match.py +90 -0
- psengine/entity_match/entity_match_mgr.py +235 -0
- psengine/entity_match/errors.py +18 -0
- psengine/entity_match/models.py +22 -0
- psengine/errors.py +41 -0
- psengine/helpers/__init__.py +23 -0
- psengine/helpers/helpers.py +471 -0
- psengine/logger/__init__.py +15 -0
- psengine/logger/constants.py +39 -0
- psengine/logger/errors.py +18 -0
- psengine/logger/rf_logger.py +148 -0
- psengine/markdown/__init__.py +21 -0
- psengine/markdown/markdown.py +169 -0
- psengine/markdown/models.py +22 -0
- psengine/playbook_alerts/__init__.py +34 -0
- psengine/playbook_alerts/constants.py +35 -0
- psengine/playbook_alerts/errors.py +35 -0
- psengine/playbook_alerts/helpers.py +80 -0
- psengine/playbook_alerts/mappings.py +44 -0
- psengine/playbook_alerts/markdown/__init__.py +13 -0
- psengine/playbook_alerts/markdown/markdown.py +98 -0
- psengine/playbook_alerts/markdown/markdown_code_repo.py +64 -0
- psengine/playbook_alerts/markdown/markdown_domain_abuse.py +118 -0
- psengine/playbook_alerts/markdown/markdown_identity_exposure.py +158 -0
- psengine/playbook_alerts/models/__init__.py +36 -0
- psengine/playbook_alerts/models/common_models.py +18 -0
- psengine/playbook_alerts/models/panel_log.py +329 -0
- psengine/playbook_alerts/models/panel_status.py +70 -0
- psengine/playbook_alerts/models/pba_code_repo_leak.py +52 -0
- psengine/playbook_alerts/models/pba_cyber_vulnerability.py +53 -0
- psengine/playbook_alerts/models/pba_domain_abuse.py +139 -0
- psengine/playbook_alerts/models/pba_identity_exposures.py +93 -0
- psengine/playbook_alerts/models/pba_third_party_risk.py +103 -0
- psengine/playbook_alerts/models/search_endpoint.py +68 -0
- psengine/playbook_alerts/pa_category.py +37 -0
- psengine/playbook_alerts/playbook_alert_mgr.py +593 -0
- psengine/playbook_alerts/playbook_alerts.py +393 -0
- psengine/rf_client.py +430 -0
- psengine/risklists/__init__.py +17 -0
- psengine/risklists/constants.py +15 -0
- psengine/risklists/errors.py +20 -0
- psengine/risklists/models.py +65 -0
- psengine/risklists/risklist_mgr.py +156 -0
- psengine/stix2/__init__.py +21 -0
- psengine/stix2/base_stix_entity.py +62 -0
- psengine/stix2/complex_entity.py +372 -0
- psengine/stix2/constants.py +81 -0
- psengine/stix2/enriched_indicator.py +261 -0
- psengine/stix2/errors.py +22 -0
- psengine/stix2/helpers.py +68 -0
- psengine/stix2/rf_bundle.py +240 -0
- psengine/stix2/simple_entity.py +145 -0
- psengine/stix2/util.py +53 -0
- psengine-2.0.4.dist-info/METADATA +189 -0
- psengine-2.0.4.dist-info/RECORD +115 -0
- psengine-2.0.4.dist-info/WHEEL +5 -0
- psengine-2.0.4.dist-info/entry_points.txt +2 -0
- psengine-2.0.4.dist-info/licenses/LICENSE +21 -0
- psengine-2.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
ENTITY_IP = 'ip'
|
|
15
|
+
ENTITY_DOMAIN = 'domain'
|
|
16
|
+
ENTITY_HASH = 'hash'
|
|
17
|
+
ENTITY_URL = 'url'
|
|
18
|
+
ENTITY_VULNERABILITY = 'vulnerability'
|
|
19
|
+
VALID_ENTITY_TYPES = [ENTITY_IP, ENTITY_DOMAIN, ENTITY_HASH, ENTITY_URL, ENTITY_VULNERABILITY]
|
|
20
|
+
|
|
21
|
+
TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
|
|
22
|
+
|
|
23
|
+
DETECTION_TYPE_CORRELATION = 'correlation'
|
|
24
|
+
DETECTION_TYPE_PLAYBOOK = 'playbook'
|
|
25
|
+
DETECTION_TYPE_RULE = 'detection_rule'
|
|
26
|
+
VALID_DETECTION_TYPES = [DETECTION_TYPE_CORRELATION, DETECTION_TYPE_PLAYBOOK, DETECTION_TYPE_RULE]
|
|
27
|
+
|
|
28
|
+
DETECTION_SUB_TYPE_SIGMA = 'sigma'
|
|
29
|
+
DETECTION_SUB_TYPE_YARA = 'yara'
|
|
30
|
+
DETECTION_SUB_TYPE_SNORT = 'snort'
|
|
31
|
+
VALID_DETECTION_RULE_SUB_TYPES = ['sigma', 'yara', 'snort']
|
|
32
|
+
DETECTION_SUB_FORMAT_MAPPING = {
|
|
33
|
+
'ioc_type': ['ioc', 'type'],
|
|
34
|
+
'ioc_value': ['ioc', 'value'],
|
|
35
|
+
'ioc_field': ['ioc', 'field'],
|
|
36
|
+
'ioc_source_type': ['ioc', 'source_type'],
|
|
37
|
+
'incident_id': ['incident', 'id'],
|
|
38
|
+
'incident_name': ['incident', 'name'],
|
|
39
|
+
'incident_type': ['incident', 'type'],
|
|
40
|
+
'detection_id': ['detection', 'id'],
|
|
41
|
+
'detection_name': ['detection', 'name'],
|
|
42
|
+
'detection_type': ['detection', 'type'],
|
|
43
|
+
}
|
|
44
|
+
SUMMARY_DEFAULT = True
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from ..errors import RecordedFutureError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CollectiveInsightsError(RecordedFutureError):
|
|
18
|
+
"""Error raised when there was an error with the Collective Insights API."""
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from functools import total_ordering
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
from ..common_models import IdNameType, RFBaseModel
|
|
19
|
+
from ..constants import TIMESTAMP_STR
|
|
20
|
+
from .models import (
|
|
21
|
+
RequestDetection,
|
|
22
|
+
RequestIOC,
|
|
23
|
+
RequestOptions,
|
|
24
|
+
SubmissionResult,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@total_ordering
|
|
29
|
+
class Insight(RFBaseModel):
|
|
30
|
+
"""Validate a single insight.
|
|
31
|
+
|
|
32
|
+
Methods:
|
|
33
|
+
__hash__:
|
|
34
|
+
Returns a hash value based on the IOC value.
|
|
35
|
+
|
|
36
|
+
__eq__:
|
|
37
|
+
Checks equality between two Insight instances based on their IOC value and timestamp.
|
|
38
|
+
|
|
39
|
+
__gt__:
|
|
40
|
+
Defines a greater-than comparison between two Insight instances based on their
|
|
41
|
+
timestamp and IOC value.
|
|
42
|
+
|
|
43
|
+
__str__:
|
|
44
|
+
Returns a string representation of the Insight instance with:
|
|
45
|
+
IOC value, timestamp, and detection type.
|
|
46
|
+
|
|
47
|
+
>>> print(insight)
|
|
48
|
+
IOC: mal_dom.com, Timestamp: 2024-05-21 10:42:30AM, Detection Type: sandbox
|
|
49
|
+
|
|
50
|
+
Total Ordering:
|
|
51
|
+
Ordering of Insight instances is determined primarily by the timestamp. If two instances
|
|
52
|
+
have the same timestamp, their IOC value is used as a secondary criterion for ordering.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
timestamp: datetime
|
|
56
|
+
ioc: RequestIOC
|
|
57
|
+
incident: Optional[IdNameType] = None
|
|
58
|
+
mitre_codes: Optional[list[str]] = None
|
|
59
|
+
malwares: Optional[list[str]] = None
|
|
60
|
+
detection: RequestDetection
|
|
61
|
+
|
|
62
|
+
def __hash__(self):
|
|
63
|
+
return hash(self.ioc.value)
|
|
64
|
+
|
|
65
|
+
def __eq__(self, other: 'Insight'):
|
|
66
|
+
return (self.ioc.value, self.timestamp) == (other.ioc.value, other.timestamp)
|
|
67
|
+
|
|
68
|
+
def __gt__(self, other: 'Insight'):
|
|
69
|
+
return (self.timestamp, self.ioc.value) > (other.timestamp, other.ioc.value)
|
|
70
|
+
|
|
71
|
+
def __str__(self):
|
|
72
|
+
return (
|
|
73
|
+
f'IOC: {self.ioc.value}, Timestamp: {self.timestamp.strftime(TIMESTAMP_STR)}, '
|
|
74
|
+
f'Detection Type: {self.detection.type_}'
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class InsightsOut(RFBaseModel):
|
|
79
|
+
"""Validate data sent to CI."""
|
|
80
|
+
|
|
81
|
+
options: Optional[RequestOptions] = None
|
|
82
|
+
organization_ids: Optional[list[str]] = None
|
|
83
|
+
data: list[Insight]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class InsightsIn(RFBaseModel):
|
|
87
|
+
"""Validate data received from CI."""
|
|
88
|
+
|
|
89
|
+
result: SubmissionResult
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from enum import Enum
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
from pydantic import Field, model_validator
|
|
18
|
+
|
|
19
|
+
from ..common_models import DetectionRuleType, IOCType, RFBaseModel
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DetectionType(Enum):
|
|
23
|
+
detection_rule = 'detection_rule'
|
|
24
|
+
correlation = 'correlation'
|
|
25
|
+
playbook = 'playbook'
|
|
26
|
+
sandbox = 'sandbox'
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SummaryProcessed(RFBaseModel):
|
|
30
|
+
ip: int
|
|
31
|
+
domain: int
|
|
32
|
+
hash_: int = Field(alias='hash')
|
|
33
|
+
vulnerability: int
|
|
34
|
+
url: int
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ResponseSummary(RFBaseModel):
|
|
38
|
+
processed: SummaryProcessed
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class RequestOptions(RFBaseModel):
|
|
42
|
+
debug: bool = False
|
|
43
|
+
summary: bool = True
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class RequestIOC(RFBaseModel):
|
|
47
|
+
type_: IOCType = Field(alias='type')
|
|
48
|
+
value: str
|
|
49
|
+
source_type: Optional[str] = None
|
|
50
|
+
field: Optional[str] = None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class RequestDetection(RFBaseModel):
|
|
54
|
+
id_: Optional[str] = Field(alias='id', default=None)
|
|
55
|
+
name: Optional[str] = None
|
|
56
|
+
type_: DetectionType = Field(alias='type')
|
|
57
|
+
sub_type: Optional[DetectionRuleType] = None
|
|
58
|
+
|
|
59
|
+
@model_validator(mode='before')
|
|
60
|
+
@classmethod
|
|
61
|
+
def validate_detection_rule(cls, data):
|
|
62
|
+
"""Validate detection rule scenario.
|
|
63
|
+
|
|
64
|
+
- id must be present.
|
|
65
|
+
- sub_type must be present and should be one of DetectionRuleType
|
|
66
|
+
"""
|
|
67
|
+
try:
|
|
68
|
+
detection_type = data['type']
|
|
69
|
+
except KeyError as e:
|
|
70
|
+
raise ValueError('type field is mandatory') from e
|
|
71
|
+
|
|
72
|
+
if detection_type == 'detection_rule' and not (data.get('id') and data.get('sub_type')):
|
|
73
|
+
raise ValueError(f'With {detection_type} the id and sub_type fields are mandatory')
|
|
74
|
+
|
|
75
|
+
return data
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class SubmissionResult(RFBaseModel):
|
|
79
|
+
status: str
|
|
80
|
+
debug: bool
|
|
81
|
+
summary: Optional[ResponseSummary] = None
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
from enum import Enum
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RFBaseModel(BaseModel):
|
|
22
|
+
model_config = ConfigDict(extra=os.environ.get('RF_MODEL_EXTRA', 'ignore'))
|
|
23
|
+
|
|
24
|
+
def json(self, by_alias=True, exclude_none=True, auto_exclude_unset=True, **kwargs):
|
|
25
|
+
"""JSON representation of models. It is inherited by every model.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
by_alias (bool): If True, writes fields with their API alias (eg. ``IpAddress``) instead
|
|
29
|
+
of the Python alias (eg. ``ip_address``). Defaults to True.
|
|
30
|
+
|
|
31
|
+
exclude_none (bool): Whether fields equal to None should be excluded from the returned
|
|
32
|
+
dictionary. Defaults to True.
|
|
33
|
+
|
|
34
|
+
auto_exclude_unset (bool): Excludes values that are not set.
|
|
35
|
+
|
|
36
|
+
- True: Based on ``RF_EXTRA_MODEL``, decides if output should have unmapped fields.
|
|
37
|
+
If the ``model_config`` extra is set to 'allow', includes unmapped values;
|
|
38
|
+
otherwise, excludes them.
|
|
39
|
+
|
|
40
|
+
- False: You need to provide the boolean ``exclude_unset`` in the kwargs.
|
|
41
|
+
|
|
42
|
+
kwargs (dict, optional): Any other parameters.
|
|
43
|
+
"""
|
|
44
|
+
if not auto_exclude_unset and kwargs.get('exclude_unset') is None:
|
|
45
|
+
raise ValueError('`auto_exclude_unset` is False, `exclude_unset has to be provided`')
|
|
46
|
+
|
|
47
|
+
exclude_unset = (
|
|
48
|
+
bool(self.model_config['extra'] != 'allow')
|
|
49
|
+
if auto_exclude_unset
|
|
50
|
+
else kwargs['exclude_unset']
|
|
51
|
+
)
|
|
52
|
+
kwargs['exclude_unset'] = exclude_unset
|
|
53
|
+
|
|
54
|
+
return self.model_dump(mode='json', by_alias=by_alias, exclude_none=exclude_none, **kwargs)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class IdName(RFBaseModel):
|
|
58
|
+
id_: str = Field(alias='id')
|
|
59
|
+
name: str
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class IdNameType(RFBaseModel):
|
|
63
|
+
id_: str = Field(alias='id', default=None)
|
|
64
|
+
name: Optional[str] = None
|
|
65
|
+
type_: str = Field(alias='type', default=None)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class IdOptionalNameType(RFBaseModel):
|
|
69
|
+
id_: str = Field(alias='id', default=None)
|
|
70
|
+
name: Optional[str] = None
|
|
71
|
+
type_: str = Field(alias='type', default=None)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class IdNameTypeDescription(IdNameType):
|
|
75
|
+
description: Optional[str] = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class IOCType(Enum):
|
|
79
|
+
ip = 'ip'
|
|
80
|
+
domain = 'domain'
|
|
81
|
+
hash = 'hash' # noqa: A003
|
|
82
|
+
vulnerability = 'vulnerability'
|
|
83
|
+
url = 'url'
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class DetectionRuleType(Enum):
|
|
87
|
+
sigma = 'sigma'
|
|
88
|
+
yara = 'yara'
|
|
89
|
+
snort = 'snort'
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from .config import Config, ConfigModel, get_config
|
|
15
|
+
from .errors import ConfigFileError
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
#################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
from copy import deepcopy
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Optional, Union
|
|
20
|
+
|
|
21
|
+
from pydantic import Field, Secret, field_validator, validate_call
|
|
22
|
+
from pydantic_settings import (
|
|
23
|
+
BaseSettings,
|
|
24
|
+
DotEnvSettingsSource,
|
|
25
|
+
EnvSettingsSource,
|
|
26
|
+
JsonConfigSettingsSource,
|
|
27
|
+
PydanticBaseSettingsSource,
|
|
28
|
+
SettingsConfigDict,
|
|
29
|
+
TomlConfigSettingsSource,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
from ..constants import (
|
|
33
|
+
BACKOFF_FACTOR,
|
|
34
|
+
POOL_MAX_SIZE,
|
|
35
|
+
REQUEST_TIMEOUT,
|
|
36
|
+
RETRY_TOTAL,
|
|
37
|
+
RF_TOKEN_ENV_VAR,
|
|
38
|
+
RF_TOKEN_VALIDATION_REGEX,
|
|
39
|
+
ROOT_DIR,
|
|
40
|
+
SSL_VERIFY,
|
|
41
|
+
STATUS_FORCELIST,
|
|
42
|
+
)
|
|
43
|
+
from ..helpers import OSHelpers
|
|
44
|
+
from .errors import ConfigFileError
|
|
45
|
+
|
|
46
|
+
PLAT_REGEX = r'^([A-Z]|[a-z])+(\/\d+)?((\.\d+)*?)$'
|
|
47
|
+
APP_ID_REGEX = r'^\S+\/\d+((\.\d+)*?)$'
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class RFToken(Secret[str]):
|
|
51
|
+
"""Recorded Future token mask."""
|
|
52
|
+
|
|
53
|
+
def _display(self) -> str:
|
|
54
|
+
return '********' + self.get_secret_value()[-4:]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ConfigModel(BaseSettings):
|
|
58
|
+
"""Global configuration settings.
|
|
59
|
+
|
|
60
|
+
This class is used to store global configuration settings for the application.
|
|
61
|
+
|
|
62
|
+
Supports config with .toml, .json and .env extensions.
|
|
63
|
+
Regular expression validation:
|
|
64
|
+
|
|
65
|
+
- app_id must be ``<str>/<int>[.<int>][.<int>]``
|
|
66
|
+
- platform_id must be ``<str>[/<int>][.<int>][.<int>]``
|
|
67
|
+
|
|
68
|
+
Example:
|
|
69
|
+
Initialize the ``Config`` with ``config_path``
|
|
70
|
+
|
|
71
|
+
.. code-block:: python
|
|
72
|
+
:linenos:
|
|
73
|
+
|
|
74
|
+
from psengine.config import Config, get_config
|
|
75
|
+
Config.init(config_path=<filepath>)
|
|
76
|
+
|
|
77
|
+
config = get_config()
|
|
78
|
+
|
|
79
|
+
Initialize the ``Config`` from python itself:
|
|
80
|
+
|
|
81
|
+
.. code-block:: python
|
|
82
|
+
:linenos:
|
|
83
|
+
|
|
84
|
+
from psengine.config import Config, get_config
|
|
85
|
+
Config.init(my_setting='example', my_second_setting='example2')
|
|
86
|
+
|
|
87
|
+
config = get_config()
|
|
88
|
+
config.my_setting
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
config_path: Union[str, Path, None] = None
|
|
92
|
+
model_config = SettingsConfigDict(arbitrary_types_allowed=True, extra='allow', frozen=True)
|
|
93
|
+
|
|
94
|
+
platform_id: Optional[str] = Field(default=None, pattern=PLAT_REGEX, examples=['Splunk/8.0.0'])
|
|
95
|
+
app_id: Optional[str] = Field(default=None, pattern=APP_ID_REGEX, examples=['get-alerts/1.0.0'])
|
|
96
|
+
rf_token: Optional[RFToken] = Field(default=os.environ.get(RF_TOKEN_ENV_VAR, ''))
|
|
97
|
+
http_proxy: Optional[str] = None
|
|
98
|
+
https_proxy: Optional[str] = None
|
|
99
|
+
client_ssl_verify: Optional[bool] = SSL_VERIFY
|
|
100
|
+
client_basic_auth: Optional[tuple[str, str]] = None
|
|
101
|
+
client_cert: Optional[Union[str, tuple[str, str]]] = None
|
|
102
|
+
client_timeout: Optional[int] = REQUEST_TIMEOUT
|
|
103
|
+
client_retries: Optional[int] = RETRY_TOTAL
|
|
104
|
+
client_backoff_factor: Optional[int] = BACKOFF_FACTOR
|
|
105
|
+
client_status_forcelist: Optional[list[int]] = STATUS_FORCELIST
|
|
106
|
+
client_pool_max_size: Optional[int] = POOL_MAX_SIZE
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def settings_customise_sources(
|
|
110
|
+
cls,
|
|
111
|
+
settings_cls: type[BaseSettings],
|
|
112
|
+
init_settings: PydanticBaseSettingsSource,
|
|
113
|
+
env_settings: PydanticBaseSettingsSource,
|
|
114
|
+
dotenv_settings: PydanticBaseSettingsSource,
|
|
115
|
+
file_secret_settings: PydanticBaseSettingsSource,
|
|
116
|
+
) -> tuple[PydanticBaseSettingsSource, ...]:
|
|
117
|
+
"""Set config file sources correctly for Config.
|
|
118
|
+
|
|
119
|
+
In the case where a value is specified for the same Settings field in multiple ways, the
|
|
120
|
+
selected value is determined as follows (in descending order of priority):
|
|
121
|
+
|
|
122
|
+
1. Arguments passed to the Config class initialiser (``Config.init``)
|
|
123
|
+
2. Environment variables
|
|
124
|
+
3. Config file specified in the config_path field
|
|
125
|
+
4. Variables loaded from the secrets directory
|
|
126
|
+
5. Default values for the Config model
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
settings_cls (Type[BaseSettings]): class settings
|
|
130
|
+
init_settings (PydanticBaseSettingsSource): initial settings callable
|
|
131
|
+
env_settings (PydanticBaseSettingsSource): environment settings callable
|
|
132
|
+
dotenv_settings (PydanticBaseSettingsSource): .env file settings callable
|
|
133
|
+
file_secret_settings (PydanticBaseSettingsSource): secrets file settings callable
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
Tuple[PydanticBaseSettingsSource, ...]: A tuple containing the sources and their order
|
|
137
|
+
for loading the settings values.
|
|
138
|
+
"""
|
|
139
|
+
env_settings = EnvSettingsSource(settings_cls, env_prefix='RF_', env_nested_delimiter='__')
|
|
140
|
+
sources = [
|
|
141
|
+
init_settings,
|
|
142
|
+
env_settings,
|
|
143
|
+
dotenv_settings,
|
|
144
|
+
file_secret_settings,
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
if config := init_settings.init_kwargs.get('config_path'):
|
|
148
|
+
config = config if isinstance(config, str) else config.as_posix()
|
|
149
|
+
if config.endswith('.toml'):
|
|
150
|
+
sources.insert(2, TomlConfigSettingsSource(settings_cls, Path(config)))
|
|
151
|
+
elif config.endswith('.json'):
|
|
152
|
+
sources.insert(2, JsonConfigSettingsSource(settings_cls, Path(config)))
|
|
153
|
+
elif config.endswith('.env'):
|
|
154
|
+
sources.insert(
|
|
155
|
+
2,
|
|
156
|
+
DotEnvSettingsSource(
|
|
157
|
+
settings_cls,
|
|
158
|
+
env_file=Path(config),
|
|
159
|
+
env_nested_delimiter='.',
|
|
160
|
+
env_prefix='RF_',
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
else:
|
|
164
|
+
raise ValueError('The config file extension must be .toml or .json or .env')
|
|
165
|
+
return tuple(sources)
|
|
166
|
+
|
|
167
|
+
@field_validator('rf_token', mode='before')
|
|
168
|
+
@classmethod
|
|
169
|
+
def validate_token(cls, rf_token):
|
|
170
|
+
"""Validate Recorded Future token.
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
rf_token (str): Recorded Future token
|
|
174
|
+
|
|
175
|
+
Raises:
|
|
176
|
+
ValueError: when token is not 32 alphanumeric characters in ``[a-f][0-9]`` range.
|
|
177
|
+
|
|
178
|
+
Returns:
|
|
179
|
+
str: token
|
|
180
|
+
"""
|
|
181
|
+
rf_token = rf_token or os.environ.get(RF_TOKEN_ENV_VAR)
|
|
182
|
+
if not rf_token:
|
|
183
|
+
# Edge case: when RF_RF_TOKEN env var is set, it is used and RF_TOKEN is ignored
|
|
184
|
+
# So we check if RF_TOKEN is set and validate it
|
|
185
|
+
return ''
|
|
186
|
+
if not re.match(RF_TOKEN_VALIDATION_REGEX, rf_token):
|
|
187
|
+
raise ValueError(
|
|
188
|
+
f'Invalid Recorded Future API token, must match regex {RF_TOKEN_VALIDATION_REGEX}'
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
return rf_token
|
|
192
|
+
|
|
193
|
+
@validate_call
|
|
194
|
+
def save_config(
|
|
195
|
+
self,
|
|
196
|
+
directory: Union[str, Path] = Path(ROOT_DIR) / 'config',
|
|
197
|
+
file: Union[str, Path] = 'config.json',
|
|
198
|
+
):
|
|
199
|
+
"""Writes the current values in Config, in the file provided as JSON.
|
|
200
|
+
|
|
201
|
+
If the file already exists, the content will be deleted.
|
|
202
|
+
The config will be saved in the ``<project_directory>/config/config.json``
|
|
203
|
+
|
|
204
|
+
"""
|
|
205
|
+
directory = Path(directory)
|
|
206
|
+
log = logging.getLogger(__name__)
|
|
207
|
+
data = self.model_dump_json(exclude='rf_token', indent=4)
|
|
208
|
+
OSHelpers.mkdir(directory)
|
|
209
|
+
config_path = directory / file
|
|
210
|
+
log.info(f'Saving config in {config_path.as_posix()}')
|
|
211
|
+
|
|
212
|
+
with config_path.open('w') as f:
|
|
213
|
+
f.write(data)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class Config:
|
|
217
|
+
"""Singleton class to manage Config instances.
|
|
218
|
+
|
|
219
|
+
Note that the config is Read Only. Once initialized, attributes cannot be changed unless you
|
|
220
|
+
initialize the config with new values.
|
|
221
|
+
"""
|
|
222
|
+
|
|
223
|
+
_instance = None
|
|
224
|
+
|
|
225
|
+
@classmethod
|
|
226
|
+
def _get_instance(cls) -> Union[ConfigModel, None]:
|
|
227
|
+
"""Get instance of ``Config``.
|
|
228
|
+
|
|
229
|
+
``get_config()`` should be used instead of calling this method directly
|
|
230
|
+
"""
|
|
231
|
+
if not cls._instance:
|
|
232
|
+
cls._instance = ConfigModel()
|
|
233
|
+
|
|
234
|
+
return cls._instance
|
|
235
|
+
|
|
236
|
+
@classmethod
|
|
237
|
+
def reset_instance(cls):
|
|
238
|
+
"""Method used for testing to clear up the previous instances of ``Config``."""
|
|
239
|
+
cls._instance = None
|
|
240
|
+
|
|
241
|
+
@classmethod
|
|
242
|
+
def init(cls, config_class=ConfigModel, **kwargs):
|
|
243
|
+
"""Initialize Config instance.
|
|
244
|
+
|
|
245
|
+
Call directly on class ``Config.init(config_path=<filepath>)``
|
|
246
|
+
|
|
247
|
+
Args:
|
|
248
|
+
config_class (ConfigModel): ConfigModel class
|
|
249
|
+
config_path (str): Path to the config file
|
|
250
|
+
platform_id (str): Name & version of the tool this integrates with, example: ES/8.0.0
|
|
251
|
+
app_id (str): Name & version of the integration itself, example: get-alerts/1.0.0
|
|
252
|
+
rf_token (str): Recorded Future API token
|
|
253
|
+
http_proxy (str): HTTP proxy
|
|
254
|
+
https_proxy (str): HTTPS proxy
|
|
255
|
+
client_ssl_verify (bool): SSL verification. Default is True
|
|
256
|
+
client_basic_auth (tuple): Basic auth credentials
|
|
257
|
+
client_cert (str or tuple): Client certificate
|
|
258
|
+
client_timeout (int): Request timeout. Default is 120
|
|
259
|
+
client_retries (int): Request retries. Default is 5
|
|
260
|
+
client_backoff_factor (int): Request backoff factor. Default is 1
|
|
261
|
+
client_status_forcelist (list): Request status forcelist. Default is [502, 503, 504]
|
|
262
|
+
client_pool_max_size (int): Request pool max size. Default is 120
|
|
263
|
+
kwargs: Additional arguments
|
|
264
|
+
|
|
265
|
+
"""
|
|
266
|
+
config_path = kwargs.get('config_path')
|
|
267
|
+
if config_path and not Path(config_path).exists():
|
|
268
|
+
raise ConfigFileError(f'File {config_path} does not exists.')
|
|
269
|
+
|
|
270
|
+
log = logging.getLogger(__name__)
|
|
271
|
+
cls._instance = config_class(**kwargs)
|
|
272
|
+
gc = cls._instance
|
|
273
|
+
|
|
274
|
+
config = deepcopy(gc.model_dump())
|
|
275
|
+
config.update(gc.model_extra)
|
|
276
|
+
log.info(f'Configuration Settings: {config}')
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def get_config():
|
|
280
|
+
"""Return an instance of ``Config``.
|
|
281
|
+
|
|
282
|
+
Use this instead of initializing Config directly, so that the same instance is used.
|
|
283
|
+
"""
|
|
284
|
+
return Config._get_instance()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from ..errors import RecordedFutureError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ConfigFileError(RecordedFutureError):
|
|
18
|
+
"""Error raised when config file provided doesnt exist or is not readbale."""
|