psengine 2.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psengine-2.0.4/LICENSE +21 -0
- psengine-2.0.4/PKG-INFO +189 -0
- psengine-2.0.4/README.rst +146 -0
- psengine-2.0.4/psengine/__init__.py +22 -0
- psengine-2.0.4/psengine/_sdk_id.py +16 -0
- psengine-2.0.4/psengine/_version.py +14 -0
- psengine-2.0.4/psengine/analyst_notes/__init__.py +32 -0
- psengine-2.0.4/psengine/analyst_notes/constants.py +15 -0
- psengine-2.0.4/psengine/analyst_notes/errors.py +42 -0
- psengine-2.0.4/psengine/analyst_notes/helpers.py +90 -0
- psengine-2.0.4/psengine/analyst_notes/models.py +219 -0
- psengine-2.0.4/psengine/analyst_notes/note.py +149 -0
- psengine-2.0.4/psengine/analyst_notes/note_mgr.py +400 -0
- psengine-2.0.4/psengine/base_http_client.py +285 -0
- psengine-2.0.4/psengine/classic_alerts/__init__.py +24 -0
- psengine-2.0.4/psengine/classic_alerts/classic_alert.py +275 -0
- psengine-2.0.4/psengine/classic_alerts/classic_alert_mgr.py +507 -0
- psengine-2.0.4/psengine/classic_alerts/constants.py +31 -0
- psengine-2.0.4/psengine/classic_alerts/errors.py +38 -0
- psengine-2.0.4/psengine/classic_alerts/helpers.py +87 -0
- psengine-2.0.4/psengine/classic_alerts/markdown/__init__.py +13 -0
- psengine-2.0.4/psengine/classic_alerts/markdown/markdown.py +359 -0
- psengine-2.0.4/psengine/classic_alerts/models.py +141 -0
- psengine-2.0.4/psengine/collective_insights/__init__.py +29 -0
- psengine-2.0.4/psengine/collective_insights/collective_insights.py +164 -0
- psengine-2.0.4/psengine/collective_insights/constants.py +44 -0
- psengine-2.0.4/psengine/collective_insights/errors.py +18 -0
- psengine-2.0.4/psengine/collective_insights/insight.py +89 -0
- psengine-2.0.4/psengine/collective_insights/models.py +81 -0
- psengine-2.0.4/psengine/common_models.py +89 -0
- psengine-2.0.4/psengine/config/__init__.py +15 -0
- psengine-2.0.4/psengine/config/config.py +284 -0
- psengine-2.0.4/psengine/config/errors.py +18 -0
- psengine-2.0.4/psengine/constants.py +63 -0
- psengine-2.0.4/psengine/detection/__init__.py +17 -0
- psengine-2.0.4/psengine/detection/detection_mgr.py +135 -0
- psengine-2.0.4/psengine/detection/detection_rule.py +85 -0
- psengine-2.0.4/psengine/detection/errors.py +26 -0
- psengine-2.0.4/psengine/detection/helpers.py +56 -0
- psengine-2.0.4/psengine/detection/models.py +47 -0
- psengine-2.0.4/psengine/endpoints.py +98 -0
- psengine-2.0.4/psengine/enrich/__init__.py +28 -0
- psengine-2.0.4/psengine/enrich/constants.py +73 -0
- psengine-2.0.4/psengine/enrich/errors.py +26 -0
- psengine-2.0.4/psengine/enrich/lookup.py +299 -0
- psengine-2.0.4/psengine/enrich/lookup_mgr.py +341 -0
- psengine-2.0.4/psengine/enrich/models/__init__.py +13 -0
- psengine-2.0.4/psengine/enrich/models/base_enriched_entity.py +43 -0
- psengine-2.0.4/psengine/enrich/models/lookup.py +271 -0
- psengine-2.0.4/psengine/enrich/models/soar.py +138 -0
- psengine-2.0.4/psengine/enrich/soar.py +89 -0
- psengine-2.0.4/psengine/enrich/soar_mgr.py +176 -0
- psengine-2.0.4/psengine/entity_lists/__init__.py +16 -0
- psengine-2.0.4/psengine/entity_lists/constants.py +19 -0
- psengine-2.0.4/psengine/entity_lists/entity_list.py +435 -0
- psengine-2.0.4/psengine/entity_lists/entity_list_mgr.py +185 -0
- psengine-2.0.4/psengine/entity_lists/errors.py +26 -0
- psengine-2.0.4/psengine/entity_lists/models.py +87 -0
- psengine-2.0.4/psengine/entity_match/__init__.py +16 -0
- psengine-2.0.4/psengine/entity_match/entity_match.py +90 -0
- psengine-2.0.4/psengine/entity_match/entity_match_mgr.py +235 -0
- psengine-2.0.4/psengine/entity_match/errors.py +18 -0
- psengine-2.0.4/psengine/entity_match/models.py +22 -0
- psengine-2.0.4/psengine/errors.py +41 -0
- psengine-2.0.4/psengine/helpers/__init__.py +23 -0
- psengine-2.0.4/psengine/helpers/helpers.py +471 -0
- psengine-2.0.4/psengine/logger/__init__.py +15 -0
- psengine-2.0.4/psengine/logger/constants.py +39 -0
- psengine-2.0.4/psengine/logger/errors.py +18 -0
- psengine-2.0.4/psengine/logger/rf_logger.py +148 -0
- psengine-2.0.4/psengine/markdown/__init__.py +21 -0
- psengine-2.0.4/psengine/markdown/markdown.py +169 -0
- psengine-2.0.4/psengine/markdown/models.py +22 -0
- psengine-2.0.4/psengine/playbook_alerts/__init__.py +34 -0
- psengine-2.0.4/psengine/playbook_alerts/constants.py +35 -0
- psengine-2.0.4/psengine/playbook_alerts/errors.py +35 -0
- psengine-2.0.4/psengine/playbook_alerts/helpers.py +80 -0
- psengine-2.0.4/psengine/playbook_alerts/mappings.py +44 -0
- psengine-2.0.4/psengine/playbook_alerts/markdown/__init__.py +13 -0
- psengine-2.0.4/psengine/playbook_alerts/markdown/markdown.py +98 -0
- psengine-2.0.4/psengine/playbook_alerts/markdown/markdown_code_repo.py +64 -0
- psengine-2.0.4/psengine/playbook_alerts/markdown/markdown_domain_abuse.py +118 -0
- psengine-2.0.4/psengine/playbook_alerts/markdown/markdown_identity_exposure.py +158 -0
- psengine-2.0.4/psengine/playbook_alerts/models/__init__.py +36 -0
- psengine-2.0.4/psengine/playbook_alerts/models/common_models.py +18 -0
- psengine-2.0.4/psengine/playbook_alerts/models/panel_log.py +329 -0
- psengine-2.0.4/psengine/playbook_alerts/models/panel_status.py +70 -0
- psengine-2.0.4/psengine/playbook_alerts/models/pba_code_repo_leak.py +52 -0
- psengine-2.0.4/psengine/playbook_alerts/models/pba_cyber_vulnerability.py +53 -0
- psengine-2.0.4/psengine/playbook_alerts/models/pba_domain_abuse.py +139 -0
- psengine-2.0.4/psengine/playbook_alerts/models/pba_identity_exposures.py +93 -0
- psengine-2.0.4/psengine/playbook_alerts/models/pba_third_party_risk.py +103 -0
- psengine-2.0.4/psengine/playbook_alerts/models/search_endpoint.py +68 -0
- psengine-2.0.4/psengine/playbook_alerts/pa_category.py +37 -0
- psengine-2.0.4/psengine/playbook_alerts/playbook_alert_mgr.py +593 -0
- psengine-2.0.4/psengine/playbook_alerts/playbook_alerts.py +393 -0
- psengine-2.0.4/psengine/rf_client.py +430 -0
- psengine-2.0.4/psengine/risklists/__init__.py +17 -0
- psengine-2.0.4/psengine/risklists/constants.py +15 -0
- psengine-2.0.4/psengine/risklists/errors.py +20 -0
- psengine-2.0.4/psengine/risklists/models.py +65 -0
- psengine-2.0.4/psengine/risklists/risklist_mgr.py +156 -0
- psengine-2.0.4/psengine/stix2/__init__.py +21 -0
- psengine-2.0.4/psengine/stix2/base_stix_entity.py +62 -0
- psengine-2.0.4/psengine/stix2/complex_entity.py +372 -0
- psengine-2.0.4/psengine/stix2/constants.py +81 -0
- psengine-2.0.4/psengine/stix2/enriched_indicator.py +261 -0
- psengine-2.0.4/psengine/stix2/errors.py +22 -0
- psengine-2.0.4/psengine/stix2/helpers.py +68 -0
- psengine-2.0.4/psengine/stix2/rf_bundle.py +240 -0
- psengine-2.0.4/psengine/stix2/simple_entity.py +145 -0
- psengine-2.0.4/psengine/stix2/util.py +53 -0
- psengine-2.0.4/psengine.egg-info/PKG-INFO +189 -0
- psengine-2.0.4/psengine.egg-info/SOURCES.txt +118 -0
- psengine-2.0.4/psengine.egg-info/dependency_links.txt +1 -0
- psengine-2.0.4/psengine.egg-info/entry_points.txt +2 -0
- psengine-2.0.4/psengine.egg-info/requires.txt +30 -0
- psengine-2.0.4/psengine.egg-info/top_level.txt +1 -0
- psengine-2.0.4/pyproject.toml +94 -0
- psengine-2.0.4/setup.cfg +4 -0
psengine-2.0.4/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 RecordedFuture-ProfessionalServices
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
psengine-2.0.4/PKG-INFO
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: psengine
|
|
3
|
+
Version: 2.0.4
|
|
4
|
+
Summary: psengine is a simple, yet elegant, library for rapid development of integrations with Recorded Future.
|
|
5
|
+
Author-email: Moise Medici <moise.medici@recordedfuture.com>, Patrick Kinsella <patrick.kinsella@recordedfuture.com>, Ernest Bartosevic <ernest.bartosevic@recordedfuture.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/RecordedFuture-ProfessionalServices/psengine-py
|
|
8
|
+
Project-URL: Changelog, https://github.com/RecordedFuture-ProfessionalServices/psengine-py/CHANGELOG.rst
|
|
9
|
+
Keywords: API,Recorded Future,Cyber Security Engineering,Threat Intelligence
|
|
10
|
+
Requires-Python: <3.14,>=3.9
|
|
11
|
+
Description-Content-Type: text/x-rst
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: requests>=2.27.1
|
|
14
|
+
Requires-Dist: jsonpath_ng<=1.6.1,>=1.5.3
|
|
15
|
+
Requires-Dist: stix2~=3.0.1
|
|
16
|
+
Requires-Dist: python-dateutil>=2.7.0
|
|
17
|
+
Requires-Dist: more-itertools<=10.2.0,>=9.0.0
|
|
18
|
+
Requires-Dist: pydantic<3.0.0,>=2.7
|
|
19
|
+
Requires-Dist: pydantic-settings[toml]~=2.5.2
|
|
20
|
+
Requires-Dist: markdown-strings==3.4.0
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: tox==4.12.1; extra == "dev"
|
|
23
|
+
Requires-Dist: build==1.0.3; extra == "dev"
|
|
24
|
+
Requires-Dist: pytest==8.3.4; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov==6.0.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-mock==3.14.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-random-order==1.1.1; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-vcr==1.0.2; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-watch==4.2.0; extra == "dev"
|
|
30
|
+
Requires-Dist: requests==2.29.0; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff~=0.7.0; extra == "dev"
|
|
32
|
+
Requires-Dist: wheel==0.37.1; extra == "dev"
|
|
33
|
+
Requires-Dist: setuptools==61.0.0; extra == "dev"
|
|
34
|
+
Requires-Dist: Sphinx==7.1.2; extra == "dev"
|
|
35
|
+
Requires-Dist: sphinxcontrib-confluencebuilder==2.3.0; extra == "dev"
|
|
36
|
+
Requires-Dist: atlassian-python-api==3.41.4; extra == "dev"
|
|
37
|
+
Requires-Dist: sphinx_autodoc_typehints==1.25.2; extra == "dev"
|
|
38
|
+
Requires-Dist: sphinxcontrib-napoleon; extra == "dev"
|
|
39
|
+
Requires-Dist: typer==0.12.5; extra == "dev"
|
|
40
|
+
Requires-Dist: cookiecutter==2.6.0; extra == "dev"
|
|
41
|
+
Requires-Dist: tomlkit>=0.13.2; extra == "dev"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
==================================================
|
|
45
|
+
PSEngine
|
|
46
|
+
==================================================
|
|
47
|
+
**PSEngine** is a simple, yet elegant, library for rapid development of integrations with Recorded Future.
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
.. code-block:: python
|
|
51
|
+
|
|
52
|
+
>>> from psengine.enrich import LookupMgr
|
|
53
|
+
>>> lookup_mgr = LookupMgr(rf_token='token')
|
|
54
|
+
>>> domain = lookup_mgr.lookup('cpejcogzznpudbsmaxxm.com', 'domain')
|
|
55
|
+
>>> domain
|
|
56
|
+
'EnrichedDomain: cpejcogzznpudbsmaxxm.com, Risk Score: 20, Last Seen: 2024-07-22 02:50:59PM'
|
|
57
|
+
>>> domain.entity
|
|
58
|
+
'cpejcogzznpudbsmaxxm.com'
|
|
59
|
+
>>> domain.content.risk
|
|
60
|
+
EntityRisk(criticality_label='Unusual', risk_string='4/52', score=20, rules=4...)
|
|
61
|
+
>>> domain.content.risk.score
|
|
62
|
+
20
|
|
63
|
+
>>>
|
|
64
|
+
domain.content.risk.risk_summary
|
|
65
|
+
'4 of 52 Risk Rules currently observed.'
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
PSEngine allows you to interact with the Recorded Future API extremely easily. There’s no need to manually build the URLs and query parameters - but nowadays, just use the modules dedicated to individual API endpoints!
|
|
69
|
+
|
|
70
|
+
PSEngine is a Python package solely built and maintained by the Cyber Security Engineering team powering a number of high profile integrations, such as: Elasticsearch, QRadar, Anomali, Jira, TheHive, etc..
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
Installation
|
|
74
|
+
==================================================
|
|
75
|
+
PSEngine is a Python package that can be installed using pip. To install PSengine, run the following command:
|
|
76
|
+
|
|
77
|
+
.. code-block:: bash
|
|
78
|
+
|
|
79
|
+
$ pip install psengine git+https://github.com/RecordedFuture-ProfessionalServices/psengine.git@main
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
PSEngine officially supports Python >= 3.9, < 3.14
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
Supported Features & Best–Practices
|
|
86
|
+
==================================================
|
|
87
|
+
|
|
88
|
+
PSEngine is ready for the demands of building robust and reliable integrations.
|
|
89
|
+
|
|
90
|
+
* Collective Insights
|
|
91
|
+
* Analyst Notes
|
|
92
|
+
* Classic & Playbook Alerts
|
|
93
|
+
* Risklists
|
|
94
|
+
* On demand IOC enrichment
|
|
95
|
+
* List management
|
|
96
|
+
* Detection Rules
|
|
97
|
+
* Built in logging
|
|
98
|
+
* Easy configuration management
|
|
99
|
+
* Proxy support
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
Quick Start
|
|
103
|
+
==================================================
|
|
104
|
+
Excited, to get started?
|
|
105
|
+
|
|
106
|
+
The section below will give you the basic building blocks to start building integrations with PSEngine.
|
|
107
|
+
|
|
108
|
+
But first ensure that:
|
|
109
|
+
|
|
110
|
+
- PSEngine is installed
|
|
111
|
+
- PSEngine is up-to-date
|
|
112
|
+
|
|
113
|
+
Let’s get started with some core concepts and practices.
|
|
114
|
+
|
|
115
|
+
Config Management
|
|
116
|
+
--------------------------------------------------
|
|
117
|
+
The key requirement when building integrations with PSEngine is initializing `Config` as early as possible in your program,
|
|
118
|
+
before initializing any PSEngine managers. This way `rf_token` `app_id` and `platform_id` you set will be used by every manager
|
|
119
|
+
initialized after the Config.
|
|
120
|
+
|
|
121
|
+
.. code-block:: python
|
|
122
|
+
|
|
123
|
+
>>> from psengine.config import Config, get_config
|
|
124
|
+
# Name & version of the integration itself
|
|
125
|
+
>>> APP_ID = 'example-app/1.0.0'
|
|
126
|
+
# Name & version of the tool this integrates with (Optional)
|
|
127
|
+
>>> PLATFORM_ID = 'PSE/1.0.0'
|
|
128
|
+
>>> Config.init(rf_token='your_token', app_id=APP_ID, platform_id=PLATFORM_ID)
|
|
129
|
+
>>> config = get_config()
|
|
130
|
+
>>> config.app_id
|
|
131
|
+
'example-app/1.0.0'
|
|
132
|
+
|
|
133
|
+
The above will result in API calls made by the managers having the following headers set:
|
|
134
|
+
|
|
135
|
+
- 'X-RFToken' Header will contain the Recorded Future API Token
|
|
136
|
+
- 'User-Agent' Header will contain APP ID and Platform ID (if supplied) which is a Recorded Future requirement, which might look like this:
|
|
137
|
+
|
|
138
|
+
example-app/1.0.0 (macOS-14.1-arm64-arm-64bit) psengine-py/2.0.1 PSE/1.0.0
|
|
139
|
+
|
|
140
|
+
Authorization
|
|
141
|
+
--------------------------------------------------
|
|
142
|
+
In the example above we saw a token passed to the Config by the caller, but you can also omit the token during initialization and let
|
|
143
|
+
Config retrieve it from the environment variable `RF_TOKEN`. Just ensure that the environment variable is set before running your program:
|
|
144
|
+
|
|
145
|
+
export RF_TOKEN=your_token
|
|
146
|
+
|
|
147
|
+
Alternatively, if you want to set an rf_token separately for a single manager, you may pass it in the constructor:
|
|
148
|
+
|
|
149
|
+
.. code-block:: python
|
|
150
|
+
|
|
151
|
+
>>> note_mgr = AnalystNoteMgr(rf_token='your_token')
|
|
152
|
+
|
|
153
|
+
Logging
|
|
154
|
+
--------------------------------------------------
|
|
155
|
+
PSEngine also provides the capability for logging to console and files. If your program needs to show log output on the terminal and keep a .log file, just import and use psengine’s logger:
|
|
156
|
+
|
|
157
|
+
.. code-block:: python
|
|
158
|
+
|
|
159
|
+
>>> from psengine.logger import RFLogger
|
|
160
|
+
>>> LOG = RFLogger().get_logger()
|
|
161
|
+
>>> LOG.info('Hello, world!')
|
|
162
|
+
|
|
163
|
+
On the other hand, if your program’s log statements already have handlers setup, just log the normal way:
|
|
164
|
+
|
|
165
|
+
.. code-block:: python
|
|
166
|
+
|
|
167
|
+
>>> import logging
|
|
168
|
+
>>> LOG = logging.getLogger(__name__)
|
|
169
|
+
>>> LOG.info('Hello, world!')
|
|
170
|
+
|
|
171
|
+
In the second example, nothing is printed to terminal or file unless a handler is setup by another program running your code.
|
|
172
|
+
|
|
173
|
+
Proxies
|
|
174
|
+
--------------------------------------------------
|
|
175
|
+
If your environment requires a proxy to access the internet, you can set the proxy in the Config:
|
|
176
|
+
|
|
177
|
+
.. code-block:: python
|
|
178
|
+
|
|
179
|
+
>>> Config.init(
|
|
180
|
+
app_id=APP_ID,
|
|
181
|
+
platform_id=PLATFORM_ID,
|
|
182
|
+
http_proxy='http://proxy:8080',
|
|
183
|
+
https_proxy='http://proxy:8080',
|
|
184
|
+
client_ssl_verify=False,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
Examples
|
|
188
|
+
--------------------------------------------------
|
|
189
|
+
Please refer to `examples <examples>`_ for usage example of each module.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
==================================================
|
|
2
|
+
PSEngine
|
|
3
|
+
==================================================
|
|
4
|
+
**PSEngine** is a simple, yet elegant, library for rapid development of integrations with Recorded Future.
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
.. code-block:: python
|
|
8
|
+
|
|
9
|
+
>>> from psengine.enrich import LookupMgr
|
|
10
|
+
>>> lookup_mgr = LookupMgr(rf_token='token')
|
|
11
|
+
>>> domain = lookup_mgr.lookup('cpejcogzznpudbsmaxxm.com', 'domain')
|
|
12
|
+
>>> domain
|
|
13
|
+
'EnrichedDomain: cpejcogzznpudbsmaxxm.com, Risk Score: 20, Last Seen: 2024-07-22 02:50:59PM'
|
|
14
|
+
>>> domain.entity
|
|
15
|
+
'cpejcogzznpudbsmaxxm.com'
|
|
16
|
+
>>> domain.content.risk
|
|
17
|
+
EntityRisk(criticality_label='Unusual', risk_string='4/52', score=20, rules=4...)
|
|
18
|
+
>>> domain.content.risk.score
|
|
19
|
+
20
|
|
20
|
+
>>>
|
|
21
|
+
domain.content.risk.risk_summary
|
|
22
|
+
'4 of 52 Risk Rules currently observed.'
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
PSEngine allows you to interact with the Recorded Future API extremely easily. There’s no need to manually build the URLs and query parameters - but nowadays, just use the modules dedicated to individual API endpoints!
|
|
26
|
+
|
|
27
|
+
PSEngine is a Python package solely built and maintained by the Cyber Security Engineering team powering a number of high profile integrations, such as: Elasticsearch, QRadar, Anomali, Jira, TheHive, etc..
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
Installation
|
|
31
|
+
==================================================
|
|
32
|
+
PSEngine is a Python package that can be installed using pip. To install PSengine, run the following command:
|
|
33
|
+
|
|
34
|
+
.. code-block:: bash
|
|
35
|
+
|
|
36
|
+
$ pip install psengine git+https://github.com/RecordedFuture-ProfessionalServices/psengine.git@main
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
PSEngine officially supports Python >= 3.9, < 3.14
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
Supported Features & Best–Practices
|
|
43
|
+
==================================================
|
|
44
|
+
|
|
45
|
+
PSEngine is ready for the demands of building robust and reliable integrations.
|
|
46
|
+
|
|
47
|
+
* Collective Insights
|
|
48
|
+
* Analyst Notes
|
|
49
|
+
* Classic & Playbook Alerts
|
|
50
|
+
* Risklists
|
|
51
|
+
* On demand IOC enrichment
|
|
52
|
+
* List management
|
|
53
|
+
* Detection Rules
|
|
54
|
+
* Built in logging
|
|
55
|
+
* Easy configuration management
|
|
56
|
+
* Proxy support
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
Quick Start
|
|
60
|
+
==================================================
|
|
61
|
+
Excited, to get started?
|
|
62
|
+
|
|
63
|
+
The section below will give you the basic building blocks to start building integrations with PSEngine.
|
|
64
|
+
|
|
65
|
+
But first ensure that:
|
|
66
|
+
|
|
67
|
+
- PSEngine is installed
|
|
68
|
+
- PSEngine is up-to-date
|
|
69
|
+
|
|
70
|
+
Let’s get started with some core concepts and practices.
|
|
71
|
+
|
|
72
|
+
Config Management
|
|
73
|
+
--------------------------------------------------
|
|
74
|
+
The key requirement when building integrations with PSEngine is initializing `Config` as early as possible in your program,
|
|
75
|
+
before initializing any PSEngine managers. This way `rf_token` `app_id` and `platform_id` you set will be used by every manager
|
|
76
|
+
initialized after the Config.
|
|
77
|
+
|
|
78
|
+
.. code-block:: python
|
|
79
|
+
|
|
80
|
+
>>> from psengine.config import Config, get_config
|
|
81
|
+
# Name & version of the integration itself
|
|
82
|
+
>>> APP_ID = 'example-app/1.0.0'
|
|
83
|
+
# Name & version of the tool this integrates with (Optional)
|
|
84
|
+
>>> PLATFORM_ID = 'PSE/1.0.0'
|
|
85
|
+
>>> Config.init(rf_token='your_token', app_id=APP_ID, platform_id=PLATFORM_ID)
|
|
86
|
+
>>> config = get_config()
|
|
87
|
+
>>> config.app_id
|
|
88
|
+
'example-app/1.0.0'
|
|
89
|
+
|
|
90
|
+
The above will result in API calls made by the managers having the following headers set:
|
|
91
|
+
|
|
92
|
+
- 'X-RFToken' Header will contain the Recorded Future API Token
|
|
93
|
+
- 'User-Agent' Header will contain APP ID and Platform ID (if supplied) which is a Recorded Future requirement, which might look like this:
|
|
94
|
+
|
|
95
|
+
example-app/1.0.0 (macOS-14.1-arm64-arm-64bit) psengine-py/2.0.1 PSE/1.0.0
|
|
96
|
+
|
|
97
|
+
Authorization
|
|
98
|
+
--------------------------------------------------
|
|
99
|
+
In the example above we saw a token passed to the Config by the caller, but you can also omit the token during initialization and let
|
|
100
|
+
Config retrieve it from the environment variable `RF_TOKEN`. Just ensure that the environment variable is set before running your program:
|
|
101
|
+
|
|
102
|
+
export RF_TOKEN=your_token
|
|
103
|
+
|
|
104
|
+
Alternatively, if you want to set an rf_token separately for a single manager, you may pass it in the constructor:
|
|
105
|
+
|
|
106
|
+
.. code-block:: python
|
|
107
|
+
|
|
108
|
+
>>> note_mgr = AnalystNoteMgr(rf_token='your_token')
|
|
109
|
+
|
|
110
|
+
Logging
|
|
111
|
+
--------------------------------------------------
|
|
112
|
+
PSEngine also provides the capability for logging to console and files. If your program needs to show log output on the terminal and keep a .log file, just import and use psengine’s logger:
|
|
113
|
+
|
|
114
|
+
.. code-block:: python
|
|
115
|
+
|
|
116
|
+
>>> from psengine.logger import RFLogger
|
|
117
|
+
>>> LOG = RFLogger().get_logger()
|
|
118
|
+
>>> LOG.info('Hello, world!')
|
|
119
|
+
|
|
120
|
+
On the other hand, if your program’s log statements already have handlers setup, just log the normal way:
|
|
121
|
+
|
|
122
|
+
.. code-block:: python
|
|
123
|
+
|
|
124
|
+
>>> import logging
|
|
125
|
+
>>> LOG = logging.getLogger(__name__)
|
|
126
|
+
>>> LOG.info('Hello, world!')
|
|
127
|
+
|
|
128
|
+
In the second example, nothing is printed to terminal or file unless a handler is setup by another program running your code.
|
|
129
|
+
|
|
130
|
+
Proxies
|
|
131
|
+
--------------------------------------------------
|
|
132
|
+
If your environment requires a proxy to access the internet, you can set the proxy in the Config:
|
|
133
|
+
|
|
134
|
+
.. code-block:: python
|
|
135
|
+
|
|
136
|
+
>>> Config.init(
|
|
137
|
+
app_id=APP_ID,
|
|
138
|
+
platform_id=PLATFORM_ID,
|
|
139
|
+
http_proxy='http://proxy:8080',
|
|
140
|
+
https_proxy='http://proxy:8080',
|
|
141
|
+
client_ssl_verify=False,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
Examples
|
|
145
|
+
--------------------------------------------------
|
|
146
|
+
Please refer to `examples <examples>`_ for usage example of each module.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
|
|
16
|
+
from ._version import __version__ as version
|
|
17
|
+
from .base_http_client import BaseHTTPClient
|
|
18
|
+
from .errors import ReadFileError, RecordedFutureError, WriteFileError
|
|
19
|
+
from .rf_client import RFClient
|
|
20
|
+
|
|
21
|
+
log = logging.getLogger('psengine')
|
|
22
|
+
log.addHandler(logging.NullHandler())
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from ._version import __version__
|
|
15
|
+
|
|
16
|
+
SDK_ID = f'psengine-py/{__version__}'
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
__version__ = '2.0.4'
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from .errors import (
|
|
15
|
+
AnalystNoteAttachmentError,
|
|
16
|
+
AnalystNoteDeleteError,
|
|
17
|
+
AnalystNoteError,
|
|
18
|
+
AnalystNoteLookupError,
|
|
19
|
+
AnalystNotePreviewError,
|
|
20
|
+
AnalystNotePublishError,
|
|
21
|
+
AnalystNoteSearchError,
|
|
22
|
+
)
|
|
23
|
+
from .helpers import save_attachment, save_note
|
|
24
|
+
from .note import (
|
|
25
|
+
AnalystNote,
|
|
26
|
+
AnalystNotePreviewIn,
|
|
27
|
+
AnalystNotePreviewOut,
|
|
28
|
+
AnalystNotePublishIn,
|
|
29
|
+
AnalystNotePublishOut,
|
|
30
|
+
AnalystNoteSearchIn,
|
|
31
|
+
)
|
|
32
|
+
from .note_mgr import AnalystNoteMgr
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
URL_TO_PORTAL = 'https://app.recordedfuture.com/portal/analyst-note/shared/true/{}'
|
|
15
|
+
NOTES_PER_PAGE = 20
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
from ..errors import RecordedFutureError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AnalystNoteError(RecordedFutureError):
|
|
18
|
+
"""Error raise when the init of AnalystNote is failing."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AnalystNoteLookupError(RecordedFutureError):
|
|
22
|
+
"""Error raise when cannot lookup an analyst note."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AnalystNoteSearchError(RecordedFutureError):
|
|
26
|
+
"""Error raise when cannot search analyst notes."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AnalystNoteAttachmentError(RecordedFutureError):
|
|
30
|
+
"""Error raise when cannot lookup an analyst note."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AnalystNoteDeleteError(RecordedFutureError):
|
|
34
|
+
"""Error raise when cannot delete an analyst note."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class AnalystNotePreviewError(RecordedFutureError):
|
|
38
|
+
"""Error raise when cannot post to preview endpoint."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class AnalystNotePublishError(RecordedFutureError):
|
|
42
|
+
"""Error raise when cannot post to publish endpoint."""
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
##################################### TERMS OF USE ###########################################
|
|
2
|
+
# The following code is provided for demonstration purpose only, and should not be used #
|
|
3
|
+
# without independent verification. Recorded Future makes no representations or warranties, #
|
|
4
|
+
# express, implied, statutory, or otherwise, regarding any aspect of this code or of the #
|
|
5
|
+
# information it may retrieve, and provides it both strictly “as-is” and without assuming #
|
|
6
|
+
# responsibility for any information it may retrieve. Recorded Future shall not be liable #
|
|
7
|
+
# for, and you assume all risk of using, the foregoing. By using this code, Customer #
|
|
8
|
+
# represents that it is solely responsible for having all necessary licenses, permissions, #
|
|
9
|
+
# rights, and/or consents to connect to third party APIs, and that it is solely responsible #
|
|
10
|
+
# for having all necessary licenses, permissions, rights, and/or consents to any data #
|
|
11
|
+
# accessed from any third party API. #
|
|
12
|
+
##############################################################################################
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Union
|
|
18
|
+
|
|
19
|
+
from pydantic import validate_call
|
|
20
|
+
|
|
21
|
+
from ..errors import WriteFileError
|
|
22
|
+
from ..helpers import OSHelpers, debug_call
|
|
23
|
+
from .note import AnalystNote
|
|
24
|
+
|
|
25
|
+
LOG = logging.getLogger('psengine.analyst_notes.helpers')
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@debug_call
|
|
29
|
+
@validate_call
|
|
30
|
+
def save_attachment(
|
|
31
|
+
note_id: str, data: Union[bytes, str], ext: str, output_directory: Union[str, Path]
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Save yara, sigma, snort or pdf to file. The file will use the extension provided and the
|
|
34
|
+
``note_id`` to create the filename.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
note_id (str): ``AnalystNote`` id
|
|
38
|
+
data (Union[bytes, str]): data, returned from ``fetch_attachment``
|
|
39
|
+
ext (str): extension of the attachment, returned by ``fetch_attachment``
|
|
40
|
+
output_directory (str, Path): the directory to save the file into
|
|
41
|
+
"""
|
|
42
|
+
output_directory = (
|
|
43
|
+
output_directory if isinstance(output_directory, str) else output_directory.as_posix()
|
|
44
|
+
)
|
|
45
|
+
_save_attachment(note_id, data, ext, output_directory)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@debug_call
|
|
49
|
+
@validate_call
|
|
50
|
+
def save_note(note: AnalystNote, output_directory: Union[str, Path]) -> None:
|
|
51
|
+
"""Save AnalystNote object on file. The file will have the name of the note id.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
note (AnalystNote): note to save.
|
|
55
|
+
output_directory (str, Path): the directory to save the file into
|
|
56
|
+
"""
|
|
57
|
+
output_directory = (
|
|
58
|
+
output_directory if isinstance(output_directory, str) else output_directory.as_posix()
|
|
59
|
+
)
|
|
60
|
+
_save_attachment(
|
|
61
|
+
note_id=note.id_,
|
|
62
|
+
data=json.dumps(note.json(), indent=4),
|
|
63
|
+
ext='json',
|
|
64
|
+
output_directory=output_directory,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _save_attachment(
|
|
69
|
+
note_id: str, data: Union[bytes, str], ext: str, output_directory: str
|
|
70
|
+
) -> None:
|
|
71
|
+
"""Save attachment from bytes or note itself from json.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
note_id (str): id of the note, will be the filename
|
|
75
|
+
data (bytes | str): content to save
|
|
76
|
+
ext (str): extension of the file.
|
|
77
|
+
output_directory (str): the directory to save the file into
|
|
78
|
+
|
|
79
|
+
Raises:
|
|
80
|
+
WriteFileError: if saving to file fails
|
|
81
|
+
"""
|
|
82
|
+
try:
|
|
83
|
+
note_id = note_id.removeprefix('doc:')
|
|
84
|
+
LOG.debug(f"Saving file related to '{note_id}' to disk")
|
|
85
|
+
|
|
86
|
+
dir_path = OSHelpers.mkdir(output_directory)
|
|
87
|
+
note_path = Path(dir_path) / f'{note_id}.{ext}'
|
|
88
|
+
note_path.write_bytes(data) if isinstance(data, bytes) else note_path.write_text(data)
|
|
89
|
+
except (FileNotFoundError, IsADirectoryError, PermissionError, OSError) as err:
|
|
90
|
+
raise WriteFileError(f'Failed to save file to disk. Cause: {err.args}') from err
|