pycti 6.1.4__py3-none-any.whl → 6.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pycti might be problematic. Click here for more details.
- pycti/__init__.py +1 -1
- pycti/api/opencti_api_client.py +10 -8
- pycti/connector/opencti_connector_helper.py +7 -2
- pycti/utils/opencti_stix2.py +265 -147
- pycti/utils/opencti_stix2_splitter.py +24 -12
- {pycti-6.1.4.dist-info → pycti-6.1.5.dist-info}/METADATA +9 -7
- {pycti-6.1.4.dist-info → pycti-6.1.5.dist-info}/RECORD +10 -10
- {pycti-6.1.4.dist-info → pycti-6.1.5.dist-info}/LICENSE +0 -0
- {pycti-6.1.4.dist-info → pycti-6.1.5.dist-info}/WHEEL +0 -0
- {pycti-6.1.4.dist-info → pycti-6.1.5.dist-info}/top_level.txt +0 -0
pycti/__init__.py
CHANGED
pycti/api/opencti_api_client.py
CHANGED
|
@@ -351,14 +351,16 @@ class OpenCTIApiClient:
|
|
|
351
351
|
if "name" in main_error
|
|
352
352
|
else main_error["message"]
|
|
353
353
|
)
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
else
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
354
|
+
error_detail = {
|
|
355
|
+
"name": error_name,
|
|
356
|
+
"error_message": main_error["message"],
|
|
357
|
+
}
|
|
358
|
+
meta_data = main_error["data"] if "data" in main_error else {}
|
|
359
|
+
# Prevent logging of input as bundle is logged differently
|
|
360
|
+
if meta_data.get("input") is not None:
|
|
361
|
+
del meta_data["input"]
|
|
362
|
+
value_error = {**error_detail, **meta_data}
|
|
363
|
+
raise ValueError(value_error)
|
|
362
364
|
else:
|
|
363
365
|
return result
|
|
364
366
|
else:
|
|
@@ -1226,9 +1226,14 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
1226
1226
|
|
|
1227
1227
|
if bypass_split:
|
|
1228
1228
|
bundles = [bundle]
|
|
1229
|
+
expectations_number = len(json.loads(bundle)["objects"])
|
|
1229
1230
|
else:
|
|
1230
1231
|
stix2_splitter = OpenCTIStix2Splitter()
|
|
1231
|
-
bundles =
|
|
1232
|
+
expectations_number, bundles = (
|
|
1233
|
+
stix2_splitter.split_bundle_with_expectations(
|
|
1234
|
+
bundle, True, event_version
|
|
1235
|
+
)
|
|
1236
|
+
)
|
|
1232
1237
|
|
|
1233
1238
|
if len(bundles) == 0:
|
|
1234
1239
|
self.metric.inc("error_count")
|
|
@@ -1236,7 +1241,7 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
1236
1241
|
|
|
1237
1242
|
if bundle_send_to_queue:
|
|
1238
1243
|
if work_id:
|
|
1239
|
-
self.api.work.add_expectations(work_id,
|
|
1244
|
+
self.api.work.add_expectations(work_id, expectations_number)
|
|
1240
1245
|
if entities_types is None:
|
|
1241
1246
|
entities_types = []
|
|
1242
1247
|
pika_credentials = pika.PlainCredentials(
|
pycti/utils/opencti_stix2.py
CHANGED
|
@@ -4,6 +4,9 @@ import base64
|
|
|
4
4
|
import datetime
|
|
5
5
|
import json
|
|
6
6
|
import os
|
|
7
|
+
import random
|
|
8
|
+
import time
|
|
9
|
+
import traceback
|
|
7
10
|
import uuid
|
|
8
11
|
from typing import Any, Dict, List, Optional, Union
|
|
9
12
|
|
|
@@ -11,6 +14,8 @@ import datefinder
|
|
|
11
14
|
import dateutil.parser
|
|
12
15
|
import pytz
|
|
13
16
|
from cachetools import LRUCache
|
|
17
|
+
from opentelemetry import metrics
|
|
18
|
+
from requests import RequestException, Timeout
|
|
14
19
|
|
|
15
20
|
from pycti.entities.opencti_identity import Identity
|
|
16
21
|
from pycti.utils.constants import (
|
|
@@ -32,11 +37,41 @@ utc = pytz.UTC
|
|
|
32
37
|
|
|
33
38
|
# Spec version
|
|
34
39
|
SPEC_VERSION = "2.1"
|
|
40
|
+
ERROR_TYPE_LOCK = "LOCK_ERROR"
|
|
41
|
+
ERROR_TYPE_MISSING_REFERENCE = "MISSING_REFERENCE_ERROR"
|
|
42
|
+
ERROR_TYPE_BAD_GATEWAY = "Bad Gateway"
|
|
35
43
|
|
|
36
44
|
# Extensions
|
|
37
45
|
STIX_EXT_OCTI = "extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba"
|
|
38
46
|
STIX_EXT_OCTI_SCO = "extension-definition--f93e2c80-4231-4f9a-af8b-95c9bd566a82"
|
|
39
47
|
STIX_EXT_MITRE = "extension-definition--322b8f77-262a-4cb8-a915-1e441e00329b"
|
|
48
|
+
PROCESSING_COUNT: int = 4
|
|
49
|
+
|
|
50
|
+
meter = metrics.get_meter(__name__)
|
|
51
|
+
bundles_timeout_error_counter = meter.create_counter(
|
|
52
|
+
name="opencti_bundles_timeout_error_counter",
|
|
53
|
+
description="number of bundles in timeout error",
|
|
54
|
+
)
|
|
55
|
+
bundles_lock_error_counter = meter.create_counter(
|
|
56
|
+
name="opencti_bundles_lock_error_counter",
|
|
57
|
+
description="number of bundles in lock error",
|
|
58
|
+
)
|
|
59
|
+
bundles_missing_reference_error_counter = meter.create_counter(
|
|
60
|
+
name="opencti_bundles_missing_reference_error_counter",
|
|
61
|
+
description="number of bundles in missing reference error",
|
|
62
|
+
)
|
|
63
|
+
bundles_bad_gateway_error_counter = meter.create_counter(
|
|
64
|
+
name="opencti_bundles_bad_gateway_error_counter",
|
|
65
|
+
description="number of bundles in bad gateway error",
|
|
66
|
+
)
|
|
67
|
+
bundles_technical_error_counter = meter.create_counter(
|
|
68
|
+
name="opencti_bundles_technical_error_counter",
|
|
69
|
+
description="number of bundles in technical error",
|
|
70
|
+
)
|
|
71
|
+
bundles_success_counter = meter.create_counter(
|
|
72
|
+
name="opencti_bundles_success_counter",
|
|
73
|
+
description="number of bundles successfully processed",
|
|
74
|
+
)
|
|
40
75
|
|
|
41
76
|
|
|
42
77
|
class OpenCTIStix2:
|
|
@@ -161,7 +196,7 @@ class OpenCTIStix2:
|
|
|
161
196
|
json_data: Union[str, bytes],
|
|
162
197
|
update: bool = False,
|
|
163
198
|
types: List = None,
|
|
164
|
-
|
|
199
|
+
work_id: str = None,
|
|
165
200
|
) -> List:
|
|
166
201
|
"""import a stix2 bundle from JSON data
|
|
167
202
|
|
|
@@ -171,16 +206,12 @@ class OpenCTIStix2:
|
|
|
171
206
|
:type update: bool, optional
|
|
172
207
|
:param types: list of stix2 types, defaults to None
|
|
173
208
|
:type types: list, optional
|
|
209
|
+
:param work_id work_id: str, optional
|
|
174
210
|
:return: list of imported stix2 objects
|
|
175
211
|
:rtype: List
|
|
176
212
|
"""
|
|
177
213
|
data = json.loads(json_data)
|
|
178
|
-
return self.import_bundle(
|
|
179
|
-
data,
|
|
180
|
-
update,
|
|
181
|
-
types,
|
|
182
|
-
retry_number,
|
|
183
|
-
)
|
|
214
|
+
return self.import_bundle(data, update, types, work_id)
|
|
184
215
|
|
|
185
216
|
def resolve_author(self, title: str) -> Optional[Identity]:
|
|
186
217
|
if "fireeye" in title.lower() or "mandiant" in title.lower():
|
|
@@ -1188,7 +1219,7 @@ class OpenCTIStix2:
|
|
|
1188
1219
|
|
|
1189
1220
|
# Create the relation
|
|
1190
1221
|
|
|
1191
|
-
|
|
1222
|
+
# Try to guess start_time / stop_time from external reference
|
|
1192
1223
|
date = None
|
|
1193
1224
|
if "external_references" in stix_relation:
|
|
1194
1225
|
for external_reference in stix_relation["external_references"]:
|
|
@@ -2362,12 +2393,235 @@ class OpenCTIStix2:
|
|
|
2362
2393
|
|
|
2363
2394
|
return bundle
|
|
2364
2395
|
|
|
2396
|
+
def import_item(
|
|
2397
|
+
self,
|
|
2398
|
+
item,
|
|
2399
|
+
update: bool = False,
|
|
2400
|
+
types: List = None,
|
|
2401
|
+
processing_count: int = 0,
|
|
2402
|
+
work_id: str = None,
|
|
2403
|
+
):
|
|
2404
|
+
worker_logger = self.opencti.logger_class("worker")
|
|
2405
|
+
try:
|
|
2406
|
+
self.opencti.set_retry_number(processing_count)
|
|
2407
|
+
if "opencti_operation" in item:
|
|
2408
|
+
if item["opencti_operation"] == "delete":
|
|
2409
|
+
delete_id = item["id"]
|
|
2410
|
+
self.opencti.stix.delete(id=delete_id)
|
|
2411
|
+
if item["opencti_operation"] == "merge":
|
|
2412
|
+
target_id = item["merge_target_id"]
|
|
2413
|
+
source_ids = item["merge_source_ids"]
|
|
2414
|
+
self.opencti.stix.merge(id=target_id, object_ids=source_ids)
|
|
2415
|
+
else:
|
|
2416
|
+
raise ValueError("Not supported opencti_operation")
|
|
2417
|
+
elif item["type"] == "relationship":
|
|
2418
|
+
# Import relationship
|
|
2419
|
+
self.import_relationship(item, update, types)
|
|
2420
|
+
elif item["type"] == "sighting":
|
|
2421
|
+
# Resolve the to
|
|
2422
|
+
to_ids = []
|
|
2423
|
+
if "where_sighted_refs" in item:
|
|
2424
|
+
for where_sighted_ref in item["where_sighted_refs"]:
|
|
2425
|
+
to_ids.append(where_sighted_ref)
|
|
2426
|
+
# Import sighting_of_ref
|
|
2427
|
+
if "x_opencti_sighting_of_ref" in item:
|
|
2428
|
+
from_id = item["x_opencti_sighting_of_ref"]
|
|
2429
|
+
if len(to_ids) > 0:
|
|
2430
|
+
for to_id in to_ids:
|
|
2431
|
+
self.import_sighting(item, from_id, to_id, update)
|
|
2432
|
+
if (
|
|
2433
|
+
self.opencti.get_attribute_in_extension("sighting_of_ref", item)
|
|
2434
|
+
is not None
|
|
2435
|
+
):
|
|
2436
|
+
from_id = self.opencti.get_attribute_in_extension(
|
|
2437
|
+
"sighting_of_ref", item
|
|
2438
|
+
)
|
|
2439
|
+
if len(to_ids) > 0:
|
|
2440
|
+
for to_id in to_ids:
|
|
2441
|
+
self.import_sighting(item, from_id, to_id, update)
|
|
2442
|
+
from_id = item["sighting_of_ref"]
|
|
2443
|
+
if len(to_ids) > 0:
|
|
2444
|
+
for to_id in to_ids:
|
|
2445
|
+
self.import_sighting(item, from_id, to_id, update)
|
|
2446
|
+
# Import observed_data_refs
|
|
2447
|
+
if "observed_data_refs" in item:
|
|
2448
|
+
for observed_data_ref in item["observed_data_refs"]:
|
|
2449
|
+
if len(to_ids) > 0:
|
|
2450
|
+
for to_id in to_ids:
|
|
2451
|
+
self.import_sighting(
|
|
2452
|
+
item, observed_data_ref, to_id, update
|
|
2453
|
+
)
|
|
2454
|
+
elif item["type"] == "label":
|
|
2455
|
+
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2456
|
+
self.opencti.label.create(
|
|
2457
|
+
stix_id=item["id"],
|
|
2458
|
+
value=item["value"],
|
|
2459
|
+
color=item["color"],
|
|
2460
|
+
x_opencti_stix_ids=stix_ids,
|
|
2461
|
+
update=update,
|
|
2462
|
+
)
|
|
2463
|
+
elif item["type"] == "vocabulary":
|
|
2464
|
+
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2465
|
+
self.opencti.vocabulary.create(
|
|
2466
|
+
stix_id=item["id"],
|
|
2467
|
+
name=item["name"],
|
|
2468
|
+
category=item["category"],
|
|
2469
|
+
description=(
|
|
2470
|
+
item["description"] if "description" in item else None
|
|
2471
|
+
),
|
|
2472
|
+
aliases=item["aliases"] if "aliases" in item else None,
|
|
2473
|
+
x_opencti_stix_ids=stix_ids,
|
|
2474
|
+
update=update,
|
|
2475
|
+
)
|
|
2476
|
+
elif item["type"] == "external-reference":
|
|
2477
|
+
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2478
|
+
self.opencti.external_reference.create(
|
|
2479
|
+
stix_id=item["id"],
|
|
2480
|
+
source_name=(
|
|
2481
|
+
item["source_name"] if "source_name" in item else None
|
|
2482
|
+
),
|
|
2483
|
+
url=item["url"] if "url" in item else None,
|
|
2484
|
+
external_id=(
|
|
2485
|
+
item["external_id"] if "external_id" in item else None
|
|
2486
|
+
),
|
|
2487
|
+
description=(
|
|
2488
|
+
item["description"] if "description" in item else None
|
|
2489
|
+
),
|
|
2490
|
+
x_opencti_stix_ids=stix_ids,
|
|
2491
|
+
update=update,
|
|
2492
|
+
)
|
|
2493
|
+
elif item["type"] == "kill-chain-phase":
|
|
2494
|
+
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2495
|
+
self.opencti.kill_chain_phase.create(
|
|
2496
|
+
stix_id=item["id"],
|
|
2497
|
+
kill_chain_name=item["kill_chain_name"],
|
|
2498
|
+
phase_name=item["phase_name"],
|
|
2499
|
+
x_opencti_order=item["order"] if "order" in item else 0,
|
|
2500
|
+
x_opencti_stix_ids=stix_ids,
|
|
2501
|
+
update=update,
|
|
2502
|
+
)
|
|
2503
|
+
elif StixCyberObservableTypes.has_value(item["type"]):
|
|
2504
|
+
if types is None or len(types) == 0:
|
|
2505
|
+
self.import_observable(item, update, types)
|
|
2506
|
+
elif item["type"] in types or "observable" in types:
|
|
2507
|
+
self.import_observable(item, update, types)
|
|
2508
|
+
else:
|
|
2509
|
+
# Check the scope
|
|
2510
|
+
if (
|
|
2511
|
+
item["type"] == "marking-definition"
|
|
2512
|
+
or types is None
|
|
2513
|
+
or len(types) == 0
|
|
2514
|
+
):
|
|
2515
|
+
self.import_object(item, update, types)
|
|
2516
|
+
# Handle identity & location if part of the scope
|
|
2517
|
+
elif item["type"] in types:
|
|
2518
|
+
self.import_object(item, update, types)
|
|
2519
|
+
else:
|
|
2520
|
+
# Specific OpenCTI scopes
|
|
2521
|
+
if item["type"] == "identity":
|
|
2522
|
+
if "identity_class" in item:
|
|
2523
|
+
if ("class" in types or "sector" in types) and item[
|
|
2524
|
+
"identity_class"
|
|
2525
|
+
] == "class":
|
|
2526
|
+
self.import_object(item, update, types)
|
|
2527
|
+
elif item["identity_class"] in types:
|
|
2528
|
+
self.import_object(item, update, types)
|
|
2529
|
+
elif item["type"] == "location":
|
|
2530
|
+
if "x_opencti_location_type" in item:
|
|
2531
|
+
if item["x_opencti_location_type"].lower() in types:
|
|
2532
|
+
self.import_object(item, update, types)
|
|
2533
|
+
elif (
|
|
2534
|
+
self.opencti.get_attribute_in_extension(
|
|
2535
|
+
"location_type", item
|
|
2536
|
+
)
|
|
2537
|
+
is not None
|
|
2538
|
+
):
|
|
2539
|
+
if (
|
|
2540
|
+
self.opencti.get_attribute_in_extension(
|
|
2541
|
+
"location_type", item
|
|
2542
|
+
).lower()
|
|
2543
|
+
in types
|
|
2544
|
+
):
|
|
2545
|
+
self.import_object(item, update, types)
|
|
2546
|
+
if work_id is not None:
|
|
2547
|
+
self.opencti.work.report_expectation(work_id, None)
|
|
2548
|
+
bundles_success_counter.add(1)
|
|
2549
|
+
return True
|
|
2550
|
+
except (RequestException, Timeout):
|
|
2551
|
+
bundles_timeout_error_counter.add(1)
|
|
2552
|
+
worker_logger.warning("A connection error or timeout occurred")
|
|
2553
|
+
# Platform is under heavy load: wait for unlock & retry almost indefinitely.
|
|
2554
|
+
sleep_jitter = round(random.uniform(10, 30), 2)
|
|
2555
|
+
time.sleep(sleep_jitter)
|
|
2556
|
+
return self.import_item(item, update, types, processing_count + 1, work_id)
|
|
2557
|
+
except Exception as ex: # pylint: disable=broad-except
|
|
2558
|
+
error = str(ex)
|
|
2559
|
+
error_msg = traceback.format_exc()
|
|
2560
|
+
in_retry = processing_count < PROCESSING_COUNT
|
|
2561
|
+
# Platform is under heavy load, wait for unlock & retry indefinitely.
|
|
2562
|
+
if ERROR_TYPE_LOCK in error_msg:
|
|
2563
|
+
worker_logger.info(
|
|
2564
|
+
"Message reprocess for lock rejection", {"count": processing_count}
|
|
2565
|
+
)
|
|
2566
|
+
bundles_lock_error_counter.add(1)
|
|
2567
|
+
sleep_jitter = round(random.uniform(1, 3), 2)
|
|
2568
|
+
time.sleep(sleep_jitter)
|
|
2569
|
+
return self.import_item(
|
|
2570
|
+
item, update, types, processing_count + 1, work_id
|
|
2571
|
+
)
|
|
2572
|
+
# Platform detects a missing reference and have to retry
|
|
2573
|
+
elif ERROR_TYPE_MISSING_REFERENCE in error_msg and in_retry:
|
|
2574
|
+
worker_logger.info(
|
|
2575
|
+
"Message reprocess for missing reference",
|
|
2576
|
+
{"count": processing_count},
|
|
2577
|
+
)
|
|
2578
|
+
bundles_missing_reference_error_counter.add(1)
|
|
2579
|
+
sleep_jitter = round(random.uniform(1, 3), 2)
|
|
2580
|
+
time.sleep(sleep_jitter)
|
|
2581
|
+
return self.import_item(
|
|
2582
|
+
item, update, types, processing_count + 1, work_id
|
|
2583
|
+
)
|
|
2584
|
+
# A bad gateway error occurs
|
|
2585
|
+
elif ERROR_TYPE_BAD_GATEWAY in error_msg:
|
|
2586
|
+
worker_logger.error("A connection error occurred")
|
|
2587
|
+
worker_logger.info(
|
|
2588
|
+
"Message reprocess for bad gateway",
|
|
2589
|
+
{"count": processing_count},
|
|
2590
|
+
)
|
|
2591
|
+
bundles_bad_gateway_error_counter.add(1)
|
|
2592
|
+
time.sleep(60)
|
|
2593
|
+
return self.import_item(
|
|
2594
|
+
item, update, types, processing_count + 1, work_id
|
|
2595
|
+
)
|
|
2596
|
+
# Platform does not know what to do and raises an error:
|
|
2597
|
+
# That also works for missing reference with too much execution
|
|
2598
|
+
else:
|
|
2599
|
+
bundles_technical_error_counter.add(1)
|
|
2600
|
+
worker_logger.error(
|
|
2601
|
+
"Error executing import",
|
|
2602
|
+
{"count": processing_count, "reason": error},
|
|
2603
|
+
)
|
|
2604
|
+
if work_id is not None:
|
|
2605
|
+
item_str = json.dumps(item)
|
|
2606
|
+
self.opencti.work.report_expectation(
|
|
2607
|
+
work_id,
|
|
2608
|
+
{
|
|
2609
|
+
"error": error,
|
|
2610
|
+
"source": (
|
|
2611
|
+
item_str
|
|
2612
|
+
if len(item_str) < 50000
|
|
2613
|
+
else "Bundle too large"
|
|
2614
|
+
),
|
|
2615
|
+
},
|
|
2616
|
+
)
|
|
2617
|
+
return False
|
|
2618
|
+
|
|
2365
2619
|
def import_bundle(
|
|
2366
2620
|
self,
|
|
2367
2621
|
stix_bundle: Dict,
|
|
2368
2622
|
update: bool = False,
|
|
2369
2623
|
types: List = None,
|
|
2370
|
-
|
|
2624
|
+
work_id: str = None,
|
|
2371
2625
|
) -> List:
|
|
2372
2626
|
# Check if the bundle is correctly formatted
|
|
2373
2627
|
if "type" not in stix_bundle or stix_bundle["type"] != "bundle":
|
|
@@ -2379,152 +2633,16 @@ class OpenCTIStix2:
|
|
|
2379
2633
|
if "x_opencti_event_version" in stix_bundle
|
|
2380
2634
|
else None
|
|
2381
2635
|
)
|
|
2382
|
-
if retry_number is not None:
|
|
2383
|
-
self.opencti.set_retry_number(retry_number)
|
|
2384
2636
|
stix2_splitter = OpenCTIStix2Splitter()
|
|
2385
2637
|
try:
|
|
2386
2638
|
bundles = stix2_splitter.split_bundle(stix_bundle, False, event_version)
|
|
2387
2639
|
except RecursionError:
|
|
2388
2640
|
bundles = [stix_bundle]
|
|
2389
|
-
# Import every
|
|
2641
|
+
# Import every element in a specific order
|
|
2390
2642
|
imported_elements = []
|
|
2391
|
-
|
|
2392
|
-
# Marking definitions
|
|
2393
2643
|
for bundle in bundles:
|
|
2394
2644
|
for item in bundle["objects"]:
|
|
2395
|
-
|
|
2396
|
-
if bundle["x_opencti_event_version"] == "3":
|
|
2397
|
-
if "x_opencti_patch" in item:
|
|
2398
|
-
self.stix2_update.process_update(item)
|
|
2399
|
-
continue
|
|
2400
|
-
if item["type"] == "relationship":
|
|
2401
|
-
self.import_relationship(item, update, types)
|
|
2402
|
-
elif item["type"] == "sighting":
|
|
2403
|
-
# Resolve the to
|
|
2404
|
-
to_ids = []
|
|
2405
|
-
if "where_sighted_refs" in item:
|
|
2406
|
-
for where_sighted_ref in item["where_sighted_refs"]:
|
|
2407
|
-
to_ids.append(where_sighted_ref)
|
|
2408
|
-
# Import sighting_of_ref
|
|
2409
|
-
if "x_opencti_sighting_of_ref" in item:
|
|
2410
|
-
from_id = item["x_opencti_sighting_of_ref"]
|
|
2411
|
-
if len(to_ids) > 0:
|
|
2412
|
-
for to_id in to_ids:
|
|
2413
|
-
self.import_sighting(item, from_id, to_id, update)
|
|
2414
|
-
if (
|
|
2415
|
-
self.opencti.get_attribute_in_extension("sighting_of_ref", item)
|
|
2416
|
-
is not None
|
|
2417
|
-
):
|
|
2418
|
-
from_id = self.opencti.get_attribute_in_extension(
|
|
2419
|
-
"sighting_of_ref", item
|
|
2420
|
-
)
|
|
2421
|
-
if len(to_ids) > 0:
|
|
2422
|
-
for to_id in to_ids:
|
|
2423
|
-
self.import_sighting(item, from_id, to_id, update)
|
|
2424
|
-
from_id = item["sighting_of_ref"]
|
|
2425
|
-
if len(to_ids) > 0:
|
|
2426
|
-
for to_id in to_ids:
|
|
2427
|
-
self.import_sighting(item, from_id, to_id, update)
|
|
2428
|
-
# Import observed_data_refs
|
|
2429
|
-
if "observed_data_refs" in item:
|
|
2430
|
-
for observed_data_ref in item["observed_data_refs"]:
|
|
2431
|
-
if len(to_ids) > 0:
|
|
2432
|
-
for to_id in to_ids:
|
|
2433
|
-
self.import_sighting(
|
|
2434
|
-
item, observed_data_ref, to_id, update
|
|
2435
|
-
)
|
|
2436
|
-
elif item["type"] == "label":
|
|
2437
|
-
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2438
|
-
self.opencti.label.create(
|
|
2439
|
-
stix_id=item["id"],
|
|
2440
|
-
value=item["value"],
|
|
2441
|
-
color=item["color"],
|
|
2442
|
-
x_opencti_stix_ids=stix_ids,
|
|
2443
|
-
update=update,
|
|
2444
|
-
)
|
|
2445
|
-
elif item["type"] == "vocabulary":
|
|
2446
|
-
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2447
|
-
self.opencti.vocabulary.create(
|
|
2448
|
-
stix_id=item["id"],
|
|
2449
|
-
name=item["name"],
|
|
2450
|
-
category=item["category"],
|
|
2451
|
-
description=(
|
|
2452
|
-
item["description"] if "description" in item else None
|
|
2453
|
-
),
|
|
2454
|
-
aliases=item["aliases"] if "aliases" in item else None,
|
|
2455
|
-
x_opencti_stix_ids=stix_ids,
|
|
2456
|
-
update=update,
|
|
2457
|
-
)
|
|
2458
|
-
elif item["type"] == "external-reference":
|
|
2459
|
-
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2460
|
-
self.opencti.external_reference.create(
|
|
2461
|
-
stix_id=item["id"],
|
|
2462
|
-
source_name=(
|
|
2463
|
-
item["source_name"] if "source_name" in item else None
|
|
2464
|
-
),
|
|
2465
|
-
url=item["url"] if "url" in item else None,
|
|
2466
|
-
external_id=(
|
|
2467
|
-
item["external_id"] if "external_id" in item else None
|
|
2468
|
-
),
|
|
2469
|
-
description=(
|
|
2470
|
-
item["description"] if "description" in item else None
|
|
2471
|
-
),
|
|
2472
|
-
x_opencti_stix_ids=stix_ids,
|
|
2473
|
-
update=update,
|
|
2474
|
-
)
|
|
2475
|
-
elif item["type"] == "kill-chain-phase":
|
|
2476
|
-
stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
|
|
2477
|
-
self.opencti.kill_chain_phase.create(
|
|
2478
|
-
stix_id=item["id"],
|
|
2479
|
-
kill_chain_name=item["kill_chain_name"],
|
|
2480
|
-
phase_name=item["phase_name"],
|
|
2481
|
-
x_opencti_order=item["order"] if "order" in item else 0,
|
|
2482
|
-
x_opencti_stix_ids=stix_ids,
|
|
2483
|
-
update=update,
|
|
2484
|
-
)
|
|
2485
|
-
elif StixCyberObservableTypes.has_value(item["type"]):
|
|
2486
|
-
if types is None or len(types) == 0:
|
|
2487
|
-
self.import_observable(item, update, types)
|
|
2488
|
-
elif item["type"] in types or "observable" in types:
|
|
2489
|
-
self.import_observable(item, update, types)
|
|
2490
|
-
else:
|
|
2491
|
-
# Check the scope
|
|
2492
|
-
if (
|
|
2493
|
-
item["type"] == "marking-definition"
|
|
2494
|
-
or types is None
|
|
2495
|
-
or len(types) == 0
|
|
2496
|
-
):
|
|
2497
|
-
self.import_object(item, update, types)
|
|
2498
|
-
# Handle identity & location if part of the scope
|
|
2499
|
-
elif item["type"] in types:
|
|
2500
|
-
self.import_object(item, update, types)
|
|
2501
|
-
else:
|
|
2502
|
-
# Specific OpenCTI scopes
|
|
2503
|
-
if item["type"] == "identity":
|
|
2504
|
-
if "identity_class" in item:
|
|
2505
|
-
if ("class" in types or "sector" in types) and item[
|
|
2506
|
-
"identity_class"
|
|
2507
|
-
] == "class":
|
|
2508
|
-
self.import_object(item, update, types)
|
|
2509
|
-
elif item["identity_class"] in types:
|
|
2510
|
-
self.import_object(item, update, types)
|
|
2511
|
-
elif item["type"] == "location":
|
|
2512
|
-
if "x_opencti_location_type" in item:
|
|
2513
|
-
if item["x_opencti_location_type"].lower() in types:
|
|
2514
|
-
self.import_object(item, update, types)
|
|
2515
|
-
elif (
|
|
2516
|
-
self.opencti.get_attribute_in_extension(
|
|
2517
|
-
"location_type", item
|
|
2518
|
-
)
|
|
2519
|
-
is not None
|
|
2520
|
-
):
|
|
2521
|
-
if (
|
|
2522
|
-
self.opencti.get_attribute_in_extension(
|
|
2523
|
-
"location_type", item
|
|
2524
|
-
).lower()
|
|
2525
|
-
in types
|
|
2526
|
-
):
|
|
2527
|
-
self.import_object(item, update, types)
|
|
2645
|
+
self.import_item(item, update, types, 0, work_id)
|
|
2528
2646
|
imported_elements.append({"id": item["id"], "type": item["type"]})
|
|
2529
2647
|
|
|
2530
2648
|
return imported_elements
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import re
|
|
3
3
|
import uuid
|
|
4
|
+
from typing import Tuple
|
|
5
|
+
|
|
6
|
+
from typing_extensions import deprecated
|
|
4
7
|
|
|
5
8
|
MITRE_X_CAPEC = (
|
|
6
9
|
"x_capec_*" # https://github.com/mitre-attack/attack-stix-data/issues/34
|
|
@@ -58,16 +61,10 @@ class OpenCTIStix2Splitter:
|
|
|
58
61
|
self.cache_index[item_id] = item # Put in cache
|
|
59
62
|
return nb_deps
|
|
60
63
|
|
|
61
|
-
def
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
:param use_json: is JSON?
|
|
66
|
-
:type use_json:
|
|
67
|
-
:raises Exception: if data is not valid JSON
|
|
68
|
-
:return: returns a list of bundles
|
|
69
|
-
:rtype: list
|
|
70
|
-
"""
|
|
64
|
+
def split_bundle_with_expectations(
|
|
65
|
+
self, bundle, use_json=True, event_version=None
|
|
66
|
+
) -> Tuple[int, list]:
|
|
67
|
+
"""splits a valid stix2 bundle into a list of bundles"""
|
|
71
68
|
if use_json:
|
|
72
69
|
try:
|
|
73
70
|
bundle_data = json.loads(bundle)
|
|
@@ -96,16 +93,31 @@ class OpenCTIStix2Splitter:
|
|
|
96
93
|
return elem["nb_deps"]
|
|
97
94
|
|
|
98
95
|
self.elements.sort(key=by_dep_size)
|
|
99
|
-
|
|
96
|
+
|
|
97
|
+
elements_with_deps = list(
|
|
98
|
+
map(lambda e: {"nb_deps": e["nb_deps"], "elements": [e]}, self.elements)
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
number_expectations = 0
|
|
102
|
+
for entity in elements_with_deps:
|
|
103
|
+
number_expectations += len(entity["elements"])
|
|
100
104
|
bundles.append(
|
|
101
105
|
self.stix2_create_bundle(
|
|
102
106
|
bundle_data["id"],
|
|
103
107
|
entity["nb_deps"],
|
|
104
|
-
[
|
|
108
|
+
entity["elements"],
|
|
105
109
|
use_json,
|
|
106
110
|
event_version,
|
|
107
111
|
)
|
|
108
112
|
)
|
|
113
|
+
|
|
114
|
+
return number_expectations, bundles
|
|
115
|
+
|
|
116
|
+
@deprecated("Use split_bundle_with_expectations instead")
|
|
117
|
+
def split_bundle(self, bundle, use_json=True, event_version=None) -> list:
|
|
118
|
+
expectations, bundles = self.split_bundle_with_expectations(
|
|
119
|
+
bundle, use_json, event_version
|
|
120
|
+
)
|
|
109
121
|
return bundles
|
|
110
122
|
|
|
111
123
|
@staticmethod
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pycti
|
|
3
|
-
Version: 6.1.
|
|
3
|
+
Version: 6.1.5
|
|
4
4
|
Summary: Python API client for OpenCTI.
|
|
5
5
|
Home-page: https://github.com/OpenCTI-Platform/client-python
|
|
6
6
|
Author: Filigran
|
|
@@ -22,15 +22,17 @@ Requires-Python: >=3.7
|
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
License-File: LICENSE
|
|
24
24
|
Requires-Dist: datefinder ~=0.7.3
|
|
25
|
-
Requires-Dist: pika ~=1.3.
|
|
26
|
-
Requires-Dist: prometheus-client ~=0.20.0
|
|
25
|
+
Requires-Dist: pika ~=1.3.0
|
|
27
26
|
Requires-Dist: python-json-logger ~=2.0.4
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist: requests ~=2.
|
|
30
|
-
Requires-Dist: setuptools ~=
|
|
27
|
+
Requires-Dist: PyYAML ~=6.0
|
|
28
|
+
Requires-Dist: requests ~=2.32.2
|
|
29
|
+
Requires-Dist: setuptools ~=70.0.0
|
|
30
|
+
Requires-Dist: cachetools ~=5.3.0
|
|
31
|
+
Requires-Dist: prometheus-client ~=0.20.0
|
|
32
|
+
Requires-Dist: opentelemetry-api ~=1.22.0
|
|
33
|
+
Requires-Dist: opentelemetry-sdk ~=1.22.0
|
|
31
34
|
Requires-Dist: filigran-sseclient ~=1.0.0
|
|
32
35
|
Requires-Dist: stix2 ~=3.0.1
|
|
33
|
-
Requires-Dist: cachetools ~=5.3.0
|
|
34
36
|
Requires-Dist: python-magic ~=0.4.27 ; sys_platform == "linux" or sys_platform == "darwin"
|
|
35
37
|
Requires-Dist: python-magic-bin ~=0.4.14 ; sys_platform == "win32"
|
|
36
38
|
Provides-Extra: dev
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
pycti/__init__.py,sha256=
|
|
1
|
+
pycti/__init__.py,sha256=b5FShCuxuJX-8qUHy2fW2-JhkP-G6wnMEjS7d683cNQ,5035
|
|
2
2
|
pycti/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
pycti/api/opencti_api_client.py,sha256=
|
|
3
|
+
pycti/api/opencti_api_client.py,sha256=oTPEdzbSTtN1NMMDFqv_DIOXuUwNz8cN7oXu3saSXyA,29701
|
|
4
4
|
pycti/api/opencti_api_connector.py,sha256=fYF0Jy9KIMFNt1RC_A1rpWomVJ-oj5HiSsBem4W0J5U,3549
|
|
5
5
|
pycti/api/opencti_api_playbook.py,sha256=OkqDawpnMYIHz5sD4djlJ_KgORkfvQ7YbJwttxE-ea8,1470
|
|
6
6
|
pycti/api/opencti_api_work.py,sha256=JLfl7oy6Cq9IrYW_kUrqwzN46FoVzyIn1JJQKyK0h_w,7615
|
|
7
7
|
pycti/connector/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
pycti/connector/opencti_connector.py,sha256=0vrZ8Y8ecbxegAP1YhpX6ybOZahYjjOkcId51D1oBi4,2449
|
|
9
|
-
pycti/connector/opencti_connector_helper.py,sha256=
|
|
9
|
+
pycti/connector/opencti_connector_helper.py,sha256=mOlakQ077mZorpR7McXNLq4UXxsM2_00ES87M-HfQY8,60496
|
|
10
10
|
pycti/connector/opencti_metric_handler.py,sha256=4jXHeJflomtHjuQ_YU0b36TG7o26vOWbY_jvU8Ezobs,3725
|
|
11
11
|
pycti/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
12
|
pycti/entities/opencti_attack_pattern.py,sha256=ycAR0cReJ1dd_edQPAL6qBrFvvTx10GJFMTezyK91cg,21471
|
|
@@ -57,12 +57,12 @@ pycti/entities/opencti_vulnerability.py,sha256=dzJ0fZB2XrkPwT-cANr6atzYOWXF5nk0a
|
|
|
57
57
|
pycti/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
58
58
|
pycti/utils/constants.py,sha256=Gj0fz94p0ApjCUCUqBQpFTfNslT021HS2w6M8azqXBY,10601
|
|
59
59
|
pycti/utils/opencti_logger.py,sha256=0dvB75V0SuPFGxL539dAQrxTt1N5Acx0A3Ogwl5WMJ8,2199
|
|
60
|
-
pycti/utils/opencti_stix2.py,sha256=
|
|
61
|
-
pycti/utils/opencti_stix2_splitter.py,sha256=
|
|
60
|
+
pycti/utils/opencti_stix2.py,sha256=Ovv8mS7Df0FTNZwO2Vful8XNWY5PwNuyrrLC3bhiOqI,117232
|
|
61
|
+
pycti/utils/opencti_stix2_splitter.py,sha256=A2GqoiFzEga8hslgA3mm4FDoObFsWgx4zK4DdcWTguc,4907
|
|
62
62
|
pycti/utils/opencti_stix2_update.py,sha256=CnMyqkeVA0jgyxEcgqna8sABU4YPMjkEJ228GVurIn4,14658
|
|
63
63
|
pycti/utils/opencti_stix2_utils.py,sha256=4r9qglN3AIN8JH1B9Ts2o20Qn3K203M4c5-lIPzRpZ4,4138
|
|
64
|
-
pycti-6.1.
|
|
65
|
-
pycti-6.1.
|
|
66
|
-
pycti-6.1.
|
|
67
|
-
pycti-6.1.
|
|
68
|
-
pycti-6.1.
|
|
64
|
+
pycti-6.1.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
65
|
+
pycti-6.1.5.dist-info/METADATA,sha256=1_z_bdOGl8Ro1xuyV5Dw9nBP335Y1QaCGQaSBX3n0tI,5397
|
|
66
|
+
pycti-6.1.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
67
|
+
pycti-6.1.5.dist-info/top_level.txt,sha256=cqEpxitAhHP4VgSA6xmrak6Yk9MeBkwoMTB6k7d2ZnE,6
|
|
68
|
+
pycti-6.1.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|