pycti 6.1.4__tar.gz → 6.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pycti might be problematic. Click here for more details.

Files changed (73) hide show
  1. {pycti-6.1.4 → pycti-6.1.5}/PKG-INFO +9 -7
  2. {pycti-6.1.4 → pycti-6.1.5}/pycti/__init__.py +1 -1
  3. {pycti-6.1.4 → pycti-6.1.5}/pycti/api/opencti_api_client.py +10 -8
  4. {pycti-6.1.4 → pycti-6.1.5}/pycti/connector/opencti_connector_helper.py +7 -2
  5. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/opencti_stix2.py +265 -147
  6. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/opencti_stix2_splitter.py +24 -12
  7. {pycti-6.1.4 → pycti-6.1.5}/pycti.egg-info/PKG-INFO +9 -7
  8. {pycti-6.1.4 → pycti-6.1.5}/pycti.egg-info/requires.txt +8 -6
  9. {pycti-6.1.4 → pycti-6.1.5}/setup.cfg +10 -8
  10. {pycti-6.1.4 → pycti-6.1.5}/LICENSE +0 -0
  11. {pycti-6.1.4 → pycti-6.1.5}/README.md +0 -0
  12. {pycti-6.1.4 → pycti-6.1.5}/pycti/api/__init__.py +0 -0
  13. {pycti-6.1.4 → pycti-6.1.5}/pycti/api/opencti_api_connector.py +0 -0
  14. {pycti-6.1.4 → pycti-6.1.5}/pycti/api/opencti_api_playbook.py +0 -0
  15. {pycti-6.1.4 → pycti-6.1.5}/pycti/api/opencti_api_work.py +0 -0
  16. {pycti-6.1.4 → pycti-6.1.5}/pycti/connector/__init__.py +0 -0
  17. {pycti-6.1.4 → pycti-6.1.5}/pycti/connector/opencti_connector.py +0 -0
  18. {pycti-6.1.4 → pycti-6.1.5}/pycti/connector/opencti_metric_handler.py +0 -0
  19. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/__init__.py +0 -0
  20. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_attack_pattern.py +0 -0
  21. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_campaign.py +0 -0
  22. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_case_incident.py +0 -0
  23. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_case_rfi.py +0 -0
  24. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_case_rft.py +0 -0
  25. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_channel.py +0 -0
  26. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_course_of_action.py +0 -0
  27. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_data_component.py +0 -0
  28. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_data_source.py +0 -0
  29. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_event.py +0 -0
  30. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_external_reference.py +0 -0
  31. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_feedback.py +0 -0
  32. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_grouping.py +0 -0
  33. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_identity.py +0 -0
  34. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_incident.py +0 -0
  35. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_indicator.py +0 -0
  36. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_infrastructure.py +0 -0
  37. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_intrusion_set.py +0 -0
  38. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_kill_chain_phase.py +0 -0
  39. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_label.py +0 -0
  40. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_language.py +0 -0
  41. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_location.py +0 -0
  42. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_malware.py +0 -0
  43. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_malware_analysis.py +0 -0
  44. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_marking_definition.py +0 -0
  45. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_narrative.py +0 -0
  46. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_note.py +0 -0
  47. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_observed_data.py +0 -0
  48. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_opinion.py +0 -0
  49. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_report.py +0 -0
  50. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix.py +0 -0
  51. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_core_object.py +0 -0
  52. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_core_relationship.py +0 -0
  53. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_cyber_observable.py +0 -0
  54. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_domain_object.py +0 -0
  55. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_nested_ref_relationship.py +0 -0
  56. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_object_or_stix_relationship.py +0 -0
  57. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_stix_sighting_relationship.py +0 -0
  58. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_task.py +0 -0
  59. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_threat_actor.py +0 -0
  60. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_threat_actor_group.py +0 -0
  61. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_threat_actor_individual.py +0 -0
  62. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_tool.py +0 -0
  63. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_vocabulary.py +0 -0
  64. {pycti-6.1.4 → pycti-6.1.5}/pycti/entities/opencti_vulnerability.py +0 -0
  65. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/__init__.py +0 -0
  66. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/constants.py +0 -0
  67. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/opencti_logger.py +0 -0
  68. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/opencti_stix2_update.py +0 -0
  69. {pycti-6.1.4 → pycti-6.1.5}/pycti/utils/opencti_stix2_utils.py +0 -0
  70. {pycti-6.1.4 → pycti-6.1.5}/pycti.egg-info/SOURCES.txt +0 -0
  71. {pycti-6.1.4 → pycti-6.1.5}/pycti.egg-info/dependency_links.txt +0 -0
  72. {pycti-6.1.4 → pycti-6.1.5}/pycti.egg-info/top_level.txt +0 -0
  73. {pycti-6.1.4 → pycti-6.1.5}/pyproject.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pycti
3
- Version: 6.1.4
3
+ Version: 6.1.5
4
4
  Summary: Python API client for OpenCTI.
5
5
  Home-page: https://github.com/OpenCTI-Platform/client-python
6
6
  Author: Filigran
@@ -22,17 +22,19 @@ Requires-Python: >=3.7
22
22
  Description-Content-Type: text/markdown
23
23
  License-File: LICENSE
24
24
  Requires-Dist: datefinder~=0.7.3
25
- Requires-Dist: pika~=1.3.1
25
+ Requires-Dist: pika~=1.3.0
26
26
  Requires-Dist: python-magic~=0.4.27; sys_platform == "linux" or sys_platform == "darwin"
27
- Requires-Dist: prometheus-client~=0.20.0
28
27
  Requires-Dist: python-magic-bin~=0.4.14; sys_platform == "win32"
29
28
  Requires-Dist: python_json_logger~=2.0.4
30
- Requires-Dist: pyyaml~=6.0
31
- Requires-Dist: requests~=2.31.0
32
- Requires-Dist: setuptools~=69.5.1
29
+ Requires-Dist: PyYAML~=6.0
30
+ Requires-Dist: requests~=2.32.2
31
+ Requires-Dist: setuptools~=70.0.0
32
+ Requires-Dist: cachetools~=5.3.0
33
+ Requires-Dist: prometheus-client~=0.20.0
34
+ Requires-Dist: opentelemetry-api~=1.22.0
35
+ Requires-Dist: opentelemetry-sdk~=1.22.0
33
36
  Requires-Dist: filigran-sseclient~=1.0.0
34
37
  Requires-Dist: stix2~=3.0.1
35
- Requires-Dist: cachetools~=5.3.0
36
38
  Provides-Extra: dev
37
39
  Requires-Dist: black~=24.4.0; extra == "dev"
38
40
  Requires-Dist: build~=1.2.1; extra == "dev"
@@ -1,5 +1,5 @@
1
1
  # -*- coding: utf-8 -*-
2
- __version__ = "6.1.4"
2
+ __version__ = "6.1.5"
3
3
 
4
4
  from .api.opencti_api_client import OpenCTIApiClient
5
5
  from .api.opencti_api_connector import OpenCTIApiConnector
@@ -351,14 +351,16 @@ class OpenCTIApiClient:
351
351
  if "name" in main_error
352
352
  else main_error["message"]
353
353
  )
354
- if "data" in main_error and "reason" in main_error["data"]:
355
- raise ValueError(
356
- {"name": error_name, "message": main_error["data"]["reason"]}
357
- )
358
- else:
359
- raise ValueError(
360
- {"name": error_name, "message": main_error["message"]}
361
- )
354
+ error_detail = {
355
+ "name": error_name,
356
+ "error_message": main_error["message"],
357
+ }
358
+ meta_data = main_error["data"] if "data" in main_error else {}
359
+ # Prevent logging of input as bundle is logged differently
360
+ if meta_data.get("input") is not None:
361
+ del meta_data["input"]
362
+ value_error = {**error_detail, **meta_data}
363
+ raise ValueError(value_error)
362
364
  else:
363
365
  return result
364
366
  else:
@@ -1226,9 +1226,14 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
1226
1226
 
1227
1227
  if bypass_split:
1228
1228
  bundles = [bundle]
1229
+ expectations_number = len(json.loads(bundle)["objects"])
1229
1230
  else:
1230
1231
  stix2_splitter = OpenCTIStix2Splitter()
1231
- bundles = stix2_splitter.split_bundle(bundle, True, event_version)
1232
+ expectations_number, bundles = (
1233
+ stix2_splitter.split_bundle_with_expectations(
1234
+ bundle, True, event_version
1235
+ )
1236
+ )
1232
1237
 
1233
1238
  if len(bundles) == 0:
1234
1239
  self.metric.inc("error_count")
@@ -1236,7 +1241,7 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
1236
1241
 
1237
1242
  if bundle_send_to_queue:
1238
1243
  if work_id:
1239
- self.api.work.add_expectations(work_id, len(bundles))
1244
+ self.api.work.add_expectations(work_id, expectations_number)
1240
1245
  if entities_types is None:
1241
1246
  entities_types = []
1242
1247
  pika_credentials = pika.PlainCredentials(
@@ -4,6 +4,9 @@ import base64
4
4
  import datetime
5
5
  import json
6
6
  import os
7
+ import random
8
+ import time
9
+ import traceback
7
10
  import uuid
8
11
  from typing import Any, Dict, List, Optional, Union
9
12
 
@@ -11,6 +14,8 @@ import datefinder
11
14
  import dateutil.parser
12
15
  import pytz
13
16
  from cachetools import LRUCache
17
+ from opentelemetry import metrics
18
+ from requests import RequestException, Timeout
14
19
 
15
20
  from pycti.entities.opencti_identity import Identity
16
21
  from pycti.utils.constants import (
@@ -32,11 +37,41 @@ utc = pytz.UTC
32
37
 
33
38
  # Spec version
34
39
  SPEC_VERSION = "2.1"
40
+ ERROR_TYPE_LOCK = "LOCK_ERROR"
41
+ ERROR_TYPE_MISSING_REFERENCE = "MISSING_REFERENCE_ERROR"
42
+ ERROR_TYPE_BAD_GATEWAY = "Bad Gateway"
35
43
 
36
44
  # Extensions
37
45
  STIX_EXT_OCTI = "extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba"
38
46
  STIX_EXT_OCTI_SCO = "extension-definition--f93e2c80-4231-4f9a-af8b-95c9bd566a82"
39
47
  STIX_EXT_MITRE = "extension-definition--322b8f77-262a-4cb8-a915-1e441e00329b"
48
+ PROCESSING_COUNT: int = 4
49
+
50
+ meter = metrics.get_meter(__name__)
51
+ bundles_timeout_error_counter = meter.create_counter(
52
+ name="opencti_bundles_timeout_error_counter",
53
+ description="number of bundles in timeout error",
54
+ )
55
+ bundles_lock_error_counter = meter.create_counter(
56
+ name="opencti_bundles_lock_error_counter",
57
+ description="number of bundles in lock error",
58
+ )
59
+ bundles_missing_reference_error_counter = meter.create_counter(
60
+ name="opencti_bundles_missing_reference_error_counter",
61
+ description="number of bundles in missing reference error",
62
+ )
63
+ bundles_bad_gateway_error_counter = meter.create_counter(
64
+ name="opencti_bundles_bad_gateway_error_counter",
65
+ description="number of bundles in bad gateway error",
66
+ )
67
+ bundles_technical_error_counter = meter.create_counter(
68
+ name="opencti_bundles_technical_error_counter",
69
+ description="number of bundles in technical error",
70
+ )
71
+ bundles_success_counter = meter.create_counter(
72
+ name="opencti_bundles_success_counter",
73
+ description="number of bundles successfully processed",
74
+ )
40
75
 
41
76
 
42
77
  class OpenCTIStix2:
@@ -161,7 +196,7 @@ class OpenCTIStix2:
161
196
  json_data: Union[str, bytes],
162
197
  update: bool = False,
163
198
  types: List = None,
164
- retry_number: int = None,
199
+ work_id: str = None,
165
200
  ) -> List:
166
201
  """import a stix2 bundle from JSON data
167
202
 
@@ -171,16 +206,12 @@ class OpenCTIStix2:
171
206
  :type update: bool, optional
172
207
  :param types: list of stix2 types, defaults to None
173
208
  :type types: list, optional
209
+ :param work_id work_id: str, optional
174
210
  :return: list of imported stix2 objects
175
211
  :rtype: List
176
212
  """
177
213
  data = json.loads(json_data)
178
- return self.import_bundle(
179
- data,
180
- update,
181
- types,
182
- retry_number,
183
- )
214
+ return self.import_bundle(data, update, types, work_id)
184
215
 
185
216
  def resolve_author(self, title: str) -> Optional[Identity]:
186
217
  if "fireeye" in title.lower() or "mandiant" in title.lower():
@@ -1188,7 +1219,7 @@ class OpenCTIStix2:
1188
1219
 
1189
1220
  # Create the relation
1190
1221
 
1191
- ## Try to guess start_time / stop_time from external reference
1222
+ # Try to guess start_time / stop_time from external reference
1192
1223
  date = None
1193
1224
  if "external_references" in stix_relation:
1194
1225
  for external_reference in stix_relation["external_references"]:
@@ -2362,12 +2393,235 @@ class OpenCTIStix2:
2362
2393
 
2363
2394
  return bundle
2364
2395
 
2396
+ def import_item(
2397
+ self,
2398
+ item,
2399
+ update: bool = False,
2400
+ types: List = None,
2401
+ processing_count: int = 0,
2402
+ work_id: str = None,
2403
+ ):
2404
+ worker_logger = self.opencti.logger_class("worker")
2405
+ try:
2406
+ self.opencti.set_retry_number(processing_count)
2407
+ if "opencti_operation" in item:
2408
+ if item["opencti_operation"] == "delete":
2409
+ delete_id = item["id"]
2410
+ self.opencti.stix.delete(id=delete_id)
2411
+ if item["opencti_operation"] == "merge":
2412
+ target_id = item["merge_target_id"]
2413
+ source_ids = item["merge_source_ids"]
2414
+ self.opencti.stix.merge(id=target_id, object_ids=source_ids)
2415
+ else:
2416
+ raise ValueError("Not supported opencti_operation")
2417
+ elif item["type"] == "relationship":
2418
+ # Import relationship
2419
+ self.import_relationship(item, update, types)
2420
+ elif item["type"] == "sighting":
2421
+ # Resolve the to
2422
+ to_ids = []
2423
+ if "where_sighted_refs" in item:
2424
+ for where_sighted_ref in item["where_sighted_refs"]:
2425
+ to_ids.append(where_sighted_ref)
2426
+ # Import sighting_of_ref
2427
+ if "x_opencti_sighting_of_ref" in item:
2428
+ from_id = item["x_opencti_sighting_of_ref"]
2429
+ if len(to_ids) > 0:
2430
+ for to_id in to_ids:
2431
+ self.import_sighting(item, from_id, to_id, update)
2432
+ if (
2433
+ self.opencti.get_attribute_in_extension("sighting_of_ref", item)
2434
+ is not None
2435
+ ):
2436
+ from_id = self.opencti.get_attribute_in_extension(
2437
+ "sighting_of_ref", item
2438
+ )
2439
+ if len(to_ids) > 0:
2440
+ for to_id in to_ids:
2441
+ self.import_sighting(item, from_id, to_id, update)
2442
+ from_id = item["sighting_of_ref"]
2443
+ if len(to_ids) > 0:
2444
+ for to_id in to_ids:
2445
+ self.import_sighting(item, from_id, to_id, update)
2446
+ # Import observed_data_refs
2447
+ if "observed_data_refs" in item:
2448
+ for observed_data_ref in item["observed_data_refs"]:
2449
+ if len(to_ids) > 0:
2450
+ for to_id in to_ids:
2451
+ self.import_sighting(
2452
+ item, observed_data_ref, to_id, update
2453
+ )
2454
+ elif item["type"] == "label":
2455
+ stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2456
+ self.opencti.label.create(
2457
+ stix_id=item["id"],
2458
+ value=item["value"],
2459
+ color=item["color"],
2460
+ x_opencti_stix_ids=stix_ids,
2461
+ update=update,
2462
+ )
2463
+ elif item["type"] == "vocabulary":
2464
+ stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2465
+ self.opencti.vocabulary.create(
2466
+ stix_id=item["id"],
2467
+ name=item["name"],
2468
+ category=item["category"],
2469
+ description=(
2470
+ item["description"] if "description" in item else None
2471
+ ),
2472
+ aliases=item["aliases"] if "aliases" in item else None,
2473
+ x_opencti_stix_ids=stix_ids,
2474
+ update=update,
2475
+ )
2476
+ elif item["type"] == "external-reference":
2477
+ stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2478
+ self.opencti.external_reference.create(
2479
+ stix_id=item["id"],
2480
+ source_name=(
2481
+ item["source_name"] if "source_name" in item else None
2482
+ ),
2483
+ url=item["url"] if "url" in item else None,
2484
+ external_id=(
2485
+ item["external_id"] if "external_id" in item else None
2486
+ ),
2487
+ description=(
2488
+ item["description"] if "description" in item else None
2489
+ ),
2490
+ x_opencti_stix_ids=stix_ids,
2491
+ update=update,
2492
+ )
2493
+ elif item["type"] == "kill-chain-phase":
2494
+ stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2495
+ self.opencti.kill_chain_phase.create(
2496
+ stix_id=item["id"],
2497
+ kill_chain_name=item["kill_chain_name"],
2498
+ phase_name=item["phase_name"],
2499
+ x_opencti_order=item["order"] if "order" in item else 0,
2500
+ x_opencti_stix_ids=stix_ids,
2501
+ update=update,
2502
+ )
2503
+ elif StixCyberObservableTypes.has_value(item["type"]):
2504
+ if types is None or len(types) == 0:
2505
+ self.import_observable(item, update, types)
2506
+ elif item["type"] in types or "observable" in types:
2507
+ self.import_observable(item, update, types)
2508
+ else:
2509
+ # Check the scope
2510
+ if (
2511
+ item["type"] == "marking-definition"
2512
+ or types is None
2513
+ or len(types) == 0
2514
+ ):
2515
+ self.import_object(item, update, types)
2516
+ # Handle identity & location if part of the scope
2517
+ elif item["type"] in types:
2518
+ self.import_object(item, update, types)
2519
+ else:
2520
+ # Specific OpenCTI scopes
2521
+ if item["type"] == "identity":
2522
+ if "identity_class" in item:
2523
+ if ("class" in types or "sector" in types) and item[
2524
+ "identity_class"
2525
+ ] == "class":
2526
+ self.import_object(item, update, types)
2527
+ elif item["identity_class"] in types:
2528
+ self.import_object(item, update, types)
2529
+ elif item["type"] == "location":
2530
+ if "x_opencti_location_type" in item:
2531
+ if item["x_opencti_location_type"].lower() in types:
2532
+ self.import_object(item, update, types)
2533
+ elif (
2534
+ self.opencti.get_attribute_in_extension(
2535
+ "location_type", item
2536
+ )
2537
+ is not None
2538
+ ):
2539
+ if (
2540
+ self.opencti.get_attribute_in_extension(
2541
+ "location_type", item
2542
+ ).lower()
2543
+ in types
2544
+ ):
2545
+ self.import_object(item, update, types)
2546
+ if work_id is not None:
2547
+ self.opencti.work.report_expectation(work_id, None)
2548
+ bundles_success_counter.add(1)
2549
+ return True
2550
+ except (RequestException, Timeout):
2551
+ bundles_timeout_error_counter.add(1)
2552
+ worker_logger.warning("A connection error or timeout occurred")
2553
+ # Platform is under heavy load: wait for unlock & retry almost indefinitely.
2554
+ sleep_jitter = round(random.uniform(10, 30), 2)
2555
+ time.sleep(sleep_jitter)
2556
+ return self.import_item(item, update, types, processing_count + 1, work_id)
2557
+ except Exception as ex: # pylint: disable=broad-except
2558
+ error = str(ex)
2559
+ error_msg = traceback.format_exc()
2560
+ in_retry = processing_count < PROCESSING_COUNT
2561
+ # Platform is under heavy load, wait for unlock & retry indefinitely.
2562
+ if ERROR_TYPE_LOCK in error_msg:
2563
+ worker_logger.info(
2564
+ "Message reprocess for lock rejection", {"count": processing_count}
2565
+ )
2566
+ bundles_lock_error_counter.add(1)
2567
+ sleep_jitter = round(random.uniform(1, 3), 2)
2568
+ time.sleep(sleep_jitter)
2569
+ return self.import_item(
2570
+ item, update, types, processing_count + 1, work_id
2571
+ )
2572
+ # Platform detects a missing reference and have to retry
2573
+ elif ERROR_TYPE_MISSING_REFERENCE in error_msg and in_retry:
2574
+ worker_logger.info(
2575
+ "Message reprocess for missing reference",
2576
+ {"count": processing_count},
2577
+ )
2578
+ bundles_missing_reference_error_counter.add(1)
2579
+ sleep_jitter = round(random.uniform(1, 3), 2)
2580
+ time.sleep(sleep_jitter)
2581
+ return self.import_item(
2582
+ item, update, types, processing_count + 1, work_id
2583
+ )
2584
+ # A bad gateway error occurs
2585
+ elif ERROR_TYPE_BAD_GATEWAY in error_msg:
2586
+ worker_logger.error("A connection error occurred")
2587
+ worker_logger.info(
2588
+ "Message reprocess for bad gateway",
2589
+ {"count": processing_count},
2590
+ )
2591
+ bundles_bad_gateway_error_counter.add(1)
2592
+ time.sleep(60)
2593
+ return self.import_item(
2594
+ item, update, types, processing_count + 1, work_id
2595
+ )
2596
+ # Platform does not know what to do and raises an error:
2597
+ # That also works for missing reference with too much execution
2598
+ else:
2599
+ bundles_technical_error_counter.add(1)
2600
+ worker_logger.error(
2601
+ "Error executing import",
2602
+ {"count": processing_count, "reason": error},
2603
+ )
2604
+ if work_id is not None:
2605
+ item_str = json.dumps(item)
2606
+ self.opencti.work.report_expectation(
2607
+ work_id,
2608
+ {
2609
+ "error": error,
2610
+ "source": (
2611
+ item_str
2612
+ if len(item_str) < 50000
2613
+ else "Bundle too large"
2614
+ ),
2615
+ },
2616
+ )
2617
+ return False
2618
+
2365
2619
  def import_bundle(
2366
2620
  self,
2367
2621
  stix_bundle: Dict,
2368
2622
  update: bool = False,
2369
2623
  types: List = None,
2370
- retry_number: int = None,
2624
+ work_id: str = None,
2371
2625
  ) -> List:
2372
2626
  # Check if the bundle is correctly formatted
2373
2627
  if "type" not in stix_bundle or stix_bundle["type"] != "bundle":
@@ -2379,152 +2633,16 @@ class OpenCTIStix2:
2379
2633
  if "x_opencti_event_version" in stix_bundle
2380
2634
  else None
2381
2635
  )
2382
- if retry_number is not None:
2383
- self.opencti.set_retry_number(retry_number)
2384
2636
  stix2_splitter = OpenCTIStix2Splitter()
2385
2637
  try:
2386
2638
  bundles = stix2_splitter.split_bundle(stix_bundle, False, event_version)
2387
2639
  except RecursionError:
2388
2640
  bundles = [stix_bundle]
2389
- # Import every elements in a specific order
2641
+ # Import every element in a specific order
2390
2642
  imported_elements = []
2391
-
2392
- # Marking definitions
2393
2643
  for bundle in bundles:
2394
2644
  for item in bundle["objects"]:
2395
- if "x_opencti_event_version" in bundle:
2396
- if bundle["x_opencti_event_version"] == "3":
2397
- if "x_opencti_patch" in item:
2398
- self.stix2_update.process_update(item)
2399
- continue
2400
- if item["type"] == "relationship":
2401
- self.import_relationship(item, update, types)
2402
- elif item["type"] == "sighting":
2403
- # Resolve the to
2404
- to_ids = []
2405
- if "where_sighted_refs" in item:
2406
- for where_sighted_ref in item["where_sighted_refs"]:
2407
- to_ids.append(where_sighted_ref)
2408
- # Import sighting_of_ref
2409
- if "x_opencti_sighting_of_ref" in item:
2410
- from_id = item["x_opencti_sighting_of_ref"]
2411
- if len(to_ids) > 0:
2412
- for to_id in to_ids:
2413
- self.import_sighting(item, from_id, to_id, update)
2414
- if (
2415
- self.opencti.get_attribute_in_extension("sighting_of_ref", item)
2416
- is not None
2417
- ):
2418
- from_id = self.opencti.get_attribute_in_extension(
2419
- "sighting_of_ref", item
2420
- )
2421
- if len(to_ids) > 0:
2422
- for to_id in to_ids:
2423
- self.import_sighting(item, from_id, to_id, update)
2424
- from_id = item["sighting_of_ref"]
2425
- if len(to_ids) > 0:
2426
- for to_id in to_ids:
2427
- self.import_sighting(item, from_id, to_id, update)
2428
- # Import observed_data_refs
2429
- if "observed_data_refs" in item:
2430
- for observed_data_ref in item["observed_data_refs"]:
2431
- if len(to_ids) > 0:
2432
- for to_id in to_ids:
2433
- self.import_sighting(
2434
- item, observed_data_ref, to_id, update
2435
- )
2436
- elif item["type"] == "label":
2437
- stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2438
- self.opencti.label.create(
2439
- stix_id=item["id"],
2440
- value=item["value"],
2441
- color=item["color"],
2442
- x_opencti_stix_ids=stix_ids,
2443
- update=update,
2444
- )
2445
- elif item["type"] == "vocabulary":
2446
- stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2447
- self.opencti.vocabulary.create(
2448
- stix_id=item["id"],
2449
- name=item["name"],
2450
- category=item["category"],
2451
- description=(
2452
- item["description"] if "description" in item else None
2453
- ),
2454
- aliases=item["aliases"] if "aliases" in item else None,
2455
- x_opencti_stix_ids=stix_ids,
2456
- update=update,
2457
- )
2458
- elif item["type"] == "external-reference":
2459
- stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2460
- self.opencti.external_reference.create(
2461
- stix_id=item["id"],
2462
- source_name=(
2463
- item["source_name"] if "source_name" in item else None
2464
- ),
2465
- url=item["url"] if "url" in item else None,
2466
- external_id=(
2467
- item["external_id"] if "external_id" in item else None
2468
- ),
2469
- description=(
2470
- item["description"] if "description" in item else None
2471
- ),
2472
- x_opencti_stix_ids=stix_ids,
2473
- update=update,
2474
- )
2475
- elif item["type"] == "kill-chain-phase":
2476
- stix_ids = self.opencti.get_attribute_in_extension("stix_ids", item)
2477
- self.opencti.kill_chain_phase.create(
2478
- stix_id=item["id"],
2479
- kill_chain_name=item["kill_chain_name"],
2480
- phase_name=item["phase_name"],
2481
- x_opencti_order=item["order"] if "order" in item else 0,
2482
- x_opencti_stix_ids=stix_ids,
2483
- update=update,
2484
- )
2485
- elif StixCyberObservableTypes.has_value(item["type"]):
2486
- if types is None or len(types) == 0:
2487
- self.import_observable(item, update, types)
2488
- elif item["type"] in types or "observable" in types:
2489
- self.import_observable(item, update, types)
2490
- else:
2491
- # Check the scope
2492
- if (
2493
- item["type"] == "marking-definition"
2494
- or types is None
2495
- or len(types) == 0
2496
- ):
2497
- self.import_object(item, update, types)
2498
- # Handle identity & location if part of the scope
2499
- elif item["type"] in types:
2500
- self.import_object(item, update, types)
2501
- else:
2502
- # Specific OpenCTI scopes
2503
- if item["type"] == "identity":
2504
- if "identity_class" in item:
2505
- if ("class" in types or "sector" in types) and item[
2506
- "identity_class"
2507
- ] == "class":
2508
- self.import_object(item, update, types)
2509
- elif item["identity_class"] in types:
2510
- self.import_object(item, update, types)
2511
- elif item["type"] == "location":
2512
- if "x_opencti_location_type" in item:
2513
- if item["x_opencti_location_type"].lower() in types:
2514
- self.import_object(item, update, types)
2515
- elif (
2516
- self.opencti.get_attribute_in_extension(
2517
- "location_type", item
2518
- )
2519
- is not None
2520
- ):
2521
- if (
2522
- self.opencti.get_attribute_in_extension(
2523
- "location_type", item
2524
- ).lower()
2525
- in types
2526
- ):
2527
- self.import_object(item, update, types)
2645
+ self.import_item(item, update, types, 0, work_id)
2528
2646
  imported_elements.append({"id": item["id"], "type": item["type"]})
2529
2647
 
2530
2648
  return imported_elements
@@ -1,6 +1,9 @@
1
1
  import json
2
2
  import re
3
3
  import uuid
4
+ from typing import Tuple
5
+
6
+ from typing_extensions import deprecated
4
7
 
5
8
  MITRE_X_CAPEC = (
6
9
  "x_capec_*" # https://github.com/mitre-attack/attack-stix-data/issues/34
@@ -58,16 +61,10 @@ class OpenCTIStix2Splitter:
58
61
  self.cache_index[item_id] = item # Put in cache
59
62
  return nb_deps
60
63
 
61
- def split_bundle(self, bundle, use_json=True, event_version=None) -> list:
62
- """splits a valid stix2 bundle into a list of bundles
63
- :param bundle: valid stix2 bundle
64
- :type bundle:
65
- :param use_json: is JSON?
66
- :type use_json:
67
- :raises Exception: if data is not valid JSON
68
- :return: returns a list of bundles
69
- :rtype: list
70
- """
64
+ def split_bundle_with_expectations(
65
+ self, bundle, use_json=True, event_version=None
66
+ ) -> Tuple[int, list]:
67
+ """splits a valid stix2 bundle into a list of bundles"""
71
68
  if use_json:
72
69
  try:
73
70
  bundle_data = json.loads(bundle)
@@ -96,16 +93,31 @@ class OpenCTIStix2Splitter:
96
93
  return elem["nb_deps"]
97
94
 
98
95
  self.elements.sort(key=by_dep_size)
99
- for entity in self.elements:
96
+
97
+ elements_with_deps = list(
98
+ map(lambda e: {"nb_deps": e["nb_deps"], "elements": [e]}, self.elements)
99
+ )
100
+
101
+ number_expectations = 0
102
+ for entity in elements_with_deps:
103
+ number_expectations += len(entity["elements"])
100
104
  bundles.append(
101
105
  self.stix2_create_bundle(
102
106
  bundle_data["id"],
103
107
  entity["nb_deps"],
104
- [entity],
108
+ entity["elements"],
105
109
  use_json,
106
110
  event_version,
107
111
  )
108
112
  )
113
+
114
+ return number_expectations, bundles
115
+
116
+ @deprecated("Use split_bundle_with_expectations instead")
117
+ def split_bundle(self, bundle, use_json=True, event_version=None) -> list:
118
+ expectations, bundles = self.split_bundle_with_expectations(
119
+ bundle, use_json, event_version
120
+ )
109
121
  return bundles
110
122
 
111
123
  @staticmethod
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pycti
3
- Version: 6.1.4
3
+ Version: 6.1.5
4
4
  Summary: Python API client for OpenCTI.
5
5
  Home-page: https://github.com/OpenCTI-Platform/client-python
6
6
  Author: Filigran
@@ -22,17 +22,19 @@ Requires-Python: >=3.7
22
22
  Description-Content-Type: text/markdown
23
23
  License-File: LICENSE
24
24
  Requires-Dist: datefinder~=0.7.3
25
- Requires-Dist: pika~=1.3.1
25
+ Requires-Dist: pika~=1.3.0
26
26
  Requires-Dist: python-magic~=0.4.27; sys_platform == "linux" or sys_platform == "darwin"
27
- Requires-Dist: prometheus-client~=0.20.0
28
27
  Requires-Dist: python-magic-bin~=0.4.14; sys_platform == "win32"
29
28
  Requires-Dist: python_json_logger~=2.0.4
30
- Requires-Dist: pyyaml~=6.0
31
- Requires-Dist: requests~=2.31.0
32
- Requires-Dist: setuptools~=69.5.1
29
+ Requires-Dist: PyYAML~=6.0
30
+ Requires-Dist: requests~=2.32.2
31
+ Requires-Dist: setuptools~=70.0.0
32
+ Requires-Dist: cachetools~=5.3.0
33
+ Requires-Dist: prometheus-client~=0.20.0
34
+ Requires-Dist: opentelemetry-api~=1.22.0
35
+ Requires-Dist: opentelemetry-sdk~=1.22.0
33
36
  Requires-Dist: filigran-sseclient~=1.0.0
34
37
  Requires-Dist: stix2~=3.0.1
35
- Requires-Dist: cachetools~=5.3.0
36
38
  Provides-Extra: dev
37
39
  Requires-Dist: black~=24.4.0; extra == "dev"
38
40
  Requires-Dist: build~=1.2.1; extra == "dev"
@@ -1,13 +1,15 @@
1
1
  datefinder~=0.7.3
2
- pika~=1.3.1
3
- prometheus-client~=0.20.0
2
+ pika~=1.3.0
4
3
  python_json_logger~=2.0.4
5
- pyyaml~=6.0
6
- requests~=2.31.0
7
- setuptools~=69.5.1
4
+ PyYAML~=6.0
5
+ requests~=2.32.2
6
+ setuptools~=70.0.0
7
+ cachetools~=5.3.0
8
+ prometheus-client~=0.20.0
9
+ opentelemetry-api~=1.22.0
10
+ opentelemetry-sdk~=1.22.0
8
11
  filigran-sseclient~=1.0.0
9
12
  stix2~=3.0.1
10
- cachetools~=5.3.0
11
13
 
12
14
  [:sys_platform == "linux" or sys_platform == "darwin"]
13
15
  python-magic~=0.4.27
@@ -33,17 +33,19 @@ packages =
33
33
  include_package_data = True
34
34
  install_requires =
35
35
  datefinder~=0.7.3
36
- pika~=1.3.1
37
- python-magic~=0.4.27; sys_platform == "linux" or sys_platform == "darwin"
38
- prometheus-client~=0.20.0
39
- python-magic-bin~=0.4.14; sys_platform == "win32"
36
+ pika~=1.3.0
37
+ python-magic~=0.4.27; sys_platform == 'linux' or sys_platform == 'darwin'
38
+ python-magic-bin~=0.4.14; sys_platform == 'win32'
40
39
  python_json_logger~=2.0.4
41
- pyyaml~=6.0
42
- requests~=2.31.0
43
- setuptools~=69.5.1
40
+ PyYAML~=6.0
41
+ requests~=2.32.2
42
+ setuptools~=70.0.0
43
+ cachetools~=5.3.0
44
+ prometheus-client~=0.20.0
45
+ opentelemetry-api~=1.22.0
46
+ opentelemetry-sdk~=1.22.0
44
47
  filigran-sseclient~=1.0.0
45
48
  stix2~=3.0.1
46
- cachetools~=5.3.0
47
49
 
48
50
  [options.extras_require]
49
51
  dev =
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes