pycti 6.3.0__tar.gz → 6.3.2__tar.gz

{pycti-6.3.0 → pycti-6.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pycti
-Version: 6.3.0
+Version: 6.3.2
 Summary: Python API client for OpenCTI.
 Home-page: https://github.com/OpenCTI-Platform/client-python
 Author: Filigran

{pycti-6.3.0 → pycti-6.3.2}/pycti/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "6.3.0"
+__version__ = "6.3.2"
 
 from .api.opencti_api_client import OpenCTIApiClient
 from .api.opencti_api_connector import OpenCTIApiConnector

{pycti-6.3.0 → pycti-6.3.2}/pycti/connector/opencti_connector_helper.py

@@ -376,7 +376,7 @@ class ListenQueue(threading.Thread):
                 self.pika_credentials = pika.PlainCredentials(self.user, self.password)
                 self.pika_parameters = pika.ConnectionParameters(
                     heartbeat=10,
-                    blocked_connection_timeout=10,
+                    blocked_connection_timeout=30,
                     host=self.host,
                     port=self.port,
                     virtual_host=self.vhost,
@@ -1556,6 +1556,8 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
         :type entities_types: list, optional
         :param update: whether to updated data in the database, defaults to False
         :type update: bool, optional
+        :param bypass_split: use to prevent splitting of the bundle. This option has been removed since 6.3 and is no longer used.
+        :type bypass_split: bool, optional
         :raises ValueError: if the bundle is empty
         :return: list of bundles
         :rtype: list
@@ -1564,11 +1566,11 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
         entities_types = kwargs.get("entities_types", None)
         update = kwargs.get("update", False)
         event_version = kwargs.get("event_version", None)
-        bypass_split = kwargs.get("bypass_split", False)
         bypass_validation = kwargs.get("bypass_validation", False)
         entity_id = kwargs.get("entity_id", None)
         file_name = kwargs.get("file_name", None)
         bundle_send_to_queue = kwargs.get("send_to_queue", self.bundle_send_to_queue)
+        cleanup_inconsistent_bundle = kwargs.get("cleanup_inconsistent_bundle", False)
         bundle_send_to_directory = kwargs.get(
             "send_to_directory", self.bundle_send_to_directory
         )
@@ -1690,17 +1692,16 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
             final_write_file = os.path.join(bundle_send_to_directory_path, bundle_file)
             os.rename(write_file, final_write_file)
 
-        if bypass_split:
-            bundles = [bundle]
-            expectations_number = len(json.loads(bundle)["objects"])
-        else:
-            stix2_splitter = OpenCTIStix2Splitter()
-            (
-                expectations_number,
-                bundles,
-            ) = stix2_splitter.split_bundle_with_expectations(
-                bundle, True, event_version
-            )
+        stix2_splitter = OpenCTIStix2Splitter()
+        (
+            expectations_number,
+            bundles,
+        ) = stix2_splitter.split_bundle_with_expectations(
+            bundle=bundle,
+            use_json=True,
+            event_version=event_version,
+            cleanup_inconsistent_bundle=cleanup_inconsistent_bundle,
+        )
 
         if len(bundles) == 0:
             self.metric.inc("error_count")
@@ -1718,7 +1719,6 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
                 )
                 pika_parameters = pika.ConnectionParameters(
                     heartbeat=10,
-                    blocked_connection_timeout=10,
                     host=self.connector_config["connection"]["host"],
                     port=self.connector_config["connection"]["port"],
                     virtual_host=self.connector_config["connection"]["vhost"],

{pycti-6.3.0 → pycti-6.3.2}/pycti/entities/opencti_case_rfi.py

@@ -747,7 +747,7 @@ class CaseRfi:
                 },
             )
             query = """
-               mutation CaseRfiEditRelationAdd($id: ID!, $input: StixRefRelationshipAddInput) {
+               mutation CaseRfiEditRelationAdd($id: ID!, $input: StixRefRelationshipAddInput!) {
                    stixDomainObjectEdit(id: $id) {
                         relationAdd(input: $input) {
                             id

{pycti-6.3.0 → pycti-6.3.2}/pycti/entities/opencti_case_rft.py

@@ -746,7 +746,7 @@ class CaseRft:
                 },
             )
             query = """
-               mutation CaseRftEditRelationAdd($id: ID!, $input: StixRefRelationshipAddInput) {
+               mutation CaseRftEditRelationAdd($id: ID!, $input: StixRefRelationshipAddInput!) {
                    stixDomainObjectEdit(id: $id) {
                         relationAdd(input: $input) {
                             id

{pycti-6.3.0 → pycti-6.3.2}/pycti/entities/opencti_kill_chain_phase.py

@@ -1,9 +1,8 @@
 # coding: utf-8
 
 import json
-import uuid
 
-from stix2.canonicalization.Canonicalize import canonicalize
+from pycti.utils.opencti_stix2_identifier import kill_chain_phase_generate_id
 
 
 class KillChainPhase:
@@ -25,10 +24,9 @@ class KillChainPhase:
 
     @staticmethod
     def generate_id(phase_name, kill_chain_name):
-        data = {"phase_name": phase_name, "kill_chain_name": kill_chain_name}
-        data = canonicalize(data, utf8=False)
-        id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
-        return "kill-chain-phase--" + id
+        return kill_chain_phase_generate_id(
+            phase_name=phase_name, kill_chain_name=kill_chain_name
+        )
 
     """
         List Kill-Chain-Phase objects

{pycti-6.3.0 → pycti-6.3.2}/pycti/utils/opencti_stix2.py

@@ -2619,10 +2619,9 @@ class OpenCTIStix2:
             else None
         )
         stix2_splitter = OpenCTIStix2Splitter()
-        try:
-            bundles = stix2_splitter.split_bundle(stix_bundle, False, event_version)
-        except RecursionError:
-            bundles = [stix_bundle]
+        _, bundles = stix2_splitter.split_bundle_with_expectations(
+            stix_bundle, False, event_version
+        )
         # Import every element in a specific order
         imported_elements = []
         for bundle in bundles:

pycti-6.3.2/pycti/utils/opencti_stix2_identifier.py

@@ -0,0 +1,22 @@
+import uuid
+
+from stix2.canonicalization.Canonicalize import canonicalize
+
+
+def external_reference_generate_id(url=None, source_name=None, external_id=None):
+    if url is not None:
+        data = {"url": url}
+    elif source_name is not None and external_id is not None:
+        data = {"source_name": source_name, "external_id": external_id}
+    else:
+        return None
+    data = canonicalize(data, utf8=False)
+    id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
+    return "external-reference--" + id
+
+
+def kill_chain_phase_generate_id(phase_name, kill_chain_name):
+    data = {"phase_name": phase_name, "kill_chain_name": kill_chain_name}
+    data = canonicalize(data, utf8=False)
+    id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
+    return "kill-chain-phase--" + id

pycti-6.3.2/pycti/utils/opencti_stix2_splitter.py

@@ -0,0 +1,263 @@
+import json
+import uuid
+from typing import Tuple
+
+from typing_extensions import deprecated
+
+from pycti.utils.opencti_stix2_identifier import (
+    external_reference_generate_id,
+    kill_chain_phase_generate_id,
+)
+from pycti.utils.opencti_stix2_utils import (
+    STIX_CYBER_OBSERVABLE_MAPPING,
+    SUPPORTED_STIX_DOMAIN_OBJECTS,
+)
+
+supported_types = (
+    SUPPORTED_STIX_DOMAIN_OBJECTS  # entities
+    + list(STIX_CYBER_OBSERVABLE_MAPPING.keys())  # observables
+    + ["relationship", "sighting"]  # relationships
+)
+
+
+def is_id_supported(key):
+    id_type = key.split("--")[0]
+    return id_type in supported_types
+
+
+class OpenCTIStix2Splitter:
+    def __init__(self):
+        self.cache_index = {}
+        self.cache_refs = {}
+        self.elements = []
+
+    def enlist_element(
+        self, item_id, raw_data, cleanup_inconsistent_bundle, parent_acc
+    ):
+        nb_deps = 1
+        if item_id not in raw_data:
+            return 0
+
+        existing_item = self.cache_index.get(item_id)
+        if existing_item is not None:
+            return existing_item["nb_deps"]
+
+        item = raw_data[item_id]
+        if self.cache_refs.get(item_id) is None:
+            self.cache_refs[item_id] = []
+        for key in list(item.keys()):
+            value = item[key]
+            # Recursive enlist for every refs
+            if key.endswith("_refs"):
+                to_keep = []
+                for element_ref in item[key]:
+                    # We need to check if this ref is not already a reference
+                    is_missing_ref = raw_data.get(element_ref) is None
+                    must_be_cleaned = is_missing_ref and cleanup_inconsistent_bundle
+                    not_dependency_ref = (
+                        self.cache_refs.get(element_ref) is None
+                        or item_id not in self.cache_refs[element_ref]
+                    )
+                    # Prevent any self reference
+                    if (
+                        is_id_supported(element_ref)
+                        and not must_be_cleaned
+                        and element_ref not in parent_acc
+                        and element_ref != item_id
+                        and not_dependency_ref
+                    ):
+                        self.cache_refs[item_id].append(element_ref)
+                        nb_deps += self.enlist_element(
+                            element_ref,
+                            raw_data,
+                            cleanup_inconsistent_bundle,
+                            parent_acc + [element_ref],
+                        )
+                        if element_ref not in to_keep:
+                            to_keep.append(element_ref)
+                    item[key] = to_keep
+            elif key.endswith("_ref"):
+                is_missing_ref = raw_data.get(value) is None
+                must_be_cleaned = is_missing_ref and cleanup_inconsistent_bundle
+                not_dependency_ref = (
+                    self.cache_refs.get(value) is None
+                    or item_id not in self.cache_refs[value]
+                )
+                # Prevent any self reference
+                if (
+                    value is not None
+                    and not must_be_cleaned
+                    and value not in parent_acc
+                    and is_id_supported(value)
+                    and value != item_id
+                    and not_dependency_ref
+                ):
+                    self.cache_refs[item_id].append(value)
+                    nb_deps += self.enlist_element(
+                        value,
+                        raw_data,
+                        cleanup_inconsistent_bundle,
+                        parent_acc + [value],
+                    )
+                else:
+                    item[key] = None
+            # Case for embedded elements (deduplicating and cleanup)
+            elif key == "external_references":
+                # specific case of splitting external references
+                # reference_ids = []
+                deduplicated_references = []
+                deduplicated_references_cache = {}
+                references = item[key]
+                for reference in references:
+                    reference_id = external_reference_generate_id(
+                        url=reference.get("url"),
+                        source_name=reference.get("source_name"),
+                        external_id=reference.get("external_id"),
+                    )
+                    if (
+                        reference_id is not None
+                        and deduplicated_references_cache.get(reference_id) is None
+                    ):
+                        deduplicated_references_cache[reference_id] = reference_id
+                        deduplicated_references.append(reference)
+                        # - Needed for a future move of splitting the elements
+                        # reference["id"] = reference_id
+                        # reference["type"] = "External-Reference"
+                        # raw_data[reference_id] = reference
+                        # if reference_id not in reference_ids:
+                        #     reference_ids.append(reference_id)
+                        # nb_deps += self.enlist_element(reference_id, raw_data)
+                item[key] = deduplicated_references
+            elif key == "kill_chain_phases":
+                # specific case of splitting kill_chain phases
+                # kill_chain_ids = []
+                deduplicated_kill_chain = []
+                deduplicated_kill_chain_cache = {}
+                kill_chains = item[key]
+                for kill_chain in kill_chains:
+                    kill_chain_id = kill_chain_phase_generate_id(
+                        kill_chain_name=kill_chain.get("kill_chain_name"),
+                        phase_name=kill_chain.get("phase_name"),
+                    )
+                    if (
+                        kill_chain_id is not None
+                        and deduplicated_kill_chain_cache.get(kill_chain_id) is None
+                    ):
+                        deduplicated_kill_chain_cache[kill_chain_id] = kill_chain_id
+                        deduplicated_kill_chain.append(kill_chain)
+                        # - Needed for a future move of splitting the elements
+                        # kill_chain["id"] = kill_chain_id
+                        # kill_chain["type"] = "Kill-Chain-Phase"
+                        # raw_data[kill_chain_id] = kill_chain
+                        # if kill_chain_id not in kill_chain_ids:
+                        #     kill_chain_ids.append(kill_chain_id)
+                        # nb_deps += self.enlist_element(kill_chain_id, raw_data)
+                item[key] = deduplicated_kill_chain
+
+        # Get the final dep counting and add in cache
+        item["nb_deps"] = nb_deps
+        # Put in cache
+        if self.cache_index.get(item_id) is None:
+            # enlist only if compatible
+            if item["type"] == "relationship":
+                is_compatible = (
+                    item["source_ref"] is not None and item["target_ref"] is not None
+                )
+            elif item["type"] == "sighting":
+                is_compatible = (
+                    item["sighting_of_ref"] is not None
+                    and len(item["where_sighted_refs"]) > 0
+                )
+            else:
+                is_compatible = is_id_supported(item_id)
+            if is_compatible:
+                self.elements.append(item)
+            self.cache_index[item_id] = item
+
+        return nb_deps
+
+    def split_bundle_with_expectations(
+        self,
+        bundle,
+        use_json=True,
+        event_version=None,
+        cleanup_inconsistent_bundle=False,
+    ) -> Tuple[int, list]:
+        """splits a valid stix2 bundle into a list of bundles"""
+        if use_json:
+            try:
+                bundle_data = json.loads(bundle)
+            except:
+                raise Exception("File data is not a valid JSON")
+        else:
+            bundle_data = bundle
+
+        if "objects" not in bundle_data:
+            raise Exception("File data is not a valid bundle")
+        if "id" not in bundle_data:
+            bundle_data["id"] = "bundle--" + str(uuid.uuid4())
+
+        raw_data = {}
+
+        # Build flat list of elements
+        for item in bundle_data["objects"]:
+            raw_data[item["id"]] = item
+        for item in bundle_data["objects"]:
+            self.enlist_element(item["id"], raw_data, cleanup_inconsistent_bundle, [])
+
+        # Build the bundles
+        bundles = []
+
+        def by_dep_size(elem):
+            return elem["nb_deps"]
+
+        self.elements.sort(key=by_dep_size)
+
+        elements_with_deps = list(
+            map(lambda e: {"nb_deps": e["nb_deps"], "elements": [e]}, self.elements)
+        )
+
+        number_expectations = 0
+        for entity in elements_with_deps:
+            number_expectations += len(entity["elements"])
+            bundles.append(
+                self.stix2_create_bundle(
+                    bundle_data["id"],
+                    entity["nb_deps"],
+                    entity["elements"],
+                    use_json,
+                    event_version,
+                )
+            )
+
+        return number_expectations, bundles
+
+    @deprecated("Use split_bundle_with_expectations instead")
+    def split_bundle(self, bundle, use_json=True, event_version=None) -> list:
+        expectations, bundles = self.split_bundle_with_expectations(
+            bundle, use_json, event_version
+        )
+        return bundles
+
+    @staticmethod
+    def stix2_create_bundle(bundle_id, bundle_seq, items, use_json, event_version=None):
+        """create a stix2 bundle with items
+
+        :param items: valid stix2 items
+        :type items:
+        :param use_json: use JSON?
+        :type use_json:
+        :return: JSON of the stix2 bundle
+        :rtype:
+        """
+
+        bundle = {
+            "type": "bundle",
+            "id": bundle_id,
+            "spec_version": "2.1",
+            "x_opencti_seq": bundle_seq,
+            "objects": items,
+        }
+        if event_version is not None:
+            bundle["x_opencti_event_version"] = event_version
+        return json.dumps(bundle) if use_json else bundle

{pycti-6.3.0 → pycti-6.3.2}/pycti/utils/opencti_stix2_utils.py

@@ -2,6 +2,46 @@ from typing import Any, Dict
 
 from stix2 import EqualityComparisonExpression, ObjectPath, ObservationExpression
 
+SUPPORTED_STIX_DOMAIN_OBJECTS = [
+    "marking-definition",
+    "attack-pattern",
+    "campaign",
+    "channel",
+    "event",
+    "note",
+    "observed-data",
+    "opinion",
+    "report",
+    "grouping",
+    "case-rfi",
+    "x-opencti-case-rfi",
+    "case-rft",
+    "x-opencti-case-rft",
+    "task",
+    "x-opencti-task",
+    "case-incident",
+    "x-opencti-case-incident",
+    "feedback",
+    "x-opencti-feedback",
+    "course-of-action",
+    "data-component",
+    "x-mitre-data-component",
+    "data-source",
+    "x-mitre-data-source",
+    "identity",
+    "indicator",
+    "infrastructure",
+    "intrusion-set",
+    "location",
+    "malware",
+    "malware-analysis",
+    "threat-actor",
+    "tool",
+    "narrative",
+    "vulnerability",
+    "incident",
+]
+
 STIX_CYBER_OBSERVABLE_MAPPING = {
     "autonomous-system": "Autonomous-System",
     "directory": "Directory",

{pycti-6.3.0 → pycti-6.3.2}/pycti.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pycti
-Version: 6.3.0
+Version: 6.3.2
 Summary: Python API client for OpenCTI.
 Home-page: https://github.com/OpenCTI-Platform/client-python
 Author: Filigran

{pycti-6.3.0 → pycti-6.3.2}/pycti.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ pycti/utils/__init__.py
 pycti/utils/constants.py
 pycti/utils/opencti_logger.py
 pycti/utils/opencti_stix2.py
+pycti/utils/opencti_stix2_identifier.py
 pycti/utils/opencti_stix2_splitter.py
 pycti/utils/opencti_stix2_update.py
 pycti/utils/opencti_stix2_utils.py

pycti-6.3.0/pycti/utils/opencti_stix2_splitter.py

@@ -1,144 +0,0 @@
-import json
-import re
-import uuid
-from typing import Tuple
-
-from typing_extensions import deprecated
-
-MITRE_X_CAPEC = (
-    "x_capec_*"  # https://github.com/mitre-attack/attack-stix-data/issues/34
-)
-unsupported_ref_patterns = [MITRE_X_CAPEC]
-
-
-class OpenCTIStix2Splitter:
-    def __init__(self):
-        self.cache_index = {}
-        self.elements = []
-        self.unsupported_patterns = list(
-            map(lambda pattern: re.compile(pattern), unsupported_ref_patterns)
-        )
-
-    def is_ref_key_supported(self, key):
-        for pattern in self.unsupported_patterns:
-            if pattern.match(key):
-                return False
-        return True
-
-    def enlist_element(self, item_id, raw_data):
-        nb_deps = 1
-        if item_id not in raw_data:
-            return 0
-        existing_item = self.cache_index.get(item_id)
-        if existing_item is not None:
-            return existing_item["nb_deps"]
-        # Recursive enlist for every refs
-        item = raw_data[item_id]
-        for key in list(item.keys()):
-            value = item[key]
-            if key.endswith("_refs") and self.is_ref_key_supported(key):
-                to_keep = []
-                for element_ref in item[key]:
-                    if element_ref != item_id:
-                        nb_deps += self.enlist_element(element_ref, raw_data)
-                        to_keep.append(element_ref)
-                    item[key] = to_keep
-            elif key.endswith("_ref") and self.is_ref_key_supported(key):
-                if item[key] == item_id:
-                    item[key] = None
-                else:
-                    # Need to handle the special case of recursive ref for created by ref
-                    is_created_by_ref = key == "created_by_ref"
-                    if is_created_by_ref:
-                        is_marking = item["id"].startswith("marking-definition--")
-                        if is_marking is False:
-                            nb_deps += self.enlist_element(value, raw_data)
-                    else:
-                        nb_deps += self.enlist_element(value, raw_data)
-        # Get the final dep counting and add in cache
-        item["nb_deps"] = nb_deps
-        self.elements.append(item)
-        self.cache_index[item_id] = item  # Put in cache
-        return nb_deps
-
-    def split_bundle_with_expectations(
-        self, bundle, use_json=True, event_version=None
-    ) -> Tuple[int, list]:
-        """splits a valid stix2 bundle into a list of bundles"""
-        if use_json:
-            try:
-                bundle_data = json.loads(bundle)
-            except:
-                raise Exception("File data is not a valid JSON")
-        else:
-            bundle_data = bundle
-
-        if "objects" not in bundle_data:
-            raise Exception("File data is not a valid bundle")
-        if "id" not in bundle_data:
-            bundle_data["id"] = "bundle--" + str(uuid.uuid4())
-
-        raw_data = {}
-
-        # Build flat list of elements
-        for item in bundle_data["objects"]:
-            raw_data[item["id"]] = item
-        for item in bundle_data["objects"]:
-            self.enlist_element(item["id"], raw_data)
-
-        # Build the bundles
-        bundles = []
-
-        def by_dep_size(elem):
-            return elem["nb_deps"]
-
-        self.elements.sort(key=by_dep_size)
-
-        elements_with_deps = list(
-            map(lambda e: {"nb_deps": e["nb_deps"], "elements": [e]}, self.elements)
-        )
-
-        number_expectations = 0
-        for entity in elements_with_deps:
-            number_expectations += len(entity["elements"])
-            bundles.append(
-                self.stix2_create_bundle(
-                    bundle_data["id"],
-                    entity["nb_deps"],
-                    entity["elements"],
-                    use_json,
-                    event_version,
-                )
-            )
-
-        return number_expectations, bundles
-
-    @deprecated("Use split_bundle_with_expectations instead")
-    def split_bundle(self, bundle, use_json=True, event_version=None) -> list:
-        expectations, bundles = self.split_bundle_with_expectations(
-            bundle, use_json, event_version
-        )
-        return bundles
-
-    @staticmethod
-    def stix2_create_bundle(bundle_id, bundle_seq, items, use_json, event_version=None):
-        """create a stix2 bundle with items
-
-        :param items: valid stix2 items
-        :type items:
-        :param use_json: use JSON?
-        :type use_json:
-        :return: JSON of the stix2 bundle
-        :rtype:
-        """
-
-        bundle = {
-            "type": "bundle",
-            "id": bundle_id,
-            "spec_version": "2.1",
-            "x_opencti_seq": bundle_seq,
-            "objects": items,
-        }
-        if event_version is not None:
-            bundle["x_opencti_event_version"] = event_version
-        return json.dumps(bundle) if use_json else bundle