pycti 6.0.9__py3-none-any.whl → 6.1.0__py3-none-any.whl

pycti/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "6.0.9"
+__version__ = "6.1.0"
 
 from .api.opencti_api_client import OpenCTIApiClient
 from .api.opencti_api_connector import OpenCTIApiConnector
@@ -53,9 +53,14 @@ from .entities.opencti_vulnerability import Vulnerability
 from .utils.constants import (
     CustomObjectCaseIncident,
     CustomObjectTask,
+    CustomObservableBankAccount,
+    CustomObservableCredential,
     CustomObservableCryptocurrencyWallet,
     CustomObservableHostname,
+    CustomObservablePaymentCard,
+    CustomObservablePhoneNumber,
     CustomObservableText,
+    CustomObservableTrackingNumber,
     CustomObservableUserAgent,
     MultipleRefRelationship,
     StixCyberObservableTypes,
@@ -128,9 +133,14 @@ __all__ = [
     "CustomObjectCaseIncident",
     "CustomObjectTask",
     "StixCyberObservableTypes",
+    "CustomObservableCredential",
     "CustomObservableHostname",
     "CustomObservableUserAgent",
+    "CustomObservableBankAccount",
     "CustomObservableCryptocurrencyWallet",
+    "CustomObservablePaymentCard",
+    "CustomObservablePhoneNumber",
+    "CustomObservableTrackingNumber",
     "CustomObservableText",
     "STIX_EXT_MITRE",
     "STIX_EXT_OCTI_SCO",

pycti/api/opencti_api_client.py

@@ -108,6 +108,7 @@ class OpenCTIApiClient:
         ssl_verify=False,
         proxies=None,
         json_logging=False,
+        bundle_send_to_queue=True,
         cert=None,
         auth=None,
         perform_health_check=True,
@@ -115,6 +116,7 @@ class OpenCTIApiClient:
         """Constructor method"""
 
         # Check configuration
+        self.bundle_send_to_queue = bundle_send_to_queue
         self.ssl_verify = ssl_verify
         self.cert = cert
         self.proxies = proxies
@@ -205,6 +207,12 @@ class OpenCTIApiClient:
     def set_applicant_id_header(self, applicant_id):
         self.request_headers["opencti-applicant-id"] = applicant_id
 
+    def set_playbook_id_header(self, playbook_id):
+        self.request_headers["opencti-playbook-id"] = playbook_id
+
+    def set_event_id(self, event_id):
+        self.request_headers["opencti-event-id"] = event_id
+
     def set_synchronized_upsert_header(self, synchronized):
         self.request_headers["synchronized-upsert"] = (
             "true" if synchronized is True else "false"
@@ -614,18 +622,19 @@ class OpenCTIApiClient:
         """upload a file to OpenCTI API
 
         :param `**kwargs`: arguments for file upload (required: `file_name` and `data`)
-        :return: returns the query respons for the file upload
+        :return: returns the query response for the file upload
         :rtype: dict
         """
 
         file_name = kwargs.get("file_name", None)
+        file_markings = kwargs.get("file_markings", None)
         data = kwargs.get("data", None)
         mime_type = kwargs.get("mime_type", "text/plain")
         if file_name is not None:
             self.app_logger.info("Uploading a file.")
             query = """
-                mutation UploadImport($file: Upload!) {
-                    uploadImport(file: $file) {
+                mutation UploadImport($file: Upload!, $fileMarkings: [String]) {
+                    uploadImport(file: $file, fileMarkings: $fileMarkings) {
                         id
                         name
                     }
@@ -637,8 +646,11 @@ class OpenCTIApiClient:
                     mime_type = "application/json"
                 else:
                     mime_type = magic.from_file(file_name, mime=True)
-
-            return self.query(query, {"file": (File(file_name, data, mime_type))})
+            query_vars = {"file": (File(file_name, data, mime_type))}
+            # optional file markings
+            if file_markings is not None:
+                query_vars["fileMarkings"] = file_markings
+            return self.query(query, query_vars)
         else:
             self.app_logger.error("[upload] Missing parameter: file_name")
             return None

pycti/api/opencti_api_playbook.py

@@ -9,14 +9,15 @@ class OpenCTIApiPlaybook:
             "Executing playbook step", {"playbook_id": playbook["playbook_id"]}
         )
         query = """
-            mutation PlaybookStepExecution($execution_id: ID!, $execution_start: DateTime!, $data_instance_id: ID!, $playbook_id: ID!, $previous_step_id: ID!, $step_id: ID!, $previous_bundle: String!, $bundle: String!) {
-                playbookStepExecution(execution_id: $execution_id, execution_start: $execution_start, data_instance_id: $data_instance_id, playbook_id: $playbook_id, previous_step_id: $previous_step_id, step_id: $step_id, previous_bundle: $previous_bundle, bundle: $bundle)
+            mutation PlaybookStepExecution($execution_id: ID!, $event_id: ID!, $execution_start: DateTime!, $data_instance_id: ID!, $playbook_id: ID!, $previous_step_id: ID!, $step_id: ID!, $previous_bundle: String!, $bundle: String!) {
+                playbookStepExecution(execution_id: $execution_id, event_id: $event_id, execution_start: $execution_start, data_instance_id: $data_instance_id, playbook_id: $playbook_id, previous_step_id: $previous_step_id, step_id: $step_id, previous_bundle: $previous_bundle, bundle: $bundle)
             }
            """
         self.api.query(
             query,
             {
                 "execution_id": playbook["execution_id"],
+                "event_id": playbook["event_id"],
                 "execution_start": playbook["execution_start"],
                 "playbook_id": playbook["playbook_id"],
                 "data_instance_id": playbook["data_instance_id"],

pycti/api/opencti_api_work.py

@@ -9,28 +9,34 @@ class OpenCTIApiWork:
         self.api = api
 
     def to_received(self, work_id: str, message: str):
-        self.api.app_logger.info("Reporting work update_received", {"work_id": work_id})
-        query = """
-            mutation workToReceived($id: ID!, $message: String) {
-                workEdit(id: $id) {
-                    toReceived (message: $message)
+        if self.api.bundle_send_to_queue:
+            self.api.app_logger.info(
+                "Reporting work update_received", {"work_id": work_id}
+            )
+            query = """
+                mutation workToReceived($id: ID!, $message: String) {
+                    workEdit(id: $id) {
+                        toReceived (message: $message)
+                    }
                 }
-            }
-           """
-        self.api.query(query, {"id": work_id, "message": message})
+               """
+            self.api.query(query, {"id": work_id, "message": message})
 
     def to_processed(self, work_id: str, message: str, in_error: bool = False):
-        self.api.app_logger.info(
-            "Reporting work update_processed", {"work_id": work_id}
-        )
-        query = """
-            mutation workToProcessed($id: ID!, $message: String, $inError: Boolean) {
-                workEdit(id: $id) {
-                    toProcessed (message: $message, inError: $inError)
+        if self.api.bundle_send_to_queue:
+            self.api.app_logger.info(
+                "Reporting work update_processed", {"work_id": work_id}
+            )
+            query = """
+                mutation workToProcessed($id: ID!, $message: String, $inError: Boolean) {
+                    workEdit(id: $id) {
+                        toProcessed (message: $message, inError: $inError)
+                    }
                 }
-            }
-           """
-        self.api.query(query, {"id": work_id, "message": message, "inError": in_error})
+               """
+            self.api.query(
+                query, {"id": work_id, "message": message, "inError": in_error}
+            )
 
     def ping(self, work_id: str):
         self.api.app_logger.info("Ping work", {"work_id": work_id})
@@ -44,49 +50,52 @@ class OpenCTIApiWork:
         self.api.query(query, {"id": work_id})
 
     def report_expectation(self, work_id: str, error):
-        self.api.app_logger.info("Report expectation", {"work_id": work_id})
-        query = """
-            mutation reportExpectation($id: ID!, $error: WorkErrorInput) {
-                workEdit(id: $id) {
-                    reportExpectation(error: $error)
+        if self.api.bundle_send_to_queue:
+            self.api.app_logger.info("Report expectation", {"work_id": work_id})
+            query = """
+                mutation reportExpectation($id: ID!, $error: WorkErrorInput) {
+                    workEdit(id: $id) {
+                        reportExpectation(error: $error)
+                    }
                 }
-            }
-           """
-        try:
-            self.api.query(query, {"id": work_id, "error": error})
-        except:
-            self.api.app_logger.error("Cannot report expectation")
+               """
+            try:
+                self.api.query(query, {"id": work_id, "error": error})
+            except:
+                self.api.app_logger.error("Cannot report expectation")
 
     def add_expectations(self, work_id: str, expectations: int):
-        self.api.app_logger.info(
-            "Update action expectations",
-            {"work_id": work_id, "expectations": expectations},
-        )
-        query = """
-            mutation addExpectations($id: ID!, $expectations: Int) {
-                workEdit(id: $id) {
-                    addExpectations(expectations: $expectations)
+        if self.api.bundle_send_to_queue:
+            self.api.app_logger.info(
+                "Update action expectations",
+                {"work_id": work_id, "expectations": expectations},
+            )
+            query = """
+                mutation addExpectations($id: ID!, $expectations: Int) {
+                    workEdit(id: $id) {
+                        addExpectations(expectations: $expectations)
+                    }
                 }
-            }
-           """
-        try:
-            self.api.query(query, {"id": work_id, "expectations": expectations})
-        except:
-            self.api.app_logger.error("Cannot report expectation")
+               """
+            try:
+                self.api.query(query, {"id": work_id, "expectations": expectations})
+            except:
+                self.api.app_logger.error("Cannot report expectation")
 
     def initiate_work(self, connector_id: str, friendly_name: str) -> str:
-        self.api.app_logger.info("Initiate work", {"connector_id": connector_id})
-        query = """
-            mutation workAdd($connectorId: String!, $friendlyName: String) {
-                workAdd(connectorId: $connectorId, friendlyName: $friendlyName) {
-                  id
+        if self.api.bundle_send_to_queue:
+            self.api.app_logger.info("Initiate work", {"connector_id": connector_id})
+            query = """
+                mutation workAdd($connectorId: String!, $friendlyName: String) {
+                    workAdd(connectorId: $connectorId, friendlyName: $friendlyName) {
+                      id
+                    }
                 }
-            }
-           """
-        work = self.api.query(
-            query, {"connectorId": connector_id, "friendlyName": friendly_name}
-        )
-        return work["data"]["workAdd"]["id"]
+               """
+            work = self.api.query(
+                query, {"connectorId": connector_id, "friendlyName": friendly_name}
+            )
+            return work["data"]["workAdd"]["id"]
 
     def delete_work(self, work_id: str):
         query = """

pycti/connector/opencti_connector_helper.py

@@ -282,18 +282,20 @@ class ListenQueue(threading.Thread):
                 is_playbook = "playbook" in json_data["internal"]
                 # If playbook, compute object on data bundle
                 if is_playbook:
-                    execution_id = json_data["internal"]["playbook"]["execution_id"]
                     execution_start = self.helper.date_now()
-                    playbook_id = json_data["internal"]["playbook"]["playbook_id"]
-                    data_instance_id = json_data["internal"]["playbook"][
+                    event_id = json_data["internal"]["playbook"].get("event_id")
+                    execution_id = json_data["internal"]["playbook"].get("execution_id")
+                    playbook_id = json_data["internal"]["playbook"].get("playbook_id")
+                    data_instance_id = json_data["internal"]["playbook"].get(
                         "data_instance_id"
-                    ]
+                    )
                     previous_bundle = json.dumps((json_data["event"]["bundle"]))
                     step_id = json_data["internal"]["playbook"]["step_id"]
                     previous_step_id = json_data["internal"]["playbook"][
                         "previous_step_id"
                     ]
                     playbook_data = {
+                        "event_id": event_id,
                         "execution_id": execution_id,
                         "execution_start": execution_start,
                         "playbook_id": playbook_id,
@@ -728,6 +730,32 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
         self.connect_auto = get_config_variable(
             "CONNECTOR_AUTO", ["connector", "auto"], config, False, False
         )
+        self.bundle_send_to_queue = get_config_variable(
+            "CONNECTOR_SEND_TO_QUEUE",
+            ["connector", "send_to_queue"],
+            config,
+            False,
+            True,
+        )
+        self.bundle_send_to_directory = get_config_variable(
+            "CONNECTOR_SEND_TO_DIRECTORY",
+            ["connector", "send_to_directory"],
+            config,
+            False,
+            False,
+        )
+        self.bundle_send_to_directory_path = get_config_variable(
+            "CONNECTOR_SEND_TO_DIRECTORY_PATH",
+            ["connector", "send_to_directory_path"],
+            config,
+        )
+        self.bundle_send_to_directory_retention = get_config_variable(
+            "CONNECTOR_SEND_TO_DIRECTORY_RETENTION",
+            ["connector", "send_to_directory_retention"],
+            config,
+            True,
+            7,
+        )
         self.connect_only_contextual = get_config_variable(
             "CONNECTOR_ONLY_CONTEXTUAL",
             ["connector", "only_contextual"],
@@ -749,6 +777,8 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
             "CONNECTOR_VALIDATE_BEFORE_IMPORT",
             ["connector", "validate_before_import"],
             config,
+            False,
+            False,
         )
         # Start up the server to expose the metrics.
         expose_metrics = get_config_variable(
@@ -769,6 +799,7 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
             self.opencti_token,
             self.log_level,
             json_logging=self.opencti_json_logging,
+            bundle_send_to_queue=self.bundle_send_to_queue,
         )
         # - Impersonate API that will use applicant id
         # Behave like standard api if applicant not found
@@ -777,6 +808,7 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
             self.opencti_token,
             self.log_level,
             json_logging=self.opencti_json_logging,
+            bundle_send_to_queue=self.bundle_send_to_queue,
         )
         self.connector_logger = self.api.logger_class(self.connect_name)
         # For retro compatibility
@@ -1075,6 +1107,16 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
         bypass_validation = kwargs.get("bypass_validation", False)
         entity_id = kwargs.get("entity_id", None)
         file_name = kwargs.get("file_name", None)
+        bundle_send_to_queue = kwargs.get("send_to_queue", self.bundle_send_to_queue)
+        bundle_send_to_directory = kwargs.get(
+            "send_to_directory", self.bundle_send_to_directory
+        )
+        bundle_send_to_directory_path = kwargs.get(
+            "send_to_directory_path", self.bundle_send_to_directory_path
+        )
+        bundle_send_to_directory_retention = kwargs.get(
+            "send_to_directory_retention", self.bundle_send_to_directory_retention
+        )
 
         # In case of enrichment ingestion, ensure the sharing if needed
         if self.enrichment_shared_organizations is not None:
@@ -1114,13 +1156,14 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
                         )
             bundle = json.dumps(bundle_data)
 
+        # If execution in playbook, callback the api
         if self.playbook is not None:
             self.api.playbook.playbook_step_execution(self.playbook, bundle)
             return [bundle]
 
+        # Upload workbench in case of pending validation
         if not file_name and work_id:
             file_name = f"{work_id}.json"
-
         if self.connect_validate_before_import and not bypass_validation and file_name:
             self.api.upload_pending_file(
                 file_name=file_name,
@@ -1130,8 +1173,61 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
             )
             return []
 
-        if entities_types is None:
-            entities_types = []
+        # If directory setup, write the bundle to the target directory
+        if bundle_send_to_directory and bundle_send_to_directory_path is not None:
+            self.connector_logger.info(
+                "The connector sending bundle to directory",
+                {
+                    "connector": self.connect_name,
+                    "directory": bundle_send_to_directory_path,
+                    "also_queuing": bundle_send_to_queue,
+                },
+            )
+            bundle_file = (
+                self.connect_name.lower().replace(" ", "_")
+                + "-"
+                + time.strftime("%Y%m%d-%H%M%S-")
+                + str(time.time())
+                + ".json"
+            )
+            write_file = os.path.join(
+                bundle_send_to_directory_path, bundle_file + ".tmp"
+            )
+            message_bundle = {
+                "bundle_type": "DIRECTORY_BUNDLE",
+                "applicant_id": self.applicant_id,
+                "connector": {
+                    "id": self.connect_id,
+                    "name": self.connect_name,
+                    "type": self.connect_type,
+                    "scope": self.connect_scope,
+                    "auto": self.connect_auto,
+                    "validate_before_import": self.connect_validate_before_import,
+                },
+                "entities_types": entities_types,
+                "bundle": json.loads(bundle),
+                "update": update,
+            }
+            # Maintains the list of files under control
+            if bundle_send_to_directory_retention > 0:  # If 0, disable the auto remove
+                current_time = time.time()
+                for f in os.listdir(bundle_send_to_directory_path):
+                    if f.endswith(".json"):
+                        file_location = os.path.join(bundle_send_to_directory_path, f)
+                        file_time = os.stat(file_location).st_mtime
+                        is_expired_file = (
+                            file_time
+                            < current_time - 86400 * bundle_send_to_directory_retention
+                        )  # 86400 = 1 day
+                        if is_expired_file:
+                            os.remove(file_location)
+            # Write the bundle to target directory
+            with open(write_file, "w") as f:
+                str_bundle = json.dumps(message_bundle)
+                f.write(str_bundle)
+            # Rename the file after full write
+            final_write_file = os.path.join(bundle_send_to_directory_path, bundle_file)
+            os.rename(write_file, final_write_file)
 
         if bypass_split:
             bundles = [bundle]
@@ -1143,44 +1239,48 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
             self.metric.inc("error_count")
             raise ValueError("Nothing to import")
 
-        if work_id:
-            self.api.work.add_expectations(work_id, len(bundles))
-
-        pika_credentials = pika.PlainCredentials(
-            self.connector_config["connection"]["user"],
-            self.connector_config["connection"]["pass"],
-        )
-        pika_parameters = pika.ConnectionParameters(
-            host=self.connector_config["connection"]["host"],
-            port=self.connector_config["connection"]["port"],
-            virtual_host=self.connector_config["connection"]["vhost"],
-            credentials=pika_credentials,
-            ssl_options=(
-                pika.SSLOptions(
-                    create_mq_ssl_context(self.config),
-                    self.connector_config["connection"]["host"],
-                )
-                if self.connector_config["connection"]["use_ssl"]
-                else None
-            ),
-        )
-        pika_connection = pika.BlockingConnection(pika_parameters)
-        channel = pika_connection.channel()
-        try:
-            channel.confirm_delivery()
-        except Exception as err:  # pylint: disable=broad-except
-            self.connector_logger.warning(str(err))
-        for sequence, bundle in enumerate(bundles, start=1):
-            self._send_bundle(
-                channel,
-                bundle,
-                work_id=work_id,
-                entities_types=entities_types,
-                sequence=sequence,
-                update=update,
+        if bundle_send_to_queue:
+            if work_id:
+                self.api.work.add_expectations(work_id, len(bundles))
+            if entities_types is None:
+                entities_types = []
+            pika_credentials = pika.PlainCredentials(
+                self.connector_config["connection"]["user"],
+                self.connector_config["connection"]["pass"],
             )
-        channel.close()
-        pika_connection.close()
+            pika_parameters = pika.ConnectionParameters(
+                host=self.connector_config["connection"]["host"],
+                port=self.connector_config["connection"]["port"],
+                virtual_host=self.connector_config["connection"]["vhost"],
+                credentials=pika_credentials,
+                ssl_options=(
+                    pika.SSLOptions(
+                        create_mq_ssl_context(self.config),
+                        self.connector_config["connection"]["host"],
+                    )
+                    if self.connector_config["connection"]["use_ssl"]
+                    else None
+                ),
+            )
+            pika_connection = pika.BlockingConnection(pika_parameters)
+            channel = pika_connection.channel()
+            try:
+                channel.confirm_delivery()
+            except Exception as err:  # pylint: disable=broad-except
+                self.connector_logger.warning(str(err))
+            self.connector_logger.info(self.connect_name + " sending bundle to queue")
+            for sequence, bundle in enumerate(bundles, start=1):
+                self._send_bundle(
+                    channel,
+                    bundle,
+                    work_id=work_id,
+                    entities_types=entities_types,
+                    sequence=sequence,
+                    update=update,
+                )
+            channel.close()
+            pika_connection.close()
+
         return bundles
 
     def _send_bundle(self, channel, bundle, **kwargs) -> None:
@@ -1212,6 +1312,7 @@ class OpenCTIConnectorHelper:  # pylint: disable=too-many-public-methods
         # if self.current_work_id is None:
         #    raise ValueError('The job id must be specified')
         message = {
+            "bundle_type": "QUEUE_BUNDLE",
             "applicant_id": self.applicant_id,
             "action_sequence": sequence,
             "entities_types": entities_types,

pycti/entities/opencti_case_incident.py

@@ -530,7 +530,7 @@ class CaseIncident:
             data = self.opencti.process_multiple(result["data"]["caseIncidents"])
             final_data = final_data + data
             while result["data"]["caseIncidents"]["pageInfo"]["hasNextPage"]:
-                after = result["date"]["caseIncidents"]["pageInfo"]["endCursor"]
+                after = result["data"]["caseIncidents"]["pageInfo"]["endCursor"]
                 self.opencti.app_logger.info("Listing Case Incidents", {"after": after})
                 result = self.opencti.query(
                     query,

pycti/entities/opencti_incident.py

@@ -458,6 +458,10 @@ class Incident:
                 stix_object["x_opencti_granted_refs"] = (
                     self.opencti.get_attribute_in_extension("granted_refs", stix_object)
                 )
+            if "x_opencti_workflow_id" not in stix_object:
+                stix_object["x_opencti_workflow_id"] = (
+                    self.opencti.get_attribute_in_extension("workflow_id", stix_object)
+                )
 
             return self.create(
                 stix_id=stix_object["id"],

pycti/entities/opencti_indicator.py

@@ -468,6 +468,7 @@ class Indicator:
         x_opencti_stix_ids = kwargs.get("x_opencti_stix_ids", None)
         create_observables = kwargs.get("x_opencti_create_observables", False)
         granted_refs = kwargs.get("objectOrganization", None)
+        x_opencti_workflow_id = kwargs.get("x_opencti_workflow_id", None)
         update = kwargs.get("update", False)
 
         if (
@@ -528,6 +529,7 @@ class Indicator:
                         "x_opencti_stix_ids": x_opencti_stix_ids,
                         "killChainPhases": kill_chain_phases,
                         "createObservables": create_observables,
+                        "x_opencti_workflow_id": x_opencti_workflow_id,
                         "update": update,
                     }
                 },
@@ -642,6 +644,10 @@ class Indicator:
                 stix_object["x_opencti_granted_refs"] = (
                     self.opencti.get_attribute_in_extension("granted_refs", stix_object)
                 )
+            if "x_opencti_workflow_id" not in stix_object:
+                stix_object["x_opencti_workflow_id"] = (
+                    self.opencti.get_attribute_in_extension("workflow_id", stix_object)
+                )
 
             return self.create(
                 stix_id=stix_object["id"],
@@ -740,6 +746,11 @@ class Indicator:
                     if "x_opencti_granted_refs" in stix_object
                     else None
                 ),
+                x_opencti_workflow_id=(
+                    stix_object["x_opencti_workflow_id"]
+                    if "x_opencti_workflow_id" in stix_object
+                    else None
+                ),
                 update=update,
             )
         else: