pycti 5.12.13__tar.gz → 5.12.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pycti might be problematic. Click here for more details.
- {pycti-5.12.13 → pycti-5.12.15}/PKG-INFO +1 -1
- {pycti-5.12.13 → pycti-5.12.15}/pycti/__init__.py +1 -1
- {pycti-5.12.13 → pycti-5.12.15}/pycti/api/opencti_api_client.py +10 -56
- {pycti-5.12.13 → pycti-5.12.15}/pycti/api/opencti_api_connector.py +1 -2
- {pycti-5.12.13 → pycti-5.12.15}/pycti/api/opencti_api_playbook.py +3 -4
- {pycti-5.12.13 → pycti-5.12.15}/pycti/api/opencti_api_work.py +15 -12
- pycti-5.12.15/pycti/connector/__init__.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/connector/opencti_connector_helper.py +63 -54
- {pycti-5.12.13 → pycti-5.12.15}/pycti/connector/opencti_metric_handler.py +8 -5
- pycti-5.12.15/pycti/entities/__init__.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_attack_pattern.py +17 -11
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_campaign.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_case_incident.py +39 -43
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_case_rfi.py +34 -46
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_case_rft.py +35 -44
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_channel.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_course_of_action.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_data_component.py +17 -14
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_data_source.py +12 -16
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_event.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_external_reference.py +20 -15
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_feedback.py +36 -46
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_grouping.py +22 -20
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_identity.py +15 -8
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_incident.py +13 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_indicator.py +18 -15
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_infrastructure.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_intrusion_set.py +12 -8
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_kill_chain_phase.py +16 -12
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_label.py +16 -12
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_language.py +13 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_location.py +13 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_malware.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_malware_analysis.py +17 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_marking_definition.py +15 -11
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_narrative.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_note.py +35 -20
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_observed_data.py +34 -19
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_opinion.py +37 -20
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_report.py +36 -20
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix.py +8 -19
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_core_object.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_core_relationship.py +69 -52
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_cyber_observable.py +78 -62
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_domain_object.py +75 -55
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_nested_ref_relationship.py +21 -14
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_object_or_stix_relationship.py +4 -7
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_stix_sighting_relationship.py +34 -26
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_task.py +37 -44
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_threat_actor.py +7 -4
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_threat_actor_group.py +10 -8
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_threat_actor_individual.py +10 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_tool.py +15 -9
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_vocabulary.py +11 -13
- {pycti-5.12.13 → pycti-5.12.15}/pycti/entities/opencti_vulnerability.py +15 -9
- pycti-5.12.15/pycti/utils/__init__.py +0 -0
- pycti-5.12.15/pycti/utils/opencti_logger.py +64 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/utils/opencti_stix2.py +30 -24
- {pycti-5.12.13 → pycti-5.12.15}/pycti/utils/opencti_stix2_update.py +2 -6
- {pycti-5.12.13 → pycti-5.12.15}/pycti.egg-info/PKG-INFO +1 -1
- {pycti-5.12.13 → pycti-5.12.15}/pycti.egg-info/SOURCES.txt +1 -0
- pycti-5.12.13/pycti/api/__init__.py +0 -3
- pycti-5.12.13/pycti/connector/__init__.py +0 -3
- pycti-5.12.13/pycti/entities/__init__.py +0 -3
- {pycti-5.12.13 → pycti-5.12.15}/LICENSE +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/README.md +0 -0
- {pycti-5.12.13/pycti/utils → pycti-5.12.15/pycti/api}/__init__.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/connector/opencti_connector.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/utils/constants.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/utils/opencti_stix2_splitter.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti/utils/opencti_stix2_utils.py +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti.egg-info/dependency_links.txt +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti.egg-info/requires.txt +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pycti.egg-info/top_level.txt +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/pyproject.toml +0 -0
- {pycti-5.12.13 → pycti-5.12.15}/setup.cfg +0 -0
|
@@ -3,16 +3,12 @@ import base64
|
|
|
3
3
|
import datetime
|
|
4
4
|
import io
|
|
5
5
|
import json
|
|
6
|
-
import logging
|
|
7
6
|
from typing import Union
|
|
8
7
|
|
|
9
8
|
import magic
|
|
10
9
|
import requests
|
|
11
|
-
import urllib3
|
|
12
|
-
from pythonjsonlogger import jsonlogger
|
|
13
10
|
|
|
14
11
|
from pycti import __version__
|
|
15
|
-
from pycti.api import LOGGER
|
|
16
12
|
from pycti.api.opencti_api_connector import OpenCTIApiConnector
|
|
17
13
|
from pycti.api.opencti_api_playbook import OpenCTIApiPlaybook
|
|
18
14
|
from pycti.api.opencti_api_work import OpenCTIApiWork
|
|
@@ -65,24 +61,10 @@ from pycti.entities.opencti_threat_actor_individual import ThreatActorIndividual
|
|
|
65
61
|
from pycti.entities.opencti_tool import Tool
|
|
66
62
|
from pycti.entities.opencti_vocabulary import Vocabulary
|
|
67
63
|
from pycti.entities.opencti_vulnerability import Vulnerability
|
|
64
|
+
from pycti.utils.opencti_logger import logger
|
|
68
65
|
from pycti.utils.opencti_stix2 import OpenCTIStix2
|
|
69
66
|
from pycti.utils.opencti_stix2_utils import OpenCTIStix2Utils
|
|
70
67
|
|
|
71
|
-
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
class CustomJsonFormatter(jsonlogger.JsonFormatter):
|
|
75
|
-
def add_fields(self, log_record, record, message_dict):
|
|
76
|
-
super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
|
|
77
|
-
if not log_record.get("timestamp"):
|
|
78
|
-
# This doesn't use record.created, so it is slightly off
|
|
79
|
-
now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
80
|
-
log_record["timestamp"] = now
|
|
81
|
-
if log_record.get("level"):
|
|
82
|
-
log_record["level"] = log_record["level"].upper()
|
|
83
|
-
else:
|
|
84
|
-
log_record["level"] = record.levelname
|
|
85
|
-
|
|
86
68
|
|
|
87
69
|
class File:
|
|
88
70
|
def __init__(self, name, data, mime="text/plain"):
|
|
@@ -141,19 +123,8 @@ class OpenCTIApiClient:
|
|
|
141
123
|
raise ValueError("A TOKEN must be set")
|
|
142
124
|
|
|
143
125
|
# Configure logger
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
if json_logging:
|
|
148
|
-
log_handler = logging.StreamHandler()
|
|
149
|
-
log_handler.setLevel(log_level)
|
|
150
|
-
formatter = CustomJsonFormatter(
|
|
151
|
-
"%(timestamp)s %(level)s %(name)s %(message)s"
|
|
152
|
-
)
|
|
153
|
-
log_handler.setFormatter(formatter)
|
|
154
|
-
logging.basicConfig(handlers=[log_handler], level=log_level, force=True)
|
|
155
|
-
else:
|
|
156
|
-
logging.basicConfig(level=log_level)
|
|
126
|
+
self.logger_class = logger(log_level.upper(), json_logging)
|
|
127
|
+
self.app_logger = self.logger_class("api")
|
|
157
128
|
|
|
158
129
|
# Define API
|
|
159
130
|
self.api_token = token
|
|
@@ -398,23 +369,6 @@ class OpenCTIApiClient:
|
|
|
398
369
|
return base64.b64encode(r.text).decode("utf-8")
|
|
399
370
|
return r.text
|
|
400
371
|
|
|
401
|
-
def log(self, level, message):
|
|
402
|
-
"""log a message with defined log level
|
|
403
|
-
:param level: must be a valid logging log level (debug, info, warning, error)
|
|
404
|
-
:type level: str
|
|
405
|
-
:param message: the message to log
|
|
406
|
-
:type message: str
|
|
407
|
-
"""
|
|
408
|
-
|
|
409
|
-
if level == "debug":
|
|
410
|
-
LOGGER.debug(message)
|
|
411
|
-
elif level == "info":
|
|
412
|
-
LOGGER.info(message)
|
|
413
|
-
elif level == "warning":
|
|
414
|
-
LOGGER.warning(message)
|
|
415
|
-
elif level == "error":
|
|
416
|
-
LOGGER.error(message)
|
|
417
|
-
|
|
418
372
|
def health_check(self):
|
|
419
373
|
"""submit an example request to the OpenCTI API.
|
|
420
374
|
|
|
@@ -426,7 +380,7 @@ class OpenCTIApiClient:
|
|
|
426
380
|
if test is not None:
|
|
427
381
|
return True
|
|
428
382
|
except Exception as err: # pylint: disable=broad-except
|
|
429
|
-
|
|
383
|
+
self.app_logger.error(str(err))
|
|
430
384
|
return False
|
|
431
385
|
return False
|
|
432
386
|
|
|
@@ -437,7 +391,7 @@ class OpenCTIApiClient:
|
|
|
437
391
|
rtype: dict
|
|
438
392
|
"""
|
|
439
393
|
|
|
440
|
-
|
|
394
|
+
self.app_logger.info("Getting logs worker config...")
|
|
441
395
|
query = """
|
|
442
396
|
query LogsWorkerConfig {
|
|
443
397
|
logsWorkerConfig {
|
|
@@ -638,7 +592,7 @@ class OpenCTIApiClient:
|
|
|
638
592
|
data = kwargs.get("data", None)
|
|
639
593
|
mime_type = kwargs.get("mime_type", "text/plain")
|
|
640
594
|
if file_name is not None:
|
|
641
|
-
|
|
595
|
+
self.app_logger.info("Uploading a file.")
|
|
642
596
|
query = """
|
|
643
597
|
mutation UploadImport($file: Upload!) {
|
|
644
598
|
uploadImport(file: $file) {
|
|
@@ -656,7 +610,7 @@ class OpenCTIApiClient:
|
|
|
656
610
|
|
|
657
611
|
return self.query(query, {"file": (File(file_name, data, mime_type))})
|
|
658
612
|
else:
|
|
659
|
-
|
|
613
|
+
self.app_logger.error("[upload] Missing parameter: file_name")
|
|
660
614
|
return None
|
|
661
615
|
|
|
662
616
|
def upload_pending_file(self, **kwargs):
|
|
@@ -673,7 +627,7 @@ class OpenCTIApiClient:
|
|
|
673
627
|
entity_id = kwargs.get("entity_id", None)
|
|
674
628
|
|
|
675
629
|
if file_name is not None:
|
|
676
|
-
|
|
630
|
+
self.app_logger.info("Uploading a file.")
|
|
677
631
|
query = """
|
|
678
632
|
mutation UploadPending($file: Upload!, $entityId: String) {
|
|
679
633
|
uploadPending(file: $file, entityId: $entityId) {
|
|
@@ -693,7 +647,7 @@ class OpenCTIApiClient:
|
|
|
693
647
|
{"file": (File(file_name, data, mime_type)), "entityId": entity_id},
|
|
694
648
|
)
|
|
695
649
|
else:
|
|
696
|
-
|
|
650
|
+
self.app_logger.error("[upload] Missing parameter: file_name")
|
|
697
651
|
return None
|
|
698
652
|
|
|
699
653
|
def get_stix_content(self, id):
|
|
@@ -703,7 +657,7 @@ class OpenCTIApiClient:
|
|
|
703
657
|
rtype: dict
|
|
704
658
|
"""
|
|
705
659
|
|
|
706
|
-
|
|
660
|
+
self.app_logger.info("Entity in JSON", {"id": id})
|
|
707
661
|
query = """
|
|
708
662
|
query StixQuery($id: String!) {
|
|
709
663
|
stix(id: $id)
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import json
|
|
2
2
|
from typing import Any, Dict
|
|
3
3
|
|
|
4
|
-
from pycti.api import LOGGER
|
|
5
4
|
from pycti.connector.opencti_connector import OpenCTIConnector
|
|
6
5
|
|
|
7
6
|
|
|
@@ -18,7 +17,7 @@ class OpenCTIApiConnector:
|
|
|
18
17
|
:rtype: dict
|
|
19
18
|
"""
|
|
20
19
|
|
|
21
|
-
|
|
20
|
+
self.api.app_logger.info("Getting connectors ...")
|
|
22
21
|
query = """
|
|
23
22
|
query GetConnectors {
|
|
24
23
|
connectorsForWorker {
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
from pycti.api import LOGGER
|
|
2
|
-
|
|
3
|
-
|
|
4
1
|
class OpenCTIApiPlaybook:
|
|
5
2
|
"""OpenCTIApiPlaybook"""
|
|
6
3
|
|
|
@@ -8,7 +5,9 @@ class OpenCTIApiPlaybook:
|
|
|
8
5
|
self.api = api
|
|
9
6
|
|
|
10
7
|
def playbook_step_execution(self, playbook: dict, bundle: str):
|
|
11
|
-
|
|
8
|
+
self.api.app_logger.info(
|
|
9
|
+
"Executing playbook step", {"playbook_id": playbook["playbook_id"]}
|
|
10
|
+
)
|
|
12
11
|
query = """
|
|
13
12
|
mutation PlaybookStepExecution($execution_id: ID!, $execution_start: DateTime!, $data_instance_id: ID!, $playbook_id: ID!, $previous_step_id: ID!, $step_id: ID!, $previous_bundle: String!, $bundle: String!) {
|
|
14
13
|
playbookStepExecution(execution_id: $execution_id, execution_start: $execution_start, data_instance_id: $data_instance_id, playbook_id: $playbook_id, previous_step_id: $previous_step_id, step_id: $step_id, previous_bundle: $previous_bundle, bundle: $bundle)
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import time
|
|
2
2
|
from typing import Dict, List
|
|
3
3
|
|
|
4
|
-
from pycti.api import LOGGER
|
|
5
|
-
|
|
6
4
|
|
|
7
5
|
class OpenCTIApiWork:
|
|
8
6
|
"""OpenCTIApiJob"""
|
|
@@ -11,7 +9,7 @@ class OpenCTIApiWork:
|
|
|
11
9
|
self.api = api
|
|
12
10
|
|
|
13
11
|
def to_received(self, work_id: str, message: str):
|
|
14
|
-
|
|
12
|
+
self.api.app_logger.info("Reporting work update_received", {"work_id": work_id})
|
|
15
13
|
query = """
|
|
16
14
|
mutation workToReceived($id: ID!, $message: String) {
|
|
17
15
|
workEdit(id: $id) {
|
|
@@ -22,7 +20,9 @@ class OpenCTIApiWork:
|
|
|
22
20
|
self.api.query(query, {"id": work_id, "message": message})
|
|
23
21
|
|
|
24
22
|
def to_processed(self, work_id: str, message: str, in_error: bool = False):
|
|
25
|
-
|
|
23
|
+
self.api.app_logger.info(
|
|
24
|
+
"Reporting work update_processed", {"work_id": work_id}
|
|
25
|
+
)
|
|
26
26
|
query = """
|
|
27
27
|
mutation workToProcessed($id: ID!, $message: String, $inError: Boolean) {
|
|
28
28
|
workEdit(id: $id) {
|
|
@@ -33,7 +33,7 @@ class OpenCTIApiWork:
|
|
|
33
33
|
self.api.query(query, {"id": work_id, "message": message, "inError": in_error})
|
|
34
34
|
|
|
35
35
|
def ping(self, work_id: str):
|
|
36
|
-
|
|
36
|
+
self.api.app_logger.info("Ping work", {"work_id": work_id})
|
|
37
37
|
query = """
|
|
38
38
|
mutation pingWork($id: ID!) {
|
|
39
39
|
workEdit(id: $id) {
|
|
@@ -44,7 +44,7 @@ class OpenCTIApiWork:
|
|
|
44
44
|
self.api.query(query, {"id": work_id})
|
|
45
45
|
|
|
46
46
|
def report_expectation(self, work_id: str, error):
|
|
47
|
-
|
|
47
|
+
self.api.app_logger.info("Report expectation", {"work_id": work_id})
|
|
48
48
|
query = """
|
|
49
49
|
mutation reportExpectation($id: ID!, $error: WorkErrorInput) {
|
|
50
50
|
workEdit(id: $id) {
|
|
@@ -55,10 +55,13 @@ class OpenCTIApiWork:
|
|
|
55
55
|
try:
|
|
56
56
|
self.api.query(query, {"id": work_id, "error": error})
|
|
57
57
|
except:
|
|
58
|
-
self.api.
|
|
58
|
+
self.api.app_logger.error("Cannot report expectation")
|
|
59
59
|
|
|
60
60
|
def add_expectations(self, work_id: str, expectations: int):
|
|
61
|
-
|
|
61
|
+
self.api.app_logger.info(
|
|
62
|
+
"Update action expectations",
|
|
63
|
+
{"work_id": work_id, "expectations": expectations},
|
|
64
|
+
)
|
|
62
65
|
query = """
|
|
63
66
|
mutation addExpectations($id: ID!, $expectations: Int) {
|
|
64
67
|
workEdit(id: $id) {
|
|
@@ -69,10 +72,10 @@ class OpenCTIApiWork:
|
|
|
69
72
|
try:
|
|
70
73
|
self.api.query(query, {"id": work_id, "expectations": expectations})
|
|
71
74
|
except:
|
|
72
|
-
self.api.
|
|
75
|
+
self.api.app_logger.error("Cannot report expectation")
|
|
73
76
|
|
|
74
77
|
def initiate_work(self, connector_id: str, friendly_name: str) -> str:
|
|
75
|
-
|
|
78
|
+
self.api.app_logger.info("Initiate work", {"connector_id": connector_id})
|
|
76
79
|
query = """
|
|
77
80
|
mutation workAdd($connectorId: String!, $friendlyName: String) {
|
|
78
81
|
workAdd(connectorId: $connectorId, friendlyName: $friendlyName) {
|
|
@@ -107,8 +110,8 @@ class OpenCTIApiWork:
|
|
|
107
110
|
status = state["status"]
|
|
108
111
|
|
|
109
112
|
if state["errors"]:
|
|
110
|
-
self.api.
|
|
111
|
-
"
|
|
113
|
+
self.api.app_logger.error(
|
|
114
|
+
"Unexpected connector error", {"state_errors": state["errors"]}
|
|
112
115
|
)
|
|
113
116
|
return ""
|
|
114
117
|
|
|
File without changes
|
|
@@ -3,7 +3,6 @@ import base64
|
|
|
3
3
|
import copy
|
|
4
4
|
import datetime
|
|
5
5
|
import json
|
|
6
|
-
import logging
|
|
7
6
|
import os
|
|
8
7
|
import queue
|
|
9
8
|
import signal
|
|
@@ -22,7 +21,6 @@ from filigran_sseclient import SSEClient
|
|
|
22
21
|
from pika.exceptions import NackError, UnroutableError
|
|
23
22
|
|
|
24
23
|
from pycti.api.opencti_api_client import OpenCTIApiClient
|
|
25
|
-
from pycti.connector import LOGGER
|
|
26
24
|
from pycti.connector.opencti_connector import OpenCTIConnector
|
|
27
25
|
from pycti.connector.opencti_metric_handler import OpenCTIMetricHandler
|
|
28
26
|
from pycti.utils.opencti_stix2_splitter import OpenCTIStix2Splitter
|
|
@@ -30,8 +28,6 @@ from pycti.utils.opencti_stix2_splitter import OpenCTIStix2Splitter
|
|
|
30
28
|
TRUTHY: List[str] = ["yes", "true", "True"]
|
|
31
29
|
FALSY: List[str] = ["no", "false", "False"]
|
|
32
30
|
|
|
33
|
-
logging.getLogger("pika").setLevel(logging.ERROR)
|
|
34
|
-
|
|
35
31
|
|
|
36
32
|
def killProgramHook(etype, value, tb):
|
|
37
33
|
traceback.print_exception(etype, value, tb)
|
|
@@ -220,7 +216,7 @@ class ListenQueue(threading.Thread):
|
|
|
220
216
|
# Not ACK the message here may lead to infinite re-deliver if the connector is broken
|
|
221
217
|
# Also ACK, will not have any impact on the blocking aspect of the following functions
|
|
222
218
|
channel.basic_ack(delivery_tag=method.delivery_tag)
|
|
223
|
-
|
|
219
|
+
self.helper.connector_logger.info("Message ack", {"tag": method.delivery_tag})
|
|
224
220
|
|
|
225
221
|
self.thread = threading.Thread(target=self._data_handler, args=[json_data])
|
|
226
222
|
self.thread.start()
|
|
@@ -237,9 +233,9 @@ class ListenQueue(threading.Thread):
|
|
|
237
233
|
else:
|
|
238
234
|
time_wait += 1
|
|
239
235
|
time.sleep(1)
|
|
240
|
-
|
|
241
|
-
"Message
|
|
242
|
-
method.delivery_tag,
|
|
236
|
+
self.helper.connector_logger.info(
|
|
237
|
+
"Message processed, thread terminated",
|
|
238
|
+
{"tag": method.delivery_tag},
|
|
243
239
|
)
|
|
244
240
|
|
|
245
241
|
def _data_handler(self, json_data) -> None:
|
|
@@ -281,13 +277,17 @@ class ListenQueue(threading.Thread):
|
|
|
281
277
|
self.helper.api.work.to_processed(work_id, message)
|
|
282
278
|
except Exception as e: # pylint: disable=broad-except
|
|
283
279
|
self.helper.metric.inc("error_count")
|
|
284
|
-
|
|
280
|
+
self.helper.connector_logger.error(
|
|
281
|
+
"Error in message processing, reporting error to API"
|
|
282
|
+
)
|
|
285
283
|
if work_id:
|
|
286
284
|
try:
|
|
287
285
|
self.helper.api.work.to_processed(work_id, str(e), True)
|
|
288
286
|
except: # pylint: disable=bare-except
|
|
289
287
|
self.helper.metric.inc("error_count")
|
|
290
|
-
|
|
288
|
+
self.helper.connector_logger.error(
|
|
289
|
+
"Failing reporting the processing"
|
|
290
|
+
)
|
|
291
291
|
|
|
292
292
|
def run(self) -> None:
|
|
293
293
|
while not self.exit_event.is_set():
|
|
@@ -310,7 +310,7 @@ class ListenQueue(threading.Thread):
|
|
|
310
310
|
try:
|
|
311
311
|
self.channel.confirm_delivery()
|
|
312
312
|
except Exception as err: # pylint: disable=broad-except
|
|
313
|
-
|
|
313
|
+
self.helper.connector_logger.warning(str(err))
|
|
314
314
|
self.channel.basic_qos(prefetch_count=1)
|
|
315
315
|
assert self.channel is not None
|
|
316
316
|
self.channel.basic_consume(
|
|
@@ -320,11 +320,11 @@ class ListenQueue(threading.Thread):
|
|
|
320
320
|
except (KeyboardInterrupt, SystemExit):
|
|
321
321
|
self.channel.stop_consuming()
|
|
322
322
|
self.pika_connection.close()
|
|
323
|
-
|
|
323
|
+
self.helper.connector_logger.info("Connector stop")
|
|
324
324
|
sys.exit(0)
|
|
325
325
|
except Exception as err: # pylint: disable=broad-except
|
|
326
326
|
self.pika_connection.close()
|
|
327
|
-
|
|
327
|
+
self.helper.connector_logger.error(str(err))
|
|
328
328
|
sys.exit(1)
|
|
329
329
|
|
|
330
330
|
def stop(self):
|
|
@@ -335,8 +335,11 @@ class ListenQueue(threading.Thread):
|
|
|
335
335
|
|
|
336
336
|
|
|
337
337
|
class PingAlive(threading.Thread):
|
|
338
|
-
def __init__(
|
|
338
|
+
def __init__(
|
|
339
|
+
self, connector_logger, connector_id, api, get_state, set_state, metric
|
|
340
|
+
) -> None:
|
|
339
341
|
threading.Thread.__init__(self)
|
|
342
|
+
self.connector_logger = connector_logger
|
|
340
343
|
self.connector_id = connector_id
|
|
341
344
|
self.in_error = False
|
|
342
345
|
self.api = api
|
|
@@ -358,38 +361,39 @@ class PingAlive(threading.Thread):
|
|
|
358
361
|
)
|
|
359
362
|
if initial_state != remote_state:
|
|
360
363
|
self.set_state(result["connector_state"])
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
self.get_state(),
|
|
364
|
+
self.connector_logger.info(
|
|
365
|
+
"Connector state has been remotely reset",
|
|
366
|
+
{"state": self.get_state()},
|
|
364
367
|
)
|
|
365
368
|
if self.in_error:
|
|
366
369
|
self.in_error = False
|
|
367
|
-
|
|
370
|
+
self.connector_logger.error("API Ping back to normal")
|
|
368
371
|
self.metric.inc("ping_api_count")
|
|
369
372
|
except Exception: # pylint: disable=broad-except
|
|
370
373
|
self.in_error = True
|
|
371
374
|
self.metric.inc("ping_api_error")
|
|
372
|
-
|
|
375
|
+
self.connector_logger.error("Error pinging the API")
|
|
373
376
|
self.exit_event.wait(40)
|
|
374
377
|
|
|
375
378
|
def run(self) -> None:
|
|
376
|
-
|
|
379
|
+
self.connector_logger.info("Starting ping alive thread")
|
|
377
380
|
self.ping()
|
|
378
381
|
|
|
379
382
|
def stop(self) -> None:
|
|
380
|
-
|
|
383
|
+
self.connector_logger.info("Preparing for clean shutdown")
|
|
381
384
|
self.exit_event.set()
|
|
382
385
|
|
|
383
386
|
|
|
384
387
|
class StreamAlive(threading.Thread):
|
|
385
|
-
def __init__(self, q) -> None:
|
|
388
|
+
def __init__(self, helper, q) -> None:
|
|
386
389
|
threading.Thread.__init__(self)
|
|
390
|
+
self.helper = helper
|
|
387
391
|
self.q = q
|
|
388
392
|
self.exit_event = threading.Event()
|
|
389
393
|
|
|
390
394
|
def run(self) -> None:
|
|
391
395
|
try:
|
|
392
|
-
|
|
396
|
+
self.helper.connector_logger.info("Starting stream alive thread")
|
|
393
397
|
time_since_last_heartbeat = 0
|
|
394
398
|
while not self.exit_event.is_set():
|
|
395
399
|
time.sleep(5)
|
|
@@ -399,7 +403,7 @@ class StreamAlive(threading.Thread):
|
|
|
399
403
|
except queue.Empty:
|
|
400
404
|
time_since_last_heartbeat = time_since_last_heartbeat + 5
|
|
401
405
|
if time_since_last_heartbeat > 45:
|
|
402
|
-
|
|
406
|
+
self.helper.connector_logger.error(
|
|
403
407
|
"Time since last heartbeat exceeded 45s, stopping the connector"
|
|
404
408
|
)
|
|
405
409
|
break
|
|
@@ -408,7 +412,7 @@ class StreamAlive(threading.Thread):
|
|
|
408
412
|
sys.excepthook(*sys.exc_info())
|
|
409
413
|
|
|
410
414
|
def stop(self) -> None:
|
|
411
|
-
|
|
415
|
+
self.helper.connector_logger.info("Preparing for clean shutdown")
|
|
412
416
|
self.exit_event.set()
|
|
413
417
|
|
|
414
418
|
|
|
@@ -474,7 +478,7 @@ class ListenStream(threading.Thread):
|
|
|
474
478
|
|
|
475
479
|
# Start the stream alive watchdog
|
|
476
480
|
q = Queue(maxsize=1)
|
|
477
|
-
stream_alive = StreamAlive(q)
|
|
481
|
+
stream_alive = StreamAlive(self.helper, q)
|
|
478
482
|
stream_alive.start()
|
|
479
483
|
# Computing args building
|
|
480
484
|
live_stream_url = self.url
|
|
@@ -491,10 +495,14 @@ class ListenStream(threading.Thread):
|
|
|
491
495
|
listen_delete = str(self.listen_delete).lower()
|
|
492
496
|
no_dependencies = str(self.no_dependencies).lower()
|
|
493
497
|
with_inferences = str(self.with_inferences).lower()
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
+
self.helper.connector_logger.info(
|
|
499
|
+
"Starting to listen stream events",
|
|
500
|
+
{
|
|
501
|
+
"live_stream_url": live_stream_url,
|
|
502
|
+
"listen_delete": listen_delete,
|
|
503
|
+
"no_dependencies": no_dependencies,
|
|
504
|
+
"with_inferences": with_inferences,
|
|
505
|
+
},
|
|
498
506
|
)
|
|
499
507
|
messages = SSEClient(
|
|
500
508
|
live_stream_url,
|
|
@@ -660,10 +668,6 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
660
668
|
metrics_port = get_config_variable(
|
|
661
669
|
"CONNECTOR_METRICS_PORT", ["connector", "metrics_port"], config, True, 9095
|
|
662
670
|
)
|
|
663
|
-
self.metric = OpenCTIMetricHandler(expose_metrics, metrics_port)
|
|
664
|
-
|
|
665
|
-
# Configure logger
|
|
666
|
-
logging.basicConfig(level=self.log_level)
|
|
667
671
|
|
|
668
672
|
# Initialize configuration
|
|
669
673
|
# - Classic API that will be directly attached to the connector rights
|
|
@@ -681,7 +685,17 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
681
685
|
self.log_level,
|
|
682
686
|
json_logging=self.opencti_json_logging,
|
|
683
687
|
)
|
|
684
|
-
|
|
688
|
+
self.connector_logger = self.api.logger_class(self.connect_name)
|
|
689
|
+
# For retro compatibility
|
|
690
|
+
self.log_debug = self.connector_logger.debug
|
|
691
|
+
self.log_info = self.connector_logger.info
|
|
692
|
+
self.log_warning = self.connector_logger.warning
|
|
693
|
+
self.log_error = self.connector_logger.warning
|
|
694
|
+
# For retro compatibility
|
|
695
|
+
|
|
696
|
+
self.metric = OpenCTIMetricHandler(
|
|
697
|
+
self.connector_logger, expose_metrics, metrics_port
|
|
698
|
+
)
|
|
685
699
|
# Register the connector in OpenCTI
|
|
686
700
|
self.connector = OpenCTIConnector(
|
|
687
701
|
self.connect_id,
|
|
@@ -693,7 +707,9 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
693
707
|
playbook_compatible,
|
|
694
708
|
)
|
|
695
709
|
connector_configuration = self.api.connector.register(self.connector)
|
|
696
|
-
|
|
710
|
+
self.connector_logger.info(
|
|
711
|
+
"Connector registered with ID", {"id": self.connect_id}
|
|
712
|
+
)
|
|
697
713
|
self.connector_id = connector_configuration["id"]
|
|
698
714
|
self.work_id = None
|
|
699
715
|
self.playbook = None
|
|
@@ -704,7 +720,12 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
704
720
|
# Start ping thread
|
|
705
721
|
if not self.connect_run_and_terminate:
|
|
706
722
|
self.ping = PingAlive(
|
|
707
|
-
self.
|
|
723
|
+
self.connector_logger,
|
|
724
|
+
self.connector.id,
|
|
725
|
+
self.api,
|
|
726
|
+
self.get_state,
|
|
727
|
+
self.set_state,
|
|
728
|
+
self.metric,
|
|
708
729
|
)
|
|
709
730
|
self.ping.start()
|
|
710
731
|
|
|
@@ -799,7 +820,7 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
799
820
|
self.api.connector.ping(self.connector_id, initial_state)
|
|
800
821
|
except Exception: # pylint: disable=broad-except
|
|
801
822
|
self.metric.inc("error_count")
|
|
802
|
-
|
|
823
|
+
self.connector_logger.error("Error pinging the API")
|
|
803
824
|
|
|
804
825
|
def listen(self, message_callback: Callable[[Dict], str]) -> None:
|
|
805
826
|
"""listen for messages and register callback function
|
|
@@ -906,18 +927,6 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
906
927
|
def get_connector(self) -> OpenCTIConnector:
|
|
907
928
|
return self.connector
|
|
908
929
|
|
|
909
|
-
def log_error(self, msg: str) -> None:
|
|
910
|
-
LOGGER.error(msg)
|
|
911
|
-
|
|
912
|
-
def log_info(self, msg: str) -> None:
|
|
913
|
-
LOGGER.info(msg)
|
|
914
|
-
|
|
915
|
-
def log_debug(self, msg: str) -> None:
|
|
916
|
-
LOGGER.debug(msg)
|
|
917
|
-
|
|
918
|
-
def log_warning(self, msg: str) -> None:
|
|
919
|
-
LOGGER.warning(msg)
|
|
920
|
-
|
|
921
930
|
def date_now(self) -> str:
|
|
922
931
|
"""get the current date (UTC)
|
|
923
932
|
:return: current datetime for utc
|
|
@@ -1018,7 +1027,7 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
1018
1027
|
try:
|
|
1019
1028
|
channel.confirm_delivery()
|
|
1020
1029
|
except Exception as err: # pylint: disable=broad-except
|
|
1021
|
-
|
|
1030
|
+
self.connector_logger.warning(str(err))
|
|
1022
1031
|
for sequence, bundle in enumerate(bundles, start=1):
|
|
1023
1032
|
self._send_bundle(
|
|
1024
1033
|
channel,
|
|
@@ -1082,10 +1091,10 @@ class OpenCTIConnectorHelper: # pylint: disable=too-many-public-methods
|
|
|
1082
1091
|
delivery_mode=2, content_encoding="utf-8" # make message persistent
|
|
1083
1092
|
),
|
|
1084
1093
|
)
|
|
1085
|
-
|
|
1094
|
+
self.connector_logger.debug("Bundle has been sent")
|
|
1086
1095
|
self.metric.inc("bundle_send")
|
|
1087
|
-
except (UnroutableError, NackError)
|
|
1088
|
-
|
|
1096
|
+
except (UnroutableError, NackError):
|
|
1097
|
+
self.connector_logger.error("Unable to send bundle, retry...")
|
|
1089
1098
|
self.metric.inc("error_count")
|
|
1090
1099
|
self._send_bundle(channel, bundle, **kwargs)
|
|
1091
1100
|
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import logging
|
|
2
1
|
from typing import Type, Union
|
|
3
2
|
|
|
4
3
|
from prometheus_client import Counter, Enum, start_http_server
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
class OpenCTIMetricHandler:
|
|
8
|
-
def __init__(self, activated: bool = False, port: int = 9095):
|
|
7
|
+
def __init__(self, connector_logger, activated: bool = False, port: int = 9095):
|
|
9
8
|
"""
|
|
10
9
|
Init of OpenCTIMetricHandler class.
|
|
11
10
|
|
|
@@ -17,8 +16,9 @@ class OpenCTIMetricHandler:
|
|
|
17
16
|
Port for prometheus server.
|
|
18
17
|
"""
|
|
19
18
|
self.activated = activated
|
|
19
|
+
self.connector_logger = connector_logger
|
|
20
20
|
if self.activated:
|
|
21
|
-
|
|
21
|
+
self.connector_logger.info("Exposing metrics on port", {"port": port})
|
|
22
22
|
start_http_server(port)
|
|
23
23
|
self._metrics = {
|
|
24
24
|
"bundle_send": Counter(
|
|
@@ -75,10 +75,13 @@ class OpenCTIMetricHandler:
|
|
|
75
75
|
True if the metric exists and is of the correct type else False.
|
|
76
76
|
"""
|
|
77
77
|
if name not in self._metrics:
|
|
78
|
-
|
|
78
|
+
self.connector_logger.error("Metric does not exist.", {"name": name})
|
|
79
79
|
return False
|
|
80
80
|
if not isinstance(self._metrics[name], expected_type):
|
|
81
|
-
|
|
81
|
+
self.connector_logger.error(
|
|
82
|
+
"Metric not of expected type",
|
|
83
|
+
{"name": name, "expected_type": expected_type},
|
|
84
|
+
)
|
|
82
85
|
return False
|
|
83
86
|
return True
|
|
84
87
|
|
|
File without changes
|