txt2detection 1.0.7__py3-none-any.whl → 1.0.9__py3-none-any.whl

txt2detection/bundler.py

@@ -15,8 +15,15 @@ from stix2 import (
 from stix2.serialization import serialize
 import hashlib
 
-from txt2detection import observables
-from txt2detection.models import AIDetection, BaseDetection, DetectionContainer, UUID_NAMESPACE, SigmaRuleDetection
+from txt2detection import attack_flow, observables
+from txt2detection.models import (
+    AIDetection,
+    BaseDetection,
+    DataContainer,
+    DetectionContainer,
+    UUID_NAMESPACE,
+    SigmaRuleDetection,
+)
 
 from datetime import UTC, datetime as dt
 import uuid
@@ -28,56 +35,57 @@ from txt2detection.utils import STATUSES, remove_rule_specific_tags
 
 logger = logging.getLogger("txt2detection.bundler")
 
+
 class Bundler:
     identity = None
     object_marking_refs = []
     uuid = None
     id_map = dict()
-    detections: DetectionContainer
+    data: DataContainer
     # https://raw.githubusercontent.com/muchdogesec/stix4doge/refs/heads/main/objects/identity/txt2detection.json
-    default_identity = Identity(**{
-        "type": "identity",
-        "spec_version": "2.1",
-        "id": "identity--a4d70b75-6f4a-5d19-9137-da863edd33d7",
-        "created_by_ref": "identity--9779a2db-f98c-5f4b-8d08-8ee04e02dbb5",
-        "created": "2020-01-01T00:00:00.000Z",
-        "modified": "2020-01-01T00:00:00.000Z",
-        "name": "txt2detection",
-        "description": "https://github.com/muchdogesec/txt2detection",
-        "identity_class": "system",
-        "sectors": [
-            "technology"
-        ],
-        "contact_information": "https://www.dogesec.com/contact/",
-        "object_marking_refs": [
-            "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
-            "marking-definition--97ba4e8b-04f6-57e8-8f6e-3a0f0a7dc0fb"
-        ]
-    })
+    default_identity = Identity(
+        **{
+            "type": "identity",
+            "spec_version": "2.1",
+            "id": "identity--a4d70b75-6f4a-5d19-9137-da863edd33d7",
+            "created_by_ref": "identity--9779a2db-f98c-5f4b-8d08-8ee04e02dbb5",
+            "created": "2020-01-01T00:00:00.000Z",
+            "modified": "2020-01-01T00:00:00.000Z",
+            "name": "txt2detection",
+            "description": "https://github.com/muchdogesec/txt2detection",
+            "identity_class": "system",
+            "sectors": ["technology"],
+            "contact_information": "https://www.dogesec.com/contact/",
+            "object_marking_refs": [
+                "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
+                "marking-definition--97ba4e8b-04f6-57e8-8f6e-3a0f0a7dc0fb",
+            ],
+        }
+    )
     # https://raw.githubusercontent.com/muchdogesec/stix4doge/refs/heads/main/objects/marking-definition/txt2detection.json
-    default_marking = MarkingDefinition(**{
-        "type": "marking-definition",
-        "spec_version": "2.1",
-        "id": "marking-definition--a4d70b75-6f4a-5d19-9137-da863edd33d7",
-        "created_by_ref": "identity--9779a2db-f98c-5f4b-8d08-8ee04e02dbb5",
-        "created": "2020-01-01T00:00:00.000Z",
-        "definition_type": "statement",
-        "definition": {
-            "statement": "This object was created using: https://github.com/muchdogesec/txt2detection"
-        },
-        "object_marking_refs": [
-            "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
-            "marking-definition--97ba4e8b-04f6-57e8-8f6e-3a0f0a7dc0fb"
-        ]
-    })
+    default_marking = MarkingDefinition(
+        **{
+            "type": "marking-definition",
+            "spec_version": "2.1",
+            "id": "marking-definition--a4d70b75-6f4a-5d19-9137-da863edd33d7",
+            "created_by_ref": "identity--9779a2db-f98c-5f4b-8d08-8ee04e02dbb5",
+            "created": "2020-01-01T00:00:00.000Z",
+            "definition_type": "statement",
+            "definition": {
+                "statement": "This object was created using: https://github.com/muchdogesec/txt2detection"
+            },
+            "object_marking_refs": [
+                "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
+                "marking-definition--97ba4e8b-04f6-57e8-8f6e-3a0f0a7dc0fb",
+            ],
+        }
+    )
 
     @classmethod
     def generate_report_id(cls, created_by_ref, created, name):
         if not created_by_ref:
-            created_by_ref = cls.default_identity['id']
-        return str(
-            uuid.uuid5(UUID_NAMESPACE, f"{created_by_ref}+{created}+{name}")
-        )
+            created_by_ref = cls.default_identity["id"]
+        return str(uuid.uuid5(UUID_NAMESPACE, f"{created_by_ref}+{created}+{name}"))
 
     def __init__(
         self,
@@ -89,7 +97,7 @@ class Bundler:
         created=None,
         modified=None,
         report_id=None,
-        external_refs: list=None,
+        external_refs: list = None,
         reference_urls=None,
         license=None,
         **kwargs,
@@ -97,15 +105,25 @@ class Bundler:
         self.created = created or dt.now(UTC)
         self.modified = modified or self.created
         self.identity = identity or self.default_identity
-        self.tlp_level = TLP_LEVEL.get(tlp_level or 'clear')
-        self.uuid = report_id or self.generate_report_id(self.identity.id, self.created, name)
+        self.tlp_level = TLP_LEVEL.get(tlp_level or "clear")
+        self.uuid = report_id or self.generate_report_id(
+            self.identity.id, self.created, name
+        )
         self.reference_urls = reference_urls or []
         self.labels = labels or []
         self.license = license
 
         self.job_id = f"report--{self.uuid}"
-        self.external_refs = (external_refs or []) + [dict(source_name='txt2detection', url=url, description='txt2detection-reference') for url in self.reference_urls]
-        
+        self.external_refs = (external_refs or []) + [
+            dict(
+                source_name="txt2detection",
+                url=url,
+                description="txt2detection-reference",
+            )
+            for url in self.reference_urls
+        ]
+        self.data = DataContainer.model_construct()
+
         self.report = Report(
             created_by_ref=self.identity.id,
             name=name,
@@ -124,7 +142,8 @@ class Bundler:
                     source_name="description_md5_hash",
                     external_id=hashlib.md5((description or "").encode()).hexdigest(),
                 )
-            ] + self.external_refs,
+            ]
+            + self.external_refs,
         )
         self.report.object_refs.clear()  # clear object refs
         self.set_defaults()
@@ -150,13 +169,15 @@ class Bundler:
         self.all_objects.add(sdo_id)
 
     def add_rule_indicator(self, detection: SigmaRuleDetection):
-        indicator_types = getattr(detection, 'indicator_types', None)
+        indicator_types = getattr(detection, "indicator_types", None)
         if isinstance(detection, AIDetection):
             detection = detection.to_sigma_rule_detection(self)
-        assert isinstance(detection, SigmaRuleDetection), f"detection of type {type(detection)} not supported"
+        assert isinstance(
+            detection, SigmaRuleDetection
+        ), f"detection of type {type(detection)} not supported"
         indicator = {
             "type": "indicator",
-            "id": "indicator--"+str(detection.detection_id),
+            "id": "indicator--" + str(detection.detection_id),
             "spec_version": "2.1",
             "created_by_ref": self.report.created_by_ref,
             "created": self.report.created,
@@ -165,62 +186,90 @@ class Bundler:
             "name": detection.title,
             "description": detection.description,
             "labels": remove_rule_specific_tags(self.labels),
-            "pattern_type": 'sigma',
+            "pattern_type": "sigma",
             "pattern": detection.make_rule(self),
             "valid_from": self.report.created,
             "object_marking_refs": self.report.object_marking_refs,
             "external_references": self.external_refs + detection.external_references,
         }
-        indicator['external_references'].append(
+        indicator["external_references"].append(
             {
-            "source_name": "rule_md5_hash",
-            "external_id": hashlib.md5(indicator['pattern'].encode()).hexdigest()
+                "source_name": "rule_md5_hash",
+                "external_id": hashlib.md5(indicator["pattern"].encode()).hexdigest(),
             }
         )
+        logsource = detection.make_data_source()
 
         logger.debug(f"===== rule {detection.detection_id} =====")
-        logger.debug("```yaml\n"+indicator['pattern']+"\n```")
+        logger.debug("```yaml\n" + indicator["pattern"] + "\n```")
         logger.debug(f" =================== end of rule =================== ")
 
+        self.data.attacks = dict.fromkeys(detection.mitre_attack_ids, "Not found")
         for obj in self.get_attack_objects(detection.mitre_attack_ids):
             self.add_ref(obj)
             self.add_relation(indicator, obj)
+            self.data.attacks[obj["external_references"][0]["external_id"]] = obj["id"]
 
+        self.data.cves = dict.fromkeys(detection.cve_ids, "Not found")
         for obj in self.get_cve_objects(detection.cve_ids):
             self.add_ref(obj)
             self.add_relation(indicator, obj)
+            self.data.cves[obj["name"]] = obj["id"]
 
         self.add_ref(parse_stix(indicator, allow_custom=True), append_report=True)
+        self.add_ref(logsource, append_report=True)
+        self.add_relation(
+            indicator,
+            logsource,
+            description=f'{indicator["name"]} is created from {make_logsouce_string(logsource)}',
+        )
 
-        for ob_type, ob_value in set(observables.find_stix_observables(detection.detection)):
+        self.data.observables = []
+        for ob_type, ob_value in set(
+            observables.find_stix_observables(detection.detection)
+        ):
+            self.data.observables.append(dict(type=ob_type, value=ob_value))
             try:
                 obj = observables.to_stix_object(ob_type, ob_value)
                 self.add_ref(obj)
-                self.add_relation(indicator, obj, 'detects', target_name=ob_value)
-            except:
+                self.add_relation(indicator, obj, "related-to", target_name=ob_value)
+            except Exception as e:
+                self.data.observables[-1]["error"] = str(e)
                 logger.exception(f"failed to process observable {ob_type}/{ob_value}")
 
-
-    def add_relation(self, indicator, target_object, relationship_type='detects', target_name=None):
+    def add_relation(
+        self,
+        indicator,
+        target_object,
+        relationship_type="related-to",
+        target_name=None,
+        description=None,
+    ):
         ext_refs = []
 
         with contextlib.suppress(Exception):
-            indicator['external_references'].append(target_object['external_references'][0])
-            ext_refs = [target_object['external_references'][0]]
-
-        target_name = target_name or f"{target_object['external_references'][0]['external_id']} ({target_object['name']})"
+            indicator["external_references"].append(
+                target_object["external_references"][0]
+            )
+            ext_refs = [target_object["external_references"][0]]
 
-        rel =  Relationship(
-            id="relationship--" + str(
-                uuid.uuid5(
-                    UUID_NAMESPACE, f"{indicator['id']}+{target_object['id']}"
-                )
+        if not description:
+            target_name = (
+                target_name
+                or f"{target_object['external_references'][0]['external_id']} ({target_object['name']})"
+            )
+            description = f"{indicator['name']} {relationship_type} {target_name}"
+
+        rel = Relationship(
+            id="relationship--"
+            + str(
+                uuid.uuid5(UUID_NAMESPACE, f"{indicator['id']}+{target_object['id']}")
             ),
-            source_ref=indicator['id'],
-            target_ref=target_object['id'],
+            source_ref=indicator["id"],
+            target_ref=target_object["id"],
             relationship_type=relationship_type,
             created_by_ref=self.report.created_by_ref,
-            description=f"{indicator['name']} {relationship_type} {target_name}",
+            description=description,
             created=self.report.created,
             modified=self.report.modified,
             object_marking_refs=self.report.object_marking_refs,
@@ -240,44 +289,82 @@ class Bundler:
         if not attack_ids:
             return []
         logger.debug(f"retrieving attack objects: {attack_ids}")
-        endpoint = urljoin(os.environ['CTIBUTLER_BASE_URL'] + '/', f"v1/attack-enterprise/objects/?attack_id="+','.join(attack_ids))
+        endpoint = urljoin(
+            os.environ["CTIBUTLER_BASE_URL"] + "/",
+            f"v1/attack-enterprise/objects/?attack_id=" + ",".join(attack_ids),
+        )
 
         headers = {}
-        if api_key := os.environ.get('CTIBUTLER_API_KEY'):
-            headers['API-KEY'] = api_key
+        if api_key := os.environ.get("CTIBUTLER_API_KEY"):
+            headers["API-KEY"] = api_key
 
         return self._get_objects(endpoint, headers)
 
-    def get_cve_objects(self, cve_ids):
+    @classmethod
+    def get_attack_tactics(cls):
+        headers = {}
+        api_root = os.environ["CTIBUTLER_BASE_URL"] + "/"
+        if api_key := os.environ.get("CTIBUTLER_API_KEY"):
+            headers["API-KEY"] = api_key
+
+        endpoint = urljoin(
+            api_root, f"v1/attack-enterprise/objects/?attack_type=Tactic"
+        )
+        version_url = urljoin(api_root, f"v1/attack-enterprise/versions/installed/")
+        tactics = cls._get_objects(endpoint, headers=headers)
+        retval = dict(
+            version=requests.get(version_url, headers=headers).json()["latest"]
+        )
+        for tac in tactics:
+            retval[tac["x_mitre_shortname"]] = tac
+            retval[tac["external_references"][0]["external_id"]] = tac
+        return retval
+
+    @classmethod
+    def get_cve_objects(cls, cve_ids):
         if not cve_ids:
             return []
         logger.debug(f"retrieving cve objects: {cve_ids}")
-        endpoint = urljoin(os.environ['VULMATCH_BASE_URL'] + '/', f"v1/cve/objects/?cve_id="+','.join(cve_ids))
+        endpoint = urljoin(
+            os.environ["VULMATCH_BASE_URL"] + "/",
+            f"v1/cve/objects/?cve_id=" + ",".join(cve_ids),
+        )
         headers = {}
-        if api_key := os.environ.get('VULMATCH_API_KEY'):
-            headers['API-KEY'] = api_key
+        if api_key := os.environ.get("VULMATCH_API_KEY"):
+            headers["API-KEY"] = api_key
 
-        return self._get_objects(endpoint, headers)
+        return cls._get_objects(endpoint, headers)
 
-    def _get_objects(self, endpoint, headers):
+    @classmethod
+    def _get_objects(cls, endpoint, headers):
         data = []
         page = 1
         while True:
-            resp = requests.get(endpoint, params=dict(page=page, page_size=1000), headers=headers)
+            resp = requests.get(
+                endpoint, params=dict(page=page, page_size=1000), headers=headers
+            )
             if resp.status_code != 200:
                 break
             d = resp.json()
-            if len(d['objects']) == 0:
+            if len(d["objects"]) == 0:
                 break
-            data.extend(d['objects'])
-            page+=1
-            if d['page_results_count'] < d['page_size']:
+            data.extend(d["objects"])
+            page += 1
+            if d["page_results_count"] < d["page_size"]:
                 break
         return data
 
     def bundle_detections(self, container: DetectionContainer):
-        self.detections = container
+        self.data = DataContainer(detections=container)
         if not container.success:
             return
         for d in container.detections:
             self.add_rule_indicator(d)
+
+
+def make_logsouce_string(source: dict):
+    d = [
+        f"{k}={v}" for k, v in source.items() if k in ["product", "service", "category"]
+    ]
+    d_str = ", ".join(d)
+    return "log-source {" + d_str + "}"

txt2detection/credential_checker.py

@@ -5,7 +5,6 @@ from urllib.parse import urljoin
 import requests
 
 
-
 def check_llms():
     from txt2detection.__main__ import parse_model
 
@@ -23,14 +22,17 @@ def check_llms():
 
 def check_ctibutler_vulmatch(service):
     session = requests.Session()
-    if service == 'vulmatch':
-        base_url = os.getenv('VULMATCH_BASE_URL')
-        url = urljoin(base_url, 'v1/cve/objects/vulnerability--f552f6f4-39da-48dc-8717-323772c99588/')
-        session.headers['API-KEY'] = os.environ.get('VULMATCH_API_KEY')
-    elif service == 'ctibutler':
-        base_url = os.getenv('CTIBUTLER_BASE_URL')
-        url = urljoin(base_url, 'v1/location/versions/available/')
-        session.headers['API-KEY'] = os.environ.get('CTIBUTLER_API_KEY')
+    if service == "vulmatch":
+        base_url = os.getenv("VULMATCH_BASE_URL")
+        url = urljoin(
+            base_url,
+            "v1/cve/objects/vulnerability--f552f6f4-39da-48dc-8717-323772c99588/",
+        )
+        session.headers["API-KEY"] = os.environ.get("VULMATCH_API_KEY")
+    elif service == "ctibutler":
+        base_url = os.getenv("CTIBUTLER_BASE_URL")
+        url = urljoin(base_url, "v1/location/versions/available/")
+        session.headers["API-KEY"] = os.environ.get("CTIBUTLER_API_KEY")
 
     try:
         resp = session.get(url)