txt2detection 0.0.2rc12__py3-none-any.whl

txt2detection/bundler.py

@@ -0,0 +1,283 @@
+import contextlib
+import enum
+import json
+import logging
+import os
+from urllib.parse import urljoin
+import requests
+from stix2 import (
+    Report,
+    Identity,
+    MarkingDefinition,
+    Relationship,
+    Bundle,
+)
+from stix2.serialization import serialize
+import hashlib
+
+from txt2detection import observables
+from txt2detection.models import AIDetection, BaseDetection, DetectionContainer, UUID_NAMESPACE, SigmaRuleDetection
+
+from datetime import UTC, datetime as dt
+import uuid
+from stix2 import parse as parse_stix
+
+from txt2detection.models import TLP_LEVEL
+from txt2detection.utils import STATUSES, remove_rule_specific_tags
+
+
+logger = logging.getLogger("txt2detection.bundler")
+
+class Bundler:
+    identity = None
+    object_marking_refs = []
+    uuid = None
+    id_map = dict()
+    detections: DetectionContainer
+    # https://raw.githubusercontent.com/muchdogesec/stix4doge/refs/heads/main/objects/identity/txt2detection.json
+    default_identity = Identity(**{
+        "type": "identity",
+        "spec_version": "2.1",
+        "id": "identity--a4d70b75-6f4a-5d19-9137-da863edd33d7",
+        "created_by_ref": "identity--9779a2db-f98c-5f4b-8d08-8ee04e02dbb5",
+        "created": "2020-01-01T00:00:00.000Z",
+        "modified": "2020-01-01T00:00:00.000Z",
+        "name": "txt2detection",
+        "description": "https://github.com/muchdogesec/txt2detection",
+        "identity_class": "system",
+        "sectors": [
+            "technology"
+        ],
+        "contact_information": "https://www.dogesec.com/contact/",
+        "object_marking_refs": [
+            "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
+            "marking-definition--97ba4e8b-04f6-57e8-8f6e-3a0f0a7dc0fb"
+        ]
+    })
+    # https://raw.githubusercontent.com/muchdogesec/stix4doge/refs/heads/main/objects/marking-definition/txt2detection.json
+    default_marking = MarkingDefinition(**{
+        "type": "marking-definition",
+        "spec_version": "2.1",
+        "id": "marking-definition--a4d70b75-6f4a-5d19-9137-da863edd33d7",
+        "created_by_ref": "identity--9779a2db-f98c-5f4b-8d08-8ee04e02dbb5",
+        "created": "2020-01-01T00:00:00.000Z",
+        "definition_type": "statement",
+        "definition": {
+            "statement": "This object was created using: https://github.com/muchdogesec/txt2detection"
+        },
+        "object_marking_refs": [
+            "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
+            "marking-definition--97ba4e8b-04f6-57e8-8f6e-3a0f0a7dc0fb"
+        ]
+    })
+
+    @classmethod
+    def generate_report_id(cls, created_by_ref, created, name):
+        if not created_by_ref:
+            created_by_ref = cls.default_identity['id']
+        return str(
+            uuid.uuid5(UUID_NAMESPACE, f"{created_by_ref}+{created}+{name}")
+        )
+
+    def __init__(
+        self,
+        name,
+        identity,
+        tlp_level,
+        description,
+        labels,
+        created=None,
+        modified=None,
+        report_id=None,
+        external_refs: list=None,
+        reference_urls=None,
+        license=None,
+        **kwargs,
+    ) -> None:
+        self.created = created or dt.now(UTC)
+        self.modified = modified or self.created
+        self.identity = identity or self.default_identity
+        self.tlp_level = TLP_LEVEL.get(tlp_level or 'clear')
+        self.uuid = report_id or self.generate_report_id(self.identity.id, self.created, name)
+        self.reference_urls = reference_urls or []
+        self.labels = labels or []
+        self.license = license
+
+        self.job_id = f"report--{self.uuid}"
+        self.external_refs = (external_refs or []) + [dict(source_name='txt2detection', url=url, description='txt2detection-reference') for url in self.reference_urls]
+        
+        self.report = Report(
+            created_by_ref=self.identity.id,
+            name=name,
+            id=self.job_id,
+            description=description,
+            object_refs=[
+                f"note--{self.uuid}"
+            ],  # won't allow creation with empty object_refs
+            created=self.created,
+            modified=self.modified,
+            object_marking_refs=[self.tlp_level.value.id],
+            labels=remove_rule_specific_tags(self.labels),
+            published=self.created,
+            external_references=[
+                dict(
+                    source_name="description_md5_hash",
+                    external_id=hashlib.md5((description or "").encode()).hexdigest(),
+                )
+            ] + self.external_refs,
+        )
+        self.report.object_refs.clear()  # clear object refs
+        self.set_defaults()
+        self.all_objects = set()
+        if not description:
+            self.report.external_references.pop(0)
+
+    def set_defaults(self):
+        # self.value.extend(TLP_LEVEL.values()) # adds all tlp levels
+        self.bundle = Bundle(objects=[self.tlp_level.value], id=f"bundle--{self.uuid}")
+
+        self.bundle.objects.extend([self.default_marking, self.identity, self.report])
+        # add default STIX 2.1 marking definition for txt2detection
+        self.report.object_marking_refs.append(self.default_marking.id)
+
+    def add_ref(self, sdo, append_report=False):
+        sdo_id = sdo["id"]
+        if sdo_id in self.all_objects:
+            return
+        self.bundle.objects.append(sdo)
+        if sdo_id not in self.report.object_refs and append_report:
+            self.report.object_refs.append(sdo_id)
+        self.all_objects.add(sdo_id)
+
+    def add_rule_indicator(self, detection: SigmaRuleDetection):
+        indicator_types = getattr(detection, 'indicator_types', None)
+        if isinstance(detection, AIDetection):
+            detection = detection.to_sigma_rule_detection(self)
+        assert isinstance(detection, SigmaRuleDetection), f"detection of type {type(detection)} not supported"
+        indicator = {
+            "type": "indicator",
+            "id": "indicator--"+str(detection.detection_id),
+            "spec_version": "2.1",
+            "created_by_ref": self.report.created_by_ref,
+            "created": self.report.created,
+            "modified": self.report.modified,
+            "indicator_types": indicator_types,
+            "name": detection.title,
+            "description": detection.description,
+            "labels": remove_rule_specific_tags(self.labels),
+            "pattern_type": 'sigma',
+            "pattern": detection.make_rule(self),
+            "valid_from": self.report.created,
+            "object_marking_refs": self.report.object_marking_refs,
+            "external_references": self.external_refs + detection.external_references,
+        }
+        indicator['external_references'].append(
+            {
+            "source_name": "rule_md5_hash",
+            "external_id": hashlib.md5(indicator['pattern'].encode()).hexdigest()
+            }
+        )
+
+        logger.debug(f"===== rule {detection.detection_id} =====")
+        logger.debug("```yaml\n"+indicator['pattern']+"\n```")
+        logger.debug(f" =================== end of rule =================== ")
+
+        for obj in self.get_attack_objects(detection.mitre_attack_ids):
+            self.add_ref(obj)
+            self.add_relation(indicator, obj)
+
+        for obj in self.get_cve_objects(detection.cve_ids):
+            self.add_ref(obj)
+            self.add_relation(indicator, obj)
+
+        self.add_ref(parse_stix(indicator, allow_custom=True), append_report=True)
+
+        for ob_type, ob_value in set(observables.find_stix_observables(detection.detection)):
+            try:
+                obj = observables.to_stix_object(ob_type, ob_value)
+                self.add_ref(obj)
+                self.add_relation(indicator, obj, 'detects', target_name=ob_value)
+            except:
+                logger.exception(f"failed to process observable {ob_type}/{ob_value}")
+
+
+    def add_relation(self, indicator, target_object, relationship_type='detects', target_name=None):
+        ext_refs = []
+
+        with contextlib.suppress(Exception):
+            indicator['external_references'].append(target_object['external_references'][0])
+            ext_refs = [target_object['external_references'][0]]
+
+        target_name = target_name or f"{target_object['external_references'][0]['external_id']} ({target_object['name']})"
+
+        rel =  Relationship(
+            id="relationship--" + str(
+                uuid.uuid5(
+                    UUID_NAMESPACE, f"{indicator['id']}+{target_object['id']}"
+                )
+            ),
+            source_ref=indicator['id'],
+            target_ref=target_object['id'],
+            relationship_type=relationship_type,
+            created_by_ref=self.report.created_by_ref,
+            description=f"{indicator['name']} {relationship_type} {target_name}",
+            created=self.report.created,
+            modified=self.report.modified,
+            object_marking_refs=self.report.object_marking_refs,
+            external_references=ext_refs,
+            allow_custom=True,
+        )
+        self.add_ref(rel)
+
+    def to_json(self):
+        return serialize(self.bundle, indent=4)
+
+    @property
+    def bundle_dict(self):
+        return json.loads(self.to_json())
+
+    def get_attack_objects(self, attack_ids):
+        if not attack_ids:
+            return []
+        logger.debug(f"retrieving attack objects: {attack_ids}")
+        endpoint = urljoin(os.environ['CTIBUTLER_BASE_URL'] + '/', f"v1/attack-enterprise/objects/?attack_id="+','.join(attack_ids))
+
+        headers = {}
+        if api_key := os.environ.get('CTIBUTLER_API_KEY'):
+            headers['Authorization'] = "Bearer " + api_key
+
+        return self._get_objects(endpoint, headers)
+
+    def get_cve_objects(self, cve_ids):
+        if not cve_ids:
+            return []
+        logger.debug(f"retrieving cve objects: {cve_ids}")
+        endpoint = urljoin(os.environ['VULMATCH_BASE_URL'] + '/', f"v1/cve/objects/?cve_id="+','.join(cve_ids))
+        headers = {}
+        if api_key := os.environ.get('VULMATCH_API_KEY'):
+            headers['Authorization'] = "Bearer " + api_key
+
+        return self._get_objects(endpoint, headers)
+
+    def _get_objects(self, endpoint, headers):
+        data = []
+        page = 1
+        while True:
+            resp = requests.get(endpoint, params=dict(page=page, page_size=1000), headers=headers)
+            if resp.status_code != 200:
+                break
+            d = resp.json()
+            if len(d['objects']) == 0:
+                break
+            data.extend(d['objects'])
+            page+=1
+            if d['page_results_count'] < d['page_size']:
+                break
+        return data
+
+    def bundle_detections(self, container: DetectionContainer):
+        self.detections = container
+        if not container.success:
+            return
+        for d in container.detections:
+            self.add_rule_indicator(d)

txt2detection/config/detection_languages.yaml

@@ -0,0 +1,14 @@
+# currently only supports Sigma
+## ===== Sigma =====
+
+sigma:
+  type: "detection_language"
+  name: "Sigma"
+  description: "https://sigmahq.io/docs/basics/rules.html"
+  products:
+    -
+  documentation: ""
+  created: 2020-01-01
+  modified:  2020-01-01
+  created_by: DOGESEC
+  version: 1.0.0

txt2detection/models.py

@@ -0,0 +1,391 @@
+import enum
+import json
+import re
+import typing
+import uuid
+from slugify import slugify
+from datetime import date as dt_date
+from typing import Any, List, Literal, Optional, Union
+from uuid import UUID
+
+import jsonschema
+from pydantic import BaseModel, Field, computed_field, field_validator
+from pydantic_core import PydanticCustomError, core_schema
+import yaml
+
+from stix2 import (
+    MarkingDefinition,
+)
+
+if typing.TYPE_CHECKING:
+    from txt2detection.bundler import Bundler
+
+UUID_NAMESPACE = uuid.UUID("a4d70b75-6f4a-5d19-9137-da863edd33d7")
+
+TAG_PATTERN = re.compile(r"^[a-z0-9_-]+\.[a-z0-9._-]+$")
+
+MITRE_TACTIC_MAP = {
+    "initial-access": "TA0001",
+    "execution": "TA0002",
+    "persistence": "TA0003",
+    "privilege-escalation": "TA0004",
+    "defense-evasion": "TA0005",
+    "credential-access": "TA0006",
+    "discovery": "TA0007",
+    "lateral-movement": "TA0008",
+    "collection": "TA0009",
+    "exfiltration": "TA0010",
+    "command-and-control": "TA0011",
+    "impact": "TA0040",
+}
+
+
+class TLP_LEVEL(enum.Enum):
+    CLEAR = MarkingDefinition(
+        spec_version="2.1",
+        id="marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
+        created="2022-10-01T00:00:00.000Z",
+        definition_type="TLP:CLEAR",
+        extensions={
+            "extension-definition--60a3c5c5-0d10-413e-aab3-9e08dde9e88d": {
+                "extension_type": "property-extension",
+                "tlp_2_0": "clear",
+            }
+        },
+    )
+    GREEN = MarkingDefinition(
+        spec_version="2.1",
+        id="marking-definition--bab4a63c-aed9-4cf5-a766-dfca5abac2bb",
+        created="2022-10-01T00:00:00.000Z",
+        definition_type="TLP:GREEN",
+        extensions={
+            "extension-definition--60a3c5c5-0d10-413e-aab3-9e08dde9e88d": {
+                "extension_type": "property-extension",
+                "tlp_2_0": "green",
+            }
+        },
+    )
+    AMBER = MarkingDefinition(
+        spec_version="2.1",
+        id="marking-definition--55d920b0-5e8b-4f79-9ee9-91f868d9b421",
+        created="2022-10-01T00:00:00.000Z",
+        definition_type="TLP:AMBER",
+        extensions={
+            "extension-definition--60a3c5c5-0d10-413e-aab3-9e08dde9e88d": {
+                "extension_type": "property-extension",
+                "tlp_2_0": "amber",
+            }
+        },
+    )
+    AMBER_STRICT = MarkingDefinition(
+        spec_version="2.1",
+        id="marking-definition--939a9414-2ddd-4d32-a0cd-375ea402b003",
+        created="2022-10-01T00:00:00.000Z",
+        definition_type="TLP:AMBER+STRICT",
+        extensions={
+            "extension-definition--60a3c5c5-0d10-413e-aab3-9e08dde9e88d": {
+                "extension_type": "property-extension",
+                "tlp_2_0": "amber+strict",
+            }
+        },
+    )
+    RED = MarkingDefinition(
+        spec_version="2.1",
+        id="marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1",
+        created="2022-10-01T00:00:00.000Z",
+        definition_type="TLP:RED",
+        extensions={
+            "extension-definition--60a3c5c5-0d10-413e-aab3-9e08dde9e88d": {
+                "extension_type": "property-extension",
+                "tlp_2_0": "red",
+            }
+        },
+    )
+
+    @classmethod
+    def levels(cls):
+        return dict(
+            clear=cls.CLEAR,
+            green=cls.GREEN,
+            amber=cls.AMBER,
+            amber_strict=cls.AMBER_STRICT,
+            red=cls.RED,
+        )
+
+    @classmethod
+    def values(cls):
+        return [
+            cls.CLEAR.value,
+            cls.GREEN.value,
+            cls.AMBER.value,
+            cls.AMBER_STRICT.value,
+            cls.RED.value,
+        ]
+
+    @classmethod
+    def get(cls, level: 'str|TLP_LEVEL'):
+        if isinstance(level, cls):
+            return level
+        level = level.lower()
+        level = level.replace('+', '_').replace('-', '_')
+        if level not in cls.levels():
+            raise Exception(f"unsupported tlp level: `{level}`")
+        return cls.levels()[level]
+
+    @property
+    def name(self):
+        return super().name.lower()
+
+class Statuses(enum.StrEnum):
+    stable = enum.auto()
+    test = enum.auto()
+    experimental = enum.auto()
+    deprecated = enum.auto()
+    unsupported = enum.auto()
+
+class Level(enum.StrEnum):
+    informational = enum.auto()
+    low = enum.auto()
+    medium = enum.auto()
+    high = enum.auto()
+    critical = enum.auto()
+
+class SigmaTag(str):
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls,
+        _source: type[Any],
+        _handler,
+    ) -> core_schema.CoreSchema:
+        return core_schema.no_info_after_validator_function(cls._validate, core_schema.str_schema())
+
+    @classmethod
+    def __get_pydantic_json_schema__(
+        cls, core_schema: core_schema.CoreSchema, handler
+    ):
+        field_schema = handler(core_schema)
+        field_schema.update(type='string', pattern=TAG_PATTERN.pattern, format='sigma-tag')
+        return field_schema
+
+    @classmethod
+    def _validate(cls, input_value: str, /) -> str:
+        if not TAG_PATTERN.match(input_value):
+            raise PydanticCustomError(
+            'value_error',
+            'value is not a valid SIGMA tag: {reason}',
+            {'reason': f'Must be in format namespace.value and match pattern {TAG_PATTERN.pattern}'},
+        )
+        return input_value
+    
+class RelatedRule(BaseModel):
+    id: UUID
+    type: Literal[
+        "derived", "obsolete", "merged", "renamed", "similar"
+    ]
+
+class BaseDetection(BaseModel):
+    title: str
+    description: str
+    detection: dict
+    logsource: dict
+    status: Statuses = Statuses.experimental
+    falsepositives: list[str]
+    tags: list[str]
+    level: Level
+    _custom_id = None
+    _extra_data: dict
+
+    def model_post_init(self, __context):
+        self.tags = self.tags or []
+        self._extra_data = dict()
+        return super().model_post_init(__context)
+
+    @property
+    def detection_id(self):
+        return str(self._custom_id or getattr(self, "id", None) or uuid.uuid4())
+
+    @detection_id.setter
+    def detection_id(self, custom_id):
+        self._custom_id = custom_id.split("--")[-1]
+
+    @property
+    def tlp_level(self):
+        return tlp_from_tags(self.tags)
+    
+    @tlp_level.setter
+    def tlp_level(self, level):
+        set_tlp_level_in_tags(self.tags, level)
+    
+    def set_labels(self, labels):
+        self.tags.extend(labels)
+
+    def set_extra_data_from_bundler(self, bundler: "Bundler"):
+        raise NotImplementedError('this class should no longer be in use')
+
+
+    def make_rule(self, bundler: "Bundler"):
+        self.set_extra_data_from_bundler(bundler)
+        self.tags = list(dict.fromkeys(self.tags))
+
+        rule = dict(
+            id=self.detection_id,
+            **self.model_dump(
+                exclude=["indicator_types", "id"], 
+                mode="json",
+                by_alias=True
+            ),
+        )
+        for k, v in list(rule.items()):
+            if not v:
+                rule.pop(k, None)
+                
+        self.validate_rule_with_json_schema(rule)
+        if getattr(self, 'date', 0):
+            rule.update(date=self.date)
+        if getattr(self, 'modified', 0):
+            rule.update(modified=self.modified)
+        return yaml.dump(rule, sort_keys=False, indent=4)
+
+    def validate_rule_with_json_schema(self, rule):
+        jsonschema.validate(
+            rule,
+            {
+                "$ref": "https://github.com/SigmaHQ/sigma-specification/raw/refs/heads/main/json-schema/sigma-detection-rule-schema.json"
+            },
+        )
+    
+    @property
+    def external_references(self):
+        refs = []
+        for attr in ['level', 'status', 'license']:
+            if attr_val := getattr(self, attr, None):
+                refs.append(dict(source_name=f'sigma-{attr}', description=attr_val))
+        return refs
+
+    @property
+    def mitre_attack_ids(self):
+        retval = []
+        for label in self.tags:
+            namespace, _, label_id = label.partition(".")
+            if namespace == "attack":
+                retval.append(MITRE_TACTIC_MAP.get(label_id, label_id.upper()))
+        return retval
+
+    @property
+    def cve_ids(self):
+        retval = []
+        for label in self.tags:
+            namespace, _, label_id = label.partition(".")
+            if namespace == "cve":
+                retval.append(namespace.upper() + "-" + label_id)
+        return retval
+
+
+class AIDetection(BaseDetection):
+    indicator_types: list[str] = Field(default_factory=list)
+    
+    def to_sigma_rule_detection(self, bundler):
+        rule_dict = {
+            **self.model_dump(exclude=['indicator_types']),
+            **dict(date=bundler.report.created.date(), modified=bundler.report.modified.date(), id=uuid.uuid4())
+        }
+        try:
+            return SigmaRuleDetection.model_validate(rule_dict)
+        except Exception as e:
+            raise ValueError(dict(message='validate ai output failed', error=e, content=rule_dict))
+
+class SigmaRuleDetection(BaseDetection):
+    title: str
+    id: Optional[UUID] = None
+    related: Optional[list[RelatedRule]] = None
+    name: Optional[str] = None
+    taxonomy: Optional[str] = None
+    status: Optional[Statuses] = None
+    description: Optional[str] = None
+    license: Optional[str] = None
+    author: Optional[str] = None
+    references: Optional[List[str]] = Field(default_factory=list)
+    date: Optional["dt_date"] = Field(alias="date", default=None)
+    modified: Optional["dt_date"] = None
+    logsource: dict
+    detection: dict
+    fields: Optional[List[str]] = None
+    falsepositives: Optional[List[str]] = None
+    level: Optional[Level] = None
+    tags: Optional[List[SigmaTag]] = Field(default_factory=[])
+    scope: Optional[List[str]] = None
+    _indicator_types: list = None
+
+    @property
+    def detection_id(self):
+        return str(self.id)
+    
+    @property
+    def indicator_types(self):
+        return self._indicator_types
+    
+    @indicator_types.setter
+    def indicator_types(self, types):
+        self._indicator_types = types
+    
+    @detection_id.setter
+    def detection_id(self, new_id):
+        if self.id and str(self.id) != str(new_id):
+            self.related = self.related or []
+            self.related.append(RelatedRule(id=self.id, type="renamed"))
+        self.id = new_id
+    
+    @field_validator('tags', mode='after')
+    @classmethod
+    def validate_tlp(cls, tags: list[str]):
+        tlps = []
+        for tag in tags:
+            if tag.startswith('tlp.'):
+                tlps.append(tag)
+        if len(tlps) > 1:
+            raise ValueError(f'tag must not contain more than one tag in tlp namespace. Got {tlps}')
+        return tags
+    
+    @field_validator('modified', mode='after')
+    @classmethod
+    def validate_modified(cls, modified, info):
+        if info.data.get('date') == modified:
+            return None
+        return modified
+    
+    def set_extra_data_from_bundler(self, bundler: "Bundler"):
+        if not bundler:
+            return
+        
+        if not self.date:
+            from .utils import as_date
+            self.date = as_date(bundler.created)
+    
+        self.set_labels(bundler.labels)
+        self.tlp_level = bundler.tlp_level.name
+        self.author = bundler.report.created_by_ref
+        self.license = bundler.license
+        self.references = bundler.reference_urls
+
+class DetectionContainer(BaseModel):
+    success: bool
+    detections: list[Union[BaseDetection , AIDetection, SigmaRuleDetection]]
+
+
+
+def tlp_from_tags(tags: list[SigmaTag]):
+    for tag in tags:
+        ns, _, level = tag.partition(".")
+        if ns != "tlp":
+            continue
+        if tlp_level := TLP_LEVEL.get(level.replace("-", "_")):
+            return tlp_level
+    return None
+
+def set_tlp_level_in_tags(tags: list[SigmaTag], level):
+    level = str(level)
+    for i, tag in enumerate(tags):
+        if tag.startswith('tlp.'):
+            tags.remove(tag)
+    tags.append('tlp.'+level.replace("_", "-"))
+    return tags