txt2detection 1.0.6__tar.gz → 1.0.8__tar.gz

{txt2detection-1.0.6 → txt2detection-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: txt2detection
-Version: 1.0.6
+Version: 1.0.8
 Summary: A command line tool that takes a txt file containing threat intelligence and turns it into a detection rule.
 Project-URL: Homepage, https://github.com/muchdogesec/txt2detection
 Project-URL: Issues, https://github.com/muchdogesec/txt2detection/issues
@@ -21,6 +21,7 @@ Requires-Dist: python-slugify
 Requires-Dist: pyyaml
 Requires-Dist: requests>=2.31.0; python_version >= '3.7'
 Requires-Dist: stix2
+Requires-Dist: stix2extensions
 Requires-Dist: tqdm>=4.66.4; python_version >= '3.7'
 Requires-Dist: validators>=0.34.0
 Provides-Extra: anthropic

{txt2detection-1.0.6 → txt2detection-1.0.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2detection"
-version = "1.0.6"
+version = "1.0.8"
 authors = [
   { name = "dogesec" }
 ]
@@ -31,6 +31,7 @@ dependencies = [
     "validators>=0.34.0",
     "llama-index-core>=0.12.42",
     'llama-index-llms-openai>=0.4.5',
+    'stix2extensions',
   ]
 [project.urls]
 Homepage = "https://github.com/muchdogesec/txt2detection"

{txt2detection-1.0.6 → txt2detection-1.0.8}/requirements.txt

@@ -155,7 +155,7 @@ jsonschema==4.25.1 ; python_version >= "3.8"
     # via txt2detection (pyproject.toml)
 jsonschema-specifications==2025.4.1
     # via jsonschema
-llama-index-core==0.12.52.post1
+llama-index-core==0.13.0
     # via
     #   llama-index-llms-anthropic
     #   llama-index-llms-gemini

{txt2detection-1.0.6 → txt2detection-1.0.8}/tests/src/test_bundler.py

@@ -16,9 +16,9 @@ def dummy_detection():
     detection = SigmaRuleDetection(
         title="Test Detection",
         description="Detects something suspicious.",
-        detection=dict(condition="selection1", selection1=dict(ip='1.1.1.1')),
+        detection=dict(condition="selection1", selection1=dict(ip="1.1.1.1")),
         tags=["tlp.red", "sigma.execution"],
-        id=str(uuid.uuid4()),
+        id="cd7ff0b1-fbf3-4c2d-ba70-5d127eb8b4be",
         external_references=[],
         logsource=dict(
             category="network-connection",
@@ -36,6 +36,8 @@ def bundler_instance():
         tlp_level="red",
         description="This is a test report.",
         labels=["tlp.red", "test.test-var"],
+        created=datetime(2025, 1, 1),
+        report_id="74e36652-00f5-4dca-bf10-9f02fc996dcc",
     )
 
 
@@ -229,7 +231,7 @@ def test_bundler_generates_valid_bundle(dummy_detection):
 
 def test_bundle_detections(dummy_detection, bundler_instance):
     container = DetectionContainer(success=False, detections=[])
-    with patch.object(Bundler, 'add_rule_indicator') as mock_add_rule_indicator:
+    with patch.object(Bundler, "add_rule_indicator") as mock_add_rule_indicator:
         bundler_instance.bundle_detections(container)
         mock_add_rule_indicator.assert_not_called()
         mock_add_rule_indicator.reset_mock()
@@ -238,3 +240,47 @@ def test_bundle_detections(dummy_detection, bundler_instance):
         container.success = True
         bundler_instance.bundle_detections(container)
         mock_add_rule_indicator.assert_called_once_with(detection)
+
+
+def test_bundle_detections__creates_log_source(dummy_detection, bundler_instance):
+    dummy_detection.detection_id = "d73e1632-c541-4b09-8281-95dc7f9c5782"
+    bundler_instance.add_rule_indicator(dummy_detection)
+    objects = [
+        obj
+        for obj in bundler_instance.bundle_dict["objects"]
+        if obj["id"]
+        in (
+            "data-source--f078a18f-0f04-5fde-b6cd-a5af90b6346b",
+            "relationship--fe0a3715-6a21-5472-840f-39ea9c61ee83",
+        )
+    ]
+    assert objects == [
+        {
+            "type": "data-source",
+            "spec_version": "2.1",
+            "id": "data-source--f078a18f-0f04-5fde-b6cd-a5af90b6346b",
+            "category": "network-connection",
+            "product": "firewall",
+            "extensions": {
+                "extension-definition--afeeb724-bce2-575e-af3d-d705842ea84b": {
+                    "extension_type": "new-sco"
+                }
+            },
+        },
+        {
+            "type": "relationship",
+            "spec_version": "2.1",
+            "id": "relationship--fe0a3715-6a21-5472-840f-39ea9c61ee83",
+            "created_by_ref": "identity--a4d70b75-6f4a-5d19-9137-da863edd33d7",
+            "created": "2025-01-01T00:00:00.000Z",
+            "modified": "2025-01-01T00:00:00.000Z",
+            "relationship_type": "related-to",
+            "description": "Test Detection is created from log-source {category=network-connection, product=firewall}",
+            "source_ref": "indicator--d73e1632-c541-4b09-8281-95dc7f9c5782",
+            "target_ref": "data-source--f078a18f-0f04-5fde-b6cd-a5af90b6346b",
+            "object_marking_refs": [
+                "marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1",
+                "marking-definition--a4d70b75-6f4a-5d19-9137-da863edd33d7",
+            ],
+        },
+    ]

{txt2detection-1.0.6 → txt2detection-1.0.8}/txt2detection/bundler.py

@@ -105,7 +105,7 @@ class Bundler:
 
         self.job_id = f"report--{self.uuid}"
         self.external_refs = (external_refs or []) + [dict(source_name='txt2detection', url=url, description='txt2detection-reference') for url in self.reference_urls]
-        
+
         self.report = Report(
             created_by_ref=self.identity.id,
             name=name,
@@ -177,6 +177,7 @@ class Bundler:
             "external_id": hashlib.md5(indicator['pattern'].encode()).hexdigest()
             }
         )
+        logsource = detection.make_data_source()
 
         logger.debug(f"===== rule {detection.detection_id} =====")
         logger.debug("```yaml\n"+indicator['pattern']+"\n```")
@@ -191,24 +192,30 @@ class Bundler:
             self.add_relation(indicator, obj)
 
         self.add_ref(parse_stix(indicator, allow_custom=True), append_report=True)
+        print('everywhere')
+        self.add_ref(logsource, append_report=True)
+        print('here')
+        self.add_relation(indicator, logsource, description=f'{indicator["name"]} is created from {make_logsouce_string(logsource)}')
+        print('there')
 
         for ob_type, ob_value in set(observables.find_stix_observables(detection.detection)):
             try:
                 obj = observables.to_stix_object(ob_type, ob_value)
                 self.add_ref(obj)
-                self.add_relation(indicator, obj, 'detects', target_name=ob_value)
+                self.add_relation(indicator, obj, 'related-to', target_name=ob_value)
             except:
                 logger.exception(f"failed to process observable {ob_type}/{ob_value}")
 
-
-    def add_relation(self, indicator, target_object, relationship_type='detects', target_name=None):
+    def add_relation(self, indicator, target_object, relationship_type='related-to', target_name=None, description=None):
         ext_refs = []
 
         with contextlib.suppress(Exception):
             indicator['external_references'].append(target_object['external_references'][0])
             ext_refs = [target_object['external_references'][0]]
 
-        target_name = target_name or f"{target_object['external_references'][0]['external_id']} ({target_object['name']})"
+        if not description:
+            target_name = target_name or f"{target_object['external_references'][0]['external_id']} ({target_object['name']})"
+            description = f"{indicator['name']} {relationship_type} {target_name}"
 
         rel =  Relationship(
             id="relationship--" + str(
@@ -220,7 +227,7 @@ class Bundler:
             target_ref=target_object['id'],
             relationship_type=relationship_type,
             created_by_ref=self.report.created_by_ref,
-            description=f"{indicator['name']} {relationship_type} {target_name}",
+            description=description,
             created=self.report.created,
             modified=self.report.modified,
             object_marking_refs=self.report.object_marking_refs,
@@ -281,3 +288,9 @@ class Bundler:
             return
         for d in container.detections:
             self.add_rule_indicator(d)
+
+def make_logsouce_string(source: dict):
+    d = [f'{k}={v}' for k, v in source.items()
+        if k in ['product', 'service', 'category']]
+    d_str = ', '.join(d)
+    return 'log-source {'+d_str+'}'

{txt2detection-1.0.6 → txt2detection-1.0.8}/txt2detection/models.py

@@ -8,6 +8,7 @@ from slugify import slugify
 from datetime import date as dt_date
 from typing import Any, ClassVar, List, Literal, Optional, Union
 from uuid import UUID
+from stix2extensions.data_source import DataSource
 
 import jsonschema
 from pydantic import BaseModel, Field, computed_field, field_validator
@@ -124,11 +125,11 @@ class TLP_LEVEL(enum.Enum):
         ]
 
     @classmethod
-    def get(cls, level: 'str|TLP_LEVEL'):
+    def get(cls, level: "str|TLP_LEVEL"):
         if isinstance(level, cls):
             return level
         level = level.lower()
-        level = level.replace('+', '_').replace('-', '_')
+        level = level.replace("+", "_").replace("-", "_")
         if level not in cls.levels():
             raise Exception(f"unsupported tlp level: `{level}`")
         return cls.levels()[level]
@@ -137,6 +138,7 @@ class TLP_LEVEL(enum.Enum):
     def name(self):
         return super().name.lower()
 
+
 class Statuses(enum.StrEnum):
     stable = enum.auto()
     test = enum.auto()
@@ -144,6 +146,7 @@ class Statuses(enum.StrEnum):
     deprecated = enum.auto()
     unsupported = enum.auto()
 
+
 class Level(enum.StrEnum):
     informational = enum.auto()
     low = enum.auto()
@@ -151,6 +154,7 @@ class Level(enum.StrEnum):
     high = enum.auto()
     critical = enum.auto()
 
+
 class SigmaTag(str):
     @classmethod
     def __get_pydantic_core_schema__(
@@ -158,31 +162,35 @@ class SigmaTag(str):
         _source: type[Any],
         _handler,
     ) -> core_schema.CoreSchema:
-        return core_schema.no_info_after_validator_function(cls._validate, core_schema.str_schema())
+        return core_schema.no_info_after_validator_function(
+            cls._validate, core_schema.str_schema()
+        )
 
     @classmethod
-    def __get_pydantic_json_schema__(
-        cls, core_schema: core_schema.CoreSchema, handler
-    ):
+    def __get_pydantic_json_schema__(cls, core_schema: core_schema.CoreSchema, handler):
         field_schema = handler(core_schema)
-        field_schema.update(type='string', pattern=TAG_PATTERN.pattern, format='sigma-tag')
+        field_schema.update(
+            type="string", pattern=TAG_PATTERN.pattern, format="sigma-tag"
+        )
         return field_schema
 
     @classmethod
     def _validate(cls, input_value: str, /) -> str:
         if not TAG_PATTERN.match(input_value):
             raise PydanticCustomError(
-            'value_error',
-            'value is not a valid SIGMA tag: {reason}',
-            {'reason': f'Must be in format namespace.value and match pattern {TAG_PATTERN.pattern}'},
-        )
+                "value_error",
+                "value is not a valid SIGMA tag: {reason}",
+                {
+                    "reason": f"Must be in format namespace.value and match pattern {TAG_PATTERN.pattern}"
+                },
+            )
         return input_value
-    
+
+
 class RelatedRule(BaseModel):
     id: UUID
-    type: Literal[
-        "derived", "obsolete", "merged", "renamed", "similar"
-    ]
+    type: Literal["derived", "obsolete", "merged", "renamed", "similar"]
+
 
 class BaseDetection(BaseModel):
     title: str
@@ -195,7 +203,9 @@ class BaseDetection(BaseModel):
     level: Level
     _custom_id = None
     _extra_data: dict
-    sigma_json_schema: ClassVar = requests.get("https://github.com/SigmaHQ/sigma-specification/raw/refs/heads/main/json-schema/sigma-detection-rule-schema.json").json()
+    sigma_json_schema: ClassVar = requests.get(
+        "https://github.com/SigmaHQ/sigma-specification/raw/refs/heads/main/json-schema/sigma-detection-rule-schema.json"
+    ).json()
 
     def model_post_init(self, __context):
         self.tags = self.tags or []
@@ -213,17 +223,16 @@ class BaseDetection(BaseModel):
     @property
     def tlp_level(self):
         return tlp_from_tags(self.tags)
-    
+
     @tlp_level.setter
     def tlp_level(self, level):
         set_tlp_level_in_tags(self.tags, level)
-    
+
     def set_labels(self, labels):
         self.tags.extend(labels)
 
     def set_extra_data_from_bundler(self, bundler: "Bundler"):
-        raise NotImplementedError('this class should no longer be in use')
-
+        raise NotImplementedError("this class should no longer be in use")
 
     def make_rule(self, bundler: "Bundler"):
         self.set_extra_data_from_bundler(bundler)
@@ -232,19 +241,17 @@ class BaseDetection(BaseModel):
         rule = dict(
             id=self.detection_id,
             **self.model_dump(
-                exclude=["indicator_types", "id"], 
-                mode="json",
-                by_alias=True
+                exclude=["indicator_types", "id"], mode="json", by_alias=True
             ),
         )
         for k, v in list(rule.items()):
             if not v:
                 rule.pop(k, None)
-                
+
         self.validate_rule_with_json_schema(rule)
-        if getattr(self, 'date', 0):
+        if getattr(self, "date", 0):
             rule.update(date=self.date)
-        if getattr(self, 'modified', 0):
+        if getattr(self, "modified", 0):
             rule.update(modified=self.modified)
         return yaml.dump(rule, sort_keys=False, indent=4)
 
@@ -253,13 +260,13 @@ class BaseDetection(BaseModel):
             rule,
             self.sigma_json_schema,
         )
-    
+
     @property
     def external_references(self):
         refs = []
-        for attr in ['level', 'status', 'license']:
+        for attr in ["level", "status", "license"]:
             if attr_val := getattr(self, attr, None):
-                refs.append(dict(source_name=f'sigma-{attr}', description=attr_val))
+                refs.append(dict(source_name=f"sigma-{attr}", description=attr_val))
         return refs
 
     @property
@@ -280,19 +287,34 @@ class BaseDetection(BaseModel):
                 retval.append(namespace.upper() + "-" + label_id)
         return retval
 
+    def make_data_source(self):
+        return DataSource(
+            category=self.logsource.get("category"),
+            product=self.logsource.get("product"),
+            service=self.logsource.get("service"),
+            definition=self.logsource.get("definition"),
+        )
+
 
 class AIDetection(BaseDetection):
     indicator_types: list[str] = Field(default_factory=list)
-    
+
     def to_sigma_rule_detection(self, bundler):
         rule_dict = {
-            **self.model_dump(exclude=['indicator_types']),
-            **dict(date=bundler.report.created.date(), modified=bundler.report.modified.date(), id=uuid.uuid4())
+            **self.model_dump(exclude=["indicator_types"]),
+            **dict(
+                date=bundler.report.created.date(),
+                modified=bundler.report.modified.date(),
+                id=uuid.uuid4(),
+            ),
         }
         try:
             return SigmaRuleDetection.model_validate(rule_dict)
         except Exception as e:
-            raise ValueError(dict(message='validate ai output failed', error=e, content=rule_dict))
+            raise ValueError(
+                dict(message="validate ai output failed", error=e, content=rule_dict)
+            )
+
 
 class SigmaRuleDetection(BaseDetection):
     title: str
@@ -319,58 +341,61 @@ class SigmaRuleDetection(BaseDetection):
     @property
     def detection_id(self):
         return str(self.id)
-    
+
     @property
     def indicator_types(self):
         return self._indicator_types
-    
+
     @indicator_types.setter
     def indicator_types(self, types):
         self._indicator_types = types
-    
+
     @detection_id.setter
     def detection_id(self, new_id):
         if self.id and str(self.id) != str(new_id):
             self.related = self.related or []
             self.related.append(RelatedRule(id=self.id, type="renamed"))
         self.id = new_id
-    
-    @field_validator('tags', mode='after')
+
+    @field_validator("tags", mode="after")
     @classmethod
     def validate_tlp(cls, tags: list[str]):
         tlps = []
         for tag in tags:
-            if tag.startswith('tlp.'):
+            if tag.startswith("tlp."):
                 tlps.append(tag)
         if len(tlps) > 1:
-            raise ValueError(f'tag must not contain more than one tag in tlp namespace. Got {tlps}')
+            raise ValueError(
+                f"tag must not contain more than one tag in tlp namespace. Got {tlps}"
+            )
         return tags
-    
-    @field_validator('modified', mode='after')
+
+    @field_validator("modified", mode="after")
     @classmethod
     def validate_modified(cls, modified, info):
-        if info.data.get('date') == modified:
+        if info.data.get("date") == modified:
             return None
         return modified
-    
+
     def set_extra_data_from_bundler(self, bundler: "Bundler"):
         if not bundler:
             return
-        
+
         if not self.date:
             from .utils import as_date
+
             self.date = as_date(bundler.created)
-    
+
         self.set_labels(bundler.labels)
         self.tlp_level = bundler.tlp_level.name
         self.author = bundler.report.created_by_ref
         self.license = bundler.license
         self.references = bundler.reference_urls
 
+
 class DetectionContainer(BaseModel):
     success: bool
-    detections: list[Union[BaseDetection , AIDetection, SigmaRuleDetection]]
-
+    detections: list[Union[BaseDetection, AIDetection, SigmaRuleDetection]]
 
 
 def tlp_from_tags(tags: list[SigmaTag]):
@@ -382,10 +407,11 @@ def tlp_from_tags(tags: list[SigmaTag]):
             return tlp_level
     return None
 
+
 def set_tlp_level_in_tags(tags: list[SigmaTag], level):
     level = str(level)
     for i, tag in enumerate(tags):
-        if tag.startswith('tlp.'):
+        if tag.startswith("tlp."):
             tags.remove(tag)
-    tags.append('tlp.'+level.replace("_", "-"))
+    tags.append("tlp." + level.replace("_", "-"))
     return tags