txt2stix 1.0.8__tar.gz → 1.0.10__tar.gz

{txt2stix-1.0.8 → txt2stix-1.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: txt2stix
-Version: 1.0.8
+Version: 1.0.10
 Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
 Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
 Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues

{txt2stix-1.0.8 → txt2stix-1.0.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2stix"
-version = "1.0.8"
+version = "1.0.10"
 authors = [{ name = "dogesec" }]
 maintainers = [{ name = "dogesec" }]
 description = "txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle."

txt2stix-1.0.10/requirements.txt

@@ -0,0 +1,233 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --output-file=requirements.txt pyproject.toml
+#
+aiohappyeyeballs==2.6.1
+    # via aiohttp
+aiohttp==3.12.15
+    # via llama-index-core
+aiosignal==1.4.0
+    # via aiohttp
+aiosqlite==0.21.0
+    # via llama-index-core
+annotated-types==0.7.0
+    # via pydantic
+antlr4-python3-runtime==4.9.3
+    # via stix2-patterns
+anyio==4.10.0
+    # via
+    #   httpx
+    #   openai
+attrs==25.3.0
+    # via aiohttp
+banks==2.2.0
+    # via llama-index-core
+base58==2.1.1
+    # via txt2stix (pyproject.toml)
+beautifulsoup4==4.13.5
+    # via txt2stix (pyproject.toml)
+certifi==2025.8.3
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.4.3
+    # via requests
+click==8.2.1
+    # via nltk
+colorama==0.4.6
+    # via griffe
+dataclasses-json==0.6.7
+    # via llama-index-core
+deprecated==1.2.18
+    # via
+    #   banks
+    #   llama-index-core
+    #   llama-index-instrumentation
+dirtyjson==1.0.8
+    # via llama-index-core
+distro==1.9.0
+    # via openai
+filelock==3.19.1
+    # via tldextract
+filetype==1.2.0
+    # via llama-index-core
+frozenlist==1.7.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2025.7.0
+    # via llama-index-core
+greenlet==3.2.4
+    # via sqlalchemy
+griffe==1.13.0
+    # via banks
+h11==0.16.0
+    # via httpcore
+httpcore==1.0.9
+    # via httpx
+httpx==0.28.1
+    # via
+    #   llama-index-core
+    #   openai
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   tldextract
+    #   yarl
+jinja2==3.1.6
+    # via banks
+jiter==0.10.0
+    # via openai
+joblib==1.5.1
+    # via nltk
+llama-index-core==0.13.3
+    # via
+    #   llama-index-llms-openai
+    #   txt2stix (pyproject.toml)
+llama-index-instrumentation==0.4.0
+    # via llama-index-workflows
+llama-index-llms-openai==0.5.4
+    # via txt2stix (pyproject.toml)
+llama-index-workflows==1.3.0
+    # via llama-index-core
+markupsafe==3.0.2
+    # via jinja2
+marshmallow==3.26.1
+    # via dataclasses-json
+mistune==3.1.3
+    # via txt2stix (pyproject.toml)
+multidict==6.6.4
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.1.0
+    # via typing-inspect
+nest-asyncio==1.6.0
+    # via llama-index-core
+networkx==3.5
+    # via llama-index-core
+nltk==3.9.1
+    # via llama-index-core
+numpy==2.3.2
+    # via llama-index-core
+openai==1.101.0
+    # via llama-index-llms-openai
+packaging==25.0
+    # via marshmallow
+pathvalidate==3.3.1
+    # via txt2stix (pyproject.toml)
+phonenumbers==9.0.12
+    # via txt2stix (pyproject.toml)
+pillow==11.3.0
+    # via llama-index-core
+platformdirs==4.3.8
+    # via
+    #   banks
+    #   llama-index-core
+propcache==0.3.2
+    # via
+    #   aiohttp
+    #   yarl
+pycountry==24.6.1
+    # via schwifty
+pydantic==2.11.7
+    # via
+    #   banks
+    #   llama-index-core
+    #   llama-index-instrumentation
+    #   llama-index-workflows
+    #   openai
+pydantic-core==2.33.2
+    # via pydantic
+python-dotenv==1.1.1
+    # via txt2stix (pyproject.toml)
+pytz==2025.2
+    # via stix2
+pyyaml==6.0.2
+    # via llama-index-core
+regex==2025.7.34
+    # via
+    #   nltk
+    #   tiktoken
+requests==2.32.5
+    # via
+    #   llama-index-core
+    #   requests-file
+    #   stix2
+    #   stix2extensions
+    #   tiktoken
+    #   tldextract
+    #   txt2stix (pyproject.toml)
+requests-file==2.1.0
+    # via tldextract
+rstr==3.2.2
+    # via schwifty
+schwifty==2025.7.0
+    # via txt2stix (pyproject.toml)
+simplejson==3.20.1
+    # via stix2
+six==1.17.0
+    # via stix2-patterns
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   openai
+soupsieve==2.7
+    # via beautifulsoup4
+sqlalchemy[asyncio]==2.0.43
+    # via llama-index-core
+stix2==3.0.1
+    # via stix2extensions
+stix2-patterns==2.0.0
+    # via stix2
+stix2extensions==1.1.1
+    # via txt2stix (pyproject.toml)
+tenacity==9.1.2
+    # via llama-index-core
+tiktoken==0.11.0
+    # via llama-index-core
+tld==0.13.1
+    # via txt2stix (pyproject.toml)
+tldextract==5.3.0
+    # via txt2stix (pyproject.toml)
+tqdm==4.67.1
+    # via
+    #   llama-index-core
+    #   nltk
+    #   openai
+typing-extensions==4.15.0
+    # via
+    #   aiosqlite
+    #   beautifulsoup4
+    #   llama-index-core
+    #   llama-index-workflows
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typing-inspect
+    #   typing-inspection
+typing-inspect==0.9.0
+    # via
+    #   dataclasses-json
+    #   llama-index-core
+typing-inspection==0.4.1
+    # via pydantic
+urllib3==2.5.0
+    # via requests
+validators==0.35.0
+    # via txt2stix (pyproject.toml)
+wrapt==1.17.3
+    # via
+    #   deprecated
+    #   llama-index-core
+yarl==1.20.1
+    # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

{txt2stix-1.0.8 → txt2stix-1.0.10}/tests/data/manually_generated_reports/attack_flow_demo.txt

@@ -4,4 +4,6 @@ Due to password protection, the zip files are able to bypass some AV detections.
 
 The zip files are extracted and usually contain a malicious document, such as a .doc, .pdf, or .xls. Some examples are malware.pdf and bad.com
 
-The extracted files contain malicious macros that connect to a C2 server 1.1.1.1
+The extracted files contain malicious macros that connect to a C2 server 1.1.1.1
+
+And here is a mobile ID to test behaviour of multiple matrices T1630.001

{txt2stix-1.0.8 → txt2stix-1.0.10}/tests/manual-tests/cases-standard-tests.md

@@ -486,6 +486,24 @@ python3 txt2stix.py \
     --report_id 3b160a8d-12dd-4e7c-aee8-5af6e371b425
 ```
 
+with two domains
+
+no indicators
+
+```shell
+python3 txt2stix.py \
+    --relationship_mode ai \
+    --ai_settings_relationships openai:gpt-5 \
+    --input_file tests/data/manually_generated_reports/attack_flow_demo.txt \
+    --name 'Test MITRE ATT&CK Flow demo' \
+    --tlp_level clear \
+    --confidence 100 \
+    --use_extractions 'ai_mitre_attack_*' \
+    --ai_settings_extractions openai:gpt-5 \
+    --ai_create_attack_flow \
+    --report_id ccc8c844-6a89-4762-b4e7-77c918fa4b8f
+```
+
 ### attack navigator demo
 
 ```shell

{txt2stix-1.0.8 → txt2stix-1.0.10}/tests/src/test_attack_flow.py

@@ -8,6 +8,7 @@ from txt2stix.attack_flow import (
     create_navigator_layer,
     get_all_tactics,
     get_techniques_from_extracted_objects,
+    parse_domain_flow,
     parse_flow,
     extract_attack_flow_and_navigator,
 )
@@ -21,31 +22,30 @@ def test_parse_flow(dummy_report, dummy_objects, dummy_flow):
     techniques = get_techniques_from_extracted_objects(dummy_objects, tactics)
     flow = dummy_flow
     report = dummy_report
-    expected_ids = set(
-        [
-            "report--9c88fbcb-8c0d-4124-868b-3dcb1e9b696c",
-            "extension-definition--fb9c968a-745b-4ade-9b25-c324172197f4",
-            "attack-flow--9c88fbcb-8c0d-4124-868b-3dcb1e9b696c",
-            "relationship--6346ead9-49cc-5ede-89e2-449f1c22ed13",
-            "x-mitre-tactic--696af733-728e-49d7-8261-75fdc590f453",
-            "attack-pattern--0fe075d5-beac-4d02-b93e-0f874997db72",
-            "attack-action--51dc1572-cb10-581b-b9ef-9e589615ecaa",
-            "x-mitre-tactic--5bc1d813-693e-4823-9961-abf9af4b0e92",
-            "attack-pattern--f9e9365a-9ca2-4d9c-8e7c-050d73d1101a",
-            "attack-action--e03c89ba-a476-5509-ac0a-049b61514be7",
-            "x-mitre-tactic--298fe907-7931-4fd2-8131-2814dd493134",
-            "attack-pattern--1b22b676-9347-4c55-9a35-ef0dc653db5b",
-            "attack-action--1fd63972-ef98-5da5-81f5-4090c7dfa585",
-            "x-mitre-tactic--2558fd61-8c75-4730-94c4-11926db2a263",
-            "attack-pattern--1a80d097-54df-41d8-9d33-34e755ec5e72",
-            "attack-action--c7e06b10-252d-520d-82eb-e32314bbec34",
-        ]
-    )
+    expected_ids = {
+        "attack-pattern--1b22b676-9347-4c55-9a35-ef0dc653db5b",
+        "x-mitre-tactic--298fe907-7931-4fd2-8131-2814dd493134",
+        "attack-action--1fd63972-ef98-5da5-81f5-4090c7dfa585",
+        "extension-definition--fb9c968a-745b-4ade-9b25-c324172197f4",
+        "attack-pattern--1a80d097-54df-41d8-9d33-34e755ec5e72",
+        "report--9c88fbcb-8c0d-4124-868b-3dcb1e9b696c",
+        "attack-flow--bb21585c-5f82-55cf-b73d-89b5217ef092",
+        "relationship--22e97298-819a-55ac-b57b-2185ffb72c62",
+        "x-mitre-tactic--2558fd61-8c75-4730-94c4-11926db2a263",
+        "attack-action--c7e06b10-252d-520d-82eb-e32314bbec34",
+        "x-mitre-tactic--696af733-728e-49d7-8261-75fdc590f453",
+        "x-mitre-tactic--5bc1d813-693e-4823-9961-abf9af4b0e92",
+        "attack-pattern--f9e9365a-9ca2-4d9c-8e7c-050d73d1101a",
+        "attack-action--51dc1572-cb10-581b-b9ef-9e589615ecaa",
+        "relationship--931f6b0d-5136-534a-93bc-8cce065e04dc",
+        "attack-pattern--0fe075d5-beac-4d02-b93e-0f874997db72",
+        "attack-flow--b48ec5d9-407c-5e4a-a6d0-fe851cd4ea0e",
+        "attack-action--e03c89ba-a476-5509-ac0a-049b61514be7",
+    }
 
     flow_objects = parse_flow(report, flow, techniques, tactics)
     assert {obj["id"] for obj in flow_objects} == expected_ids
 
-
 def test_parse_flow__no_success(dummy_report):
     flow_objects = parse_flow(
         dummy_report,
@@ -61,6 +61,48 @@ def test_parse_flow__no_success(dummy_report):
     assert len(flow_objects) == 0
 
 
+@pytest.mark.parametrize(
+    ["domain", "expected_ids"],
+    [
+        ("mobile-attack", set()),
+        [
+            "ics-attack",
+            {
+                "x-mitre-tactic--298fe907-7931-4fd2-8131-2814dd493134",
+                "x-mitre-tactic--696af733-728e-49d7-8261-75fdc590f453",
+                "attack-pattern--1b22b676-9347-4c55-9a35-ef0dc653db5b",
+                "relationship--22e97298-819a-55ac-b57b-2185ffb72c62",
+                "attack-action--1fd63972-ef98-5da5-81f5-4090c7dfa585",
+                "attack-pattern--0fe075d5-beac-4d02-b93e-0f874997db72",
+                "attack-flow--bb21585c-5f82-55cf-b73d-89b5217ef092",
+                "attack-action--51dc1572-cb10-581b-b9ef-9e589615ecaa",
+            },
+        ],
+        [
+            "enterprise-attack",
+            {
+                "attack-action--e03c89ba-a476-5509-ac0a-049b61514be7",
+                "relationship--931f6b0d-5136-534a-93bc-8cce065e04dc",
+                "attack-flow--b48ec5d9-407c-5e4a-a6d0-fe851cd4ea0e",
+                "attack-pattern--f9e9365a-9ca2-4d9c-8e7c-050d73d1101a",
+                "x-mitre-tactic--2558fd61-8c75-4730-94c4-11926db2a263",
+                "attack-pattern--1a80d097-54df-41d8-9d33-34e755ec5e72",
+                "x-mitre-tactic--5bc1d813-693e-4823-9961-abf9af4b0e92",
+                "attack-action--c7e06b10-252d-520d-82eb-e32314bbec34",
+            },
+        ],
+    ],
+)
+def test_parse_domain_flow(dummy_report, dummy_objects, dummy_flow, domain, expected_ids):
+    tactics = get_all_tactics()
+    techniques = get_techniques_from_extracted_objects(dummy_objects, tactics)
+    flow = dummy_flow
+    report = dummy_report
+    flow_objects = parse_domain_flow(report, flow, techniques, tactics, domain)
+    print([domain, {obj["id"] for obj in flow_objects}], ",")
+    assert {obj["id"] for obj in flow_objects} == expected_ids
+
+
 def test_get_techniques_from_extracted_objects(dummy_objects):
     tactics = get_all_tactics()
     techniques = get_techniques_from_extracted_objects(dummy_objects, tactics)
@@ -142,7 +184,11 @@ def test_extract_attack_flow_and_navigator(dummy_objects, dummy_report):
         mock_extract_flow.assert_called_once_with(text, techniques)
 
         mock_create_navigator_layer.assert_called_once_with(
-            bundler.report, bundler.summary, mock_extract_flow.return_value, techniques, tactics
+            bundler.report,
+            bundler.summary,
+            mock_extract_flow.return_value,
+            techniques,
+            tactics,
         )
 
         ### reset mocks
@@ -181,7 +227,11 @@ def test_extract_attack_flow_and_navigator(dummy_objects, dummy_report):
         mock_parse_flow.assert_not_called()
 
         mock_create_navigator_layer.assert_called_once_with(
-            bundler.report, bundler.summary, mock_extract_flow.return_value, techniques, tactics
+            bundler.report,
+            bundler.summary,
+            mock_extract_flow.return_value,
+            techniques,
+            tactics,
         )
 
         ### reset mocks
@@ -281,7 +331,7 @@ def test_create_navigator_layer(dummy_report):
         {
             "versions": {
                 "layer": "4.5",
-                "attack": '16.1',
+                "attack": "16.1",
                 "navigator": "5.1.0",
             },
             "name": "some markdown document",
@@ -311,7 +361,7 @@ def test_create_navigator_layer(dummy_report):
         {
             "versions": {
                 "layer": "4.5",
-                "attack": '17.0',
+                "attack": "17.0",
                 "navigator": "5.1.0",
             },
             "name": "some markdown document",
@@ -341,7 +391,7 @@ def test_create_navigator_layer(dummy_report):
         {
             "versions": {
                 "layer": "4.5",
-                "attack": '13.1',
+                "attack": "13.1",
                 "navigator": "5.1.0",
             },
             "name": "some markdown document",

{txt2stix-1.0.8 → txt2stix-1.0.10}/txt2stix/ai_extractor/openai.py

@@ -1,4 +1,5 @@
    
+import logging
 import os
 from txt2stix.ai_extractor.base import BaseAIExtractor
 from llama_index.llms.openai import OpenAI
@@ -11,5 +12,9 @@ class OpenAIExtractor(BaseAIExtractor, provider="openai"):
         super().__init__()
 
     def count_tokens(self, text):
-        return len(self.llm._tokenizer.encode(text))
+        try:
+            return len(self.llm._tokenizer.encode(text))
+        except Exception as e:
+            logging.warning(e)
+            return super().count_tokens(text)
     

{txt2stix-1.0.8 → txt2stix-1.0.10}/txt2stix/attack_flow.py

@@ -16,15 +16,26 @@ def parse_flow(report, flow: AttackFlowList, techniques, tactics):
     logging.info(f"flow.success = {flow.success}")
     if not flow.success:
         return []
-    flow_objects = [report, attack_flow_ExtensionDefinitionSMO]
+    objects = [report, attack_flow_ExtensionDefinitionSMO]
+    for domain in ["enterprise-attack", "mobile-attack", "ics-attack"]:
+        flow_objects = parse_domain_flow(report, flow, techniques, tactics, domain)
+        objects.extend(flow_objects)
+    return objects
+
+def parse_domain_flow(report, flow: AttackFlowList, techniques, tactics, domain):
+    flow_objects = []
+    flow_obj = None
     last_action = None
     for i, item in enumerate(flow.items):
         try:
             technique = techniques[item.attack_technique_id]
+            if technique["domain"] != domain:
+                continue
             tactic_id = technique["possible_tactics"][
                 flow.tactic_mapping[item.attack_technique_id]
             ]
             technique_obj = technique["stix_obj"]
+
             tactic_obj = tactics[technique["domain"]][tactic_id]
             action_obj = AttackAction(
                 **{
@@ -40,16 +51,16 @@ def parse_flow(report, flow: AttackFlowList, techniques, tactics):
                 allow_custom=True,
             )
             action_obj.effect_refs.clear()
-            if i == 0:
+            if not flow_obj:
                 flow_obj = {
                     "type": "attack-flow",
-                    "id": report.id.replace("report", "attack-flow"),
+                    "id": "attack-flow--"+str(uuid.uuid5(UUID_NAMESPACE, f"attack-flow+{domain}+{report.id}")),
                     "spec_version": "2.1",
                     "created": report.created,
                     "modified": report.modified,
                     "created_by_ref": report.created_by_ref,
                     "start_refs": [action_obj["id"]],
-                    "name": report.name,
+                    "name": f"[{domain.split('-')[0].upper()}] {report.name}",
                     "description": report.description,
                     "scope": "malware",
                     "external_references": report.external_references,
@@ -61,7 +72,7 @@ def parse_flow(report, flow: AttackFlowList, techniques, tactics):
                         type="relationship",
                         spec_version="2.1",
                         id="relationship--"
-                        + str(uuid.uuid5(UUID_NAMESPACE, f"attack-flow+{report.id}")),
+                        + str(uuid.uuid5(UUID_NAMESPACE, f"attack-flow+{report.id}+{flow_obj['id']}")),
                         created_by_ref=report.created_by_ref,
                         created=report.created,
                         modified=report.modified,