txt2stix 0.0.4__tar.gz → 1.0.1__tar.gz

{txt2stix-0.0.4 → txt2stix-1.0.1}/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.4
 Name: txt2stix
-Version: 0.0.4
+Version: 1.0.1
 Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
 Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
 Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues
-Author-email: DOGESEC <support@dogesec.com>
+Project-URL: dogesec HQ, https://dogesec.com
+Author: dogesec
+Maintainer: dogesec
 License-File: LICENSE
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
@@ -16,7 +18,6 @@ Requires-Dist: llama-index-core>=0.12.42
 Requires-Dist: llama-index-llms-anthropic>=0.7.2
 Requires-Dist: llama-index-llms-deepseek>=0.1.2
 Requires-Dist: llama-index-llms-gemini>=0.5.0
-Requires-Dist: llama-index-llms-openai-like>=0.4.0
 Requires-Dist: llama-index-llms-openai>=0.4.5
 Requires-Dist: llama-index-llms-openrouter>=0.3.2
 Requires-Dist: mistune>=3.0.2

{txt2stix-0.0.4 → txt2stix-1.0.1}/pyproject.toml

@@ -4,8 +4,13 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2stix"
-version = "0.0.4"
-authors = [{ name = "DOGESEC", email = "support@dogesec.com" }]
+version = "1.0.1"
+authors = [
+  { name = "dogesec" }
+]
+maintainers = [
+  { name = "dogesec" }
+]
 description = "txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -32,7 +37,6 @@ dependencies = [
   'llama-index-llms-anthropic>=0.7.2',
   'llama-index-llms-gemini>=0.5.0',
   'llama-index-llms-openai>=0.4.5',
-  'llama-index-llms-openai-like>=0.4.0',
   'llama-index-llms-deepseek>=0.1.2',
   'llama-index-llms-openrouter>=0.3.2',
   'mistune>=3.0.2',
@@ -46,7 +50,7 @@ allow-direct-references = true
 [project.urls]
 Homepage = "https://github.com/muchdogesec/txt2stix"
 Issues = "https://github.com/muchdogesec/txt2stix/issues"
-
+"dogesec HQ" = "https://dogesec.com"
 
 [project.scripts]
 stix2arango = "txt2stix.txt2stix:main"

{txt2stix-0.0.4 → txt2stix-1.0.1}/requirements.txt

@@ -6,15 +6,15 @@
 #
 aiohappyeyeballs==2.6.1
     # via aiohttp
-aiohttp==3.12.12
+aiohttp==3.12.13
     # via llama-index-core
-aiosignal==1.3.2
+aiosignal==1.4.0
     # via aiohttp
 aiosqlite==0.21.0
     # via llama-index-core
 annotated-types==0.7.0
     # via pydantic
-anthropic[bedrock,vertex]==0.54.0
+anthropic[bedrock,vertex]==0.57.1
     # via llama-index-llms-anthropic
 antlr4-python3-runtime==4.9.3
     # via stix2-patterns
@@ -25,22 +25,22 @@ anyio==4.9.0
     #   openai
 attrs==25.3.0
     # via aiohttp
-banks==2.1.2
+banks==2.1.3
     # via llama-index-core
 base58==2.1.1
     # via txt2stix (pyproject.toml)
 beautifulsoup4==4.13.4
     # via txt2stix (pyproject.toml)
-boto3==1.38.35
+boto3==1.39.3
     # via anthropic
-botocore==1.38.35
+botocore==1.39.3
     # via
     #   anthropic
     #   boto3
     #   s3transfer
 cachetools==5.5.2
     # via google-auth
-certifi==2025.4.26
+certifi==2025.6.15
     # via
     #   httpcore
     #   httpx
@@ -57,6 +57,7 @@ deprecated==1.2.18
     # via
     #   banks
     #   llama-index-core
+    #   llama-index-instrumentation
 dirtyjson==1.0.8
     # via llama-index-core
 distro==1.9.0
@@ -80,12 +81,12 @@ fsspec==2025.5.1
     #   llama-index-core
 google-ai-generativelanguage==0.6.15
     # via google-generativeai
-google-api-core[grpc]==2.25.0
+google-api-core[grpc]==2.25.1
     # via
     #   google-ai-generativelanguage
     #   google-api-python-client
     #   google-generativeai
-google-api-python-client==2.172.0
+google-api-python-client==2.175.0
     # via google-generativeai
 google-auth[requests]==2.40.3
     # via
@@ -107,15 +108,15 @@ greenlet==3.2.3
     # via sqlalchemy
 griffe==1.7.3
     # via banks
-grpcio==1.73.0
+grpcio==1.73.1
     # via
     #   google-api-core
     #   grpcio-status
-grpcio-status==1.71.0
+grpcio-status==1.71.2
     # via google-api-core
 h11==0.16.0
     # via httpcore
-hf-xet==1.1.3
+hf-xet==1.1.5
     # via huggingface-hub
 httpcore==1.0.9
     # via httpx
@@ -128,7 +129,7 @@ httpx==0.28.1
     #   anthropic
     #   llama-index-core
     #   openai
-huggingface-hub==0.33.0
+huggingface-hub==0.33.2
     # via
     #   tokenizers
     #   transformers
@@ -151,7 +152,7 @@ jmespath==1.0.1
     #   botocore
 joblib==1.5.1
     # via nltk
-llama-index-core==0.12.42
+llama-index-core==0.12.47
     # via
     #   llama-index-llms-anthropic
     #   llama-index-llms-gemini
@@ -159,13 +160,15 @@ llama-index-core==0.12.42
     #   llama-index-llms-openai-like
     #   llama-index-llms-openrouter
     #   txt2stix (pyproject.toml)
-llama-index-llms-anthropic==0.7.2
+llama-index-instrumentation==0.2.0
+    # via llama-index-workflows
+llama-index-llms-anthropic==0.7.6
     # via txt2stix (pyproject.toml)
 llama-index-llms-deepseek==0.1.2
     # via txt2stix (pyproject.toml)
 llama-index-llms-gemini==0.5.0
     # via txt2stix (pyproject.toml)
-llama-index-llms-openai==0.4.5
+llama-index-llms-openai==0.4.7
     # via
     #   llama-index-llms-openai-like
     #   txt2stix (pyproject.toml)
@@ -173,16 +176,17 @@ llama-index-llms-openai-like==0.4.0
     # via
     #   llama-index-llms-deepseek
     #   llama-index-llms-openrouter
-    #   txt2stix (pyproject.toml)
 llama-index-llms-openrouter==0.3.2
     # via txt2stix (pyproject.toml)
+llama-index-workflows==1.0.1
+    # via llama-index-core
 markupsafe==3.0.2
     # via jinja2
 marshmallow==3.26.1
     # via dataclasses-json
 mistune==3.1.3
     # via txt2stix (pyproject.toml)
-multidict==6.4.4
+multidict==6.6.3
     # via
     #   aiohttp
     #   yarl
@@ -194,20 +198,20 @@ networkx==3.5
     # via llama-index-core
 nltk==3.9.1
     # via llama-index-core
-numpy==2.3.0
+numpy==2.3.1
     # via
     #   llama-index-core
     #   transformers
-openai==1.86.0
+openai==1.93.2
     # via llama-index-llms-openai
 packaging==25.0
     # via
     #   huggingface-hub
     #   marshmallow
     #   transformers
-pathvalidate==3.2.3
+pathvalidate==3.3.1
     # via txt2stix (pyproject.toml)
-phonenumbers==9.0.7
+phonenumbers==9.0.9
     # via txt2stix (pyproject.toml)
 pillow==10.4.0
     # via
@@ -239,12 +243,14 @@ pyasn1-modules==0.4.2
     # via google-auth
 pycountry==24.6.1
     # via schwifty
-pydantic==2.11.5
+pydantic==2.11.7
     # via
     #   anthropic
     #   banks
     #   google-generativeai
     #   llama-index-core
+    #   llama-index-instrumentation
+    #   llama-index-workflows
     #   openai
 pydantic-core==2.33.2
     # via pydantic
@@ -252,7 +258,7 @@ pyparsing==3.2.3
     # via httplib2
 python-dateutil==2.9.0.post0
     # via botocore
-python-dotenv==1.1.0
+python-dotenv==1.1.1
     # via txt2stix (pyproject.toml)
 pytz==2025.2
     # via stix2
@@ -289,7 +295,7 @@ s3transfer==0.13.0
     # via boto3
 safetensors==0.5.3
     # via transformers
-schwifty==2025.1.0
+schwifty==2025.6.0
     # via txt2stix (pyproject.toml)
 simplejson==3.20.1
     # via stix2
@@ -310,7 +316,7 @@ stix2==3.0.1
     # via stix2extensions
 stix2-patterns==2.0.0
     # via stix2
-stix2extensions @ https://github.com/muchdogesec/stix2extensions/releases/download/main-2025-05-19-15-30-06/stix2extensions-0.0.3-py3-none-any.whl
+stix2extensions==1.0.0
     # via txt2stix (pyproject.toml)
 tenacity==9.1.2
     # via llama-index-core
@@ -320,7 +326,7 @@ tld==0.13.1
     # via txt2stix (pyproject.toml)
 tldextract==5.3.0
     # via txt2stix (pyproject.toml)
-tokenizers==0.21.1
+tokenizers==0.21.2
     # via transformers
 tqdm==4.67.1
     # via
@@ -330,9 +336,9 @@ tqdm==4.67.1
     #   nltk
     #   openai
     #   transformers
-transformers==4.52.4
+transformers==4.53.1
     # via llama-index-llms-openai-like
-typing-extensions==4.14.0
+typing-extensions==4.14.1
     # via
     #   aiosqlite
     #   anthropic
@@ -366,3 +372,6 @@ wrapt==1.17.2
     #   llama-index-core
 yarl==1.20.1
     # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

txt2stix-1.0.1/tests/src/test_bundler.py

@@ -0,0 +1,297 @@
+from unittest.mock import MagicMock, call, patch
+import uuid
+
+import pytest
+from txt2stix.bundler import txt2stixBundler, TLP_LEVEL
+from txt2stix.common import MinorException
+from . import utils
+from dateutil.parser import parse as parse_date
+from stix2 import Identity, Relationship
+from stix2extensions import Weakness, BankCard
+
+dummy_identity = Identity(
+    **{
+        "type": "identity",
+        "spec_version": "2.1",
+        "id": "identity--b1ae1a15-abcd-431e-b990-1b9678f35e15",
+        "name": "Test Identity",
+    }
+)
+
+
+@pytest.mark.parametrize(
+    ["tlp_level", "identity", "created", "modified"],
+    [
+        ("red", None, None, "2024-01-01T16:00:00.000Z"),
+        ("green", None, "2024-10-09T16:00:00.000Z", None),
+        ("amber", dummy_identity, None, "2024-01-01T16:00:00.000Z"),
+        ("clear", None, "2024-10-09T16:00:00.000Z", "2024-01-01T16:00:00.000Z"),
+    ],
+)
+def test_constructor(tlp_level, identity, created, modified):
+    report_id = str(uuid.uuid4())
+    bundler = txt2stixBundler(
+        "n",
+        identity,
+        tlp_level,
+        "",
+        0,
+        [],
+        [],
+        report_id,
+        created=created,
+        modified=modified,
+    )
+    assert bundler.tlp_level.name == tlp_level
+    assert bundler.report.id == "report--" + report_id
+    assert bundler.tlp_level.value["id"] in bundler.report.object_marking_refs
+    assert bundler.report.published == bundler.report.created
+    if identity:
+        assert identity == bundler.identity, "passed identity ignored"
+    else:
+        assert (
+            bundler.identity == bundler.default_identity
+        ), "default identity must be used if no identity is passed"
+    assert (
+        bundler.report.created_by_ref == bundler.identity["id"]
+    ), "report not using bundler.identity"
+    if not modified:
+        assert (
+            bundler.report.modified == bundler.report.created
+        ), "modified and created should be the same if modified is not passed"
+    else:
+        assert bundler.report.modified == parse_date(modified)
+    if created:
+        assert bundler.report.created == parse_date(created)
+
+    assert (
+        bundler.identity in bundler.bundle.objects
+    ), "identity must be in bundle.objects"
+    assert (
+        bundler.tlp_level.value in bundler.bundle.objects
+    ), "tlp_level marking definition must be in bundle.objects"
+    assert bundler.report in bundler.bundle.objects, "report must be in bundle.objects"
+
+
+@pytest.mark.parametrize(
+    "obj",
+    [
+        Weakness(name="test weakness"),
+        BankCard(number="1234567891011"),
+    ],
+)
+def test_add_ref(obj):
+    bundler = utils.dummy_bundler()
+    bundler.add_ref(obj)
+    assert obj in bundler.bundle.objects
+    assert obj.id in bundler.added_objects
+
+
+def test_add_indicator():
+    bundler = utils.dummy_bundler()
+    mocked_extractor = MagicMock()
+    bundler.all_extractors = dict(placeholder_extractor=mocked_extractor)
+    mocked_related_refs = MagicMock()
+    mocked_observables = MagicMock()
+    with patch.object(txt2stixBundler, 'new_indicator') as mock_new_indicator, patch(
+        "txt2stix.bundler.build_observables"
+    ) as mock_build_observables:
+        extracted_dict = dict(type='placeholder_extractor', value='test value', id='extract-19')
+        mock_build_observables.return_value = mocked_observables, mocked_related_refs
+        bundler.add_indicator(extracted_dict, True)
+
+
+        mock_new_indicator.assert_called_once_with(mocked_extractor, mocked_extractor.stix_mapping, extracted_dict['value'])
+        mock_build_observables.assert_called_once_with(bundler, mocked_extractor.stix_mapping, mock_new_indicator.return_value, extracted_dict['value'], mocked_extractor)
+        assert bundler.id_map[extracted_dict["id"]] == mocked_related_refs
+
+
+def test_add_indicator_sets_id_map():
+    extractor = MagicMock()
+    extractor.stix_mapping = "domain-name"
+    extractor.slug = "testslug"
+    extractor.version = "1.0"
+
+    bundler = txt2stixBundler(
+        name="Test",
+        identity=None,
+        tlp_level="red",
+        description="desc",
+        confidence=20,
+        extractors={"domain": extractor},
+        labels=[]
+    )
+
+    # patch build_observables to return one object
+    with patch("txt2stix.bundler.build_observables") as mock_build:
+        mock_build.return_value = ([{"type": "domain-name", "id": "domain-name--926a1335-a4d7-40d3-804c-3aa53da6fc9e", "value": "test.com"}], ["domain-name--926a1335-a4d7-40d3-804c-3aa53da6fc9e"])
+
+        extracted = {"type": "domain", "value": "test.com", "id": "testid"}
+        bundler.add_indicator(extracted, add_standard_relationship=False)
+
+        assert "testid" in bundler.id_map
+        assert bundler.id_map["testid"] == ["domain-name--926a1335-a4d7-40d3-804c-3aa53da6fc9e"]
+        assert "domain-name--926a1335-a4d7-40d3-804c-3aa53da6fc9e" in bundler.id_value_map
+
+def test_add_indicator_raises_minor_exception():
+    extractor = MagicMock()
+    extractor.stix_mapping = "domain-name"
+    extractor.slug = "testslug"
+    extractor.version = "1.0"
+
+    bundler = txt2stixBundler(
+        name="Test",
+        identity=None,
+        tlp_level="red",
+        description="desc",
+        confidence=20,
+        extractors={"domain": extractor},
+        labels=[]
+    )
+
+    # patch build_observables to return one object
+    with patch("txt2stix.bundler.build_observables") as mock_build, pytest.raises(MinorException):
+        mock_build.return_value = ([], [])
+
+        extracted = {"type": "domain", "value": "test.com", "id": "testid"}
+        bundler.add_indicator(extracted, add_standard_relationship=False)
+
+
+
+def test_flow_objects():
+    bundler = txt2stixBundler(
+        name="FlowTest",
+        identity=None,
+        tlp_level="clear",
+        description="desc",
+        confidence=10,
+        extractors={},
+        labels=[]
+    )
+
+    obj = {"id": "indicator--123", "type": "indicator", "name": "x"}
+    bundler.flow_objects = [obj, bundler.report]
+
+    assert "indicator--123" in bundler.id_value_map
+    assert obj in bundler.bundle.objects
+    assert bundler.flow_objects == [obj, bundler.report]
+
+
+
+
+def test_add_standard_relationship():
+    bundler = txt2stixBundler(
+        name="Test",
+        identity=None,
+        tlp_level="amber",
+        description="desc",
+        confidence=30,
+        extractors={},
+        labels=[],
+        report_id="d9f3b306-e7fe-4074-b89a-33ce54280718"
+    )
+
+    bundler.id_value_map["identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c"] = "valA"
+    bundler.id_value_map["phone-number--8764871f-6521-4401-bbf2-a17538435f49"] = "valB"
+
+    bundler.add_standard_relationship("identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c", "phone-number--8764871f-6521-4401-bbf2-a17538435f49", "xx-related-to")
+    assert "relationship--290e1319-7a95-5f51-b2c5-463f896cb35a" in bundler.added_objects
+    
+
+    found = [obj for obj in bundler.bundle.objects if obj['id'] == "relationship--290e1319-7a95-5f51-b2c5-463f896cb35a"][0]
+    assert found.description == "valA xx related to valB"
+    assert found.relationship_type == "xx-related-to"
+    assert found.source_ref == "identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c"
+    assert found.target_ref == "phone-number--8764871f-6521-4401-bbf2-a17538435f49"
+
+def test_add_ai_relationship():
+    bundler = txt2stixBundler(
+        name="Test",
+        identity=None,
+        tlp_level="amber",
+        description="desc",
+        confidence=30,
+        extractors={},
+        labels=[],
+        report_id="d9f3b306-e7fe-4074-b89a-33ce54280718"
+    )
+    bundler.id_map['ex1'] = ["phone-number--8764871f-6521-4401-bbf2-a17538435f49", "indicator--401855b2-bd7a-444f-95b1-723efbdba33b"]
+    bundler.id_map['ex2'] = ["identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c"]
+    bundler.id_value_map["identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c"] = "valA"
+    bundler.id_value_map["phone-number--8764871f-6521-4401-bbf2-a17538435f49"] = "valB"
+
+    with patch.object(txt2stixBundler, 'add_standard_relationship') as mock_add_standard_relationship:
+        bundler.add_ai_relationship(dict(source_ref='ex1', target_ref='ex2', relationship_type='in-use-by'))
+        mock_add_standard_relationship.assert_any_call("phone-number--8764871f-6521-4401-bbf2-a17538435f49", "identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c", "in-use-by")
+        mock_add_standard_relationship.assert_any_call("indicator--401855b2-bd7a-444f-95b1-723efbdba33b", "identity--6493ad42-ec4d-4260-b2e9-3f3a1110193c", "in-use-by")
+
+
+
+def test_add_summary():
+    bundler = txt2stixBundler(
+        name="Test",
+        identity=None,
+        tlp_level="amber",
+        description="desc",
+        confidence=30,
+        extractors={},
+        labels=['report-label1'],
+        report_id="d9f3b306-e7fe-4074-b89a-33ce54280718"
+    )
+    summary = "This is a summary"
+    bundler.add_summary(summary, "some-random-ai-provider")
+    assert 'note--d9f3b306-e7fe-4074-b89a-33ce54280718' in bundler.added_objects
+    assert 'relationship--fc73fe53-9487-540f-bd65-582d9d2d1b54' in bundler.added_objects
+    note_obj = [obj for obj in bundler.bundle.objects if obj['id'] == 'note--d9f3b306-e7fe-4074-b89a-33ce54280718'][0]
+    assert note_obj.content == summary
+    assert note_obj.object_refs == ["report--d9f3b306-e7fe-4074-b89a-33ce54280718"]
+    for k in ['created', 'modified', 'created_by_ref', 'object_marking_refs', 'labels', 'confidence']:
+        assert bundler.report[k] == note_obj[k]
+
+    ref_obj = [obj for obj in bundler.bundle.objects if obj['id'] == 'relationship--fc73fe53-9487-540f-bd65-582d9d2d1b54'][0]
+    assert ref_obj.description == "AI generated summary for Test"
+    assert ref_obj.external_references == note_obj.external_references
+    
+    
+def test_process_observables_and_process_relationships():
+    extractor = MagicMock()
+    extractor.stix_mapping = "domain-name"
+    extractor.slug = "testslug"
+    extractor.version = "1.0"
+
+    bundler = txt2stixBundler(
+        name="Test Process",
+        identity=None,
+        tlp_level="amber_strict",
+        description="desc",
+        confidence=90,
+        extractors={"domain": extractor},
+        labels=[]
+    )
+
+    with patch("txt2stix.bundler.build_observables") as mock_build:
+        mock_build.return_value = ([{"type": "domain-name", "id": "domain-name--5b35eddb-c7fc-43c6-859b-36bb859ebb7c", "value": "foo.com"}], ["domain-name--5b35eddb-c7fc-43c6-859b-36bb859ebb7c"])
+        bundler.process_observables([{"type": "domain", "value": "foo.com"}])
+
+        assert bundler.observables_processed == 1
+        assert "domain-name--5b35eddb-c7fc-43c6-859b-36bb859ebb7c" in bundler.id_value_map
+
+    # test process_relationships
+    bundler.id_map = {"ai_src": ["a"], "ai_tgt": ["b"]}
+    with patch.object(bundler, "add_standard_relationship") as mock_add_rel:
+        bundler.process_relationships([{"source_ref": "ai_src", "target_ref": "ai_tgt", "relationship_type": "controls"}])
+        mock_add_rel.assert_called_once_with("a", "b", "controls")
+
+def test_tlp_level_values():
+    values = TLP_LEVEL.values()
+    assert all(v.type == "marking-definition" for v in values)
+    assert len(values) == 5
+
+def test_tlp_level_get_by_enum():
+    assert TLP_LEVEL.get(TLP_LEVEL.CLEAR) == TLP_LEVEL.CLEAR
+
+def test_tlp_level_get_by_string():
+    assert TLP_LEVEL.get("clear") == TLP_LEVEL.CLEAR
+    assert TLP_LEVEL.get("amber_strict") == TLP_LEVEL.AMBER_STRICT
+    assert TLP_LEVEL.get("amber+strict") == TLP_LEVEL.AMBER_STRICT
+    assert TLP_LEVEL.get("amber-strict") == TLP_LEVEL.AMBER_STRICT