txt2stix 1.2.3__tar.gz → 1.2.5__tar.gz

{txt2stix-1.2.3 → txt2stix-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: txt2stix
-Version: 1.2.3
+Version: 1.2.5
 Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
 Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
 Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues

{txt2stix-1.2.3 → txt2stix-1.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2stix"
-version = "1.2.3"
+version = "1.2.5"
 authors = [{ name = "dogesec" }]
 maintainers = [{ name = "dogesec" }]
 description = "txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle."

{txt2stix-1.2.3 → txt2stix-1.2.5}/tests/src/test_bundler.py

@@ -114,6 +114,20 @@ def test_add_indicator(bundler):
         assert bundler.id_map[extracted_dict["id"]] == mocked_related_refs
 
 
+def test_add_indicator_rejects_empty_values(bundler):
+    """Test that add_indicator raises MinorException for None or empty extracted values."""    
+    extracted_dict_empty = {
+        "id": "ex-1",
+        "type": "test_type",
+        "value": ""
+    }
+    mocked_extractor = MagicMock()
+    bundler.all_extractors = dict(test_type=mocked_extractor)
+    
+    with pytest.raises(MinorException, match="extracted value is empty"):
+        bundler.add_indicator(extracted_dict_empty, add_standard_relationship=False)
+
+
 def test_add_indicator_sets_id_map():
     extractor = MagicMock()
     extractor.stix_mapping = "domain-name"

{txt2stix-1.2.3 → txt2stix-1.2.5}/tests/src/test_main.py

@@ -30,8 +30,10 @@ from txt2stix.txt2stix import (
     processing_phase,
     run_extractors,
     extract_relationships,
+    validate_token_count,
 )
-from txt2stix.common import FatalException
+from txt2stix.common import FatalException, MinorException
+from txt2stix.bundler import txt2stixBundler
 import argparse
 
 
@@ -419,7 +421,9 @@ def test_processing_phase_applies_extracts_and_relationships():
     preprocessed_text = "SOME TEXT"
     # prepare data object similar to Txt2StixData
     data = SimpleNamespace()
-    data.extractions = {'lookup': ['l1'], 'pattern': ['p1']}
+    mock1 = {"name": "observable1", "error": "some error"}
+    mock2 = {"name": "observable2", "error": "some error"}
+    data.extractions = {'lookup': [mock1], 'pattern': [mock2]}
     data.relationships = {'relationships': ['r1']}
     data.content_check = None
 
@@ -428,6 +432,97 @@ def test_processing_phase_applies_extracts_and_relationships():
 
     processing_phase(bundler, preprocessed_text, data, ai_create_attack_flow=False, ai_create_attack_navigator_layer=False)
 
-    bundler.process_observables.assert_any_call(['l1'])
-    bundler.process_observables.assert_any_call(['p1'])
-    bundler.process_relationships.assert_called_once_with(['r1'])
+    bundler.process_observables.assert_any_call([mock1])
+    bundler.process_observables.assert_any_call([mock2])
+    assert "error" not in mock2  # ensure no error field removal in this test
+    assert "error" not in mock1  # ensure no error field removal in this test
+    bundler.process_relationships.assert_called_once_with(['r1'])
+
+
+def test_run_extractors():
+    """Test run_extractors returns dict with id fields and handles AI extractor failures."""
+    text_content = "SAMPLE TEXT"
+    extractors_map = {
+        'lookup': {'lookup1': MagicMock()},
+        'pattern': {'pattern1': MagicMock()},
+        'ai': {'ai1': MagicMock()}
+    }
+    
+    mock_ai_extractor1 = MagicMock()
+    mock_ai_extractor1.extractor_name = "ai_extractor_1"
+    mock_ai_extractor1.extract_objects.return_value = [{'type': 'ipv4', 'value': '1.1.1.1'}]
+    
+    mock_ai_extractor2 = MagicMock()
+    mock_ai_extractor2.extractor_name = "ai_extractor_2"
+    mock_ai_extractor2.extract_objects.side_effect = Exception("AI extraction failed")
+    
+    with patch('txt2stix.txt2stix.lookups.extract_all') as mock_lookup, \
+         patch('txt2stix.txt2stix.pattern.extract_all') as mock_pattern:
+        
+        mock_lookup.return_value = [{'type': 'domain', 'value': 'example.com'}]
+        mock_pattern.return_value = [{'type': 'url', 'value': 'http://test.com'}]
+        
+        result = run_extractors(extractors_map, text_content, ai_extractors=[mock_ai_extractor1, mock_ai_extractor2])
+        
+        # Verify lookup and pattern were called
+        assert 'lookup' in result
+        assert 'pattern' in result
+        assert 'ai-ai_extractor_1' in result
+        
+        # Verify all extracts have 'id' field
+        all_extracts = []
+        for extracts in result.values():
+            all_extracts.extend(extracts)
+        
+        for i, extract in enumerate(all_extracts):
+            assert 'id' in extract
+            assert extract['id'].startswith('ex-')
+        
+        # Verify AI extractor 2 failure was handled (should not be in results)
+        assert 'ai-ai_extractor_2' not in result
+
+
+def test_extract_relationships():
+    """Test extract_relationships calls AI session and handles exceptions."""
+    text = "TEXT_CONTENT"
+    all_extracts = {"lookup": [{'id': 'ex-0'}, {'id': 'ex-1'}], "ai": [{'id': 'ex-2'}, {'id': 'ex-3'}]}
+    mock_ai_session = MagicMock()
+    mock_ai_session.extract_relationships.return_value.model_dump.return_value = {
+        "relationships": [{'source': 'ex-0', 'target': 'ex-2'}]
+    }
+
+    relationships = extract_relationships(text, all_extracts, mock_ai_session)
+
+    # Verify extract_relationships was called with flattened list
+    called_args = mock_ai_session.extract_relationships.call_args
+    assert len(called_args[0][1]) == 4  # flattened list of 4 extracts
+    assert called_args[0][2] == RELATIONSHIP_TYPES
+    
+    mock_ai_session.extract_relationships.return_value.model_dump.assert_called()
+    assert relationships == {"relationships": [{'source': 'ex-0', 'target': 'ex-2'}]}
+
+    # Test exception handling - should return None
+    mock_ai_session.extract_relationships.side_effect = Exception("AI failed")
+    assert extract_relationships(text, all_extracts, mock_ai_session) is None
+
+
+def test_validate_token_count():
+    """Test validate_token_count raises FatalException when limit exceeded."""
+    
+    # Test case 1: Should not raise when under limit
+    mock_extractor1 = MagicMock()
+    mock_extractor1.extractor_name = "test_extractor_under_limit"
+    mock_extractor1.count_tokens.return_value = 500
+    
+    # Use different input text to avoid lru_cache collision
+    validate_token_count(1000, "test input 1", [mock_extractor1])
+    
+    # Test case 2: Should raise when over limit
+    mock_extractor2 = MagicMock()
+    mock_extractor2.extractor_name = "test_extractor_over_limit"
+    mock_extractor2.count_tokens.return_value = 1500
+    
+    with pytest.raises(FatalException, match="exceeds INPUT_TOKEN_LIMIT"):
+        # Use different input text to avoid lru_cache collision
+        validate_token_count(1000, "test input 2", [mock_extractor2])
+

{txt2stix-1.2.3 → txt2stix-1.2.5}/txt2stix/bundler.py

@@ -307,6 +307,9 @@ class txt2stixBundler:
         extracted_value = extracted_dict["value"]
         extracted_id = extracted_dict["id"]
 
+        if extracted_value is None or extracted_value == "":
+            raise MinorException(f"extracted value is empty")
+
         indicator = self.new_indicator(extractor, stix_mapping, extracted_value)
         # set id so it doesn''t need to be created in build_observables
         if extracted_dict.get("indexes"):

{txt2stix-1.2.3 → txt2stix-1.2.5}/txt2stix/txt2stix.py

@@ -556,6 +556,8 @@ def processing_phase(
     ai_content_check_provider=None,
 ):
     """Process extracted `data` into the given `bundler` without invoking LLMs."""
+    for d in itertools.chain([], *(data.extractions or {}).values()):
+        d.pop("error", None)
     try:
         if data.content_check:
             cc = data.content_check