txt2stix 1.0.11__tar.gz → 1.0.13__tar.gz

{txt2stix-1.0.11 → txt2stix-1.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: txt2stix
-Version: 1.0.11
+Version: 1.0.13
 Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
 Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
 Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues

{txt2stix-1.0.11 → txt2stix-1.0.13}/includes/extractions/ai/config.yaml

@@ -28,7 +28,7 @@ ai_ipv4_address_cidr:
   modified:  2020-01-01
   created_by: DOGESEC
   version: 1.0.0
-  prompt_base: 'Extract all IPv4 addresses with a CIDR from the text.'
+  prompt_base: 'Extract all IPv4 addresses with a CIDR from the text. CIDR part must be >=0 <=32.'
   prompt_helper: 'Do not include any IPs that do not have a CIDR.'
   prompt_conversion: ''
   test_cases: ipv4_address_cidr
@@ -44,8 +44,8 @@ ai_ipv4_address_port:
   modified:  2020-01-01
   created_by: DOGESEC
   version: 1.0.0
-  prompt_base: 'Extract all IPv4 addresses with a port from the text.'
-  prompt_helper: 'Do not include any IPv4s that do not contain a port number'
+  prompt_base: 'Extract all IPv4 addresses with a port from the text. Port number part must be >=0 <=65535.'
+  prompt_helper: 'Do not include any IPv4s that do not contain a port number.'
   prompt_conversion: ''
   test_cases: ipv4_address_port
   stix_mapping: ipv4-addr-port
@@ -72,13 +72,13 @@ ai_ipv6_address_cidr:
   type: ai
   dogesec_web: true
   name: 'IPv6 Address with CIDR'
-  description: 'Extracts IPv6 addresses with CIDRs'
+  description: 'Extracts IPv6 addresses with CIDRs.'
   notes: 'pattern_ipv6_address_cidr legacy extraction also exists if you cannot use AI'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: DOGESEC
   version: 1.0.0
-  prompt_base: 'Extract all IPv6 addresses with a CIDR from the text.'
+  prompt_base: 'Extract all IPv6 addresses with a CIDR from the text. CIDR part must be >=0 <=128.'
   prompt_helper: 'Do not include any IPv6s that do not contain a CIDR'
   prompt_conversion: ''
   test_cases: generic_ipv6_address_cidr
@@ -94,7 +94,7 @@ ai_ipv6_address_port:
   modified:  2020-01-01
   created_by: DOGESEC
   version: 1.0.0
-  prompt_base: 'Extract all IPv6 addresses with a CIDR from the text.'
+  prompt_base: 'Extract all IPv6 addresses with a CIDR from the text. Port number part must be >=0 <=65535.'
   prompt_helper: 'Do not include any IPv6s that do not contain a port number'
   prompt_conversion: ''
   test_cases: generic_ipv6_address_port

{txt2stix-1.0.11 → txt2stix-1.0.13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2stix"
-version = "1.0.11"
+version = "1.0.13"
 authors = [{ name = "dogesec" }]
 maintainers = [{ name = "dogesec" }]
 description = "txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle."

txt2stix-1.0.13/tests/data/manually_generated_reports/ai_index_position.txt

@@ -0,0 +1,2 @@
+1.1.1.1
+1.1.1.1

{txt2stix-1.0.11 → txt2stix-1.0.13}/tests/manual-tests/cases-standard-tests.md

@@ -540,4 +540,21 @@ python3 txt2stix.py \
     --ai_create_attack_flow \
     --ai_create_attack_navigator_layer \
     --report_id c0d48262-1d9f-42d2-aa29-f0cba1bfa2e0
+```
+
+### test AI extraction position
+
+same extraction twice in doc
+
+```shell
+python3 txt2stix.py \
+    --relationship_mode ai \
+    --ai_settings_relationships openai:gpt-4o \
+    --input_file tests/data/manually_generated_reports/ai_index_position.txt \
+    --name 'Extraction index' \
+    --tlp_level clear \
+    --confidence 100 \
+    --use_extractions 'ai_ipv4_address_only' \
+    --ai_settings_extractions openai:gpt-4o \
+    --report_id 2b3326b4-dfcf-4391-b550-e91652f9ffcd
 ```

{txt2stix-1.0.11 → txt2stix-1.0.13}/txt2stix/ai_extractor/base.py

@@ -10,6 +10,8 @@ from txt2stix.ai_extractor.prompts import DEFAULT_CONTENT_CHECKER_WITH_SUMMARY_T
 from txt2stix.ai_extractor.utils import AttackFlowList, DescribesIncident, ExtractionList, ParserWithLogging, RelationshipList, get_extractors_str
 from llama_index.core.utils import get_tokenizer
 
+from txt2stix.lookups import find_get_indexes
+
 
 _ai_extractor_registry: dict[str, 'Type[BaseAIExtractor]'] = {}
 class BaseAIExtractor():
@@ -67,8 +69,11 @@ class BaseAIExtractor():
     def extract_relationships(self, input_text, extractions, relationship_types: list[str]) -> RelationshipList:
         return self._get_relationship_program()(relationship_types=relationship_types, input_file=input_text, extractions=extractions)
 
-    def extract_objects(self, input_text, extractors) -> ExtractionList:
-        extraction_list = self._get_extraction_program()(extractors=get_extractors_str(extractors), input_file=input_text)
+    def extract_objects(self, input_text: str, extractors) -> ExtractionList:
+        extraction_list: ExtractionList = self._get_extraction_program()(extractors=get_extractors_str(extractors), input_file=input_text)
+        for extract in extraction_list.extractions:
+            extract.start_index = list(find_get_indexes(extract.original_text, input_text))
+
         return extraction_list.model_dump().get('extractions', [])
 
     def __init__(self, *args, **kwargs) -> None:

{txt2stix-1.0.11 → txt2stix-1.0.13}/txt2stix/ai_extractor/utils.py

@@ -15,7 +15,8 @@ class Extraction(BaseModel):
     id: str =  Field(description='is the id of the extraction of the format `"ai-%d" %(position in list)`, it should start from 1 (e.g `"ai-1", "ai-2", ..., "ai-n"`)')
     value: str  =  Field(description='is the value extracted from the text')
     original_text: str =  Field(description='is the original text the extraction was made from')
-    # start_index: list[str|int] =  Field(description='a list of the index positions of the first character for each matching extraction. Some documents might capture many extractions where `key` and `value` are the same for many entries. This property allows the user to identify how many extractions happened, and where they are in the document.')
+    start_index: list[str|int] =  Field(default_factory=list, description='no result expected')
+
 
 class Relationship(BaseModel):
     source_ref: str = Field(description='is the id for the source extraction for the relationship (e.g. extraction_1).')
@@ -85,9 +86,3 @@ def get_extractors_str(extractors):
     logging.debug(buffer.getvalue())
     logging.debug("======== extractors end ======")
     return buffer.getvalue()
-
-
-
-if __name__ == '__main__':
-    a = ExtractionList(extractions=[Extraction(type="yes", id="1", value="2", original_text="3")], success=True)
-    print(a.model_dump())