txt2stix 1.0.10__tar.gz → 1.0.12__tar.gz

{txt2stix-1.0.10 → txt2stix-1.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: txt2stix
-Version: 1.0.10
+Version: 1.0.12
 Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
 Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
 Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues

{txt2stix-1.0.10 → txt2stix-1.0.12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "txt2stix"
-version = "1.0.10"
+version = "1.0.12"
 authors = [{ name = "dogesec" }]
 maintainers = [{ name = "dogesec" }]
 description = "txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle."

txt2stix-1.0.12/tests/data/manually_generated_reports/ai_index_position.txt

@@ -0,0 +1,2 @@
+1.1.1.1
+1.1.1.1

{txt2stix-1.0.10 → txt2stix-1.0.12}/tests/manual-tests/cases-standard-tests.md

@@ -540,4 +540,21 @@ python3 txt2stix.py \
     --ai_create_attack_flow \
     --ai_create_attack_navigator_layer \
     --report_id c0d48262-1d9f-42d2-aa29-f0cba1bfa2e0
+```
+
+### test AI extraction position
+
+same extraction twice in doc
+
+```shell
+python3 txt2stix.py \
+    --relationship_mode ai \
+    --ai_settings_relationships openai:gpt-4o \
+    --input_file tests/data/manually_generated_reports/ai_index_position.txt \
+    --name 'Extraction index' \
+    --tlp_level clear \
+    --confidence 100 \
+    --use_extractions 'ai_ipv4_address_only' \
+    --ai_settings_extractions openai:gpt-4o \
+    --report_id 2b3326b4-dfcf-4391-b550-e91652f9ffcd
 ```

{txt2stix-1.0.10 → txt2stix-1.0.12}/txt2stix/ai_extractor/base.py

@@ -10,6 +10,8 @@ from txt2stix.ai_extractor.prompts import DEFAULT_CONTENT_CHECKER_WITH_SUMMARY_T
 from txt2stix.ai_extractor.utils import AttackFlowList, DescribesIncident, ExtractionList, ParserWithLogging, RelationshipList, get_extractors_str
 from llama_index.core.utils import get_tokenizer
 
+from txt2stix.lookups import find_get_indexes
+
 
 _ai_extractor_registry: dict[str, 'Type[BaseAIExtractor]'] = {}
 class BaseAIExtractor():
@@ -67,8 +69,11 @@ class BaseAIExtractor():
     def extract_relationships(self, input_text, extractions, relationship_types: list[str]) -> RelationshipList:
         return self._get_relationship_program()(relationship_types=relationship_types, input_file=input_text, extractions=extractions)
 
-    def extract_objects(self, input_text, extractors) -> ExtractionList:
-        extraction_list = self._get_extraction_program()(extractors=get_extractors_str(extractors), input_file=input_text)
+    def extract_objects(self, input_text: str, extractors) -> ExtractionList:
+        extraction_list: ExtractionList = self._get_extraction_program()(extractors=get_extractors_str(extractors), input_file=input_text)
+        for extract in extraction_list.extractions:
+            extract.start_index = list(find_get_indexes(extract.original_text, input_text))
+
         return extraction_list.model_dump().get('extractions', [])
 
     def __init__(self, *args, **kwargs) -> None:

{txt2stix-1.0.10 → txt2stix-1.0.12}/txt2stix/ai_extractor/utils.py

@@ -15,7 +15,8 @@ class Extraction(BaseModel):
     id: str =  Field(description='is the id of the extraction of the format `"ai-%d" %(position in list)`, it should start from 1 (e.g `"ai-1", "ai-2", ..., "ai-n"`)')
     value: str  =  Field(description='is the value extracted from the text')
     original_text: str =  Field(description='is the original text the extraction was made from')
-    # start_index: list[str|int] =  Field(description='a list of the index positions of the first character for each matching extraction. Some documents might capture many extractions where `key` and `value` are the same for many entries. This property allows the user to identify how many extractions happened, and where they are in the document.')
+    start_index: list[str|int] =  Field(default_factory=list, description='no result expected')
+
 
 class Relationship(BaseModel):
     source_ref: str = Field(description='is the id for the source extraction for the relationship (e.g. extraction_1).')
@@ -85,9 +86,3 @@ def get_extractors_str(extractors):
     logging.debug(buffer.getvalue())
     logging.debug("======== extractors end ======")
     return buffer.getvalue()
-
-
-
-if __name__ == '__main__':
-    a = ExtractionList(extractions=[Extraction(type="yes", id="1", value="2", original_text="3")], success=True)
-    print(a.model_dump())

{txt2stix-1.0.10 → txt2stix-1.0.12}/txt2stix/txt2stix.py

@@ -439,18 +439,18 @@ def main():
 
         ## write outputs
         out = bundler.to_json()
-        output_dir = Path("./output")/str(job_id)
+        output_dir = Path("./output")/str(bundler.uuid)
         with contextlib.suppress(BaseException):
             shutil.rmtree(output_dir)
         output_dir.mkdir(exist_ok=True, parents=True)
         output_path = output_dir/f"{bundler.bundle.id}.json"
         output_path.write_text(out)
         logger.info(f"Wrote bundle output to `{output_path}`")
-        data_path = output_dir/f"data--{args.report_id}.json"
+        data_path = output_dir/f"data--{bundler.uuid}.json"
         data_path.write_text(data.model_dump_json(indent=4))
         logger.info(f"Wrote data output to `{data_path}`")
         for nav_layer in data.navigator_layer or []:
-            nav_path = output_dir/f"navigator-{nav_layer['domain']}----{args.report_id}.json"
+            nav_path = output_dir/f"navigator-{nav_layer['domain']}----{bundler.uuid}.json"
             nav_path.write_text(json.dumps(nav_layer, indent=4))
             logger.info(f"Wrote navigator output to `{nav_path}`")
     except argparse.ArgumentError as e: