txt2stix 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- txt2stix/ai_extractor/openai.py +1 -1
- txt2stix/attack_flow.py +1 -3
- txt2stix/bundler.py +58 -26
- txt2stix/includes/extractions/ai/config.yaml +8 -8
- txt2stix/includes/extractions/pattern/config.yaml +14 -14
- txt2stix/indicator.py +47 -32
- txt2stix/pattern/extractors/card/discover_card_extractor.py +1 -1
- txt2stix/retriever.py +8 -2
- {txt2stix-1.1.2.dist-info → txt2stix-1.1.4.dist-info}/METADATA +1 -1
- {txt2stix-1.1.2.dist-info → txt2stix-1.1.4.dist-info}/RECORD +13 -13
- {txt2stix-1.1.2.dist-info → txt2stix-1.1.4.dist-info}/WHEEL +0 -0
- {txt2stix-1.1.2.dist-info → txt2stix-1.1.4.dist-info}/entry_points.txt +0 -0
- {txt2stix-1.1.2.dist-info → txt2stix-1.1.4.dist-info}/licenses/LICENSE +0 -0
txt2stix/ai_extractor/openai.py
CHANGED
|
@@ -8,7 +8,7 @@ from llama_index.llms.openai import OpenAI
|
|
|
8
8
|
class OpenAIExtractor(BaseAIExtractor, provider="openai"):
|
|
9
9
|
def __init__(self, **kwargs) -> None:
|
|
10
10
|
kwargs.setdefault('temperature', float(os.environ.get('TEMPERATURE', 0.0)))
|
|
11
|
-
self.llm = OpenAI(system_prompt=self.system_prompt, **kwargs)
|
|
11
|
+
self.llm = OpenAI(system_prompt=self.system_prompt, timeout=300, **kwargs)
|
|
12
12
|
super().__init__()
|
|
13
13
|
|
|
14
14
|
def count_tokens(self, text):
|
txt2stix/attack_flow.py
CHANGED
|
@@ -8,15 +8,13 @@ from txt2stix.ai_extractor.base import BaseAIExtractor
|
|
|
8
8
|
from txt2stix.common import UUID_NAMESPACE
|
|
9
9
|
from txt2stix.retriever import STIXObjectRetriever
|
|
10
10
|
from stix2extensions.attack_action import AttackAction, AttackFlow
|
|
11
|
-
from stix2extensions._extensions import attack_flow_ExtensionDefinitionSMO
|
|
12
11
|
from .utils import AttackFlowList
|
|
13
12
|
|
|
14
|
-
|
|
15
13
|
def parse_flow(report, flow: AttackFlowList, techniques, tactics):
|
|
16
14
|
logging.info(f"flow.success = {flow.success}")
|
|
17
15
|
if not flow.success:
|
|
18
16
|
return []
|
|
19
|
-
objects = [report
|
|
17
|
+
objects = [report]
|
|
20
18
|
for domain in ["enterprise-attack", "mobile-attack", "ics-attack"]:
|
|
21
19
|
flow_objects = parse_domain_flow(report, flow, techniques, tactics, domain)
|
|
22
20
|
objects.extend(flow_objects)
|
txt2stix/bundler.py
CHANGED
|
@@ -113,7 +113,7 @@ class TLP_LEVEL(enum.Enum):
|
|
|
113
113
|
@classmethod
|
|
114
114
|
def get(cls, level):
|
|
115
115
|
if isinstance(level, str):
|
|
116
|
-
level = level.replace(
|
|
116
|
+
level = level.replace("-", "_").replace("+", "_")
|
|
117
117
|
if isinstance(level, cls):
|
|
118
118
|
return level
|
|
119
119
|
return cls.levels()[level]
|
|
@@ -128,12 +128,13 @@ class txt2stixBundler:
|
|
|
128
128
|
"user-agent": None,
|
|
129
129
|
"cryptocurrency-wallet": None,
|
|
130
130
|
"cryptocurrency-transaction": None,
|
|
131
|
-
"
|
|
131
|
+
"payment-card": None,
|
|
132
132
|
"bank-account": None,
|
|
133
133
|
"phone-number": None,
|
|
134
134
|
"weakness": None,
|
|
135
135
|
}
|
|
136
136
|
EXTENSION_DEFINITION_BASE_URL = "https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions"
|
|
137
|
+
ATTACK_FLOW_SMO_URL = "https://github.com/muchdogesec/stix2extensions/raw/refs/heads/main/remote-definitions/attack-flow.json"
|
|
137
138
|
report = None
|
|
138
139
|
identity = None
|
|
139
140
|
object_marking_refs = []
|
|
@@ -204,7 +205,7 @@ class txt2stixBundler:
|
|
|
204
205
|
)
|
|
205
206
|
external_references = external_references or []
|
|
206
207
|
labels = labels or []
|
|
207
|
-
labels.append(
|
|
208
|
+
labels.append("placeholder_label")
|
|
208
209
|
|
|
209
210
|
self.job_id = f"report--{self.uuid}"
|
|
210
211
|
self.report_md5 = hashlib.md5(description.encode()).hexdigest()
|
|
@@ -230,11 +231,12 @@ class txt2stixBundler:
|
|
|
230
231
|
"source_name": "txt2stix_report_md5",
|
|
231
232
|
"description": self.report_md5,
|
|
232
233
|
},
|
|
233
|
-
]
|
|
234
|
+
]
|
|
235
|
+
+ external_references,
|
|
234
236
|
confidence=confidence,
|
|
235
237
|
)
|
|
236
238
|
self.report.object_refs.clear() # clear object refs
|
|
237
|
-
self.report.labels.pop(-1)
|
|
239
|
+
self.report.labels.pop(-1) # remove txt2stix placeholder
|
|
238
240
|
self.added_objects = set()
|
|
239
241
|
self.set_defaults()
|
|
240
242
|
|
|
@@ -277,7 +279,17 @@ class txt2stixBundler:
|
|
|
277
279
|
self.bundle.objects.append(sdo)
|
|
278
280
|
|
|
279
281
|
sdo_value = ""
|
|
280
|
-
for key in [
|
|
282
|
+
for key in [
|
|
283
|
+
"name",
|
|
284
|
+
"value",
|
|
285
|
+
"path",
|
|
286
|
+
"key",
|
|
287
|
+
"string",
|
|
288
|
+
"number",
|
|
289
|
+
"iban_number",
|
|
290
|
+
"address",
|
|
291
|
+
"hashes",
|
|
292
|
+
]:
|
|
281
293
|
if v := sdo.get(key):
|
|
282
294
|
sdo_value = v
|
|
283
295
|
break
|
|
@@ -305,7 +317,7 @@ class txt2stixBundler:
|
|
|
305
317
|
)
|
|
306
318
|
)
|
|
307
319
|
objects, related_refs = build_observables(
|
|
308
|
-
self, stix_mapping, indicator, extracted_dict[
|
|
320
|
+
self, stix_mapping, indicator, extracted_dict["value"], extractor
|
|
309
321
|
)
|
|
310
322
|
if not objects:
|
|
311
323
|
raise MinorException(
|
|
@@ -349,17 +361,30 @@ class txt2stixBundler:
|
|
|
349
361
|
for source_ref in self.id_map.get(gpt_out["source_ref"], []):
|
|
350
362
|
for target_ref in self.id_map.get(gpt_out["target_ref"], []):
|
|
351
363
|
self.add_standard_relationship(
|
|
352
|
-
source_ref,
|
|
364
|
+
source_ref,
|
|
365
|
+
target_ref,
|
|
366
|
+
gpt_out["relationship_type"],
|
|
353
367
|
)
|
|
354
368
|
|
|
355
369
|
def add_standard_relationship(self, source_ref, target_ref, relationship_type):
|
|
356
|
-
descriptor =
|
|
357
|
-
self.add_ref(
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
370
|
+
descriptor = " ".join(relationship_type.split("-"))
|
|
371
|
+
self.add_ref(
|
|
372
|
+
self.new_relationship(
|
|
373
|
+
source_ref,
|
|
374
|
+
target_ref,
|
|
375
|
+
relationship_type,
|
|
376
|
+
description=f"{self.id_value_map.get(source_ref, source_ref)} {descriptor} {self.id_value_map.get(target_ref, target_ref)}",
|
|
377
|
+
)
|
|
378
|
+
)
|
|
361
379
|
|
|
362
|
-
def new_relationship(
|
|
380
|
+
def new_relationship(
|
|
381
|
+
self,
|
|
382
|
+
source_ref,
|
|
383
|
+
target_ref,
|
|
384
|
+
relationship_type,
|
|
385
|
+
description=None,
|
|
386
|
+
external_references=None,
|
|
387
|
+
):
|
|
363
388
|
relationship = dict(
|
|
364
389
|
id="relationship--"
|
|
365
390
|
+ str(
|
|
@@ -367,8 +392,8 @@ class txt2stixBundler:
|
|
|
367
392
|
UUID_NAMESPACE, f"{relationship_type}+{source_ref}+{target_ref}"
|
|
368
393
|
)
|
|
369
394
|
),
|
|
370
|
-
type=
|
|
371
|
-
spec_version=
|
|
395
|
+
type="relationship",
|
|
396
|
+
spec_version="2.1",
|
|
372
397
|
source_ref=source_ref,
|
|
373
398
|
target_ref=target_ref,
|
|
374
399
|
relationship_type=relationship_type,
|
|
@@ -377,7 +402,8 @@ class txt2stixBundler:
|
|
|
377
402
|
description=description,
|
|
378
403
|
modified=self.report.modified,
|
|
379
404
|
object_marking_refs=self.report.object_marking_refs,
|
|
380
|
-
external_references=external_references
|
|
405
|
+
external_references=external_references
|
|
406
|
+
or [
|
|
381
407
|
{
|
|
382
408
|
"source_name": "txt2stix_report_id",
|
|
383
409
|
"external_id": self.uuid,
|
|
@@ -386,7 +412,7 @@ class txt2stixBundler:
|
|
|
386
412
|
)
|
|
387
413
|
error = relationships_strict(relationship)
|
|
388
414
|
if error:
|
|
389
|
-
relationship[
|
|
415
|
+
relationship["relationship_type"] = "related-to"
|
|
390
416
|
logger.debug(error)
|
|
391
417
|
return parse_stix(relationship, allow_custom=True)
|
|
392
418
|
|
|
@@ -396,7 +422,9 @@ class txt2stixBundler:
|
|
|
396
422
|
def process_observables(self, extractions, add_standard_relationship=False):
|
|
397
423
|
for ex in extractions:
|
|
398
424
|
try:
|
|
399
|
-
if ex.get(
|
|
425
|
+
if ex.get("id", "").startswith(
|
|
426
|
+
"ai"
|
|
427
|
+
): # so id is distinct across multiple AIExtractors
|
|
400
428
|
ex["id"] = f'{ex["id"]}_{self.observables_processed}'
|
|
401
429
|
ex["id"] = ex.get("id", f"ex_{self.observables_processed}")
|
|
402
430
|
self.observables_processed += 1
|
|
@@ -406,7 +434,7 @@ class txt2stixBundler:
|
|
|
406
434
|
f"ran into exception while processing observable `{ex}`. {e}",
|
|
407
435
|
exc_info=True,
|
|
408
436
|
)
|
|
409
|
-
ex[
|
|
437
|
+
ex["error"] = str(e)
|
|
410
438
|
|
|
411
439
|
def process_relationships(self, observables):
|
|
412
440
|
for relationship in observables:
|
|
@@ -420,29 +448,33 @@ class txt2stixBundler:
|
|
|
420
448
|
|
|
421
449
|
def indicator_id_from_value(self, value, stix_mapping):
|
|
422
450
|
return "indicator--" + str(
|
|
423
|
-
uuid.uuid5(
|
|
451
|
+
uuid.uuid5(
|
|
452
|
+
UUID_NAMESPACE,
|
|
453
|
+
f"txt2stix+{self.identity['id']}+{self.report_md5}+{stix_mapping}+{value}",
|
|
454
|
+
)
|
|
424
455
|
)
|
|
425
456
|
|
|
426
457
|
def add_summary(self, summary, ai_summary_provider):
|
|
427
458
|
self.report.external_references.append(
|
|
428
459
|
dict(
|
|
429
|
-
source_name=
|
|
460
|
+
source_name="txt2stix_ai_summary",
|
|
430
461
|
external_id=ai_summary_provider,
|
|
431
|
-
description=summary
|
|
462
|
+
description=summary,
|
|
432
463
|
)
|
|
433
464
|
)
|
|
434
465
|
self.summary = summary
|
|
435
466
|
|
|
436
|
-
|
|
437
467
|
@property
|
|
438
468
|
def flow_objects(self):
|
|
439
469
|
return self._flow_objects
|
|
440
470
|
|
|
441
471
|
@flow_objects.setter
|
|
442
472
|
def flow_objects(self, objects):
|
|
473
|
+
smo_objects = self.load_stix_object_from_url(self.ATTACK_FLOW_SMO_URL)["objects"]
|
|
474
|
+
objects.extend(smo_objects)
|
|
443
475
|
for obj in objects:
|
|
444
|
-
if obj[
|
|
476
|
+
if obj["id"] == self.report.id:
|
|
445
477
|
continue
|
|
446
|
-
is_report_object = obj[
|
|
478
|
+
is_report_object = obj["type"] not in ["extension-definition", "identity"]
|
|
447
479
|
self.add_ref(obj, is_report_object=is_report_object)
|
|
448
480
|
self._flow_objects = objects
|
|
@@ -622,7 +622,7 @@ ai_bank_card_all:
|
|
|
622
622
|
prompt_helper: ''
|
|
623
623
|
prompt_conversion: ''
|
|
624
624
|
test_cases: generic_bank_card_mastercard
|
|
625
|
-
stix_mapping:
|
|
625
|
+
stix_mapping: payment-card
|
|
626
626
|
|
|
627
627
|
ai_bank_card_mastercard:
|
|
628
628
|
type: ai
|
|
@@ -638,7 +638,7 @@ ai_bank_card_mastercard:
|
|
|
638
638
|
prompt_helper: ''
|
|
639
639
|
prompt_conversion: ''
|
|
640
640
|
test_cases: generic_bank_card_mastercard
|
|
641
|
-
stix_mapping:
|
|
641
|
+
stix_mapping: payment-card
|
|
642
642
|
|
|
643
643
|
ai_bank_card_visa:
|
|
644
644
|
type: ai
|
|
@@ -654,7 +654,7 @@ ai_bank_card_visa:
|
|
|
654
654
|
prompt_helper: ''
|
|
655
655
|
prompt_conversion: ''
|
|
656
656
|
test_cases: generic_bank_card_visa
|
|
657
|
-
stix_mapping:
|
|
657
|
+
stix_mapping: payment-card
|
|
658
658
|
|
|
659
659
|
ai_bank_card_amex:
|
|
660
660
|
type: ai
|
|
@@ -670,7 +670,7 @@ ai_bank_card_amex:
|
|
|
670
670
|
prompt_helper: ''
|
|
671
671
|
prompt_conversion: ''
|
|
672
672
|
test_cases: generic_bank_card_amex
|
|
673
|
-
stix_mapping:
|
|
673
|
+
stix_mapping: payment-card
|
|
674
674
|
|
|
675
675
|
ai_bank_card_union_pay:
|
|
676
676
|
type: ai
|
|
@@ -686,7 +686,7 @@ ai_bank_card_union_pay:
|
|
|
686
686
|
prompt_helper: ''
|
|
687
687
|
prompt_conversion: ''
|
|
688
688
|
test_cases: generic_bank_card_union_pay
|
|
689
|
-
stix_mapping:
|
|
689
|
+
stix_mapping: payment-card
|
|
690
690
|
|
|
691
691
|
ai_bank_card_diners:
|
|
692
692
|
type: ai
|
|
@@ -702,7 +702,7 @@ ai_bank_card_diners:
|
|
|
702
702
|
prompt_helper: ''
|
|
703
703
|
prompt_conversion: ''
|
|
704
704
|
test_cases: generic_bank_card_diners
|
|
705
|
-
stix_mapping:
|
|
705
|
+
stix_mapping: payment-card
|
|
706
706
|
|
|
707
707
|
ai_bank_card_jcb:
|
|
708
708
|
type: ai
|
|
@@ -718,7 +718,7 @@ ai_bank_card_jcb:
|
|
|
718
718
|
prompt_helper: ''
|
|
719
719
|
prompt_conversion: ''
|
|
720
720
|
test_cases: generic_bank_card_jcb
|
|
721
|
-
stix_mapping:
|
|
721
|
+
stix_mapping: payment-card
|
|
722
722
|
|
|
723
723
|
ai_bank_card_discover:
|
|
724
724
|
type: ai
|
|
@@ -734,7 +734,7 @@ ai_bank_card_discover:
|
|
|
734
734
|
prompt_helper: ''
|
|
735
735
|
prompt_conversion: ''
|
|
736
736
|
test_cases: generic_bank_card_discover
|
|
737
|
-
stix_mapping:
|
|
737
|
+
stix_mapping: payment-card
|
|
738
738
|
|
|
739
739
|
####### IBAN Extractions #######
|
|
740
740
|
|
|
@@ -491,92 +491,92 @@ pattern_bank_card_mastercard:
|
|
|
491
491
|
type: pattern
|
|
492
492
|
dogesec_web: true
|
|
493
493
|
name: 'Bank Card Mastercard'
|
|
494
|
-
description: 'Will extract card numbers and create a
|
|
494
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
495
495
|
notes: 'Also available: ai_bank_card_mastercard'
|
|
496
496
|
created: 2020-01-01
|
|
497
497
|
modified: 2020-01-01
|
|
498
498
|
created_by: DOGESEC
|
|
499
499
|
version: 1.0.0
|
|
500
500
|
test_cases: generic_bank_card_mastercard
|
|
501
|
-
stix_mapping:
|
|
501
|
+
stix_mapping: payment-card
|
|
502
502
|
|
|
503
503
|
pattern_bank_card_visa:
|
|
504
504
|
type: pattern
|
|
505
505
|
dogesec_web: true
|
|
506
506
|
name: 'Bank Card Visa'
|
|
507
|
-
description: 'Will extract card numbers and create a
|
|
507
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
508
508
|
notes: 'Also available: ai_bank_card_visa'
|
|
509
509
|
created: 2020-01-01
|
|
510
510
|
modified: 2020-01-01
|
|
511
511
|
created_by: DOGESEC
|
|
512
512
|
version: 1.0.0
|
|
513
513
|
test_cases: generic_bank_card_visa
|
|
514
|
-
stix_mapping:
|
|
514
|
+
stix_mapping: payment-card
|
|
515
515
|
|
|
516
516
|
pattern_bank_card_amex:
|
|
517
517
|
type: pattern
|
|
518
518
|
dogesec_web: true
|
|
519
519
|
name: 'Bank Card American Express'
|
|
520
|
-
description: 'Will extract card numbers and create a
|
|
520
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
521
521
|
notes: 'Also available: ai_bank_card_amex'
|
|
522
522
|
created: 2020-01-01
|
|
523
523
|
modified: 2020-01-01
|
|
524
524
|
created_by: DOGESEC
|
|
525
525
|
version: 1.0.0
|
|
526
526
|
test_cases: generic_bank_card_amex
|
|
527
|
-
stix_mapping:
|
|
527
|
+
stix_mapping: payment-card
|
|
528
528
|
|
|
529
529
|
pattern_bank_card_union_pay:
|
|
530
530
|
type: pattern
|
|
531
531
|
dogesec_web: true
|
|
532
532
|
name: 'Bank Card Union Pay'
|
|
533
|
-
description: 'Will extract card numbers and create a
|
|
533
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
534
534
|
notes: 'Also available: ai_bank_card_union_pay'
|
|
535
535
|
created: 2020-01-01
|
|
536
536
|
modified: 2020-01-01
|
|
537
537
|
created_by: DOGESEC
|
|
538
538
|
version: 1.0.0
|
|
539
539
|
test_cases: generic_bank_card_union_pay
|
|
540
|
-
stix_mapping:
|
|
540
|
+
stix_mapping: payment-card
|
|
541
541
|
|
|
542
542
|
pattern_bank_card_diners:
|
|
543
543
|
type: pattern
|
|
544
544
|
dogesec_web: true
|
|
545
545
|
name: 'Bank Card Diners'
|
|
546
|
-
description: 'Will extract card numbers and create a
|
|
546
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
547
547
|
notes: 'Also available: ai_bank_card_diners'
|
|
548
548
|
created: 2020-01-01
|
|
549
549
|
modified: 2020-01-01
|
|
550
550
|
created_by: DOGESEC
|
|
551
551
|
version: 1.0.0
|
|
552
552
|
test_cases: generic_bank_card_diners
|
|
553
|
-
stix_mapping:
|
|
553
|
+
stix_mapping: payment-card
|
|
554
554
|
|
|
555
555
|
pattern_bank_card_jcb:
|
|
556
556
|
type: pattern
|
|
557
557
|
dogesec_web: true
|
|
558
558
|
name: 'Bank Card JCB'
|
|
559
|
-
description: 'Will extract card numbers and create a
|
|
559
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
560
560
|
notes: 'Also available: ai_bank_card_jcb'
|
|
561
561
|
created: 2020-01-01
|
|
562
562
|
modified: 2020-01-01
|
|
563
563
|
created_by: DOGESEC
|
|
564
564
|
version: 1.0.0
|
|
565
565
|
test_cases: generic_bank_card_jcb
|
|
566
|
-
stix_mapping:
|
|
566
|
+
stix_mapping: payment-card
|
|
567
567
|
|
|
568
568
|
pattern_bank_card_discover:
|
|
569
569
|
type: pattern
|
|
570
570
|
dogesec_web: true
|
|
571
571
|
name: 'Bank Card Discover'
|
|
572
|
-
description: 'Will extract card numbers and create a
|
|
572
|
+
description: 'Will extract card numbers and create a payment-card object. Will also enrich card information if BIN List API key set'
|
|
573
573
|
notes: 'Also available: ai_bank_card_discover'
|
|
574
574
|
created: 2020-01-01
|
|
575
575
|
modified: 2020-01-01
|
|
576
576
|
created_by: DOGESEC
|
|
577
577
|
version: 1.0.0
|
|
578
578
|
test_cases: generic_bank_card_discover
|
|
579
|
-
stix_mapping:
|
|
579
|
+
stix_mapping: payment-card
|
|
580
580
|
|
|
581
581
|
####### IBAN Extractions #######
|
|
582
582
|
|
txt2stix/indicator.py
CHANGED
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
4
|
from stix2.parsing import dict_to_stix2
|
|
5
|
-
from stix2 import HashConstant
|
|
5
|
+
from stix2 import HashConstant, File
|
|
6
6
|
from stix2.v21.vocab import HASHING_ALGORITHM
|
|
7
7
|
from stix2.patterns import _HASH_REGEX as HASHING_ALGORITHM_2
|
|
8
8
|
from ipaddress import ip_address
|
|
@@ -14,7 +14,9 @@ from typing import TYPE_CHECKING
|
|
|
14
14
|
|
|
15
15
|
import validators
|
|
16
16
|
|
|
17
|
-
from txt2stix.pattern.extractors.others.phonenumber_extractor import
|
|
17
|
+
from txt2stix.pattern.extractors.others.phonenumber_extractor import (
|
|
18
|
+
PhoneNumberExtractor,
|
|
19
|
+
)
|
|
18
20
|
from txt2stix.utils import validate_file_mimetype, validate_reg_key
|
|
19
21
|
|
|
20
22
|
if TYPE_CHECKING:
|
|
@@ -24,7 +26,7 @@ if TYPE_CHECKING:
|
|
|
24
26
|
|
|
25
27
|
from .common import MinorException
|
|
26
28
|
|
|
27
|
-
from .retriever import
|
|
29
|
+
from .retriever import retrieve_stix_objects
|
|
28
30
|
|
|
29
31
|
logger = logging.getLogger("txt2stix.indicator")
|
|
30
32
|
|
|
@@ -71,12 +73,13 @@ def split_ip_port(ip_port: str):
|
|
|
71
73
|
|
|
72
74
|
return ip.exploded, int(port)
|
|
73
75
|
|
|
76
|
+
|
|
74
77
|
def get_country_code(number: str) -> str:
|
|
75
78
|
phone = PhoneNumberExtractor.parse_phone_number(number)
|
|
76
79
|
if phone:
|
|
77
80
|
return geocoder.region_codes_for_country_code(phone.country_code)[0]
|
|
78
81
|
else:
|
|
79
|
-
raise BadDataException(
|
|
82
|
+
raise BadDataException("bad phone number")
|
|
80
83
|
|
|
81
84
|
|
|
82
85
|
def get_iban_details(number) -> tuple[str, str]:
|
|
@@ -93,7 +96,7 @@ def build_observables(
|
|
|
93
96
|
except BadDataException:
|
|
94
97
|
raise
|
|
95
98
|
except BaseException as e:
|
|
96
|
-
raise BadDataException(
|
|
99
|
+
raise BadDataException("unknown data error") from e
|
|
97
100
|
|
|
98
101
|
|
|
99
102
|
def _build_observables(
|
|
@@ -102,6 +105,13 @@ def _build_observables(
|
|
|
102
105
|
retrieved_objects = retrieve_stix_objects(stix_mapping, extracted_value)
|
|
103
106
|
if retrieved_objects:
|
|
104
107
|
return retrieved_objects, [sdo["id"] for sdo in retrieved_objects]
|
|
108
|
+
if retrieved_objects == []:
|
|
109
|
+
logger.warning(
|
|
110
|
+
f"could not find `{stix_mapping}` with id=`{extracted_value}` in remote"
|
|
111
|
+
)
|
|
112
|
+
raise BadDataException(
|
|
113
|
+
f"could not find `{stix_mapping}` with id=`{extracted_value}` in remote"
|
|
114
|
+
)
|
|
105
115
|
|
|
106
116
|
stix_objects = [indicator]
|
|
107
117
|
|
|
@@ -321,7 +331,9 @@ def _build_observables(
|
|
|
321
331
|
}
|
|
322
332
|
)
|
|
323
333
|
indicator["name"] = f"Directory File: {extracted_value}"
|
|
324
|
-
indicator["pattern"] =
|
|
334
|
+
indicator["pattern"] = (
|
|
335
|
+
f"[ directory:path = { repr(dir_obj.path) } OR file:name = {repr(file.name)}]"
|
|
336
|
+
)
|
|
325
337
|
|
|
326
338
|
stix_objects.append(dir_obj)
|
|
327
339
|
stix_objects.append(file)
|
|
@@ -440,11 +452,11 @@ def _build_observables(
|
|
|
440
452
|
|
|
441
453
|
if stix_mapping == "user-agent":
|
|
442
454
|
indicator["name"] = f"User Agent: {extracted_value}"
|
|
443
|
-
indicator["pattern"] = f"[ user-agent:
|
|
455
|
+
indicator["pattern"] = f"[ user-agent:value = { repr(extracted_value) } ]"
|
|
444
456
|
|
|
445
457
|
stix_objects.append(
|
|
446
458
|
dict_to_stix2(
|
|
447
|
-
{"type": "user-agent", "spec_version": "2.1", "
|
|
459
|
+
{"type": "user-agent", "spec_version": "2.1", "value": extracted_value}
|
|
448
460
|
)
|
|
449
461
|
)
|
|
450
462
|
stix_objects.append(
|
|
@@ -464,7 +476,9 @@ def _build_observables(
|
|
|
464
476
|
f"AS Number must contain a number, got `{extracted_value}`"
|
|
465
477
|
)
|
|
466
478
|
extracted_value = int(match.group(0))
|
|
467
|
-
assert
|
|
479
|
+
assert (
|
|
480
|
+
extracted_value >= 1 and extracted_value <= 65535
|
|
481
|
+
), "AS Number must be between 1 and 65535"
|
|
468
482
|
indicator["name"] = f"AS{extracted_value}"
|
|
469
483
|
indicator["pattern"] = (
|
|
470
484
|
f"[ autonomous-system:number = { repr(extracted_value) } ]"
|
|
@@ -497,7 +511,7 @@ def _build_observables(
|
|
|
497
511
|
btc2stix = crypto2stix.BTC2Stix()
|
|
498
512
|
indicator["name"] = f"{currency_symbol} Wallet: {extracted_value}"
|
|
499
513
|
indicator["pattern"] = (
|
|
500
|
-
f"[ cryptocurrency-wallet:
|
|
514
|
+
f"[ cryptocurrency-wallet:value = { repr(extracted_value) } ]"
|
|
501
515
|
)
|
|
502
516
|
wallet_obj, *other_objects = btc2stix.process_wallet(
|
|
503
517
|
extracted_value, wallet_only=True, transactions_only=False
|
|
@@ -524,7 +538,7 @@ def _build_observables(
|
|
|
524
538
|
txn_object, *other_objects = btc2stix.process_transaction(extracted_value)
|
|
525
539
|
indicator["name"] = f"{currency_symbol} Transaction: {extracted_value}"
|
|
526
540
|
indicator["pattern"] = (
|
|
527
|
-
f"[ cryptocurrency-transaction:
|
|
541
|
+
f"[ cryptocurrency-transaction:value = { repr(extracted_value) } ]"
|
|
528
542
|
)
|
|
529
543
|
|
|
530
544
|
stix_objects.append(txn_object)
|
|
@@ -549,7 +563,7 @@ def _build_observables(
|
|
|
549
563
|
btc2stix = crypto2stix.BTC2Stix()
|
|
550
564
|
indicator["name"] = f"{currency_symbol} Wallet: {extracted_value}"
|
|
551
565
|
indicator["pattern"] = (
|
|
552
|
-
f"[ cryptocurrency-wallet:
|
|
566
|
+
f"[ cryptocurrency-wallet:value = { repr(extracted_value) } ]"
|
|
553
567
|
)
|
|
554
568
|
wallet_obj, *other_objects = btc2stix.process_wallet(
|
|
555
569
|
extracted_value, wallet_only=False, transactions_only=True
|
|
@@ -567,7 +581,7 @@ def _build_observables(
|
|
|
567
581
|
)
|
|
568
582
|
)
|
|
569
583
|
return stix_objects, [wallet_obj.id]
|
|
570
|
-
if stix_mapping == "
|
|
584
|
+
if stix_mapping == "payment-card":
|
|
571
585
|
# TODO
|
|
572
586
|
card_type = extractor.name
|
|
573
587
|
if "Bank Card" in extractor.name:
|
|
@@ -585,7 +599,7 @@ def _build_observables(
|
|
|
585
599
|
card_type = card_object["scheme"]
|
|
586
600
|
|
|
587
601
|
indicator["name"] = f"{card_type}: {extracted_value}"
|
|
588
|
-
indicator["pattern"] = f"[
|
|
602
|
+
indicator["pattern"] = f"[ payment-card:value = { repr(extracted_value) } ]"
|
|
589
603
|
|
|
590
604
|
stix_objects.append(
|
|
591
605
|
bundler.new_relationship(
|
|
@@ -601,41 +615,42 @@ def _build_observables(
|
|
|
601
615
|
if stix_mapping == "bank-account":
|
|
602
616
|
q = validators.iban(extracted_value)
|
|
603
617
|
if q != True:
|
|
604
|
-
raise BadDataException(
|
|
618
|
+
raise BadDataException("invalid iban number") from q
|
|
605
619
|
indicator["name"] = f"Bank account: {extracted_value}"
|
|
606
|
-
indicator["pattern"] = (
|
|
607
|
-
f"[ bank-account:iban_number = { repr(extracted_value) } ]"
|
|
608
|
-
)
|
|
620
|
+
indicator["pattern"] = f"[ bank-account:iban = { repr(extracted_value) } ]"
|
|
609
621
|
extracted_value = extracted_value.replace("-", "").replace(" ", "")
|
|
610
622
|
|
|
611
623
|
country_code, bank_code = get_iban_details(extracted_value)
|
|
624
|
+
location = retrieve_stix_objects("location", country_code)[0]
|
|
625
|
+
stix_objects.append(location)
|
|
612
626
|
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
}
|
|
621
|
-
)
|
|
627
|
+
bank_acc = dict_to_stix2(
|
|
628
|
+
{
|
|
629
|
+
"type": "bank-account",
|
|
630
|
+
"spec_version": "2.1",
|
|
631
|
+
"iban": extracted_value,
|
|
632
|
+
"country_ref": location["id"],
|
|
633
|
+
}
|
|
622
634
|
)
|
|
635
|
+
|
|
636
|
+
stix_objects.append(bank_acc)
|
|
623
637
|
stix_objects.append(
|
|
624
638
|
bundler.new_relationship(
|
|
625
639
|
indicator["id"],
|
|
626
|
-
|
|
640
|
+
bank_acc.id,
|
|
627
641
|
"related-to",
|
|
628
642
|
description=f"STIX pattern contains {extracted_value}",
|
|
629
643
|
external_references=indicator["external_references"],
|
|
630
644
|
)
|
|
631
645
|
)
|
|
646
|
+
return stix_objects, [bank_acc.id]
|
|
632
647
|
|
|
633
648
|
if stix_mapping == "phone-number":
|
|
634
649
|
country_code = get_country_code(extracted_value)
|
|
635
650
|
if not country_code:
|
|
636
|
-
raise BadDataException(
|
|
651
|
+
raise BadDataException("parse phone number failed")
|
|
637
652
|
indicator["name"] = f"Phone Number: {extracted_value}"
|
|
638
|
-
indicator["pattern"] = f"[ phone-number:
|
|
653
|
+
indicator["pattern"] = f"[ phone-number:value = { repr(extracted_value) }"
|
|
639
654
|
if country_code:
|
|
640
655
|
indicator["pattern"] += f" AND phone-number:country = '{country_code}' "
|
|
641
656
|
indicator["pattern"] += " ]"
|
|
@@ -645,7 +660,7 @@ def _build_observables(
|
|
|
645
660
|
{
|
|
646
661
|
"type": "phone-number",
|
|
647
662
|
"spec_version": "2.1",
|
|
648
|
-
"
|
|
663
|
+
"value": extracted_value,
|
|
649
664
|
"country": country_code,
|
|
650
665
|
}
|
|
651
666
|
)
|
|
@@ -825,7 +840,7 @@ def _build_observables(
|
|
|
825
840
|
"user-agent",
|
|
826
841
|
"cryptocurrency-wallet",
|
|
827
842
|
"cryptocurrency-transaction",
|
|
828
|
-
"
|
|
843
|
+
"payment-card",
|
|
829
844
|
"bank-account",
|
|
830
845
|
"phone-number",
|
|
831
846
|
"attack-pattern",
|
|
@@ -7,7 +7,7 @@ class BankCardDiscoverExtractor(BaseExtractor):
|
|
|
7
7
|
A class for extracting Discover credit card numbers from text using regular expressions.
|
|
8
8
|
|
|
9
9
|
Attributes:
|
|
10
|
-
name (str): The name of the extractor, set to "
|
|
10
|
+
name (str): The name of the extractor, set to "payment-card-discover".
|
|
11
11
|
extraction_regex (str): The regular expression pattern used for extracting Discover credit card numbers.
|
|
12
12
|
"""
|
|
13
13
|
|
txt2stix/retriever.py
CHANGED
|
@@ -58,6 +58,12 @@ class STIXObjectRetriever:
|
|
|
58
58
|
return self._retrieve_objects(
|
|
59
59
|
urljoin(self.api_root, f"v1/{type}/objects/{id}/")
|
|
60
60
|
)
|
|
61
|
+
|
|
62
|
+
def retrieve_object_by_id(self, id, type):
|
|
63
|
+
endpoint = urljoin(self.api_root, f"v1/{type}/objects/{id}/")
|
|
64
|
+
resp = self.session.get(endpoint)
|
|
65
|
+
resp.raise_for_status()
|
|
66
|
+
return [resp.json()]
|
|
61
67
|
|
|
62
68
|
def get_location_objects(self, id):
|
|
63
69
|
return self._retrieve_objects(
|
|
@@ -111,9 +117,9 @@ def _retrieve_stix_objects(host, knowledge_base, filter_value):
|
|
|
111
117
|
case "mitre-cwe-id":
|
|
112
118
|
return retreiver.get_objects_by_id(filter_value, "cwe")
|
|
113
119
|
case "cve-id":
|
|
114
|
-
return retreiver.
|
|
120
|
+
return retreiver.retrieve_object_by_id(filter_value, "cve")
|
|
115
121
|
case "cpe-id":
|
|
116
|
-
return retreiver.
|
|
122
|
+
return retreiver.retrieve_object_by_id(filter_value, "cpe")
|
|
117
123
|
case "location":
|
|
118
124
|
return retreiver.get_location_objects(filter_value)
|
|
119
125
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: txt2stix
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.4
|
|
4
4
|
Summary: txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle.
|
|
5
5
|
Project-URL: Homepage, https://github.com/muchdogesec/txt2stix
|
|
6
6
|
Project-URL: Issues, https://github.com/muchdogesec/txt2stix/issues
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
txt2stix/__init__.py,sha256=Sm_VT913IFuAZ6dJEdVz3baPwC5VYtHySVfBAOUG92w,803
|
|
2
|
-
txt2stix/attack_flow.py,sha256=
|
|
3
|
-
txt2stix/bundler.py,sha256=
|
|
2
|
+
txt2stix/attack_flow.py,sha256=Y0zl-2YVkiVhbtUu6rsrX7ujnfdDTArSkrR_WS_fSc8,9052
|
|
3
|
+
txt2stix/bundler.py,sha256=L5s8IvNLd6VwjnTEqU71hEAR3j_Vj4NdXlufavTLuKM,16917
|
|
4
4
|
txt2stix/common.py,sha256=ISnGNKqJPE1EcfhL-x_4G18mcwt1urmorkW-ru9kV-0,585
|
|
5
5
|
txt2stix/credential_checker.py,sha256=eWDP-jY3-jm8zI0JMoUcyoQZ_JqPNfCIr_HAO8nVYz0,3044
|
|
6
6
|
txt2stix/extractions.py,sha256=_tlsqYHhfAoV-PJzxRHysrX47uxCsMlSg7PQWxww1u0,2171
|
|
7
|
-
txt2stix/indicator.py,sha256=
|
|
7
|
+
txt2stix/indicator.py,sha256=dyf4wbvVrZRitZpm6t7UusSM98bVW1qc5UkdGpVm3ls,30025
|
|
8
8
|
txt2stix/lookups.py,sha256=h42YVtYUkWZm6ZPv2h5hHDHDzDs3yBqrT_T7pj2MDZI,2301
|
|
9
|
-
txt2stix/retriever.py,sha256=
|
|
9
|
+
txt2stix/retriever.py,sha256=0eoLzabGrcR0wdQuEYdU8ZPomq42lAsGwP4gY6RLgww,6410
|
|
10
10
|
txt2stix/stix.py,sha256=9nXD9a2dCY4uaatl-mlIA1k3srwQBhGW-tUSho3iYe0,30
|
|
11
11
|
txt2stix/txt2stix.py,sha256=4iVvzlLbUeDIKUPPHGUWZufsy-LIMPk6ejrw8kSI1o8,18595
|
|
12
12
|
txt2stix/utils.py,sha256=n6mh4t9ZRJ7iT4Jvp9ai_dfCXjgXNcRtF_zXO7nkpnk,3304
|
|
@@ -15,7 +15,7 @@ txt2stix/ai_extractor/anthropic.py,sha256=mdz-8CB-BSCEqnK5l35DRZURVPUf508ef2b48X
|
|
|
15
15
|
txt2stix/ai_extractor/base.py,sha256=t0SCh24FeDEDzXsrGFada6ux9F6m0ILwXtPSaleDiv8,4172
|
|
16
16
|
txt2stix/ai_extractor/deepseek.py,sha256=2XehIYbWXG6Odq68nQX4CNtl5GdmBlAmjLP_lG2eEFo,660
|
|
17
17
|
txt2stix/ai_extractor/gemini.py,sha256=yJC7knYzl-TScyCBd-MTpUf-NT6znC25E7vXxNMqjLU,578
|
|
18
|
-
txt2stix/ai_extractor/openai.py,sha256=
|
|
18
|
+
txt2stix/ai_extractor/openai.py,sha256=1RxaLy0TJ4GjNKmcJoi6ZiBrCS_gt5ql9jpeE-SOy8g,642
|
|
19
19
|
txt2stix/ai_extractor/openrouter.py,sha256=hAA6mTOMcpA28XYsOCvuJH7WMJqXCxfqZGJf_VrDsIk,628
|
|
20
20
|
txt2stix/ai_extractor/prompts.py,sha256=NtqtVyPPtShPlVZ5SrFmo-LCkfpANIIi4H9rjqaxqDo,10559
|
|
21
21
|
txt2stix/ai_extractor/utils.py,sha256=jG5tPuS2xfiH7xCxlaEkAOboNOOMDah9JpGDXrUUJBA,4342
|
|
@@ -27,7 +27,7 @@ txt2stix/pattern/extractors/card/README.md,sha256=VRtmsIhbwswI0mP9axgV-czTOgpG6-
|
|
|
27
27
|
txt2stix/pattern/extractors/card/__init__.py,sha256=KyYIaHA2igtJZfTDebwM6ohism_dJNh5Velexzhhcts,678
|
|
28
28
|
txt2stix/pattern/extractors/card/amex_card_extractor.py,sha256=NrWqwv2cEeOK1LxZZ5AwxLyZ1lyYB7CmxpT40t5dUIk,1999
|
|
29
29
|
txt2stix/pattern/extractors/card/diners_card_extractor.py,sha256=e7Q7_g1iXyBD4439iz4uMLBrEf5ykL143dqwYtQMJPM,1802
|
|
30
|
-
txt2stix/pattern/extractors/card/discover_card_extractor.py,sha256=
|
|
30
|
+
txt2stix/pattern/extractors/card/discover_card_extractor.py,sha256=wYaP1Juzqc5wcvJcxGGi2ni9ENY8NL9dc9a2itqi_mw,1912
|
|
31
31
|
txt2stix/pattern/extractors/card/jcb_card_extractor.py,sha256=6wKrJmkhsm8gnQUtyLE2JxHuCO0NzUaDl3uyfU4livw,1681
|
|
32
32
|
txt2stix/pattern/extractors/card/master_card_extractor.py,sha256=STzq09lHV1W7nZg8NjToS0D3e04QwME2i0CUSUbL7BM,2736
|
|
33
33
|
txt2stix/pattern/extractors/card/union_card_extractor.py,sha256=94Z2shAnBB0fvGHm_0DgUPwzULyftAmmWpMLZdq0WB8,1374
|
|
@@ -74,9 +74,9 @@ txt2stix/pattern/extractors/url/url_extractor.py,sha256=-SH1WvxbViaRZ1on8lRlzNAc
|
|
|
74
74
|
txt2stix/pattern/extractors/url/url_file_extractor.py,sha256=_VDu_BX3Ys9SKhZlscZPp9xSOKCxNKKvJ2gbe7Nvuv0,881
|
|
75
75
|
txt2stix/pattern/extractors/url/url_path_extractor.py,sha256=FNKcMZRnJpcZZF44T8DHsDWzhBqPq5a23h7T7l2osac,2459
|
|
76
76
|
txt2stix/includes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
|
-
txt2stix/includes/extractions/ai/config.yaml,sha256=
|
|
77
|
+
txt2stix/includes/extractions/ai/config.yaml,sha256=r98Sa93GaZNcY60NuZ2XlIM8BpFl33PUBkqFvGLqTOo,41564
|
|
78
78
|
txt2stix/includes/extractions/lookup/config.yaml,sha256=lZoJ-vHig30TpfiwNEl4fiT-AwdOlhm7h0pE8b_G6jg,12059
|
|
79
|
-
txt2stix/includes/extractions/pattern/config.yaml,sha256=
|
|
79
|
+
txt2stix/includes/extractions/pattern/config.yaml,sha256=UICpzQN7SOr5yT7jmhJYtpDuQxQxA-cxoltI4GW0nFs,20196
|
|
80
80
|
txt2stix/includes/helpers/mimetype_filename_extension_list.csv,sha256=kgozjMyp7y87CqRcoedfDwNXSLKrDgC9r9YKDYK0EbY,27593
|
|
81
81
|
txt2stix/includes/helpers/stix_relationship_types.txt,sha256=PQytANVSrWepdK_SLEZtfiTe1eoxj6YMGUZslO_C1oc,505
|
|
82
82
|
txt2stix/includes/helpers/tlds.txt,sha256=Va_awj-FQiKgs5ace6C0kC5xxAHIl9yAIBhvT08Q7Q0,9551
|
|
@@ -113,8 +113,8 @@ txt2stix/includes/lookups/threat_actor.txt,sha256=QfDO9maQuqKBgW_Sdd7VGv1SHZ9Ra-
|
|
|
113
113
|
txt2stix/includes/lookups/tld.txt,sha256=-MEgJea2NMG_KDsnc4BVvI8eRk5Dm93L-t8SGYx5wMo,8598
|
|
114
114
|
txt2stix/includes/lookups/tool.txt,sha256=HGKG6JpUE26w6ezzSxOjBkp15UpSaB7N-mZ_NU_3G7A,6
|
|
115
115
|
txt2stix/includes/tests/test_cases.yaml,sha256=QD1FdIunpPkOpsn6wJRqs2vil_hv8OSVaqUp4a96aZg,22247
|
|
116
|
-
txt2stix-1.1.
|
|
117
|
-
txt2stix-1.1.
|
|
118
|
-
txt2stix-1.1.
|
|
119
|
-
txt2stix-1.1.
|
|
120
|
-
txt2stix-1.1.
|
|
116
|
+
txt2stix-1.1.4.dist-info/METADATA,sha256=_LzscyEWJvO91-3mIQkAq7jie7uFEZ8GcLVmfrcUhJM,15513
|
|
117
|
+
txt2stix-1.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
118
|
+
txt2stix-1.1.4.dist-info/entry_points.txt,sha256=x6QPtt65hWeomw4IpJ_wQUesBl1M4WOLODbhOKyWMFg,55
|
|
119
|
+
txt2stix-1.1.4.dist-info/licenses/LICENSE,sha256=BK8Ppqlc4pdgnNzIxnxde0taoQ1BgicdyqmBvMiNYgY,11364
|
|
120
|
+
txt2stix-1.1.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|