txt2stix 1.1.8__py3-none-any.whl → 1.1.10__py3-none-any.whl

txt2stix/includes/extractions/ai/config.yaml

@@ -876,6 +876,26 @@ ai_mitre_cwe:
   test_cases: ai_mitre_cwe
   stix_mapping: ctibutler-mitre-cwe-id
 
+####### Sector #######
+
+ai_sector:
+  type: ai
+  dogesec_web: true
+  name: 'Sectors'
+  description: 'Detect which sectors from a fixed sector list are referenced anywhere in the input text.'
+  notes: 'lookup_sector_aliases legacy extraction also exists if you cannot use AI'
+  created: 2020-01-01
+  modified:  2020-01-01
+  created_by: dogesec
+  version: 1.0.0
+  prompt_base: |
+      You will be given a list of possible sectors. Check whatever sector best matches what is being referenced in the text
+      (either directly or via obvious common synonyms).
+  prompt_helper: ''
+  prompt_conversion: 'Should respond with one of ["Road transport","Hospitality","Finance","Lodging industry","Hosting service providers","Technologies","Information Technologies Consulting","Travel and tourism industry","Food and drinks businesses","Banking institutions","Education","Defense industry","Diplomacy","Schools","Non-Governmental Organizations (NGOs)","Healthcare research","Defense ministries (including the military)","Oil","Research","Think tanks","Political parties","Legislative branch (parliamentary chambers)","Mining","Defense research and development","Gambling","Air transport","Nuclear power (civilian use)","Energy","Consulting","Maritime transport","Insurance services","Government and administrations","Market infrastructures","Transport","Gas","Citizens","Digital infrastructures","Heavy industries","Engineering consulting","Construction","Local administrations","Ministries of foreign affairs","Telecommunications","Sports","Agriculture and agribusiness","Entertainment industry","Logistics","Central administration and government","Retail (distribution)","Internet Service providers","Pharmacy and drugs manufacturing","Financial organizations","Manufacturing","Universities","Defense","International organizations","Legal consulting","Renewable energies","Health","Rail transport","Culture","Civil society","Culture and entertainment","Medias and audiovisual","Unknown","Healthcare services","Dissidents","Water distribution and supply","Electricity","Judicial power (justice)","High-tech"]'
+  test_cases: ai_sector_names
+  stix_mapping: ctibutler-sector-name
+
 ####### Generic Extractions #######
 
 ai_attack_pattern:

txt2stix/includes/extractions/lookup/config.yaml

@@ -24,7 +24,7 @@ lookup_mitre_attack_enterprise_id:
   name: 'MITRE ATT&CK Enterprise IDs'
   description: 'Extracts MITRE ATT&CK Enterprise IDs from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_enterprise also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_enterprise_id_v16_0.txt'
+  file: 'lookups/mitre_attack_enterprise_id_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -38,7 +38,7 @@ lookup_mitre_attack_enterprise_name:
   name: 'MITRE ATT&CK Enterprise names'
   description: 'Extracts MITRE ATT&CK Enterprise names from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_enterprise also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_enterprise_name_v16_0.txt'
+  file: 'lookups/mitre_attack_enterprise_name_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -52,7 +52,7 @@ lookup_mitre_attack_enterprise_alias:
   name: 'MITRE ATT&CK Enterprise alias'
   description: 'Extracts MITRE ATT&CK Enterprise aliases from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_enterprise also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_enterprise_aliases_v16_0.txt'
+  file: 'lookups/mitre_attack_enterprise_aliases_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -66,7 +66,7 @@ lookup_mitre_attack_mobile_id:
   name: 'MITRE ATT&CK Mobile IDs'
   description: 'Extracts MITRE ATT&CK Mobile IDs from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_mobile also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_mobile_id_v16_0.txt'
+  file: 'lookups/mitre_attack_mobile_id_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -80,7 +80,7 @@ lookup_mitre_attack_mobile_name:
   name: 'MITRE ATT&CK Mobile names'
   description: 'Extracts MITRE ATT&CK Mobile names from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_mobile also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_mobile_name_v16_0.txt'
+  file: 'lookups/mitre_attack_mobile_name_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -94,7 +94,7 @@ lookup_mitre_attack_mobile_alias:
   name: 'MITRE ATT&CK Mobile alias'
   description: 'Extracts MITRE ATT&CK Mobile aliases from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_mobile also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_mobile_aliases_v16_0.txt'
+  file: 'lookups/mitre_attack_mobile_aliases_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -108,7 +108,7 @@ lookup_mitre_attack_ics_id:
   name: 'MITRE ATT&CK ICS IDs'
   description: 'Extracts MITRE ATT&CK ICS names from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_ics also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_ics_id_v16_0.txt'
+  file: 'lookups/mitre_attack_ics_id_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -122,7 +122,7 @@ lookup_mitre_attack_ics_name:
   name: 'MITRE ATT&CK ICS names'
   description: 'Extracts MITRE ATT&CK ICS names from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_ics also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_ics_name_v16_0.txt'
+  file: 'lookups/mitre_attack_ics_name_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -136,7 +136,7 @@ lookup_mitre_attack_ics_alias:
   name: 'MITRE ATT&CK ICS alias'
   description: 'Extracts MITRE ATT&CK ICS aliases from text. See lookup name for version used.'
   notes: 'ai_mitre_attack_ics also exists but beware of hallucinations'
-  file: 'lookups/mitre_attack_ics_aliases_v16_0.txt'
+  file: 'lookups/mitre_attack_ics_aliases_v18_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -182,7 +182,7 @@ lookup_mitre_cwe_id:
   name: MITRE CWE IDs
   description: 'Extracts MITRE CWE IDs from text. See lookup name for version used.'
   notes: 'ai_mitre_cwe also exists but beware of hallucinations'
-  file: 'lookups/mitre_cwe_id_v4_15.txt'
+  file: 'lookups/mitre_cwe_id_v4_18.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -196,7 +196,7 @@ lookup_mitre_cwe_name:
   name: MITRE CWE names
   description: 'Extracts MITRE CWE names from text. See lookup name for version used.'
   notes: 'ai_mitre_cwe also exists but beware of hallucinations'
-  file: 'lookups/mitre_cwe_name_v4_15.txt'
+  file: 'lookups/mitre_cwe_name_v4_18.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -212,7 +212,7 @@ lookup_mitre_atlas_id:
   name: MITRE ATLAS IDs
   description: 'Extracts MITRE ATLAS IDs from text. See lookup name for version used.'
   notes: 'No corresponding AI version yet due to poor AI performance'
-  file: 'lookups/mitre_atlas_id_v4_5_2.txt'
+  file: 'lookups/mitre_atlas_id_v4_9_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -226,7 +226,7 @@ lookup_mitre_atlas_name:
   name: MITRE ATLAS names
   description: 'Extracts MITRE ATLAS names from text. See lookup name for version used.'
   notes: 'No corresponding AI version yet due to poor AI performance'
-  file: 'lookups/mitre_atlas_name_v4_5_2.txt'
+  file: 'lookups/mitre_atlas_name_v4_9_0.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -242,7 +242,7 @@ lookup_disarm_id:
   name: DISARM names
   description: 'Extracts DISARM IDs from text. See lookup name for version used.'
   notes: 'No corresponding AI version yet due to poor AI performance'
-  file: 'lookups/disarm_id_v1_5.txt'
+  file: 'lookups/disarm_id_v1_6.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -256,7 +256,7 @@ lookup_disarm_name:
   name: DISARM IDs
   description: 'Extracts DISARM names from text. See lookup name for version used.'
   notes: 'No corresponding AI version yet due to poor AI performance'
-  file: 'lookups/disarm_name_v1_5.txt'
+  file: 'lookups/disarm_name_v1_6.txt'
   created: 2020-01-01
   modified:  2020-01-01
   created_by: dogesec
@@ -264,6 +264,22 @@ lookup_disarm_name:
   test_cases: generic_disarm_name
   stix_mapping: ctibutler-disarm-name
 
+####### Sector #######
+
+lookup_sector_alias:
+  type: lookup
+  dogesec_web: false
+  name: 'Sector names'
+  description: 'Extracts Sector alias from text.'
+  notes: 'ai_mitre_attack_ics also exists but beware of hallucinations'
+  file: 'lookups/sector_aliases_v1_0.txt'
+  created: 2020-01-01
+  modified:  2020-01-01
+  created_by: dogesec
+  version: 1.0.0
+  test_cases: generic_sector_aliases
+  stix_mapping: ctibutler-sector-alias
+
 ####### Generic Extractions #######
 
 lookup_attack_pattern:
@@ -391,3 +407,5 @@ lookup_tool:
   version: 1.0.0
   test_cases: lookup_tool
   stix_mapping: tool
+
+

txt2stix/includes/lookups/_generate_lookups.py

@@ -12,10 +12,10 @@ script_dir = os.path.dirname(os.path.abspath(__file__))
 
 # Define queries and output files
 queries = {
-    "mitre_cwe_id_v4_15.txt":
+    "mitre_cwe_id_v4_18.txt":
       """
         FOR doc IN mitre_cwe_vertex_collection
-          FILTER doc._stix2arango_note == "version=4_15"
+          FILTER doc._stix2arango_note == "version=4_18"
           AND IS_ARRAY(doc.external_references)
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -27,10 +27,10 @@ queries = {
             SORT reference.external_id ASC
             RETURN reference.external_id
       """,
-    "mitre_cwe_name_v4_15.txt":
+    "mitre_cwe_name_v4_18.txt":
       """
         FOR doc IN mitre_cwe_vertex_collection
-          FILTER doc._stix2arango_note == "version=4_15"
+          FILTER doc._stix2arango_note == "version=4_18"
           AND IS_ARRAY(doc.external_references)
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -65,10 +65,10 @@ queries = {
           AND !CONTAINS(doc.name, "DEPRECATED:")
           RETURN doc.name
       """,
-    "mitre_attack_enterprise_id_v16_0.txt":
+    "mitre_attack_enterprise_id_v18_0.txt":
       """
         FOR doc IN mitre_attack_enterprise_vertex_collection
-          FILTER doc._stix2arango_note == "version=16_0"
+          FILTER doc._stix2arango_note == "version=18_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -78,21 +78,21 @@ queries = {
             SORT reference.external_id ASC
             RETURN reference.external_id
       """,
-    "mitre_attack_enterprise_name_v16_0.txt":
+    "mitre_attack_enterprise_name_v18_0.txt":
       """
         FOR doc IN mitre_attack_enterprise_vertex_collection
-          FILTER doc._stix2arango_note == "version=16_0"
+          FILTER doc._stix2arango_note == "version=18_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
           RETURN doc.name
       """,
-    "mitre_attack_enterprise_aliases_v16_0.txt":
+    "mitre_attack_enterprise_aliases_v18_0.txt":
       """
         FOR alias IN UNIQUE(
           FLATTEN(
             FOR doc IN mitre_attack_enterprise_vertex_collection
-              FILTER doc._stix2arango_note == "version=16_0"
+              FILTER doc._stix2arango_note == "version=18_0"
               AND doc.type != "x-mitre-matrix"
               AND doc.x_mitre_deprecated != true
               AND doc.revoked != true
@@ -106,10 +106,10 @@ queries = {
         )
         RETURN alias
       """,
-    "mitre_attack_ics_id_v16_0.txt":
+    "mitre_attack_ics_id_v18_0.txt":
       """
         FOR doc IN mitre_attack_ics_vertex_collection
-          FILTER doc._stix2arango_note == "version=16_0"
+          FILTER doc._stix2arango_note == "version=18_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -119,21 +119,21 @@ queries = {
             SORT reference.external_id ASC
             RETURN reference.external_id
       """,
-    "mitre_attack_ics_name_v16_0.txt":
+    "mitre_attack_ics_name_v18_0.txt":
       """
         FOR doc IN mitre_attack_ics_vertex_collection
-          FILTER doc._stix2arango_note == "version=16_0"
+          FILTER doc._stix2arango_note == "version=18_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
           RETURN doc.name
       """,
-    "mitre_attack_ics_aliases_v16_0.txt":
+    "mitre_attack_ics_aliases_v18_0.txt":
       """
         FOR alias IN UNIQUE(
           FLATTEN(
             FOR doc IN mitre_attack_ics_vertex_collection
-              FILTER doc._stix2arango_note == "version=16_0"
+              FILTER doc._stix2arango_note == "version=18_0"
               AND doc.type != "x-mitre-matrix"
               AND doc.x_mitre_deprecated != true
               AND doc.revoked != true
@@ -147,10 +147,10 @@ queries = {
         )
         RETURN alias
       """,
-    "mitre_attack_mobile_id_v16_0.txt":
+    "mitre_attack_mobile_id_v18_0.txt":
       """
         FOR doc IN mitre_attack_mobile_vertex_collection
-          FILTER doc._stix2arango_note == "version=16_0"
+          FILTER doc._stix2arango_note == "version=18_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -160,21 +160,21 @@ queries = {
             SORT reference.external_id ASC
             RETURN reference.external_id
       """,
-    "mitre_attack_mobile_name_v16_0.txt":
+    "mitre_attack_mobile_name_v18_0.txt":
       """
         FOR doc IN mitre_attack_mobile_vertex_collection
-          FILTER doc._stix2arango_note == "version=16_0"
+          FILTER doc._stix2arango_note == "version=18_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
           RETURN doc.name
       """,
-    "mitre_attack_mobile_aliases_v16_0.txt":
+    "mitre_attack_mobile_aliases_v18_0.txt":
       """
         FOR alias IN UNIQUE(
           FLATTEN(
             FOR doc IN mitre_attack_mobile_vertex_collection
-              FILTER doc._stix2arango_note == "version=16_0"
+              FILTER doc._stix2arango_note == "version=18_0"
               AND doc.type != "x-mitre-matrix"
               AND doc.x_mitre_deprecated != true
               AND doc.revoked != true
@@ -188,10 +188,10 @@ queries = {
         )
         RETURN alias
       """,
-    "mitre_atlas_id_v4_5_2.txt":
+    "mitre_atlas_id_v4_9_0.txt":
       """
         FOR doc IN mitre_atlas_vertex_collection
-          FILTER doc._stix2arango_note == "version=4_5_2"
+          FILTER doc._stix2arango_note == "version=4_9_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -201,19 +201,19 @@ queries = {
             SORT reference.external_id ASC
             RETURN reference.external_id
       """,
-    "mitre_atlas_name_v4_5_2.txt":
+    "mitre_atlas_name_v4_9_0.txt":
       """
         FOR doc IN mitre_atlas_vertex_collection
-          FILTER doc._stix2arango_note == "version=4_5_2"
+          FILTER doc._stix2arango_note == "version=4_9_0"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
           RETURN doc.name
       """,
-    "disarm_id_v1_5.txt":
+    "disarm_id_v1_6.txt":
       """
         FOR doc IN disarm_vertex_collection
-          FILTER doc._stix2arango_note == "version=1_5"
+          FILTER doc._stix2arango_note == "version=1_6"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
@@ -223,14 +223,23 @@ queries = {
             SORT reference.external_id ASC
             RETURN reference.external_id
     """,
-    "disarm_name_v1_5.txt":
+    "disarm_name_v1_6.txt":
       """
         FOR doc IN disarm_vertex_collection
-          FILTER doc._stix2arango_note == "version=1_5"
+          FILTER doc._stix2arango_note == "version=1_6"
           AND doc.type != "x-mitre-matrix"
           AND doc.x_mitre_deprecated != true
           AND doc.revoked != true
           RETURN doc.name
+      """,
+    "sector_aliases_v1_0.txt":
+      """
+        FOR doc IN sector_vertex_collection
+          FILTER doc._stix2arango_note == "version=1_0"
+            AND doc.revoked != true
+          FOR alias IN (doc.x_opencti_aliases ? doc.x_opencti_aliases : [])
+            COLLECT alias_dedup = alias
+            RETURN alias_dedup
       """
 }
 

txt2stix/includes/lookups/{disarm_id_v1_5.txt → disarm_id_v1_6.txt}

@@ -1,13 +1,14 @@
+DISARM
 T0002
 T0003
 T0004
-T0007
 T0010
-T0013
 T0014
 T0014.001
 T0014.002
 T0015
+T0015.001
+T0015.002
 T0016
 T0017
 T0017.001
@@ -23,9 +24,6 @@ T0029
 T0039
 T0040
 T0042
-T0043
-T0043.001
-T0043.002
 T0044
 T0045
 T0046
@@ -115,11 +113,6 @@ T0088.002
 T0089
 T0089.001
 T0089.003
-T0090
-T0090.001
-T0090.002
-T0090.003
-T0090.004
 T0091
 T0091.001
 T0091.002
@@ -173,31 +166,13 @@ T0102
 T0102.001
 T0102.002
 T0102.003
-T0103
-T0103.001
-T0103.002
-T0104
-T0104.001
-T0104.002
-T0104.003
-T0104.004
-T0104.005
-T0104.006
-T0105
-T0105.001
-T0105.002
-T0105.003
-T0106
-T0106.001
 T0107
-T0108
 T0109
 T0110
 T0111
 T0111.001
 T0111.002
 T0111.003
-T0112
 T0113
 T0114
 T0114.001
@@ -250,7 +225,6 @@ T0129.004
 T0129.005
 T0129.006
 T0129.007
-T0129.008
 T0129.009
 T0129.010
 T0130
@@ -308,9 +282,6 @@ T0140
 T0140.001
 T0140.002
 T0140.003
-T0141
-T0141.001
-T0141.002
 T0143
 T0143.001
 T0143.002
@@ -327,6 +298,98 @@ T0145.004
 T0145.005
 T0145.006
 T0145.007
+T0146
+T0146.001
+T0146.002
+T0146.003
+T0146.004
+T0146.005
+T0146.006
+T0146.007
+T0147
+T0147.001
+T0147.002
+T0147.003
+T0147.004
+T0148
+T0148.001
+T0148.002
+T0148.003
+T0148.004
+T0148.005
+T0148.006
+T0148.007
+T0148.008
+T0148.009
+T0149
+T0149.001
+T0149.002
+T0149.003
+T0149.004
+T0149.005
+T0149.006
+T0149.007
+T0149.008
+T0149.009
+T0150
+T0150.001
+T0150.002
+T0150.003
+T0150.004
+T0150.005
+T0150.006
+T0150.007
+T0150.008
+T0151
+T0151.001
+T0151.002
+T0151.003
+T0151.004
+T0151.005
+T0151.006
+T0151.007
+T0151.008
+T0151.009
+T0151.010
+T0151.011
+T0151.012
+T0151.013
+T0151.014
+T0151.015
+T0151.016
+T0151.017
+T0152
+T0152.001
+T0152.002
+T0152.003
+T0152.004
+T0152.005
+T0152.006
+T0152.007
+T0152.008
+T0152.009
+T0152.010
+T0152.011
+T0152.012
+T0153
+T0153.001
+T0153.002
+T0153.003
+T0153.004
+T0153.005
+T0153.006
+T0153.007
+T0154
+T0154.001
+T0154.002
+T0155
+T0155.001
+T0155.002
+T0155.003
+T0155.004
+T0155.005
+T0155.006
+T0155.007
 TA01
 TA02
 TA05