txt2detection 1.0.8__tar.gz → 1.0.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of txt2detection might be problematic. Click here for more details.

Files changed (61) hide show
  1. {txt2detection-1.0.8 → txt2detection-1.0.9}/PKG-INFO +6 -8
  2. {txt2detection-1.0.8 → txt2detection-1.0.9}/README.md +5 -7
  3. {txt2detection-1.0.8 → txt2detection-1.0.9}/pyproject.toml +1 -1
  4. txt2detection-1.0.9/tests/files/sigma-rule-attack-enterprise.yml +27 -0
  5. txt2detection-1.0.9/tests/files/sigma-rule-attack-flow.yml +25 -0
  6. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-custom-tags.yml +1 -1
  7. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-master.yml +1 -1
  8. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-description.yml +0 -1
  9. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-level.yml +0 -1
  10. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-license.yml +0 -1
  11. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-status.yml +0 -1
  12. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-observables.yml +0 -1
  13. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/manual-tests/README.md +30 -1
  14. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_bundler.py +10 -11
  15. txt2detection-1.0.9/txt2detection/__main__.py +347 -0
  16. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/base.py +41 -13
  17. txt2detection-1.0.9/txt2detection/ai_extractor/models.py +34 -0
  18. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/openai.py +1 -3
  19. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/openrouter.py +4 -4
  20. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/prompts.py +130 -3
  21. txt2detection-1.0.9/txt2detection/attack_flow.py +233 -0
  22. txt2detection-1.0.9/txt2detection/bundler.py +370 -0
  23. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/credential_checker.py +11 -9
  24. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/models.py +11 -0
  25. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/observables.py +0 -1
  26. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/utils.py +24 -12
  27. txt2detection-1.0.8/txt2detection/__main__.py +0 -196
  28. txt2detection-1.0.8/txt2detection/bundler.py +0 -296
  29. {txt2detection-1.0.8 → txt2detection-1.0.9}/.env.example +0 -0
  30. {txt2detection-1.0.8 → txt2detection-1.0.9}/.env.markdown +0 -0
  31. {txt2detection-1.0.8 → txt2detection-1.0.9}/.github/workflows/create-release.yml +0 -0
  32. {txt2detection-1.0.8 → txt2detection-1.0.9}/.github/workflows/run-tests.yml +0 -0
  33. {txt2detection-1.0.8 → txt2detection-1.0.9}/.gitignore +0 -0
  34. {txt2detection-1.0.8 → txt2detection-1.0.9}/LICENSE +0 -0
  35. {txt2detection-1.0.8 → txt2detection-1.0.9}/config/detection_languages.yaml +0 -0
  36. {txt2detection-1.0.8 → txt2detection-1.0.9}/docs/README.md +0 -0
  37. {txt2detection-1.0.8 → txt2detection-1.0.9}/docs/txt2detection.png +0 -0
  38. {txt2detection-1.0.8 → txt2detection-1.0.9}/requirements.txt +0 -0
  39. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/CVE-2024-56520.txt +0 -0
  40. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/EC2-exfil.txt +0 -0
  41. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/observables.txt +0 -0
  42. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-existing-related.yml +0 -0
  43. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-author.yml +0 -0
  44. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-date.yml +0 -0
  45. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-tags.yml +0 -0
  46. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-title.yml +0 -0
  47. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-one-date.yml +0 -0
  48. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/__init__.py +0 -0
  49. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/requirements.txt +0 -0
  50. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_main.py +0 -0
  51. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_main_run_txt2detction.py +0 -0
  52. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_models.py +0 -0
  53. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_observables.py +0 -0
  54. {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_utils.py +0 -0
  55. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/__init__.py +0 -0
  56. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/__init__.py +0 -0
  57. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/anthropic.py +0 -0
  58. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/deepseek.py +0 -0
  59. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/gemini.py +0 -0
  60. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/utils.py +0 -0
  61. {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: txt2detection
3
- Version: 1.0.8
3
+ Version: 1.0.9
4
4
  Summary: A command line tool that takes a txt file containing threat intelligence and turns it into a detection rule.
5
5
  Project-URL: Homepage, https://github.com/muchdogesec/txt2detection
6
6
  Project-URL: Issues, https://github.com/muchdogesec/txt2detection/issues
@@ -72,12 +72,6 @@ txt2detection allows a user to enter some threat intelligence as a file to consi
72
72
  2. Based on the user input, AI prompts structured and sent to produce an intelligence rule
73
73
  3. Rules converted into STIX objects
74
74
 
75
- ## tl;dr
76
-
77
- [![txt2detection](https://img.youtube.com/vi/uJWXYKyu3Xg/0.jpg)](https://www.youtube.com/watch?v=uJWXYKyu3Xg)
78
-
79
- [Watch the demo](https://www.youtube.com/watch?v=uJWXYKyu3Xg).
80
-
81
75
  ## Usage
82
76
 
83
77
  ### Setup
@@ -162,12 +156,14 @@ Use this mode to generate a set of rules from an input text file;
162
156
  * `--license` (optional): [License of the rule according the SPDX ID specification](https://spdx.org/licenses/). Will be added to the rule.
163
157
  * `--reference_urls` (optional): A list of URLs to be added as `references` in the Sigma Rule property and in the `external_references` property of the Indicator and Report STIX object created. e.g `"https://www.google.com/" "https://www.facebook.com/"`
164
158
  * `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
165
- * `ai_provider` (required): defines the `provider:model` to be used to generate the rule. Select one option. Currently supports:
159
+ * `--ai_provider` (required): defines the `provider:model` to be used to generate the rule. Select one option. Currently supports:
166
160
  * Provider (env var required `OPENROUTER_API_KEY`): `openrouter:`, providers/models `openai/gpt-4o`, `deepseek/deepseek-chat` ([More here](https://openrouter.ai/models))
167
161
  * Provider (env var required `OPENAI_API_KEY`): `openai:`, models e.g.: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `gpt-4` ([More here](https://platform.openai.com/docs/models))
168
162
  * Provider (env var required `ANTHROPIC_API_KEY`): `anthropic:`, models e.g.: `claude-3-5-sonnet-latest`, `claude-3-5-haiku-latest`, `claude-3-opus-latest` ([More here](https://docs.anthropic.com/en/docs/about-claude/models))
169
163
  * Provider (env var required `GOOGLE_API_KEY`): `gemini:models/`, models: `gemini-1.5-pro-latest`, `gemini-1.5-flash-latest` ([More here](https://ai.google.dev/gemini-api/docs/models/gemini))
170
164
  * Provider (env var required `DEEPSEEK_API_KEY`): `deepseek:`, models `deepseek-chat` ([More here](https://api-docs.deepseek.com/quick_start/pricing))
165
+ * `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
166
+ * `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
171
167
 
172
168
  Note, in this mode, the following values will be automatically assigned to the rule
173
169
 
@@ -194,6 +190,8 @@ Note, in this mode you should be aware of a few things;
194
190
  * `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
195
191
  * `status` (optional): either `stable`, `test`, `experimental`, `deprecated`, `unsupported`. If passed, will overwrite any existing `status` recorded in the rule
196
192
  * `level` (optional): either `informational`, `low`, `medium`, `high`, `critical`. If passed, will overwrite any existing `level` recorded in the rule
193
+ * `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
194
+ * `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
197
195
 
198
196
  ### A note on observable extraction
199
197
 
@@ -31,12 +31,6 @@ txt2detection allows a user to enter some threat intelligence as a file to consi
31
31
  2. Based on the user input, AI prompts structured and sent to produce an intelligence rule
32
32
  3. Rules converted into STIX objects
33
33
 
34
- ## tl;dr
35
-
36
- [![txt2detection](https://img.youtube.com/vi/uJWXYKyu3Xg/0.jpg)](https://www.youtube.com/watch?v=uJWXYKyu3Xg)
37
-
38
- [Watch the demo](https://www.youtube.com/watch?v=uJWXYKyu3Xg).
39
-
40
34
  ## Usage
41
35
 
42
36
  ### Setup
@@ -121,12 +115,14 @@ Use this mode to generate a set of rules from an input text file;
121
115
  * `--license` (optional): [License of the rule according the SPDX ID specification](https://spdx.org/licenses/). Will be added to the rule.
122
116
  * `--reference_urls` (optional): A list of URLs to be added as `references` in the Sigma Rule property and in the `external_references` property of the Indicator and Report STIX object created. e.g `"https://www.google.com/" "https://www.facebook.com/"`
123
117
  * `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
124
- * `ai_provider` (required): defines the `provider:model` to be used to generate the rule. Select one option. Currently supports:
118
+ * `--ai_provider` (required): defines the `provider:model` to be used to generate the rule. Select one option. Currently supports:
125
119
  * Provider (env var required `OPENROUTER_API_KEY`): `openrouter:`, providers/models `openai/gpt-4o`, `deepseek/deepseek-chat` ([More here](https://openrouter.ai/models))
126
120
  * Provider (env var required `OPENAI_API_KEY`): `openai:`, models e.g.: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `gpt-4` ([More here](https://platform.openai.com/docs/models))
127
121
  * Provider (env var required `ANTHROPIC_API_KEY`): `anthropic:`, models e.g.: `claude-3-5-sonnet-latest`, `claude-3-5-haiku-latest`, `claude-3-opus-latest` ([More here](https://docs.anthropic.com/en/docs/about-claude/models))
128
122
  * Provider (env var required `GOOGLE_API_KEY`): `gemini:models/`, models: `gemini-1.5-pro-latest`, `gemini-1.5-flash-latest` ([More here](https://ai.google.dev/gemini-api/docs/models/gemini))
129
123
  * Provider (env var required `DEEPSEEK_API_KEY`): `deepseek:`, models `deepseek-chat` ([More here](https://api-docs.deepseek.com/quick_start/pricing))
124
+ * `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
125
+ * `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
130
126
 
131
127
  Note, in this mode, the following values will be automatically assigned to the rule
132
128
 
@@ -153,6 +149,8 @@ Note, in this mode you should be aware of a few things;
153
149
  * `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
154
150
  * `status` (optional): either `stable`, `test`, `experimental`, `deprecated`, `unsupported`. If passed, will overwrite any existing `status` recorded in the rule
155
151
  * `level` (optional): either `informational`, `low`, `medium`, `high`, `critical`. If passed, will overwrite any existing `level` recorded in the rule
152
+ * `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
153
+ * `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
156
154
 
157
155
  ### A note on observable extraction
158
156
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "txt2detection"
7
- version = "1.0.8"
7
+ version = "1.0.9"
8
8
  authors = [
9
9
  { name = "dogesec" }
10
10
  ]
@@ -0,0 +1,27 @@
1
+ title: Attack Navigator Enterprise
2
+ id: a18e76d1-f152-4b87-a552-d46f41afd637
3
+ status: test
4
+ description: Build an Attack Enterprise Navigator layer
5
+ references:
6
+ - https://www.dogesec.com
7
+ author: dogesec
8
+ date: 2020-01-01
9
+ modified: 2020-01-02
10
+ tags:
11
+ - tlp.clear
12
+ - attack.t1547
13
+ - attack.t1671
14
+ - attack.t1025
15
+ - attack.command-and-control
16
+ - attack.t1661 # will fail is mobile
17
+ logsource:
18
+ product: okta
19
+ service: okta
20
+ detection:
21
+ selection:
22
+ eventtype:
23
+ - policy.lifecycle.update
24
+ - policy.lifecycle.delete
25
+ condition: selection
26
+ level: low
27
+ license: MIT
@@ -0,0 +1,25 @@
1
+ title: Attack Flow demo
2
+ id: 7894eba6-b0e5-48d9-be52-26bf5a556e45
3
+ status: test
4
+ description: Build an Attack Flow from a Sigma Rule
5
+ references:
6
+ - https://www.dogesec.com
7
+ author: dogesec
8
+ date: 2020-01-01
9
+ modified: 2020-01-02
10
+ tags:
11
+ - tlp.clear
12
+ - attack.t1547
13
+ - attack.t1671
14
+ - attack.t1025
15
+ logsource:
16
+ product: okta
17
+ service: okta
18
+ detection:
19
+ selection:
20
+ eventtype:
21
+ - policy.lifecycle.update
22
+ - policy.lifecycle.delete
23
+ condition: selection
24
+ level: low
25
+ license: MIT
@@ -11,7 +11,7 @@ modified: 2022-10-09
11
11
  tags:
12
12
  - tlp.red
13
13
  - attack.t1547
14
- - attack.command_and_control
14
+ - attack.command-and-control
15
15
  - cve.2024-56520
16
16
  - detection.xyz
17
17
  logsource:
@@ -11,7 +11,7 @@ modified: 2022-10-09
11
11
  tags:
12
12
  - tlp.red
13
13
  - attack.t1547
14
- - attack.command_and_control
14
+ - attack.command-and-control
15
15
  - cve.2024-56520
16
16
  logsource:
17
17
  product: okta
@@ -10,7 +10,6 @@ modified: 2022-10-09
10
10
  tags:
11
11
  - tlp.red
12
12
  - attack.t1547
13
- - attack.command_and_control
14
13
  - cve.2024-56520
15
14
  logsource:
16
15
  product: okta
@@ -11,7 +11,6 @@ modified: 2022-10-09
11
11
  tags:
12
12
  - tlp.red
13
13
  - attack.t1547
14
- - attack.command_and_control
15
14
  - cve.2024-56520
16
15
  logsource:
17
16
  product: okta
@@ -11,7 +11,6 @@ modified: 2022-10-09
11
11
  tags:
12
12
  - tlp.red
13
13
  - attack.t1547
14
- - attack.command_and_control
15
14
  - cve.2024-56520
16
15
  logsource:
17
16
  product: okta
@@ -10,7 +10,6 @@ modified: 2022-10-09
10
10
  tags:
11
11
  - tlp.red
12
12
  - attack.t1547
13
- - attack.command_and_control
14
13
  - cve.2024-56520
15
14
  logsource:
16
15
  product: okta
@@ -11,7 +11,6 @@ modified: 2022-10-09
11
11
  tags:
12
12
  - tlp.red
13
13
  - attack.t1547
14
- - attack.command_and_control
15
14
  - cve.2024-56520
16
15
  logsource:
17
16
  product: okta
@@ -445,4 +445,33 @@ python3 txt2detection.py sigma \
445
445
  --name "Overwrite status" \
446
446
  --status unsupported \
447
447
  --report_id d2d01afa-dc55-4a80-8d62-15d154450112
448
- ```
448
+ ```
449
+
450
+
451
+ ## Attack Flow
452
+
453
+ ```shell
454
+ python3 txt2detection.py sigma \
455
+ --sigma_file tests/files/sigma-rule-attack-flow.yml \
456
+ --name "Create ATT&CK Flow" \
457
+ --report_id 330e2030-1dc2-45e6-be13-9342b102621b \
458
+ --ai_provider openai:gpt-5 \
459
+ --ai_create_attack_flow
460
+ ```
461
+
462
+ ## Attack Navigator
463
+
464
+ ### Enterprise
465
+
466
+ ```shell
467
+ python3 txt2detection.py sigma \
468
+ --sigma_file tests/files/sigma-rule-attack-enterprise.yml \
469
+ --name "Attack Navigator Enterprise" \
470
+ --report_id a18e76d1-f152-4b87-a552-d46f41afd637 \
471
+ --ai_provider openai:gpt-5 \
472
+ --ai_create_attack_navigator_layer
473
+ ```
474
+
475
+ ### Mobile / ICS
476
+
477
+ Not currently supported by Sigma.
@@ -28,17 +28,6 @@ def dummy_detection():
28
28
  return detection
29
29
 
30
30
 
31
- @pytest.fixture
32
- def bundler_instance():
33
- return Bundler(
34
- name="Test Report",
35
- identity=None,
36
- tlp_level="red",
37
- description="This is a test report.",
38
- labels=["tlp.red", "test.test-var"],
39
- created=datetime(2025, 1, 1),
40
- report_id="74e36652-00f5-4dca-bf10-9f02fc996dcc",
41
- )
42
31
 
43
32
 
44
33
  def test_bundler_initialization(bundler_instance):
@@ -284,3 +273,13 @@ def test_bundle_detections__creates_log_source(dummy_detection, bundler_instance
284
273
  ],
285
274
  },
286
275
  ]
276
+
277
+ def test_get_attack_objects(bundler_instance):
278
+ retval = bundler_instance.get_attack_objects(['T1190', 'T1547'])
279
+ print({r['id'] for r in retval})
280
+ assert {r['id'] for r in retval} == {'attack-pattern--1ecb2399-e8ba-4f6b-8ba7-5c27d49405cf', 'attack-pattern--3f886f2a-874f-4333-b794-aa6075009b1c'}
281
+
282
+ def test_get_cve_objects(bundler_instance):
283
+ cves = ['CVE-2025-1234', 'CVE-2024-1234']
284
+ retval = bundler_instance.get_cve_objects(cves)
285
+ assert {r['name'] for r in retval} == set(cves)
@@ -0,0 +1,347 @@
1
+ import argparse
2
+ from datetime import UTC, datetime
3
+
4
+ from dataclasses import dataclass
5
+ import io
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ import logging
10
+ import re
11
+ import sys
12
+ import uuid
13
+ from stix2 import Identity
14
+ import yaml
15
+
16
+ from txt2detection import attack_flow, credential_checker
17
+ from txt2detection.ai_extractor.base import BaseAIExtractor
18
+ from txt2detection.models import (
19
+ TAG_PATTERN,
20
+ DetectionContainer,
21
+ Level,
22
+ SigmaRuleDetection,
23
+ )
24
+ from txt2detection.utils import validate_token_count
25
+
26
+
27
+ def configureLogging():
28
+ # Configure logging
29
+ stream_handler = logging.StreamHandler() # Log to stdout and stderr
30
+ stream_handler.setLevel(logging.INFO)
31
+ logging.basicConfig(
32
+ level=logging.DEBUG, # Set the desired logging level
33
+ format=f"%(asctime)s [%(levelname)s] %(message)s",
34
+ handlers=[stream_handler],
35
+ datefmt="%d-%b-%y %H:%M:%S",
36
+ )
37
+
38
+ return logging.root
39
+
40
+
41
+ configureLogging()
42
+
43
+
44
+ def setLogFile(logger, file: Path):
45
+ file.parent.mkdir(parents=True, exist_ok=True)
46
+ logger.info(f"Saving log to `{file.absolute()}`")
47
+ handler = logging.FileHandler(file, "w")
48
+ handler.formatter = logging.Formatter(
49
+ fmt="%(levelname)s %(asctime)s - %(message)s", datefmt="%d-%b-%y %H:%M:%S"
50
+ )
51
+ handler.setLevel(logging.DEBUG)
52
+ logger.addHandler(handler)
53
+ logger.info("=====================txt2detection======================")
54
+
55
+
56
+ from .bundler import Bundler
57
+ import shutil
58
+
59
+
60
+ from .utils import STATUSES, as_date, make_identity, valid_licenses, parse_model
61
+
62
+
63
+ def parse_identity(str):
64
+ return Identity(**json.loads(str))
65
+
66
+
67
+ @dataclass
68
+ class Args:
69
+ input_file: str
70
+ input_text: str
71
+ name: str
72
+ tlp_level: str
73
+ labels: list[str]
74
+ created: datetime
75
+ use_identity: Identity
76
+ ai_provider: BaseAIExtractor
77
+ report_id: uuid.UUID
78
+ external_refs: dict[str, str]
79
+ reference_urls: list[str]
80
+
81
+
82
+ def parse_created(value):
83
+ """Convert the created timestamp to a datetime object."""
84
+ try:
85
+ return datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC)
86
+ except ValueError:
87
+ raise argparse.ArgumentTypeError(
88
+ "Invalid date format. Use YYYY-MM-DDTHH:MM:SS."
89
+ )
90
+
91
+
92
+ def parse_ref(value):
93
+ m = re.compile(r"(.+?)=(.+)").match(value)
94
+ if not m:
95
+ raise argparse.ArgumentTypeError("must be in format key=value")
96
+ return dict(source_name=m.group(1), external_id=m.group(2))
97
+
98
+
99
+ def parse_label(label: str):
100
+ if not TAG_PATTERN.match(label):
101
+ raise argparse.ArgumentTypeError(
102
+ "Invalid label format. Must follow sigma tag format {namespace}.{label}"
103
+ )
104
+ namespace, _, _ = label.partition(".")
105
+ if namespace in ["tlp"]:
106
+ raise argparse.ArgumentTypeError(f"Unsupported tag namespace `{namespace}`")
107
+ return label
108
+
109
+
110
+ def parse_args():
111
+ parser = argparse.ArgumentParser(
112
+ description="Convert text file to detection format."
113
+ )
114
+ mode = parser.add_subparsers(
115
+ title="process-mode", dest="mode", description="mode to use"
116
+ )
117
+ file = mode.add_parser("file", help="process a file input using ai")
118
+ text = mode.add_parser("text", help="process a text argument using ai")
119
+ sigma = mode.add_parser("sigma", help="process a sigma file without ai")
120
+ check_credentials = mode.add_parser(
121
+ "check-credentials",
122
+ help="show status of external services with respect to credentials",
123
+ )
124
+
125
+ for mode_parser in [file, text, sigma]:
126
+ mode_parser.add_argument(
127
+ "--report_id", type=uuid.UUID, help="report_id to use for generated report"
128
+ )
129
+ mode_parser.add_argument(
130
+ "--name",
131
+ required=True,
132
+ help="Name of file, max 72 chars. Will be used in the STIX Report Object created.",
133
+ )
134
+ mode_parser.add_argument(
135
+ "--tlp_level",
136
+ choices=["clear", "green", "amber", "amber_strict", "red"],
137
+ help="Options are clear, green, amber, amber_strict, red. Default is clear if not passed.",
138
+ )
139
+ mode_parser.add_argument(
140
+ "--labels",
141
+ type=parse_label,
142
+ action="extend",
143
+ nargs="+",
144
+ help="Comma-separated list of labels. Case-insensitive (will be converted to lower-case). Allowed a-z, 0-9.",
145
+ )
146
+ mode_parser.add_argument(
147
+ "--created",
148
+ type=parse_created,
149
+ help="Explicitly set created time in format YYYY-MM-DDTHH:MM:SS.sssZ. Default is current time.",
150
+ )
151
+ mode_parser.add_argument(
152
+ "--use_identity",
153
+ type=parse_identity,
154
+ help="Pass a full STIX 2.1 identity object (properly escaped). Validated by the STIX2 library. Default is SIEM Rules identity.",
155
+ )
156
+ mode_parser.add_argument(
157
+ "--ai_provider",
158
+ required=False,
159
+ type=parse_model,
160
+ help="(required): defines the `provider:model` to be used. Select one option.",
161
+ metavar="provider[:model]",
162
+ )
163
+ mode_parser.add_argument(
164
+ "--external_refs",
165
+ type=parse_ref,
166
+ help="pass additional `external_references` entry (or entries) to the report object created. e.g --external_ref author=dogesec link=https://dkjjadhdaj.net",
167
+ default=[],
168
+ metavar="{source_name}={external_id}",
169
+ action="extend",
170
+ nargs="+",
171
+ )
172
+ mode_parser.add_argument(
173
+ "--reference_urls",
174
+ help="pass additional `external_references` url entry (or entries) to the report object created.",
175
+ default=[],
176
+ metavar="{url}",
177
+ action="extend",
178
+ nargs="+",
179
+ )
180
+ mode_parser.add_argument(
181
+ "--license",
182
+ help="Valid SPDX license for the rule",
183
+ default=None,
184
+ metavar="[LICENSE]",
185
+ choices=valid_licenses(),
186
+ )
187
+ mode_parser.add_argument(
188
+ "--ai_create_attack_navigator_layer",
189
+ help="Create navigator layer",
190
+ action="store_true",
191
+ default=False,
192
+ )
193
+ mode_parser.add_argument(
194
+ "--ai_create_attack_flow",
195
+ help="Create attack flow",
196
+ action="store_true",
197
+ default=False,
198
+ )
199
+
200
+ file.add_argument(
201
+ "--input_file",
202
+ help="The file to be converted. Must be .txt",
203
+ type=lambda x: Path(x).read_text(),
204
+ )
205
+ text.add_argument("--input_text", help="The text to be converted")
206
+ sigma.add_argument(
207
+ "--sigma_file",
208
+ help="The sigma file to be converted. Must be .yml",
209
+ type=lambda x: Path(x).read_text(),
210
+ )
211
+ sigma.add_argument(
212
+ "--status",
213
+ help="If passed, will overwrite any existing `status` recorded in the rule",
214
+ choices=STATUSES,
215
+ )
216
+ sigma.add_argument(
217
+ "--level",
218
+ help="If passed, will overwrite any existing `level` recorded in the rule",
219
+ choices=Level._member_names_,
220
+ )
221
+
222
+ args: Args = parser.parse_args()
223
+ if args.mode == "check-credentials":
224
+ statuses = credential_checker.check_statuses(test_llms=True)
225
+ credential_checker.format_statuses(statuses)
226
+ sys.exit(0)
227
+
228
+ if args.mode != "sigma":
229
+ assert args.ai_provider, "--ai_provider is required in file or txt mode"
230
+
231
+ if args.ai_create_attack_navigator_layer or args.ai_create_attack_flow:
232
+ assert (
233
+ args.ai_provider
234
+ ), "--ai_provider is required when --ai_create_attack_navigator_layer/--ai_create_attack_flow is passed"
235
+
236
+ if args.mode == "file":
237
+ args.input_text = args.input_file
238
+
239
+ args.input_text = getattr(args, "input_text", "")
240
+ if not args.report_id:
241
+ args.report_id = Bundler.generate_report_id(
242
+ args.use_identity.id if args.use_identity else None, args.created, args.name
243
+ )
244
+
245
+ return args
246
+
247
+
248
+ def run_txt2detection(
249
+ name,
250
+ identity,
251
+ tlp_level,
252
+ input_text: str,
253
+ labels: list[str],
254
+ report_id: str | uuid.UUID,
255
+ ai_provider: BaseAIExtractor,
256
+ ai_create_attack_flow=False,
257
+ ai_create_attack_navigator_layer=False,
258
+ **kwargs,
259
+ ) -> Bundler:
260
+ if (
261
+ kwargs.get("sigma_file") != "sigma_file"
262
+ or ai_create_attack_flow
263
+ or ai_create_attack_navigator_layer
264
+ ):
265
+ validate_token_count(
266
+ int(os.getenv("INPUT_TOKEN_LIMIT", 0)), input_text, ai_provider
267
+ )
268
+
269
+ if sigma := kwargs.get("sigma_file"):
270
+ detection = get_sigma_detections(sigma)
271
+ if not identity and detection.author:
272
+ identity = make_identity(detection.author)
273
+ kwargs.update(
274
+ reference_urls=kwargs.setdefault("reference_urls", [])
275
+ + detection.references
276
+ )
277
+ if not kwargs.get("created"):
278
+ # only consider rule.date and rule.modified if user does not pass --created
279
+ kwargs.update(
280
+ created=detection.date,
281
+ modified=detection.modified,
282
+ )
283
+ detection.level = kwargs.get("level", detection.level)
284
+ detection.status = kwargs.get("status", detection.status)
285
+ detection.date = as_date(kwargs.get("created"))
286
+ detection.modified = as_date(kwargs.get("modified"))
287
+ detection.references = kwargs["reference_urls"]
288
+ detection.detection_id = str(report_id).removeprefix("report--")
289
+ bundler = Bundler(
290
+ name or detection.title,
291
+ identity,
292
+ tlp_level or detection.tlp_level or "clear",
293
+ detection.description,
294
+ (labels or []) + detection.tags,
295
+ report_id=report_id,
296
+ **kwargs,
297
+ )
298
+ detections = DetectionContainer(success=True, detections=[])
299
+ detections.detections.append(detection)
300
+ else:
301
+ bundler = Bundler(
302
+ name, identity, tlp_level, input_text, labels, report_id=report_id, **kwargs
303
+ )
304
+ detections = ai_provider.get_detections(input_text)
305
+ bundler.bundle_detections(detections)
306
+
307
+ if ai_create_attack_flow or ai_create_attack_navigator_layer:
308
+ bundler.data.attack_flow, bundler.data.navigator_layer = (
309
+ attack_flow.extract_attack_flow_and_navigator(
310
+ bundler,
311
+ bundler.report.description,
312
+ ai_create_attack_flow,
313
+ ai_create_attack_navigator_layer,
314
+ ai_provider,
315
+ )
316
+ )
317
+ return bundler
318
+
319
+
320
+ def get_sigma_detections(sigma: str) -> SigmaRuleDetection:
321
+ obj = yaml.safe_load(io.StringIO(sigma))
322
+ return SigmaRuleDetection.model_validate(obj)
323
+
324
+
325
+ def main(args: Args):
326
+
327
+ setLogFile(logging.root, Path(f"logs/log-{args.report_id}.log"))
328
+ logging.info(f"starting argument: {json.dumps(sys.argv[1:])}")
329
+ kwargs = args.__dict__
330
+ kwargs["identity"] = args.use_identity
331
+ bundler = run_txt2detection(**kwargs)
332
+
333
+ output_dir = Path("./output") / str(bundler.bundle.id)
334
+ shutil.rmtree(output_dir, ignore_errors=True)
335
+ rules_dir = output_dir / "rules"
336
+ rules_dir.mkdir(exist_ok=True, parents=True)
337
+
338
+ output_path = output_dir / "bundle.json"
339
+ data_path = output_dir / f"data.json"
340
+ output_path.write_text(bundler.to_json())
341
+ data_path.write_text(bundler.data.model_dump_json(indent=4))
342
+ for obj in bundler.bundle["objects"]:
343
+ if obj["type"] != "indicator" or obj["pattern_type"] != "sigma":
344
+ continue
345
+ name = obj["id"].replace("indicator", "rule") + ".yml"
346
+ (rules_dir / name).write_text(obj["pattern"])
347
+ logging.info(f"Writing bundle output to `{output_path}`")