txt2detection 1.0.8__tar.gz → 1.0.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of txt2detection might be problematic. Click here for more details.
- {txt2detection-1.0.8 → txt2detection-1.0.9}/PKG-INFO +6 -8
- {txt2detection-1.0.8 → txt2detection-1.0.9}/README.md +5 -7
- {txt2detection-1.0.8 → txt2detection-1.0.9}/pyproject.toml +1 -1
- txt2detection-1.0.9/tests/files/sigma-rule-attack-enterprise.yml +27 -0
- txt2detection-1.0.9/tests/files/sigma-rule-attack-flow.yml +25 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-custom-tags.yml +1 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-master.yml +1 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-description.yml +0 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-level.yml +0 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-license.yml +0 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-status.yml +0 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-observables.yml +0 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/manual-tests/README.md +30 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_bundler.py +10 -11
- txt2detection-1.0.9/txt2detection/__main__.py +347 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/base.py +41 -13
- txt2detection-1.0.9/txt2detection/ai_extractor/models.py +34 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/openai.py +1 -3
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/openrouter.py +4 -4
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/prompts.py +130 -3
- txt2detection-1.0.9/txt2detection/attack_flow.py +233 -0
- txt2detection-1.0.9/txt2detection/bundler.py +370 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/credential_checker.py +11 -9
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/models.py +11 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/observables.py +0 -1
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/utils.py +24 -12
- txt2detection-1.0.8/txt2detection/__main__.py +0 -196
- txt2detection-1.0.8/txt2detection/bundler.py +0 -296
- {txt2detection-1.0.8 → txt2detection-1.0.9}/.env.example +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/.env.markdown +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/.github/workflows/create-release.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/.github/workflows/run-tests.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/.gitignore +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/LICENSE +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/config/detection_languages.yaml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/docs/README.md +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/docs/txt2detection.png +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/requirements.txt +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/CVE-2024-56520.txt +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/EC2-exfil.txt +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/observables.txt +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-existing-related.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-author.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-date.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-tags.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-no-title.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/files/sigma-rule-one-date.yml +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/__init__.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/requirements.txt +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_main.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_main_run_txt2detction.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_models.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_observables.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/tests/src/test_utils.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/__init__.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/__init__.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/anthropic.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/deepseek.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/gemini.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection/ai_extractor/utils.py +0 -0
- {txt2detection-1.0.8 → txt2detection-1.0.9}/txt2detection.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: txt2detection
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.9
|
|
4
4
|
Summary: A command line tool that takes a txt file containing threat intelligence and turns it into a detection rule.
|
|
5
5
|
Project-URL: Homepage, https://github.com/muchdogesec/txt2detection
|
|
6
6
|
Project-URL: Issues, https://github.com/muchdogesec/txt2detection/issues
|
|
@@ -72,12 +72,6 @@ txt2detection allows a user to enter some threat intelligence as a file to consi
|
|
|
72
72
|
2. Based on the user input, AI prompts structured and sent to produce an intelligence rule
|
|
73
73
|
3. Rules converted into STIX objects
|
|
74
74
|
|
|
75
|
-
## tl;dr
|
|
76
|
-
|
|
77
|
-
[](https://www.youtube.com/watch?v=uJWXYKyu3Xg)
|
|
78
|
-
|
|
79
|
-
[Watch the demo](https://www.youtube.com/watch?v=uJWXYKyu3Xg).
|
|
80
|
-
|
|
81
75
|
## Usage
|
|
82
76
|
|
|
83
77
|
### Setup
|
|
@@ -162,12 +156,14 @@ Use this mode to generate a set of rules from an input text file;
|
|
|
162
156
|
* `--license` (optional): [License of the rule according the SPDX ID specification](https://spdx.org/licenses/). Will be added to the rule.
|
|
163
157
|
* `--reference_urls` (optional): A list of URLs to be added as `references` in the Sigma Rule property and in the `external_references` property of the Indicator and Report STIX object created. e.g `"https://www.google.com/" "https://www.facebook.com/"`
|
|
164
158
|
* `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
|
|
165
|
-
*
|
|
159
|
+
* `--ai_provider` (required): defines the `provider:model` to be used to generate the rule. Select one option. Currently supports:
|
|
166
160
|
* Provider (env var required `OPENROUTER_API_KEY`): `openrouter:`, providers/models `openai/gpt-4o`, `deepseek/deepseek-chat` ([More here](https://openrouter.ai/models))
|
|
167
161
|
* Provider (env var required `OPENAI_API_KEY`): `openai:`, models e.g.: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `gpt-4` ([More here](https://platform.openai.com/docs/models))
|
|
168
162
|
* Provider (env var required `ANTHROPIC_API_KEY`): `anthropic:`, models e.g.: `claude-3-5-sonnet-latest`, `claude-3-5-haiku-latest`, `claude-3-opus-latest` ([More here](https://docs.anthropic.com/en/docs/about-claude/models))
|
|
169
163
|
* Provider (env var required `GOOGLE_API_KEY`): `gemini:models/`, models: `gemini-1.5-pro-latest`, `gemini-1.5-flash-latest` ([More here](https://ai.google.dev/gemini-api/docs/models/gemini))
|
|
170
164
|
* Provider (env var required `DEEPSEEK_API_KEY`): `deepseek:`, models `deepseek-chat` ([More here](https://api-docs.deepseek.com/quick_start/pricing))
|
|
165
|
+
* `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
|
|
166
|
+
* `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
|
|
171
167
|
|
|
172
168
|
Note, in this mode, the following values will be automatically assigned to the rule
|
|
173
169
|
|
|
@@ -194,6 +190,8 @@ Note, in this mode you should be aware of a few things;
|
|
|
194
190
|
* `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
|
|
195
191
|
* `status` (optional): either `stable`, `test`, `experimental`, `deprecated`, `unsupported`. If passed, will overwrite any existing `status` recorded in the rule
|
|
196
192
|
* `level` (optional): either `informational`, `low`, `medium`, `high`, `critical`. If passed, will overwrite any existing `level` recorded in the rule
|
|
193
|
+
* `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
|
|
194
|
+
* `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
|
|
197
195
|
|
|
198
196
|
### A note on observable extraction
|
|
199
197
|
|
|
@@ -31,12 +31,6 @@ txt2detection allows a user to enter some threat intelligence as a file to consi
|
|
|
31
31
|
2. Based on the user input, AI prompts structured and sent to produce an intelligence rule
|
|
32
32
|
3. Rules converted into STIX objects
|
|
33
33
|
|
|
34
|
-
## tl;dr
|
|
35
|
-
|
|
36
|
-
[](https://www.youtube.com/watch?v=uJWXYKyu3Xg)
|
|
37
|
-
|
|
38
|
-
[Watch the demo](https://www.youtube.com/watch?v=uJWXYKyu3Xg).
|
|
39
|
-
|
|
40
34
|
## Usage
|
|
41
35
|
|
|
42
36
|
### Setup
|
|
@@ -121,12 +115,14 @@ Use this mode to generate a set of rules from an input text file;
|
|
|
121
115
|
* `--license` (optional): [License of the rule according the SPDX ID specification](https://spdx.org/licenses/). Will be added to the rule.
|
|
122
116
|
* `--reference_urls` (optional): A list of URLs to be added as `references` in the Sigma Rule property and in the `external_references` property of the Indicator and Report STIX object created. e.g `"https://www.google.com/" "https://www.facebook.com/"`
|
|
123
117
|
* `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
|
|
124
|
-
*
|
|
118
|
+
* `--ai_provider` (required): defines the `provider:model` to be used to generate the rule. Select one option. Currently supports:
|
|
125
119
|
* Provider (env var required `OPENROUTER_API_KEY`): `openrouter:`, providers/models `openai/gpt-4o`, `deepseek/deepseek-chat` ([More here](https://openrouter.ai/models))
|
|
126
120
|
* Provider (env var required `OPENAI_API_KEY`): `openai:`, models e.g.: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `gpt-4` ([More here](https://platform.openai.com/docs/models))
|
|
127
121
|
* Provider (env var required `ANTHROPIC_API_KEY`): `anthropic:`, models e.g.: `claude-3-5-sonnet-latest`, `claude-3-5-haiku-latest`, `claude-3-opus-latest` ([More here](https://docs.anthropic.com/en/docs/about-claude/models))
|
|
128
122
|
* Provider (env var required `GOOGLE_API_KEY`): `gemini:models/`, models: `gemini-1.5-pro-latest`, `gemini-1.5-flash-latest` ([More here](https://ai.google.dev/gemini-api/docs/models/gemini))
|
|
129
123
|
* Provider (env var required `DEEPSEEK_API_KEY`): `deepseek:`, models `deepseek-chat` ([More here](https://api-docs.deepseek.com/quick_start/pricing))
|
|
124
|
+
* `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
|
|
125
|
+
* `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
|
|
130
126
|
|
|
131
127
|
Note, in this mode, the following values will be automatically assigned to the rule
|
|
132
128
|
|
|
@@ -153,6 +149,8 @@ Note, in this mode you should be aware of a few things;
|
|
|
153
149
|
* `--external_refs` (optional): txt2detection will automatically populate the `external_references` of the report object it creates for the input. You can use this value to add additional objects to `external_references`. Note, you can only add `source_name` and `external_id` values currently. Pass as `source_name=external_id`. e.g. `--external_refs txt2stix=demo1 source=id` would create the following objects under the `external_references` property: `{"source_name":"txt2stix","external_id":"demo1"},{"source_name":"source","external_id":"id"}`
|
|
154
150
|
* `status` (optional): either `stable`, `test`, `experimental`, `deprecated`, `unsupported`. If passed, will overwrite any existing `status` recorded in the rule
|
|
155
151
|
* `level` (optional): either `informational`, `low`, `medium`, `high`, `critical`. If passed, will overwrite any existing `level` recorded in the rule
|
|
152
|
+
* `--ai_create_attack_flow` (boolean): passing this flag will also prompt the AI model (the same entered for `--ai_provider`, default `false`) to generate an [Attack Flow](https://center-for-threat-informed-defense.github.io/attack-flow/) for the MITRE ATT&CK tags to define the logical order in which they are being described. Note, Sigma currently supports ATT&CK Enterprise only.
|
|
153
|
+
* `--ai_create_attack_navigator_layer` (boolean, default `false`): passing this flag will generate a [MITRE ATT&CK Navigator layer](https://mitre-attack.github.io/attack-navigator/) for MITRE ATT&CK tags. Note, Sigma currently supports ATT&CK Enterprise only. You don't need to pass this if `--ai_create_attack_flow` is set to `true` (as this mode relies on this setting being true)
|
|
156
154
|
|
|
157
155
|
### A note on observable extraction
|
|
158
156
|
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
title: Attack Navigator Enterprise
|
|
2
|
+
id: a18e76d1-f152-4b87-a552-d46f41afd637
|
|
3
|
+
status: test
|
|
4
|
+
description: Build an Attack Enterprise Navigator layer
|
|
5
|
+
references:
|
|
6
|
+
- https://www.dogesec.com
|
|
7
|
+
author: dogesec
|
|
8
|
+
date: 2020-01-01
|
|
9
|
+
modified: 2020-01-02
|
|
10
|
+
tags:
|
|
11
|
+
- tlp.clear
|
|
12
|
+
- attack.t1547
|
|
13
|
+
- attack.t1671
|
|
14
|
+
- attack.t1025
|
|
15
|
+
- attack.command-and-control
|
|
16
|
+
- attack.t1661 # will fail is mobile
|
|
17
|
+
logsource:
|
|
18
|
+
product: okta
|
|
19
|
+
service: okta
|
|
20
|
+
detection:
|
|
21
|
+
selection:
|
|
22
|
+
eventtype:
|
|
23
|
+
- policy.lifecycle.update
|
|
24
|
+
- policy.lifecycle.delete
|
|
25
|
+
condition: selection
|
|
26
|
+
level: low
|
|
27
|
+
license: MIT
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
title: Attack Flow demo
|
|
2
|
+
id: 7894eba6-b0e5-48d9-be52-26bf5a556e45
|
|
3
|
+
status: test
|
|
4
|
+
description: Build an Attack Flow from a Sigma Rule
|
|
5
|
+
references:
|
|
6
|
+
- https://www.dogesec.com
|
|
7
|
+
author: dogesec
|
|
8
|
+
date: 2020-01-01
|
|
9
|
+
modified: 2020-01-02
|
|
10
|
+
tags:
|
|
11
|
+
- tlp.clear
|
|
12
|
+
- attack.t1547
|
|
13
|
+
- attack.t1671
|
|
14
|
+
- attack.t1025
|
|
15
|
+
logsource:
|
|
16
|
+
product: okta
|
|
17
|
+
service: okta
|
|
18
|
+
detection:
|
|
19
|
+
selection:
|
|
20
|
+
eventtype:
|
|
21
|
+
- policy.lifecycle.update
|
|
22
|
+
- policy.lifecycle.delete
|
|
23
|
+
condition: selection
|
|
24
|
+
level: low
|
|
25
|
+
license: MIT
|
|
@@ -445,4 +445,33 @@ python3 txt2detection.py sigma \
|
|
|
445
445
|
--name "Overwrite status" \
|
|
446
446
|
--status unsupported \
|
|
447
447
|
--report_id d2d01afa-dc55-4a80-8d62-15d154450112
|
|
448
|
-
```
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
## Attack Flow
|
|
452
|
+
|
|
453
|
+
```shell
|
|
454
|
+
python3 txt2detection.py sigma \
|
|
455
|
+
--sigma_file tests/files/sigma-rule-attack-flow.yml \
|
|
456
|
+
--name "Create ATT&CK Flow" \
|
|
457
|
+
--report_id 330e2030-1dc2-45e6-be13-9342b102621b \
|
|
458
|
+
--ai_provider openai:gpt-5 \
|
|
459
|
+
--ai_create_attack_flow
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
## Attack Navigator
|
|
463
|
+
|
|
464
|
+
### Enterprise
|
|
465
|
+
|
|
466
|
+
```shell
|
|
467
|
+
python3 txt2detection.py sigma \
|
|
468
|
+
--sigma_file tests/files/sigma-rule-attack-enterprise.yml \
|
|
469
|
+
--name "Attack Navigator Enterprise" \
|
|
470
|
+
--report_id a18e76d1-f152-4b87-a552-d46f41afd637 \
|
|
471
|
+
--ai_provider openai:gpt-5 \
|
|
472
|
+
--ai_create_attack_navigator_layer
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
### Mobile / ICS
|
|
476
|
+
|
|
477
|
+
Not currently supported by Sigma.
|
|
@@ -28,17 +28,6 @@ def dummy_detection():
|
|
|
28
28
|
return detection
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
@pytest.fixture
|
|
32
|
-
def bundler_instance():
|
|
33
|
-
return Bundler(
|
|
34
|
-
name="Test Report",
|
|
35
|
-
identity=None,
|
|
36
|
-
tlp_level="red",
|
|
37
|
-
description="This is a test report.",
|
|
38
|
-
labels=["tlp.red", "test.test-var"],
|
|
39
|
-
created=datetime(2025, 1, 1),
|
|
40
|
-
report_id="74e36652-00f5-4dca-bf10-9f02fc996dcc",
|
|
41
|
-
)
|
|
42
31
|
|
|
43
32
|
|
|
44
33
|
def test_bundler_initialization(bundler_instance):
|
|
@@ -284,3 +273,13 @@ def test_bundle_detections__creates_log_source(dummy_detection, bundler_instance
|
|
|
284
273
|
],
|
|
285
274
|
},
|
|
286
275
|
]
|
|
276
|
+
|
|
277
|
+
def test_get_attack_objects(bundler_instance):
|
|
278
|
+
retval = bundler_instance.get_attack_objects(['T1190', 'T1547'])
|
|
279
|
+
print({r['id'] for r in retval})
|
|
280
|
+
assert {r['id'] for r in retval} == {'attack-pattern--1ecb2399-e8ba-4f6b-8ba7-5c27d49405cf', 'attack-pattern--3f886f2a-874f-4333-b794-aa6075009b1c'}
|
|
281
|
+
|
|
282
|
+
def test_get_cve_objects(bundler_instance):
|
|
283
|
+
cves = ['CVE-2025-1234', 'CVE-2024-1234']
|
|
284
|
+
retval = bundler_instance.get_cve_objects(cves)
|
|
285
|
+
assert {r['name'] for r in retval} == set(cves)
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
from datetime import UTC, datetime
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import logging
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import uuid
|
|
13
|
+
from stix2 import Identity
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from txt2detection import attack_flow, credential_checker
|
|
17
|
+
from txt2detection.ai_extractor.base import BaseAIExtractor
|
|
18
|
+
from txt2detection.models import (
|
|
19
|
+
TAG_PATTERN,
|
|
20
|
+
DetectionContainer,
|
|
21
|
+
Level,
|
|
22
|
+
SigmaRuleDetection,
|
|
23
|
+
)
|
|
24
|
+
from txt2detection.utils import validate_token_count
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def configureLogging():
|
|
28
|
+
# Configure logging
|
|
29
|
+
stream_handler = logging.StreamHandler() # Log to stdout and stderr
|
|
30
|
+
stream_handler.setLevel(logging.INFO)
|
|
31
|
+
logging.basicConfig(
|
|
32
|
+
level=logging.DEBUG, # Set the desired logging level
|
|
33
|
+
format=f"%(asctime)s [%(levelname)s] %(message)s",
|
|
34
|
+
handlers=[stream_handler],
|
|
35
|
+
datefmt="%d-%b-%y %H:%M:%S",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
return logging.root
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
configureLogging()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def setLogFile(logger, file: Path):
|
|
45
|
+
file.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
logger.info(f"Saving log to `{file.absolute()}`")
|
|
47
|
+
handler = logging.FileHandler(file, "w")
|
|
48
|
+
handler.formatter = logging.Formatter(
|
|
49
|
+
fmt="%(levelname)s %(asctime)s - %(message)s", datefmt="%d-%b-%y %H:%M:%S"
|
|
50
|
+
)
|
|
51
|
+
handler.setLevel(logging.DEBUG)
|
|
52
|
+
logger.addHandler(handler)
|
|
53
|
+
logger.info("=====================txt2detection======================")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
from .bundler import Bundler
|
|
57
|
+
import shutil
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
from .utils import STATUSES, as_date, make_identity, valid_licenses, parse_model
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def parse_identity(str):
|
|
64
|
+
return Identity(**json.loads(str))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class Args:
|
|
69
|
+
input_file: str
|
|
70
|
+
input_text: str
|
|
71
|
+
name: str
|
|
72
|
+
tlp_level: str
|
|
73
|
+
labels: list[str]
|
|
74
|
+
created: datetime
|
|
75
|
+
use_identity: Identity
|
|
76
|
+
ai_provider: BaseAIExtractor
|
|
77
|
+
report_id: uuid.UUID
|
|
78
|
+
external_refs: dict[str, str]
|
|
79
|
+
reference_urls: list[str]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def parse_created(value):
|
|
83
|
+
"""Convert the created timestamp to a datetime object."""
|
|
84
|
+
try:
|
|
85
|
+
return datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC)
|
|
86
|
+
except ValueError:
|
|
87
|
+
raise argparse.ArgumentTypeError(
|
|
88
|
+
"Invalid date format. Use YYYY-MM-DDTHH:MM:SS."
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def parse_ref(value):
|
|
93
|
+
m = re.compile(r"(.+?)=(.+)").match(value)
|
|
94
|
+
if not m:
|
|
95
|
+
raise argparse.ArgumentTypeError("must be in format key=value")
|
|
96
|
+
return dict(source_name=m.group(1), external_id=m.group(2))
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def parse_label(label: str):
|
|
100
|
+
if not TAG_PATTERN.match(label):
|
|
101
|
+
raise argparse.ArgumentTypeError(
|
|
102
|
+
"Invalid label format. Must follow sigma tag format {namespace}.{label}"
|
|
103
|
+
)
|
|
104
|
+
namespace, _, _ = label.partition(".")
|
|
105
|
+
if namespace in ["tlp"]:
|
|
106
|
+
raise argparse.ArgumentTypeError(f"Unsupported tag namespace `{namespace}`")
|
|
107
|
+
return label
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def parse_args():
|
|
111
|
+
parser = argparse.ArgumentParser(
|
|
112
|
+
description="Convert text file to detection format."
|
|
113
|
+
)
|
|
114
|
+
mode = parser.add_subparsers(
|
|
115
|
+
title="process-mode", dest="mode", description="mode to use"
|
|
116
|
+
)
|
|
117
|
+
file = mode.add_parser("file", help="process a file input using ai")
|
|
118
|
+
text = mode.add_parser("text", help="process a text argument using ai")
|
|
119
|
+
sigma = mode.add_parser("sigma", help="process a sigma file without ai")
|
|
120
|
+
check_credentials = mode.add_parser(
|
|
121
|
+
"check-credentials",
|
|
122
|
+
help="show status of external services with respect to credentials",
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
for mode_parser in [file, text, sigma]:
|
|
126
|
+
mode_parser.add_argument(
|
|
127
|
+
"--report_id", type=uuid.UUID, help="report_id to use for generated report"
|
|
128
|
+
)
|
|
129
|
+
mode_parser.add_argument(
|
|
130
|
+
"--name",
|
|
131
|
+
required=True,
|
|
132
|
+
help="Name of file, max 72 chars. Will be used in the STIX Report Object created.",
|
|
133
|
+
)
|
|
134
|
+
mode_parser.add_argument(
|
|
135
|
+
"--tlp_level",
|
|
136
|
+
choices=["clear", "green", "amber", "amber_strict", "red"],
|
|
137
|
+
help="Options are clear, green, amber, amber_strict, red. Default is clear if not passed.",
|
|
138
|
+
)
|
|
139
|
+
mode_parser.add_argument(
|
|
140
|
+
"--labels",
|
|
141
|
+
type=parse_label,
|
|
142
|
+
action="extend",
|
|
143
|
+
nargs="+",
|
|
144
|
+
help="Comma-separated list of labels. Case-insensitive (will be converted to lower-case). Allowed a-z, 0-9.",
|
|
145
|
+
)
|
|
146
|
+
mode_parser.add_argument(
|
|
147
|
+
"--created",
|
|
148
|
+
type=parse_created,
|
|
149
|
+
help="Explicitly set created time in format YYYY-MM-DDTHH:MM:SS.sssZ. Default is current time.",
|
|
150
|
+
)
|
|
151
|
+
mode_parser.add_argument(
|
|
152
|
+
"--use_identity",
|
|
153
|
+
type=parse_identity,
|
|
154
|
+
help="Pass a full STIX 2.1 identity object (properly escaped). Validated by the STIX2 library. Default is SIEM Rules identity.",
|
|
155
|
+
)
|
|
156
|
+
mode_parser.add_argument(
|
|
157
|
+
"--ai_provider",
|
|
158
|
+
required=False,
|
|
159
|
+
type=parse_model,
|
|
160
|
+
help="(required): defines the `provider:model` to be used. Select one option.",
|
|
161
|
+
metavar="provider[:model]",
|
|
162
|
+
)
|
|
163
|
+
mode_parser.add_argument(
|
|
164
|
+
"--external_refs",
|
|
165
|
+
type=parse_ref,
|
|
166
|
+
help="pass additional `external_references` entry (or entries) to the report object created. e.g --external_ref author=dogesec link=https://dkjjadhdaj.net",
|
|
167
|
+
default=[],
|
|
168
|
+
metavar="{source_name}={external_id}",
|
|
169
|
+
action="extend",
|
|
170
|
+
nargs="+",
|
|
171
|
+
)
|
|
172
|
+
mode_parser.add_argument(
|
|
173
|
+
"--reference_urls",
|
|
174
|
+
help="pass additional `external_references` url entry (or entries) to the report object created.",
|
|
175
|
+
default=[],
|
|
176
|
+
metavar="{url}",
|
|
177
|
+
action="extend",
|
|
178
|
+
nargs="+",
|
|
179
|
+
)
|
|
180
|
+
mode_parser.add_argument(
|
|
181
|
+
"--license",
|
|
182
|
+
help="Valid SPDX license for the rule",
|
|
183
|
+
default=None,
|
|
184
|
+
metavar="[LICENSE]",
|
|
185
|
+
choices=valid_licenses(),
|
|
186
|
+
)
|
|
187
|
+
mode_parser.add_argument(
|
|
188
|
+
"--ai_create_attack_navigator_layer",
|
|
189
|
+
help="Create navigator layer",
|
|
190
|
+
action="store_true",
|
|
191
|
+
default=False,
|
|
192
|
+
)
|
|
193
|
+
mode_parser.add_argument(
|
|
194
|
+
"--ai_create_attack_flow",
|
|
195
|
+
help="Create attack flow",
|
|
196
|
+
action="store_true",
|
|
197
|
+
default=False,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
file.add_argument(
|
|
201
|
+
"--input_file",
|
|
202
|
+
help="The file to be converted. Must be .txt",
|
|
203
|
+
type=lambda x: Path(x).read_text(),
|
|
204
|
+
)
|
|
205
|
+
text.add_argument("--input_text", help="The text to be converted")
|
|
206
|
+
sigma.add_argument(
|
|
207
|
+
"--sigma_file",
|
|
208
|
+
help="The sigma file to be converted. Must be .yml",
|
|
209
|
+
type=lambda x: Path(x).read_text(),
|
|
210
|
+
)
|
|
211
|
+
sigma.add_argument(
|
|
212
|
+
"--status",
|
|
213
|
+
help="If passed, will overwrite any existing `status` recorded in the rule",
|
|
214
|
+
choices=STATUSES,
|
|
215
|
+
)
|
|
216
|
+
sigma.add_argument(
|
|
217
|
+
"--level",
|
|
218
|
+
help="If passed, will overwrite any existing `level` recorded in the rule",
|
|
219
|
+
choices=Level._member_names_,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
args: Args = parser.parse_args()
|
|
223
|
+
if args.mode == "check-credentials":
|
|
224
|
+
statuses = credential_checker.check_statuses(test_llms=True)
|
|
225
|
+
credential_checker.format_statuses(statuses)
|
|
226
|
+
sys.exit(0)
|
|
227
|
+
|
|
228
|
+
if args.mode != "sigma":
|
|
229
|
+
assert args.ai_provider, "--ai_provider is required in file or txt mode"
|
|
230
|
+
|
|
231
|
+
if args.ai_create_attack_navigator_layer or args.ai_create_attack_flow:
|
|
232
|
+
assert (
|
|
233
|
+
args.ai_provider
|
|
234
|
+
), "--ai_provider is required when --ai_create_attack_navigator_layer/--ai_create_attack_flow is passed"
|
|
235
|
+
|
|
236
|
+
if args.mode == "file":
|
|
237
|
+
args.input_text = args.input_file
|
|
238
|
+
|
|
239
|
+
args.input_text = getattr(args, "input_text", "")
|
|
240
|
+
if not args.report_id:
|
|
241
|
+
args.report_id = Bundler.generate_report_id(
|
|
242
|
+
args.use_identity.id if args.use_identity else None, args.created, args.name
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
return args
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def run_txt2detection(
|
|
249
|
+
name,
|
|
250
|
+
identity,
|
|
251
|
+
tlp_level,
|
|
252
|
+
input_text: str,
|
|
253
|
+
labels: list[str],
|
|
254
|
+
report_id: str | uuid.UUID,
|
|
255
|
+
ai_provider: BaseAIExtractor,
|
|
256
|
+
ai_create_attack_flow=False,
|
|
257
|
+
ai_create_attack_navigator_layer=False,
|
|
258
|
+
**kwargs,
|
|
259
|
+
) -> Bundler:
|
|
260
|
+
if (
|
|
261
|
+
kwargs.get("sigma_file") != "sigma_file"
|
|
262
|
+
or ai_create_attack_flow
|
|
263
|
+
or ai_create_attack_navigator_layer
|
|
264
|
+
):
|
|
265
|
+
validate_token_count(
|
|
266
|
+
int(os.getenv("INPUT_TOKEN_LIMIT", 0)), input_text, ai_provider
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
if sigma := kwargs.get("sigma_file"):
|
|
270
|
+
detection = get_sigma_detections(sigma)
|
|
271
|
+
if not identity and detection.author:
|
|
272
|
+
identity = make_identity(detection.author)
|
|
273
|
+
kwargs.update(
|
|
274
|
+
reference_urls=kwargs.setdefault("reference_urls", [])
|
|
275
|
+
+ detection.references
|
|
276
|
+
)
|
|
277
|
+
if not kwargs.get("created"):
|
|
278
|
+
# only consider rule.date and rule.modified if user does not pass --created
|
|
279
|
+
kwargs.update(
|
|
280
|
+
created=detection.date,
|
|
281
|
+
modified=detection.modified,
|
|
282
|
+
)
|
|
283
|
+
detection.level = kwargs.get("level", detection.level)
|
|
284
|
+
detection.status = kwargs.get("status", detection.status)
|
|
285
|
+
detection.date = as_date(kwargs.get("created"))
|
|
286
|
+
detection.modified = as_date(kwargs.get("modified"))
|
|
287
|
+
detection.references = kwargs["reference_urls"]
|
|
288
|
+
detection.detection_id = str(report_id).removeprefix("report--")
|
|
289
|
+
bundler = Bundler(
|
|
290
|
+
name or detection.title,
|
|
291
|
+
identity,
|
|
292
|
+
tlp_level or detection.tlp_level or "clear",
|
|
293
|
+
detection.description,
|
|
294
|
+
(labels or []) + detection.tags,
|
|
295
|
+
report_id=report_id,
|
|
296
|
+
**kwargs,
|
|
297
|
+
)
|
|
298
|
+
detections = DetectionContainer(success=True, detections=[])
|
|
299
|
+
detections.detections.append(detection)
|
|
300
|
+
else:
|
|
301
|
+
bundler = Bundler(
|
|
302
|
+
name, identity, tlp_level, input_text, labels, report_id=report_id, **kwargs
|
|
303
|
+
)
|
|
304
|
+
detections = ai_provider.get_detections(input_text)
|
|
305
|
+
bundler.bundle_detections(detections)
|
|
306
|
+
|
|
307
|
+
if ai_create_attack_flow or ai_create_attack_navigator_layer:
|
|
308
|
+
bundler.data.attack_flow, bundler.data.navigator_layer = (
|
|
309
|
+
attack_flow.extract_attack_flow_and_navigator(
|
|
310
|
+
bundler,
|
|
311
|
+
bundler.report.description,
|
|
312
|
+
ai_create_attack_flow,
|
|
313
|
+
ai_create_attack_navigator_layer,
|
|
314
|
+
ai_provider,
|
|
315
|
+
)
|
|
316
|
+
)
|
|
317
|
+
return bundler
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def get_sigma_detections(sigma: str) -> SigmaRuleDetection:
|
|
321
|
+
obj = yaml.safe_load(io.StringIO(sigma))
|
|
322
|
+
return SigmaRuleDetection.model_validate(obj)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def main(args: Args):
|
|
326
|
+
|
|
327
|
+
setLogFile(logging.root, Path(f"logs/log-{args.report_id}.log"))
|
|
328
|
+
logging.info(f"starting argument: {json.dumps(sys.argv[1:])}")
|
|
329
|
+
kwargs = args.__dict__
|
|
330
|
+
kwargs["identity"] = args.use_identity
|
|
331
|
+
bundler = run_txt2detection(**kwargs)
|
|
332
|
+
|
|
333
|
+
output_dir = Path("./output") / str(bundler.bundle.id)
|
|
334
|
+
shutil.rmtree(output_dir, ignore_errors=True)
|
|
335
|
+
rules_dir = output_dir / "rules"
|
|
336
|
+
rules_dir.mkdir(exist_ok=True, parents=True)
|
|
337
|
+
|
|
338
|
+
output_path = output_dir / "bundle.json"
|
|
339
|
+
data_path = output_dir / f"data.json"
|
|
340
|
+
output_path.write_text(bundler.to_json())
|
|
341
|
+
data_path.write_text(bundler.data.model_dump_json(indent=4))
|
|
342
|
+
for obj in bundler.bundle["objects"]:
|
|
343
|
+
if obj["type"] != "indicator" or obj["pattern_type"] != "sigma":
|
|
344
|
+
continue
|
|
345
|
+
name = obj["id"].replace("indicator", "rule") + ".yml"
|
|
346
|
+
(rules_dir / name).write_text(obj["pattern"])
|
|
347
|
+
logging.info(f"Writing bundle output to `{output_path}`")
|