pySigma-backend-sqlite 0.2.0__tar.gz → 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. pysigma_backend_sqlite-1.0.0/PKG-INFO +166 -0
  2. pysigma_backend_sqlite-1.0.0/README.md +145 -0
  3. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.0.0}/pyproject.toml +20 -21
  4. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.0.0}/sigma/backends/sqlite/sqlite.py +675 -431
  5. pysigma_backend_sqlite-0.2.0/PKG-INFO +0 -99
  6. pysigma_backend_sqlite-0.2.0/README.md +0 -79
  7. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.0.0}/LICENSE +0 -0
  8. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.0.0}/sigma/backends/sqlite/__init__.py +0 -0
pysigma_backend_sqlite-1.0.0/PKG-INFO ADDED
@@ -0,0 +1,166 @@
1
+ Metadata-Version: 2.4
2
+ Name: pySigma-backend-sqlite
3
+ Version: 1.0.0
4
+ Summary: pySigma sqlite backend
5
+ License-Expression: LGPL-3.0-only
6
+ License-File: LICENSE
7
+ Author: wagga
8
+ Requires-Python: >=3.10,<4.0
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Requires-Dist: pysigma (==1.0.2)
16
+ Requires-Dist: pysigma-pipeline-sysmon (>=2.0.0,<3.0.0)
17
+ Requires-Dist: pysigma-pipeline-windows (>=2.0.0,<3.0.0)
18
+ Project-URL: Repository, https://github.com/SigmaHQ/pySigma-backend-sqlite
19
+ Description-Content-Type: text/markdown
20
+
21
+ ![Tests](https://github.com/wagga40/pySigma-backend-sqlite/actions/workflows/test.yml/badge.svg)
22
+ ![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/wagga40/2ec45ded898fa11f2c42bcb9d2b163cf/raw/test.json)
23
+ ![Status](https://img.shields.io/badge/Status-pre--release-orange)
24
+
25
+ # pySigma SQLite Backend
26
+
27
+ This is the SQLite backend for pySigma. It provides the package `sigma.backends.sqlite` with the `sqliteBackend` class.
28
+
29
+ This backend also aims to be compatible with [Zircolite](https://github.com/wagga40/Zircolite) which uses **pure SQLite queries** to perform SIGMA-based detection on EVTX, Auditd, Sysmon for linux, XML or JSONL/NDJSON Logs.
30
+
31
+ It supports the following output formats:
32
+
33
+ * **default**: plain SQLite queries
34
+ * **zircolite** : SQLite queries in JSON format for Zircolite
35
+
36
+ This backend is currently maintained by:
37
+
38
+ * [wagga](https://github.com/wagga40/)
39
+
40
+ ## Supported Features
41
+
42
+ ### Sigma Modifiers
43
+
44
+ | Modifier | Description | SQLite Implementation |
45
+ |----------|-------------|----------------------|
46
+ | `contains` | Substring matching | `LIKE '%value%'` |
47
+ | `startswith` | Prefix matching | `LIKE 'value%'` |
48
+ | `endswith` | Suffix matching | `LIKE '%value'` |
49
+ | `all` | All values must match | Multiple `AND` conditions |
50
+ | `re` | Regular expressions | `REGEXP` |
51
+ | `cidr` | CIDR network matching | Expanded to `LIKE` patterns |
52
+ | `cased` | Case-sensitive matching | `GLOB` |
53
+ | `fieldref` | Compare two fields | `field1=field2` or with `LIKE` for startswith/endswith/contains |
54
+ | `exists` | Field existence check | `field = field` |
55
+ | `gt`, `gte`, `lt`, `lte` | Numeric comparisons | `>`, `>=`, `<`, `<=` |
56
+ | `hour`, `minute`, `day`, `week`, `month`, `year` | Timestamp part extraction | `strftime()` |
57
+
58
+ ### Correlation Rules
59
+
60
+ The backend supports Sigma correlation rules with the following types:
61
+
62
+ | Correlation Type | Description |
63
+ |-----------------|-------------|
64
+ | `event_count` | Count events matching conditions |
65
+ | `value_count` | Count distinct field values |
66
+ | `temporal` | Events from multiple rules occurring within a timespan |
67
+ | `temporal_ordered` | Events occurring in a specific order within a timespan |
68
+ | `value_sum` | Sum of field values |
69
+ | `value_avg` | Average of field values |
70
+
71
+ Correlation rules support `group-by` for grouping results and `timespan` for temporal constraints.
72
+
73
+ #### SQLite Requirements for Correlation
74
+
75
+ For correlation rules to work properly, your SQLite database must meet the following requirements:
76
+
77
+ | Requirement | Description |
78
+ |-------------|-------------|
79
+ | **Timestamp field** | Required for temporal correlations. Must be in a format compatible with SQLite's `julianday()` function (ISO8601, Julian day number, or Unix timestamp) |
80
+
81
+ **Configurable Parameters:**
82
+
83
+ The backend provides configurable parameters for correlation queries:
84
+
85
+ | Parameter | Default | Description |
86
+ |-----------|---------|-------------|
87
+ | `timestamp_field` | `timestamp` | Field name containing the event timestamp |
88
+
89
+ **Example usage with custom parameters:**
90
+
91
+ ```python
92
+ backend = sqliteBackend(correlation_methods=["default"])
93
+ backend.timestamp_field = "event_time"
94
+ ```
95
+
96
+ **Notes:**
97
+ - The timestamp field is used with `julianday()` for time difference calculations in temporal correlations
98
+ - For multi-rule correlations (`temporal`, `temporal_ordered`), the backend automatically adds a `sigma_rule_id` column to identify which rule matched each event
99
+ - Timespan values are converted to seconds internally for comparison
100
+
101
+ ### Other Features
102
+
103
+ * **NULL value handling**: `field: null` → `field IS NULL`
104
+ * **Boolean values**: `true`/`false` support
105
+ * **Field name quoting**: Special characters in field names are quoted with backticks
106
+ * **Wildcard escaping**: Proper escaping of `%` and `_` characters in values
107
+
108
+ ## Known issues/limitations
109
+
110
+ * Full text search support will need some work and is not a priority since it needs virtual tables on SQLite side
111
+
112
+ # Quick Start
113
+
114
+ ## Example script (default output) with sysmon pipeline
115
+
116
+ ### Add pipelines
117
+
118
+ ```shell
119
+ poetry add pysigma-pipeline-sysmon
120
+ poetry add pysigma-pipeline-windows
121
+ ```
122
+
123
+ ### Convert a rule
124
+
125
+ ```python
126
+ from sigma.collection import SigmaCollection
127
+ from sigma.backends.sqlite import sqlite
128
+ from sigma.pipelines.sysmon import sysmon_pipeline
129
+ from sigma.pipelines.windows import windows_logsource_pipeline
130
+
131
+ from sigma.processing.resolver import ProcessingPipelineResolver
132
+
133
+ # Create the pipeline resolver
134
+ piperesolver = ProcessingPipelineResolver()
135
+ # Add pipelines
136
+ piperesolver.add_pipeline_class(sysmon_pipeline()) # Syssmon
137
+ piperesolver.add_pipeline_class(windows_logsource_pipeline()) # Windows
138
+ # Create a combined pipeline
139
+ combined_pipeline = piperesolver.resolve(piperesolver.pipelines)
140
+ # Instantiate backend using the combined pipeline
141
+ sqlite_backend = sqlite.sqliteBackend(combined_pipeline)
142
+
143
+ rule = SigmaCollection.from_yaml(
144
+ r"""
145
+ title: Test
146
+ status: test
147
+ logsource:
148
+ category: test_category
149
+ product: test_product
150
+ detection:
151
+ sel:
152
+ fieldA: valueA
153
+ fieldB: valueB
154
+ condition: sel
155
+ """)
156
+
157
+ print(sqlite_backend.convert(rule)[0])
158
+
159
+ ```
160
+
161
+ ## Running
162
+
163
+ ```shell
164
+ poetry run python3 example.py
165
+ ```
166
+
pysigma_backend_sqlite-1.0.0/README.md ADDED
@@ -0,0 +1,145 @@
1
+ ![Tests](https://github.com/wagga40/pySigma-backend-sqlite/actions/workflows/test.yml/badge.svg)
2
+ ![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/wagga40/2ec45ded898fa11f2c42bcb9d2b163cf/raw/test.json)
3
+ ![Status](https://img.shields.io/badge/Status-pre--release-orange)
4
+
5
+ # pySigma SQLite Backend
6
+
7
+ This is the SQLite backend for pySigma. It provides the package `sigma.backends.sqlite` with the `sqliteBackend` class.
8
+
9
+ This backend also aims to be compatible with [Zircolite](https://github.com/wagga40/Zircolite) which uses **pure SQLite queries** to perform SIGMA-based detection on EVTX, Auditd, Sysmon for linux, XML or JSONL/NDJSON Logs.
10
+
11
+ It supports the following output formats:
12
+
13
+ * **default**: plain SQLite queries
14
+ * **zircolite** : SQLite queries in JSON format for Zircolite
15
+
16
+ This backend is currently maintained by:
17
+
18
+ * [wagga](https://github.com/wagga40/)
19
+
20
+ ## Supported Features
21
+
22
+ ### Sigma Modifiers
23
+
24
+ | Modifier | Description | SQLite Implementation |
25
+ |----------|-------------|----------------------|
26
+ | `contains` | Substring matching | `LIKE '%value%'` |
27
+ | `startswith` | Prefix matching | `LIKE 'value%'` |
28
+ | `endswith` | Suffix matching | `LIKE '%value'` |
29
+ | `all` | All values must match | Multiple `AND` conditions |
30
+ | `re` | Regular expressions | `REGEXP` |
31
+ | `cidr` | CIDR network matching | Expanded to `LIKE` patterns |
32
+ | `cased` | Case-sensitive matching | `GLOB` |
33
+ | `fieldref` | Compare two fields | `field1=field2` or with `LIKE` for startswith/endswith/contains |
34
+ | `exists` | Field existence check | `field = field` |
35
+ | `gt`, `gte`, `lt`, `lte` | Numeric comparisons | `>`, `>=`, `<`, `<=` |
36
+ | `hour`, `minute`, `day`, `week`, `month`, `year` | Timestamp part extraction | `strftime()` |
37
+
38
+ ### Correlation Rules
39
+
40
+ The backend supports Sigma correlation rules with the following types:
41
+
42
+ | Correlation Type | Description |
43
+ |-----------------|-------------|
44
+ | `event_count` | Count events matching conditions |
45
+ | `value_count` | Count distinct field values |
46
+ | `temporal` | Events from multiple rules occurring within a timespan |
47
+ | `temporal_ordered` | Events occurring in a specific order within a timespan |
48
+ | `value_sum` | Sum of field values |
49
+ | `value_avg` | Average of field values |
50
+
51
+ Correlation rules support `group-by` for grouping results and `timespan` for temporal constraints.
52
+
53
+ #### SQLite Requirements for Correlation
54
+
55
+ For correlation rules to work properly, your SQLite database must meet the following requirements:
56
+
57
+ | Requirement | Description |
58
+ |-------------|-------------|
59
+ | **Timestamp field** | Required for temporal correlations. Must be in a format compatible with SQLite's `julianday()` function (ISO8601, Julian day number, or Unix timestamp) |
60
+
61
+ **Configurable Parameters:**
62
+
63
+ The backend provides configurable parameters for correlation queries:
64
+
65
+ | Parameter | Default | Description |
66
+ |-----------|---------|-------------|
67
+ | `timestamp_field` | `timestamp` | Field name containing the event timestamp |
68
+
69
+ **Example usage with custom parameters:**
70
+
71
+ ```python
72
+ backend = sqliteBackend(correlation_methods=["default"])
73
+ backend.timestamp_field = "event_time"
74
+ ```
75
+
76
+ **Notes:**
77
+ - The timestamp field is used with `julianday()` for time difference calculations in temporal correlations
78
+ - For multi-rule correlations (`temporal`, `temporal_ordered`), the backend automatically adds a `sigma_rule_id` column to identify which rule matched each event
79
+ - Timespan values are converted to seconds internally for comparison
80
+
81
+ ### Other Features
82
+
83
+ * **NULL value handling**: `field: null` → `field IS NULL`
84
+ * **Boolean values**: `true`/`false` support
85
+ * **Field name quoting**: Special characters in field names are quoted with backticks
86
+ * **Wildcard escaping**: Proper escaping of `%` and `_` characters in values
87
+
88
+ ## Known issues/limitations
89
+
90
+ * Full text search support will need some work and is not a priority since it needs virtual tables on SQLite side
91
+
92
+ # Quick Start
93
+
94
+ ## Example script (default output) with sysmon pipeline
95
+
96
+ ### Add pipelines
97
+
98
+ ```shell
99
+ poetry add pysigma-pipeline-sysmon
100
+ poetry add pysigma-pipeline-windows
101
+ ```
102
+
103
+ ### Convert a rule
104
+
105
+ ```python
106
+ from sigma.collection import SigmaCollection
107
+ from sigma.backends.sqlite import sqlite
108
+ from sigma.pipelines.sysmon import sysmon_pipeline
109
+ from sigma.pipelines.windows import windows_logsource_pipeline
110
+
111
+ from sigma.processing.resolver import ProcessingPipelineResolver
112
+
113
+ # Create the pipeline resolver
114
+ piperesolver = ProcessingPipelineResolver()
115
+ # Add pipelines
116
+ piperesolver.add_pipeline_class(sysmon_pipeline()) # Syssmon
117
+ piperesolver.add_pipeline_class(windows_logsource_pipeline()) # Windows
118
+ # Create a combined pipeline
119
+ combined_pipeline = piperesolver.resolve(piperesolver.pipelines)
120
+ # Instantiate backend using the combined pipeline
121
+ sqlite_backend = sqlite.sqliteBackend(combined_pipeline)
122
+
123
+ rule = SigmaCollection.from_yaml(
124
+ r"""
125
+ title: Test
126
+ status: test
127
+ logsource:
128
+ category: test_category
129
+ product: test_product
130
+ detection:
131
+ sel:
132
+ fieldA: valueA
133
+ fieldB: valueB
134
+ condition: sel
135
+ """)
136
+
137
+ print(sqlite_backend.convert(rule)[0])
138
+
139
+ ```
140
+
141
+ ## Running
142
+
143
+ ```shell
144
+ poetry run python3 example.py
145
+ ```
{pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.0.0}/pyproject.toml RENAMED
@@ -1,18 +1,23 @@
1
- [tool.poetry]
1
+ [project]
2
2
  name = "pySigma-backend-sqlite"
3
- version = "0.2.0"
3
+ version = "1.0.0"
4
4
  description = "pySigma sqlite backend"
5
5
  readme = "README.md"
6
- authors = ["wagga"]
6
+ authors = [
7
+ { name = "wagga" }
8
+ ]
7
9
  license = "LGPL-3.0-only"
8
- repository = "https://github.com/SigmaHQ/pySigma-backend-sqlite"
10
+
11
+ [tool.poetry]
9
12
  packages = [
10
13
  { include = "sigma" }
11
14
  ]
12
15
 
13
16
  [tool.poetry.dependencies]
14
- python = "^3.9"
15
- pysigma = "^0.11.17"
17
+ python = "^3.10"
18
+ pysigma = "1.0.2"
19
+ pysigma-pipeline-windows = "^2.0.0"
20
+ pysigma-pipeline-sysmon = "^2.0.0"
16
21
 
17
22
  [tool.poetry.group.dev.dependencies]
18
23
  black = "^24.1"
@@ -21,6 +26,12 @@ pytest-cov = "^4.1"
21
26
  coverage = "^7.4"
22
27
 
23
28
 
29
+ [tool.pytest.ini_options]
30
+ filterwarnings = [
31
+ # Suppress pyparsing deprecation warning from pySigma library
32
+ "ignore:'parseString' deprecated:DeprecationWarning",
33
+ ]
34
+
24
35
  [tool.pdm]
25
36
  [tool.pdm.dev-dependencies]
26
37
  dev = []
@@ -31,24 +42,12 @@ includes = ["sigma"]
31
42
  requires = ["poetry-core>=1.8.1"]
32
43
  build-backend = "poetry.core.masonry.api"
33
44
 
34
- [project]
35
- authors = [
36
- {name = "wagga", email = "wagga"},
37
- ]
38
- license = {text = "LGPL-3.0-only"}
39
- requires-python = ">=3.9,<4.0"
40
- dependencies = [
41
- "pysigma>=0.11.17",
42
- ]
43
-
44
- name = "pySigma-backend-sqlite"
45
- version = "0.2.0"
46
- description = "pySigma sqlite backend"
47
-
48
45
  [project.urls]
49
- repository = "https://github.com/SigmaHQ/pySigma-backend-sqlite"
46
+ Repository = "https://github.com/SigmaHQ/pySigma-backend-sqlite"
50
47
 
51
48
  [project.group.dev.dependencies]
52
49
  pytest = "^8.0"
53
50
  pytest-cov = "^4.1"
54
51
  coverage = "^7.4"
52
+
53
+