pySigma-backend-sqlite 0.2.0__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. pysigma_backend_sqlite-1.1.0/PKG-INFO +164 -0
  2. pysigma_backend_sqlite-1.1.0/README.md +145 -0
  3. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.1.0}/pyproject.toml +19 -21
  4. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.1.0}/sigma/backends/sqlite/sqlite.py +675 -431
  5. pysigma_backend_sqlite-0.2.0/PKG-INFO +0 -99
  6. pysigma_backend_sqlite-0.2.0/README.md +0 -79
  7. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.1.0}/LICENSE +0 -0
  8. {pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.1.0}/sigma/backends/sqlite/__init__.py +0 -0
pysigma_backend_sqlite-1.1.0/PKG-INFO ADDED
@@ -0,0 +1,164 @@
1
+ Metadata-Version: 2.4
2
+ Name: pySigma-backend-sqlite
3
+ Version: 1.1.0
4
+ Summary: pySigma sqlite backend
5
+ License-Expression: LGPL-3.0-only
6
+ License-File: LICENSE
7
+ Author: wagga
8
+ Requires-Python: >=3.10,<4.0
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Requires-Dist: pysigma (>=1.0.2,<1.1.0)
16
+ Project-URL: Repository, https://github.com/SigmaHQ/pySigma-backend-sqlite
17
+ Description-Content-Type: text/markdown
18
+
19
+ ![Tests](https://github.com/wagga40/pySigma-backend-sqlite/actions/workflows/test.yml/badge.svg)
20
+ ![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/wagga40/2ec45ded898fa11f2c42bcb9d2b163cf/raw/test.json)
21
+ ![Status](https://img.shields.io/badge/Status-pre--release-orange)
22
+
23
+ # pySigma SQLite Backend
24
+
25
+ This is the SQLite backend for pySigma. It provides the package `sigma.backends.sqlite` with the `sqliteBackend` class.
26
+
27
+ This backend also aims to be compatible with [Zircolite](https://github.com/wagga40/Zircolite) which uses **pure SQLite queries** to perform SIGMA-based detection on EVTX, Auditd, Sysmon for linux, XML or JSONL/NDJSON Logs.
28
+
29
+ It supports the following output formats:
30
+
31
+ * **default**: plain SQLite queries
32
+ * **zircolite** : SQLite queries in JSON format for Zircolite
33
+
34
+ This backend is currently maintained by:
35
+
36
+ * [wagga](https://github.com/wagga40/)
37
+
38
+ ## Supported Features
39
+
40
+ ### Sigma Modifiers
41
+
42
+ | Modifier | Description | SQLite Implementation |
43
+ |----------|-------------|----------------------|
44
+ | `contains` | Substring matching | `LIKE '%value%'` |
45
+ | `startswith` | Prefix matching | `LIKE 'value%'` |
46
+ | `endswith` | Suffix matching | `LIKE '%value'` |
47
+ | `all` | All values must match | Multiple `AND` conditions |
48
+ | `re` | Regular expressions | `REGEXP` |
49
+ | `cidr` | CIDR network matching | Expanded to `LIKE` patterns |
50
+ | `cased` | Case-sensitive matching | `GLOB` |
51
+ | `fieldref` | Compare two fields | `field1=field2` or with `LIKE` for startswith/endswith/contains |
52
+ | `exists` | Field existence check | `field = field` |
53
+ | `gt`, `gte`, `lt`, `lte` | Numeric comparisons | `>`, `>=`, `<`, `<=` |
54
+ | `hour`, `minute`, `day`, `week`, `month`, `year` | Timestamp part extraction | `strftime()` |
55
+
56
+ ### Correlation Rules
57
+
58
+ The backend supports Sigma correlation rules with the following types:
59
+
60
+ | Correlation Type | Description |
61
+ |-----------------|-------------|
62
+ | `event_count` | Count events matching conditions |
63
+ | `value_count` | Count distinct field values |
64
+ | `temporal` | Events from multiple rules occurring within a timespan |
65
+ | `temporal_ordered` | Events occurring in a specific order within a timespan |
66
+ | `value_sum` | Sum of field values |
67
+ | `value_avg` | Average of field values |
68
+
69
+ Correlation rules support `group-by` for grouping results and `timespan` for temporal constraints.
70
+
71
+ #### SQLite Requirements for Correlation
72
+
73
+ For correlation rules to work properly, your SQLite database must meet the following requirements:
74
+
75
+ | Requirement | Description |
76
+ |-------------|-------------|
77
+ | **Timestamp field** | Required for temporal correlations. Must be in a format compatible with SQLite's `julianday()` function (ISO8601, Julian day number, or Unix timestamp) |
78
+
79
+ **Configurable Parameters:**
80
+
81
+ The backend provides configurable parameters for correlation queries:
82
+
83
+ | Parameter | Default | Description |
84
+ |-----------|---------|-------------|
85
+ | `timestamp_field` | `timestamp` | Field name containing the event timestamp |
86
+
87
+ **Example usage with custom parameters:**
88
+
89
+ ```python
90
+ backend = sqliteBackend(correlation_methods=["default"])
91
+ backend.timestamp_field = "event_time"
92
+ ```
93
+
94
+ **Notes:**
95
+ - The timestamp field is used with `julianday()` for time difference calculations in temporal correlations
96
+ - For multi-rule correlations (`temporal`, `temporal_ordered`), the backend automatically adds a `sigma_rule_id` column to identify which rule matched each event
97
+ - Timespan values are converted to seconds internally for comparison
98
+
99
+ ### Other Features
100
+
101
+ * **NULL value handling**: `field: null` → `field IS NULL`
102
+ * **Boolean values**: `true`/`false` support
103
+ * **Field name quoting**: Special characters in field names are quoted with backticks
104
+ * **Wildcard escaping**: Proper escaping of `%` and `_` characters in values
105
+
106
+ ## Known issues/limitations
107
+
108
+ * Full text search support will need some work and is not a priority since it needs virtual tables on SQLite side
109
+
110
+ # Quick Start
111
+
112
+ ## Example script (default output) with sysmon pipeline
113
+
114
+ ### Add pipelines
115
+
116
+ ```shell
117
+ poetry add pysigma-pipeline-sysmon
118
+ poetry add pysigma-pipeline-windows
119
+ ```
120
+
121
+ ### Convert a rule
122
+
123
+ ```python
124
+ from sigma.collection import SigmaCollection
125
+ from sigma.backends.sqlite import sqlite
126
+ from sigma.pipelines.sysmon import sysmon_pipeline
127
+ from sigma.pipelines.windows import windows_logsource_pipeline
128
+
129
+ from sigma.processing.resolver import ProcessingPipelineResolver
130
+
131
+ # Create the pipeline resolver
132
+ piperesolver = ProcessingPipelineResolver()
133
+ # Add pipelines
134
+ piperesolver.add_pipeline_class(sysmon_pipeline()) # Syssmon
135
+ piperesolver.add_pipeline_class(windows_logsource_pipeline()) # Windows
136
+ # Create a combined pipeline
137
+ combined_pipeline = piperesolver.resolve(piperesolver.pipelines)
138
+ # Instantiate backend using the combined pipeline
139
+ sqlite_backend = sqlite.sqliteBackend(combined_pipeline)
140
+
141
+ rule = SigmaCollection.from_yaml(
142
+ r"""
143
+ title: Test
144
+ status: test
145
+ logsource:
146
+ category: test_category
147
+ product: test_product
148
+ detection:
149
+ sel:
150
+ fieldA: valueA
151
+ fieldB: valueB
152
+ condition: sel
153
+ """)
154
+
155
+ print(sqlite_backend.convert(rule)[0])
156
+
157
+ ```
158
+
159
+ ## Running
160
+
161
+ ```shell
162
+ poetry run python3 example.py
163
+ ```
164
+
pysigma_backend_sqlite-1.1.0/README.md ADDED
@@ -0,0 +1,145 @@
1
+ ![Tests](https://github.com/wagga40/pySigma-backend-sqlite/actions/workflows/test.yml/badge.svg)
2
+ ![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/wagga40/2ec45ded898fa11f2c42bcb9d2b163cf/raw/test.json)
3
+ ![Status](https://img.shields.io/badge/Status-pre--release-orange)
4
+
5
+ # pySigma SQLite Backend
6
+
7
+ This is the SQLite backend for pySigma. It provides the package `sigma.backends.sqlite` with the `sqliteBackend` class.
8
+
9
+ This backend also aims to be compatible with [Zircolite](https://github.com/wagga40/Zircolite) which uses **pure SQLite queries** to perform SIGMA-based detection on EVTX, Auditd, Sysmon for linux, XML or JSONL/NDJSON Logs.
10
+
11
+ It supports the following output formats:
12
+
13
+ * **default**: plain SQLite queries
14
+ * **zircolite** : SQLite queries in JSON format for Zircolite
15
+
16
+ This backend is currently maintained by:
17
+
18
+ * [wagga](https://github.com/wagga40/)
19
+
20
+ ## Supported Features
21
+
22
+ ### Sigma Modifiers
23
+
24
+ | Modifier | Description | SQLite Implementation |
25
+ |----------|-------------|----------------------|
26
+ | `contains` | Substring matching | `LIKE '%value%'` |
27
+ | `startswith` | Prefix matching | `LIKE 'value%'` |
28
+ | `endswith` | Suffix matching | `LIKE '%value'` |
29
+ | `all` | All values must match | Multiple `AND` conditions |
30
+ | `re` | Regular expressions | `REGEXP` |
31
+ | `cidr` | CIDR network matching | Expanded to `LIKE` patterns |
32
+ | `cased` | Case-sensitive matching | `GLOB` |
33
+ | `fieldref` | Compare two fields | `field1=field2` or with `LIKE` for startswith/endswith/contains |
34
+ | `exists` | Field existence check | `field = field` |
35
+ | `gt`, `gte`, `lt`, `lte` | Numeric comparisons | `>`, `>=`, `<`, `<=` |
36
+ | `hour`, `minute`, `day`, `week`, `month`, `year` | Timestamp part extraction | `strftime()` |
37
+
38
+ ### Correlation Rules
39
+
40
+ The backend supports Sigma correlation rules with the following types:
41
+
42
+ | Correlation Type | Description |
43
+ |-----------------|-------------|
44
+ | `event_count` | Count events matching conditions |
45
+ | `value_count` | Count distinct field values |
46
+ | `temporal` | Events from multiple rules occurring within a timespan |
47
+ | `temporal_ordered` | Events occurring in a specific order within a timespan |
48
+ | `value_sum` | Sum of field values |
49
+ | `value_avg` | Average of field values |
50
+
51
+ Correlation rules support `group-by` for grouping results and `timespan` for temporal constraints.
52
+
53
+ #### SQLite Requirements for Correlation
54
+
55
+ For correlation rules to work properly, your SQLite database must meet the following requirements:
56
+
57
+ | Requirement | Description |
58
+ |-------------|-------------|
59
+ | **Timestamp field** | Required for temporal correlations. Must be in a format compatible with SQLite's `julianday()` function (ISO8601, Julian day number, or Unix timestamp) |
60
+
61
+ **Configurable Parameters:**
62
+
63
+ The backend provides configurable parameters for correlation queries:
64
+
65
+ | Parameter | Default | Description |
66
+ |-----------|---------|-------------|
67
+ | `timestamp_field` | `timestamp` | Field name containing the event timestamp |
68
+
69
+ **Example usage with custom parameters:**
70
+
71
+ ```python
72
+ backend = sqliteBackend(correlation_methods=["default"])
73
+ backend.timestamp_field = "event_time"
74
+ ```
75
+
76
+ **Notes:**
77
+ - The timestamp field is used with `julianday()` for time difference calculations in temporal correlations
78
+ - For multi-rule correlations (`temporal`, `temporal_ordered`), the backend automatically adds a `sigma_rule_id` column to identify which rule matched each event
79
+ - Timespan values are converted to seconds internally for comparison
80
+
81
+ ### Other Features
82
+
83
+ * **NULL value handling**: `field: null` → `field IS NULL`
84
+ * **Boolean values**: `true`/`false` support
85
+ * **Field name quoting**: Special characters in field names are quoted with backticks
86
+ * **Wildcard escaping**: Proper escaping of `%` and `_` characters in values
87
+
88
+ ## Known issues/limitations
89
+
90
+ * Full text search support will need some work and is not a priority since it needs virtual tables on SQLite side
91
+
92
+ # Quick Start
93
+
94
+ ## Example script (default output) with sysmon pipeline
95
+
96
+ ### Add pipelines
97
+
98
+ ```shell
99
+ poetry add pysigma-pipeline-sysmon
100
+ poetry add pysigma-pipeline-windows
101
+ ```
102
+
103
+ ### Convert a rule
104
+
105
+ ```python
106
+ from sigma.collection import SigmaCollection
107
+ from sigma.backends.sqlite import sqlite
108
+ from sigma.pipelines.sysmon import sysmon_pipeline
109
+ from sigma.pipelines.windows import windows_logsource_pipeline
110
+
111
+ from sigma.processing.resolver import ProcessingPipelineResolver
112
+
113
+ # Create the pipeline resolver
114
+ piperesolver = ProcessingPipelineResolver()
115
+ # Add pipelines
116
+ piperesolver.add_pipeline_class(sysmon_pipeline()) # Syssmon
117
+ piperesolver.add_pipeline_class(windows_logsource_pipeline()) # Windows
118
+ # Create a combined pipeline
119
+ combined_pipeline = piperesolver.resolve(piperesolver.pipelines)
120
+ # Instantiate backend using the combined pipeline
121
+ sqlite_backend = sqlite.sqliteBackend(combined_pipeline)
122
+
123
+ rule = SigmaCollection.from_yaml(
124
+ r"""
125
+ title: Test
126
+ status: test
127
+ logsource:
128
+ category: test_category
129
+ product: test_product
130
+ detection:
131
+ sel:
132
+ fieldA: valueA
133
+ fieldB: valueB
134
+ condition: sel
135
+ """)
136
+
137
+ print(sqlite_backend.convert(rule)[0])
138
+
139
+ ```
140
+
141
+ ## Running
142
+
143
+ ```shell
144
+ poetry run python3 example.py
145
+ ```
{pysigma_backend_sqlite-0.2.0 → pysigma_backend_sqlite-1.1.0}/pyproject.toml RENAMED
@@ -1,18 +1,21 @@
1
- [tool.poetry]
1
+ [project]
2
2
  name = "pySigma-backend-sqlite"
3
- version = "0.2.0"
3
+ version = "1.1.0"
4
4
  description = "pySigma sqlite backend"
5
5
  readme = "README.md"
6
- authors = ["wagga"]
6
+ authors = [
7
+ { name = "wagga" }
8
+ ]
7
9
  license = "LGPL-3.0-only"
8
- repository = "https://github.com/SigmaHQ/pySigma-backend-sqlite"
10
+
11
+ [tool.poetry]
9
12
  packages = [
10
13
  { include = "sigma" }
11
14
  ]
12
15
 
13
16
  [tool.poetry.dependencies]
14
- python = "^3.9"
15
- pysigma = "^0.11.17"
17
+ python = "^3.10"
18
+ pysigma = ">=1.0.2,<1.1.0"
16
19
 
17
20
  [tool.poetry.group.dev.dependencies]
18
21
  black = "^24.1"
@@ -21,6 +24,12 @@ pytest-cov = "^4.1"
21
24
  coverage = "^7.4"
22
25
 
23
26
 
27
+ [tool.pytest.ini_options]
28
+ filterwarnings = [
29
+ # Suppress pyparsing deprecation warning from pySigma library
30
+ "ignore:'parseString' deprecated:DeprecationWarning",
31
+ ]
32
+
24
33
  [tool.pdm]
25
34
  [tool.pdm.dev-dependencies]
26
35
  dev = []
@@ -31,24 +40,13 @@ includes = ["sigma"]
31
40
  requires = ["poetry-core>=1.8.1"]
32
41
  build-backend = "poetry.core.masonry.api"
33
42
 
34
- [project]
35
- authors = [
36
- {name = "wagga", email = "wagga"},
37
- ]
38
- license = {text = "LGPL-3.0-only"}
39
- requires-python = ">=3.9,<4.0"
40
- dependencies = [
41
- "pysigma>=0.11.17",
42
- ]
43
-
44
- name = "pySigma-backend-sqlite"
45
- version = "0.2.0"
46
- description = "pySigma sqlite backend"
47
-
48
43
  [project.urls]
49
- repository = "https://github.com/SigmaHQ/pySigma-backend-sqlite"
44
+ Repository = "https://github.com/SigmaHQ/pySigma-backend-sqlite"
50
45
 
51
46
  [project.group.dev.dependencies]
52
47
  pytest = "^8.0"
53
48
  pytest-cov = "^4.1"
54
49
  coverage = "^7.4"
50
+
51
+
52
+