pysigma-backend-parseable 0.1.0__tar.gz

pysigma_backend_parseable-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,108 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install test dependencies
+        run: python -m pip install --upgrade pip && python -m pip install -e '.[test]' sigma-cli build
+      - name: Lint
+        run: ruff check .
+      - name: Unit tests
+        run: pytest tests -m 'not integration' --cov=sigma.backends.parseable --cov=sigma.pipelines.parseable --cov-report=term-missing
+      - name: Sigma CLI discovery and conversion
+        run: |
+          sigma list targets
+          sigma list pipelines parseable
+          sigma convert -t parseable -O dataset=windows-events examples/powershell.yml > /tmp/parseable-query.sql
+          sigma convert -t parseable -p parseable_otlp -O dataset=otel-events examples/powershell.yml > /tmp/parseable-otlp-query.sql
+          sigma convert -t parseable -p parseable_sysmon -p parseable_ecs -O dataset=ecs-events examples/powershell.yml > /tmp/parseable-ecs-query.sql
+          test -s /tmp/parseable-query.sql
+          test -s /tmp/parseable-otlp-query.sql
+          test -s /tmp/parseable-ecs-query.sql
+      - name: Build distributions
+        if: matrix.python-version == '3.14'
+        run: python -m build
+
+  corpus-regression:
+    name: Sigma corpus regression
+    runs-on: ubuntu-latest
+    env:
+      SIGMA_CORPUS_COMMIT: 5c9b21756f4e3ba137c1773ac9ba5a8332188961
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install backend
+        run: python -m pip install --upgrade pip && python -m pip install -e '.[test]'
+      - name: Fetch pinned Sigma corpus
+        run: |
+          git init "$RUNNER_TEMP/sigma-corpus"
+          git -C "$RUNNER_TEMP/sigma-corpus" remote add origin https://github.com/SigmaHQ/sigma.git
+          git -C "$RUNNER_TEMP/sigma-corpus" fetch --depth 1 origin "$SIGMA_CORPUS_COMMIT"
+          git -C "$RUNNER_TEMP/sigma-corpus" checkout --detach FETCH_HEAD
+      - name: Run compatibility regression gate
+        run: |
+          python scripts/corpus_sweep.py "$RUNNER_TEMP/sigma-corpus" \
+            --baseline reports/corpus-baseline.json \
+            --output-dir "$RUNNER_TEMP/corpus-report"
+      - name: Upload full corpus report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: corpus-report
+          path: ${{ runner.temp }}/corpus-report/
+          if-no-files-found: error
+
+  publish:
+    name: Publish to PyPI
+    needs: [test, corpus-regression]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - name: Install build tools
+        run: python -m pip install --upgrade pip build twine
+      - name: Verify release version
+        run: |
+          package_version="$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+          test "v${package_version}" = "${GITHUB_REF_NAME}"
+      - name: Build distributions
+        run: python -m build
+      - name: Check distributions
+        run: twine check dist/*
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: twine upload dist/* --verbose

pysigma_backend_parseable-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+/.coverage
+/.env
+/.pytest_cache/
+/.ruff_cache/
+/build/
+/dist/
+/reports/corpus-report.json
+*.egg-info/
+__pycache__/
+*.py[cod]

pysigma_backend_parseable-0.1.0/CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+All notable changes to this project are documented here.
+
+## 0.1.0 - 2026-09-08
+
+Initial release.
+
+### Added
+
+- Parseable SQL and predicate output formats.
+- Sigma CLI and Python API integration through pySigma plugin discovery.
+- Case-sensitive and case-insensitive string matching, wildcards, lists, comparisons,
+  regular expressions, null checks, existence checks, field references, and fieldless
+  keyword searches.
+- IPv4 and IPv6 CIDR conversion using Parseable's `ip_in_cidr` SQL function. CIDR queries
+  require Parseable v3.2.1 or newer.
+- Built-in processing pipelines for Parseable OTLP, flattened ECS, and native Sysmon schemas.
+- Custom placeholder and field-mapping pipeline support.
+- Unit and live Parseable integration tests.
+- Pinned Sigma corpus compatibility regression checks in GitHub Actions.
+
+### Known limitations
+
+- Sigma correlation rules are not supported.
+- Timestamp-part modifiers are not supported.
+- Fieldless regular expressions are not supported.
+- Deployment-specific Sigma placeholders require values supplied through a processing
+  pipeline.

pysigma_backend_parseable-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Parseable pySigma contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pysigma_backend_parseable-0.1.0/PKG-INFO

@@ -0,0 +1,327 @@
+Metadata-Version: 2.5
+Name: pysigma-backend-parseable
+Version: 0.1.0
+Summary: pySigma backend for Parseable SQL queries
+Project-URL: Homepage, https://github.com/parseablehq/pySigma-backend-parseable
+Project-URL: Issues, https://github.com/parseablehq/pySigma-backend-parseable/issues
+Author: Parseable pySigma contributors
+License: MIT
+License-File: LICENSE
+Keywords: parseable,pysigma,security,sigma,sql
+Requires-Python: >=3.10
+Requires-Dist: pysigma<2,>=1.0.0
+Provides-Extra: test
+Requires-Dist: coverage[toml]>=7.8; extra == 'test'
+Requires-Dist: pytest-cov>=6.1; extra == 'test'
+Requires-Dist: pytest>=8.3; extra == 'test'
+Requires-Dist: ruff>=0.12; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pySigma backend for Parseable
+
+Convert [Sigma](https://github.com/SigmaHQ/sigma) detection rules into SQL accepted by
+[Parseable](https://github.com/parseablehq/parseable).
+
+The backend generates SQL only. It does not create alerts, send queries, discover datasets,
+or infer how fields are stored in your Parseable instance.
+
+## Requirements
+
+- Python 3.10 or newer
+- pySigma 1.x
+- Parseable v3.2.1 or newer when executing generated CIDR queries
+- A Parseable dataset when executing the generated SQL
+
+## Installation
+
+The package is not yet published. Install it from a checkout:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install -e .
+```
+
+For the `sigma` command-line interface:
+
+```bash
+python -m pip install sigma-cli
+```
+
+Confirm that the plugin is available:
+
+```bash
+sigma list targets
+sigma list pipelines parseable
+```
+
+## Quick start
+
+Convert a rule into a complete query:
+
+```bash
+sigma convert \
+  --target parseable \
+  --backend-option dataset=windows-events \
+  examples/powershell.yml
+```
+
+Example output:
+
+```sql
+SELECT "Image", "CommandLine", "User"
+FROM "windows-events"
+WHERE LOWER("Image") LIKE '%\\powershell.exe' ESCAPE '\'
+  AND LOWER("CommandLine") LIKE '%-encodedcommand%' ESCAPE '\'
+```
+
+The backend quotes dataset and column names, so dotted names such as `service.name` are
+rendered as one SQL identifier: `"service.name"`.
+
+### Python API
+
+```python
+from pathlib import Path
+
+from sigma.backends.parseable import ParseableBackend
+from sigma.collection import SigmaCollection
+
+rules = SigmaCollection.from_yaml(Path("rule.yml").read_text())
+backend = ParseableBackend(dataset="windows-events")
+
+for query in backend.convert(rules):
+    print(query)
+```
+
+`convert()` returns a list because a Sigma document may contain multiple rules or
+conditions.
+
+## Output formats
+
+The default format produces a complete query and requires a dataset:
+
+```bash
+sigma convert -t parseable -O dataset=windows-events rule.yml
+```
+
+```sql
+SELECT * FROM "windows-events" WHERE "EventID" = 4625
+```
+
+The `predicate` format produces only the condition for embedding in another query. It does
+not require a dataset:
+
+```bash
+sigma convert -t parseable -f predicate rule.yml
+```
+
+```sql
+"EventID" = 4625
+```
+
+## Backend options
+
+| Option                  | Default                       | Description                                                  |
+| ----------------------- | ----------------------------- | ------------------------------------------------------------ |
+| `dataset`               | none                          | Dataset in the `FROM` clause; required for default output    |
+| `limit`                 | none                          | Positive integer appended as `LIMIT`                         |
+| `default_search_fields` | `body,message,event.original` | Comma-separated columns searched by fieldless Sigma keywords |
+
+Example:
+
+```bash
+sigma convert \
+  -t parseable \
+  -O dataset=application-logs \
+  -O limit=500 \
+  -O default_search_fields=body,message,log \
+  rule.yml
+```
+
+Every configured search field must exist in the target dataset. DataFusion rejects a query
+that references a missing column.
+
+## Field mapping pipelines
+
+Sigma rules use abstract field names. Parseable queries must use the exact columns created
+at ingestion. Select the pipeline matching your stored event schema:
+
+| Pipeline           | Use when                                                              |
+| ------------------ | --------------------------------------------------------------------- |
+| `parseable_otlp`   | OTLP log attributes are stored as literal semantic-convention columns |
+| `parseable_ecs`    | Nested ECS documents are flattened by Parseable using underscores     |
+| `parseable_sysmon` | Events use native Sysmon fields                                       |
+
+### OpenTelemetry
+
+```bash
+sigma convert \
+  -t parseable \
+  -p parseable_otlp \
+  -O dataset=otel-events \
+  rule.yml
+```
+
+Representative mappings:
+
+| Sigma             | Parseable OTLP            |
+| ----------------- | ------------------------- |
+| `Image`           | `process.executable.path` |
+| `CommandLine`     | `process.command_line`    |
+| `ProcessId`       | `process.pid`             |
+| `ParentProcessId` | `process.parent_pid`      |
+| `SourceIp`        | `source.address`          |
+| `DestinationIp`   | `destination.address`     |
+| `DestinationPort` | `destination.port`        |
+| `TargetFilename`  | `file.path`               |
+| `QueryName`       | `dns.question.name`       |
+| `Computer`        | `host.name`               |
+
+Mappings are scoped by Sigma log source where field meaning changes. The pipeline does not
+invent fields without a standard OpenTelemetry equivalent.
+
+### ECS
+
+```bash
+sigma convert \
+  -t parseable \
+  -p parseable_ecs \
+  -O dataset=ecs-events \
+  rule.yml
+```
+
+Nested ECS input such as `{"source":{"ip":"192.0.2.1"}}` becomes the Parseable column
+`source_ip`.
+
+| Sigma            | Flattened ECS               |
+| ---------------- | --------------------------- |
+| `EventID`        | `event_code`                |
+| `Channel`        | `winlog_channel`            |
+| `Image`          | `process_executable`        |
+| `CommandLine`    | `process_command_line`      |
+| `ParentImage`    | `process_parent_executable` |
+| `User`           | `user_name`                 |
+| `SourceIp`       | `source_ip`                 |
+| `DestinationIp`  | `destination_ip`            |
+| `TargetFilename` | `file_path`                 |
+| `QueryName`      | `dns_question_name`         |
+
+Use a custom pipeline if your events contain literal dotted ECS keys or use different column
+names.
+
+### Sysmon
+
+```bash
+sigma convert \
+  -t parseable \
+  -p parseable_sysmon \
+  -O dataset=sysmon-events \
+  rule.yml
+```
+
+This pipeline retains native Sysmon fields and adds the appropriate `Channel` and `EventID`
+conditions for generic Windows log sources. For Sysmon normalized to nested ECS before
+ingestion, chain the pipelines:
+
+```bash
+sigma convert \
+  -t parseable \
+  -p parseable_sysmon \
+  -p parseable_ecs \
+  -O dataset=ecs-sysmon-events \
+  rule.yml
+```
+
+### Custom schemas
+
+Built-in pipelines cannot know organization-specific column names. Define a pySigma pipeline
+for the schema actually present in your dataset:
+
+```yaml
+name: My Parseable field mapping
+priority: 30
+allowed_backends:
+  - parseable
+transformations:
+  - id: organization_fields
+    type: field_name_mapping
+    mapping:
+      Image: exe_path
+      CommandLine: command
+      User: username
+      SourceIp: client_ip
+```
+
+```bash
+sigma convert \
+  -t parseable \
+  -p company-parseable.yml \
+  -O dataset=company-events \
+  rule.yml
+```
+
+Always compare generated columns with the Parseable dataset schema before deploying rules.
+
+## Supported Sigma features
+
+The backend supports:
+
+- Case-insensitive Sigma string matching and the `cased` modifier
+- `contains`, `startswith`, `endswith`, `exists`, `fieldref`, and comparison modifiers
+- Sigma `*` and `?` wildcards
+- String and numeric lists
+- Regular expressions bound to a field
+- Null checks and Boolean conditions
+- Fieldless string and numeric keyword searches
+- IPv4 and IPv6 CIDR expressions through Parseable's `ip_in_cidr` SQL function
+
+Unsupported constructs fail explicitly with `SigmaFeatureNotSupportedByBackendError`:
+
+- Sigma correlation rules
+- Timestamp-part modifiers such as `minute` and `hour`
+- Fieldless regular expressions
+
+CIDR conversion requires a Parseable deployment that provides `ip_in_cidr(ip, cidr)`. The
+function correctly parses IPv4 and IPv6 rather than approximating address ranges as text.
+
+## Placeholders
+
+Sigma placeholders such as `%Administrators%` are deployment-specific values. Resolve them
+with a processing pipeline before conversion:
+
+```bash
+sigma convert \
+  -t parseable \
+  -p examples/placeholder-pipeline.yml \
+  -O dataset=windows-events \
+  examples/placeholder-rule.yml
+```
+
+Do not replace unknown placeholders with wildcards; that changes the detection's meaning.
+
+## Development
+
+Install development dependencies and run local checks:
+
+```bash
+python -m pip install -e '.[test]'
+pytest tests -m 'not integration'
+ruff check .
+python -m build
+```
+
+Live tests require `PARSEABLE_URL`, `PARSEABLE_INGESTION_URL`, and `PARSEABLE_API_KEY`.
+They use fixture datasets and are intentionally excluded from the default test command:
+
+```bash
+pytest tests/integration -m integration
+```
+
+The GitHub Actions corpus job checks conversion against a pinned Sigma corpus. Detailed
+results and regression thresholds live in [`reports/`](reports/) rather than this README.
+Successful conversion means valid SQL was generated; it does not prove that a deployment has
+matching columns or representative data.
+
+## License
+
+MIT