clickdetect 0.1.0__tar.gz

clickdetect-0.1.0/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: clickdetect
+Version: 0.1.0
+Summary: Generic SIEM detector
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp[speedups]>=3.13.2
+Requires-Dist: apscheduler>=3.11.1
+Requires-Dist: asyncpg>=0.31.0
+Requires-Dist: clickhouse-connect>=0.10.0
+Requires-Dist: colorlog>=6.10.1
+Requires-Dist: fastapi[standard]>=0.127.1
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: matrix-nio[e2e]>=0.25.2
+Requires-Dist: pyyaml>=6.0.3
+
+# Overview
+
+**Clickdetect** is a framework for threshold-based detection and alerting. It periodically queries your data sources, evaluates rules against the results, and sends alerts to one or more destinations when conditions are met.
+
+You can pull events from any DataSource implemented, and push alerts to any webhook.
+
+If you use elastalert, you will like this!
+
+## Documentation
+
+Documentation: [https://clickdetect.souzo.me](https://clickdetect.souzo.me)
+
+## Next steps
+
+* Implement timeframe []
+    * Grouping alerts []
+    * Suppress alerts []
+* Hot reload rules []
+* Add new rules using api []
+* Add api endpoint to silence detectors []
+* Sigma converter in rule (sigma: true) []
+* Sync schedulers for scalability []
+* Get rules from S3 
+
+
+## Installation
+
+### Using uv
+
+Download here:
+
+1. https://docs.astral.sh/uv/getting-started/installation/
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+uv sync --no-dev
+uv run clickdetect
+```
+
+### Using Docker / Podman
+
+From repository
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+podman build -t clickdetect .
+podman run --rm -v ./runner.yml:/app/runner.yml -p 8080:8080 clickdetect --api
+```
+
+## Contribution
+
+* Like this project
+* Help me to create a sigma converter for clickhouse.
+* Report bugs in the issues
+
+## Contact
+
+* E-mail: me@souzo.me <vinicius morais>
+* [Linkedin](https://www.linkedin.com/in/vinicius-f-a76ba51b5/)
+

clickdetect-0.1.0/README.md

@@ -0,0 +1,62 @@
+# Overview
+
+**Clickdetect** is a framework for threshold-based detection and alerting. It periodically queries your data sources, evaluates rules against the results, and sends alerts to one or more destinations when conditions are met.
+
+You can pull events from any DataSource implemented, and push alerts to any webhook.
+
+If you use elastalert, you will like this!
+
+## Documentation
+
+Documentation: [https://clickdetect.souzo.me](https://clickdetect.souzo.me)
+
+## Next steps
+
+* Implement timeframe []
+    * Grouping alerts []
+    * Suppress alerts []
+* Hot reload rules []
+* Add new rules using api []
+* Add api endpoint to silence detectors []
+* Sigma converter in rule (sigma: true) []
+* Sync schedulers for scalability []
+* Get rules from S3 
+
+
+## Installation
+
+### Using uv
+
+Download here:
+
+1. https://docs.astral.sh/uv/getting-started/installation/
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+uv sync --no-dev
+uv run clickdetect
+```
+
+### Using Docker / Podman
+
+From repository
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+podman build -t clickdetect .
+podman run --rm -v ./runner.yml:/app/runner.yml -p 8080:8080 clickdetect --api
+```
+
+## Contribution
+
+* Like this project
+* Help me to create a sigma converter for clickhouse.
+* Report bugs in the issues
+
+## Contact
+
+* E-mail: me@souzo.me <vinicius morais>
+* [Linkedin](https://www.linkedin.com/in/vinicius-f-a76ba51b5/)
+

clickdetect-0.1.0/api/detector.py

@@ -0,0 +1,70 @@
+from fastapi import APIRouter, HTTPException
+from detector.manager import get_manager_instance
+from detector.detector import Detector
+from datetime import datetime
+
+router = APIRouter(prefix='/detector')
+
+def detector_to_dict(job_id: str, d: Detector):
+    return {
+        'id': job_id,
+        'name': d.name,
+        'description': d.description,
+        'tenant': d.tenant,
+        'active': d.active,
+        'for_time': d.for_time,
+        'rules_count': len(d._rules),
+        'webhooks': d.webhooks,
+        'last_time_exec': datetime.fromtimestamp(d._last_time).isoformat(),
+        'next_time_exec': datetime.fromtimestamp(d._next_time).isoformat()
+    }
+
+
+@router.get('/list')
+async def listDetectors():
+    manager = get_manager_instance()
+    detectors = await manager.get_detectors()
+    return [detector_to_dict(job_id, d) for job_id, d in detectors.items()]
+
+
+@router.get('/tenant/{tenant}')
+async def getDetectorsByTenant(tenant: str):
+    manager = get_manager_instance()
+    detectors = await manager.get_detectors()
+    return [detector_to_dict(job_id, d) for job_id, d in detectors.items() if d.tenant == tenant]
+
+
+@router.get('/{id}')
+async def getDetector(id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return detector_to_dict(id, detector)
+
+
+@router.delete('/{id}')
+async def deleteDetector(id: str):
+    manager = get_manager_instance()
+    result = await manager.remove_scheduler(id)
+    if not result:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return {'deleted': id}
+
+
+@router.post('/{id}/stop')
+async def stopDetector(id: str):
+    manager = get_manager_instance()
+    result = await manager.stop_scheduler(id)
+    if not result:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return {'stopped': id}
+
+
+@router.post('/{id}/resume')
+async def resumeDetector(id: str):
+    manager = get_manager_instance()
+    result = await manager.resume_scheduler(id)
+    if not result:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return {'resumed': id}

clickdetect-0.1.0/api/health.py

@@ -0,0 +1,6 @@
+from fastapi import APIRouter
+router = APIRouter(prefix='/health')
+
+@router.get('/ok')
+def isOk():
+    return { 'ok': True}

clickdetect-0.1.0/api/rules.py

@@ -0,0 +1,45 @@
+from fastapi import APIRouter, HTTPException
+from detector.manager import get_manager_instance
+router = APIRouter(prefix='/rules')
+
+@router.get('/{detector_id}')
+async def listRules(detector_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return [x.to_dict() for x in detector._rules]
+
+@router.get('/{detector_id}/{rule_id}')
+async def getRuleById(detector_id: str, rule_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    rule = await detector.get_rule_by_id(rule_id)
+    if not rule:
+        raise HTTPException(status_code=404, detail='Rule not found')
+    return rule.to_dict()
+
+@router.get('/{detector_id}/{rule_id}/pause')
+async def pauseRule(detector_id: str, rule_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    ok = await detector.setRuleActive(rule_id, False)
+    if not ok:
+        raise HTTPException(status_code=404, detail='Rule not found')
+    return { 'ok': True }
+
+@router.get('/{detector_id}/{rule_id}/resume')
+async def resumeRule(detector_id: str, rule_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    ok = await detector.setRuleActive(rule_id, True)
+    if not ok:
+        raise HTTPException(status_code=404, detail='Rule not found')
+    return { 'ok': True }
+    

clickdetect-0.1.0/clickdetect.egg-info/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: clickdetect
+Version: 0.1.0
+Summary: Generic SIEM detector
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp[speedups]>=3.13.2
+Requires-Dist: apscheduler>=3.11.1
+Requires-Dist: asyncpg>=0.31.0
+Requires-Dist: clickhouse-connect>=0.10.0
+Requires-Dist: colorlog>=6.10.1
+Requires-Dist: fastapi[standard]>=0.127.1
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: matrix-nio[e2e]>=0.25.2
+Requires-Dist: pyyaml>=6.0.3
+
+# Overview
+
+**Clickdetect** is a framework for threshold-based detection and alerting. It periodically queries your data sources, evaluates rules against the results, and sends alerts to one or more destinations when conditions are met.
+
+You can pull events from any DataSource implemented, and push alerts to any webhook.
+
+If you use elastalert, you will like this!
+
+## Documentation
+
+Documentation: [https://clickdetect.souzo.me](https://clickdetect.souzo.me)
+
+## Next steps
+
+* Implement timeframe []
+    * Grouping alerts []
+    * Suppress alerts []
+* Hot reload rules []
+* Add new rules using api []
+* Add api endpoint to silence detectors []
+* Sigma converter in rule (sigma: true) []
+* Sync schedulers for scalability []
+* Get rules from S3 
+
+
+## Installation
+
+### Using uv
+
+Download here:
+
+1. https://docs.astral.sh/uv/getting-started/installation/
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+uv sync --no-dev
+uv run clickdetect
+```
+
+### Using Docker / Podman
+
+From repository
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+podman build -t clickdetect .
+podman run --rm -v ./runner.yml:/app/runner.yml -p 8080:8080 clickdetect --api
+```
+
+## Contribution
+
+* Like this project
+* Help me to create a sigma converter for clickhouse.
+* Report bugs in the issues
+
+## Contact
+
+* E-mail: me@souzo.me <vinicius morais>
+* [Linkedin](https://www.linkedin.com/in/vinicius-f-a76ba51b5/)
+

clickdetect-0.1.0/clickdetect.egg-info/SOURCES.txt

@@ -0,0 +1,31 @@
+README.md
+clickdetect.py
+pyproject.toml
+api/detector.py
+api/health.py
+api/rules.py
+clickdetect.egg-info/PKG-INFO
+clickdetect.egg-info/SOURCES.txt
+clickdetect.egg-info/dependency_links.txt
+clickdetect.egg-info/entry_points.txt
+clickdetect.egg-info/requires.txt
+clickdetect.egg-info/top_level.txt
+detector/__init__.py
+detector/config.py
+detector/detector.py
+detector/manager.py
+detector/rules.py
+detector/runner.py
+detector/utils.py
+detector/datasource/__init__.py
+detector/datasource/base.py
+detector/datasource/clickhouse.py
+detector/datasource/elasticsearch.py
+detector/datasource/loki.py
+detector/datasource/postgresql.py
+detector/webhooks/__init__.py
+detector/webhooks/base.py
+detector/webhooks/email.py
+detector/webhooks/generic.py
+detector/webhooks/matrix.py
+detector/webhooks/teams.py

clickdetect-0.1.0/clickdetect.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

clickdetect-0.1.0/clickdetect.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+clickdetect = clickdetect:run

clickdetect-0.1.0/clickdetect.egg-info/requires.txt

@@ -0,0 +1,9 @@
+aiohttp[speedups]>=3.13.2
+apscheduler>=3.11.1
+asyncpg>=0.31.0
+clickhouse-connect>=0.10.0
+colorlog>=6.10.1
+fastapi[standard]>=0.127.1
+jinja2>=3.1.6
+matrix-nio[e2e]>=0.25.2
+pyyaml>=6.0.3

clickdetect-0.1.0/clickdetect.egg-info/top_level.txt

@@ -0,0 +1,3 @@
+api
+clickdetect
+detector

clickdetect-0.1.0/clickdetect.py

@@ -0,0 +1,79 @@
+import asyncio
+import uvicorn
+import detector.config as config
+import argparse
+from typing import Any
+from detector.runner import Runner
+from detector.manager import Manager, get_manager_instance
+from logging import getLogger
+from fastapi import FastAPI
+from api.detector import router as detector_router
+from api.rules import router as rules_router
+from os.path import exists as f_exists
+
+config.logConfig()
+logger = getLogger(__name__)
+
+async def load_api(args: Any):
+    app = FastAPI(title=config.app_name)
+    app.include_router(detector_router)
+    app.include_router(rules_router)
+
+    server_config = uvicorn.Config(app, host='0.0.0.0', port=args.port, log_level='info')
+    server = uvicorn.Server(server_config)
+    await server.serve()
+
+async def load_runner(args: Any) -> Runner | None:
+    runner = await Runner(args.runner).init()
+    if not runner:
+        logger.debug('Runner not loaded')
+        return None
+
+    detectors = await runner.get_detectors()
+    manager = Manager()
+
+    if not detectors:
+        logger.error('No detector found')
+        return None
+
+    for detector in detectors:
+        await manager.run_detector(detector)
+
+    return runner
+
+async def loop_run(runner: Runner | None = None):
+    try:
+        while await config.is_running():
+            await asyncio.sleep(1)
+    except asyncio.CancelledError:
+        logger.warning('received kill event')
+    finally:
+        await get_manager_instance().shutdown()
+        if runner:
+            await runner.close()
+
+async def main():
+    parser = argparse.ArgumentParser(description=f'{config.app_name} is a tool to detect patterns and alerts in clickhouse and others database')
+    parser.add_argument('--api', required=False, default=False, action='store_true', help='Enable api, required for clicksiem-backend')
+    parser.add_argument('-p', '--port', default=config.default_port, type=int, help=f'specify api port, default: {config.default_port}')
+    parser.add_argument('-r', '--runner', default=config.default_runner, type=str, help=f'Runner file containing webhook, datasources, detectors and rules. Default: {config.default_runner}')
+    parser.add_argument('--stdin', default=False, type=bool, help='Read file from stdin')
+    args = parser.parse_args()
+
+    tasks = []
+    if args.runner:
+        if not f_exists(args.runner):
+            logger.fatal(f'File {args.runner} does not exists')
+            exit(1)
+        tasks.append(await load_runner(args))
+    if args.api:
+        tasks.append(load_api(args))
+    if tasks:
+        await asyncio.gather(*tasks)
+
+
+def run():
+    asyncio.run(main())
+
+if __name__ == "__main__":
+    run()

clickdetect-0.1.0/detector/__init__.py

@@ -0,0 +1 @@
+

clickdetect-0.1.0/detector/config.py

@@ -0,0 +1,36 @@
+import logging
+from asyncio import Lock
+import colorlog
+
+_lock = Lock()
+running = True
+rule_eval_semaphore = 7
+webhook_send_semaphore = 7
+
+def logConfig():
+    colorlog.basicConfig(
+        level=colorlog.DEBUG,
+        format="%(asctime)s | %(log_color)s%(levelname)-8s%(reset)s | %(name)s | %(message)s",
+        log_colors={
+            'DEBUG': 'cyan',
+            'INFO': 'green',
+            'WARNING': 'yellow',
+            'ERROR': 'red',
+            'CRITICAL': 'red,bg_white',
+        }
+    )
+
+async def is_running():
+    global running
+    async with _lock:
+        return running
+
+async def stop_running():
+    global running
+    async with _lock:
+        running = False
+
+
+app_name = 'ClickDetector'
+default_runner = 'runner.yml'
+default_port = 8080

clickdetect-0.1.0/detector/datasource/__init__.py

@@ -0,0 +1,14 @@
+from typing import Dict, List, Type
+from detector.datasource.base import BaseDataSource
+from detector.datasource.clickhouse import ClickhouseDataSource
+from detector.datasource.loki import LokiDataSource
+from detector.datasource.elasticsearch import ElasticsearchDataSource
+from detector.datasource.postgresql import PostgreSQLDataSource
+
+
+datasources: List[Type[BaseDataSource]] = [
+    ClickhouseDataSource,
+    LokiDataSource,
+    ElasticsearchDataSource,
+    PostgreSQLDataSource,
+]

clickdetect-0.1.0/detector/datasource/base.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from typing import Any, Dict
+
+
+@dataclass()
+class DataSourceQueryResult:
+    len: int
+    value: Any
+
+    def to_dict(self) -> Dict[str, Any]:
+        return { 
+            'len': self.len,
+            'value': self.value
+        }
+
+class BaseDataSource:
+    async def connect(self):
+        raise NotImplementedError()
+
+    async def query(self, data: str) -> DataSourceQueryResult | None:
+        raise NotImplementedError()
+
+    @classmethod
+    def _name(cls) -> str:
+        raise NotImplementedError()
+
+    async def _parse(self, _obj: Any):
+        raise NotImplementedError()
+
+    def to_dict(self) -> Dict:
+        raise NotImplementedError()

clickdetect-0.1.0/detector/datasource/clickhouse.py

@@ -0,0 +1,76 @@
+from typing import Any, Dict
+from clickhouse_connect import get_async_client
+from clickhouse_connect.driver.asyncclient import AsyncClient
+from logging import getLogger
+
+from .base import BaseDataSource, DataSourceQueryResult
+
+logger = getLogger(__name__)
+
+class ClickhouseDataSource(BaseDataSource):
+    database: str
+    host: str
+    port: int
+    username: str
+    password: str
+    verify: bool = False
+    client: AsyncClient | None = None
+
+    async def connect(self):
+        try:
+            self.client = await get_async_client(
+                    database=self.database,
+                    host=self.host,
+                    username=self.username,
+                    password=self.password,
+                    port=self.port,
+                    secure=self.verify
+            )
+        except Exception as ex:
+            logger.error(f'Failed to connect to ClickHouse at {self.host}:{self.port} | {ex}')
+            self.client = None
+
+    async def query(self, data: str) -> DataSourceQueryResult | None:
+        if not self.client:
+            await self.connect()
+        if not self.client:
+            return None
+        try:
+            result = await self.client.query(data)
+            return DataSourceQueryResult(result.row_count, list(result.named_results()))
+        except Exception as ex:
+            logger.error(f'Query failed, resetting client | {ex}')
+            self.client = None
+            return None
+
+    @classmethod
+    def _name(cls) -> str:
+        return 'clickhouse'
+
+    def to_dict(self) -> Dict:
+        return {
+            'database': self.database,
+            'host': self.host,
+            'port': self.port,
+            'username': self.username,
+            'password': self.password,
+            'verify': self.verify
+        }
+
+    async def _parse(self, _obj: Any):
+        database = _obj.get('database', 'default')
+        host = _obj.get('host')
+        port = _obj.get('port')
+        username = _obj.get('username')
+        password = _obj.get('password')
+        verify = _obj.get('verify', False)
+
+        if not host or not port or not username or not password:
+            raise Exception(f'Invalid parameters: {self.to_dict().items()}')
+
+        self.database = database
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.verify = verify