clickdetect 0.1.0__py3-none-any.whl

api/detector.py

@@ -0,0 +1,70 @@
+from fastapi import APIRouter, HTTPException
+from detector.manager import get_manager_instance
+from detector.detector import Detector
+from datetime import datetime
+
+router = APIRouter(prefix='/detector')
+
+def detector_to_dict(job_id: str, d: Detector):
+    return {
+        'id': job_id,
+        'name': d.name,
+        'description': d.description,
+        'tenant': d.tenant,
+        'active': d.active,
+        'for_time': d.for_time,
+        'rules_count': len(d._rules),
+        'webhooks': d.webhooks,
+        'last_time_exec': datetime.fromtimestamp(d._last_time).isoformat(),
+        'next_time_exec': datetime.fromtimestamp(d._next_time).isoformat()
+    }
+
+
+@router.get('/list')
+async def listDetectors():
+    manager = get_manager_instance()
+    detectors = await manager.get_detectors()
+    return [detector_to_dict(job_id, d) for job_id, d in detectors.items()]
+
+
+@router.get('/tenant/{tenant}')
+async def getDetectorsByTenant(tenant: str):
+    manager = get_manager_instance()
+    detectors = await manager.get_detectors()
+    return [detector_to_dict(job_id, d) for job_id, d in detectors.items() if d.tenant == tenant]
+
+
+@router.get('/{id}')
+async def getDetector(id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return detector_to_dict(id, detector)
+
+
+@router.delete('/{id}')
+async def deleteDetector(id: str):
+    manager = get_manager_instance()
+    result = await manager.remove_scheduler(id)
+    if not result:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return {'deleted': id}
+
+
+@router.post('/{id}/stop')
+async def stopDetector(id: str):
+    manager = get_manager_instance()
+    result = await manager.stop_scheduler(id)
+    if not result:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return {'stopped': id}
+
+
+@router.post('/{id}/resume')
+async def resumeDetector(id: str):
+    manager = get_manager_instance()
+    result = await manager.resume_scheduler(id)
+    if not result:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return {'resumed': id}

api/health.py

@@ -0,0 +1,6 @@
+from fastapi import APIRouter
+router = APIRouter(prefix='/health')
+
+@router.get('/ok')
+def isOk():
+    return { 'ok': True}

api/rules.py

@@ -0,0 +1,45 @@
+from fastapi import APIRouter, HTTPException
+from detector.manager import get_manager_instance
+router = APIRouter(prefix='/rules')
+
+@router.get('/{detector_id}')
+async def listRules(detector_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    return [x.to_dict() for x in detector._rules]
+
+@router.get('/{detector_id}/{rule_id}')
+async def getRuleById(detector_id: str, rule_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    rule = await detector.get_rule_by_id(rule_id)
+    if not rule:
+        raise HTTPException(status_code=404, detail='Rule not found')
+    return rule.to_dict()
+
+@router.get('/{detector_id}/{rule_id}/pause')
+async def pauseRule(detector_id: str, rule_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    ok = await detector.setRuleActive(rule_id, False)
+    if not ok:
+        raise HTTPException(status_code=404, detail='Rule not found')
+    return { 'ok': True }
+
+@router.get('/{detector_id}/{rule_id}/resume')
+async def resumeRule(detector_id: str, rule_id: str):
+    manager = get_manager_instance()
+    detector = await manager.get_detector_by_id(detector_id)
+    if not detector:
+        raise HTTPException(status_code=404, detail='Detector not found')
+    ok = await detector.setRuleActive(rule_id, True)
+    if not ok:
+        raise HTTPException(status_code=404, detail='Rule not found')
+    return { 'ok': True }
+    

clickdetect-0.1.0.dist-info/METADATA

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: clickdetect
+Version: 0.1.0
+Summary: Generic SIEM detector
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp[speedups]>=3.13.2
+Requires-Dist: apscheduler>=3.11.1
+Requires-Dist: asyncpg>=0.31.0
+Requires-Dist: clickhouse-connect>=0.10.0
+Requires-Dist: colorlog>=6.10.1
+Requires-Dist: fastapi[standard]>=0.127.1
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: matrix-nio[e2e]>=0.25.2
+Requires-Dist: pyyaml>=6.0.3
+
+# Overview
+
+**Clickdetect** is a framework for threshold-based detection and alerting. It periodically queries your data sources, evaluates rules against the results, and sends alerts to one or more destinations when conditions are met.
+
+You can pull events from any DataSource implemented, and push alerts to any webhook.
+
+If you use elastalert, you will like this!
+
+## Documentation
+
+Documentation: [https://clickdetect.souzo.me](https://clickdetect.souzo.me)
+
+## Next steps
+
+* Implement timeframe []
+    * Grouping alerts []
+    * Suppress alerts []
+* Hot reload rules []
+* Add new rules using api []
+* Add api endpoint to silence detectors []
+* Sigma converter in rule (sigma: true) []
+* Sync schedulers for scalability []
+* Get rules from S3 
+
+
+## Installation
+
+### Using uv
+
+Download here:
+
+1. https://docs.astral.sh/uv/getting-started/installation/
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+uv sync --no-dev
+uv run clickdetect
+```
+
+### Using Docker / Podman
+
+From repository
+
+```sh
+git clone https://github.com/clicksiem/clickdetect
+cd clickdetect
+podman build -t clickdetect .
+podman run --rm -v ./runner.yml:/app/runner.yml -p 8080:8080 clickdetect --api
+```
+
+## Contribution
+
+* Like this project
+* Help me to create a sigma converter for clickhouse.
+* Report bugs in the issues
+
+## Contact
+
+* E-mail: me@souzo.me <vinicius morais>
+* [Linkedin](https://www.linkedin.com/in/vinicius-f-a76ba51b5/)
+

clickdetect-0.1.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+clickdetect.py,sha256=DoHaFBaRYsyTGwjU7jbynWC3uLfJf7CD_9FUSixbxo0,2640
+api/detector.py,sha256=4FNsF_ORmsPXaCk-IFtUBPeDCHXUff5gHkNZM5HEDu0,2217
+api/health.py,sha256=r7c6FNfnvrfb7Pt84HY1obCeyKsWe4QesmAuZEaJTG0,124
+api/rules.py,sha256=AKMyy8MNfeRC1JM_95i9ybIRZNmxpVkypQ6deiSOvS0,1818
+detector/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+detector/config.py,sha256=XUerloUPlY9adeQKFkCy0cwP9MlPdUXsS7ABCY8G6Hk,763
+detector/detector.py,sha256=WymJitGhbfPT1GDPgF3RmCjdzH_HRnWqmxXiT6RTuTA,8726
+detector/manager.py,sha256=0PC3VD4bkl25U-SnCDTv-1b_26a-kaC3vYlBDB-uVNg,2767
+detector/rules.py,sha256=ZbKAm7a52zhskk6Uht-LwVGuQgyZIzcdYsOmFtB9vtM,1896
+detector/runner.py,sha256=hBMQ3lrar4yHriW7XhLMAXmZlURDiQGlGmJZv_Bh7Bk,4630
+detector/utils.py,sha256=QqvTPzlixY62xE6VFX37ViEY-z7WydVAgiV9MdvdGyw,1831
+detector/datasource/__init__.py,sha256=_1WhW6-ggEMlv-RQM4sjXoqjr45KvuhGjXEzSLGX730,487
+detector/datasource/base.py,sha256=3LUMpXKoddkFYp_TIinHUXXGKsQmAf_HwuF8Bio68kw,686
+detector/datasource/clickhouse.py,sha256=Og365v4T1etJ3xV3C_GJRFLvdEm4qVyi47BU_02zz-g,2390
+detector/datasource/elasticsearch.py,sha256=wFV8l1PzOd5xt-VuZVmNYAuxILFE7FQDPdynp3RQwlU,3734
+detector/datasource/loki.py,sha256=1xXk2t2VmzWpu4loTm0dQtTKZTPpTaFddN09qLDHQgA,4332
+detector/datasource/postgresql.py,sha256=iYz12W6zD7IArHJNj3l_SX-PvafkgX1Xsb5B6yN3FWg,2281
+detector/webhooks/__init__.py,sha256=_GqWAgBXVbmF2HL9A7PAlU06kCPX6rl4H4TZCLapDOc,396
+detector/webhooks/base.py,sha256=0blxe5C04ik7eL6THvJfwtob9xnFWxPnuOA1z531kPY,668
+detector/webhooks/email.py,sha256=Fi-uLWTvHTgMUiSZGav7H2NV40kFTCid8G2XXYpjPsc,3430
+detector/webhooks/generic.py,sha256=iy6jruuxZIgqBT8PJOeChltYfNidjTU7KBHfNcB3F1c,2176
+detector/webhooks/matrix.py,sha256=-7qUvA7CTmOxKUwN818m08MMiJZkcjQSh_eek9ho-CY,2932
+detector/webhooks/teams.py,sha256=ZCk7qfnDKySxCys8SEFwaBsdvQy9B_A1qWVbaR4sOHM,2766
+clickdetect-0.1.0.dist-info/METADATA,sha256=fE8u4mCnheqXFuq37_3BDpeQKkMv_5aXMkrjDrJrGAI,1908
+clickdetect-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+clickdetect-0.1.0.dist-info/entry_points.txt,sha256=8_PkJbEHQgVjvJHjmBQnMP6uvD0sUzM-Gd5xTi0gafQ,48
+clickdetect-0.1.0.dist-info/top_level.txt,sha256=zV-XLGS2Ry805jitX8Ib3o663-AQG86vrgUlVQGaGt0,25
+clickdetect-0.1.0.dist-info/RECORD,,

clickdetect-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

clickdetect-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+clickdetect = clickdetect:run

clickdetect-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+api
+clickdetect
+detector

clickdetect.py

@@ -0,0 +1,79 @@
+import asyncio
+import uvicorn
+import detector.config as config
+import argparse
+from typing import Any
+from detector.runner import Runner
+from detector.manager import Manager, get_manager_instance
+from logging import getLogger
+from fastapi import FastAPI
+from api.detector import router as detector_router
+from api.rules import router as rules_router
+from os.path import exists as f_exists
+
+config.logConfig()
+logger = getLogger(__name__)
+
+async def load_api(args: Any):
+    app = FastAPI(title=config.app_name)
+    app.include_router(detector_router)
+    app.include_router(rules_router)
+
+    server_config = uvicorn.Config(app, host='0.0.0.0', port=args.port, log_level='info')
+    server = uvicorn.Server(server_config)
+    await server.serve()
+
+async def load_runner(args: Any) -> Runner | None:
+    runner = await Runner(args.runner).init()
+    if not runner:
+        logger.debug('Runner not loaded')
+        return None
+
+    detectors = await runner.get_detectors()
+    manager = Manager()
+
+    if not detectors:
+        logger.error('No detector found')
+        return None
+
+    for detector in detectors:
+        await manager.run_detector(detector)
+
+    return runner
+
+async def loop_run(runner: Runner | None = None):
+    try:
+        while await config.is_running():
+            await asyncio.sleep(1)
+    except asyncio.CancelledError:
+        logger.warning('received kill event')
+    finally:
+        await get_manager_instance().shutdown()
+        if runner:
+            await runner.close()
+
+async def main():
+    parser = argparse.ArgumentParser(description=f'{config.app_name} is a tool to detect patterns and alerts in clickhouse and others database')
+    parser.add_argument('--api', required=False, default=False, action='store_true', help='Enable api, required for clicksiem-backend')
+    parser.add_argument('-p', '--port', default=config.default_port, type=int, help=f'specify api port, default: {config.default_port}')
+    parser.add_argument('-r', '--runner', default=config.default_runner, type=str, help=f'Runner file containing webhook, datasources, detectors and rules. Default: {config.default_runner}')
+    parser.add_argument('--stdin', default=False, type=bool, help='Read file from stdin')
+    args = parser.parse_args()
+
+    tasks = []
+    if args.runner:
+        if not f_exists(args.runner):
+            logger.fatal(f'File {args.runner} does not exists')
+            exit(1)
+        tasks.append(await load_runner(args))
+    if args.api:
+        tasks.append(load_api(args))
+    if tasks:
+        await asyncio.gather(*tasks)
+
+
+def run():
+    asyncio.run(main())
+
+if __name__ == "__main__":
+    run()

detector/__init__.py

@@ -0,0 +1 @@
+

detector/config.py

@@ -0,0 +1,36 @@
+import logging
+from asyncio import Lock
+import colorlog
+
+_lock = Lock()
+running = True
+rule_eval_semaphore = 7
+webhook_send_semaphore = 7
+
+def logConfig():
+    colorlog.basicConfig(
+        level=colorlog.DEBUG,
+        format="%(asctime)s | %(log_color)s%(levelname)-8s%(reset)s | %(name)s | %(message)s",
+        log_colors={
+            'DEBUG': 'cyan',
+            'INFO': 'green',
+            'WARNING': 'yellow',
+            'ERROR': 'red',
+            'CRITICAL': 'red,bg_white',
+        }
+    )
+
+async def is_running():
+    global running
+    async with _lock:
+        return running
+
+async def stop_running():
+    global running
+    async with _lock:
+        running = False
+
+
+app_name = 'ClickDetector'
+default_runner = 'runner.yml'
+default_port = 8080

detector/datasource/__init__.py

@@ -0,0 +1,14 @@
+from typing import Dict, List, Type
+from detector.datasource.base import BaseDataSource
+from detector.datasource.clickhouse import ClickhouseDataSource
+from detector.datasource.loki import LokiDataSource
+from detector.datasource.elasticsearch import ElasticsearchDataSource
+from detector.datasource.postgresql import PostgreSQLDataSource
+
+
+datasources: List[Type[BaseDataSource]] = [
+    ClickhouseDataSource,
+    LokiDataSource,
+    ElasticsearchDataSource,
+    PostgreSQLDataSource,
+]

detector/datasource/base.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from typing import Any, Dict
+
+
+@dataclass()
+class DataSourceQueryResult:
+    len: int
+    value: Any
+
+    def to_dict(self) -> Dict[str, Any]:
+        return { 
+            'len': self.len,
+            'value': self.value
+        }
+
+class BaseDataSource:
+    async def connect(self):
+        raise NotImplementedError()
+
+    async def query(self, data: str) -> DataSourceQueryResult | None:
+        raise NotImplementedError()
+
+    @classmethod
+    def _name(cls) -> str:
+        raise NotImplementedError()
+
+    async def _parse(self, _obj: Any):
+        raise NotImplementedError()
+
+    def to_dict(self) -> Dict:
+        raise NotImplementedError()

detector/datasource/clickhouse.py

@@ -0,0 +1,76 @@
+from typing import Any, Dict
+from clickhouse_connect import get_async_client
+from clickhouse_connect.driver.asyncclient import AsyncClient
+from logging import getLogger
+
+from .base import BaseDataSource, DataSourceQueryResult
+
+logger = getLogger(__name__)
+
+class ClickhouseDataSource(BaseDataSource):
+    database: str
+    host: str
+    port: int
+    username: str
+    password: str
+    verify: bool = False
+    client: AsyncClient | None = None
+
+    async def connect(self):
+        try:
+            self.client = await get_async_client(
+                    database=self.database,
+                    host=self.host,
+                    username=self.username,
+                    password=self.password,
+                    port=self.port,
+                    secure=self.verify
+            )
+        except Exception as ex:
+            logger.error(f'Failed to connect to ClickHouse at {self.host}:{self.port} | {ex}')
+            self.client = None
+
+    async def query(self, data: str) -> DataSourceQueryResult | None:
+        if not self.client:
+            await self.connect()
+        if not self.client:
+            return None
+        try:
+            result = await self.client.query(data)
+            return DataSourceQueryResult(result.row_count, list(result.named_results()))
+        except Exception as ex:
+            logger.error(f'Query failed, resetting client | {ex}')
+            self.client = None
+            return None
+
+    @classmethod
+    def _name(cls) -> str:
+        return 'clickhouse'
+
+    def to_dict(self) -> Dict:
+        return {
+            'database': self.database,
+            'host': self.host,
+            'port': self.port,
+            'username': self.username,
+            'password': self.password,
+            'verify': self.verify
+        }
+
+    async def _parse(self, _obj: Any):
+        database = _obj.get('database', 'default')
+        host = _obj.get('host')
+        port = _obj.get('port')
+        username = _obj.get('username')
+        password = _obj.get('password')
+        verify = _obj.get('verify', False)
+
+        if not host or not port or not username or not password:
+            raise Exception(f'Invalid parameters: {self.to_dict().items()}')
+
+        self.database = database
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.verify = verify

detector/datasource/elasticsearch.py

@@ -0,0 +1,109 @@
+from typing import Any, Dict
+from logging import getLogger
+import json
+import aiohttp
+
+from .base import BaseDataSource, DataSourceQueryResult
+
+logger = getLogger(__name__)
+
+class ElasticsearchDataSource(BaseDataSource):
+    host: str
+    port: int
+    index: str
+    username: str | None = None
+    password: str | None = None
+    api_key: str | None = None
+    verify: bool = False
+    _session: aiohttp.ClientSession | None = None
+
+    def _base_url(self) -> str:
+        scheme = 'https' if self.verify else 'http'
+        return f'{scheme}://{self.host}:{self.port}'
+
+    def _headers(self) -> Dict[str, str]:
+        headers = {'Content-Type': 'application/json'}
+        if self.api_key:
+            headers['Authorization'] = f'ApiKey {self.api_key}'
+        return headers
+
+    def _auth(self) -> aiohttp.BasicAuth | None:
+        if self.username and self.password:
+            return aiohttp.BasicAuth(self.username, self.password)
+        return None
+
+    async def connect(self):
+        try:
+            connector = aiohttp.TCPConnector(ssl=False)
+            self._session = aiohttp.ClientSession(
+                connector=connector,
+                auth=self._auth(),
+                headers=self._headers()
+            )
+            async with self._session.get(f'{self._base_url()}/_cluster/health') as resp:
+                if resp.status != 200:
+                    raise Exception(f'Elasticsearch not healthy, status: {resp.status}')
+        except Exception as ex:
+            logger.error(f'Failed to connect to Elasticsearch at {self.host}:{self.port} | {ex}')
+            if self._session:
+                await self._session.close()
+            self._session = None
+
+    async def query(self, data: str) -> DataSourceQueryResult | None:
+        if not self._session:
+            await self.connect()
+        if not self._session:
+            return None
+        try:
+            body = json.loads(data)
+            async with self._session.post(
+                f'{self._base_url()}/{self.index}/_search',
+                json=body
+            ) as resp:
+                if resp.status != 200:
+                    text = await resp.text()
+                    raise Exception(f'HTTP {resp.status}: {text}')
+                payload = await resp.json()
+                return self._parse_result(payload)
+        except Exception as ex:
+            logger.error(f'Query failed, resetting session | {ex}')
+            if self._session:
+                await self._session.close()
+            self._session = None
+            return None
+
+    def _parse_result(self, payload: Any) -> DataSourceQueryResult:
+        hits = payload.get('hits', {}).get('hits', [])
+        rows = [{'_id': h['_id'], '_index': h['_index'], **h['_source']} for h in hits]
+        return DataSourceQueryResult(len(rows), rows)
+
+    @classmethod
+    def _name(cls) -> str:
+        return 'elasticsearch'
+
+    def to_dict(self) -> Dict:
+        return {
+            'host': self.host,
+            'port': self.port,
+            'index': self.index,
+            'username': self.username,
+            'password': self.password,
+            'api_key': self.api_key,
+            'verify': self.verify,
+        }
+
+    async def _parse(self, _obj: Any):
+        host = _obj.get('host')
+        port = _obj.get('port')
+        index = _obj.get('index')
+
+        if not host or not port or not index:
+            raise Exception('Invalid parameters: host, port and index are required')
+
+        self.host = host
+        self.port = port
+        self.index = index
+        self.username = _obj.get('username')
+        self.password = _obj.get('password')
+        self.api_key = _obj.get('api_key')
+        self.verify = _obj.get('verify', False)