clickdetect 1.2.0__tar.gz → 1.3.0__tar.gz

{clickdetect-1.2.0 → clickdetect-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: clickdetect
-Version: 1.2.0
+Version: 1.3.0
 Summary: Generic SIEM engine detector
 Author: Vinicius Morais
 Author-email: Vinicius Morais <me@souzo.me>

{clickdetect-1.2.0 → clickdetect-1.3.0}/clickdetect/__init__.py

@@ -20,7 +20,7 @@ logger = getLogger(__name__)
 
 def print_webhooks():
     for w in webhooks:
-        print(f"  Webhook: {w._name()}")
+        print(f"Webhook: {w._name()}")
         for param in w._params():
             print(
                 f"\tName: {param.name}({param.type.__name__}) {'Required' if param.required else 'Optional'}. {f'Help: {param.help}' if param.help else ''}", end=' '
@@ -34,7 +34,7 @@ def print_webhooks():
 
 def print_datasources():
     for ds in datasources:
-        print(f"  Webhook: {ds._name()}")
+        print(f"Datasources: {ds._name()}")
         for param in ds._params():
             print(
                 f"\tName: {param.name}({param.type.__name__}) {'Required' if param.required else 'Optional'}. {f'Help: {param.help}' if param.help else ''}", end=' '

{clickdetect-1.2.0 → clickdetect-1.3.0}/clickdetect/detector/datasource/__init__.py

@@ -4,6 +4,7 @@ from .clickhouse import ClickhouseDataSource
 from .loki import LokiDataSource
 from .elasticsearch import ElasticsearchDataSource
 from .postgresql import PostgreSQLDataSource
+from .victorialogs import VictoriaLogsDataSource
 
 
 datasources: List[Type[BaseDataSource]] = [
@@ -11,4 +12,5 @@ datasources: List[Type[BaseDataSource]] = [
     LokiDataSource,
     ElasticsearchDataSource,
     PostgreSQLDataSource,
+    VictoriaLogsDataSource
 ]

{clickdetect-1.2.0 → clickdetect-1.3.0}/clickdetect/detector/datasource/base.py

@@ -1,3 +1,4 @@
+from ..rules import Rule
 from ..utils import Parameters
 from dataclasses import dataclass
 from typing import Any, Dict, List
@@ -19,6 +20,9 @@ class BaseDataSource:
     async def connect(self):
         raise NotImplementedError()
 
+    async def _query(self, data: str, rule: Rule) -> DataSourceQueryResult | None:
+        return await self.query(data)
+
     async def query(self, data: str) -> DataSourceQueryResult | None:
         raise NotImplementedError()
 
@@ -26,6 +30,10 @@ class BaseDataSource:
     def _name(cls) -> str:
         raise NotImplementedError()
 
+    @classmethod
+    def _params(cls) -> List[Parameters]:
+        raise NotImplementedError()
+
     async def _parse(self, data: Any):
         self.params = self._params()
 
@@ -54,6 +62,3 @@ class BaseDataSource:
             result[param.name] = getattr(self, attr, param.default)
         return result
 
-    @classmethod
-    def _params(cls) -> List[Parameters]:
-        raise NotImplementedError()

{clickdetect-1.2.0 → clickdetect-1.3.0}/clickdetect/detector/datasource/loki.py

@@ -1,25 +1,20 @@
 from typing import Any, List
 from logging import getLogger
 from .base import BaseDataSource, DataSourceQueryResult
-import aiohttp
-
 from ..utils import Parameters
+import aiohttp
 
 logger = getLogger(__name__)
 
 
 class LokiDataSource(BaseDataSource):
-    host: str
-    port: int
-    username: str | None = None
-    password: str | None = None
-    verify: bool = False
-    org_id: str | None = None
+    url: str
+    username: str
+    password: str
+    verify: bool
+    org_id: str
     _session: aiohttp.ClientSession | None = None
-
-    def _base_url(self) -> str:
-        scheme = "https" if self.verify else "http"
-        return f"{scheme}://{self.host}:{self.port}"
+    _base_url: str
 
     def _headers(self) -> dict:
         headers = {"Content-Type": "application/json"}
@@ -33,16 +28,17 @@ class LokiDataSource(BaseDataSource):
         return None
 
     async def connect(self):
+        logger.debug(f"Connecting to {self.url}")
+        self._base_url = self.url.rstrip("/")
         try:
-            connector = aiohttp.TCPConnector(ssl=False)
+            connector = aiohttp.TCPConnector(verify_ssl=self.verify)
             self._session = aiohttp.ClientSession(
                 connector=connector, auth=self._auth(), headers=self._headers()
             )
-            async with self._session.get(f"{self._base_url()}/ready") as resp:
-                if resp.status != 200:
-                    raise Exception(f"Loki not ready, status: {resp.status}")
+            async with self._session.get(f"{self._base_url}/ready") as resp:
+                resp.raise_for_status()
         except Exception as ex:
-            logger.error(f"Failed to connect to Loki at {self.host}:{self.port} | {ex}")
+            logger.error(f"Datasource connect exception. url: {self.url} | {ex}")
             if self._session:
                 await self._session.close()
             self._session = None
@@ -53,13 +49,12 @@ class LokiDataSource(BaseDataSource):
         if not self._session:
             return None
         try:
+            logger.debug(f"Sending query to {self.url}")
             params = {"query": data, "limit": 5000}
             async with self._session.get(
-                f"{self._base_url()}/loki/api/v1/query_range", params=params
+                f"{self._base_url}/loki/api/v1/query_range", params=params
             ) as resp:
-                if resp.status != 200:
-                    body = await resp.text()
-                    raise Exception(f"HTTP {resp.status}: {body}")
+                resp.raise_for_status()
                 payload = await resp.json()
                 return await self._parse_result(payload)
         except Exception as ex:
@@ -72,9 +67,9 @@ class LokiDataSource(BaseDataSource):
     async def _parse_result(self, payload: Any) -> DataSourceQueryResult:
         result_type = payload.get("data", {}).get("resultType", "streams")
         results = payload.get("data", {}).get("result", [])
+        rows = []
 
         if result_type == "matrix":
-            rows = []
             total = 0
             for series in results:
                 metric = series.get("metric", {})
@@ -85,7 +80,6 @@ class LokiDataSource(BaseDataSource):
                     total += float(values[-1][1])
             return DataSourceQueryResult(int(total), rows, self._name())
 
-        rows = []
         for stream in results:
             labels = stream.get("stream", {})
             for ts, line in stream.get("values", []):
@@ -99,10 +93,11 @@ class LokiDataSource(BaseDataSource):
     @classmethod
     def _params(cls) -> List[Parameters]:
         return [
-            Parameters('host', str, True, 'Loki host'),
-            Parameters('port', int, False, 'Loki port', 3100),
-            Parameters('username', str, False, 'Username', is_sensive_field=True),
-            Parameters('password', str, False, 'Password', is_sensive_field=True),
-            Parameters('verify', bool, False, 'Verify SSL', False),
-            Parameters('org_id', str, False, 'Loki org ID (X-Scope-OrgID)'),
+            Parameters("url", str, True, "Loki url: http://localhost:3100"),
+            Parameters("username", str, False, "Username", is_sensive_field=True),
+            Parameters("password", str, False, "Password", is_sensive_field=True),
+            Parameters("verify", bool, False, "Verify SSL", False),
+            Parameters(
+                "org_id", str, False, "Loki org ID (X-Scope-OrgID)", default="fake"
+            ),
         ]

clickdetect-1.3.0/clickdetect/detector/datasource/victorialogs.py

@@ -0,0 +1,101 @@
+from typing import List
+from logging import getLogger
+from .base import BaseDataSource, DataSourceQueryResult
+from ..utils import Parameters
+
+import aiohttp
+import json
+
+logger = getLogger(__name__)
+
+
+class VictoriaLogsDataSource(BaseDataSource):
+    url: str
+    username: str
+    password: str
+    verify: bool
+    account_id: str
+    project_id: str
+    _session: aiohttp.ClientSession | None = None
+    _base_url: str
+
+    def _headers(self) -> dict:
+        headers = {"Content-Type": "application/json"}
+        if self.account_id:
+            headers["X-VictoriaMetrics-Account-Id"] = self.account_id
+        if self.project_id:
+            headers["X-VictoriaMetrics-Project-Id"] = self.project_id
+        return headers
+
+    def _auth(self) -> aiohttp.BasicAuth | None:
+        if self.username and self.password:
+            return aiohttp.BasicAuth(self.username, self.password)
+        return None
+
+    async def connect(self):
+        logger.debug(f"Connecting to {self.url}")
+        self._base_url = self.url.rstrip("/")
+        try:
+            connector = aiohttp.TCPConnector(ssl=False)
+            self._session = aiohttp.ClientSession(
+                connector=connector, auth=self._auth(), headers=self._headers()
+            )
+            async with self._session.get(f"{self._base_url}/health") as resp:
+                if resp.status != 200:
+                    raise Exception(f"VictoriaLogs not ready, status: {resp.status}")
+        except Exception as ex:
+            logger.error(f"Failed to connect to VictoriaLogs at {self.url} | {ex}")
+            if self._session:
+                await self._session.close()
+            self._session = None
+
+    async def query(self, data: str) -> DataSourceQueryResult | None:
+        if not self._session:
+            await self.connect()
+        if not self._session:
+            return None
+        try:
+            logger.debug(f"Sending query to {self.url}")
+            params = {"query": data, "limit": 5000}
+            async with self._session.get(
+                f"{self._base_url}/select/logsql/query", params=params
+            ) as resp:
+                resp.raise_for_status()
+                body = await resp.text()
+                return self._parse_result(body)
+        except Exception as ex:
+            logger.error(f"Query failed, resetting session | {ex}")
+            if self._session:
+                await self._session.close()
+            self._session = None
+            return None
+
+    def _parse_result(self, body: str) -> DataSourceQueryResult:
+        rows = []
+        for line in body.splitlines():
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+        return DataSourceQueryResult(len(rows), rows, self._name())
+
+    @classmethod
+    def _name(cls) -> str:
+        logger.warning("This datasource integration hasn't been tested yet")
+        logger.warning("I only did the integration based on victorialogs api documentation")
+        logger.warning("Please open an issue if you can validate this datasource")
+        return "victorialogs"
+
+    @classmethod
+    def _params(cls) -> List[Parameters]:
+        return [
+            Parameters("url", str, True, "VictoriaLogs url"),
+            Parameters("username", str, False, "Username", is_sensive_field=True),
+            Parameters("password", str, False, "Password", is_sensive_field=True),
+            Parameters("verify", bool, False, "Verify SSL", False),
+            Parameters(
+                "account_id", str, False, "X-VictoriaMetrics-Account-Id header", "0"
+            ),
+            Parameters(
+                "project_id", str, False, "X-VictoriaMetrics-Project-Id header", "0"
+            ),
+        ]

{clickdetect-1.2.0 → clickdetect-1.3.0}/clickdetect/detector/detector.py

@@ -120,7 +120,7 @@ class Detector:
                 logger.debug(f"Rule id: {rule.id}")
                 logger.debug(f"Rule name: {rule.name}")
                 logger.debug(f"rule query:\n {query}\n")
-                result = await self.datasource.query(query)
+                result = await self.datasource._query(query, rule)
 
                 if result is None:
                     logger.warning("Datasource unavailable, skipping rule evaluation")

{clickdetect-1.2.0 → clickdetect-1.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "clickdetect"
-version = "1.2.0"
+version = "1.3.0"
 description = "Generic SIEM engine detector"
 readme = "README.md"
 requires-python = ">=3.13"