mobsec-modules-library 0.0.1__tar.gz → 0.0.2__tar.gz

{mobsec_modules_library-0.0.1 → mobsec_modules_library-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mobsec_modules_library
-Version: 0.0.1
+Version: 0.0.2
 Summary: Library for writing MobSec Modules using Python
 Author-email: Stormtrooperroman <stormtrooperroman@gmail.com>
 License-Expression: MIT
@@ -11,9 +11,11 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: redis>=4.0.2
+Requires-Dist: redis>=4.2.0
 Requires-Dist: fastapi>=0.68.0
 Requires-Dist: uvicorn>=0.15.0
+Requires-Dist: httpx>=0.23.0
+Requires-Dist: PyYAML>=6.0
 Dynamic: license-file
 
 # MobSec modules library

{mobsec_modules_library-0.0.1 → mobsec_modules_library-0.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mobsec_modules_library"
-version = "0.0.1"
+version = "0.0.2"
 authors = [
   { name="Stormtrooperroman", email="stormtrooperroman@gmail.com" },
 ]
@@ -15,9 +15,11 @@ license = "MIT"
 license-files = ["LICEN[CS]E*"]
 
 dependencies = [
-    "redis>=4.0.2",
+    "redis>=4.2.0",
     "fastapi>=0.68.0",
-    "uvicorn>=0.15.0"
+    "uvicorn>=0.15.0",
+    "httpx>=0.23.0",
+    "PyYAML>=6.0",
 
 ]
 

mobsec_modules_library-0.0.2/src/mobsec_modules_library/external/__init__.py

@@ -0,0 +1,3 @@
+from .external_module import ExternalModule, make_finding
+
+__all__ = ["ExternalModule", "make_finding"]

mobsec_modules_library-0.0.2/src/mobsec_modules_library/external/external_module.py

@@ -0,0 +1,246 @@
+import asyncio
+import json
+import logging
+import os
+import re
+import shutil
+import tarfile
+from abc import ABC, abstractmethod
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import httpx
+import redis.asyncio as aioredis
+import uvicorn
+import yaml
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class OperationRequest(BaseModel):
+    task_id: str
+    chain_task_id: Optional[str] = None
+    data: Dict[str, Any] = {}
+    file_hash: Optional[str] = None
+
+
+class OperationResponse(BaseModel):
+    status: str
+    data: Dict[str, Any]
+
+
+def make_finding(
+    *,
+    rule_id: str,
+    name: str,
+    message: str,
+    file: str,
+    severity: str = "WARNING",
+    path: Optional[str] = None,
+    start_line: int = 0,
+    end_line: Optional[int] = None,
+    code: str = "",
+    **metadata: Any,
+) -> Dict[str, Any]:
+    return {
+        "rule_id": rule_id,
+        "name": name,
+        "message": message,
+        "severity": severity,
+        "location": {
+            "file": file,
+            "path": path or file,
+            "start_line": start_line,
+            "end_line": end_line if end_line is not None else start_line,
+            "code": code,
+        },
+        "metadata": metadata,
+    }
+
+
+def _safe_extract(tar_path: Path, dest: Path) -> None:
+    dest = dest.resolve()
+    with tarfile.open(tar_path, "r:gz") as tar:
+        members = []
+        for m in tar.getmembers():
+            if not (dest / m.name).resolve().is_relative_to(dest):
+                raise ValueError(f"Unsafe path in archive: {m.name}")
+            if m.issym() or m.islnk() or m.isdev():
+                continue
+            members.append(m)
+        tar.extractall(dest, members=members)
+
+
+class ExternalModule(ABC):
+    def __init__(
+        self,
+        config_path: str = "config.yaml",
+        ui_component: Optional[str] = None,
+    ):
+        with open(config_path) as f:
+            self.config: Dict[str, Any] = yaml.safe_load(f)
+
+        slug = re.sub(r"[^a-z0-9]+", "-", self.config["name"].lower()).strip("-")
+        self.module_id = os.getenv("MODULE_ID", slug)
+        self.base_url = os.environ["MODULE_BASE_URL"].rstrip("/")
+        self.api_url = os.getenv("MOBSEC_API_URL", "http://backend:8000/api/v1")
+        self.redis = aioredis.Redis.from_url(
+            os.getenv("REDIS_URL", "redis://redis:6379/0"), decode_responses=True
+        )
+        self.temp_dir = Path(os.getenv("MODULE_TEMP_DIR", f"/tmp/{self.module_id}"))
+        self.temp_dir.mkdir(parents=True, exist_ok=True)
+        self.ui_component = Path(ui_component) if ui_component else None
+
+    @abstractmethod
+    async def scan(self, work_dir: Path, task: Dict[str, Any]) -> List[Dict[str, Any]]:
+        pass
+
+    async def fetch_files(self, file_hash: str, work_dir: Path) -> None:
+        tar_path = work_dir / "files.tar.gz"
+        timeout = httpx.Timeout(30.0, read=300.0)
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            async with client.stream(
+                "GET", f"{self.api_url}/artifacts/{file_hash}/archive"
+            ) as resp:
+                resp.raise_for_status()
+                with tar_path.open("wb") as f:
+                    async for chunk in resp.aiter_bytes():
+                        f.write(chunk)
+        try:
+            await asyncio.to_thread(_safe_extract, tar_path, work_dir)
+        finally:
+            tar_path.unlink(missing_ok=True)
+
+    def _registration_payload(self) -> Dict[str, Any]:
+        config = {
+            "name": self.config["name"],
+            "version": self.config["version"],
+            "description": self.config.get("description", ""),
+            "input_formats": self.config.get("input_formats", []),
+            "has_custom_ui": bool(self.ui_component and self.ui_component.exists()),
+        }
+        if config["has_custom_ui"]:
+            config["ui_component"] = {
+                "name": self.ui_component.stem,
+                "endpoint": f"{self.base_url}/ui-component",
+            }
+        return {
+            "module_id": self.module_id,
+            "base_url": self.base_url,
+            "config": config,
+            "healthcheck_url": f"{self.base_url}/health",
+            "status": "active",
+        }
+
+    async def register(self, retries: int = 10, delay: float = 5.0) -> bool:
+        payload = self._registration_payload()
+        for attempt in range(1, retries + 1):
+            try:
+                async with httpx.AsyncClient(timeout=30.0) as client:
+                    resp = await client.post(
+                        f"{self.api_url}/external-modules/register", json=payload
+                    )
+                if resp.status_code == 200:
+                    logger.info("Module %s registered", self.module_id)
+                    return True
+                logger.error("Registration failed: %s %s", resp.status_code, resp.text)
+            except Exception as e:
+                logger.error("Registration error: %s", e)
+            if attempt < retries:
+                await asyncio.sleep(delay)
+        logger.error("Could not register %s after %d attempts", self.module_id, retries)
+        return False
+
+    async def _handle_task(self, task_id: str) -> None:
+        raw = await self.redis.get(f"task:{task_id}")
+        if not raw:
+            logger.error("Task data not found for %s", task_id)
+            return
+        task = json.loads(raw)
+        file_hash = task.get("file_hash")
+        if not file_hash:
+            logger.error("No file_hash in task %s", task_id)
+            return
+
+        work_dir = self.temp_dir / f"scan_{task_id}"
+        try:
+            work_dir.mkdir(parents=True, exist_ok=True)
+            await self.fetch_files(file_hash, work_dir)
+            findings = await self.scan(work_dir, task)
+            result = {"status": "success", "results": findings}
+            logger.info("Task %s: %d findings", task_id, len(findings))
+        except Exception as e:
+            logger.exception("Task %s failed", task_id)
+            result = {"status": "error", "error": str(e), "results": []}
+        finally:
+            shutil.rmtree(work_dir, ignore_errors=True)
+
+        await self.redis.set(f"result:{self.module_id}:{file_hash}", json.dumps(result))
+
+    async def _worker(self) -> None:
+        queue_key = f"module:{self.module_id}:queue"
+        logger.info("Worker started on %s", queue_key)
+        while True:
+            try:
+                task_id = await self.redis.lpop(queue_key)
+                if task_id:
+                    await self._handle_task(task_id)
+                else:
+                    await asyncio.sleep(1)
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.exception("Worker loop error")
+                await asyncio.sleep(5)
+
+    def create_app(self) -> FastAPI:
+        @asynccontextmanager
+        async def lifespan(app: FastAPI):
+            tasks = [
+                asyncio.create_task(self._worker()),
+                asyncio.create_task(self.register()),
+            ]
+            yield
+            for t in tasks:
+                t.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+        app = FastAPI(title=self.config["name"], lifespan=lifespan)
+
+        @app.get("/health")
+        async def health():
+            return {"status": "healthy"}
+
+        @app.get("/ui-component")
+        async def ui_component():
+            if not self.ui_component or not self.ui_component.exists():
+                raise HTTPException(404, "UI component file not found")
+            return {
+                "component_name": self.ui_component.stem,
+                "component_content": self.ui_component.read_text(),
+            }
+
+        @app.post("/operations/process", response_model=OperationResponse)
+        async def process(request: OperationRequest):
+            if not request.file_hash:
+                raise HTTPException(400, "Missing file_hash parameter")
+            await self.redis.set(
+                f"task:{request.task_id}",
+                json.dumps({**request.data, "file_hash": request.file_hash}),
+                ex=3600,
+            )
+            await self.redis.rpush(f"module:{self.module_id}:queue", request.task_id)
+            return OperationResponse(
+                status="success",
+                data={"task_id": request.task_id, "message": "Task added to queue"},
+            )
+
+        return app
+
+    def run(self) -> None:
+        uvicorn.run(
+            self.create_app(), host="0.0.0.0", port=int(os.getenv("PORT", "8080"))
+        )

{mobsec_modules_library-0.0.1 → mobsec_modules_library-0.0.2}/src/mobsec_modules_library.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mobsec_modules_library
-Version: 0.0.1
+Version: 0.0.2
 Summary: Library for writing MobSec Modules using Python
 Author-email: Stormtrooperroman <stormtrooperroman@gmail.com>
 License-Expression: MIT
@@ -11,9 +11,11 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: redis>=4.0.2
+Requires-Dist: redis>=4.2.0
 Requires-Dist: fastapi>=0.68.0
 Requires-Dist: uvicorn>=0.15.0
+Requires-Dist: httpx>=0.23.0
+Requires-Dist: PyYAML>=6.0
 Dynamic: license-file
 
 # MobSec modules library

{mobsec_modules_library-0.0.1 → mobsec_modules_library-0.0.2}/src/mobsec_modules_library.egg-info/SOURCES.txt

@@ -12,4 +12,6 @@ src/mobsec_modules_library/dynamic/adb_utils.py
 src/mobsec_modules_library/dynamic/base_module_manager.py
 src/mobsec_modules_library/dynamic/base_server.py
 src/mobsec_modules_library/dynamic/base_websocket_manager.py
+src/mobsec_modules_library/external/__init__.py
+src/mobsec_modules_library/external/external_module.py
 src/mobsec_modules_library/static/static_module.py

mobsec_modules_library-0.0.2/src/mobsec_modules_library.egg-info/requires.txt

@@ -0,0 +1,5 @@
+redis>=4.2.0
+fastapi>=0.68.0
+uvicorn>=0.15.0
+httpx>=0.23.0
+PyYAML>=6.0

mobsec_modules_library-0.0.1/src/mobsec_modules_library.egg-info/requires.txt

@@ -1,3 +0,0 @@
-redis>=4.0.2
-fastapi>=0.68.0
-uvicorn>=0.15.0