goosebit 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

goosebit/updater/controller/v1/routes.py

@@ -2,11 +2,17 @@ import logging
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.requests import Request
-from fastapi.responses import FileResponse, RedirectResponse, Response
+from fastapi.responses import (
+    FileResponse,
+    RedirectResponse,
+    Response,
+    StreamingResponse,
+)
 
-from goosebit.db.models import Software, UpdateStateEnum
+from goosebit.db.models import Device, Software, UpdateStateEnum
+from goosebit.device_manager import DeviceManager, HandlingType, get_device
 from goosebit.settings import config
-from goosebit.updater.manager import HandlingType, UpdateManager, get_update_manager
+from goosebit.storage import storage
 from goosebit.updates import generate_chunk
 
 from .schema import (
@@ -22,48 +28,64 @@ router = APIRouter(prefix="/v1")
 
 
 @router.get("/{dev_id}")
-async def polling(request: Request, dev_id: str, updater: UpdateManager = Depends(get_update_manager)):
+async def polling(request: Request, device: Device = Depends(get_device)):
     links: dict[str, dict[str, str]] = {}
 
-    sleep = updater.poll_time
-    device = await updater.get_device()
-
     if device is None:
         raise HTTPException(404)
 
+    sleep = config.poll_time
+
     if device.last_state == UpdateStateEnum.UNKNOWN:
-        # device registration
-        sleep = config.poll_time_registration
+        # device registration: force device to poll again in 10s. After registration, an update might be available
+        sleep = "00:00:10"
         links["configData"] = {
             "href": str(
                 request.url_for(
                     "config_data",
-                    dev_id=dev_id,
+                    dev_id=device.id,
                 )
             )
         }
-        logger.info(f"Skip: registration required, device={updater.dev_id}")
+        logger.info(f"Skip: registration required, device={device.id}")
 
     elif device.last_state == UpdateStateEnum.ERROR and not device.force_update:
-        logger.warning(f"Skip: device in error state, device={updater.dev_id}")
-        pass
+        logger.info(f"Skip: device in error state, device={device.id}")
 
     else:
         # provide update if available. Note: this is also required while in state "running", otherwise swupdate
         # won't confirm a successful testing (might be a bug/problem in swupdate)
-        handling_type, software = await updater.get_update()
+        handling_type, software = await DeviceManager.get_update(device)
         if handling_type != HandlingType.SKIP and software is not None:
-            links["deploymentBase"] = {
-                "href": str(
-                    request.url_for(
-                        "deployment_base",
-                        dev_id=dev_id,
-                        action_id=software.id,
+            number_of_running = await Device.filter(last_state=UpdateStateEnum.RUNNING).count()
+            if number_of_running < config.max_concurrent_updates or device.last_state == UpdateStateEnum.RUNNING:
+                links["deploymentBase"] = {
+                    "href": str(
+                        request.url_for(
+                            "deployment_base",
+                            dev_id=device.id,
+                            action_id=software.id,
+                        )
                     )
-                )
-            }
-            logger.info(f"Forced: update available, device={updater.dev_id}")
-
+                }
+                logger.info(f"Forced: update available, device={device.id}")
+        else:
+            number_of_running = await Device.filter(last_state=UpdateStateEnum.RUNNING).count()
+            if number_of_running < config.max_concurrent_updates or device.last_state == UpdateStateEnum.RUNNING:
+                plugin_sources = await DeviceManager.get_alt_src_updates(request, device)
+                for handling_type, _ in plugin_sources:
+                    if handling_type == HandlingType.SKIP:
+                        continue
+                    links["deploymentBase"] = {
+                        "href": str(
+                            request.url_for(
+                                "deployment_base",
+                                dev_id=device.id,
+                                action_id=-1,  # custom plugin
+                            )
+                        )
+                    }
+                    break
     return {
         "config": {"polling": {"sleep": sleep}},
         "_links": links,
@@ -71,9 +93,9 @@ async def polling(request: Request, dev_id: str, updater: UpdateManager = Depend
 
 
 @router.put("/{dev_id}/configData")
-async def config_data(_: Request, cfg: ConfigDataSchema, updater: UpdateManager = Depends(get_update_manager)):
-    await updater.update_config_data(**cfg.data)
-    logger.info(f"Updating config data, device={updater.dev_id}")
+async def config_data(_: Request, cfg: ConfigDataSchema, device: Device = Depends(get_device)):
+    await DeviceManager.update_config_data(device, **cfg.data)
+    logger.info(f"Updating config data, device={device.id}")
     return {"success": True, "message": "Updated swupdate data."}
 
 
@@ -81,91 +103,106 @@ async def config_data(_: Request, cfg: ConfigDataSchema, updater: UpdateManager
 async def deployment_base(
     request: Request,
     action_id: int,
-    updater: UpdateManager = Depends(get_update_manager),
+    device: Device = Depends(get_device),
 ):
-    handling_type, software = await updater.get_update()
-
-    logger.info(f"Request deployment base, device={updater.dev_id}")
-
-    return {
-        "id": str(action_id),
-        "deployment": {
-            "download": str(handling_type),
-            "update": str(handling_type),
-            "chunks": await generate_chunk(request, updater),
-        },
-    }
+    handling_type, software = await DeviceManager.get_update(device)
+
+    logger.info(f"Request deployment base, device={device.id}")
+    if not handling_type == HandlingType.SKIP:
+        return {
+            "id": str(action_id),
+            "deployment": {
+                "download": str(handling_type),
+                "update": str(handling_type),
+                "chunks": [chunk.model_dump(by_alias=True) for chunk in (await generate_chunk(request, device))],
+            },
+        }
+    else:
+        plugin_sources = await DeviceManager.get_alt_src_updates(request, device)
+        for handling_type, chunk in plugin_sources:
+            if handling_type == HandlingType.SKIP or chunk is None:
+                continue
+            return {
+                "id": str(action_id),
+                "deployment": {
+                    "download": str(handling_type),
+                    "update": str(handling_type),
+                    "chunks": [chunk.model_dump(by_alias=True)],
+                },
+            }
 
 
 @router.post("/{dev_id}/deploymentBase/{action_id}/feedback")
-async def deployment_feedback(
-    _: Request, data: FeedbackSchema, action_id: int, updater: UpdateManager = Depends(get_update_manager)
-):
+async def deployment_feedback(_: Request, data: FeedbackSchema, action_id: int, device: Device = Depends(get_device)):
     if data.status.execution == FeedbackStatusExecutionState.PROCEEDING:
-        await updater.update_device_state(UpdateStateEnum.RUNNING)
-        logger.debug(f"Installation in progress, device={updater.dev_id}")
+        if device and device.last_state != UpdateStateEnum.RUNNING:
+            await DeviceManager.deployment_action_start(device)
+            await DeviceManager.update_device_state(device, UpdateStateEnum.RUNNING)
+
+        logger.debug(f"Installation in progress, device={device.id}")
 
     elif data.status.execution == FeedbackStatusExecutionState.CLOSED:
-        await updater.update_force_update(False)
-        await updater.update_log_complete(True)
+        await DeviceManager.update_force_update(device, False)
 
         reported_software = await Software.get_or_none(id=action_id)
 
         # From hawkBit docu: DDI defines also a status NONE which will not be interpreted by the update server
         # and handled like SUCCESS.
         if data.status.result.finished in [FeedbackStatusResultFinished.SUCCESS, FeedbackStatusResultFinished.NONE]:
-            await updater.update_device_state(UpdateStateEnum.FINISHED)
+            await DeviceManager.deployment_action_success(device)
+            await DeviceManager.update_device_state(device, UpdateStateEnum.FINISHED)
 
-            # not guaranteed to be the correct rollout - see next comment.
-            rollout = await updater.get_rollout()
+            rollout = await DeviceManager.get_rollout(device)
             if rollout:
                 if rollout.software == reported_software:
                     rollout.success_count += 1
                     await rollout.save()
                 else:
+                    # edge case where device update mode got changed while update was running
                     logging.warning(
-                        f"Updating rollout success stats failed, software={reported_software.id}, device={updater.dev_id}"  # noqa: E501
+                        f"Updating rollout success stats failed, action_id={action_id}, device={device.id}"
+                        # noqa: E501
                     )
 
-            # setting the currently installed version based on the current assigned software / existing rollouts
-            # is problematic. Better to assign custom action_id for each update (rollout id? software id? new id?).
-            # Alternatively - but requires customization on the gateway side - use version reported by the gateway.
-            await updater.update_sw_version(reported_software.version)
-            logger.debug(f"Installation successful, software={reported_software.version}, device={updater.dev_id}")
+            if reported_software:
+                await DeviceManager.update_sw_version(device, reported_software.version)
+
+            software_version = reported_software.version if reported_software else None
+            logger.debug(f"Installation successful, software={software_version}, device={device.id}")
 
         elif data.status.result.finished == FeedbackStatusResultFinished.FAILURE:
-            await updater.update_device_state(UpdateStateEnum.ERROR)
+            await DeviceManager.update_device_state(device, UpdateStateEnum.ERROR)
 
-            # not guaranteed to be the correct rollout - see comment above.
-            rollout = await updater.get_rollout()
+            rollout = await DeviceManager.get_rollout(device)
             if rollout:
                 if rollout.software == reported_software:
                     rollout.failure_count += 1
                     await rollout.save()
                 else:
+                    # edge case where device update mode got changed while update was running
                     logging.warning(
-                        f"Updating rollout failure stats failed, software={reported_software.id}, device={updater.dev_id}"  # noqa: E501
+                        f"Updating rollout failure stats failed, action_id={action_id}, device={device.id}"
+                        # noqa: E501
                     )
 
-            logger.debug(f"Installation failed, software={reported_software.version}, device={updater.dev_id}")
+            software_version = reported_software.version if reported_software else None
+            logger.debug(f"Installation failed, software={software_version}, device={device.id}")
     else:
-        logging.warning(
-            f"Device reported unhandled execution state, state={data.status.execution}, device={updater.dev_id}"
-        )
+        logging.error(f"Device reported unhandled execution state, state={data.status.execution}, device={device.id}")
 
     try:
         log = data.status.details
         if log is not None:
-            await updater.update_log("\n".join(log))
+            await DeviceManager.update_log(device, "\n".join(log))
     except AttributeError:
-        logging.warning(f"No details to update device update log, device={updater.dev_id}")
+        logging.warning(f"No details to update device update log, device={device.id}")
 
     return {"id": str(action_id)}
 
 
 @router.head("/{dev_id}/download")
-async def download_artifact_head(_: Request, updater: UpdateManager = Depends(get_update_manager)):
-    _, software = await updater.get_update()
+async def download_artifact_head(_: Request, device: Device = Depends(get_device)):
+    _, software = await DeviceManager.get_update(device)
     if software is None:
         raise HTTPException(404)
 
@@ -175,15 +212,26 @@ async def download_artifact_head(_: Request, updater: UpdateManager = Depends(ge
 
 
 @router.get("/{dev_id}/download")
-async def download_artifact(_: Request, updater: UpdateManager = Depends(get_update_manager)):
-    _, software = await updater.get_update()
+async def download_artifact(_: Request, device: Device = Depends(get_device)):
+    _, software = await DeviceManager.get_update(device)
     if software is None:
         raise HTTPException(404)
+
     if software.local:
         return FileResponse(
             software.path,
             media_type="application/octet-stream",
             filename=software.path.name,
         )
-    else:
-        return RedirectResponse(url=software.uri)
+
+    try:
+        url = await storage.get_download_url(software.uri)
+        return RedirectResponse(url=url)
+    except ValueError:
+        # Fallback to streaming if redirect fails.
+        file_stream = storage.get_file_stream(software.uri)
+        return StreamingResponse(
+            file_stream,
+            media_type="application/octet-stream",
+            headers={"Content-Disposition": f"attachment; filename={software.path.name}"},
+        )

goosebit/updater/routes.py

@@ -1,21 +1,100 @@
 import time
 
-from fastapi import APIRouter, Depends
+import httpx
+from fastapi import APIRouter, Depends, HTTPException
 from fastapi.requests import Request
 
+from goosebit.device_manager import DeviceManager, get_device_or_none
+from goosebit.settings import config
+from goosebit.settings.schema import DeviceAuthMode, ExternalAuthMode
+
+from ..db import Device
 from . import controller
-from .manager import get_update_manager
 
 
 async def log_last_connection(request: Request, dev_id: str):
-    host = request.client.host
-    updater = await get_update_manager(dev_id)
-    await updater.update_last_connection(round(time.time()), host)
+    device = await get_device_or_none(dev_id)
+
+    if not device:
+        return
+
+    if request.scope["config"].track_device_ip:
+        await DeviceManager.update_last_connection(device, round(time.time()), request.client.host)
+    else:
+        await DeviceManager.update_last_connection(device, round(time.time()))
+
+
+async def validate_device_token(request: Request, dev_id: str):
+    if not request.scope["config"].device_auth.enable:
+        return
+
+    # parse device token, needs to be the `TargetToken`
+    device_token = request.headers.get("Authorization")
+    if device_token is not None:
+        if device_token.startswith("TargetToken"):
+            device_token = device_token.replace("TargetToken ", "")
+        else:
+            device_token = None
+
+    # setup mode should register devices and set up their auth token
+    if request.scope["config"].device_auth.mode == DeviceAuthMode.SETUP:
+        device = await DeviceManager.get_device(dev_id)
+        if device_token is None:
+            return
+        await DeviceManager.update_auth_token(device, device_token)
+
+    # lax mode should register devices and check their token if they have one, but not register their tokens
+    elif request.scope["config"].device_auth.mode == DeviceAuthMode.LAX:
+        device = await DeviceManager.get_device(dev_id)
+        # should not be possible
+        assert device is not None
+
+        if not device.auth_token == device_token:
+            raise HTTPException(401, "Device authentication token does not match.")
+
+    # strict mode should ensure all device are already set up and have a token, then check the token
+    elif request.scope["config"].device_auth.mode == DeviceAuthMode.STRICT:
+        if device_token is None:
+            raise HTTPException(401, "Device authentication token is required in strict mode.")
+        # do not create a device in strict mode
+        device = await Device.get_or_none(id=dev_id)
+        if device is None:
+            raise HTTPException(401, "Cannot register a new device in strict mode.")
+        if not device.auth_token == device_token:
+            raise HTTPException(401, "Device authentication token does not match.")
+
+    # external mode should check the token with an external service
+    elif request.scope["config"].device_auth.mode == DeviceAuthMode.EXTERNAL:
+        if device_token is None:
+            raise HTTPException(401, "Device authentication token is required in external mode.")
+
+        try:
+            async with httpx.AsyncClient() as client:
+                if request.scope["config"].device_auth.external_mode == ExternalAuthMode.BEARER:
+                    response = await client.post(
+                        request.scope["config"].device_auth.external_url,
+                        headers={"Authorization": f"Bearer {device_token}"},
+                    )
+                elif request.scope["config"].device_auth.external_mode == ExternalAuthMode.JSON:
+                    json = {request.scope["config"].device_auth.external_json_key: device_token}
+                    response = await client.post(
+                        request.scope["config"].device_auth.external_url,
+                        json=json,
+                    )
+
+                if response.status_code != 200:
+                    raise HTTPException(401, "Device authentication token is invalid.")
+                else:
+                    await DeviceManager.get_device(dev_id)
+        except httpx.RequestError as e:
+            raise HTTPException(401, f"Error communicating with authentication service: {str(e)}")
+        except Exception as e:
+            raise HTTPException(401, f"Error: {str(e)}")
 
 
 router = APIRouter(
-    prefix="/ddi",
-    dependencies=[Depends(log_last_connection)],
+    prefix=f"/{config.tenant}",
+    dependencies=[Depends(log_last_connection), Depends(validate_device_token)],
     tags=["ddi"],
 )
 router.include_router(controller.router)

goosebit/updates/__init__.py

@@ -8,10 +8,11 @@ from fastapi import HTTPException
 from fastapi.requests import Request
 from tortoise.expressions import Q
 
-from goosebit.db.models import Hardware, Software
-from goosebit.updater.manager import UpdateManager
+from goosebit.db.models import Device, Hardware, Software
+from goosebit.device_manager import DeviceManager
+from goosebit.schema.updates import UpdateChunk, UpdateChunkArtifact
+from goosebit.storage import storage
 
-from ..settings import config
 from . import swdesc
 
 
@@ -48,11 +49,9 @@ async def create_software_update(uri: str, temp_file: Path | None) -> Software:
         if temp_file is None:
             raise HTTPException(500, "Temporary file missing, cannot parse file information")
         filename = Path(url2pathname(unquote(parsed_uri.path))).name
-        path = Path(config.artifacts_dir).joinpath(update_info["hash"], filename)
-        await path.parent.mkdir(parents=True, exist_ok=True)
-        await temp_file.replace(path)
-        absolute = await path.absolute()
-        uri = absolute.as_uri()
+
+        dest_path = Path(update_info["hash"]).joinpath(filename)
+        uri = await storage.store_file(temp_file, dest_path)
 
     # create software
     software = await Software.create(
@@ -92,31 +91,25 @@ async def _is_software_colliding(update_info):
     return is_colliding
 
 
-async def generate_chunk(request: Request, updater: UpdateManager) -> list:
-    _, software = await updater.get_update()
+async def generate_chunk(request: Request, device: Device) -> list[UpdateChunk]:
+    _, software = await DeviceManager.get_update(device)
     if software is None:
         return []
-    if software.local:
-        href = str(
-            request.url_for(
-                "download_artifact",
-                dev_id=updater.dev_id,
-            )
-        )
-    else:
-        href = software.uri
+
+    # Always use the download endpoint for consistency, the endpoint
+    # will handle both local and remote files appropriately.
+    href = str(request.url_for("download_artifact", dev_id=device.id))
+
     return [
-        {
-            "part": "os",
-            "version": "1",
-            "name": software.path.name,
-            "artifacts": [
-                {
-                    "filename": software.path.name,
-                    "hashes": {"sha1": software.hash},
-                    "size": software.size,
-                    "_links": {"download": {"href": href}},
-                }
+        UpdateChunk(
+            name=software.path.name,
+            artifacts=[
+                UpdateChunkArtifact(
+                    filename=software.path.name,
+                    hashes={"sha1": software.hash},
+                    size=software.size,
+                    links={"download": {"href": href}},
+                )
             ],
-        }
+        )
     ]

goosebit/updates/swdesc.py

@@ -6,15 +6,17 @@ from typing import Any
 
 import httpx
 import libconf
-import semver
 from anyio import AsyncFile, Path, open_file
 
-from goosebit.settings import config
+from goosebit.storage import storage
+from goosebit.util.version import Version
 
 logger = logging.getLogger(__name__)
 
 
 def _append_compatibility(boardname, value, compatibility):
+    if not isinstance(value, dict):
+        return
     if "hardware-compatibility" in value:
         for revision in value["hardware-compatibility"]:
             compatibility.append({"hw_model": boardname, "hw_revision": revision})
@@ -23,7 +25,7 @@ def _append_compatibility(boardname, value, compatibility):
 def parse_descriptor(swdesc: libconf.AttrDict[Any, Any | None]):
     swdesc_attrs = {}
     try:
-        swdesc_attrs["version"] = semver.Version.parse(swdesc["software"]["version"])
+        swdesc_attrs["version"] = Version.parse(swdesc["software"]["version"])
         compatibility: list[dict[str, str]] = []
         _append_compatibility("default", swdesc["software"], compatibility)
 
@@ -35,6 +37,10 @@ def parse_descriptor(swdesc: libconf.AttrDict[Any, Any | None]):
                 for key2 in element:
                     _append_compatibility(key, element[key2], compatibility)
 
+        if len(compatibility) == 0:
+            # if nothing is specified, assume compatibility with default / default boards
+            compatibility.append({"hw_model": "default", "hw_revision": "default"})
+
         swdesc_attrs["compatibility"] = compatibility
     except KeyError as e:
         logging.warning(f"Parsing swu descriptor failed, error={e}")
@@ -71,15 +77,16 @@ async def parse_file(file: Path):
 async def parse_remote(url: str):
     async with httpx.AsyncClient() as c:
         file = await c.get(url)
-        artifacts_dir = Path(config.artifacts_dir)
-        tmp_file_path = artifacts_dir.joinpath("tmp", ("".join(random.choices(string.ascii_lowercase, k=12)) + ".tmp"))
-        await tmp_file_path.parent.mkdir(parents=True, exist_ok=True)
+        temp_dir = Path(storage.get_temp_dir())
+        tmp_file_path = temp_dir.joinpath("".join(random.choices(string.ascii_lowercase, k=12)) + ".tmp")
         try:
             async with await open_file(tmp_file_path, "w+b") as f:
                 await f.write(file.content)
-            file_data = await parse_file(Path(str(f.name)))
+            file_data = await parse_file(tmp_file_path)  # Use anyio.Path for parse_file
+        except Exception:
+            raise
         finally:
-            await tmp_file_path.unlink()
+            await tmp_file_path.unlink(missing_ok=True)
         return file_data
 
 

goosebit/users/__init__.py

@@ -0,0 +1,63 @@
+from aiocache import caches
+
+from goosebit.api.telemetry.metrics import users_count
+from goosebit.db.models import User
+from goosebit.settings import PWD_CXT
+
+
+async def create_user(username: str, password: str, permissions: list[str]) -> User:
+    return await UserManager.setup_user(username=username, hashed_pwd=PWD_CXT.hash(password), permissions=permissions)
+
+
+async def create_initial_user(username: str, hashed_pwd: str) -> User:
+    return await UserManager.setup_user(username=username, hashed_pwd=hashed_pwd, permissions=["*"])
+
+
+class UserManager:
+    @staticmethod
+    async def save_user(user: User, update_fields: list[str]) -> None:
+        await user.save(update_fields=update_fields)
+
+        # only update cache after a successful database save
+        result = await caches.get("default").set(user.username, user, ttl=600)
+        assert result, "user being cached"
+
+    @staticmethod
+    async def update_enabled(user: User, enabled: bool) -> None:
+        user.enabled = enabled
+        await UserManager.save_user(user, update_fields=["enabled"])
+
+    @classmethod
+    async def setup_user(cls, username: str, hashed_pwd: str, permissions: list[str]) -> User:
+        user = (
+            await User.get_or_create(
+                username=username,
+                defaults={
+                    "hashed_pwd": hashed_pwd,
+                    "permissions": permissions,
+                },
+            )
+        )[0]
+        users_count.set(await User.all().count())
+        return user
+
+    @staticmethod
+    async def get_user(username: str) -> User:
+        cache = caches.get("default")
+        user = await cache.get(username)
+        if user:
+            return user
+
+        user = await User.get_or_none(username=username)
+        if user is not None:
+            result = await cache.set(user.username, user, ttl=600)
+            assert result, "user being cached"
+
+        return user
+
+    @staticmethod
+    async def delete_users(usernames: list[str]):
+        await User.filter(username__in=usernames).delete()
+        for username in usernames:
+            await caches.get("default").delete(username)
+        users_count.set(await User.all().count())

goosebit/util/path.py

@@ -0,0 +1,42 @@
+from anyio import Path
+
+
+async def validate_filename(filename: str, temp_dir: Path) -> Path:
+    if not filename or not isinstance(filename, str):
+        raise ValueError("Filename must be a non-empty string")
+
+    filename = filename.strip()
+    if not filename:
+        raise ValueError("Filename cannot be empty or whitespace only")
+
+    # Check for dangerous patterns that could indicate path traversal
+    # This includes both Unix (..) and Windows (..\) style traversal
+    dangerous_patterns = ["../", "..\\", "../", "..\\"]
+    if any(pattern in filename for pattern in dangerous_patterns):
+        raise ValueError("Filename contains invalid path traversal components")
+
+    # Check for Windows drive letters (C:, D:, etc.)
+    if len(filename) >= 2 and filename[1] == ":" and filename[0].isalpha():
+        raise ValueError("Filename cannot contain Windows drive letters")
+
+    # Create a path from the filename
+    filename_path = Path(filename)
+
+    # Check if it's an absolute path
+    if filename_path.is_absolute():
+        raise ValueError("Filename cannot be an absolute path")
+
+    # Construct the full path within temp directory
+    file_path = temp_dir / filename_path
+
+    # Resolve both paths to check for path traversal
+    resolved_file = await file_path.resolve()
+    resolved_temp = await temp_dir.resolve()
+
+    # Ensure the resolved file path is within the temp directory
+    try:
+        resolved_file.relative_to(resolved_temp)
+    except ValueError:
+        raise ValueError("Filename contains invalid path traversal components")
+
+    return file_path