goosebit 0.1.2__py3-none-any.whl → 0.2.1__py3-none-any.whl

goosebit/updater/manager.py

@@ -6,20 +6,29 @@ from abc import ABC, abstractmethod
 from contextlib import asynccontextmanager
 from datetime import datetime
 from enum import StrEnum
-from typing import Callable, Optional
+from typing import Callable
 
-from aiocache import Cache, cached
-from aiocache.serializers import PickleSerializer
+from aiocache import cached, caches
 
-from goosebit.models import (
+from goosebit.db.models import (
     Device,
-    Firmware,
     Hardware,
     Rollout,
+    Software,
     UpdateModeEnum,
     UpdateStateEnum,
 )
-from goosebit.settings import POLL_TIME, POLL_TIME_UPDATING
+from goosebit.settings import config
+
+caches.set_config(
+    {
+        "default": {
+            "cache": "aiocache.SimpleMemoryCache",
+            "serializer": {"class": "aiocache.serializers.PickleSerializer"},
+            "ttl": 600,
+        },
+    }
+)
 
 
 class HandlingType(StrEnum):
@@ -41,7 +50,7 @@ class UpdateManager(ABC):
     async def update_force_update(self, force_update: bool) -> None:
         return
 
-    async def update_fw_version(self, version: str) -> None:
+    async def update_sw_version(self, version: str) -> None:
         return
 
     async def update_hardware(self, hardware: Hardware) -> None:
@@ -53,7 +62,7 @@ class UpdateManager(ABC):
     async def update_last_connection(self, last_seen: int, last_ip: str) -> None:
         return
 
-    async def update_update(self, update_mode: UpdateModeEnum, firmware: Firmware | None):
+    async def update_update(self, update_mode: UpdateModeEnum, software: Software | None):
         return
 
     async def update_name(self, name: str):
@@ -62,16 +71,13 @@ class UpdateManager(ABC):
     async def update_feed(self, feed: str):
         return
 
-    async def update_flavor(self, flavor: str):
-        return
-
     async def update_config_data(self, **kwargs):
         return
 
     async def update_log_complete(self, log_complete: bool):
         return
 
-    async def get_rollout(self) -> Optional[Rollout]:
+    async def get_rollout(self) -> Rollout | None:
         return None
 
     @asynccontextmanager
@@ -105,11 +111,11 @@ class UpdateManager(ABC):
 
     @property
     def poll_time(self):
-        return UpdateManager.device_poll_time.get(self.dev_id, POLL_TIME)
+        return UpdateManager.device_poll_time.get(self.dev_id, config.poll_time_default)
 
     @poll_time.setter
     def poll_time(self, value: str):
-        if not value == POLL_TIME:
+        if not value == config.poll_time_default:
             UpdateManager.device_poll_time[self.dev_id] = value
             return
         if self.dev_id in UpdateManager.device_poll_time:
@@ -120,7 +126,7 @@ class UpdateManager(ABC):
             await cb(log_data)
 
     @abstractmethod
-    async def get_update(self) -> tuple[HandlingType, Firmware]: ...
+    async def get_update(self) -> tuple[HandlingType, Software]: ...
 
     @abstractmethod
     async def update_log(self, log_data: str) -> None: ...
@@ -129,14 +135,14 @@ class UpdateManager(ABC):
 class UnknownUpdateManager(UpdateManager):
     def __init__(self, dev_id: str):
         super().__init__(dev_id)
-        self.poll_time = POLL_TIME_UPDATING
+        self.poll_time = config.poll_time_updating
 
-    async def _get_firmware(self) -> Firmware:
-        return await Firmware.latest(await self.get_device())
+    async def _get_software(self) -> Software:
+        return await Software.latest(await self.get_device())
 
-    async def get_update(self) -> tuple[HandlingType, Firmware]:
-        firmware = await self._get_firmware()
-        return HandlingType.FORCED, firmware
+    async def get_update(self) -> tuple[HandlingType, Software]:
+        software = await self._get_software()
+        return HandlingType.FORCED, software
 
     async def update_log(self, log_data: str) -> None:
         return
@@ -145,13 +151,7 @@ class UnknownUpdateManager(UpdateManager):
 class DeviceUpdateManager(UpdateManager):
     hardware_default = None
 
-    @cached(
-        ttl=600,
-        key_builder=lambda fn, self: self.dev_id,
-        cache=Cache.MEMORY,
-        serializer=PickleSerializer(),
-        namespace="main",
-    )
+    @cached(key_builder=lambda fn, self: self.dev_id, alias="default")
     async def get_device(self) -> Device:
         hardware = DeviceUpdateManager.hardware_default
         if hardware is None:
@@ -161,8 +161,8 @@ class DeviceUpdateManager(UpdateManager):
         return (await Device.get_or_create(uuid=self.dev_id, defaults={"hardware": hardware}))[0]
 
     async def save_device(self, device: Device, update_fields: list[str]):
-        cache = Cache(namespace="main")
-        await cache.set(self.dev_id, device, ttl=600)
+        result = await caches.get("default").set(self.dev_id, device, ttl=600)
+        assert result, "device being cached"
         await device.save(update_fields=update_fields)
 
     async def update_force_update(self, force_update: bool) -> None:
@@ -170,10 +170,10 @@ class DeviceUpdateManager(UpdateManager):
         device.force_update = force_update
         await self.save_device(device, update_fields=["force_update"])
 
-    async def update_fw_version(self, version: str) -> None:
+    async def update_sw_version(self, version: str) -> None:
         device = await self.get_device()
-        device.fw_version = version
-        await self.save_device(device, update_fields=["fw_version"])
+        device.sw_version = version
+        await self.save_device(device, update_fields=["sw_version"])
 
     async def update_hardware(self, hardware: Hardware) -> None:
         device = await self.get_device()
@@ -195,11 +195,11 @@ class DeviceUpdateManager(UpdateManager):
             device.last_ip = last_ip
             await self.save_device(device, update_fields=["last_seen", "last_ip"])
 
-    async def update_update(self, update_mode: UpdateModeEnum, firmware: Firmware | None):
+    async def update_update(self, update_mode: UpdateModeEnum, software: Software | None):
         device = await self.get_device()
-        device.assigned_firmware = firmware
+        device.assigned_software = software
         device.update_mode = update_mode
-        await self.save_device(device, update_fields=["assigned_firmware_id", "update_mode"])
+        await self.save_device(device, update_fields=["assigned_software_id", "update_mode"])
 
     async def update_name(self, name: str):
         device = await self.get_device()
@@ -211,14 +211,11 @@ class DeviceUpdateManager(UpdateManager):
         device.feed = feed
         await self.save_device(device, update_fields=["feed"])
 
-    async def update_flavor(self, flavor: str):
-        device = await self.get_device()
-        device.flavor = flavor
-        await self.save_device(device, update_fields=["flavor"])
-
     async def update_config_data(self, **kwargs):
-        model = kwargs.get("hw_model") or "default"
+        model = kwargs.get("hw_boardname") or "default"
         revision = kwargs.get("hw_revision") or "default"
+        sw_version = kwargs.get("sw_version")
+
         hardware = (await Hardware.get_or_create(model=model, revision=revision))[0]
         device = await self.get_device()
         modified = False
@@ -231,76 +228,79 @@ class DeviceUpdateManager(UpdateManager):
             device.last_state = UpdateStateEnum.REGISTERED
             modified = True
 
+        if device.sw_version != sw_version:
+            device.sw_version = sw_version
+            modified = True
+
         if modified:
-            await self.save_device(device, update_fields=["hardware_id", "last_state"])
+            await self.save_device(device, update_fields=["hardware_id", "last_state", "sw_version"])
 
     async def update_log_complete(self, log_complete: bool):
         device = await self.get_device()
         device.log_complete = log_complete
         await self.save_device(device, update_fields=["log_complete"])
 
-    async def get_rollout(self) -> Optional[Rollout]:
+    async def get_rollout(self) -> Rollout | None:
         device = await self.get_device()
 
         if device.update_mode == UpdateModeEnum.ROLLOUT:
             return (
                 await Rollout.filter(
                     feed=device.feed,
-                    flavor=device.flavor,
                     paused=False,
-                    firmware__compatibility__devices__uuid=device.uuid,
+                    software__compatibility__devices__uuid=device.uuid,
                 )
                 .order_by("-created_at")
                 .first()
-                .prefetch_related("firmware")
+                .prefetch_related("software")
             )
 
         return None
 
-    async def _get_firmware(self) -> Firmware | None:
+    async def _get_software(self) -> Software | None:
         device = await self.get_device()
 
         if device.update_mode == UpdateModeEnum.ROLLOUT:
             rollout = await self.get_rollout()
             if not rollout or rollout.paused:
                 return None
-            await rollout.fetch_related("firmware")
-            return rollout.firmware
+            await rollout.fetch_related("software")
+            return rollout.software
         if device.update_mode == UpdateModeEnum.ASSIGNED:
-            await device.fetch_related("assigned_firmware")
-            return device.assigned_firmware
+            await device.fetch_related("assigned_software")
+            return device.assigned_software
 
         if device.update_mode == UpdateModeEnum.LATEST:
-            return await Firmware.latest(device)
+            return await Software.latest(device)
 
         assert device.update_mode == UpdateModeEnum.PINNED
         return None
 
-    async def get_update(self) -> tuple[HandlingType, Firmware]:
+    async def get_update(self) -> tuple[HandlingType, Software]:
         device = await self.get_device()
-        firmware = await self._get_firmware()
+        software = await self._get_software()
 
-        if firmware is None:
+        if software is None:
             handling_type = HandlingType.SKIP
-            self.poll_time = POLL_TIME
+            self.poll_time = config.poll_time_default
 
-        elif firmware.version == device.fw_version and not device.force_update:
+        elif software.version == device.sw_version and not device.force_update:
             handling_type = HandlingType.SKIP
-            self.poll_time = POLL_TIME
+            self.poll_time = config.poll_time_default
 
         elif device.last_state == UpdateStateEnum.ERROR and not device.force_update:
             handling_type = HandlingType.SKIP
-            self.poll_time = POLL_TIME
+            self.poll_time = config.poll_time_default
 
         else:
             handling_type = HandlingType.FORCED
-            self.poll_time = POLL_TIME_UPDATING
+            self.poll_time = config.poll_time_updating
 
             if device.log_complete:
                 await self.update_log_complete(False)
                 await self.clear_log()
 
-        return handling_type, firmware
+        return handling_type, software
 
     async def update_log(self, log_data: str) -> None:
         if log_data is None:
@@ -343,7 +343,7 @@ async def get_update_manager(dev_id: str) -> UpdateManager:
 
 
 async def delete_devices(ids: list[str]):
-    await Device.filter(id__in=ids).delete()
-    cache = Cache(namespace="main")
+    await Device.filter(uuid__in=ids).delete()
     for dev_id in ids:
-        await cache.delete(dev_id)
+        result = await caches.get("default").delete(dev_id)
+        assert result == 1, "device has been cached"

goosebit/updater/routes.py

@@ -1,20 +1,12 @@
 import time
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends
 from fastapi.requests import Request
 
-from goosebit.settings import TENANT
-
 from . import controller
 from .manager import get_update_manager
 
 
-async def verify_tenant(tenant: str):
-    if not tenant == TENANT:
-        raise HTTPException(404)
-    return tenant
-
-
 async def log_last_connection(request: Request, dev_id: str):
     host = request.client.host
     updater = await get_update_manager(dev_id)
@@ -22,8 +14,8 @@ async def log_last_connection(request: Request, dev_id: str):
 
 
 router = APIRouter(
-    prefix="/{tenant}",
-    dependencies=[Depends(verify_tenant), Depends(log_last_connection)],
+    prefix="/ddi",
+    dependencies=[Depends(log_last_connection)],
     tags=["ddi"],
 )
 router.include_router(controller.router)

goosebit/updates/__init__.py

@@ -1,68 +1,127 @@
+import shutil
 from pathlib import Path
 from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
 
+from fastapi import HTTPException
 from fastapi.requests import Request
+from tortoise.expressions import Q
 
-from goosebit.models import Firmware, Hardware
+from goosebit.db.models import Hardware, Software
+from goosebit.updater.manager import UpdateManager
 
 from . import swdesc
 
 
-async def create_firmware_update(uri: str):
+async def create_software_update(uri: str, temp_file: Path | None) -> Software:
     parsed_uri = urlparse(uri)
 
+    # parse swu header into update_info
     if parsed_uri.scheme == "file":
-        file_path = Path(url2pathname(unquote(parsed_uri.path)))
-        update_info = await swdesc.parse_file(file_path)
-        if update_info is None:
-            return
+        try:
+            update_info = await swdesc.parse_file(temp_file)
+        except Exception:
+            raise HTTPException(422, "Software swu header cannot be parsed")
+
     elif parsed_uri.scheme.startswith("http"):
-        update_info = await swdesc.parse_remote(uri)
-        if update_info is None:
-            return
+        try:
+            update_info = await swdesc.parse_remote(uri)
+        except Exception:
+            raise HTTPException(422, "Software swu header cannot be parsed")
+
     else:
-        return
-
-    firmware = (
-        await Firmware.get_or_create(
-            uri=uri,
-            version=str(update_info["version"]),
-            size=update_info["size"],
-            hash=update_info["hash"],
-        )
-    )[0]
+        raise HTTPException(422, "Software URI protocol unknown")
+
+    if update_info is None:
+        raise HTTPException(422, "Software swu header contains invalid data")
+
+    # check for collisions
+    is_colliding = await _is_software_colliding(update_info)
+    if is_colliding:
+        raise HTTPException(409, "Software with same version and overlapping compatibility already exists")
 
+    # for local file: rename temp file to final name
+    if parsed_uri.scheme == "file":
+        path = _unique_path(parsed_uri)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy(temp_file, path)
+        uri = path.absolute().as_uri()
+
+    # create software
+    software = await Software.create(
+        uri=uri,
+        version=str(update_info["version"]),
+        size=update_info["size"],
+        hash=update_info["hash"],
+    )
+
+    # create compatibility information
     for comp in update_info["compatibility"]:
         model = comp.get("hw_model", "default")
         revision = comp.get("hw_revision", "default")
-        await firmware.compatibility.add((await Hardware.get_or_create(model=model, revision=revision))[0])
-    await firmware.save()
-    return firmware
+        await software.compatibility.add((await Hardware.get_or_create(model=model, revision=revision))[0])
+    await software.save()
+    return software
+
+
+async def _is_software_colliding(update_info):
+    version = update_info["version"]
+    compatibility = update_info["compatibility"]
+
+    # Extract hardware model and revision from compatibility info
+    hw_model_revisions = [(hw["hw_model"], hw["hw_revision"]) for hw in compatibility]
+
+    # Build the query for hardware matching the provided model and revision
+    hardware_query = Q()
+    for hw_model, hw_revision in hw_model_revisions:
+        hardware_query |= Q(model=hw_model, revision=hw_revision)
+
+    # Find the hardware IDs matching the provided model and revision
+    hardware_ids = await Hardware.filter(hardware_query).values_list("id", flat=True)
+
+    # Check if any existing software with the same version is compatible with any of these hardware IDs
+    is_colliding = await Software.filter(version=version, compatibility__in=hardware_ids).exists()
+
+    return is_colliding
+
+
+def _unique_path(uri):
+    path = Path(url2pathname(unquote(uri.path)))
+    if not path.exists():
+        return path
+
+    counter = 1
+    new_path = path.with_name(f"{path.stem}-{counter}{path.suffix}")
+    while new_path.exists():
+        counter += 1
+        new_path = path.with_name(f"{path.stem}-{counter}{path.suffix}")
+
+    return new_path
 
 
-def generate_chunk(request: Request, firmware: Firmware | None) -> list:
-    if firmware is None:
+async def generate_chunk(request: Request, updater: UpdateManager) -> list:
+    _, software = await updater.get_update()
+    if software is None:
         return []
-    if firmware.local:
+    if software.local:
         href = str(
             request.url_for(
-                "download_file",
-                file_id=firmware.id,
+                "download_artifact",
+                dev_id=updater.dev_id,
             )
         )
     else:
-        href = firmware.uri
+        href = software.uri
     return [
         {
             "part": "os",
             "version": "1",
-            "name": firmware.path.name,
+            "name": software.path.name,
             "artifacts": [
                 {
-                    "filename": firmware.path.name,
-                    "hashes": {"sha1": firmware.hash},
-                    "size": firmware.size,
+                    "filename": software.path.name,
+                    "hashes": {"sha1": software.hash},
+                    "size": software.size,
                     "_links": {"download": {"href": href}},
                 }
             ],

goosebit/updates/swdesc.py

@@ -8,8 +8,6 @@ import httpx
 import libconf
 import semver
 
-from goosebit.settings import UPDATES_DIR
-
 logger = logging.getLogger(__name__)
 
 
@@ -69,12 +67,9 @@ async def parse_file(file: Path):
 async def parse_remote(url: str):
     async with httpx.AsyncClient() as c:
         file = await c.get(url)
-        temp_file = UPDATES_DIR.joinpath("temp.swu")
-        async with aiofiles.open(temp_file, "w+b") as f:
+        async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f:
             await f.write(file.content)
-    parsed_file = await parse_file(temp_file)
-    temp_file.unlink()
-    return parsed_file
+            return await parse_file(Path(f.name))
 
 
 def _sha1_hash_file(file_path: Path):

goosebit-0.2.1.dist-info/METADATA

@@ -0,0 +1,173 @@
+Metadata-Version: 2.1
+Name: goosebit
+Version: 0.2.1
+Summary: 
+Author: Upstream Data
+Author-email: brett@upstreamdata.ca
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: postgresql
+Requires-Dist: aerich (>=0.7.2,<0.8.0)
+Requires-Dist: aiocache (>=0.12.2,<0.13.0)
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
+Requires-Dist: argon2-cffi (>=23.1.0,<24.0.0)
+Requires-Dist: asyncpg (>=0.29.0,<0.30.0) ; extra == "postgresql"
+Requires-Dist: fastapi[uvicorn] (>=0.111.0,<0.112.0)
+Requires-Dist: httpx (>=0.27.0,<0.28.0)
+Requires-Dist: itsdangerous (>=2.2.0,<3.0.0)
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: joserfc (>=1.0.0,<2.0.0)
+Requires-Dist: libconf (>=2.0.1,<3.0.0)
+Requires-Dist: opentelemetry-distro (>=0.47b0,<0.48)
+Requires-Dist: opentelemetry-exporter-prometheus (>=0.47b0,<0.48)
+Requires-Dist: opentelemetry-instrumentation-fastapi (>=0.47b0,<0.48)
+Requires-Dist: pydantic-settings (>=2.4.0,<3.0.0)
+Requires-Dist: python-multipart (>=0.0.9,<0.0.10)
+Requires-Dist: semver (>=3.0.2,<4.0.0)
+Requires-Dist: tortoise-orm (>=0.21.4,<0.22.0)
+Requires-Dist: websockets (>=12.0,<13.0)
+Description-Content-Type: text/markdown
+
+# gooseBit
+
+<img src="docs/img/goosebit-logo.png" style="width: 100px; height: 100px; display: block;">
+
+---
+
+A simplistic, opinionated remote update server implementing hawkBit™'s [DDI API](https://eclipse.dev/hawkbit/apis/ddi_api/).
+
+## Quick Start
+
+### Installation
+
+1. Install dependencies using [Poetry](https://python-poetry.org/):
+    ```bash
+    poetry install
+    ```
+2. Launch gooseBit:
+    ```bash
+    python main.py
+    ```
+
+### Initial Configuration
+
+Before running gooseBit for the first time, update the default credentials in `settings.yaml`. The default login for testing purposes is:
+
+-   **Username:** `admin@goosebit.local`
+-   **Password:** `admin`
+
+## Assumptions
+
+-   Devices use [SWUpdate](https://swupdate.org) for managing software updates.
+
+## Features
+
+### Device Registry
+
+When a device connects to gooseBit for the first time, it is automatically added to the device registry. The server will then request the device's configuration data, including:
+
+-   `hw_model` and `hw_revision`: Used to match compatible software.
+-   `sw_version`: Indicates the currently installed software version.
+
+The registry tracks each device's status, including the last online timestamp, installed software version, update state, and more.
+
+### Software Repository
+
+Software packages (`*.swu` files) can be hosted directly on the gooseBit server or on an external server. gooseBit parses the software metadata to determine compatibility with specific hardware models and revisions.
+
+### Device Update Modes
+
+Devices can be configured with different update modes. The default mode is `Rollout`.
+
+#### 1. Manual Update to Specified Software
+
+Assign specific software to a device manually. Once installed, no further updates will be triggered.
+
+#### 2. Automatic Update to Latest Software
+
+Automatically updates the device to the latest compatible software, based on the reported `hw_model` and `hw_revision`. Note: versions are interpreted as [SemVer](https://semver.org) versions.
+
+#### 3. Software Rollout
+
+Rollouts target all devices with a specified "feed" value, ensuring that the assigned software is installed on all matching devices. Rollouts also track success and error rates, with future plans for automatic aborts. If multiple rollouts exist for the same feed, the most recent rollout takes precedence.
+
+### Pause Updates
+
+Devices can be pinned to their current software version, preventing any updates from being applied.
+
+### Real-time Update Logs
+
+While updates are in progress, gooseBit captures real-time logs, which are accessible through the device repository.
+
+## Development
+
+### Database
+
+Create or upgrade database
+
+```bash
+poetry run aerich upgrade
+```
+
+After a model change create the migration
+
+```bash
+poetry run aerich migrate
+```
+
+### Code formatting and linting
+
+Code is formatted using different tools
+
+-   black and isort for `*.py`
+-   biomejs for `*.js`, `*.json`
+-   prettier for `*.html`, `*.md`, `*.yml`, `*.yaml`
+
+Code is linted using different tools as well
+
+-   flake8 for `*.py`
+-   biomejs for `*.js`
+
+Best to have pre-commit install git hooks that run all those tools before a commit:
+
+```bash
+poetry run pre-commit install
+```
+
+To manually apply the hooks to all files use:
+
+```bash
+pre-commit run --all-files
+```
+
+### Testing
+
+Tests are implemented using pytest. To run all tests
+
+```bash
+poetry run pytest
+```
+
+### Structure
+
+The structure of gooseBit is as follows:
+
+-   `api`: Files for the API.
+-   `ui`: Files for the UI.
+    -   `bff`: Backend for frontend API.
+    -   `static`: Static files.
+    -   `templates`: Jinja2 formatted templates.
+    -   `nav`: Navbar handler.
+-   `updater`: DDI API handler and device update manager.
+-   `updates`: SWUpdate file parsing.
+-   `realtime`: Realtime API functionality with websockets.
+-   `auth`: Authentication functions and permission handling.
+-   `models`: Database models.
+-   `db`: Database config and initialization.
+-   `schema`: Pydantic models used for API type hinting.
+-   `settings`: Settings loader and handler.
+-   `telemetry`: Telemetry data handlers.
+-   `routes`: Routes for a giving endpoint, including the router.
+