goosebit 0.2.4__tar.gz → 0.2.6__tar.gz

goosebit-0.2.6/PKG-INFO

@@ -0,0 +1,280 @@
+Metadata-Version: 2.3
+Name: goosebit
+Version: 0.2.6
+Summary: 
+Author: Brett Rowan
+Author-email: 121075405+b-rowan@users.noreply.github.com
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: postgresql
+Requires-Dist: aerich (>=0.9.1,<0.10.0)
+Requires-Dist: aiocache (>=0.12.3,<0.13.0)
+Requires-Dist: argon2-cffi (>=25.1.0,<26.0.0)
+Requires-Dist: asyncpg (>=0.30.0,<0.31.0) ; extra == "postgresql"
+Requires-Dist: boto3 (>=1.40.8,<2.0.0)
+Requires-Dist: fastapi (>=0.116.1,<0.117.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: itsdangerous (>=2.2.0,<3.0.0)
+Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
+Requires-Dist: joserfc (>=1.2.2,<2.0.0)
+Requires-Dist: libconf (>=2.0.1,<3.0.0)
+Requires-Dist: opentelemetry-distro (>=0.57b0,<0.58)
+Requires-Dist: opentelemetry-exporter-prometheus (>=0.57b0,<0.58)
+Requires-Dist: opentelemetry-instrumentation-fastapi (>=0.57b0,<0.58)
+Requires-Dist: pydantic-settings[yaml] (>=2.10.1,<3.0.0)
+Requires-Dist: python-multipart (>=0.0.20,<0.0.21)
+Requires-Dist: semver (>=3.0.4,<4.0.0)
+Requires-Dist: tortoise-orm (>=0.25.1,<0.26.0)
+Requires-Dist: uvicorn (>=0.35.0,<0.36.0)
+Description-Content-Type: text/markdown
+
+# gooseBit
+
+<img src="https://upstreamdatainc.github.io/goosebit/img/goosebit-logo.png" style="width: 100px; height: 100px; display: block;">
+
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/UpstreamDataInc/goosebit/badge)](https://scorecard.dev/viewer/?uri=github.com/UpstreamDataInc/goosebit)
+
+---
+
+A simplistic, opinionated remote update server implementing hawkBit™'s [DDI API](https://eclipse.dev/hawkbit/apis/ddi_api/).
+
+## Deployment
+
+### Docker Compose Demo
+
+The Docker Compose demo [docker/demo/docker-compose.yml] may serve as inspiration for a containerized (cloud) deployment.
+It uses PostgreSQL as the database and NGINX as a reverse proxy.
+
+> [!WARNING]
+> Do not use the demo (as-is) in production!
+
+Make sure you have [Docker](https://www.docker.com/get-started/) (and Docker Compose) installed.
+Then run:
+
+```txt
+docker compose -f docker/demo/docker-compose.yml up
+```
+
+Visit gooseBit at: https://localhost
+
+[docker/demo/docker-compose.yml]: https://github.com/UpstreamDataInc/goosebit/blob/master/docker/docker-compose-dev.yml
+
+### Configuration
+
+gooseBit can be configured through a configuration file (`/etc/goosebit.yaml`) or by setting environment variables.
+For the available options and their defaults, see [goosebit.yaml].
+The environment variable corresponding to e.g. the `poll_time` YAML setting would be `GOOSEBIT_POLL_TIME`.
+Environment variables for nested settings are constructed using `__` as the separator:
+
+```txt
+GOOSEBIT_DEVICE_AUTH__ENABLE=true
+```
+
+Alternatively, a JSON string can be assigned:
+
+```txt
+GOOSEBIT_DEVICE_AUTH='{"enable": true}'
+```
+
+[goosebit.yaml]: https://github.com/UpstreamDataInc/goosebit/blob/master/goosebit.yaml
+
+### Database
+
+By default, SQLite is used as the database. For more sophisticated setups, PostgreSQL is supported.
+To use PostgreSQL, set `db_uri` or the `GOSSEBIT_DB_URI` environment variable to something like:
+
+```txt
+postgres://user:password@host:5432/db_name
+```
+
+### Artifact Storage
+
+The software packages managed by gooseBit are either stored on the local filesystem (`artifacts_dir` setting) or an S3-compatible object storage.
+
+## Assumptions
+
+- Devices use [SWUpdate](https://swupdate.org) for managing software updates.
+
+## Features
+
+### Device Registry
+
+When a device connects to gooseBit for the first time, it is automatically added to the device registry. The server will then request the device's configuration data, including:
+
+- `hw_model` and `hw_revision`: Used to match compatible software.
+- `sw_version`: Indicates the currently installed software version.
+
+The registry tracks each device's status, including the last online timestamp, installed software version, update state, and more.
+
+### Software Repository
+
+Software packages (`*.swu` files) can be hosted directly on the gooseBit server or on an external server. gooseBit parses the software metadata to determine compatibility with specific hardware models and revisions.
+
+### Device Update Modes
+
+Devices can be configured with different update modes. The default mode is `Rollout`.
+
+#### 1. Manual Update to Specified Software
+
+Assign specific software to a device manually. Once installed, no further updates will be triggered.
+
+#### 2. Automatic Update to Latest Software
+
+Automatically updates the device to the latest compatible software, based on the reported `hw_model` and `hw_revision`. Note: versions are interpreted as [SemVer](https://semver.org) versions.
+
+#### 3. Software Rollout
+
+Rollouts target all devices with a specified "feed" value, ensuring that the assigned software is installed on all matching devices. Rollouts also track success and error rates, with future plans for automatic aborts. If multiple rollouts exist for the same feed, the most recent rollout takes precedence.
+
+### Pause Updates
+
+Devices can be pinned to their current software version, preventing any updates from being applied.
+
+### Real-time Update Logs
+
+While updates are in progress, gooseBit captures real-time logs, which are accessible through the device repository.
+
+## Development with Poetry
+
+### Initial Setup
+
+Install Poetry as described [here](https://python-poetry.org/docs/#installation).
+
+Then, to install gooseBit's dependencies, run:
+
+```txt
+poetry install
+```
+
+Initialize the database:
+
+```txt
+poetry run aerich upgrade
+```
+
+Launch gooseBit:
+
+```txt
+poetry run python -m goosebit
+```
+
+The service is now available at: http://localhost:60053
+
+### Database
+
+Initialize or migrate database:
+
+```txt
+poetry run aerich upgrade
+```
+
+After a model change create the migration:
+
+```txt
+poetry run aerich migrate
+```
+
+To seed some sample data (attention: drops all current data) use:
+
+```txt
+poetry run generate-sample-data
+```
+
+### Code formatting and linting
+
+Code is formatted using different tools
+
+- black and isort for `*.py`
+- biomejs for `*.js`, `*.json`
+- prettier for `*.html`, `*.md`, `*.yml`, `*.yaml`
+
+Code is linted using different tools as well
+
+- flake8 for `*.py`
+- biomejs for `*.js`
+
+Best to have pre-commit install git hooks that run all those tools before a commit:
+
+```txt
+poetry run pre-commit install
+```
+
+To manually apply the hooks to all files use:
+
+```txt
+poetry run pre-commit run --all-files
+```
+
+### Testing
+
+Tests are implemented using pytest. You can run all the tests with:
+
+```txt
+poetry run pytest
+```
+
+To run only the unit tests:
+
+```txt
+poetry run pytest tests/unit
+```
+
+To run only the end-to-end tests:
+
+```txt
+poetry run pytest tests/e2e
+```
+
+## Development with Docker (and PostgreSQL)
+
+### Running the Containers
+
+```txt
+docker compose -f docker/docker-compose-dev.yml up --build
+```
+
+### Applying the Migrations
+
+```txt
+docker exec goosebit-dev python -m aerich upgrade
+```
+
+### Using the Interactive Debugger
+
+You might need [rlwrap](https://github.com/hanslub42/rlwrap) to fix readline support.
+
+Place `breakpoint()` before the code you want to debug. The server will reload automatically.
+Then, connect to remote PDB (when the breakpoint has been hit):
+
+```txt
+rlwrap telnet localhost 4444
+```
+
+To exit the debugger, press `Ctrl + ]` and then `q`.
+
+## Architecture
+
+### Structure
+
+The structure of gooseBit is as follows:
+
+- `api`: Files for the API.
+- `ui`: Files for the UI.
+    - `bff`: Backend for frontend API.
+    - `static`: Static files.
+    - `templates`: Jinja2 formatted templates.
+    - `nav`: Navbar handler.
+- `updater`: DDI API handler and device update manager.
+- `updates`: SWUpdate file parsing.
+- `auth`: Authentication functions and permission handling.
+- `models`: Database models.
+- `db`: Database config and initialization.
+- `schema`: Pydantic models used for API type hinting.
+- `settings`: Settings loader and handler.
+- `storage`: Storage for software artifacts.
+- `telemetry`: Telemetry data handlers.
+- `routes`: Routes for a giving endpoint, including the router.
+

goosebit-0.2.6/README.md

@@ -0,0 +1,246 @@
+# gooseBit
+
+<img src="https://upstreamdatainc.github.io/goosebit/img/goosebit-logo.png" style="width: 100px; height: 100px; display: block;">
+
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/UpstreamDataInc/goosebit/badge)](https://scorecard.dev/viewer/?uri=github.com/UpstreamDataInc/goosebit)
+
+---
+
+A simplistic, opinionated remote update server implementing hawkBit™'s [DDI API](https://eclipse.dev/hawkbit/apis/ddi_api/).
+
+## Deployment
+
+### Docker Compose Demo
+
+The Docker Compose demo [docker/demo/docker-compose.yml] may serve as inspiration for a containerized (cloud) deployment.
+It uses PostgreSQL as the database and NGINX as a reverse proxy.
+
+> [!WARNING]
+> Do not use the demo (as-is) in production!
+
+Make sure you have [Docker](https://www.docker.com/get-started/) (and Docker Compose) installed.
+Then run:
+
+```txt
+docker compose -f docker/demo/docker-compose.yml up
+```
+
+Visit gooseBit at: https://localhost
+
+[docker/demo/docker-compose.yml]: https://github.com/UpstreamDataInc/goosebit/blob/master/docker/docker-compose-dev.yml
+
+### Configuration
+
+gooseBit can be configured through a configuration file (`/etc/goosebit.yaml`) or by setting environment variables.
+For the available options and their defaults, see [goosebit.yaml].
+The environment variable corresponding to e.g. the `poll_time` YAML setting would be `GOOSEBIT_POLL_TIME`.
+Environment variables for nested settings are constructed using `__` as the separator:
+
+```txt
+GOOSEBIT_DEVICE_AUTH__ENABLE=true
+```
+
+Alternatively, a JSON string can be assigned:
+
+```txt
+GOOSEBIT_DEVICE_AUTH='{"enable": true}'
+```
+
+[goosebit.yaml]: https://github.com/UpstreamDataInc/goosebit/blob/master/goosebit.yaml
+
+### Database
+
+By default, SQLite is used as the database. For more sophisticated setups, PostgreSQL is supported.
+To use PostgreSQL, set `db_uri` or the `GOSSEBIT_DB_URI` environment variable to something like:
+
+```txt
+postgres://user:password@host:5432/db_name
+```
+
+### Artifact Storage
+
+The software packages managed by gooseBit are either stored on the local filesystem (`artifacts_dir` setting) or an S3-compatible object storage.
+
+## Assumptions
+
+- Devices use [SWUpdate](https://swupdate.org) for managing software updates.
+
+## Features
+
+### Device Registry
+
+When a device connects to gooseBit for the first time, it is automatically added to the device registry. The server will then request the device's configuration data, including:
+
+- `hw_model` and `hw_revision`: Used to match compatible software.
+- `sw_version`: Indicates the currently installed software version.
+
+The registry tracks each device's status, including the last online timestamp, installed software version, update state, and more.
+
+### Software Repository
+
+Software packages (`*.swu` files) can be hosted directly on the gooseBit server or on an external server. gooseBit parses the software metadata to determine compatibility with specific hardware models and revisions.
+
+### Device Update Modes
+
+Devices can be configured with different update modes. The default mode is `Rollout`.
+
+#### 1. Manual Update to Specified Software
+
+Assign specific software to a device manually. Once installed, no further updates will be triggered.
+
+#### 2. Automatic Update to Latest Software
+
+Automatically updates the device to the latest compatible software, based on the reported `hw_model` and `hw_revision`. Note: versions are interpreted as [SemVer](https://semver.org) versions.
+
+#### 3. Software Rollout
+
+Rollouts target all devices with a specified "feed" value, ensuring that the assigned software is installed on all matching devices. Rollouts also track success and error rates, with future plans for automatic aborts. If multiple rollouts exist for the same feed, the most recent rollout takes precedence.
+
+### Pause Updates
+
+Devices can be pinned to their current software version, preventing any updates from being applied.
+
+### Real-time Update Logs
+
+While updates are in progress, gooseBit captures real-time logs, which are accessible through the device repository.
+
+## Development with Poetry
+
+### Initial Setup
+
+Install Poetry as described [here](https://python-poetry.org/docs/#installation).
+
+Then, to install gooseBit's dependencies, run:
+
+```txt
+poetry install
+```
+
+Initialize the database:
+
+```txt
+poetry run aerich upgrade
+```
+
+Launch gooseBit:
+
+```txt
+poetry run python -m goosebit
+```
+
+The service is now available at: http://localhost:60053
+
+### Database
+
+Initialize or migrate database:
+
+```txt
+poetry run aerich upgrade
+```
+
+After a model change create the migration:
+
+```txt
+poetry run aerich migrate
+```
+
+To seed some sample data (attention: drops all current data) use:
+
+```txt
+poetry run generate-sample-data
+```
+
+### Code formatting and linting
+
+Code is formatted using different tools
+
+- black and isort for `*.py`
+- biomejs for `*.js`, `*.json`
+- prettier for `*.html`, `*.md`, `*.yml`, `*.yaml`
+
+Code is linted using different tools as well
+
+- flake8 for `*.py`
+- biomejs for `*.js`
+
+Best to have pre-commit install git hooks that run all those tools before a commit:
+
+```txt
+poetry run pre-commit install
+```
+
+To manually apply the hooks to all files use:
+
+```txt
+poetry run pre-commit run --all-files
+```
+
+### Testing
+
+Tests are implemented using pytest. You can run all the tests with:
+
+```txt
+poetry run pytest
+```
+
+To run only the unit tests:
+
+```txt
+poetry run pytest tests/unit
+```
+
+To run only the end-to-end tests:
+
+```txt
+poetry run pytest tests/e2e
+```
+
+## Development with Docker (and PostgreSQL)
+
+### Running the Containers
+
+```txt
+docker compose -f docker/docker-compose-dev.yml up --build
+```
+
+### Applying the Migrations
+
+```txt
+docker exec goosebit-dev python -m aerich upgrade
+```
+
+### Using the Interactive Debugger
+
+You might need [rlwrap](https://github.com/hanslub42/rlwrap) to fix readline support.
+
+Place `breakpoint()` before the code you want to debug. The server will reload automatically.
+Then, connect to remote PDB (when the breakpoint has been hit):
+
+```txt
+rlwrap telnet localhost 4444
+```
+
+To exit the debugger, press `Ctrl + ]` and then `q`.
+
+## Architecture
+
+### Structure
+
+The structure of gooseBit is as follows:
+
+- `api`: Files for the API.
+- `ui`: Files for the UI.
+    - `bff`: Backend for frontend API.
+    - `static`: Static files.
+    - `templates`: Jinja2 formatted templates.
+    - `nav`: Navbar handler.
+- `updater`: DDI API handler and device update manager.
+- `updates`: SWUpdate file parsing.
+- `auth`: Authentication functions and permission handling.
+- `models`: Database models.
+- `db`: Database config and initialization.
+- `schema`: Pydantic models used for API type hinting.
+- `settings`: Settings loader and handler.
+- `storage`: Storage for software artifacts.
+- `telemetry`: Telemetry data handlers.
+- `routes`: Routes for a giving endpoint, including the router.

{goosebit-0.2.4 → goosebit-0.2.6}/goosebit/__init__.py

@@ -4,19 +4,23 @@ from logging import getLogger
 from typing import Annotated
 
 from fastapi import Depends, FastAPI, HTTPException
+from fastapi.exception_handlers import http_exception_handler
 from fastapi.openapi.docs import get_swagger_ui_html
 from fastapi.requests import Request
 from fastapi.responses import RedirectResponse
 from fastapi.security import OAuth2PasswordRequestForm
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as Instrumentor
+from starlette.exceptions import HTTPException as StarletteHTTPException
 from tortoise.exceptions import ValidationError
 
-from goosebit import api, db, realtime, ui, updater
-from goosebit.api.telemetry import metrics
+from goosebit import api, db, plugins, ui, updater
 from goosebit.auth import get_user_from_request, login_user, redirect_if_authenticated
+from goosebit.device_manager import DeviceManager
+from goosebit.settings import PWD_CXT, config
 from goosebit.ui.nav import nav
 from goosebit.ui.static import static
 from goosebit.ui.templates import templates
+from goosebit.users import create_initial_user
 
 logger = getLogger(__name__)
 
@@ -26,7 +30,9 @@ async def lifespan(_: FastAPI):
     db_ready = await db.init()
     if not db_ready:
         logger.exception("DB does not exist, try running `poetry run aerich upgrade`.")
-    await metrics.init()
+
+    logger.debug(f"Initialized storage backend: {config.storage.backend}")
+
     if db_ready:
         yield
     await db.close()
@@ -54,10 +60,30 @@ app = FastAPI(
 app.include_router(updater.router)
 app.include_router(ui.router)
 app.include_router(api.router)
-app.include_router(realtime.router)
 app.mount("/static", static, name="static")
 Instrumentor.instrument_app(app)
 
+for plugin in plugins.load():
+    if plugin.middleware is not None:
+        logger.info(f"Adding middleware for plugin: {plugin.name}")
+        app.add_middleware(plugin.middleware)
+    if plugin.router is not None:
+        logger.info(f"Adding routing handler for plugin: {plugin.name}")
+        app.include_router(router=plugin.router, prefix=plugin.url_prefix)
+    if plugin.db_model_path is not None:
+        logger.info(f"Adding db handler for plugin: {plugin.name}")
+        db.config.add_models(plugin.db_model_path)
+    if plugin.static_files is not None:
+        logger.info(f"Adding static files handler for plugin: {plugin.name}")
+        app.mount(f"{plugin.url_prefix}/static", plugin.static_files, name=plugin.static_files_name)
+    if plugin.templates is not None:
+        logger.info(f"Adding template handler for plugin: {plugin.name}")
+        templates.add_template_handler(plugin.templates)
+    if plugin.update_source_hook is not None:
+        DeviceManager.add_update_source(plugin.update_source_hook)
+    if plugin.config_data_hook is not None:
+        DeviceManager.add_config_callback(plugin.config_data_hook)
+
 
 # Custom exception handler for Tortoise ValidationError
 @app.exception_handler(ValidationError)
@@ -65,6 +91,13 @@ async def tortoise_validation_exception_handler(request: Request, exc: Validatio
     raise HTTPException(422, str(exc))
 
 
+# Extend default handler to do logging
+@app.exception_handler(StarletteHTTPException)
+async def custom_http_exception_handler(request, exc):
+    logger.warning(f"HTTPException, request={request.url}, status={exc.status_code}, detail={exc.detail}")
+    return await http_exception_handler(request, exc)
+
+
 @app.middleware("http")
 async def attach_user(request: Request, call_next):
     request.scope["user"] = await get_user_from_request(request)
@@ -77,6 +110,12 @@ async def attach_nav(request: Request, call_next):
     return await call_next(request)
 
 
+@app.middleware("http")
+async def attach_config(request: Request, call_next):
+    request.scope["config"] = config
+    return await call_next(request)
+
+
 @app.get("/", include_in_schema=False)
 def root_redirect(request: Request):
     return RedirectResponse(request.url_for("ui_root"))
@@ -89,7 +128,18 @@ async def login_get(request: Request):
 
 @app.post("/login", tags=["login"])
 async def login_post(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]):
-    return {"access_token": login_user(form_data.username, form_data.password), "token_type": "bearer"}
+    return {"access_token": await login_user(form_data.username, form_data.password), "token_type": "bearer"}
+
+
+@app.get("/setup", include_in_schema=False)
+async def setup_get(request: Request):
+    return templates.TemplateResponse(request, "setup.html.jinja")
+
+
+@app.post("/setup", include_in_schema=False)
+async def setup_post(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]):
+    await create_initial_user(form_data.username, PWD_CXT.hash(form_data.password))
+    return {"access_token": await login_user(form_data.username, form_data.password), "token_type": "bearer"}
 
 
 @app.get("/logout", include_in_schema=False)
@@ -99,7 +149,7 @@ async def logout(request: Request):
     return resp
 
 
-@app.get("/docs")
+@app.get("/docs", include_in_schema=False)
 async def swagger_docs(request: Request):
     return get_swagger_ui_html(
         title="gooseBit docs",

{goosebit-0.2.4 → goosebit-0.2.6}/goosebit/api/telemetry/metrics.py

@@ -2,7 +2,7 @@ from opentelemetry import metrics
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 
-from goosebit.settings import USERS, config
+from goosebit.settings import config
 
 from . import prometheus
 
@@ -28,7 +28,3 @@ users_count = meter.create_gauge(
     "users.count",
     description="The number of registered users",
 )
-
-
-async def init():
-    users_count.set(len(USERS))

{goosebit-0.2.4 → goosebit-0.2.6}/goosebit/api/v1/devices/device/responses.py

@@ -7,6 +7,7 @@ from goosebit.schema.devices import DeviceSchema
 
 class DeviceLogResponse(BaseModel):
     log: str | None
+    progress: int | None
 
 
 class DeviceResponse(DeviceSchema):

goosebit-0.2.6/goosebit/api/v1/devices/device/routes.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Security
+from fastapi.requests import Request
+
+from goosebit.api.v1.devices.device.responses import DeviceLogResponse, DeviceResponse
+from goosebit.auth import validate_user_permissions
+from goosebit.auth.permissions import GOOSEBIT_PERMISSIONS
+from goosebit.db import Device
+from goosebit.device_manager import get_device
+
+router = APIRouter(prefix="/{dev_id}")
+
+
+@router.get(
+    "",
+    dependencies=[Security(validate_user_permissions, scopes=[GOOSEBIT_PERMISSIONS["device"]["read"]()])],
+)
+async def device_get(_: Request, device: Device = Depends(get_device)) -> DeviceResponse:
+    if device is None:
+        raise HTTPException(404)
+    await device.fetch_related("assigned_software", "hardware")
+    return DeviceResponse.model_validate(device)
+
+
+@router.get(
+    "/log",
+    dependencies=[Security(validate_user_permissions, scopes=[GOOSEBIT_PERMISSIONS["device"]["read"]()])],
+)
+async def device_logs(_: Request, device: Device = Depends(get_device)) -> DeviceLogResponse:
+    if device is None:
+        raise HTTPException(404)
+    return DeviceLogResponse(log=device.last_log, progress=device.progress)