cofy-api 0.1.0__py3-none-any.whl

cofy/cofy_api.py

@@ -0,0 +1,43 @@
+from typing import Any
+
+from fastapi import FastAPI
+from fastapi.openapi.utils import get_openapi
+
+from src.cofy.docs_router import DocsRouter
+from src.shared.module import Module
+
+DEFAULT_ARGS: dict[str, Any] = {
+    "title": "Cofy API",
+    "version": "0.1.0",
+    "description": "Modular cloud API for energy data",
+    "docs_url": None,
+    "redoc_url": None,
+    "openapi_url": None,
+}
+
+
+class CofyApi(FastAPI):
+    def __init__(self, **kwargs):
+        super().__init__(**(DEFAULT_ARGS | kwargs))
+        self._modules: list[Module] = []
+        self.include_router(DocsRouter(self.openapi))
+
+    def openapi(self):
+        return get_openapi(
+            title=self.title,
+            version=self.version,
+            routes=self.routes,
+            tags=self.tags_metadata,
+        )
+
+    def register_module(self, module: Module):
+        self._modules.append(module)
+        self.include_router(module)
+
+    @property
+    def tags_metadata(self) -> list[dict[str, Any]]:
+        return [module.tag for module in self._modules]
+
+    @property
+    def modules(self) -> tuple[Module, ...]:
+        return tuple(self._modules)

cofy/db/alembic/env.py

@@ -0,0 +1,52 @@
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from src.cofy.db.base import Base
+
+config = context.config
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+configured_metadata = config.attributes.get("target_metadata")
+if isinstance(configured_metadata, list):
+    target_metadata = configured_metadata[0] if configured_metadata else Base.metadata
+elif configured_metadata is not None:
+    target_metadata = configured_metadata
+else:
+    target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

cofy/db/base.py

@@ -0,0 +1,5 @@
+from sqlalchemy.orm import DeclarativeBase
+
+
+class Base(DeclarativeBase):
+    pass

cofy/db/cofy_db.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from importlib import resources
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import MetaData, create_engine
+
+from src.cofy.db.database_backed_source import DatabaseBackedSource
+
+if TYPE_CHECKING:
+    from sqlalchemy.engine import Engine
+
+    from src.cofy.cofy_api import CofyApi
+    from src.shared.module import Module
+
+
+class CofyDB:
+    def __init__(self, url: str | None = None, **engine_kwargs):
+        if url is None:
+            raise ValueError("CofyDB requires a database URL to be configured.")
+        self._sources: list[DatabaseBackedSource] = []
+        self._url = url
+        self.engine: Engine = create_engine(self._url, **engine_kwargs)
+
+    def register_module(self, module: Module):
+        source = getattr(module, "source", None)
+        if isinstance(source, DatabaseBackedSource):
+            self._sources.append(source)
+
+    def register_modules(self, modules: Iterable[Module]):
+        for module in modules:
+            self.register_module(module)
+
+    def bind_api(self, api: CofyApi):
+        self.register_modules(api.modules)
+
+    @property
+    def migration_locations(self) -> list[str]:
+        locations: list[str] = []
+        for source in self._sources:
+            for location in source.migration_locations:
+                resolved_location = str(Path(location).resolve())
+                if resolved_location not in locations:
+                    locations.append(resolved_location)
+        return locations
+
+    @property
+    def target_metadata(self) -> list[Any]:
+        metadata: list[Any] = []
+        for source in self._sources:
+            if source.target_metadata is not None and source.target_metadata not in metadata:
+                metadata.append(source.target_metadata)
+        return metadata
+
+    def _build_config(self) -> Config:
+        config = Config()
+        config.set_main_option("script_location", str(resources.files("src.cofy.db").joinpath("alembic")))
+        config.set_main_option("sqlalchemy.url", self._url)
+        config.set_main_option("path_separator", "os")
+
+        if self.migration_locations:
+            config.set_main_option("version_locations", " ".join(self.migration_locations))
+
+        config.attributes["target_metadata"] = self.target_metadata
+        return config
+
+    def run_migrations(self, revision: str = "heads") -> None:
+        command.upgrade(self._build_config(), revision)
+
+    def generate_migration(
+        self,
+        message: str,
+        head: str,
+        rev_id: str | None = None,
+        autogenerate: bool = True,
+    ) -> None:
+        """Generate a new Alembic migration revision for a specific module branch.
+
+        Args:
+            message: Description of the migration.
+            head: The branch head to extend, e.g. "members_core@head".
+            rev_id: Optional custom revision ID. If None, Alembic generates one.
+            autogenerate: Whether to autogenerate the migration from model changes.
+        """
+        command.revision(
+            self._build_config(),
+            message=message,
+            autogenerate=autogenerate,
+            head=head,
+            rev_id=rev_id,
+        )
+
+    def reset(self) -> None:
+        """usefull in development to reset the database to a clean state
+        Warning: this will delete all data in the database, use with caution!
+        """
+        meta = MetaData()
+        meta.reflect(bind=self.engine)
+        meta.drop_all(bind=self.engine)
+        self.run_migrations()

cofy/db/database_backed_source.py

@@ -0,0 +1,13 @@
+from collections.abc import Sequence
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class DatabaseBackedSource(Protocol):
+    @property
+    def migration_locations(self) -> Sequence[str]:
+        """Filesystem paths containing alembic version files."""
+
+    @property
+    def target_metadata(self) -> Any | None:
+        """SQLAlchemy metadata used for migration autogeneration."""

cofy/db/timestamp_mixin.py

@@ -0,0 +1,9 @@
+import datetime as dt
+
+from sqlalchemy import DateTime, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+
+class TimestampMixin:
+    created_at: Mapped[dt.datetime] = mapped_column(DateTime, nullable=False, default=func.now())
+    updated_at: Mapped[dt.datetime] = mapped_column(DateTime, nullable=False, default=func.now(), onupdate=func.now())

cofy/docs_router.py

@@ -0,0 +1,37 @@
+from collections.abc import Callable
+
+from fastapi import APIRouter, Request
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import HTMLResponse
+
+
+class DocsRouter(APIRouter):
+    def __init__(self, get_openapi: Callable[[], dict]):
+        super().__init__()
+        self.get_openapi = get_openapi
+        self.add_api_route(
+            "/docs",
+            self.get_swagger_ui_html,
+            include_in_schema=False,
+        )
+        self.add_api_route("/openapi.json", self.get_openapi, include_in_schema=False)
+
+    async def get_swagger_ui_html(self, request: Request):
+        response = get_swagger_ui_html(
+            openapi_url="/openapi.json",
+            title="Docs",
+            swagger_ui_parameters={
+                "spec": self.get_openapi(),
+                "onComplete": "AUTHORIZE_API",
+            },
+        )
+
+        if hasattr(request.state, "auth_info"):
+            assert isinstance(response.body, bytes)
+            content = response.body.decode()
+            content = content.replace(
+                '"AUTHORIZE_API"',
+                f'() => ui.preauthorizeApiKey("{request.state.auth_info["scheme"]}", "{request.state.auth_info["content"]}")',
+            )
+            response = HTMLResponse(content=content, status_code=response.status_code)
+        return response

cofy/token_auth.py

@@ -0,0 +1,70 @@
+from datetime import UTC, datetime
+
+from fastapi import Depends, HTTPException, Request
+from fastapi.security import APIKeyHeader, APIKeyQuery
+from pydantic import BaseModel
+from starlette.status import HTTP_401_UNAUTHORIZED
+
+
+class TokenInfo(BaseModel):
+    name: str
+    expires: str | None = None
+
+    def __init__(self, info: dict):
+        if "name" not in info or not info["name"]:
+            raise ValueError("Token info must include a name")
+        if "expires" in info and info["expires"] is not None:
+            try:
+                datetime.fromisoformat(info["expires"])
+            except Exception:
+                raise ValueError("Token expires must be in ISO8601 format") from None
+        super().__init__(**info)
+
+    def is_expired(self) -> bool:
+        if self.expires:
+            expires_dt = datetime.fromisoformat(self.expires)
+            if expires_dt.tzinfo is None:
+                expires_dt = expires_dt.replace(tzinfo=UTC)
+
+            return datetime.now(UTC) > expires_dt
+        return False
+
+
+def token_verifier(tokens: dict):
+    tokens = {token: TokenInfo(info) for token, info in tokens.items()}
+
+    def verify(
+        request: Request,
+        header_token: str = Depends(APIKeyHeader(name="Authorization", auto_error=False, scheme_name="header")),
+        query_token: str = Depends(APIKeyQuery(name="token", auto_error=False, scheme_name="query")),
+    ):
+        token = None
+        auth_info = None
+        if header_token and header_token.lower().startswith("bearer "):
+            token = header_token[7:]
+            auth_info = {
+                "scheme": "header",
+                "content": header_token,
+            }
+        elif query_token:
+            token = query_token
+            auth_info = {
+                "scheme": "query",
+                "content": query_token,
+            }
+        if not token:
+            if header_token:
+                raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail="Invalid token format")
+
+            raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail="Missing token")
+
+        token_info = tokens.get(token)
+        if not token_info:
+            raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail="Invalid token")
+
+        if token_info.is_expired():
+            raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail="Token expired")
+        request.state.token = token
+        request.state.auth_info = auth_info
+
+    return verify

cofy/worker.py

@@ -0,0 +1,187 @@
+"""
+Provides CofyWorker — a thin scaffolding layer over SAQ for background job processing.
+
+Each community (deployment) creates its own worker with the jobs and cron
+schedules it needs. Modules provide reusable job functions; the community
+decides which ones to activate and how to schedule them.
+
+Architecture:
+    ┌─────────────┐     ┌─────────┐     ┌──────────────┐
+    │ FastAPI API │────▶│  Redis  │◀────│  SAQ Worker  │
+    │  (enqueue)  │     │  Queue  │     │  (process)   │
+    └─────────────┘     └─────────┘     └──────────────┘
+
+Both the API and the worker connect to the same Redis queue.
+The API enqueues jobs; the worker processes them.
+
+Example — community worker (e.g. src/demo/worker.py):
+
+    from src.cofy.jobs.worker import CofyWorker
+
+    worker = CofyWorker(url="redis://localhost:6379")
+    worker.register(my_job_function)
+    worker.schedule(my_job_function, cron="0 2 * * *")
+    settings = worker.settings  # SAQ entry point: saq src.demo.worker.settings
+
+Example — enqueue from a FastAPI endpoint:
+
+    from src.cofy.worker import create_queue
+
+    queue = create_queue("redis://localhost:6379")
+
+    @app.post("/upload")
+    async def upload(file: UploadFile):
+        await queue.enqueue("process_upload", file_id="...")
+"""
+
+import asyncio
+import functools
+import inspect
+from collections.abc import Callable, Coroutine
+from typing import Any, Protocol
+
+from saq import CronJob, Queue
+
+type JobFunction = Callable[..., Coroutine[Any, Any, Any]]
+
+
+class NamedCallable(Protocol):
+    """A callable that exposes a ``__name__`` attribute (i.e. a function)."""
+
+    __name__: str
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any: ...
+
+
+def create_queue(url: str = "redis://localhost:6379", **kwargs: Any) -> Queue:
+    """Create a SAQ queue for enqueuing jobs (e.g. from the API process).
+
+    Use this in the API process when you only need to enqueue jobs,
+    not process them. Points to the same Redis as the worker.
+    """
+    return Queue.from_url(url, **kwargs)
+
+
+def to_task(function: Callable, **fixed_kwargs: Any) -> JobFunction:
+    """Wrap a plain function into a SAQ-compatible task.
+
+    The wrapped function does NOT need to know about SAQ's `ctx` dict.
+    It simply declares the arguments it needs::
+
+        async def sync_members(db_engine, file_path: str) -> dict: ...
+
+    Arguments are resolved in this priority order (last wins):
+        1. fixed_kwargs — bound at registration time via ``to_task(fn, file_path="/data/x.csv")``
+        2. ctx values — set by the worker's on_startup hooks (e.g. ``db_engine``)
+        3. enqueue kwargs — passed at ``queue.enqueue("sync_members", file_path="/other.csv")``
+
+    Only arguments that match the function's signature are passed through,
+    so SAQ internals in ctx (``queue``, ``job``) don't leak in.
+
+    Works with both sync and async functions. Sync functions are
+    automatically run in a thread via ``asyncio.to_thread``.
+    """
+    sig = inspect.signature(function)
+    params = sig.parameters
+    accepts_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
+    is_async = inspect.iscoroutinefunction(function)
+
+    @functools.wraps(function)
+    async def task(ctx: dict, **kwargs: Any) -> Any:
+        merged = {**fixed_kwargs, **ctx, **kwargs}
+
+        call_kwargs = merged if accepts_var_keyword else {k: v for k, v in merged.items() if k in params}
+
+        if is_async:
+            return await function(**call_kwargs)
+        return await asyncio.to_thread(function, **call_kwargs)
+
+    return task
+
+
+class CofyWorker:
+    """Builds SAQ worker settings from registered job functions and cron schedules.
+
+    The worker collects job functions and cron schedules, then exposes them as a `settings` dict for the SAQ CLI.
+
+    It also provides lifecycle hooks (on_startup / on_shutdown) for setting up shared resources like database engines
+    """
+
+    def __init__(self, url: str = "redis://localhost:6379", **queue_kwargs: Any):
+        self.queue = Queue.from_url(url, **queue_kwargs)
+        self._functions: dict[str, JobFunction] = {}
+        self._cron_jobs: list[CronJob] = []
+        self._startup_hooks: list[JobFunction] = []
+        self._shutdown_hooks: list[JobFunction] = []
+
+    def register(self, func: NamedCallable, **fixed_kwargs: Any) -> JobFunction:
+        """Register a job function so it can be enqueued and processed by this worker.
+
+        Registered functions become available for ``queue.enqueue("function_name", ...)``.
+        """
+        if func.__name__ not in self._functions:
+            self._functions[func.__name__] = to_task(func, **fixed_kwargs)
+        return self._functions[func.__name__]
+
+    def schedule(
+        self,
+        func: NamedCallable,
+        cron: str,
+        function_kwargs: dict | None = None,
+        **kwargs: Any,
+    ) -> JobFunction:
+        """Schedule a function to run on a cron expression."""
+        task = self.register(func)
+        self._cron_jobs.append(
+            CronJob(task, cron=cron, kwargs=function_kwargs or {}, **kwargs),
+        )
+        return task
+
+    def on_startup(self, func: JobFunction) -> JobFunction:
+        """Register a hook that runs when the worker starts.
+
+        Use this to set up shared resources (DB engines, HTTP clients, etc.)
+        that jobs can access through `ctx`.
+        """
+        self._startup_hooks.append(func)
+        return func
+
+    def on_shutdown(self, func: JobFunction) -> JobFunction:
+        """Register a hook that runs when the worker stops. Use for cleanup."""
+        self._shutdown_hooks.append(func)
+        return func
+
+    @property
+    def settings(self) -> dict[str, Any]:
+        """SAQ worker settings dict.
+
+        Export this as a module-level `settings` variable for the SAQ CLI:
+
+            settings = worker.settings
+            # Run with: saq src.demo.worker.settings
+        """
+        result: dict[str, Any] = {
+            "queue": self.queue,
+            "functions": list(self._functions.values()),
+            "cron_jobs": self._cron_jobs,
+        }
+
+        if self._startup_hooks:
+            startup_hooks = list(self._startup_hooks)
+
+            async def startup(ctx: dict) -> None:
+                for hook in startup_hooks:
+                    await hook(ctx)
+
+            result["startup"] = startup
+
+        if self._shutdown_hooks:
+            shutdown_hooks = list(self._shutdown_hooks)
+
+            async def shutdown(ctx: dict) -> None:
+                for hook in shutdown_hooks:
+                    await hook(ctx)
+
+            result["shutdown"] = shutdown
+
+        return result

cofy_api-0.1.0.dist-info/METADATA

@@ -0,0 +1,163 @@
+Metadata-Version: 2.4
+Name: cofy-api
+Version: 0.1.0
+Summary: An open-source modular framework for ingesting, standardising, storing, and computing energy-related data.
+Author: Jorg
+Author-email: EnergyID <info@energieid.be>
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: alembic>=1.18.4
+Requires-Dist: entsoe-py>=0.7.10
+Requires-Dist: fastapi[standard]>=0.128.0
+Requires-Dist: narwhals>=2.15.0
+Requires-Dist: pandas>=3.0.0
+Requires-Dist: polars>=1.37.1
+Requires-Dist: saq[redis,web]>=0.26.1
+
+**Cofy Cloud** is an open-source modular framework for ingesting, standardising, storing, and computing energy-related data, designed to run anywhere from local deployments to cloud environments.
+
+Right now this is very much a work in progress.
+With the development of a first proof of concept. This is not ready for production use and the api is likely to change significantly in the near future.
+
+## Setup
+### Install
+*TODO*
+### Configure
+*TODO*
+### Development
+We use [astral](https://docs.astral.sh/) python tooling for our development environment.
+We use [poethepoet](https://poethepoet.natn.io) to define some essential tasks.
+The demo run task is also available as vscode execution task, making it easy to run and debug the demo application from within vscode.
+
+#### Install/update dependencies:
+First install [uv](https://docs.astral.sh/uv/) if you don't have it yet.
+
+Then install/update dependencies:
+```sh
+uv sync
+```
+
+Install [poethepoet](https://poethepoet.natn.io) and [pre-commit](https://pre-commit.com/)
+```sh
+uv tool install poethepoet
+uv tool install pre-commit
+```
+
+Activate [pre-commit](https://pre-commit.com/) hooks that enforce code style on every commit:
+```sh
+pre-commit install
+```
+
+#### Configure environment variables:
+Our demo application uses some API keys for external services. You can provide these `.env.local` file in the root of the repository, following the structure of `.env.example`.
+
+#### Set up the database:
+The demo application uses a SQLite database. A different database can be configured via the environment variables.
+To create the database and seed it with example data:
+
+```sh
+poe db-seed
+```
+
+This will run all pending migrations and load the example CSV data into the database.
+
+#### Run development demo application:
+
+```sh
+poe demo
+```
+
+#### Run background worker:
+We use Cofy Worker to run background jobs that perform data ingestion and processing tasks asynchronously.
+
+The background worker requires a Redis instance to run. You can configure the Redis URL via the environment variables.
+The simplest way to run Redis locally is via Docker:
+
+```sh
+docker run -p 6379:6379 redis
+```
+
+Then start the worker:
+
+```sh
+poe worker
+```
+
+#### Code style checks:
+```sh
+poe lint    # Check code style
+poe format  # Format code
+poe check   # Run type checks
+```
+
+#### Run tests:
+```sh
+poe test
+```
+#### Build & publish
+TODO
+
+## Database & Migrations
+
+Cofy uses [Alembic](https://alembic.sqlalchemy.org/) for database migrations. Each module that needs database storage owns its own migration branch, keeping schemas independent and composable.
+
+### Commands
+
+| Command | Description |
+|---|---|
+| `poe db-seed` | Run all migrations and seed the database with example data |
+| `poe db-migrate` | Run all pending migrations |
+| `poe db-reset` | Drop all tables and re-run migrations (⚠️ destroys all data) |
+| `poe db-generate` | Generate a new migration file for a specific module branch |
+
+### Generating a migration
+
+When you change a module's SQLAlchemy model, generate a migration for that module's branch:
+
+```sh
+poe db-generate "add phone number to member" --head members_core@head --rev-id members_core_0002
+```
+
+| Argument | Required | Description |
+|---|---|---|
+| `message` (positional) | ✅ | Short description of the change |
+| `--head` | ✅ | Branch to extend, e.g. `members_core@head` |
+| `--rev-id` | ❌ | Custom revision ID (Alembic generates one if omitted) |
+| `--no-autogenerate` | ❌ | Create an empty migration instead of diffing model changes |
+
+The generated file will be placed in the module's own `migrations/versions/` directory automatically.
+
+### How branches work
+
+Each module declares its own Alembic branch label in its initial migration. This allows multiple modules to coexist in the same database without interfering with each other.
+
+For example, the members module uses branch `members_core`:
+
+```
+src/modules/members/migrations/versions/
+├── members_core_0001_members_core_initial.py   # branch_labels = ("members_core",)
+└── members_core_0002_add_phone_number.py       # extends members_core@head
+```
+
+A separate module `foo` would have its own branch `foo_core` with revisions in its own directory:
+
+```
+src/modules/foo/migrations/versions/
+├── foo_core_0001_initial.py                    # branch_labels = ("foo_core",)
+└── foo_core_0002_add_index.py                  # extends foo_core@head
+```
+
+When `poe db-migrate` runs, Alembic upgrades all branches to their latest head — both `members_core` and `foo_core` are applied independently.
+
+### Extending a module's schema
+
+If you build a custom implementation that extends an existing module's schema, you can create a new branch that depends on a specific revision:
+
+```python
+revision = "members_custom_0001"
+down_revision = None
+branch_labels = ("members_custom",)
+depends_on = "members_core_0001"  # ensures the base schema exists first
+```
+
+This guarantees the base tables are created before your extension runs, while keeping both migration chains independent.