lucihub-common 0.1.0__tar.gz

lucihub_common-0.1.0/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.1
+Name: lucihub-common
+Version: 0.1.0
+Summary: Common data models and storage components for LuciHub
+Author-Email: Anabrid <info@anabrid.com>
+Requires-Python: <4.0,>=3.11
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: celery<6.0.0,>=5.3.0
+Requires-Dist: numpy>=2.4
+Requires-Dist: pybrid-computing
+Requires-Dist: pybrid-computing-native
+Requires-Dist: pyredacc
+Requires-Dist: sympy>=1.13.3
+Requires-Dist: sqlalchemy[asyncio]>=2.0
+Requires-Dist: asyncpg>=0.29
+Requires-Dist: alembic>=1.13
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
+# Models
+
+This component contains data models that are used to communicate between
+at least two different components of the project.
+
+This includes the following models:
+- `database`: models/classes for interfacing with hot and cold storage
+  - `hot`: use plain Python classes that abstract away the path variables and include static types
+  - `cold`: data models in SQL alchemy
+
+Note that the `hot` storage essentially caches parts of the data structures
+of the `cold` storage. This should be reflected in the SQL alchemy model: 
+
+## Principles
+
+- make sure to use object-oriented expressons and techniques, for example
+abstract models and classes which have different implementations to account for versioning.
+- consider the fact that the API itself is mostly consisting of CRUD-type of endpoints
+- for hot storage, favor small field with primitive data types
+- for cold storage, prefer large objects that enable fast data read without too many joins
+when reading/writing through the API endpoints
+
+## Modelled data
+
+This is a very rough first draft which does _not_ correspond to an 1:1 ORM:
+
+**Cold storage**
+- run 
+    - id
+    - type of job (compile, run, verify...)
+    - state changes (array of timestamp/status)
+    - logs
+    - number of channels
+    - measurements (per channel) for run/verify jobs
+    - device id
+    - input type
+    - input (JSON data)
+    - output (JSON data)
+- device
+    - id
+    - type
+    - input type
+    - parameter schema (as a data type that will simplify the model checking later)
+
+**Hot storage**
+- run (by UUID)
+  - state changes (array of pairs timestamp/status)
+  - logs (array of lines, lines are appended)
+  - measurements (on different channels, each channel is a time series of floating point values)
+- list of active devices (device IDs), regularly updated from scanner

lucihub_common-0.1.0/README.md

@@ -0,0 +1,50 @@
+# Models
+
+This component contains data models that are used to communicate between
+at least two different components of the project.
+
+This includes the following models:
+- `database`: models/classes for interfacing with hot and cold storage
+  - `hot`: use plain Python classes that abstract away the path variables and include static types
+  - `cold`: data models in SQL alchemy
+
+Note that the `hot` storage essentially caches parts of the data structures
+of the `cold` storage. This should be reflected in the SQL alchemy model: 
+
+## Principles
+
+- make sure to use object-oriented expressons and techniques, for example
+abstract models and classes which have different implementations to account for versioning.
+- consider the fact that the API itself is mostly consisting of CRUD-type of endpoints
+- for hot storage, favor small field with primitive data types
+- for cold storage, prefer large objects that enable fast data read without too many joins
+when reading/writing through the API endpoints
+
+## Modelled data
+
+This is a very rough first draft which does _not_ correspond to an 1:1 ORM:
+
+**Cold storage**
+- run 
+    - id
+    - type of job (compile, run, verify...)
+    - state changes (array of timestamp/status)
+    - logs
+    - number of channels
+    - measurements (per channel) for run/verify jobs
+    - device id
+    - input type
+    - input (JSON data)
+    - output (JSON data)
+- device
+    - id
+    - type
+    - input type
+    - parameter schema (as a data type that will simplify the model checking later)
+
+**Hot storage**
+- run (by UUID)
+  - state changes (array of pairs timestamp/status)
+  - logs (array of lines, lines are appended)
+  - measurements (on different channels, each channel is a time series of floating point values)
+- list of active devices (device IDs), regularly updated from scanner

lucihub_common-0.1.0/pyproject.toml

@@ -0,0 +1,86 @@
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = [
+    "tests",
+]
+pythonpath = [
+    "src",
+]
+python_classes = [
+    "Test*",
+    "!_*",
+]
+python_functions = [
+    "test_*",
+]
+
+[tool.black]
+line-length = 88
+target-version = [
+    "py313",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py313"
+
+[tool.pdm.build]
+includes = [
+    "src/anabrid",
+]
+
+[tool.uv]
+prerelease = "allow"
+
+[tool.uv.sources.pybrid-computing]
+path = "/home/dthuerck/dev/software/pybrid-computing/packages/pybrid-computing"
+editable = true
+
+[tool.uv.sources.pybrid-computing-native]
+path = "/home/dthuerck/dev/software/pybrid-computing/packages/pybrid-computing-native"
+editable = false
+
+[tool.uv.sources.pyredacc]
+path = "/home/dthuerck/dev/software/redacc/src/frontends/pyredacc"
+editable = true
+
+[project]
+authors = [
+    { name = "Anabrid", email = "info@anabrid.com" },
+]
+requires-python = "<4.0,>=3.11"
+dependencies = [
+    "pydantic<3.0.0,>=2.0.0",
+    "celery<6.0.0,>=5.3.0",
+    "numpy>=2.4",
+    "pybrid-computing",
+    "pybrid-computing-native",
+    "pyredacc",
+    "sympy>=1.13.3",
+    "sqlalchemy[asyncio]>=2.0",
+    "asyncpg>=0.29",
+    "alembic>=1.13",
+    "pyyaml>=6.0",
+]
+name = "lucihub-common"
+version = "0.1.0"
+description = "Common data models and storage components for LuciHub"
+readme = "README.md"
+
+[project.scripts]
+lucihub-deploy = "anabrid.lucihub.deploy.generate:main"
+
+[dependency-groups]
+dev = [
+    "pytest<8.0.0,>=7.0.0",
+    "pytest-asyncio<1.0.0,>=0.21.0",
+    "black>=24.0.0",
+    "ruff<1.0.0,>=0.0.280",
+    "psycopg[binary]>=3.1",
+]

lucihub_common-0.1.0/src/anabrid/lucihub/deploy/generate.py

@@ -0,0 +1,363 @@
+"""Deploy configuration generator for LuciHub worker services.
+
+Converts a ``LUCIHUBConfig`` instance into two deployment artefacts written
+to the same output directory:
+
+* A Docker Compose YAML fragment (``docker-compose.workers.yml``) — one
+  service per worker process (compile and run).  SIMULATOR run-workers
+  receive an additional bind-mount of the simulator binary.
+
+* A supervisord configuration file (``supervisord.conf``) — one program
+  per worker process plus the API server.  Programs use ``watchfiles`` (for
+  workers) and uvicorn ``--reload`` (for the API) so source edits hot-reload
+  without restarting supervisord.
+
+Both artefacts are generated from the same ``LUCIHUBConfig``; a
+duplicate-port guard fires before either is written.
+
+The database DSN is never baked into the generated files. The compose
+fragment emits a compose-level ``${POSTGRES_PASSWORD}`` interpolation so
+docker resolves the secret from ``.env`` at container start; the
+supervisord fragment emits ``%(ENV_POSTGRES_PASSWORD)s`` (and the
+matching user/host/port/db placeholders) so supervisord resolves the
+secret from its own environment at program-start. No plaintext password
+appears in either artefact.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import yaml
+
+from anabrid.lucihub.models.config import LUCIHUBConfig
+from anabrid.lucihub.models.enums import DeviceTypeEnum
+
+DEFAULT_WORKER_IMAGE = "lucihub-worker:latest"
+_RESTART = "unless-stopped"
+_NETWORKS: tuple[str, ...] = ("internal",)
+_VOLUMES: tuple[str, ...] = (
+    "${LUCIHUB_BASE_DIR}/config:/config:ro",
+    "${LUCIHUB_BASE_DIR}/data:/data",
+    "${LUCIHUB_BASE_DIR}/logs:/logs",
+)
+_BROKER_URL = "amqp://rabbitmq:5672//"
+_REDACC_EXECUTABLE = "/usr/local/bin/redacc"
+_SIMULATOR_EXECUTABLE = "/usr/local/bin/compiler_sim"
+
+# Compose-level interpolation literal: docker resolves ${...} at container
+# start, pulling the DB secret from the compose process's environment
+# (which is loaded from the project's .env file).
+_COMPOSE_DATABASE_URL = (
+    "postgresql+asyncpg://"
+    "${POSTGRES_USER:-lucihub}:${POSTGRES_PASSWORD}"
+    "@postgres:${POSTGRES_PORT:-5432}"
+    "/${POSTGRES_DB:-lucihub}"
+)
+
+# Supervisord runs in the dev-host-api topology only; the API and workers
+# reach postgres via the published port on the loopback. Supervisord's
+# %(ENV_X)s syntax has no default-value form, so non-secret fields are
+# inlined here and only the password remains a placeholder resolved from
+# the shell (which sources .env before launching supervisord).
+_SUPERVISORD_DATABASE_URL = (
+    "postgresql+asyncpg://lucihub:%(ENV_POSTGRES_PASSWORD)s"
+    "@localhost:5432/lucihub"
+)
+
+# Supervisord-specific constants
+_SUPERVISORD_BROKER_URL = "amqp://localhost:5672//"
+_SUPERVISORD_SOCK = "/tmp/supervisor.sock"
+
+_API_COMMAND = (
+    "uv run uvicorn anabrid.lucihub.server.main:app"
+    " --reload"
+    " --reload-dir %(ENV_LUCIHUB_BASE_DIR)s/components/server/src"
+    " --reload-dir %(ENV_LUCIHUB_BASE_DIR)s/components/common/src"
+    " --port 8000"
+)
+_WORKER_COMMAND = (
+    "uv run watchfiles --filter python"
+    ' "python -m anabrid.lucihub.worker.main"'
+    " %(ENV_LUCIHUB_BASE_DIR)s/components/worker/src"
+    " %(ENV_LUCIHUB_BASE_DIR)s/components/common/src"
+)
+
+
+def _check_duplicate_simulator_ports(config: LUCIHUBConfig) -> None:
+    """Raise ``ValueError`` if two SIMULATOR devices share the same (host, port).
+
+    This guard fires before any output is produced, so a misconfigured file
+    fails fast with a clear message naming both devices and the conflicting
+    address.
+    """
+    port_map: dict[tuple[str, int], list[str]] = {}
+    for entry in config.devices.available:
+        if entry.type == DeviceTypeEnum.SIMULATOR:
+            key = (entry.host, entry.port)
+            port_map.setdefault(key, []).append(entry.name)
+
+    for (host, port), names in port_map.items():
+        if len(names) > 1:
+            first, second = names[0], names[1]
+            suffix = f" and {len(names) - 2} others" if len(names) > 2 else ""
+            raise ValueError(
+                f"SIMULATOR devices {first!r} and {second!r}{suffix} share"
+                f" host:port {host}:{port}"
+            )
+
+
+def _base_service(
+    worker_type: str, worker_image: str
+) -> dict[str, object]:
+    """Return a service dict with common fields populated for the given worker type.
+
+    The service's ``LUCIHUB_DATABASE_URL`` is a compose-interpolation
+    literal; docker resolves it from the compose-process environment
+    (loaded from ``.env``) at container start.
+    """
+    return {
+        "image": worker_image,
+        "restart": _RESTART,
+        "user": "${LUCIHUB_UID:-1000}:${LUCIHUB_GID:-1000}",
+        "networks": list(_NETWORKS),
+        "volumes": list(_VOLUMES),
+        "environment": {
+            "WORKER_TYPE": worker_type,
+            "BROKER_URL": _BROKER_URL,
+            "LUCIHUB_DATABASE_URL": _COMPOSE_DATABASE_URL,
+            "LUCIHUB_BASE_DIR": "/",
+        },
+    }
+
+
+def generate_compose(
+    config: LUCIHUBConfig,
+    worker_image: str = DEFAULT_WORKER_IMAGE,
+) -> str:
+    """Generate a Docker Compose YAML string for all worker services.
+
+    Emits one service per compile instance and one per device-worker slot.
+    SIMULATOR run-workers receive ``LUCIHUB_SIMULATOR_BINARY`` pointing at
+    the ``compiler_sim`` binary baked into the worker image. Raises
+    ``ValueError`` if two SIMULATOR devices share the same ``(host, port)``
+    pair, as that would cause a runtime port conflict.
+
+    ``worker_image`` controls the image tag on every generated service.
+    Defaults to the local ``lucihub-worker:latest`` tag built by
+    ``init_workers.sh``; production bundles pass a registry-qualified
+    reference so the prod host can pull directly from the registry.
+
+    The returned string is valid Compose YAML that can be combined with the
+    main ``docker-compose.yml``.
+
+    Security:
+        The generated file carries the DB DSN as a compose-level
+        interpolation literal (``${POSTGRES_PASSWORD}`` etc). Docker
+        resolves the secret from the ``.env`` file at container start,
+        so no plaintext password is ever written to disk by this
+        generator.
+    """
+    _check_duplicate_simulator_ports(config)
+
+    services: dict = {}
+
+    for i in range(config.workers.compile_instances):
+        name = f"worker-compile-{i}"
+        svc = _base_service("compile", worker_image)
+        svc["environment"]["LUCIHUB_LOG_NAME"] = name
+        svc["environment"]["REDACC_EXECUTABLE"] = _REDACC_EXECUTABLE
+        services[name] = svc
+
+    for entry in config.devices.available:
+        for i in range(entry.workers):
+            name = f"worker-run-{entry.name}-{i}"
+            svc = _base_service("run", worker_image)
+            svc["environment"]["LUCIHUB_LOG_NAME"] = name
+            svc["environment"]["DEVICE_NAME"] = entry.name
+            if entry.type == DeviceTypeEnum.SIMULATOR:
+                svc["environment"]["LUCIHUB_SIMULATOR_BINARY"] = (
+                    _SIMULATOR_EXECUTABLE
+                )
+            services[name] = svc
+
+    compose = {
+        "services": services,
+        "networks": {
+            "internal": {
+                "external": True,
+                "name": "lucihub_internal",
+            }
+        },
+    }
+
+    return yaml.safe_dump(compose, sort_keys=False, default_flow_style=False)
+
+
+def generate_supervisord(config: LUCIHUBConfig) -> str:
+    """Generate a supervisord.conf string for the dev-host-API topology.
+
+    Emits four harness stanzas (supervisord, unix_http_server,
+    rpcinterface:supervisor, supervisorctl), one ``[program:api]`` stanza
+    running uvicorn with --reload, one ``[program:worker-compile-N]`` per
+    compile instance running the worker entrypoint wrapped by watchfiles,
+    and one ``[program:worker-run-<device>-N]`` per device-worker slot.
+
+    All program paths reference ``%(ENV_LUCIHUB_BASE_DIR)s`` so the
+    generated file is independent of where the project is checked out.
+    Log files are written to
+    ``%(ENV_LUCIHUB_BASE_DIR)s/logs/<program-name>.{out,err}``.
+
+    Raises ``ValueError`` if two SIMULATOR devices share the same
+    ``(host, port)`` pair.
+    """
+    _check_duplicate_simulator_ports(config)
+
+    common_env_pairs = [
+        f'BROKER_URL="{_SUPERVISORD_BROKER_URL}"',
+        f'LUCIHUB_DATABASE_URL="{_SUPERVISORD_DATABASE_URL}"',
+        'LUCIHUB_BASE_DIR="%(ENV_LUCIHUB_BASE_DIR)s"',
+    ]
+
+    def _log_base(name: str) -> str:
+        return "%(ENV_LUCIHUB_BASE_DIR)s/logs/" + name
+
+    def _program_block(
+        name: str,
+        command: str,
+        directory: str,
+        extra_env_pairs: list[str] | None = None,
+    ) -> str:
+        env_pairs = list(extra_env_pairs or []) + common_env_pairs
+        env_line = ",".join(env_pairs)
+        lines = [
+            f"[program:{name}]",
+            f"command={command}",
+            f"directory={directory}",
+            "autostart=true",
+            "autorestart=true",
+            f"stdout_logfile={_log_base(name)}.out",
+            f"stderr_logfile={_log_base(name)}.err",
+            f"environment={env_line}",
+            "",
+        ]
+        return "\n".join(lines)
+
+    base_dir = "%(ENV_LUCIHUB_BASE_DIR)s"
+    server_dir = f"{base_dir}/components/server"
+    worker_dir = f"{base_dir}/components/worker"
+
+    sections: list[str] = []
+
+    # Harness stanzas
+    sections.append(
+        "[supervisord]\n"
+        "nodaemon=true\n"
+        "\n"
+    )
+    sections.append(
+        "[unix_http_server]\n"
+        f"file={_SUPERVISORD_SOCK}\n"
+        "\n"
+    )
+    sections.append(
+        "[rpcinterface:supervisor]\n"
+        "supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface\n"
+        "\n"
+    )
+    sections.append(
+        "[supervisorctl]\n"
+        f"serverurl=unix://{_SUPERVISORD_SOCK}\n"
+        "\n"
+    )
+
+    # API program
+    sections.append(
+        _program_block(
+            name="api",
+            command=_API_COMMAND,
+            directory=server_dir,
+        )
+    )
+
+    # Compile workers
+    for i in range(config.workers.compile_instances):
+        name = f"worker-compile-{i}"
+        sections.append(
+            _program_block(
+                name=name,
+                command=_WORKER_COMMAND,
+                directory=worker_dir,
+                extra_env_pairs=[
+                    'WORKER_TYPE="compile"',
+                    'REDACC_EXECUTABLE="%(ENV_LUCIHUB_BASE_DIR)s/bin/redacc"',
+                ],
+            )
+        )
+
+    # Run workers
+    simulator_binary = config.workers.simulator_binary_path
+    for entry in config.devices.available:
+        for i in range(entry.workers):
+            name = f"worker-run-{entry.name}-{i}"
+            env_pairs = [
+                'WORKER_TYPE="run"',
+                f'DEVICE_NAME="{entry.name}"',
+            ]
+            if entry.type == DeviceTypeEnum.SIMULATOR:
+                env_pairs.append(f'LUCIHUB_SIMULATOR_BINARY="{simulator_binary}"')
+            sections.append(
+                _program_block(
+                    name=name,
+                    command=_WORKER_COMMAND,
+                    directory=worker_dir,
+                    extra_env_pairs=env_pairs,
+                )
+            )
+
+    return "\n".join(sections)
+
+
+def main() -> None:
+    """Entry point for the ``lucihub-deploy`` CLI tool."""
+    parser = argparse.ArgumentParser(
+        description="Generate Docker Compose worker configuration from a LuciHub config file."
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    gen = subparsers.add_parser("generate", help="Generate a worker compose file.")
+    gen.add_argument("-c", "--config", required=True, help="Path to lucihub.yaml.")
+    gen.add_argument(
+        "-o", "--output", required=True, help="Path for the output compose file."
+    )
+    gen.add_argument(
+        "--worker-image",
+        default=DEFAULT_WORKER_IMAGE,
+        help=(
+            "Worker image reference to emit on every generated service. "
+            f"Defaults to {DEFAULT_WORKER_IMAGE!r}. Pass a registry-qualified "
+            "ref (e.g. lab.analogparadigm.com:5050/.../worker:latest) when "
+            "preparing a production bundle."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    config_path = Path(args.config)
+    config = LUCIHUBConfig.model_validate(
+        yaml.safe_load(config_path.read_text(encoding="utf-8"))
+    )
+    try:
+        yaml_text = generate_compose(config, worker_image=args.worker_image)
+        supervisord_text = generate_supervisord(config)
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        sys.exit(2)
+
+    output_path = Path(args.output)
+    output_path.write_text(yaml_text, encoding="utf-8")
+
+    supervisord_path = output_path.parent / "supervisord.conf"
+    supervisord_path.write_text(supervisord_text, encoding="utf-8")