gini-toolkit 6.0.1.dev0__py3-none-any.whl

gini/server/app.py

@@ -0,0 +1,140 @@
+"""The GINI server request router + a stdlib HTTP wrapper.
+
+`GiniServer.handle(method, path, token, body)` is the whole API as a pure function of its
+inputs (so it's unit-testable with a fake orchestrator, no Docker). The HTTP layer below is
+a thin shell that parses requests and calls it.
+
+Flow for /run: take the student's **topology** -> GINI's own compiler -> policy.enforce ->
+the student's namespaced orchestrator. The student never supplies a compose file or a
+Docker command.
+"""
+from __future__ import annotations
+
+import json
+import threading
+
+from ..domain.topology import Topology
+from ..services.compiler import RuntimeCompiler
+from .auth import Tokens, UserStore
+from .policy import PolicyError, default_allowed_images, enforce
+from .session import SessionManager
+
+
+class GiniServer:
+    def __init__(self, users: UserStore, tokens: Tokens, sessions: SessionManager,
+                 orchestrator_factory, allowed_images=None, max_cpus: float = 2.0) -> None:
+        self.users = users
+        self.tokens = tokens
+        self.sessions = sessions
+        self._make_orch = orchestrator_factory        # (project, workdir) -> orchestrator
+        self.allowed_images = set(allowed_images) if allowed_images else default_allowed_images()
+        self.max_cpus = max_cpus
+        self._orch: dict = {}                          # user -> orchestrator
+        self._runs: dict = {}                          # user -> {"state", "message"}
+        self._lock = threading.Lock()
+
+    def _orch_for(self, user: str):
+        if user not in self._orch:
+            self._orch[user] = self._make_orch(
+                self.sessions.project_name(user), self.sessions.workdir(user))
+        return self._orch[user]
+
+    # the entire API as a pure (status, payload) function -------------------- #
+    def handle(self, method: str, path: str, token: str | None, body: dict) -> tuple[int, dict]:
+        if method == "POST" and path == "/login":
+            u, p = body.get("username"), body.get("password")
+            if self.users.verify(u, p):
+                return 200, {"token": self.tokens.mint(u)}
+            return 401, {"error": "invalid credentials"}
+
+        user = self.tokens.verify(token or "")
+        if not user:
+            return 401, {"error": "authentication required"}
+
+        try:
+            if method == "POST" and path == "/run":
+                return self._run(user, body)
+            if method == "POST" and path == "/stop":
+                ok, msg = self._orch_for(user).down()
+                with self._lock:
+                    self._runs[user] = {"state": "stopped", "message": msg}
+                return (200 if ok else 500), {"ok": ok, "message": msg}
+            if method == "GET" and path == "/status":
+                with self._lock:
+                    run = dict(self._runs.get(user, {"state": "stopped", "message": ""}))
+                return 200, {"status": self._orch_for(user).status(), "run": run}
+            if method == "GET" and path == "/metrics":
+                o = self._orch_for(user)
+                return 200, {"stats": o.stats_all(), "startup": o.startup_times()}
+            if method == "GET" and path == "/capabilities":
+                return 200, {"kata": self._orch_for(user).runtime_available("kata")}
+        except PolicyError as e:
+            return 400, {"error": str(e)}
+        except Exception as e:                          # noqa: BLE001 — report, don't crash
+            return 500, {"error": str(e)}
+        return 404, {"error": "not found"}
+
+    def _run(self, user: str, body: dict) -> tuple[int, dict]:
+        if "topology" not in body:
+            return 400, {"error": "missing topology"}
+        topo = Topology.from_dict(body["topology"])     # the student described elements + links
+        cfg = RuntimeCompiler().compile(topo)           # WE compile it, with our trusted compiler
+        enforce(cfg, self.allowed_images, self.max_cpus)  # raises PolicyError on a violation (fast)
+        o = self._orch_for(user)
+        workdir = str(self.sessions.workdir(user))
+        with self._lock:
+            self._runs[user] = {"state": "starting", "message": ""}
+
+        # `docker compose up` can take minutes on first run (image pulls + builds), so run it
+        # in the background and let the client poll /status — never block the HTTP request.
+        def launch():
+            try:
+                ok, msg = o.up(cfg, workdir)
+                state = "running" if ok else "error"
+            except Exception as e:                       # noqa: BLE001
+                ok, msg, state = False, str(e), "error"
+            with self._lock:
+                self._runs[user] = {"state": state, "message": msg}
+
+        threading.Thread(target=launch, daemon=True).start()
+        return 202, {"ok": True, "state": "starting"}
+
+
+# --------------------------------------------------------------------------- #
+# Thin HTTP shell (stdlib). Production entry point lives in __main__.py.
+# --------------------------------------------------------------------------- #
+def serve(server: "GiniServer", host: str = "0.0.0.0", port: int = 10000) -> None:
+    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *a):                      # quiet
+            pass
+
+        def _token(self) -> str | None:
+            auth = self.headers.get("Authorization", "")
+            return auth[7:] if auth.startswith("Bearer ") else None
+
+        def _dispatch(self, method: str) -> None:
+            n = int(self.headers.get("Content-Length", 0) or 0)
+            raw = self.rfile.read(n).decode("utf-8") if n else ""
+            try:
+                body = json.loads(raw) if raw else {}
+            except json.JSONDecodeError:
+                body = {}
+            status, payload = server.handle(method, self.path.split("?", 1)[0],
+                                            self._token(), body if isinstance(body, dict) else {})
+            data = json.dumps(payload).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+
+        def do_GET(self):
+            self._dispatch("GET")
+
+        def do_POST(self):
+            self._dispatch("POST")
+
+    print(f"[gini-server] listening on {host}:{port}", flush=True)
+    ThreadingHTTPServer((host, port), Handler).serve_forever()

gini/server/auth.py

@@ -0,0 +1,82 @@
+"""Authentication for the GINI server: a flat user file + signed session tokens.
+
+The user file is JSON: ``{"<username>": {"salt": "<hex>", "hash": "<hex>"}}``. Passwords
+are PBKDF2-HMAC-SHA256 — the server never stores (or sees, after login) a plaintext
+password. Tokens are HMAC-signed with the server secret and carry an expiry, so the
+server stays stateless: no session table, nothing to leak.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import os
+import time
+from pathlib import Path
+
+_ITER = 200_000        # PBKDF2 iterations
+
+
+def hash_password(password: str, salt: bytes | None = None) -> dict:
+    """Make a {salt, hash} record for the user file (used by the add-user admin tool)."""
+    salt = salt or os.urandom(16)
+    h = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, _ITER)
+    return {"salt": salt.hex(), "hash": h.hex()}
+
+
+def _check(password: str, rec: dict) -> bool:
+    try:
+        salt = bytes.fromhex(rec["salt"])
+        h = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, _ITER)
+        return hmac.compare_digest(h.hex(), rec["hash"])
+    except (KeyError, ValueError, TypeError):
+        return False
+
+
+class UserStore:
+    """A flat username -> {salt, hash} table."""
+
+    def __init__(self, users: dict | None = None) -> None:
+        self._users = dict(users or {})
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> "UserStore":
+        p = Path(path)
+        return cls(json.loads(p.read_text()) if p.exists() else {})
+
+    def verify(self, username: str, password: str) -> bool:
+        rec = self._users.get(username or "")
+        return bool(rec) and _check(password or "", rec)
+
+
+class Tokens:
+    """Mint / verify HMAC-signed bearer tokens (`<payload>.<sig>`), so auth is stateless."""
+
+    def __init__(self, secret: bytes, ttl: int = 8 * 3600) -> None:
+        self._secret = secret
+        self._ttl = ttl
+
+    def mint(self, username: str) -> str:
+        payload = {"u": username, "exp": int(time.time()) + self._ttl}
+        body = base64.urlsafe_b64encode(json.dumps(payload).encode())
+        sig = hmac.new(self._secret, body, hashlib.sha256).digest()
+        return body.decode() + "." + base64.urlsafe_b64encode(sig).decode()
+
+    def verify(self, token: str) -> str | None:
+        """Return the username if the token is valid + unexpired, else None."""
+        try:
+            body_s, sig_s = (token or "").split(".", 1)
+            body = body_s.encode()
+            # compare the base64 signature strings directly (b64decode is lenient about
+            # trailing junk, which would let a tampered token slip through).
+            expect = base64.urlsafe_b64encode(
+                hmac.new(self._secret, body, hashlib.sha256).digest()).decode()
+            if not hmac.compare_digest(sig_s, expect):
+                return None
+            payload = json.loads(base64.urlsafe_b64decode(body))
+            if int(payload.get("exp", 0)) < time.time():
+                return None
+            return payload.get("u")
+        except (ValueError, KeyError, json.JSONDecodeError):
+            return None

gini/server/policy.py

@@ -0,0 +1,57 @@
+"""Server-side policy — the single point that decides what may actually run.
+
+The student only ever sends a *topology*; the GINI server compiles it with GINI's own
+trusted compiler, then this validates + sanitizes the compiled `RuntimeConfig` before it
+can touch Docker. A student therefore cannot make the daemon do anything dangerous: no
+arbitrary images, no privileged containers, no host bind mounts, no non-Kata runtimes, no
+fabric/router/SDN/k8s/serverless (out of scope for a Kata experiment box), and resource
+use is capped. Published host ports are stripped — the server proxies consoles, which also
+avoids port collisions between students.
+"""
+from __future__ import annotations
+
+ALLOWED_RUNTIMES = {"", "kata"}        # plain container (runc) or a Kata microVM
+# only the cloud-plane experiment subset runs here:
+_FORBIDDEN_SECTIONS = ("machines", "routers", "ovs_switches", "controllers", "k8s", "faas")
+
+
+class PolicyError(Exception):
+    """A topology that compiled to something the server refuses to run."""
+
+
+def _image_base(image: str) -> str:
+    return (image or "").split("@", 1)[0]    # drop any digest; keep repo:tag
+
+
+def enforce(config, allowed_images, max_cpus: float = 2.0):
+    """Validate + sanitize a compiled RuntimeConfig *in place*; raise PolicyError on any
+    violation. Returns the (sanitized) config on success."""
+    for sec in _FORBIDDEN_SECTIONS:
+        if getattr(config, sec, None):
+            raise PolicyError(
+                f"this backend runs only Kata/container service topologies (found {sec})")
+    allow = set(allowed_images)
+    for s in config.services:
+        if _image_base(s.image) not in allow:
+            raise PolicyError(f"image not allowed: {s.image}")
+        if s.runtime not in ALLOWED_RUNTIMES:
+            raise PolicyError(f"runtime not allowed: {s.runtime!r}")
+        if s.privileged:
+            raise PolicyError("privileged containers are not allowed")
+        for v in s.volumes:
+            src = v.split(":", 1)[0]
+            if not src.startswith("."):           # only project-relative mounts
+                raise PolicyError(f"host bind mount not allowed: {v}")
+        if s.cpus and s.cpus > max_cpus:
+            s.cpus = max_cpus                      # clamp to the per-student cap
+        s.ports = []                               # never publish to the host (proxy instead)
+    return config
+
+
+def default_allowed_images() -> set:
+    """The curated image set students may run: exactly the images GINI itself uses (the
+    managed-service catalog) plus the compute base images. Anything else is rejected."""
+    from ..services.cloud_catalog import CATALOG
+    base = {_image_base(svc.image) for svc in CATALOG.values()}
+    base |= {"ubuntu:22.04", "alpine:latest"}      # Instance / Kata Instance / Container
+    return base

gini/server/session.py

@@ -0,0 +1,23 @@
+"""Per-student isolation: a stable namespaced compose project + workdir per user, so two
+students sharing one Kata host never collide (project names, container names, files)."""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+
+def _slug(name: str) -> str:
+    return re.sub(r"[^a-z0-9]", "", (name or "").lower()) or "user"
+
+
+class SessionManager:
+    def __init__(self, base_dir: str | Path) -> None:
+        self._base = Path(base_dir)
+
+    def project_name(self, user: str) -> str:
+        """The docker compose project (`-p`) for this user — keeps stacks separate."""
+        return "gini-" + _slug(user)
+
+    def workdir(self, user: str) -> Path:
+        """A per-user project directory under the server's base dir."""
+        return self._base / _slug(user)

gini/services/__init__.py

@@ -0,0 +1,15 @@
+from .compiler import RuntimeCompiler, RuntimeConfig
+from .gloader import GLoader
+from .orchestrator import Orchestrator, Sim, simulate, write_project
+from .persistence import PROJECT_EXT, load_project, save_project
+from .project import (
+    delete_experiment, experiment_path, is_project_dir, list_experiments, list_projects,
+    load_experiment, load_project_dir, rename_experiment, safe_name, save_experiment,
+    save_project_dir,
+)
+from .terminal import open_terminal
+
+__all__ = ["RuntimeCompiler", "RuntimeConfig", "GLoader", "Orchestrator", "Sim",
+           "simulate", "write_project", "open_terminal",
+           "PROJECT_EXT", "load_project", "save_project",
+           "is_project_dir", "list_projects", "load_project_dir", "save_project_dir"]

gini/services/boardflash.py

@@ -0,0 +1,248 @@
+"""Put firmware on a GINI32 board over USB, from inside gBuilder.
+
+This closes a gap that made the Hardware menu untrue. `boardsetup` drives the board's
+`gini> ` console — which only exists once firmware is already on the board. So a student
+holding a *virgin* board could not start from gBuilder at all: they needed a terminal,
+a 2 GB ESP-IDF install and the `gini32` CLI. Flashing is the missing first step.
+
+**Why flashing can live here but building cannot.** Building needs ESP-IDF: the whole
+toolchain, `export.sh` sourced into the environment, a per-chip `set-target`, and network
+access for the managed components. That is an instructor-grade install and no UI hides
+it. Flashing needs only `esptool` — pure Python, no toolchain — plus ~870 KB of prebuilt
+images. It is also the right split for the architecture: the firmware is built ONCE and
+is identical on every board, so building is a rare developer act while flashing is a
+routine student one.
+
+**The offsets are load-bearing — do not merge the images.** The flash layout is::
+
+    0x00000  bootloader.bin
+    0x08000  partition-table.bin
+    0x09000  nvs            <-- the board's identity: id, lab Wi-Fi, owner, LED pin
+    0x10000  gbridge.bin    (the app)
+
+`esptool merge_bin` would produce one image from 0x0 and pad the gaps with 0xFF, which
+runs straight over NVS at 0x9000 and silently unpairs the board, forgets its id and
+forgets the lab Wi-Fi. Writing the three files at their own offsets leaves 0x9000
+untouched, so re-flashing a board that is already set up keeps it set up. For the same
+reason there is deliberately no `erase_flash` here.
+
+Standard library plus `esptool`, invoked as a subprocess rather than imported: esptool's
+Python API has changed shape across major versions, while its command line has been
+stable for years, and a subprocess cannot take the GUI down with it.
+"""
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+# (offset, filename) — see the module docstring on why these are separate writes.
+IMAGES: tuple[tuple[int, str], ...] = (
+    (0x0000, "bootloader.bin"),
+    (0x8000, "partition-table.bin"),
+    (0x10000, "gbridge.bin"),
+)
+
+# NVS lives here. Nothing in this module may write at or across it.
+NVS_OFFSET = 0x9000
+
+# esptool reports chips in human form ("ESP32-S3"); the build tree names them the way
+# `idf.py set-target` does ("esp32s3"). One canonical direction, computed not hardcoded.
+def _canonical_target(chip: str) -> str:
+    return re.sub(r"[^a-z0-9]", "", (chip or "").strip().lower())
+
+
+def firmware_root() -> Path:
+    """Where prebuilt images live: ``<repo>/backend/gini32/firmware``.
+
+    Same repo-relative convention the orchestrator uses to find the backend, so a source
+    checkout works with no configuration. Override with GINI_FIRMWARE_DIR (a packaged
+    build, or an instructor pointing a lab at a freshly built image).
+    """
+    env = os.environ.get("GINI_FIRMWARE_DIR")
+    if env:
+        return Path(env).expanduser()
+    return Path(__file__).resolve().parents[4] / "backend" / "gini32" / "firmware"
+
+
+@dataclass
+class Firmware:
+    """A complete, flashable set of images for one chip."""
+    target: str
+    directory: Path
+    files: list[tuple[int, Path]] = field(default_factory=list)
+    build: str = ""            # the GB_BUILD marker read out of the app image
+
+    @property
+    def total_bytes(self) -> int:
+        return sum(p.stat().st_size for _, p in self.files if p.exists())
+
+
+def read_build_marker(app_bin: Path) -> str:
+    """Pull the GB_BUILD string out of a built app image.
+
+    A shipped binary can drift from the source beside it, and a stale flash masquerading
+    as a fresh one has already cost this project two debugging sessions. The firmware
+    stamps GB_BUILD into its boot log; finding the same string here lets the UI say
+    exactly what it is about to install, and lets a human compare the two.
+    """
+    try:
+        blob = app_bin.read_bytes()
+    except OSError:
+        return ""
+    m = re.search(rb"gbridge-\d+ \([^)]{0,40}\)", blob)
+    return m.group(0).decode("ascii", "replace") if m else ""
+
+
+def available(target: str, root: Path | None = None) -> Firmware | None:
+    """The flashable image set for `target`, or None if it is not shipped."""
+    root = root or firmware_root()
+    d = root / _canonical_target(target)
+    if not d.is_dir():
+        return None
+    files = [(off, d / name) for off, name in IMAGES]
+    if not all(p.is_file() for _, p in files):
+        return None                       # a partial set is worse than none
+    app = d / IMAGES[-1][1]
+    return Firmware(target=_canonical_target(target), directory=d, files=files,
+                    build=read_build_marker(app))
+
+
+def available_targets(root: Path | None = None) -> list[str]:
+    root = root or firmware_root()
+    if not root.is_dir():
+        return []
+    return sorted(d.name for d in root.iterdir()
+                  if d.is_dir() and available(d.name, root) is not None)
+
+
+# --------------------------------------------------------------------- esptool
+
+def esptool_argv() -> list[str]:
+    """How to invoke esptool. ``-m esptool`` uses the SAME interpreter running gBuilder,
+    so a virtualenv install is found without depending on PATH."""
+    return [sys.executable, "-m", "esptool"]
+
+
+def esptool_available(run=None) -> bool:
+    run = run or _run
+    try:
+        rc, _ = run(esptool_argv() + ["version"], timeout=20)
+        return rc == 0
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+
+def _run(argv: list[str], timeout: float = 300.0) -> tuple[int, str]:
+    p = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                       timeout=timeout)
+    return p.returncode, p.stdout.decode("utf-8", "replace")
+
+
+_CHIP_RE = re.compile(r"(ESP32(?:-[A-Z0-9]+)?)\b")
+
+
+def detect_chip(port: str, run=None) -> str:
+    """Ask the board what it is, so we flash an image built for that chip.
+
+    Returns a canonical target ("esp32s3") or "" if nothing answered. Flashing an image
+    built for the wrong chip produces a board that is bricked-looking rather than
+    obviously wrong — it boots into a reset loop with no console — so this is checked
+    rather than assumed.
+    """
+    run = run or _run
+    try:
+        rc, out = run(esptool_argv() + ["--port", port, "chip_id"], timeout=60)
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    if rc != 0:
+        return ""
+    # "Detecting chip type... ESP32-S3" / "Chip is ESP32-S3 (revision v0.2)"
+    for line in out.splitlines():
+        if "chip" not in line.lower():
+            continue
+        m = _CHIP_RE.search(line)
+        if m:
+            return _canonical_target(m.group(1))
+    return ""
+
+
+@dataclass
+class FlashResult:
+    ok: bool
+    message: str
+    output: str = ""
+    build: str = ""
+
+
+def flash(port: str, firmware: Firmware, baud: int = 460800,
+          run=None, on_progress=None) -> FlashResult:
+    """Write the three images at their own offsets. NVS is not touched.
+
+    `on_progress` is called with short human sentences; the caller decides where they go.
+    """
+    run = run or _run
+    say = on_progress or (lambda _m: None)
+
+    missing = [str(p) for _, p in firmware.files if not p.is_file()]
+    if missing:
+        return FlashResult(False, f"firmware image missing: {', '.join(missing)}")
+
+    # Belt and braces: if anyone ever edits IMAGES, catch an overlap with NVS here
+    # rather than discovering it as boards mysteriously forgetting who they are.
+    for off, path in firmware.files:
+        size = path.stat().st_size
+        if off < NVS_OFFSET < off + size:
+            return FlashResult(
+                False,
+                f"{path.name} at 0x{off:x} would run over NVS at 0x{NVS_OFFSET:x} — "
+                f"that would erase the board's identity, so refusing to flash.")
+
+    argv = esptool_argv() + ["--chip", firmware.target, "--port", port,
+                            "--baud", str(baud), "write_flash"]
+    for off, path in firmware.files:
+        argv += [hex(off), str(path)]
+
+    say(f"writing {firmware.total_bytes // 1024} KB to {port} …")
+    try:
+        rc, out = run(argv, timeout=600)
+    except subprocess.TimeoutExpired:
+        return FlashResult(False, "esptool did not finish — is the board still plugged in?")
+    except FileNotFoundError:
+        return FlashResult(False, "esptool is not installed (pip install esptool)")
+    except (OSError, subprocess.SubprocessError) as exc:
+        return FlashResult(False, f"could not run esptool: {exc}")
+
+    if rc != 0:
+        return FlashResult(False, _explain_failure(out), output=out)
+    return FlashResult(True, f"flashed {firmware.build or 'firmware'} — "
+                             f"the board keeps its id and lab Wi-Fi",
+                       output=out, build=firmware.build)
+
+
+_HINTS = (
+    ("Failed to connect", "the board did not answer. Hold BOOT, tap RESET, release BOOT, "
+                          "then try again — some boards cannot be reset over USB alone."),
+    ("Permission denied", "no permission for the serial port. On Linux: "
+                          "`sudo usermod -aG dialout $USER`, then log out and back in."),
+    ("could not open port", "the port is busy — close any serial monitor "
+                            "(including gBuilder's own Set Up a Board) and retry."),
+    ("Resource busy", "the port is busy — close any serial monitor and retry."),
+    ("does not match", "this image was built for a different chip than the board reports."),
+)
+
+
+def _explain_failure(out: str) -> str:
+    """Turn esptool's output into one sentence a student can act on.
+
+    Every failure here is physical — a cable, a button, a permission — and the raw output
+    buries that under a stack trace.
+    """
+    for needle, hint in _HINTS:
+        if needle.lower() in out.lower():
+            return hint
+    tail = [l for l in out.strip().splitlines() if l.strip()]
+    return tail[-1] if tail else "esptool failed with no output"