labtasker-server 2.0.0__py3-none-any.whl

labtasker_server/cli.py

@@ -0,0 +1,371 @@
+from __future__ import annotations
+
+import json
+import os
+import signal
+import socket
+import time
+from contextlib import suppress
+from pathlib import Path
+from typing import Annotated
+
+import typer
+import uvicorn
+
+from labtasker_server import __version__
+from labtasker_server.app import create_app
+from labtasker_server.config import ServerSettings
+from labtasker_server.database import DatabaseOwnershipError
+from labtasker_server.local import (
+    database_identity,
+    database_is_free,
+    ensure_local_daemon,
+    ensure_runtime_directory,
+    has_runtime_artifacts,
+    local_paths,
+    make_metadata,
+    metadata_matches_database,
+    metadata_owner_is_verified,
+    read_metadata,
+    remove_generation_artifacts,
+    remove_generation_socket,
+    remove_stale_artifacts,
+    require_local_capabilities,
+    socket_health,
+    startup_age,
+    throttle_remaining,
+    write_metadata,
+)
+from labtasker_server.logging import uvicorn_log_config
+
+app = typer.Typer(
+    help="Run the Labtasker v2 HTTP Server.",
+    add_completion=False,
+    no_args_is_help=True,
+    pretty_exceptions_enable=False,
+    rich_markup_mode=None,
+)
+
+
+@app.callback()
+def main() -> None:
+    """Run and manage the Labtasker v2 Server."""
+
+
+@app.command()
+def serve(
+    host: Annotated[
+        str,
+        typer.Option(help="Address to bind; a non-loopback address requires a token."),
+    ] = "127.0.0.1",
+    port: Annotated[
+        int,
+        typer.Option(min=1, max=65535, help="TCP port to listen on."),
+    ] = 8000,
+    database: Annotated[
+        Path,
+        typer.Option(help="SQLite database file owned by this Server process."),
+    ] = Path(".labtasker/server.db"),
+) -> None:
+    """Initialize the database and run one Labtasker v2 Server process.
+
+    Authentication is optional on loopback. For any non-loopback bind, set the
+    shared token with LABTASKER_SERVER_TOKEN; tokens are never accepted as a
+    command-line option. Run only one Server process for each SQLite file.
+
+    Examples:
+
+    
+      labtasker-server serve
+      LABTASKER_SERVER_TOKEN=secret labtasker-server serve \\
+        --host 0.0.0.0 --database /data/labtasker.db
+    """
+    try:
+        settings = ServerSettings.from_values(host=host, port=port, database=database)
+    except ValueError as error:
+        typer.echo(f"[labtasker-server] Server configuration error: {error}", err=True)
+        raise typer.Exit(1) from error
+    try:
+        application = create_app(settings)
+    except (DatabaseOwnershipError, OSError, RuntimeError) as error:
+        typer.echo(f"[labtasker-server] Server startup error: {error}", err=True)
+        raise typer.Exit(1) from error
+    uvicorn.run(
+        application,
+        host=settings.host,
+        port=settings.port,
+        log_level="info",
+        log_config=uvicorn_log_config(),
+    )
+
+
+@app.command()
+def start() -> None:
+    """Start the current directory's local daemon, or report the existing one."""
+    try:
+        paths = local_paths()
+        started, metadata = ensure_local_daemon(
+            paths.directory,
+            bypass_throttle=True,
+            server_version=__version__,
+            emit=lambda message: typer.echo(f"[labtasker-server] {message}", err=True),
+        )
+    except (OSError, RuntimeError) as error:
+        typer.echo(f"[labtasker-server] Local Server startup error: {error}", err=True)
+        raise typer.Exit(1) from error
+    action = "started" if started else "already running"
+    pid = metadata.pid if metadata is not None else "unknown"
+    typer.echo(
+        f"[labtasker-server] {action} local daemon pid={pid} socket={paths.socket}",
+        err=True,
+    )
+
+
+@app.command("_ensure-daemon", hidden=True)
+def ensure_daemon(
+    directory: Annotated[Path, typer.Option(hidden=True)],
+) -> None:
+    """Ensure one healthy local daemon for an automatic Client request."""
+    try:
+        paths = local_paths(directory)
+        started, metadata = ensure_local_daemon(
+            paths.directory,
+            bypass_throttle=False,
+            server_version=__version__,
+            emit=lambda message: typer.echo(f"[labtasker-server] {message}", err=True),
+        )
+    except (OSError, RuntimeError) as error:
+        typer.echo(f"[labtasker-server] automatic local startup error: {error}", err=True)
+        try:
+            result = _local_status(directory)
+        except (OSError, RuntimeError):
+            result = {
+                "state": "unhealthy",
+                "directory": str(directory.resolve()),
+                "database": None,
+                "socket": None,
+                "log": None,
+                "pid": None,
+                "version": None,
+                "retry_after_seconds": None,
+            }
+        result.update({"ok": False, "message": str(error)})
+        typer.echo(json.dumps(result, ensure_ascii=False))
+        raise typer.Exit(1) from error
+    result = _local_status(paths.directory)
+    result.update(
+        {
+            "ok": True,
+            "started": started,
+            "pid": metadata.pid if metadata is not None else result["pid"],
+            "version": (metadata.server_version if metadata is not None else result["version"]),
+        }
+    )
+    typer.echo(json.dumps(result, ensure_ascii=False))
+
+
+@app.command()
+def status() -> None:
+    """Print the current directory's local daemon status as JSON."""
+    try:
+        paths = local_paths()
+        result = _local_status(paths.directory)
+    except (OSError, RuntimeError) as error:
+        typer.echo(f"[labtasker-server] Local Server status error: {error}", err=True)
+        raise typer.Exit(1) from error
+    typer.echo(json.dumps(result, indent=2, ensure_ascii=False) + "\n", nl=False)
+
+
+@app.command()
+def stop(
+    force: Annotated[
+        bool,
+        typer.Option(help="Send SIGKILL after the 30-second graceful deadline."),
+    ] = False,
+) -> None:
+    """Stop the current directory's verified local daemon."""
+    try:
+        paths = local_paths()
+    except RuntimeError as error:
+        typer.echo(f"[labtasker-server] Local Server stop error: {error}", err=True)
+        raise typer.Exit(1) from error
+    metadata = read_metadata(paths)
+    if database_is_free(paths):
+        if has_runtime_artifacts(paths):
+            try:
+                remove_stale_artifacts(paths)
+            except RuntimeError as error:
+                typer.echo(f"[labtasker-server] Local Server stop error: {error}", err=True)
+                raise typer.Exit(1) from error
+        typer.echo("[labtasker-server] local daemon is already stopped", err=True)
+        return
+    if (
+        metadata is None
+        or metadata.role != "daemon"
+        or not metadata_owner_is_verified(paths, metadata)
+    ):
+        typer.echo(
+            "[labtasker-server] Local Server stop error: "
+            "database owner is not a verified local daemon.",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    with suppress(ProcessLookupError):
+        os.kill(metadata.pid, signal.SIGTERM)
+    typer.echo(f"[labtasker-server] stopping local daemon pid={metadata.pid}", err=True)
+    if _wait_for_exit(paths.directory, timeout=30.0):
+        remove_generation_artifacts(paths, metadata.generation)
+        typer.echo(f"[labtasker-server] stopped local daemon pid={metadata.pid}", err=True)
+        return
+    if not force:
+        typer.echo(
+            "[labtasker-server] Local Server stop error: daemon did not stop within "
+            "30 seconds; retry with --force.",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    current = read_metadata(paths)
+    if (
+        current is None
+        or current.generation != metadata.generation
+        or not metadata_owner_is_verified(paths, current)
+    ):
+        typer.echo(
+            "[labtasker-server] Local Server stop error: "
+            "daemon identity changed; refusing SIGKILL.",
+            err=True,
+        )
+        raise typer.Exit(1)
+    os.kill(current.pid, signal.SIGKILL)
+    typer.echo(f"[labtasker-server] force-stopping local daemon pid={current.pid}", err=True)
+    if not _wait_for_exit(paths.directory, timeout=5.0):
+        typer.echo(
+            "[labtasker-server] Local Server stop error: database ownership was not released.",
+            err=True,
+        )
+        raise typer.Exit(1)
+    remove_generation_artifacts(paths, current.generation)
+    typer.echo(f"[labtasker-server] stopped local daemon pid={current.pid}", err=True)
+
+
+@app.command()
+def logs() -> None:
+    """Print the current directory's complete local Server log."""
+    try:
+        path = local_paths().log
+        typer.echo(path.read_text(encoding="utf-8"), nl=False)
+    except FileNotFoundError:
+        return
+    except (OSError, UnicodeError) as error:
+        typer.echo(f"[labtasker-server] Local Server log error: {error}", err=True)
+        raise typer.Exit(1) from error
+
+
+@app.command("_daemon", hidden=True)
+def daemon(
+    directory: Annotated[Path, typer.Option(hidden=True)],
+    database_fd: Annotated[int, typer.Option(hidden=True)],
+    generation: Annotated[str, typer.Option(hidden=True)],
+    automatic_attempt_at: Annotated[float, typer.Option(hidden=True)],
+) -> None:
+    """Run one private local daemon process."""
+    paths = local_paths(directory)
+    listener: socket.socket | None = None
+    try:
+        require_local_capabilities()
+        ensure_runtime_directory(paths)
+        metadata = make_metadata(
+            paths,
+            generation=generation,
+            role="daemon",
+            pid=os.getpid(),
+            automatic_attempt_at=automatic_attempt_at,
+            database_fd=database_fd,
+            server_version=__version__,
+        )
+        write_metadata(paths, metadata)
+        if database_identity(database_fd) != (
+            metadata.database_device,
+            metadata.database_inode,
+        ):
+            raise RuntimeError("Inherited database identity changed.")
+        settings = ServerSettings(database=paths.database, token=None, database_fd=database_fd)
+        application = create_app(settings)
+        os.close(database_fd)
+        database_fd = -1
+
+        listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        listener.bind(str(paths.socket))
+        os.chmod(paths.socket, 0o600)
+        uvicorn.run(
+            application,
+            fd=listener.fileno(),
+            log_level="info",
+            log_config=uvicorn_log_config(),
+        )
+    except BaseException as error:
+        typer.echo(f"[labtasker-server] Local daemon failed: {error}", err=True)
+        raise
+    finally:
+        if database_fd >= 0:
+            os.close(database_fd)
+        if listener is not None:
+            listener.close()
+        # Preserve the attempt metadata so an unexpected exit remains throttled.
+        # An explicit successful stop removes the full generation itself.
+        remove_generation_socket(paths, generation)
+
+
+def _local_status(directory: Path) -> dict[str, object]:
+    paths = local_paths(directory)
+    metadata = read_metadata(paths)
+    if socket_health(paths):
+        state = "running"
+        retry_after: float | None = None
+    elif not database_is_free(paths):
+        state = (
+            "starting"
+            if metadata is not None
+            and metadata_owner_is_verified(paths, metadata)
+            and startup_age(metadata) is not None
+            else "unhealthy"
+        )
+        retry_after = None
+    else:
+        remaining = (
+            throttle_remaining(metadata)
+            if metadata is not None and metadata_matches_database(paths, metadata)
+            else 0.0
+        )
+        if remaining > 0:
+            state = "backoff"
+            retry_after = round(remaining, 3)
+        elif has_runtime_artifacts(paths):
+            state = "stale"
+            retry_after = None
+        else:
+            state = "stopped"
+            retry_after = None
+    verified = metadata is not None and metadata_owner_is_verified(paths, metadata)
+    return {
+        "state": state,
+        "directory": str(paths.directory),
+        "database": str(paths.database),
+        "socket": str(paths.socket),
+        "log": str(paths.log),
+        "pid": metadata.pid if verified and metadata is not None else None,
+        "version": metadata.server_version if verified and metadata is not None else None,
+        "retry_after_seconds": retry_after,
+    }
+
+
+def _wait_for_exit(directory: Path, *, timeout: float) -> bool:
+    paths = local_paths(directory)
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if database_is_free(paths):
+            return True
+        time.sleep(0.05)
+    return database_is_free(paths)

labtasker_server/config.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import ipaddress
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True, slots=True)
+class ServerSettings:
+    host: str = "127.0.0.1"
+    port: int = 8000
+    database: Path = Path(".labtasker/server.db")
+    token: str | None = None
+    database_fd: int | None = None
+
+    @classmethod
+    def from_values(
+        cls,
+        *,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        database: str | Path = ".labtasker/server.db",
+        token: str | None = None,
+    ) -> ServerSettings:
+        effective_token = os.environ.get("LABTASKER_SERVER_TOKEN") if token is None else token
+        if effective_token == "":
+            raise ValueError("LABTASKER_SERVER_TOKEN must not be empty.")
+        if not 1 <= port <= 65535:
+            raise ValueError("port must be between 1 and 65535.")
+        if effective_token is None and not _is_tokenless_host_allowed(host):
+            raise ValueError("A token is required when binding to a non-loopback host.")
+        return cls(host=host, port=port, database=Path(database), token=effective_token)
+
+
+def _is_tokenless_host_allowed(host: str) -> bool:
+    if host.lower() == "localhost":
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False

labtasker_server/database.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - explicit HTTP remains best effort off POSIX
+    fcntl = None  # type: ignore[assignment]
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import Engine, create_engine, event, inspect, text
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session, sessionmaker
+
+from labtasker_server.errors import DomainError
+from labtasker_server.models import QueueRow
+
+LOCAL_GITIGNORE = "*\n!.gitignore\n"
+
+
+class DatabaseOwnershipError(RuntimeError):
+    pass
+
+
+class Database:
+    def __init__(self, path: Path, *, ownership_fd: int | None = None) -> None:
+        self.path = path.resolve()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        labtasker_dir = next(
+            (parent for parent in self.path.parents if parent.name == ".labtasker"),
+            None,
+        )
+        if labtasker_dir is not None:
+            _ensure_local_gitignore(labtasker_dir)
+        self._ownership_fd: int | None = _acquire_database_ownership(self.path, ownership_fd)
+        self.engine = _create_sqlite_engine(self.path)
+        self._session_factory = sessionmaker(self.engine, expire_on_commit=False)
+
+    def initialize(self) -> None:
+        existing_tables = set(inspect(self.engine).get_table_names())
+        is_fresh = not existing_tables
+        if existing_tables and "alembic_version" not in existing_tables:
+            raise RuntimeError("Database has tables but is not a recognized Labtasker v2 schema.")
+
+        alembic_config = Config()
+        alembic_config.set_main_option(
+            "script_location",
+            str(Path(__file__).resolve().parent / "migrations"),
+        )
+        with self.engine.begin() as connection:
+            alembic_config.attributes["connection"] = connection
+            command.upgrade(alembic_config, "head")
+        _verify_sqlite_settings(self.engine)
+
+        if is_fresh:
+            with self.write_session() as session:
+                session.add(QueueRow(name="default"))
+
+    @contextmanager
+    def read_session(self) -> Iterator[Session]:
+        with self._session_factory() as session:
+            yield session
+
+    @contextmanager
+    def write_session(self) -> Iterator[Session]:
+        with self._session_factory() as session:
+            try:
+                session.execute(text("BEGIN IMMEDIATE"))
+                yield session
+                session.commit()
+            except OperationalError as error:
+                session.rollback()
+                if _is_sqlite_busy(error):
+                    raise DomainError(
+                        503,
+                        "database_busy",
+                        "The database is busy; retry the operation.",
+                        {},
+                    ) from error
+                raise
+            except BaseException:
+                session.rollback()
+                raise
+
+    def dispose(self) -> None:
+        self.engine.dispose()
+        if self._ownership_fd is not None:
+            os.close(self._ownership_fd)
+            self._ownership_fd = None
+
+
+def _acquire_database_ownership(path: Path, inherited_fd: int | None) -> int:
+    if inherited_fd is None:
+        fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600)
+        if fcntl is not None:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError as error:
+                os.close(fd)
+                raise DatabaseOwnershipError(
+                    f"Another Server process already owns database {path}."
+                ) from error
+    else:
+        fd = os.dup(inherited_fd)
+        os.set_inheritable(fd, False)
+
+    descriptor_stat = os.fstat(fd)
+    try:
+        path_stat = path.stat()
+    except OSError:
+        os.close(fd)
+        raise
+    if (descriptor_stat.st_dev, descriptor_stat.st_ino) != (
+        path_stat.st_dev,
+        path_stat.st_ino,
+    ):
+        os.close(fd)
+        raise DatabaseOwnershipError(
+            f"Database descriptor does not identify configured path {path}."
+        )
+    return fd
+
+
+def _create_sqlite_engine(path: Path) -> Engine:
+    engine = create_engine(
+        f"sqlite+pysqlite:///{path}",
+        connect_args={"check_same_thread": False, "timeout": 5.0},
+    )
+
+    @event.listens_for(engine, "connect")
+    def configure_connection(dbapi_connection: object, _: object) -> None:
+        cursor = dbapi_connection.cursor()  # type: ignore[attr-defined]
+        try:
+            cursor.execute("PRAGMA foreign_keys=ON")
+            cursor.execute("PRAGMA busy_timeout=5000")
+            cursor.execute("PRAGMA synchronous=FULL")
+            cursor.execute("PRAGMA journal_mode=WAL")
+        finally:
+            cursor.close()
+
+    return engine
+
+
+def _ensure_local_gitignore(labtasker_dir: Path) -> None:
+    try:
+        with (labtasker_dir / ".gitignore").open("x", encoding="utf-8", newline="\n") as stream:
+            stream.write(LOCAL_GITIGNORE)
+    except FileExistsError:
+        pass
+
+
+def _is_sqlite_busy(error: OperationalError) -> bool:
+    code = getattr(error.orig, "sqlite_errorcode", None)
+    return code in {5, 6} or "database is locked" in str(error.orig).lower()
+
+
+def _verify_sqlite_settings(engine: Engine) -> None:
+    with engine.connect() as connection:
+        actual = {
+            "journal_mode": connection.scalar(text("PRAGMA journal_mode")),
+            "foreign_keys": connection.scalar(text("PRAGMA foreign_keys")),
+            "busy_timeout": connection.scalar(text("PRAGMA busy_timeout")),
+            "synchronous": connection.scalar(text("PRAGMA synchronous")),
+        }
+    expected = {
+        "journal_mode": "wal",
+        "foreign_keys": 1,
+        "busy_timeout": 5000,
+        "synchronous": 2,
+    }
+    if actual != expected:
+        raise RuntimeError(f"Required SQLite settings were not applied: {actual!r}")

labtasker_server/errors.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class DomainError(Exception):
+    status_code: int
+    code: str
+    message: str
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def __str__(self) -> str:
+        return self.message
+
+
+def not_found(code: str, message: str, **details: Any) -> DomainError:
+    return DomainError(404, code, message, details)
+
+
+def conflict(code: str, message: str, **details: Any) -> DomainError:
+    return DomainError(409, code, message, details)
+
+
+def invalid(code: str, message: str, **details: Any) -> DomainError:
+    return DomainError(422, code, message, details)