workerz 0.0.0__py3-none-any.whl

workerz/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.0"

workerz/coordinator/__main__.py

@@ -0,0 +1,3 @@
+import os, uvicorn
+from workerz.coordinator.app import app
+uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("WORKERZ_HTTP_PORT", 8000)), log_level="info")

workerz/coordinator/app.py

@@ -0,0 +1,376 @@
+"""
+Coordinator — stateless re: jobs (all job state lives in Redis).
+In-memory: only TCP writer handles.
+
+HTTP routes:
+  POST   /job          submit job
+  GET    /job/{id}     get job (status + result envelope)
+  DELETE /job/{id}     cancel job
+
+TCP: workers connect, coordinator pushes Dispatch messages.
+"""
+
+import asyncio
+import json
+import os
+import uuid
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+
+import redis.asyncio as aioredis
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+from starlette.routing import Route
+
+from workerz.protocol import (
+    Cancel, Dispatch, JobStatus, JobUpdate, Ping, Pong, Register,
+    recv_msg, send_msg,
+)
+
+TCP_HOST  = os.environ.get("WORKERZ_TCP_HOST", "0.0.0.0")
+TCP_PORT  = int(os.environ.get("WORKERZ_TCP_PORT", 7777))
+HTTP_PORT = int(os.environ.get("WORKERZ_HTTP_PORT", 8000))
+REDIS_URL = os.environ.get("WORKERZ_REDIS_URL", "redis://localhost:6379")
+
+PING_INTERVAL = 10
+PONG_TIMEOUT  = 15
+
+# Only thing kept in memory: writer handles keyed by worker_id
+_writers: dict[str, asyncio.StreamWriter] = {}
+
+rdb: aioredis.Redis  = None
+queue: asyncio.Queue = None
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+# ── Redis helpers ─────────────────────────────────────────────────────────────
+
+async def _save_worker(worker_id: str, labels: list[str], status: str, current_job: str = ""):
+    await rdb.sadd("workerz:workers", worker_id)
+    await rdb.hset(f"workerz:worker:{worker_id}", mapping={
+        "id":          worker_id,
+        "labels":      json.dumps(labels),
+        "status":      status,
+        "current_job": current_job,
+        "last_seen":   _now(),
+    })
+
+
+async def _delete_worker(worker_id: str):
+    await rdb.srem("workerz:workers", worker_id)
+    await rdb.delete(f"workerz:worker:{worker_id}")
+
+
+async def _save_job(job: dict):
+    await rdb.sadd("workerz:jobs", job["id"])
+    await rdb.hset(f"workerz:job:{job['id']}", mapping={
+        "id":          job["id"],
+        "task":        job["task"],
+        "args":        json.dumps(job["args"]),
+        "kwargs":      json.dumps(job["kwargs"]),
+        "labels":      json.dumps(job.get("labels") or []),
+        "status":      job["status"],
+        "worker_id":   job.get("worker_id") or "",
+        "result":      job.get("result") or "",
+        "error":       job.get("error") or "",
+        "warnings":    json.dumps(job.get("warnings") or []),
+        "infos":       json.dumps(job.get("infos") or []),
+        "debug":       json.dumps(job.get("debug") or []),
+        "meta":        json.dumps(job.get("meta") or {}),
+        "created_at":  job["created_at"],
+        "finished_at": job.get("finished_at") or "",
+    })
+
+
+async def _load_job(job_id: str) -> dict | None:
+    raw = await rdb.hgetall(f"workerz:job:{job_id}")
+    if not raw:
+        return None
+    return {
+        "id":          raw["id"],
+        "task":        raw["task"],
+        "args":        json.loads(raw["args"]),
+        "kwargs":      json.loads(raw["kwargs"]),
+        "labels":      json.loads(raw.get("labels", "[]")),
+        "status":      raw["status"],
+        "worker_id":   raw.get("worker_id") or None,
+        "result":      json.loads(raw["result"]) if raw.get("result") else None,
+        "error":       raw.get("error") or None,
+        "warnings":    json.loads(raw.get("warnings", "[]")),
+        "infos":       json.loads(raw.get("infos", "[]")),
+        "debug":       json.loads(raw.get("debug", "[]")),
+        "meta":        json.loads(raw.get("meta", "{}")),
+        "created_at":  raw["created_at"],
+        "finished_at": raw.get("finished_at") or None,
+    }
+
+
+async def _load_all_workers() -> list[dict]:
+    ids  = await rdb.smembers("workerz:workers")
+    out  = []
+    for wid in ids:
+        raw = await rdb.hgetall(f"workerz:worker:{wid}")
+        if raw:
+            out.append({
+                "id":          raw["id"],
+                "labels":      json.loads(raw.get("labels", "[]")),
+                "status":      raw.get("status", "offline"),
+                "current_job": raw.get("current_job") or None,
+                "last_seen":   raw.get("last_seen"),
+            })
+    return out
+
+
+# ── Dispatch ──────────────────────────────────────────────────────────────────
+
+async def _find_idle_worker(labels: list[str]) -> tuple[str, list[str]] | None:
+    """Return (worker_id, worker_labels) of an idle worker matching all requested labels, or None."""
+    for wid, writer in _writers.items():
+        raw = await rdb.hgetall(f"workerz:worker:{wid}")
+        if not raw or raw.get("status") != "idle":
+            continue
+        worker_labels = set(json.loads(raw.get("labels", "[]")))
+        if not labels or set(labels).issubset(worker_labels):
+            return wid, list(worker_labels)
+    return None
+
+
+async def _dispatch_next():
+    """Try to assign queued jobs to idle workers."""
+    pending = []
+    while not queue.empty():
+        job = await queue.get()
+        match = await _find_idle_worker(job.get("labels") or [])
+        if match:
+            wid, _ = match
+            await _assign(wid, job)
+        else:
+            pending.append(job)
+    for job in pending:
+        await queue.put(job)
+
+
+async def _assign(worker_id: str, job: dict):
+    writer = _writers.get(worker_id)
+    if not writer:
+        await queue.put(job)
+        return
+
+    raw = await rdb.hgetall(f"workerz:worker:{worker_id}")
+    labels = json.loads(raw.get("labels", "[]")) if raw else []
+
+    job["status"]    = "dispatched"
+    job["worker_id"] = worker_id
+    await _save_job(job)
+    await _save_worker(worker_id, labels, "busy", job["id"])
+    await send_msg(writer, Dispatch(
+        job_id=job["id"], task=job["task"],
+        args=job["args"], kwargs=job["kwargs"],
+    ))
+
+
+# ── TCP ───────────────────────────────────────────────────────────────────────
+
+async def handle_worker(reader, writer):
+    worker_id = None
+    labels    = []
+    try:
+        msg = await recv_msg(reader)
+        if not isinstance(msg, Register):
+            writer.close()
+            return
+
+        worker_id = msg.worker_id
+        labels    = msg.labels
+        _writers[worker_id] = writer
+
+        await _save_worker(worker_id, labels, "idle")
+        print(f"[coordinator] worker {worker_id} connected labels={labels}")
+
+        await _dispatch_next()
+        asyncio.create_task(_ping_loop(worker_id, writer))
+
+        while True:
+            msg = await recv_msg(reader)
+
+            if isinstance(msg, Pong):
+                pass  # heartbeat ack, nothing to do
+
+            elif isinstance(msg, JobStatus):
+                job = await _load_job(msg.job_id)
+                if not job:
+                    continue
+
+                if msg.status == "running":
+                    job["status"] = "running"
+                    await _save_job(job)
+                    await _save_worker(worker_id, labels, "busy", msg.job_id)
+
+                elif msg.status in ("done", "error", "cancelled"):
+                    job.update({
+                        "status":      msg.status,
+                        "result":      json.loads(msg.result) if msg.result else None,
+                        "error":       msg.error,
+                        "warnings":    msg.warnings,
+                        "infos":       msg.infos,
+                        "debug":       msg.debug,
+                        "finished_at": _now(),
+                    })
+                    await _save_job(job)
+                    await _save_worker(worker_id, labels, "idle")
+                    await _dispatch_next()
+
+            elif isinstance(msg, JobUpdate):
+                # Merge meta into job
+                job = await _load_job(msg.job_id)
+                if job:
+                    job["meta"].update(msg.meta)
+                    await _save_job(job)
+
+    except (asyncio.IncompleteReadError, ConnectionResetError):
+        pass
+    finally:
+        if worker_id:
+            _writers.pop(worker_id, None)
+            await _save_worker(worker_id, labels, "offline")
+            print(f"[coordinator] worker {worker_id} disconnected")
+
+
+async def _ping_loop(worker_id: str, writer):
+    loop = asyncio.get_event_loop()
+    last_pong = loop.time()
+    try:
+        while _writers.get(worker_id) is writer:
+            await asyncio.sleep(PING_INTERVAL)
+            if _writers.get(worker_id) is not writer:
+                break
+            await send_msg(writer, Ping())
+            await asyncio.sleep(PONG_TIMEOUT)
+            # If no pong within window, drop connection
+            raw = await rdb.hgetall(f"workerz:worker:{worker_id}")
+            if raw.get("status") == "offline":
+                break
+    except Exception:
+        pass
+
+
+# ── HTTP ──────────────────────────────────────────────────────────────────────
+
+async def route_post_job(request: Request):
+    body = await request.json()
+    labels = body.get("labels") or []
+
+    # Fail fast if no worker with these labels is registered at all
+    if labels:
+        all_workers = await _load_all_workers()
+        label_set   = set(labels)
+        if not any(label_set.issubset(set(w["labels"])) for w in all_workers):
+            return JSONResponse(
+                {"error": "no_worker", "detail": f"no worker registered with labels {labels}"},
+                status_code=409,
+            )
+
+    job_id = str(uuid.uuid4())
+    job = {
+        "id":          job_id,
+        "task":        body["task"],
+        "args":        body.get("args", []),
+        "kwargs":      body.get("kwargs", {}),
+        "labels":      labels,
+        "status":      "queued",
+        "worker_id":   None,
+        "result":      None,
+        "error":       None,
+        "warnings":    [],
+        "infos":       [],
+        "debug":       [],
+        "meta":        {},
+        "created_at":  _now(),
+        "finished_at": None,
+    }
+    await _save_job(job)
+
+    match = await _find_idle_worker(labels)
+    if match:
+        wid, _ = match
+        await _assign(wid, job)
+    else:
+        await queue.put(job)
+
+    return JSONResponse({"job_id": job_id})
+
+
+async def route_get_job(request: Request):
+    job = await _load_job(request.path_params["job_id"])
+    if not job:
+        return JSONResponse({"error": "not found"}, status_code=404)
+    return JSONResponse(job)
+
+
+async def route_cancel_job(request: Request):
+    job_id = request.path_params["job_id"]
+    job    = await _load_job(job_id)
+    if not job:
+        return JSONResponse({"error": "not found"}, status_code=404)
+
+    if job["status"] == "queued":
+        job.update({"status": "cancelled", "finished_at": _now()})
+        await _save_job(job)
+        return JSONResponse({"cancelled": True})
+
+    if job["status"] in ("dispatched", "running"):
+        wid = job.get("worker_id")
+        if wid and _writers.get(wid):
+            await send_msg(_writers[wid], Cancel(job_id=job_id))
+        job.update({"status": "cancelled", "finished_at": _now()})
+        await _save_job(job)
+        return JSONResponse({"cancelling": True})
+
+    return JSONResponse({"error": "not cancellable", "status": job["status"]})
+
+
+# ── Lifespan ──────────────────────────────────────────────────────────────────
+
+@asynccontextmanager
+async def lifespan(app):
+    global rdb, queue
+    queue = asyncio.Queue()
+    rdb   = aioredis.from_url(REDIS_URL, decode_responses=True)
+
+    # Re-queue jobs that were in-flight when coordinator last died
+    for jid in await rdb.smembers("workerz:jobs"):
+        raw = await rdb.hgetall(f"workerz:job:{jid}")
+        if raw and raw.get("status") in ("queued", "dispatched"):
+            job = await _load_job(jid)
+            if job:
+                job["status"]    = "queued"
+                job["worker_id"] = None
+                await _save_job(job)
+                await queue.put(job)
+
+    # Mark all workers offline (they'll re-register via TCP)
+    for wid in await rdb.smembers("workerz:workers"):
+        raw = await rdb.hgetall(f"workerz:worker:{wid}")
+        if raw:
+            await rdb.hset(f"workerz:worker:{wid}", mapping={"status": "offline", "current_job": ""})
+
+    tcp = await asyncio.start_server(handle_worker, TCP_HOST, TCP_PORT)
+    asyncio.get_event_loop().create_task(tcp.serve_forever())
+    print(f"coordinator  http://0.0.0.0:{HTTP_PORT}  tcp://0.0.0.0:{TCP_PORT}  redis={REDIS_URL}")
+    yield
+    tcp.close()
+    await rdb.aclose()
+
+
+app = Starlette(
+    lifespan=lifespan,
+    routes=[
+        Route("/job",          route_post_job,    methods=["POST"]),
+        Route("/job/{job_id}", route_get_job,     methods=["GET"]),
+        Route("/job/{job_id}", route_cancel_job,  methods=["DELETE"]),
+    ],
+)

workerz/dashboard/__main__.py

@@ -0,0 +1,3 @@
+import os, uvicorn
+from workerz.dashboard.app import app
+uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("WORKERZ_DASHBOARD_PORT", 8080)), log_level="info")

workerz/dashboard/app.py

@@ -0,0 +1,80 @@
+import json
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+import redis.asyncio as aioredis
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import HTMLResponse, JSONResponse
+from starlette.routing import Route
+
+REDIS_URL = os.environ.get("WORKERZ_REDIS_URL", "redis://localhost:6379")
+HTTP_PORT = int(os.environ.get("WORKERZ_DASHBOARD_PORT", 8080))
+
+rdb: aioredis.Redis = None
+
+
+async def route_dashboard(request: Request):
+    html = (Path(__file__).parent / "index.html").read_text()
+    return HTMLResponse(html)
+
+
+async def route_jobs(request: Request):
+    ids  = await rdb.smembers("workerz:jobs")
+    jobs = []
+    for jid in ids:
+        raw = await rdb.hgetall(f"workerz:job:{jid}")
+        if raw:
+            jobs.append({
+                "id":          raw["id"],
+                "task":        raw["task"],
+                "labels":      json.loads(raw.get("labels", "[]")),
+                "status":      raw["status"],
+                "worker_id":   raw.get("worker_id") or None,
+                "result":      json.loads(raw["result"]) if raw.get("result") else None,
+                "error":       raw.get("error") or None,
+                "warnings":    json.loads(raw.get("warnings", "[]")),
+                "infos":       json.loads(raw.get("infos", "[]")),
+                "debug":       json.loads(raw.get("debug", "[]")),
+                "meta":        json.loads(raw.get("meta", "{}")),
+                "created_at":  raw["created_at"],
+                "finished_at": raw.get("finished_at") or None,
+            })
+    jobs.sort(key=lambda j: j["created_at"], reverse=True)
+    return JSONResponse(jobs)
+
+
+async def route_workers(request: Request):
+    ids     = await rdb.smembers("workerz:workers")
+    workers = []
+    for wid in ids:
+        raw = await rdb.hgetall(f"workerz:worker:{wid}")
+        if raw:
+            workers.append({
+                "id":          raw["id"],
+                "labels":      json.loads(raw.get("labels", "[]")),
+                "status":      raw.get("status", "offline"),
+                "current_job": raw.get("current_job") or None,
+                "last_seen":   raw.get("last_seen"),
+            })
+    return JSONResponse(workers)
+
+
+@asynccontextmanager
+async def lifespan(app):
+    global rdb
+    rdb = aioredis.from_url(REDIS_URL, decode_responses=True)
+    print(f"dashboard  http://0.0.0.0:{HTTP_PORT}  redis={REDIS_URL}")
+    yield
+    await rdb.aclose()
+
+
+app = Starlette(
+    lifespan=lifespan,
+    routes=[
+        Route("/",        route_dashboard),
+        Route("/jobs",    route_jobs),
+        Route("/workers", route_workers),
+    ],
+)

workerz/dashboard/index.html

@@ -0,0 +1,149 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8" />
+        <title>workerz</title>
+        <link rel="stylesheet" href="https://assets.ubuntu.com/v1/vanilla_framework_version_4.51.0.min.css" />
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
+        <style>
+            html { background:#666; padding:0.5in; overflow:auto; min-width:10in; display:flex; justify-content:center; }
+            body { box-sizing:border-box; width:8.27in; min-height:11.69in; margin:0; padding:0.4in;
+                   background:#fff; border-radius:1px; box-shadow:0 0 1in -0.25in rgba(0,0,0,.5); overflow:hidden; }
+            .p-table td, .p-table th { padding:0.25rem 0.5rem !important; font-size:0.72rem; }
+            .p-table th { font-size:0.65rem; }
+            .p-muted-heading { margin-bottom:0.4rem !important; font-size:0.65rem; }
+            .p-status-label { font-size:0.6rem; padding:1px 5px; }
+        </style>
+    </head>
+    <body class="u-no-margin u-no-padding">
+        <div style="background:#000;color:#fff;padding:0.35rem 0.5rem;border-radius:0.2em;
+                    margin-bottom:0.8rem;display:flex;align-items:center;justify-content:space-between;">
+            <div>
+                <strong style="letter-spacing:0.4em;text-transform:uppercase;font-size:0.8rem;">Workerz</strong>
+                <span class="u-text--muted" style="font-size:0.68rem;margin-left:0.8rem;">
+                    Dashboard — <span id="hdr-host"></span>
+                </span>
+            </div>
+            <div style="font-size:0.68rem;color:#aaa;text-align:right;line-height:1.8;">
+                Workers&nbsp;<strong id="hdr-worker-count" style="color:#fff">0</strong>
+                &ensp;Queued&nbsp;<strong id="hdr-queued" style="color:#fff">0</strong>
+                &ensp;<span id="hdr-time" style="color:#aaa"></span>
+            </div>
+        </div>
+
+        <!-- Stats -->
+        <div class="row u-no-margin--bottom" style="margin-bottom:0.8rem;">
+            <div class="col-2"><div class="p-card u-no-margin u-align--center" style="padding:0.4rem 0.2rem;">
+                <div class="p-heading--4 u-no-margin" id="stat-total">0</div>
+                <div class="u-text--muted" style="font-size:0.6rem;text-transform:uppercase;letter-spacing:.08em;">Total</div>
+            </div></div>
+            <div class="col-2"><div class="p-card u-no-margin u-align--center" style="padding:0.4rem 0.2rem;">
+                <div class="p-heading--4 u-no-margin" id="stat-running">0</div>
+                <div class="u-text--muted" style="font-size:0.6rem;text-transform:uppercase;letter-spacing:.08em;">Running</div>
+            </div></div>
+            <div class="col-2"><div class="p-card u-no-margin u-align--center" style="padding:0.4rem 0.2rem;">
+                <div class="p-heading--4 u-no-margin" id="stat-queued">0</div>
+                <div class="u-text--muted" style="font-size:0.6rem;text-transform:uppercase;letter-spacing:.08em;">Queued</div>
+            </div></div>
+            <div class="col-2"><div class="p-card u-no-margin u-align--center" style="padding:0.4rem 0.2rem;">
+                <div class="p-heading--4 u-no-margin" id="stat-done">0</div>
+                <div class="u-text--muted" style="font-size:0.6rem;text-transform:uppercase;letter-spacing:.08em;">Done</div>
+            </div></div>
+            <div class="col-2"><div class="p-card u-no-margin u-align--center" style="padding:0.4rem 0.2rem;">
+                <div class="p-heading--4 u-no-margin" id="stat-error">0</div>
+                <div class="u-text--muted" style="font-size:0.6rem;text-transform:uppercase;letter-spacing:.08em;">Errors</div>
+            </div></div>
+            <div class="col-2"><div class="p-card u-no-margin u-align--center" style="padding:0.4rem 0.2rem;">
+                <div class="p-heading--4 u-no-margin" id="stat-idle">0</div>
+                <div class="u-text--muted" style="font-size:0.6rem;text-transform:uppercase;letter-spacing:.08em;">Idle</div>
+            </div></div>
+        </div>
+
+        <!-- Workers -->
+        <div class="row" style="margin-bottom:0.8rem;">
+            <div class="col-12">
+                <p class="p-muted-heading">Workers</p>
+                <table class="p-table" style="width:100%">
+                    <thead><tr><th>ID</th><th>Labels</th><th>Status</th><th>Current Job</th><th>Last Seen</th></tr></thead>
+                    <tbody id="workers-body">
+                        <tr><td colspan="5" class="u-align--center u-text--muted"><em>no workers</em></td></tr>
+                    </tbody>
+                </table>
+            </div>
+        </div>
+
+        <!-- Jobs -->
+        <div class="row">
+            <div class="col-12">
+                <p class="p-muted-heading">Jobs</p>
+                <table class="p-table" style="width:100%">
+                    <thead><tr><th>Job ID</th><th>Task</th><th>Labels</th><th>Status</th><th>Worker</th><th>Created</th><th>Finished</th></tr></thead>
+                    <tbody id="jobs-body">
+                        <tr><td colspan="7" class="u-align--center u-text--muted"><em>no jobs</em></td></tr>
+                    </tbody>
+                </table>
+            </div>
+        </div>
+
+        <hr class="u-sv1" style="margin-top:0.8rem;" />
+        <p class="u-text--muted u-align--center u-no-margin--bottom" style="font-size:0.65rem;">
+            workerz dashboard — polling 1s — read only
+        </p>
+
+        <script>
+            const BADGE = {
+                idle:"positive", busy:"caution", offline:"", queued:"information",
+                dispatched:"", running:"caution", done:"positive", error:"negative", cancelled:""
+            };
+            const shortId = id => id ? id.slice(0,8) : "—";
+            const fmtTime = iso => iso ? new Date(iso).toLocaleTimeString("en-GB",{hour12:false}) : "—";
+            const badge   = s => { const t=BADGE[s]??""; return t ? `<span class="p-status-label--${t}">${s}</span>` : `<span class="p-status-label">${s}</span>`; };
+            const dot     = s => { const c={idle:"#0e8420",busy:"#f99b11",offline:"#aaa"}[s]??"#aaa"; return `<span style="color:${c};margin-right:4px;">&#x25cf;</span>`; };
+            const mono    = t => `<span style="font-family:monospace;color:#999;">${t}</span>`;
+            const chips   = arr => (arr||[]).map(l=>`<span style="background:#e5e5e5;border-radius:3px;padding:1px 5px;font-size:0.6rem;margin-right:2px;">${l}</span>`).join("");
+
+            function renderWorkers(workers) {
+                const $b = $("#workers-body").empty();
+                if (!workers.length) { $b.html('<tr><td colspan="5" class="u-align--center u-text--muted"><em>no workers</em></td></tr>'); return; }
+                workers.forEach(w => $b.append(`<tr>
+                    <td>${mono(shortId(w.id))}</td>
+                    <td>${chips(w.labels)}</td>
+                    <td>${dot(w.status)}${badge(w.status)}</td>
+                    <td>${w.current_job ? mono(shortId(w.current_job)) : "—"}</td>
+                    <td class="u-text--muted">${fmtTime(w.last_seen)}</td></tr>`));
+                $("#hdr-worker-count").text(workers.filter(w=>w.status!=="offline").length);
+                $("#stat-idle").text(workers.filter(w=>w.status==="idle").length);
+            }
+
+            function renderJobs(jobs) {
+                ["total","running","queued","done","error"].forEach(k => {
+                    $("#stat-"+k).text(k==="total" ? jobs.length : jobs.filter(j=>j.status===k).length);
+                });
+                $("#hdr-queued").text(jobs.filter(j=>j.status==="queued").length);
+                const $b = $("#jobs-body").empty();
+                if (!jobs.length) { $b.html('<tr><td colspan="7" class="u-align--center u-text--muted"><em>no jobs</em></td></tr>'); return; }
+                jobs.forEach(j => $b.append(`<tr>
+                    <td>${mono(shortId(j.id))}</td>
+                    <td><strong>${j.task}</strong></td>
+                    <td>${chips(j.labels)}</td>
+                    <td>${badge(j.status)}</td>
+                    <td>${j.worker_id ? mono(shortId(j.worker_id)) : "—"}</td>
+                    <td>${fmtTime(j.created_at)}</td>
+                    <td>${fmtTime(j.finished_at)}</td></tr>`));
+            }
+
+            function poll() {
+                $.when($.get("/workers"), $.get("/jobs")).done((w,j) => {
+                    renderWorkers(w[0]);
+                    renderJobs(j[0]);
+                });
+            }
+
+            $("#hdr-host").text(window.location.host);
+            $("#hdr-time").text(new Date().toLocaleTimeString("en-GB",{hour12:false}));
+            setInterval(()=>$("#hdr-time").text(new Date().toLocaleTimeString("en-GB",{hour12:false})), 1000);
+            poll();
+            setInterval(poll, 1000);
+        </script>
+    </body>
+</html>

workerz/exceptions.py

@@ -0,0 +1,11 @@
+class WorkerzError(Exception):
+    pass
+
+class JobNotFound(WorkerzError):
+    pass
+
+class WorkerError(WorkerzError):
+    pass
+
+class NoWorkerAvailable(WorkerzError):
+    pass

workerz/protocol.py

@@ -0,0 +1,61 @@
+import struct
+import msgspec
+
+
+class Register(msgspec.Struct, tag=True):
+    worker_id: str
+    labels:    list[str]
+
+
+class Dispatch(msgspec.Struct, tag=True):
+    job_id: str
+    task:   str
+    args:   list
+    kwargs: dict
+
+
+class Cancel(msgspec.Struct, tag=True):
+    job_id: str
+
+
+class JobStatus(msgspec.Struct, tag=True):
+    job_id:   str
+    status:   str                    # running | done | error | cancelled
+    result:   str | None = None      # json-encoded
+    error:    str | None = None
+    warnings: list[str] = []
+    infos:    list[str] = []
+    debug:    list[str] = []
+
+
+class JobUpdate(msgspec.Struct, tag=True):
+    """Worker pushes arbitrary meta mid-run."""
+    job_id: str
+    meta:   dict
+
+
+class Ping(msgspec.Struct, tag=True):
+    pass
+
+
+class Pong(msgspec.Struct, tag=True):
+    worker_id: str
+
+
+Message = Register | Dispatch | Cancel | JobStatus | JobUpdate | Ping | Pong
+
+_encoder = msgspec.json.Encoder()
+_decoder = msgspec.json.Decoder(Message)
+
+
+async def send_msg(writer, msg):
+    data = _encoder.encode(msg)
+    writer.write(struct.pack(">I", len(data)) + data)
+    await writer.drain()
+
+
+async def recv_msg(reader):
+    raw_len = await reader.readexactly(4)
+    length  = struct.unpack(">I", raw_len)[0]
+    data    = await reader.readexactly(length)
+    return _decoder.decode(data)

workerz/sdk/client.py

@@ -0,0 +1,93 @@
+import time
+import httpx
+from workerz.task import task  # re-exported for convenience
+from workerz.exceptions import JobNotFound, WorkerError, NoWorkerAvailable
+
+
+class Job:
+    def __init__(self, data: dict, client: "Client"):
+        self._data   = data
+        self._client = client
+
+    @property
+    def id(self) -> str:
+        return self._data["id"]
+
+    @property
+    def status(self) -> str:
+        return self._data["status"]
+
+    @property
+    def done(self) -> bool:
+        return self._data["status"] in ("done", "error", "cancelled")
+
+    def _refresh(self):
+        self._data = self._client._get_job_raw(self.id)
+
+    def wait(self, poll: float = 0.5, timeout: float = None) -> "Job":
+        start = time.monotonic()
+        while not self.done:
+            if timeout and time.monotonic() - start > timeout:
+                raise TimeoutError(f"job {self.id} did not finish within {timeout}s")
+            time.sleep(poll)
+            self._refresh()
+        return self
+
+    def get(self):
+        if not self.done:
+            self.wait()
+        return self._data.get("result")
+
+    def get_or_raise(self):
+        if not self.done:
+            self.wait()
+        if self._data["status"] == "error":
+            raise WorkerError(self._data.get("error") or "job failed")
+        return self._data.get("result")
+
+    @property
+    def result(self):        return self._data.get("result")
+    @property
+    def error(self):         return self._data.get("error")
+    @property
+    def warnings(self):      return self._data.get("warnings", [])
+    @property
+    def infos(self):         return self._data.get("infos", [])
+    @property
+    def debug(self):         return self._data.get("debug", [])
+    @property
+    def meta(self):          return self._data.get("meta", {})
+
+    def __repr__(self):
+        return f"<Job {self.id[:8]} status={self.status}>"
+
+
+class Client:
+    def __init__(self, base_url: str = "http://127.0.0.1:8000"):
+        self.base = base_url.rstrip("/")
+
+    def _get_job_raw(self, job_id: str) -> dict:
+        resp = httpx.get(f"{self.base}/job/{job_id}")
+        if resp.status_code == 404:
+            raise JobNotFound(job_id)
+        resp.raise_for_status()
+        return resp.json()
+
+    def run(self, task: str, args: list = None, kwargs: dict = None, labels: list[str] = None) -> Job:
+        resp = httpx.post(f"{self.base}/job", json={
+            "task": task, "args": args or [], "kwargs": kwargs or {}, "labels": labels or [],
+        })
+        if resp.status_code == 409:
+            raise NoWorkerAvailable(resp.json().get("detail", "no worker available"))
+        resp.raise_for_status()
+        return Job(self._get_job_raw(resp.json()["job_id"]), self)
+
+    def job(self, job_id: str) -> Job:
+        return Job(self._get_job_raw(job_id), self)
+
+    def cancel(self, job_id: str) -> dict:
+        resp = httpx.delete(f"{self.base}/job/{job_id}")
+        if resp.status_code == 404:
+            raise JobNotFound(job_id)
+        resp.raise_for_status()
+        return resp.json()

workerz/task.py

@@ -0,0 +1,3 @@
+def task(fn):
+    fn._is_workerz_task = True
+    return fn

workerz/worker/__main__.py

@@ -0,0 +1,15 @@
+import asyncio
+import argparse
+import os
+import sys
+
+parser = argparse.ArgumentParser(prog="python -m workerz.worker")
+parser.add_argument("file", help="path to tasks .py file")
+args = parser.parse_args()
+
+if not os.environ.get("WORKERZ_LABELS"):
+    print("ERROR: WORKERZ_LABELS env var is required", file=sys.stderr)
+    sys.exit(1)
+
+from workerz.worker.worker import main
+asyncio.run(main(filepath=args.file))

workerz/worker/worker.py

@@ -0,0 +1,130 @@
+import asyncio
+import importlib.util
+import inspect
+import json
+import os
+import uuid
+from pathlib import Path
+
+from workerz.protocol import Cancel, Dispatch, JobStatus, JobUpdate, Ping, Pong, Register, recv_msg, send_msg
+
+COORDINATOR_HOST = os.environ.get("WORKERZ_COORDINATOR_HOST", "127.0.0.1")
+COORDINATOR_TCP  = int(os.environ.get("WORKERZ_COORDINATOR_TCP", 7777))
+LABELS           = [l.strip() for l in os.environ.get("WORKERZ_LABELS", "").split(",") if l.strip()]
+
+
+class TaskContext:
+    """Injected as first arg into every task. Collects logs + allows mid-run meta pushes."""
+
+    def __init__(self, job_id: str, writer):
+        self.job_id   = job_id
+        self._writer  = writer
+        self.warnings: list[str] = []
+        self.infos:    list[str] = []
+        self.debug:    list[str] = []
+
+    def warn(self, msg: str):
+        self.warnings.append(str(msg))
+
+    def info(self, msg: str):
+        self.infos.append(str(msg))
+
+    def debug(self, msg: str):
+        self.debug.append(str(msg))
+
+    async def update(self, **meta):
+        """Push arbitrary meta to coordinator immediately (visible in dashboard)."""
+        await send_msg(self._writer, JobUpdate(job_id=self.job_id, meta=meta))
+
+
+def load_tasks(filepath: str) -> dict:
+    path   = Path(filepath).resolve()
+    spec   = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return {
+        name: fn
+        for name, fn in inspect.getmembers(module, inspect.isfunction)
+        if getattr(fn, "_is_workerz_task", False)
+    }
+
+
+async def run_task(fn, ctx, args, kwargs):
+    if asyncio.iscoroutinefunction(fn):
+        return await fn(ctx, *args, **kwargs)
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(None, lambda: fn(ctx, *args, **kwargs))
+
+
+async def main(filepath: str):
+    if not LABELS:
+        raise RuntimeError("WORKERZ_LABELS is required (comma-separated list)")
+
+    tasks     = load_tasks(filepath)
+    worker_id = f"worker-{uuid.uuid4().hex[:8]}"
+    active: dict[str, asyncio.Task] = {}
+
+    print(f"[{worker_id}] labels={LABELS} tasks={list(tasks.keys())}")
+    print(f"[{worker_id}] connecting {COORDINATOR_HOST}:{COORDINATOR_TCP}")
+
+    reader, writer = await asyncio.open_connection(COORDINATOR_HOST, COORDINATOR_TCP)
+
+    await send_msg(writer, Register(worker_id=worker_id, labels=LABELS))
+    print(f"[{worker_id}] registered")
+
+    async def execute(msg: Dispatch):
+        fn  = tasks.get(msg.task)
+        ctx = TaskContext(job_id=msg.job_id, writer=writer)
+
+        if not fn:
+            await send_msg(writer, JobStatus(
+                job_id=msg.job_id, status="error",
+                error=f"unknown task: {msg.task}",
+            ))
+            return
+
+        await send_msg(writer, JobStatus(job_id=msg.job_id, status="running"))
+        handle = asyncio.create_task(run_task(fn, ctx, msg.args, msg.kwargs))
+        active[msg.job_id] = handle
+
+        try:
+            result = await handle
+            await send_msg(writer, JobStatus(
+                job_id=msg.job_id, status="done",
+                result=json.dumps(result) if result is not None else None,
+                warnings=ctx.warnings, infos=ctx.infos, debug=ctx.debug,
+            ))
+
+        except asyncio.CancelledError:
+            await send_msg(writer, JobStatus(
+                job_id=msg.job_id, status="cancelled",
+                warnings=ctx.warnings, infos=ctx.infos, debug=ctx.debug,
+            ))
+
+        except Exception as e:
+            await send_msg(writer, JobStatus(
+                job_id=msg.job_id, status="error",
+                error=str(e),
+                warnings=ctx.warnings, infos=ctx.infos, debug=ctx.debug,
+            ))
+
+        finally:
+            active.pop(msg.job_id, None)
+
+    while True:
+        try:
+            msg = await recv_msg(reader)
+        except (asyncio.IncompleteReadError, ConnectionResetError):
+            print(f"[{worker_id}] coordinator disconnected, reconnecting...")
+            break
+
+        if isinstance(msg, Dispatch):
+            asyncio.create_task(execute(msg))
+        elif isinstance(msg, Cancel):
+            handle = active.get(msg.job_id)
+            if handle:
+                handle.cancel()
+        elif isinstance(msg, Ping):
+            await send_msg(writer, Pong(worker_id=worker_id))
+
+    writer.close()

workerz-0.0.0.dist-info/METADATA

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: workerz
+Version: 0.0.0
+Summary: Better Workers.
+Requires-Python: >=3.11
+Provides-Extra: coordinator
+Requires-Dist: starlette>=0.41; extra == "coordinator"
+Requires-Dist: uvicorn>=0.29; extra == "coordinator"
+Requires-Dist: redis>=5.0; extra == "coordinator"
+Requires-Dist: msgspec>=0.18; extra == "coordinator"
+Provides-Extra: dashboard
+Requires-Dist: starlette>=0.41; extra == "dashboard"
+Requires-Dist: uvicorn>=0.29; extra == "dashboard"
+Requires-Dist: redis>=5.0; extra == "dashboard"
+Provides-Extra: worker
+Requires-Dist: msgspec>=0.18; extra == "worker"
+Provides-Extra: sdk
+Requires-Dist: httpx>=0.27; extra == "sdk"
+Provides-Extra: full
+Requires-Dist: workerz[coordinator]; extra == "full"
+Requires-Dist: workerz[dashboard]; extra == "full"
+Requires-Dist: workerz[worker]; extra == "full"
+Requires-Dist: workerz[sdk]; extra == "full"

workerz-0.0.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+workerz/__init__.py,sha256=ShXQBVjyiSOHxoQJS2BvNG395W4KZfqMxZWBAR0MZrE,22
+workerz/exceptions.py,sha256=tKRcA4IBy3xfuz9Fr3OzlTlVa0v3IadPLSP5EAIefHw,175
+workerz/protocol.py,sha256=zmTxV_axUQIbxgU88V1nj03uFh9Su3Ot0ac4-AE4ZLA,1326
+workerz/task.py,sha256=CYwEBiN5ODnrZe61_ADaTxet22DKYmBgTNj4G7mmyJY,59
+workerz/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+workerz/coordinator/__main__.py,sha256=T4wx16RdIa8GAvqvmgDMLl-lvPACehlm3jGaKeSrNzk,163
+workerz/coordinator/app.py,sha256=ZRHLinyrmmMyQgz6kK4q373WiOB9egDcfjY7sRY3iCQ,13456
+workerz/dashboard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+workerz/dashboard/__main__.py,sha256=tAfaAda45mkOLNXoxnmJOMjU3MP-s2I92QiztJICcGY,166
+workerz/dashboard/app.py,sha256=f4ftKN319mUtupaP2FmgO0RNf-kkrtNqeiY5lp6cu9o,2710
+workerz/dashboard/index.html,sha256=qGn-KFq437AbMn6biJgiv00sVyndmmDzt5jKAnPEAMc,8913
+workerz/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+workerz/sdk/client.py,sha256=Q1bXejD_RcnEloP03uIDwn2aKKRpyK1srflF5XHrV94,3052
+workerz/worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+workerz/worker/__main__.py,sha256=qVts2Air4ZlG0-BeDV0INcRtPwigHs546SB9LjpnBXo,413
+workerz/worker/worker.py,sha256=5IKPuWeTNzj0eyG0h-yEb1YTlqbH3GTKCj4OWVJ1I6o,4404
+workerz-0.0.0.dist-info/METADATA,sha256=l_iduMGWzlieXtfKuZ0gz4q0eLYDgt6NAvMnLNIHy7A,869
+workerz-0.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+workerz-0.0.0.dist-info/top_level.txt,sha256=zu2JHkULJfxCuVXG-7-8Bgqj9aEq7WmWixQ0QrEzgF0,8
+workerz-0.0.0.dist-info/RECORD,,

workerz-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

workerz-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+workerz