funmill-api 0.1.2__py3-none-any.whl

funmill/api/backends/service.py

@@ -0,0 +1,107 @@
+import os
+import signal
+import subprocess
+import time
+from pathlib import Path
+
+
+def _pid_path(name: str, directory: Path) -> Path:
+    return directory / f"{name}.pid"
+
+
+def _read_pid(path: Path) -> int:
+    try:
+        pid = int(path.read_text(encoding="utf-8").strip())
+        if pid <= 1:
+            raise ValueError
+        return pid
+    except (OSError, ValueError) as exc:
+        raise RuntimeError(f"invalid service PID file: {path}") from exc
+
+
+def _is_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def start_background(
+    name: str, command: list[str], environment: dict[str, str], directory: Path
+) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    pid_path = _pid_path(name, directory)
+    if pid_path.exists():
+        pid = _read_pid(pid_path)
+        if _is_running(pid):
+            raise RuntimeError(f"{name} is already running with pid {pid}")
+        pid_path.unlink()
+
+    log_path = directory / f"{name}.log"
+    with log_path.open("ab") as output:
+        process = subprocess.Popen(
+            command,
+            env=environment,
+            stdin=subprocess.DEVNULL,
+            stdout=output,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+    pid_path.write_text(f"{process.pid}\n", encoding="utf-8")
+    try:
+        return_code = process.wait(timeout=0.2)
+    except subprocess.TimeoutExpired:
+        print(f"started {name}: pid={process.pid}, log={log_path}")
+        return
+    pid_path.unlink(missing_ok=True)
+    raise RuntimeError(
+        f"{name} exited during startup with code {return_code}; see {log_path}"
+    )
+
+
+def status_background(name: str, directory: Path) -> bool:
+    pid_path = _pid_path(name, directory)
+    if not pid_path.exists():
+        print(f"{name} is stopped")
+        return False
+    pid = _read_pid(pid_path)
+    if not _is_running(pid):
+        pid_path.unlink()
+        print(f"{name} is stopped")
+        return False
+    print(f"{name} is running: pid={pid}")
+    return True
+
+
+def stop_background(name: str, directory: Path, timeout: float = 10) -> None:
+    pid_path = _pid_path(name, directory)
+    if not pid_path.exists():
+        print(f"{name} is already stopped")
+        return
+    pid = _read_pid(pid_path)
+    if not _is_running(pid):
+        pid_path.unlink()
+        print(f"{name} is already stopped")
+        return
+    try:
+        group_id = os.getpgid(pid)
+    except ProcessLookupError:
+        pid_path.unlink()
+        print(f"{name} is already stopped")
+        return
+    if group_id != pid:
+        raise RuntimeError(
+            f"refusing to stop {name}: pid {pid} is not its process group"
+        )
+
+    os.killpg(group_id, signal.SIGTERM)
+    deadline = time.monotonic() + timeout
+    while _is_running(pid):
+        if time.monotonic() >= deadline:
+            raise RuntimeError(f"timed out waiting for {name} pid {pid} to stop")
+        time.sleep(0.1)
+    pid_path.unlink(missing_ok=True)
+    print(f"stopped {name}: pid={pid}")

funmill/api/backends/windmill/README.md

@@ -0,0 +1,112 @@
+# Windmill 简单部署
+
+以下方式不使用 Docker，适用于 Linux x86_64。需要一套可用的 PostgreSQL，
+以及 `python3`、`uv` 和 Bash；Windmill 会自动初始化数据库表。
+
+## 1. 安装 Funmill 和 Windmill
+
+```bash
+uv sync
+uv run funmill install windmill
+```
+
+Windmill 的二进制和配置文件都会放在
+`~/.farfarfun/funmill/services/windmill/`。安装器会校验官方发行包的
+SHA-256；设置 `FUNMILL_HOME` 可以修改 Funmill 的数据根目录。其他第三方服务
+同样使用 `~/.farfarfun/funmill/services/<service>/` 目录。
+
+## 2. 准备数据库
+
+在 PostgreSQL 中创建用户和数据库：
+
+```bash
+sudo -u postgres createuser --pwprompt windmill
+sudo -u postgres createdb --owner=windmill windmill
+```
+
+编辑 `~/.farfarfun/funmill/services/windmill/.env`，写入连接地址：
+
+```dotenv
+DATABASE_URL=postgresql://windmill:数据库密码@127.0.0.1:5432/windmill
+MODE=standalone
+SERVER_BIND_ADDR=0.0.0.0
+```
+
+安装器首次创建该文件时会设置 `0600` 权限，且不会覆盖已有配置。
+Funmill 启动 Windmill 时会固定使用 `SERVER_BIND_ADDR=0.0.0.0`。
+
+## 3. 启动 Windmill
+
+`standalone` 模式会在一个进程中同时运行 Server 和一个 Worker：
+
+```bash
+uv run funmill start windmill
+```
+
+该命令会在后台启动 Windmill，并打印 PID 和日志路径。PID 与日志分别保存在
+`~/.farfarfun/funmill/services/windmill/windmill.pid` 和 `windmill.log`。
+Windmill 的 Web 界面和原生 API 固定监听 `0.0.0.0:8813`。本机打开
+<http://127.0.0.1:8813>，首次登录使用：
+
+```text
+admin@windmill.dev / changeme
+```
+
+立即修改密码，并在 `admins` workspace 中创建 API Token。
+
+## 4. 启动 Funmill API
+
+回到 Funmill 仓库根目录执行：
+
+```bash
+FUNMILL_API_KEY='自行设置的接口密钥' \
+FUNMILL_BACKEND=windmill \
+WINDMILL_URL='http://127.0.0.1:8813' \
+WINDMILL_WORKSPACE=admins \
+WINDMILL_TOKEN='刚创建的Windmill-Token' \
+uv run funmill start
+```
+
+Funmill API 固定监听 `0.0.0.0:8812`，并保持前台运行。
+
+验证：
+
+```bash
+curl http://127.0.0.1:8812/health
+FUNMILL_API_KEY='自行设置的接口密钥' ./scripts/smoke.sh
+```
+
+## 5. 管理后台服务
+
+```bash
+uv run funmill status windmill
+uv run funmill restart windmill
+uv run funmill stop windmill
+tail -f ~/.farfarfun/funmill/services/windmill/windmill.log
+```
+
+重复启动会被 PID 文件拦截。`stop` 会向 Windmill 的独立进程组发送 `SIGTERM`，
+等待正常退出后删除 PID 文件；服务异常退出后，`status` 会清理失效 PID。
+
+## 增加 Worker
+
+需要更高并发时，在同一台或其他机器额外启动 Worker。每个进程使用不同的
+`WORKER_SUFFIX`：
+
+```bash
+MODE=worker WORKER_SUFFIX=worker2 uv run funmill start windmill
+MODE=worker WORKER_SUFFIX=worker3 uv run funmill start windmill
+```
+
+每个 Worker 使用 `windmill-<WORKER_SUFFIX>.pid` 和同名日志。查询或停止某个
+Worker 时需要传入相同环境变量，例如：
+
+```bash
+MODE=worker WORKER_SUFFIX=worker2 uv run funmill status windmill
+MODE=worker WORKER_SUFFIX=worker2 uv run funmill stop windmill
+```
+
+需要开机启动、自动重启和日志轮转时，使用现有的 systemd 或进程管理器直接
+管理已安装的 Windmill 二进制。不要在
+一个普通 Worker 进程中设置 `NUM_WORKERS>1`；Windmill 会因为隔离安全限制将
+它回退为 1，多个独立 Worker 进程更明确。

funmill/api/backends/windmill/__init__.py

@@ -0,0 +1,414 @@
+import json
+import os
+from typing import Any
+from uuid import UUID
+
+import httpx
+
+from funmill.api.models import (
+    TaskDefinition,
+    TaskInfo,
+    TaskLanguage,
+    TaskLogs,
+    TaskProgress,
+    TaskResult,
+    TaskStatus,
+    TaskSubmit,
+    WorkflowSubmit,
+)
+from funmill.api.ports import THIRD_PARTY_WEB_PORT
+
+from ..base import BackendError, TaskBackend
+
+_CALLBACK_SOURCE = """import json
+import os
+from urllib.request import Request, urlopen
+
+def main(callback_url: str, status: str, payload):
+    body = {
+        "task_id": os.environ["WM_ROOT_JOB_ID"],
+        "status": status,
+        "payload": payload,
+    }
+    request = Request(
+        callback_url,
+        data=json.dumps(body, default=str).encode(),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urlopen(request, timeout=10):
+        return payload
+"""
+
+_DEPENDENCY_SOURCE = """from datetime import datetime, timezone
+import os
+from wmill import get_job
+
+def main(dependencies: list[str], timeout_seconds: int):
+    root = get_job(os.environ["WM_ROOT_JOB_ID"])
+    created_at = datetime.fromisoformat(root["created_at"].replace("Z", "+00:00"))
+    if (datetime.now(timezone.utc) - created_at).total_seconds() > timeout_seconds:
+        raise TimeoutError("dependency wait timed out")
+
+    for task_id in dependencies:
+        job = get_job(task_id)
+        if "success" not in job:
+            return False
+        if job.get("canceled"):
+            raise RuntimeError(f"dependency {task_id} was canceled")
+        if not job["success"]:
+            raise RuntimeError(f"dependency {task_id} failed")
+    return True
+"""
+
+_RESULT_SOURCE = """def main(results):
+    return results
+"""
+
+_LANGUAGES = {
+    TaskLanguage.PYTHON: "python3",
+    TaskLanguage.BASH: "bash",
+}
+
+
+class WindmillBackend(TaskBackend):
+    name = "windmill"
+
+    def __init__(
+        self,
+        base_url: str,
+        workspace: str,
+        token: str,
+        timeout: float = 30,
+        client: httpx.Client | None = None,
+    ) -> None:
+        self.token = token
+        self.base_url = base_url.rstrip("/")
+        self.client = client or httpx.Client(
+            base_url=f"{self.base_url}/api/w/{workspace}/",
+            timeout=timeout,
+        )
+
+    @classmethod
+    def from_env(cls) -> "WindmillBackend":
+        return cls(
+            base_url=os.getenv(
+                "WINDMILL_URL", f"http://127.0.0.1:{THIRD_PARTY_WEB_PORT}"
+            ),
+            workspace=os.getenv("WINDMILL_WORKSPACE", "admins"),
+            token=os.getenv("WINDMILL_TOKEN", ""),
+            timeout=float(os.getenv("WINDMILL_TIMEOUT", "30")),
+        )
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
+        if not self.token:
+            raise BackendError("WINDMILL_TOKEN is not configured", 503)
+        try:
+            response = self.client.request(
+                method,
+                path,
+                headers={"Authorization": f"Bearer {self.token}"},
+                **kwargs,
+            )
+        except httpx.TimeoutException as exc:
+            raise BackendError("Windmill request timed out", 504) from exc
+        except httpx.HTTPError as exc:
+            raise BackendError(f"Windmill is unavailable: {exc}", 502) from exc
+        if response.is_error:
+            status_code = 404 if response.status_code == 404 else 502
+            detail = response.text.strip()[:500] or f"HTTP {response.status_code}"
+            raise BackendError(f"Windmill rejected the request: {detail}", status_code)
+        return response
+
+    def health_check(self) -> None:
+        if not self.token:
+            raise BackendError("WINDMILL_TOKEN is not configured", 503)
+        try:
+            response = self.client.get(f"{self.base_url}/api/health/status")
+        except httpx.TimeoutException as exc:
+            raise BackendError("Windmill health check timed out", 504) from exc
+        except httpx.HTTPError as exc:
+            raise BackendError(f"Windmill is unavailable: {exc}", 502) from exc
+        if response.is_error:
+            raise BackendError(
+                f"Windmill health check failed: HTTP {response.status_code}", 502
+            )
+        status = response.json().get("status")
+        if status not in {"healthy", "ok"}:
+            raise BackendError(f"Windmill reports status {status!r}", 502)
+
+    def _submit_flow(self, value: dict[str, Any], args: dict[str, Any]) -> str:
+        response = self._request(
+            "POST", "jobs/run/preview_flow", json={"value": value, "args": args}
+        )
+        return self._response_task_id(response)
+
+    @staticmethod
+    def _response_task_id(response: httpx.Response) -> str:
+        lines = response.text.strip().splitlines()
+        if not lines:
+            raise BackendError("Windmill returned an empty task ID")
+        try:
+            return str(UUID(lines[0]))
+        except ValueError as exc:
+            raise BackendError(
+                f"Windmill returned an invalid task ID: {lines[0]!r}"
+            ) from exc
+
+    @staticmethod
+    def _task_module(task_id: str, task: TaskDefinition) -> dict[str, Any]:
+        module: dict[str, Any] = {
+            "id": task_id,
+            "value": {
+                "type": "rawscript",
+                "language": _LANGUAGES[task.language],
+                "content": task.source,
+                "input_transforms": {
+                    key: {"type": "static", "value": value}
+                    for key, value in task.args.items()
+                },
+            },
+        }
+        if task.retry.attempts:
+            module["retry"] = {
+                "constant": {
+                    "attempts": task.retry.attempts,
+                    "seconds": task.retry.delay_seconds,
+                }
+            }
+        if task.timeout_seconds:
+            module["timeout"] = {"type": "static", "value": task.timeout_seconds}
+        return module
+
+    @staticmethod
+    def _dependency_module(
+        dependencies: list[str], timeout_seconds: int
+    ) -> dict[str, Any]:
+        # ponytail: polling creates one short Windmill job per interval; replace with
+        # backend completion events when dependency volume makes that measurable.
+        return {
+            "id": "funmill_wait",
+            "value": {
+                "type": "whileloopflow",
+                "skip_failures": False,
+                "modules": [
+                    {
+                        "id": "funmill_check_dependencies",
+                        "value": {
+                            "type": "rawscript",
+                            "language": "python3",
+                            "content": _DEPENDENCY_SOURCE,
+                            "input_transforms": {
+                                "dependencies": {
+                                    "type": "static",
+                                    "value": dependencies,
+                                },
+                                "timeout_seconds": {
+                                    "type": "static",
+                                    "value": timeout_seconds,
+                                },
+                            },
+                        },
+                        "sleep": {"type": "static", "value": 2},
+                        "stop_after_if": {"expr": "result === true"},
+                    }
+                ],
+            },
+        }
+
+    @staticmethod
+    def _add_callback(
+        value: dict[str, Any], callback_url: str | None, result_id: str
+    ) -> None:
+        if callback_url is None:
+            return
+
+        def callback_module(module_id: str, status: str, payload_expr: str):
+            return {
+                "id": module_id,
+                "value": {
+                    "type": "rawscript",
+                    "language": "python3",
+                    "content": _CALLBACK_SOURCE,
+                    "input_transforms": {
+                        "callback_url": {"type": "static", "value": callback_url},
+                        "status": {"type": "static", "value": status},
+                        "payload": {"type": "javascript", "expr": payload_expr},
+                    },
+                },
+                "retry": {"constant": {"attempts": 3, "seconds": 2}},
+            }
+
+        result_expr = f"results[{json.dumps(result_id)}]"
+        value["modules"].append(
+            callback_module("funmill_callback", "succeeded", result_expr)
+        )
+        value["failure_module"] = callback_module("failure", "failed", "error")
+
+    @staticmethod
+    def _result_module(task_results: dict[str, str]) -> dict[str, Any]:
+        entries = ", ".join(
+            f"{json.dumps(task_id)}: {result_expr}"
+            for task_id, result_expr in task_results.items()
+        )
+        return {
+            "id": "funmill_result",
+            "value": {
+                "type": "rawscript",
+                "language": "python3",
+                "content": _RESULT_SOURCE,
+                "input_transforms": {
+                    "results": {
+                        "type": "javascript",
+                        "expr": f"({{{entries}}})",
+                    }
+                },
+            },
+        }
+
+    def submit_task(self, task: TaskSubmit) -> str:
+        modules = []
+        if task.depends_on:
+            modules.append(
+                self._dependency_module(
+                    task.depends_on, task.dependency_timeout_seconds
+                )
+            )
+        modules.append(self._task_module("task", task))
+        value = {"modules": modules}
+        self._add_callback(
+            value,
+            str(task.callback_url) if task.callback_url else None,
+            "task",
+        )
+        return self._submit_flow(value, {})
+
+    def submit_workflow(self, workflow: WorkflowSubmit) -> str:
+        modules = []
+        task_results = {}
+        if workflow.depends_on:
+            modules.append(
+                self._dependency_module(
+                    workflow.depends_on, workflow.dependency_timeout_seconds
+                )
+            )
+
+        for index, layer in enumerate(workflow.topological_layers()):
+            if len(layer) == 1:
+                module = self._task_module(layer[0].key, layer[0])
+                task_results[layer[0].key] = f"results[{json.dumps(layer[0].key)}]"
+            else:
+                layer_id = f"funmill_layer_{index}"
+                module = {
+                    "id": layer_id,
+                    "value": {
+                        "type": "branchall",
+                        "parallel": True,
+                        "branches": [
+                            {
+                                "summary": task.key,
+                                "modules": [self._task_module(task.key, task)],
+                            }
+                            for task in layer
+                        ],
+                    },
+                }
+                for task_index, task in enumerate(layer):
+                    task_results[task.key] = (
+                        f"results[{json.dumps(layer_id)}][{task_index}]"
+                    )
+            modules.append(module)
+
+        result_module = self._result_module(task_results)
+        modules.append(result_module)
+
+        value = {"modules": modules}
+        self._add_callback(
+            value,
+            str(workflow.callback_url) if workflow.callback_url else None,
+            result_module["id"],
+        )
+        return self._submit_flow(value, {})
+
+    def _get_job(self, task_id: str) -> dict[str, Any]:
+        return self._request("GET", f"jobs_u/get/{task_id}").json()
+
+    @staticmethod
+    def _status(job: dict[str, Any]) -> TaskStatus:
+        if job.get("canceled"):
+            return TaskStatus.CANCELED
+        if "success" in job:
+            return TaskStatus.SUCCEEDED if job["success"] else TaskStatus.FAILED
+        return TaskStatus.RUNNING if job.get("running") else TaskStatus.QUEUED
+
+    def get_task(self, task_id: str) -> TaskInfo:
+        job = self._get_job(task_id)
+        return TaskInfo(
+            task_id=task_id,
+            status=self._status(job),
+            created_at=job.get("created_at"),
+            started_at=job.get("started_at"),
+            completed_at=job.get("completed_at"),
+            duration_ms=job.get("duration_ms"),
+        )
+
+    def get_progress(self, task_id: str) -> TaskProgress:
+        job = self._get_job(task_id)
+        status = self._status(job)
+        if status == TaskStatus.SUCCEEDED:
+            return TaskProgress(task_id=task_id, progress=100)
+
+        modules = (job.get("flow_status") or {}).get("modules") or []
+        scores = []
+        for module in modules:
+            if module.get("type") == "Success":
+                scores.append(100)
+            elif module.get("type") == "InProgress":
+                scores.append(module.get("progress") or 0)
+            else:
+                scores.append(0)
+        progress = int(sum(scores) / len(scores)) if scores else None
+        return TaskProgress(task_id=task_id, progress=progress)
+
+    def get_logs(self, task_id: str) -> TaskLogs:
+        job = self._get_job(task_id)
+        if job.get("job_kind") in {"flow", "flowpreview", "singlestepflow"}:
+            response = self._request("GET", f"jobs_u/get_flow_all_logs/{task_id}")
+            logs = response.text
+        else:
+            logs = self._request("GET", f"jobs_u/get_logs/{task_id}").text
+        return TaskLogs(task_id=task_id, logs=logs)
+
+    def get_result(self, task_id: str) -> TaskResult:
+        result = self._request("GET", f"jobs_u/completed/get_result/{task_id}").json()
+        return TaskResult(task_id=task_id, result=result)
+
+    def cancel(self, task_id: str, reason: str) -> None:
+        self._request("POST", f"jobs_u/queue/cancel/{task_id}", json={"reason": reason})
+
+    def rerun(self, task_id: str) -> str:
+        job = self._get_job(task_id)
+        raw_flow = job.get("raw_flow")
+        if raw_flow:
+            return self._submit_flow(raw_flow, job.get("args") or {})
+
+        raw_code = job.get("raw_code")
+        if raw_code:
+            response = self._request(
+                "POST",
+                "jobs/run/preview",
+                json={
+                    "content": raw_code,
+                    "language": job.get("language"),
+                    "kind": "code",
+                    "args": job.get("args") or {},
+                },
+            )
+            return self._response_task_id(response)
+
+        raise BackendError(
+            "task cannot be rerun because its source is unavailable", 409
+        )
+
+    def close(self) -> None:
+        self.client.close()