funmill-api 0.1.2__py3-none-any.whl

funmill/api/backends/windmill/service.py

@@ -0,0 +1,151 @@
+import hashlib
+import os
+import platform
+import re
+import shutil
+import tempfile
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+from funmill.api.ports import SERVICE_BIND_HOST, THIRD_PARTY_WEB_PORT
+
+from ..service import start_background, status_background, stop_background
+
+VERSION = "v1.808.0"
+URL = f"https://github.com/windmill-labs/windmill/releases/download/{VERSION}/windmill-amd64"
+SHA256 = "ed48bfb9a391daa437f0c867376f009c7186855530de7fe2f58cf557ff1f7c3a"
+_ENV_TEMPLATE = f"""DATABASE_URL=
+MODE=standalone
+SERVER_BIND_ADDR={SERVICE_BIND_HOST}
+"""
+
+
+def _home() -> Path:
+    return Path(os.getenv("FUNMILL_HOME", Path.home() / ".farfarfun" / "funmill"))
+
+
+def _target() -> Path:
+    return _home() / "services" / "windmill" / "windmill"
+
+
+def _config_path() -> Path:
+    return _target().parent / ".env"
+
+
+def _ensure_config() -> Path:
+    path = _config_path()
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(_ENV_TEMPLATE, encoding="utf-8")
+        path.chmod(0o600)
+        print(f"created config: {path}")
+    return path
+
+
+def _read_config(path: Path) -> dict[str, str]:
+    values = {}
+    for line_number, raw_line in enumerate(
+        path.read_text(encoding="utf-8").splitlines(), 1
+    ):
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        key, separator, value = line.partition("=")
+        key = key.strip()
+        if not separator or not key.isidentifier():
+            raise RuntimeError(f"invalid config at {path}:{line_number}")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _sha256(path: Path) -> str:
+    with path.open("rb") as file:
+        return hashlib.file_digest(file, "sha256").hexdigest()
+
+
+def _environment() -> dict[str, str]:
+    path = _config_path()
+    environment = _read_config(path) if path.exists() else {}
+    environment.update(os.environ)
+    return environment
+
+
+def _instance_name(environment: dict[str, str]) -> str:
+    if environment.get("MODE", "standalone") != "worker":
+        return "windmill"
+    suffix = environment.get("WORKER_SUFFIX", "worker")
+    if not re.fullmatch(r"[A-Za-z0-9_-]+", suffix):
+        raise RuntimeError("WORKER_SUFFIX must contain only letters, numbers, _ or -")
+    return f"windmill-{suffix}"
+
+
+def install(force: bool = False) -> Path:
+    if platform.system() != "Linux" or platform.machine().lower() not in {
+        "x86_64",
+        "amd64",
+    }:
+        raise RuntimeError("Windmill v1.808.0 installer only supports Linux x86_64")
+
+    target = _target()
+    _ensure_config()
+    if target.exists() and _sha256(target) == SHA256:
+        return target
+    if target.exists() and not force:
+        raise RuntimeError(
+            f"{target} exists but has an unexpected checksum; use --force"
+        )
+
+    target.parent.mkdir(parents=True, exist_ok=True)
+    descriptor, temporary_name = tempfile.mkstemp(dir=target.parent)
+    os.close(descriptor)
+    temporary = Path(temporary_name)
+    try:
+        print(f"downloading Windmill {VERSION}...", flush=True)
+        request = Request(URL, headers={"User-Agent": "funmill"})
+        with urlopen(request, timeout=30) as response, temporary.open("wb") as file:
+            shutil.copyfileobj(response, file)
+        if _sha256(temporary) != SHA256:
+            raise RuntimeError("downloaded Windmill binary failed SHA-256 verification")
+        temporary.chmod(0o755)
+        temporary.replace(target)
+    finally:
+        temporary.unlink(missing_ok=True)
+    return target
+
+
+def start() -> None:
+    executable = _target()
+    if not executable.exists():
+        system_executable = shutil.which("windmill")
+        if system_executable is None:
+            raise RuntimeError(
+                "Windmill is not installed; run: funmill install windmill"
+            )
+        executable = Path(system_executable)
+
+    config = _ensure_config()
+    environment = _environment()
+    if not environment.get("DATABASE_URL"):
+        raise RuntimeError(f"DATABASE_URL is required; edit {config}")
+
+    mode = environment.setdefault("MODE", "standalone")
+    if mode in {"standalone", "server"}:
+        environment["PORT"] = str(THIRD_PARTY_WEB_PORT)
+        environment.setdefault("BASE_URL", f"http://127.0.0.1:{THIRD_PARTY_WEB_PORT}")
+    environment["SERVER_BIND_ADDR"] = SERVICE_BIND_HOST
+    start_background(
+        _instance_name(environment), [str(executable)], environment, _target().parent
+    )
+
+
+def stop() -> None:
+    environment = _environment()
+    stop_background(_instance_name(environment), _target().parent)
+
+
+def status() -> bool:
+    environment = _environment()
+    return status_background(_instance_name(environment), _target().parent)

funmill/api/cli.py

@@ -0,0 +1,83 @@
+import argparse
+import importlib
+from collections.abc import Sequence
+from types import ModuleType
+
+import uvicorn
+
+from funmill.api.backends import BACKEND_SPECS
+from funmill.api.ports import FUNMILL_API_PORT, SERVICE_BIND_HOST
+
+
+def _service_names() -> list[str]:
+    return [name for name, spec in BACKEND_SPECS.items() if spec.service]
+
+
+def _service(name: str) -> ModuleType:
+    spec = BACKEND_SPECS[name]
+    return importlib.import_module(spec.service, "funmill.api.backends")
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="funmill")
+    commands = parser.add_subparsers(dest="command")
+
+    commands.add_parser("services", help="列出可安装的第三方服务")
+
+    install = commands.add_parser("install", help="安装第三方服务")
+    install.add_argument("service", choices=_service_names())
+    install.add_argument("--force", action="store_true", help="覆盖现有安装")
+
+    start = commands.add_parser("start", help="启动 Funmill 或第三方服务")
+    start.add_argument(
+        "service", nargs="?", choices=["api", *_service_names()], default="api"
+    )
+
+    for command in ("stop", "status", "restart"):
+        service_command = commands.add_parser(
+            command,
+            help={
+                "stop": "停止第三方服务",
+                "status": "查看第三方服务状态",
+                "restart": "重启第三方服务",
+            }[command],
+        )
+        service_command.add_argument("service", choices=_service_names())
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = _parser()
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "services":
+            print("\n".join(_service_names()))
+        elif args.command == "install":
+            path = _service(args.service).install(force=args.force)
+            print(f"installed {args.service}: {path}")
+        elif args.command == "start" and args.service != "api":
+            _service(args.service).start()
+        elif args.command == "stop":
+            _service(args.service).stop()
+        elif args.command == "status":
+            if not _service(args.service).status():
+                raise SystemExit(1)
+        elif args.command == "restart":
+            service = _service(args.service)
+            service.stop()
+            service.start()
+        elif args.command == "start":
+            uvicorn.run(
+                "funmill.api:app",
+                host=SERVICE_BIND_HOST,
+                port=FUNMILL_API_PORT,
+            )
+        else:
+            parser.print_help()
+    except (OSError, RuntimeError, ValueError) as exc:
+        parser.exit(1, f"error: {exc}\n")
+
+
+if __name__ == "__main__":
+    main()

funmill/api/models.py

@@ -0,0 +1,120 @@
+from datetime import datetime
+from enum import StrEnum
+from typing import Annotated, Any, Self
+
+from pydantic import AnyHttpUrl, BaseModel, Field, model_validator
+
+DependencyId = Annotated[str, Field(min_length=1, max_length=200)]
+
+
+class TaskLanguage(StrEnum):
+    PYTHON = "python"
+    BASH = "bash"
+
+
+class TaskStatus(StrEnum):
+    QUEUED = "queued"
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    CANCELED = "canceled"
+
+
+class RetryPolicy(BaseModel):
+    attempts: int = Field(default=0, ge=0, le=10)
+    delay_seconds: int = Field(default=1, ge=0, le=3600)
+
+
+class TaskDefinition(BaseModel):
+    language: TaskLanguage
+    source: str = Field(min_length=1, max_length=1_000_000)
+    args: dict[str, Any] = Field(default_factory=dict)
+    retry: RetryPolicy = Field(default_factory=RetryPolicy)
+    timeout_seconds: int | None = Field(default=None, ge=1, le=86_400)
+
+
+class TaskSubmit(TaskDefinition):
+    depends_on: list[DependencyId] = Field(default_factory=list, max_length=100)
+    dependency_timeout_seconds: int = Field(default=86_400, ge=1, le=604_800)
+    callback_url: AnyHttpUrl | None = None
+
+
+class WorkflowTask(TaskDefinition):
+    key: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_-]{0,63}$")
+    depends_on: list[str] = Field(default_factory=list, max_length=100)
+
+
+class WorkflowSubmit(BaseModel):
+    tasks: list[WorkflowTask] = Field(min_length=1, max_length=100)
+    depends_on: list[DependencyId] = Field(default_factory=list, max_length=100)
+    dependency_timeout_seconds: int = Field(default=86_400, ge=1, le=604_800)
+    callback_url: AnyHttpUrl | None = None
+
+    @model_validator(mode="after")
+    def validate_dependencies(self) -> Self:
+        keys = [task.key for task in self.tasks]
+        if len(keys) != len(set(keys)):
+            raise ValueError("workflow task keys must be unique")
+        if any(key == "failure" or key.startswith("funmill_") for key in keys):
+            raise ValueError("task keys cannot be 'failure' or start with 'funmill_'")
+
+        known = set(keys)
+        for task in self.tasks:
+            unknown = set(task.depends_on) - known
+            if unknown:
+                raise ValueError(
+                    f"task {task.key!r} has unknown dependencies: {sorted(unknown)}"
+                )
+        self.topological_layers()
+        return self
+
+    def topological_layers(self) -> list[list[WorkflowTask]]:
+        by_key = {task.key: task for task in self.tasks}
+        remaining = {task.key: set(task.depends_on) for task in self.tasks}
+        layers: list[list[WorkflowTask]] = []
+        while remaining:
+            ready = [key for key, dependencies in remaining.items() if not dependencies]
+            if not ready:
+                raise ValueError("workflow contains a dependency cycle")
+            layers.append([by_key[key] for key in ready])
+            ready_set = set(ready)
+            remaining = {
+                key: dependencies - ready_set
+                for key, dependencies in remaining.items()
+                if key not in ready_set
+            }
+        return layers
+
+
+class TaskAccepted(BaseModel):
+    task_id: str
+    status: TaskStatus = TaskStatus.QUEUED
+    rerun_of: str | None = None
+
+
+class TaskInfo(BaseModel):
+    task_id: str
+    status: TaskStatus
+    created_at: datetime | None = None
+    started_at: datetime | None = None
+    completed_at: datetime | None = None
+    duration_ms: int | None = None
+
+
+class TaskProgress(BaseModel):
+    task_id: str
+    progress: int | None
+
+
+class TaskLogs(BaseModel):
+    task_id: str
+    logs: str
+
+
+class TaskResult(BaseModel):
+    task_id: str
+    result: Any
+
+
+class CancelRequest(BaseModel):
+    reason: str = Field(default="canceled through Funmill", max_length=500)

funmill/api/ports.py

@@ -0,0 +1,3 @@
+SERVICE_BIND_HOST = "0.0.0.0"
+FUNMILL_API_PORT = 8812
+THIRD_PARTY_WEB_PORT = 8813

funmill_api-0.1.2.dist-info/METADATA

@@ -0,0 +1,211 @@
+Metadata-Version: 2.5
+Name: funmill-api
+Version: 0.1.2
+Summary: Backend-neutral task execution API
+Project-URL: Organization, https://github.com/farfarfun
+Project-URL: Repository, https://github.com/farfarfun/funmill-api
+Project-URL: Releases, https://github.com/farfarfun/funmill-api/releases
+Author-email: 牛哥 <niuliangtao@qq.com>, farfarfun <farfarfun@qq.com>
+Maintainer-email: 牛哥 <niuliangtao@qq.com>, farfarfun <farfarfun@qq.com>
+License-Expression: MIT
+Requires-Python: >=3.12
+Requires-Dist: fastapi<1,>=0.115
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: uvicorn<1,>=0.30
+Description-Content-Type: text/markdown
+
+# Funmill
+
+Funmill provides one stable task API while execution is delegated to a
+replaceable backend. The recommended local backend is self-hosted
+[Dagu](https://github.com/dagucloud/dagu), pinned to `v2.16.3`. Windmill remains
+available for existing deployments.
+
+```text
+client -> Funmill /v1 -> TaskBackend -> Dagu
+                                  -> Windmill
+```
+
+Funmill owns the public request and response models. Backend job IDs remain
+opaque strings, and no backend-specific routes or payloads are exposed to clients.
+
+## Start
+
+Install the project and the Dagu binary. The installer supports macOS and Linux
+on Intel/AMD and ARM64:
+
+```bash
+uv sync
+uv run funmill install dagu
+```
+
+Start Dagu in the background. It stores state under
+`~/.farfarfun/funmill/services/dagu/data/` and needs no external database:
+
+```bash
+uv run funmill start dagu
+```
+
+The command reports its PID and log path. Open <http://localhost:8813>, then
+start Funmill:
+
+```bash
+FUNMILL_API_KEY='replace-me' \
+FUNMILL_BACKEND=dagu \
+DAGU_URL='http://127.0.0.1:8813' \
+uv run funmill start
+```
+
+Managed HTTP services bind to `0.0.0.0`: Funmill uses port `8812` and the active
+third-party service uses `8813`. Local client URLs still use `127.0.0.1` or
+`localhost`; `0.0.0.0` is a listen address, not a client destination.
+
+Dagu starts without authentication. Because it listens on every interface,
+restrict port `8813` with a firewall or enable Dagu authentication. When
+authentication is enabled, set `DAGU_TOKEN` for both `funmill start dagu` and
+`funmill start` so cross-run dependencies can query Dagu from worker processes.
+
+Dagu runs submitted source with the service user's host permissions. Keep both
+services private and accept only trusted code; use isolated workers or
+containers before accepting untrusted jobs.
+
+The Funmill API port is fixed at `8812`; the active third-party UI/API port is
+fixed at `8813`. OpenAPI docs are at <http://localhost:8812/docs>. All `/v1`
+routes require `X-API-Key`. See the
+[Dagu deployment guide](src/funmill/api/backends/dagu/README.md) or the
+[Windmill deployment guide](src/funmill/api/backends/windmill/README.md) for
+backend-specific setup.
+
+Run the end-to-end task and DAG checks with:
+
+```bash
+FUNMILL_API_KEY=the-value-from-env ./scripts/smoke.sh
+```
+
+Set `FUNMILL_CALLBACK_URL` to test callbacks. The URL must be reachable from
+the backend workers.
+
+## API
+
+| Operation | Route |
+| --- | --- |
+| Health check | `GET /health` |
+| Submit Python/Bash | `POST /v1/tasks` |
+| Submit a DAG | `POST /v1/workflows` |
+| Status | `GET /v1/tasks/{task_id}` |
+| Logs | `GET /v1/tasks/{task_id}/logs` |
+| Progress | `GET /v1/tasks/{task_id}/progress` |
+| Result | `GET /v1/tasks/{task_id}/result` |
+| Cancel | `POST /v1/tasks/{task_id}/cancel` |
+| Rerun | `POST /v1/tasks/{task_id}/rerun` |
+
+`GET /health` is unauthenticated and reports whether the Funmill process and
+its configured backend are reachable; it returns `{"status": "ok", "backend":
+"..."}` on success and a 503 with a `detail` message when the backend is
+unreachable or misconfigured.
+
+Submit one task, optionally waiting for existing task IDs:
+
+```json
+{
+  "language": "python",
+  "source": "def main(value: int):\n    return value * 2\n",
+  "args": {"value": 21},
+  "depends_on": ["EXISTING_TASK_ID"],
+  "dependency_timeout_seconds": 3600,
+  "retry": {"attempts": 2, "delay_seconds": 5},
+  "timeout_seconds": 300,
+  "callback_url": "https://example.internal/task-callback"
+}
+```
+
+Submit `A -> [B, C]` as one workflow:
+
+```json
+{
+  "tasks": [
+    {"key": "a", "language": "python", "source": "def main(): return 1"},
+    {"key": "b", "language": "python", "source": "def main(): return 2", "depends_on": ["a"]},
+    {"key": "c", "language": "bash", "source": "main() { echo 3; }", "depends_on": ["a"]}
+  ]
+}
+```
+
+Dependencies inside a workflow are task keys. Top-level `depends_on` values are
+IDs returned by earlier Funmill submissions. Backends check those dependencies
+from worker jobs, so waiting does not hold the Funmill API process. Failure or
+cancellation of a dependency fails the waiting task.
+Workflow results and callback payloads are objects keyed by every workflow task
+key. Each topological layer is a synchronization barrier; tasks in the same
+layer run in parallel.
+
+Callbacks contain `task_id`, `status`, and `payload`; success and failure
+delivery retry three times. Delivery is at least once, so callback receivers
+must be idempotent.
+
+## Python SDK
+
+A Python client for the `/v1` routes plus `/health` lives in the separate
+[`funmill-sdk`](https://github.com/farfarfun/funmill-sdk) repository, published
+to PyPI as `funmill` (imported as `funmill.client`):
+
+```bash
+pip install funmill
+```
+
+```python
+from funmill.client import FunmillClient, TaskSubmit
+
+with FunmillClient(base_url="http://127.0.0.1:8812", api_key="replace-me") as client:
+    client.health()  # {"status": "ok", "backend": "dagu"}
+
+    accepted = client.submit_task(
+        TaskSubmit(language="python", source="def main(): return 21 * 2")
+    )
+    task = client.get_task(accepted.task_id)
+    result = client.get_result(accepted.task_id)
+```
+
+See the `funmill-sdk` README for the full client API and error handling.
+
+## Backends
+
+The public contract is `TaskBackend` in `src/funmill/api/backends/base.py`. Backend
+selection uses `FUNMILL_BACKEND`; registration lives in
+`src/funmill/api/backends/__init__.py`, following the same driver pattern as
+`fundrive`. Each third-party adapter lives in its own directory, such as
+`src/funmill/api/backends/windmill/`.
+
+Every third-party adapter directory must include a `README.md` covering its
+supported platforms, installation, configuration, startup, verification, and
+security or operational constraints.
+
+Every managed HTTP service must bind to `SERVICE_BIND_HOST` (`0.0.0.0`). Every
+third-party service must use the shared background lifecycle in
+`src/funmill/api/backends/service.py` and expose `start`, `stop`, and `status`;
+`restart` is composed from `stop` and `start` by the CLI.
+
+Changing the backend does not change `/v1`, but it does not migrate old jobs or
+their IDs. Add a Funmill-owned ID mapping database only when jobs must remain
+queryable after a live backend migration.
+
+## Operations
+
+```bash
+funmill services
+funmill install dagu
+funmill start dagu
+funmill status dagu
+funmill restart dagu
+funmill stop dagu
+funmill install windmill
+funmill start windmill
+funmill start
+```
+
+Third-party services run in the background with PID and log files under
+`~/.farfarfun/funmill/services/<service>/`; replace `dagu` with `windmill` in
+the lifecycle commands as needed. The Funmill API remains in the foreground and
+stops with `Ctrl+C`. Add authentication, TLS, firewall rules, PostgreSQL
+backups, callback egress restrictions, and a secrets manager before network
+exposure.

funmill_api-0.1.2.dist-info/RECORD

@@ -0,0 +1,18 @@
+funmill/api/__init__.py,sha256=bOFRmVJ0Fum8eBNn31EyOkjpDwqUzlQmbHOa2kArp9k,765
+funmill/api/app.py,sha256=McZUCWKuQho1Di4001qzD8MbuQ-7U0sAlE8kJYaKZqg,3145
+funmill/api/cli.py,sha256=MO2iFLOsjny6JBAiTUTbtczHfL-vKn_iqDonc1CBNeo,2751
+funmill/api/models.py,sha256=c0R-gJi2Tfo8TZKDz-Gwmd5Qd4fGmrNE_n5AjV2jmzM,3756
+funmill/api/ports.py,sha256=xfBREmyI6vBOhF_E6HQgUDjysSRzy_0GPfjfua7vHRI,82
+funmill/api/backends/__init__.py,sha256=ZXD-BqF4t59W_AhoLV3l20n0MxBJEM6zrPz6TrtxP4k,1066
+funmill/api/backends/base.py,sha256=8hXbv3Bw3wJ4WF75MZqf7hA-ZRKpKgjDNFhzlRFyL7M,1208
+funmill/api/backends/service.py,sha256=8OEb4Uepo3l6fudscVjYfhO9ISYWQIy-ANNwOmxVWLk,3122
+funmill/api/backends/dagu/README.md,sha256=wzER_C0XPDILL-Urlgh5uvIDjU2RUAfBteK-2qKdc04,2945
+funmill/api/backends/dagu/__init__.py,sha256=27eRanZ2aX_46_T6uPMheKxewWCGOz2GsTioOx8sdQg,16407
+funmill/api/backends/dagu/service.py,sha256=7ddkGzOVX1bfxHOzNVZ3mG_mjXSDya2i9ltoxwcqlME,5147
+funmill/api/backends/windmill/README.md,sha256=nifEkIMoqXr77Jct7ZrET9tuwtwUkmlRLTDK7OcGt8Y,3437
+funmill/api/backends/windmill/__init__.py,sha256=o5j3Yi9TNuY9w6wW4REaq5VEQSlDf1aAgmqzdACF05s,14689
+funmill/api/backends/windmill/service.py,sha256=gf7ZVPHVW6JXSipbkTHmc3U8KN4qA0Ulk2fHe-bEzco,4887
+funmill_api-0.1.2.dist-info/METADATA,sha256=q-xGYmlStuU1eriIDjslACmBFlxnSFXLZ1Q_jLJUvFk,7435
+funmill_api-0.1.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+funmill_api-0.1.2.dist-info/entry_points.txt,sha256=O6hMrrwUkrlQDnKfgSs_joamaUzVrfxfH2_gyfO0Nrg,49
+funmill_api-0.1.2.dist-info/RECORD,,

funmill_api-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

funmill_api-0.1.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+funmill = funmill.api.cli:main