fleet-framework 0.1.0__py3-none-any.whl

fleet/__init__.py

@@ -0,0 +1 @@
+# namespace marker

fleet/cli.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import socket
+import sys
+
+import click
+import uvicorn
+
+from fleet.core.logging import configure_logging
+
+configure_logging()
+
+
+@click.group()
+def cli() -> None:
+    # fleet — master + worker CLI.
+    pass
+
+
+@cli.command()
+@click.option("--host", default="0.0.0.0", show_default=True)
+@click.option("--port", default=8000, show_default=True)
+@click.option(
+    "--backend-url",
+    envvar="FLEET_BACKEND_URL",
+    default=None,
+    show_default=True,
+    help="Backend URL: redis://..., memory://, or any registered scheme. "
+         "Defaults to $FLEET_BACKEND_URL, then $REDIS_URL, then redis://localhost:6379/0.",
+)
+@click.option(
+    "--redis-url",
+    envvar="REDIS_URL",
+    default=None,
+    show_default=True,
+    help="(Legacy) Use --backend-url instead.",
+)
+def master(host: str, port: int, backend_url: str | None, redis_url: str | None) -> None:
+    """Start the master process. Binds REST + WS + dashboard."""
+    from fleet.master import create_app
+
+    url = backend_url or redis_url or "redis://localhost:6379/0"
+    if redis_url:
+        os.environ.setdefault("REDIS_URL", redis_url)
+    os.environ.setdefault("FLEET_BACKEND_URL", url)
+    app = create_app(url)
+    uvicorn.run(app, host=host, port=port, log_level=os.environ.get("LOG_LEVEL", "info").lower())
+
+
+@cli.command()
+@click.option("--type", "automation_type", required=True, help="automation type name (entry-point key)")
+@click.option("--id", "worker_id", default=None, help="worker id (defaults to hostname)")
+@click.option("--master-url", envvar="MASTER_URL", required=True)
+@click.option("--worker-token", envvar="FLEET_WORKER_TOKEN", required=True)
+@click.option(
+    "--backend-url",
+    envvar="FLEET_BACKEND_URL",
+    default=None,
+    help="Backend URL; falls back to --redis-url then redis://localhost:6379/0.",
+)
+@click.option(
+    "--redis-url",
+    envvar="REDIS_URL",
+    default=None,
+    help="(Legacy) Use --backend-url instead.",
+)
+def worker(
+    automation_type: str,
+    worker_id: str | None,
+    master_url: str,
+    worker_token: str,
+    backend_url: str | None,
+    redis_url: str | None,
+) -> None:
+    """Start a worker process pinned to one automation type."""
+    from fleet.worker import Agent
+
+    url = backend_url or redis_url or "redis://localhost:6379/0"
+    wid = worker_id or socket.gethostname()
+    agent = Agent(
+        automation_type=automation_type,
+        worker_id=wid,
+        master_url=master_url,
+        worker_token=worker_token,
+        redis_url=url,
+    )
+    try:
+        asyncio.run(agent.run())
+    except KeyboardInterrupt:
+        sys.exit(0)
+
+
+def _list_plugins() -> None:
+    from fleet.core.automation import load_entry_points
+    from fleet.core.proxy import load_provider_entry_points
+
+    reg = load_entry_points()
+    if reg:
+        click.echo("automations:")
+        for name, cls in reg.items():
+            kind = cls.__mro__[1].__name__
+            click.echo(f"  {name:30} {cls.__module__}.{cls.__name__}  ({kind})")
+    else:
+        click.echo("automations: (none installed)")
+    providers = load_provider_entry_points()
+    if providers:
+        click.echo("proxy providers:")
+        for name in providers:
+            click.echo(f"  {name}")
+
+
+@cli.command()
+def plugins() -> None:
+    """List installed automations + proxy providers discovered via entry-points."""
+    _list_plugins()
+
+
+@cli.command(name="info")
+def info_alias() -> None:
+    """Alias of `plugins`. Kept for backwards compatibility."""
+    _list_plugins()
+
+
+@cli.command()
+@click.argument("automation_type")
+@click.option("--master-url", envvar="MASTER_URL", required=True)
+@click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", required=True)
+@click.option("--n", default=10, show_default=True, help="entries per pop")
+@click.option("--interval", default=1.0, show_default=True, help="seconds between polls")
+@click.option("--peek", is_flag=True, help="non-destructive read instead of pop")
+def tail(
+    automation_type: str,
+    master_url: str,
+    admin_token: str,
+    n: int,
+    interval: float,
+    peek: bool,
+) -> None:
+    """Stream the output of an automation type to stdout, one JSON object per line."""
+    import httpx
+
+    headers = {"Authorization": f"Bearer {admin_token}"}
+    op = "peek" if peek else "pop"
+    base = master_url.rstrip("/")
+    try:
+        while True:
+            try:
+                if peek:
+                    url = f"{base}/api/v1/automations/{automation_type}/output/peek?n={n}"
+                    r = httpx.get(url, headers=headers, timeout=10)
+                else:
+                    url = f"{base}/api/v1/automations/{automation_type}/output/pop"
+                    r = httpx.post(url, headers=headers, json={"n": n}, timeout=10)
+                r.raise_for_status()
+                for env in r.json():
+                    click.echo(json.dumps(env, default=str))
+            except Exception as e:
+                click.echo(f"# {op} failed: {e}", err=True)
+            import time as _t
+            _t.sleep(max(0.1, interval))
+    except KeyboardInterrupt:
+        sys.exit(0)
+
+
+@cli.command()
+@click.argument("automation_type")
+@click.argument("payload")  # JSON string or @path
+@click.option("--master-url", envvar="MASTER_URL", required=True)
+@click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", required=True)
+@click.option("--task-id", default=None, help="override the framework's UUID")
+@click.option("--max-attempts", default=None, type=int, help="override per-task retry budget")
+def submit(
+    automation_type: str,
+    payload: str,
+    master_url: str,
+    admin_token: str,
+    task_id: str | None,
+    max_attempts: int | None,
+) -> None:
+    """Submit a single task to a BatchAutomation.
+
+    PAYLOAD is a JSON string or @path/to/file.json. Validated against the
+    automation's TaskPayload schema server-side.
+    """
+    import httpx
+
+    body_text = payload
+    if payload.startswith("@"):
+        body_text = open(payload[1:]).read()
+    try:
+        parsed = json.loads(body_text)
+    except json.JSONDecodeError as e:
+        click.echo(f"payload is not valid JSON: {e}", err=True)
+        sys.exit(2)
+    req: dict = {"payload": parsed}
+    if task_id:
+        req["task_id"] = task_id
+    if max_attempts is not None:
+        req["max_attempts"] = max_attempts
+    url = master_url.rstrip("/") + f"/api/v1/automations/{automation_type}/tasks"
+    r = httpx.post(url, headers={"Authorization": f"Bearer {admin_token}"}, json=req, timeout=10)
+    if r.status_code >= 400:
+        click.echo(f"submit failed ({r.status_code}): {r.text}", err=True)
+        sys.exit(1)
+    click.echo(json.dumps(r.json()))
+
+
+@cli.command()
+@click.option("--master-url", envvar="MASTER_URL", default=None)
+@click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", default=None)
+@click.option("--worker-token", envvar="FLEET_WORKER_TOKEN", default=None)
+@click.option("--backend-url", envvar="FLEET_BACKEND_URL", default=None)
+def doctor(
+    master_url: str | None,
+    admin_token: str | None,
+    worker_token: str | None,
+    backend_url: str | None,
+) -> None:
+    """Pre-flight sanity checks: env tokens, plugins, backend, master reachable."""
+    asyncio.run(_doctor(master_url, admin_token, worker_token, backend_url))
+
+
+async def _doctor(master_url, admin_token, worker_token, backend_url) -> None:
+    import httpx
+
+    ok = True
+
+    def _check(label: str, condition: bool, detail: str = "") -> None:
+        nonlocal ok
+        mark = "✓" if condition else "✗"
+        ok = ok and condition
+        click.echo(f"{mark} {label}" + (f" — {detail}" if detail else ""))
+
+    _check("FLEET_ADMIN_TOKEN set", bool(admin_token))
+    _check("FLEET_WORKER_TOKEN set", bool(worker_token))
+
+    from fleet.core.automation import load_entry_points
+    reg = load_entry_points()
+    _check("automations discovered", bool(reg), f"{len(reg)} found")
+    for name in reg:
+        click.echo(f"    {name}")
+
+    if backend_url:
+        try:
+            from fleet.core.backend import RedisBackend, backend_from_url
+            be = backend_from_url(backend_url)
+            _check("backend constructs", True, type(be).__name__)
+            if isinstance(be, RedisBackend):
+                try:
+                    pong = await be.r.ping()
+                    _check("redis ping", bool(pong))
+                    info = await be.r.info("persistence")
+                    aof = info.get(b"aof_enabled", info.get("aof_enabled"))
+                    aof_on = str(aof).strip() in ("1", "True", "true")
+                    _check(
+                        "redis AOF on", aof_on,
+                        "" if aof_on else "data lives only in RAM; configs/pools die on restart",
+                    )
+                except Exception as e:
+                    _check("redis reachable", False, str(e))
+                finally:
+                    await be.aclose()
+        except Exception as e:
+            _check("backend constructs", False, str(e))
+    else:
+        click.echo("· backend-url not set (skipping backend checks)")
+
+    if master_url and admin_token:
+        try:
+            async with httpx.AsyncClient(timeout=3) as c:
+                r = await c.get(master_url.rstrip("/") + "/healthz")
+                _check("master /healthz reachable", r.status_code == 200, f"HTTP {r.status_code}")
+                r = await c.get(
+                    master_url.rstrip("/") + "/api/v1/automations",
+                    headers={"Authorization": f"Bearer {admin_token}"},
+                )
+                _check("admin token accepted", r.status_code == 200, f"HTTP {r.status_code}")
+        except Exception as e:
+            _check("master reachable", False, str(e))
+    else:
+        click.echo("· master-url or admin-token not set (skipping master checks)")
+
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    cli()

fleet/core/__init__.py

@@ -0,0 +1,69 @@
+from fleet.core.automation import (
+    BaseAutomation,
+    BatchAutomation,
+    ContinuousAutomation,
+    Task,
+    catalog_doc,
+    get_registry,
+    load_entry_points,
+    register,
+)
+from fleet.core.backend import Backend, RedisBackend
+from fleet.core.config import BaseConfig
+from fleet.core.context import Context
+from fleet.core.contract import Pool, Queue, Stream
+from fleet.core.country_presets import known_presets, resolve_countries
+from fleet.core.primitives import (
+    KV,
+    Counter,
+    Lock,
+    PoolFilter,
+    PoolHandle,
+    PoolItem,
+    QueueHandle,
+    StreamReader,
+)
+from fleet.core.proxy import (
+    ProxyHandle,
+    ProxyProvider,
+    ProxySession,
+    build_provider,
+    get_provider_registry,
+    load_provider_entry_points,
+    register_provider,
+)
+
+__all__ = [
+    "Backend",
+    "BaseAutomation",
+    "BaseConfig",
+    "BatchAutomation",
+    "ContinuousAutomation",
+    "Context",
+    "Counter",
+    "KV",
+    "Lock",
+    "Pool",
+    "PoolFilter",
+    "PoolHandle",
+    "PoolItem",
+    "ProxyHandle",
+    "ProxyProvider",
+    "ProxySession",
+    "Queue",
+    "QueueHandle",
+    "RedisBackend",
+    "Stream",
+    "StreamReader",
+    "Task",
+    "build_provider",
+    "catalog_doc",
+    "get_provider_registry",
+    "get_registry",
+    "known_presets",
+    "load_entry_points",
+    "load_provider_entry_points",
+    "register",
+    "register_provider",
+    "resolve_countries",
+]

fleet/core/automation.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import importlib.metadata as md
+import logging
+from dataclasses import dataclass
+from typing import Any, ClassVar, Generic, Optional, TypeVar
+
+from pydantic import BaseModel
+
+from fleet.core.config import BaseConfig
+from fleet.core.context import Context
+from fleet.core.contract import Pool, Queue, Stream
+
+C = TypeVar("C", bound=BaseConfig)
+
+logger = logging.getLogger(__name__)
+
+_REGISTRY: dict[str, type["BaseAutomation"]] = {}
+
+
+@dataclass
+class Task:
+    id: str
+    payload: BaseModel  # typed if the automation declared TaskPayload, else a fallback model
+
+
+class BaseAutomation(Generic[C]):
+    # subclass + decorate with @register("name"). do not instantiate directly.
+    Config: ClassVar[type[BaseConfig]] = BaseConfig
+
+    # what other plugins (and the master catalog) need to know about this
+    # automation. None means this automation doesn't have that kind of resource.
+    Output: ClassVar[Optional[type[BaseModel]]] = None      # stream payload type (one stream per automation, by convention)
+    TaskPayload: ClassVar[Optional[type[BaseModel]]] = None # BatchAutomation task payload
+    Pools: ClassVar[dict[str, Pool]] = {}                   # pools this automation owns
+
+    automation_type: ClassVar[str] = ""
+
+    async def cleanup(self, ctx: Context[C]) -> None:
+        return None
+
+    @classmethod
+    def own_stream(cls) -> Optional[Stream]:
+        if cls.Output is None:
+            return None
+        return Stream(cls.automation_type, payload=cls.Output)
+
+    @classmethod
+    def own_queue(cls) -> Optional[Queue]:
+        if cls.TaskPayload is None:
+            return None
+        return Queue(cls.automation_type, payload=cls.TaskPayload)
+
+
+class ContinuousAutomation(BaseAutomation[C]):
+    async def run_slot(self, ctx: Context[C]) -> None:
+        raise NotImplementedError
+
+
+class BatchAutomation(BaseAutomation[C]):
+    async def run_one(self, task: Task, ctx: Context[C]) -> None:
+        raise NotImplementedError
+
+
+def register(name: str):
+    def _wrap(cls: type[BaseAutomation]) -> type[BaseAutomation]:
+        if not issubclass(cls, BaseAutomation):
+            raise TypeError(f"{cls.__name__} must subclass BaseAutomation")
+        # BatchAutomation must declare TaskPayload; ContinuousAutomation should declare Output.
+        if issubclass(cls, BatchAutomation) and cls.TaskPayload is None:
+            raise TypeError(
+                f"{cls.__name__} is a BatchAutomation but did not declare 'TaskPayload'"
+            )
+        cls.automation_type = name
+        if name in _REGISTRY and _REGISTRY[name] is not cls:
+            raise ValueError(f"automation '{name}' already registered: {_REGISTRY[name]}")
+        _REGISTRY[name] = cls
+        logger.debug("registered automation '%s' -> %s", name, cls.__name__)
+        return cls
+
+    return _wrap
+
+
+def get_registry() -> dict[str, type[BaseAutomation]]:
+    return dict(_REGISTRY)
+
+
+def load_entry_points(group: str = "fleet.automations") -> dict[str, type[BaseAutomation]]:
+    try:
+        eps = md.entry_points(group=group)
+    except TypeError:
+        eps = md.entry_points().get(group, [])  # type: ignore[assignment]
+    for ep in eps:
+        try:
+            ep.load()
+        except Exception:
+            logger.exception("failed to load entry-point %s", ep.name)
+    return get_registry()
+
+
+def catalog_doc(cls: type[BaseAutomation]) -> dict[str, Any]:
+    # the master serves this. consumers don't need it for typing (they import
+    # the same Pool/Queue/Stream specs from the shared contracts package),
+    # but it's useful for the dashboard and for write-side validation.
+    out: dict[str, Any] = {
+        "type": cls.automation_type,
+        "kind": cls.__mro__[1].__name__,
+        "config": cls.Config.model_json_schema(),
+    }
+    if cls.Output is not None:
+        out["stream"] = {
+            "name": cls.automation_type,
+            "payload": cls.Output.model_json_schema(),
+        }
+    if cls.TaskPayload is not None:
+        out["queue"] = {
+            "name": cls.automation_type,
+            "payload": cls.TaskPayload.model_json_schema(),
+        }
+    if cls.Pools:
+        out["pools"] = {
+            n: {"payload": p.payload.model_json_schema(), "tags": p.tags.model_json_schema()}
+            for n, p in cls.Pools.items()
+        }
+    return out