fleet-framework 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fleet_framework-0.1.0/LICENSE +21 -0
- fleet_framework-0.1.0/PKG-INFO +160 -0
- fleet_framework-0.1.0/README.md +124 -0
- fleet_framework-0.1.0/fleet/__init__.py +1 -0
- fleet_framework-0.1.0/fleet/cli.py +290 -0
- fleet_framework-0.1.0/fleet/core/__init__.py +69 -0
- fleet_framework-0.1.0/fleet/core/automation.py +125 -0
- fleet_framework-0.1.0/fleet/core/backend.py +736 -0
- fleet_framework-0.1.0/fleet/core/config.py +38 -0
- fleet_framework-0.1.0/fleet/core/context.py +102 -0
- fleet_framework-0.1.0/fleet/core/contract.py +87 -0
- fleet_framework-0.1.0/fleet/core/country_presets.py +50 -0
- fleet_framework-0.1.0/fleet/core/events.py +55 -0
- fleet_framework-0.1.0/fleet/core/logging.py +97 -0
- fleet_framework-0.1.0/fleet/core/memory_backend.py +492 -0
- fleet_framework-0.1.0/fleet/core/metrics.py +61 -0
- fleet_framework-0.1.0/fleet/core/otel.py +97 -0
- fleet_framework-0.1.0/fleet/core/primitives.py +310 -0
- fleet_framework-0.1.0/fleet/core/protocol.py +171 -0
- fleet_framework-0.1.0/fleet/core/proxy.py +166 -0
- fleet_framework-0.1.0/fleet/core/reconcile.py +75 -0
- fleet_framework-0.1.0/fleet/core/sqlite_backend.py +1117 -0
- fleet_framework-0.1.0/fleet/core/store.py +104 -0
- fleet_framework-0.1.0/fleet/master/__init__.py +3 -0
- fleet_framework-0.1.0/fleet/master/api.py +324 -0
- fleet_framework-0.1.0/fleet/master/app.py +105 -0
- fleet_framework-0.1.0/fleet/master/auth.py +132 -0
- fleet_framework-0.1.0/fleet/master/broadcaster.py +37 -0
- fleet_framework-0.1.0/fleet/master/dashboard/__init__.py +4 -0
- fleet_framework-0.1.0/fleet/master/dashboard/router.py +36 -0
- fleet_framework-0.1.0/fleet/master/dashboard/static/style.css +97 -0
- fleet_framework-0.1.0/fleet/master/dashboard/templates/index.html +372 -0
- fleet_framework-0.1.0/fleet/master/metrics_route.py +141 -0
- fleet_framework-0.1.0/fleet/master/ratelimit.py +55 -0
- fleet_framework-0.1.0/fleet/master/ws_router.py +142 -0
- fleet_framework-0.1.0/fleet/worker/__init__.py +3 -0
- fleet_framework-0.1.0/fleet/worker/agent.py +173 -0
- fleet_framework-0.1.0/fleet/worker/reconcile_loop.py +246 -0
- fleet_framework-0.1.0/fleet/worker/slot_runner.py +256 -0
- fleet_framework-0.1.0/fleet/worker/ws_client.py +164 -0
- fleet_framework-0.1.0/fleet_browser/__init__.py +21 -0
- fleet_framework-0.1.0/fleet_browser/browser.py +277 -0
- fleet_framework-0.1.0/fleet_browser/cert.py +68 -0
- fleet_framework-0.1.0/fleet_browser/fingerprint.py +327 -0
- fleet_framework-0.1.0/fleet_browser/humanizer.py +157 -0
- fleet_framework-0.1.0/fleet_browser/pool.py +241 -0
- fleet_framework-0.1.0/fleet_browser/proxy_extension.py +122 -0
- fleet_framework-0.1.0/fleet_browser/solver.py +51 -0
- fleet_framework-0.1.0/fleet_browser/stealth.py +80 -0
- fleet_framework-0.1.0/fleet_cloudflare/__init__.py +22 -0
- fleet_framework-0.1.0/fleet_cloudflare/bypasser.py +168 -0
- fleet_framework-0.1.0/fleet_cloudflare/harvest.py +266 -0
- fleet_framework-0.1.0/fleet_cloudflare/replay.py +82 -0
- fleet_framework-0.1.0/fleet_cloudflare/solver.py +28 -0
- fleet_framework-0.1.0/fleet_content/__init__.py +24 -0
- fleet_framework-0.1.0/fleet_content/automation.py +43 -0
- fleet_framework-0.1.0/fleet_content/contracts.py +76 -0
- fleet_framework-0.1.0/fleet_detect/__init__.py +26 -0
- fleet_framework-0.1.0/fleet_detect/contracts.py +67 -0
- fleet_framework-0.1.0/fleet_detect/detect.py +126 -0
- fleet_framework-0.1.0/fleet_framework.egg-info/PKG-INFO +160 -0
- fleet_framework-0.1.0/fleet_framework.egg-info/SOURCES.txt +88 -0
- fleet_framework-0.1.0/fleet_framework.egg-info/dependency_links.txt +1 -0
- fleet_framework-0.1.0/fleet_framework.egg-info/entry_points.txt +9 -0
- fleet_framework-0.1.0/fleet_framework.egg-info/requires.txt +28 -0
- fleet_framework-0.1.0/fleet_framework.egg-info/top_level.txt +14 -0
- fleet_framework-0.1.0/fleet_headers/__init__.py +28 -0
- fleet_framework-0.1.0/fleet_headers/profiles.py +131 -0
- fleet_framework-0.1.0/fleet_jobs/__init__.py +28 -0
- fleet_framework-0.1.0/fleet_jobs/automation.py +34 -0
- fleet_framework-0.1.0/fleet_jobs/contracts.py +143 -0
- fleet_framework-0.1.0/fleet_marketplace/__init__.py +33 -0
- fleet_framework-0.1.0/fleet_marketplace/automation.py +32 -0
- fleet_framework-0.1.0/fleet_marketplace/contracts.py +151 -0
- fleet_framework-0.1.0/fleet_news/__init__.py +21 -0
- fleet_framework-0.1.0/fleet_news/automation.py +51 -0
- fleet_framework-0.1.0/fleet_news/contracts.py +59 -0
- fleet_framework-0.1.0/fleet_place/__init__.py +33 -0
- fleet_framework-0.1.0/fleet_place/automation.py +37 -0
- fleet_framework-0.1.0/fleet_place/contracts.py +156 -0
- fleet_framework-0.1.0/fleet_provider_dataimpulse/__init__.py +82 -0
- fleet_framework-0.1.0/fleet_provider_evomi/__init__.py +76 -0
- fleet_framework-0.1.0/fleet_serp/__init__.py +30 -0
- fleet_framework-0.1.0/fleet_serp/automation.py +47 -0
- fleet_framework-0.1.0/fleet_serp/contracts.py +100 -0
- fleet_framework-0.1.0/fleet_social/__init__.py +34 -0
- fleet_framework-0.1.0/fleet_social/automation.py +44 -0
- fleet_framework-0.1.0/fleet_social/contracts.py +172 -0
- fleet_framework-0.1.0/pyproject.toml +97 -0
- fleet_framework-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 fleet contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fleet-framework
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: generic distributed-automation framework — master/worker, browser pool, anti-bot helpers, and abstract automation contracts (SERP, content, news, place, marketplace, jobs, social)
|
|
5
|
+
Author: Sarper Avci
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sarperavci/fleet
|
|
8
|
+
Project-URL: Repository, https://github.com/sarperavci/fleet
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: fastapi>=0.115.0
|
|
13
|
+
Requires-Dist: uvicorn[standard]>=0.30.0
|
|
14
|
+
Requires-Dist: pydantic>=2.7.0
|
|
15
|
+
Requires-Dist: websockets>=12.0
|
|
16
|
+
Requires-Dist: httpx>=0.27.0
|
|
17
|
+
Requires-Dist: redis>=5.0.0
|
|
18
|
+
Requires-Dist: jinja2>=3.1.0
|
|
19
|
+
Requires-Dist: psutil>=5.9.0
|
|
20
|
+
Requires-Dist: click>=8.1.0
|
|
21
|
+
Provides-Extra: browser
|
|
22
|
+
Requires-Dist: DrissionPage>=4.1.0; extra == "browser"
|
|
23
|
+
Requires-Dist: cryptography>=42.0.0; extra == "browser"
|
|
24
|
+
Provides-Extra: cloudflare
|
|
25
|
+
Requires-Dist: fleet-framework[browser]; extra == "cloudflare"
|
|
26
|
+
Provides-Extra: otel
|
|
27
|
+
Requires-Dist: opentelemetry-api>=1.27.0; extra == "otel"
|
|
28
|
+
Requires-Dist: opentelemetry-sdk>=1.27.0; extra == "otel"
|
|
29
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27.0; extra == "otel"
|
|
30
|
+
Provides-Extra: test
|
|
31
|
+
Requires-Dist: pytest>=8.0; extra == "test"
|
|
32
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "test"
|
|
33
|
+
Provides-Extra: all
|
|
34
|
+
Requires-Dist: fleet-framework[browser,cloudflare,otel]; extra == "all"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# Fleet
|
|
38
|
+
|
|
39
|
+
A framework for distributed automation fleets. One master, many workers, typed output streams between them. Born out of a production CAPTCHA-solving farm, generalized to fit any "N machines each doing M parallel things" workload.
|
|
40
|
+
|
|
41
|
+
## What it is
|
|
42
|
+
|
|
43
|
+
Two pip packages. `fleet-core` is the framework: master, worker shell, reconcile loop, Redis-backed store, output streams, event bus, a small dashboard. `fleet-browser` is an optional add-on that ships a battle-tested Chromium pool with fingerprint rotation, proxy auth, and orphan-process cleanup.
|
|
44
|
+
|
|
45
|
+
You write a Python class subclassing `ContinuousAutomation` or `BatchAutomation` and a Pydantic config. Ship it as a pip package with one entry-point line. Install it on the master and on each worker host and the framework wires up everything else: config push, validation, slot lifecycle, recycle, heal, stats, dashboard, stream-based inter-automation comms.
|
|
46
|
+
|
|
47
|
+
## What's already in the box
|
|
48
|
+
|
|
49
|
+
- Master process: REST + WebSocket API, auto-discovered plugins, dashboard, Redis-backed state.
|
|
50
|
+
- Worker process: pulls config, runs N concurrent slots, reconciles to desired state, heals on failure.
|
|
51
|
+
- Reconcile loop: pure-functional diff, smallest-converging-plan semantics, slot recycle, self-heal.
|
|
52
|
+
- Typed output streams: ZSET-backed, bounded by length and TTL, one per automation type, pop or peek over REST or in-process.
|
|
53
|
+
- Event bus: Redis pubsub for lightweight signals.
|
|
54
|
+
- Auth: bearer tokens, admin vs worker, constant-time comparison. WebSocket handshake validated before accept.
|
|
55
|
+
- Hardening from the original farm: tini in the worker container reaps zombies, `psutil`-based process-tree cleanup catches orphans, recycle's heal step closes the "zero-slot lockout" failure mode.
|
|
56
|
+
|
|
57
|
+
## What you write
|
|
58
|
+
|
|
59
|
+
Roughly 30 lines per automation. Here's a fully working example:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
import asyncio
|
|
63
|
+
import httpx
|
|
64
|
+
from fleet.core import BaseConfig, ContinuousAutomation, register
|
|
65
|
+
|
|
66
|
+
class PingerConfig(BaseConfig):
|
|
67
|
+
url: str
|
|
68
|
+
interval_seconds: float = 5.0
|
|
69
|
+
|
|
70
|
+
@register("pinger")
|
|
71
|
+
class Pinger(ContinuousAutomation[PingerConfig]):
|
|
72
|
+
Config = PingerConfig
|
|
73
|
+
|
|
74
|
+
async def run_slot(self, ctx):
|
|
75
|
+
async with httpx.AsyncClient(proxy=ctx.proxy) as client:
|
|
76
|
+
while not ctx.shutdown.is_set():
|
|
77
|
+
r = await client.get(ctx.config.url)
|
|
78
|
+
await ctx.emit({"status": r.status_code})
|
|
79
|
+
await asyncio.sleep(ctx.config.interval_seconds)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Plus one line in your `pyproject.toml`:
|
|
83
|
+
|
|
84
|
+
```toml
|
|
85
|
+
[project.entry-points."fleet.automations"]
|
|
86
|
+
pinger = "my_package:Pinger"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Quickstart
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
git clone <repo-url> fleet && cd fleet
|
|
93
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
94
|
+
pip install -e packages/fleet-core
|
|
95
|
+
pip install -e examples/hello-world
|
|
96
|
+
|
|
97
|
+
# in one terminal:
|
|
98
|
+
redis-server --port 6379 --appendonly yes
|
|
99
|
+
# in another:
|
|
100
|
+
FLEET_ADMIN_TOKEN=admin FLEET_WORKER_TOKEN=worker fleet master
|
|
101
|
+
# in another:
|
|
102
|
+
MASTER_URL=http://localhost:8000 FLEET_WORKER_TOKEN=worker fleet worker --type hello-world --id w1
|
|
103
|
+
# in another:
|
|
104
|
+
curl -X PATCH http://localhost:8000/api/v1/automations/hello-world/workers/w1/config \
|
|
105
|
+
-H "Authorization: Bearer admin" -H "Content-Type: application/json" \
|
|
106
|
+
-d '{"enabled": true, "slots": 2}'
|
|
107
|
+
curl -X POST http://localhost:8000/api/v1/automations/hello-world/output/pop \
|
|
108
|
+
-H "Authorization: Bearer admin" -H "Content-Type: application/json" \
|
|
109
|
+
-d '{"n": 5}'
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Walkthrough with explanations: [docs/getting-started/quickstart.md](docs/getting-started/quickstart.md).
|
|
113
|
+
|
|
114
|
+
## Repo layout
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
fleet/
|
|
118
|
+
├── packages/
|
|
119
|
+
│ ├── fleet-core/ master, worker, reconcile, store, streams, event bus, dashboard, CLI
|
|
120
|
+
│ └── fleet-browser/ optional Chromium pool, fingerprint, humanizer, proxy auth, cert
|
|
121
|
+
├── examples/
|
|
122
|
+
│ ├── hello-world/ minimal ContinuousAutomation (emit greetings on a timer)
|
|
123
|
+
│ └── echo-consumer/ consumes another automation's output stream
|
|
124
|
+
└── docs/ GitBook-format documentation (also lives on the `docs` branch)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Documentation
|
|
128
|
+
|
|
129
|
+
Full docs in [`docs/`](docs/), rendered as a GitBook from the `docs` branch.
|
|
130
|
+
|
|
131
|
+
- [Introduction](docs/README.md)
|
|
132
|
+
- [Installation](docs/getting-started/installation.md)
|
|
133
|
+
- [Quickstart](docs/getting-started/quickstart.md)
|
|
134
|
+
- [Your first automation](docs/getting-started/first-automation.md)
|
|
135
|
+
- [Architecture](docs/concepts/architecture.md)
|
|
136
|
+
- [Writing a continuous automation](docs/guides/continuous-automation.md)
|
|
137
|
+
- [Writing a batch automation](docs/guides/batch-automation.md)
|
|
138
|
+
- [Making two automations talk](docs/guides/inter-automation.md)
|
|
139
|
+
- [Browser-based automations](docs/guides/browser-automations.md)
|
|
140
|
+
- [Deployment](docs/guides/deployment.md)
|
|
141
|
+
- [REST API](docs/reference/rest-api.md)
|
|
142
|
+
- [WebSocket protocol](docs/reference/ws-protocol.md)
|
|
143
|
+
- [Authentication](docs/operations/auth.md)
|
|
144
|
+
- [Troubleshooting](docs/operations/troubleshooting.md)
|
|
145
|
+
|
|
146
|
+
## Origins
|
|
147
|
+
|
|
148
|
+
Fleet started as the control plane for a Cloudflare Turnstile-solving farm. That farm hit every interesting failure mode in distributed automation: zombie subprocess accumulation, master state wipe on Redis restart, gen-counter regression, slot-recycle drops with no heal, and config drift across heterogeneous workers. Each of those has a fix baked into the framework.
|
|
149
|
+
|
|
150
|
+
The framework is what the original farm should have been from day one. It's deliberately small. If you want a scheduler, an orchestrator, or a service mesh, this isn't it. If you want a thin layer that turns "N hosts doing repetitive work" into a manageable system, it is.
|
|
151
|
+
|
|
152
|
+
## Status
|
|
153
|
+
|
|
154
|
+
Pre-alpha. The public API will change. No PyPI release yet — install from source. Production deployments are running on it but with a lot of manual operational knowledge to back them up. Watch the changelog before relying on anything sensitive.
|
|
155
|
+
|
|
156
|
+
See [ROADMAP.md](ROADMAP.md) for the path to v0.1.
|
|
157
|
+
|
|
158
|
+
## License
|
|
159
|
+
|
|
160
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Fleet
|
|
2
|
+
|
|
3
|
+
A framework for distributed automation fleets. One master, many workers, typed output streams between them. Born out of a production CAPTCHA-solving farm, generalized to fit any "N machines each doing M parallel things" workload.
|
|
4
|
+
|
|
5
|
+
## What it is
|
|
6
|
+
|
|
7
|
+
Two pip packages. `fleet-core` is the framework: master, worker shell, reconcile loop, Redis-backed store, output streams, event bus, a small dashboard. `fleet-browser` is an optional add-on that ships a battle-tested Chromium pool with fingerprint rotation, proxy auth, and orphan-process cleanup.
|
|
8
|
+
|
|
9
|
+
You write a Python class subclassing `ContinuousAutomation` or `BatchAutomation` and a Pydantic config. Ship it as a pip package with one entry-point line. Install it on the master and on each worker host and the framework wires up everything else: config push, validation, slot lifecycle, recycle, heal, stats, dashboard, stream-based inter-automation comms.
|
|
10
|
+
|
|
11
|
+
## What's already in the box
|
|
12
|
+
|
|
13
|
+
- Master process: REST + WebSocket API, auto-discovered plugins, dashboard, Redis-backed state.
|
|
14
|
+
- Worker process: pulls config, runs N concurrent slots, reconciles to desired state, heals on failure.
|
|
15
|
+
- Reconcile loop: pure-functional diff, smallest-converging-plan semantics, slot recycle, self-heal.
|
|
16
|
+
- Typed output streams: ZSET-backed, bounded by length and TTL, one per automation type, pop or peek over REST or in-process.
|
|
17
|
+
- Event bus: Redis pubsub for lightweight signals.
|
|
18
|
+
- Auth: bearer tokens, admin vs worker, constant-time comparison. WebSocket handshake validated before accept.
|
|
19
|
+
- Hardening from the original farm: tini in the worker container reaps zombies, `psutil`-based process-tree cleanup catches orphans, recycle's heal step closes the "zero-slot lockout" failure mode.
|
|
20
|
+
|
|
21
|
+
## What you write
|
|
22
|
+
|
|
23
|
+
Roughly 30 lines per automation. Here's a fully working example:
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
import asyncio
|
|
27
|
+
import httpx
|
|
28
|
+
from fleet.core import BaseConfig, ContinuousAutomation, register
|
|
29
|
+
|
|
30
|
+
class PingerConfig(BaseConfig):
|
|
31
|
+
url: str
|
|
32
|
+
interval_seconds: float = 5.0
|
|
33
|
+
|
|
34
|
+
@register("pinger")
|
|
35
|
+
class Pinger(ContinuousAutomation[PingerConfig]):
|
|
36
|
+
Config = PingerConfig
|
|
37
|
+
|
|
38
|
+
async def run_slot(self, ctx):
|
|
39
|
+
async with httpx.AsyncClient(proxy=ctx.proxy) as client:
|
|
40
|
+
while not ctx.shutdown.is_set():
|
|
41
|
+
r = await client.get(ctx.config.url)
|
|
42
|
+
await ctx.emit({"status": r.status_code})
|
|
43
|
+
await asyncio.sleep(ctx.config.interval_seconds)
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Plus one line in your `pyproject.toml`:
|
|
47
|
+
|
|
48
|
+
```toml
|
|
49
|
+
[project.entry-points."fleet.automations"]
|
|
50
|
+
pinger = "my_package:Pinger"
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Quickstart
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
git clone <repo-url> fleet && cd fleet
|
|
57
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
58
|
+
pip install -e packages/fleet-core
|
|
59
|
+
pip install -e examples/hello-world
|
|
60
|
+
|
|
61
|
+
# in one terminal:
|
|
62
|
+
redis-server --port 6379 --appendonly yes
|
|
63
|
+
# in another:
|
|
64
|
+
FLEET_ADMIN_TOKEN=admin FLEET_WORKER_TOKEN=worker fleet master
|
|
65
|
+
# in another:
|
|
66
|
+
MASTER_URL=http://localhost:8000 FLEET_WORKER_TOKEN=worker fleet worker --type hello-world --id w1
|
|
67
|
+
# in another:
|
|
68
|
+
curl -X PATCH http://localhost:8000/api/v1/automations/hello-world/workers/w1/config \
|
|
69
|
+
-H "Authorization: Bearer admin" -H "Content-Type: application/json" \
|
|
70
|
+
-d '{"enabled": true, "slots": 2}'
|
|
71
|
+
curl -X POST http://localhost:8000/api/v1/automations/hello-world/output/pop \
|
|
72
|
+
-H "Authorization: Bearer admin" -H "Content-Type: application/json" \
|
|
73
|
+
-d '{"n": 5}'
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Walkthrough with explanations: [docs/getting-started/quickstart.md](docs/getting-started/quickstart.md).
|
|
77
|
+
|
|
78
|
+
## Repo layout
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
fleet/
|
|
82
|
+
├── packages/
|
|
83
|
+
│ ├── fleet-core/ master, worker, reconcile, store, streams, event bus, dashboard, CLI
|
|
84
|
+
│ └── fleet-browser/ optional Chromium pool, fingerprint, humanizer, proxy auth, cert
|
|
85
|
+
├── examples/
|
|
86
|
+
│ ├── hello-world/ minimal ContinuousAutomation (emit greetings on a timer)
|
|
87
|
+
│ └── echo-consumer/ consumes another automation's output stream
|
|
88
|
+
└── docs/ GitBook-format documentation (also lives on the `docs` branch)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Documentation
|
|
92
|
+
|
|
93
|
+
Full docs in [`docs/`](docs/), rendered as a GitBook from the `docs` branch.
|
|
94
|
+
|
|
95
|
+
- [Introduction](docs/README.md)
|
|
96
|
+
- [Installation](docs/getting-started/installation.md)
|
|
97
|
+
- [Quickstart](docs/getting-started/quickstart.md)
|
|
98
|
+
- [Your first automation](docs/getting-started/first-automation.md)
|
|
99
|
+
- [Architecture](docs/concepts/architecture.md)
|
|
100
|
+
- [Writing a continuous automation](docs/guides/continuous-automation.md)
|
|
101
|
+
- [Writing a batch automation](docs/guides/batch-automation.md)
|
|
102
|
+
- [Making two automations talk](docs/guides/inter-automation.md)
|
|
103
|
+
- [Browser-based automations](docs/guides/browser-automations.md)
|
|
104
|
+
- [Deployment](docs/guides/deployment.md)
|
|
105
|
+
- [REST API](docs/reference/rest-api.md)
|
|
106
|
+
- [WebSocket protocol](docs/reference/ws-protocol.md)
|
|
107
|
+
- [Authentication](docs/operations/auth.md)
|
|
108
|
+
- [Troubleshooting](docs/operations/troubleshooting.md)
|
|
109
|
+
|
|
110
|
+
## Origins
|
|
111
|
+
|
|
112
|
+
Fleet started as the control plane for a Cloudflare Turnstile-solving farm. That farm hit every interesting failure mode in distributed automation: zombie subprocess accumulation, master state wipe on Redis restart, gen-counter regression, slot-recycle drops with no heal, and config drift across heterogeneous workers. Each of those has a fix baked into the framework.
|
|
113
|
+
|
|
114
|
+
The framework is what the original farm should have been from day one. It's deliberately small. If you want a scheduler, an orchestrator, or a service mesh, this isn't it. If you want a thin layer that turns "N hosts doing repetitive work" into a manageable system, it is.
|
|
115
|
+
|
|
116
|
+
## Status
|
|
117
|
+
|
|
118
|
+
Pre-alpha. The public API will change. No PyPI release yet — install from source. Production deployments are running on it but with a lot of manual operational knowledge to back them up. Watch the changelog before relying on anything sensitive.
|
|
119
|
+
|
|
120
|
+
See [ROADMAP.md](ROADMAP.md) for the path to v0.1.
|
|
121
|
+
|
|
122
|
+
## License
|
|
123
|
+
|
|
124
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# namespace marker
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import socket
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
import click
|
|
10
|
+
import uvicorn
|
|
11
|
+
|
|
12
|
+
from fleet.core.logging import configure_logging
|
|
13
|
+
|
|
14
|
+
configure_logging()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.group()
|
|
18
|
+
def cli() -> None:
|
|
19
|
+
# fleet — master + worker CLI.
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@cli.command()
|
|
24
|
+
@click.option("--host", default="0.0.0.0", show_default=True)
|
|
25
|
+
@click.option("--port", default=8000, show_default=True)
|
|
26
|
+
@click.option(
|
|
27
|
+
"--backend-url",
|
|
28
|
+
envvar="FLEET_BACKEND_URL",
|
|
29
|
+
default=None,
|
|
30
|
+
show_default=True,
|
|
31
|
+
help="Backend URL: redis://..., memory://, or any registered scheme. "
|
|
32
|
+
"Defaults to $FLEET_BACKEND_URL, then $REDIS_URL, then redis://localhost:6379/0.",
|
|
33
|
+
)
|
|
34
|
+
@click.option(
|
|
35
|
+
"--redis-url",
|
|
36
|
+
envvar="REDIS_URL",
|
|
37
|
+
default=None,
|
|
38
|
+
show_default=True,
|
|
39
|
+
help="(Legacy) Use --backend-url instead.",
|
|
40
|
+
)
|
|
41
|
+
def master(host: str, port: int, backend_url: str | None, redis_url: str | None) -> None:
|
|
42
|
+
"""Start the master process. Binds REST + WS + dashboard."""
|
|
43
|
+
from fleet.master import create_app
|
|
44
|
+
|
|
45
|
+
url = backend_url or redis_url or "redis://localhost:6379/0"
|
|
46
|
+
if redis_url:
|
|
47
|
+
os.environ.setdefault("REDIS_URL", redis_url)
|
|
48
|
+
os.environ.setdefault("FLEET_BACKEND_URL", url)
|
|
49
|
+
app = create_app(url)
|
|
50
|
+
uvicorn.run(app, host=host, port=port, log_level=os.environ.get("LOG_LEVEL", "info").lower())
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@cli.command()
|
|
54
|
+
@click.option("--type", "automation_type", required=True, help="automation type name (entry-point key)")
|
|
55
|
+
@click.option("--id", "worker_id", default=None, help="worker id (defaults to hostname)")
|
|
56
|
+
@click.option("--master-url", envvar="MASTER_URL", required=True)
|
|
57
|
+
@click.option("--worker-token", envvar="FLEET_WORKER_TOKEN", required=True)
|
|
58
|
+
@click.option(
|
|
59
|
+
"--backend-url",
|
|
60
|
+
envvar="FLEET_BACKEND_URL",
|
|
61
|
+
default=None,
|
|
62
|
+
help="Backend URL; falls back to --redis-url then redis://localhost:6379/0.",
|
|
63
|
+
)
|
|
64
|
+
@click.option(
|
|
65
|
+
"--redis-url",
|
|
66
|
+
envvar="REDIS_URL",
|
|
67
|
+
default=None,
|
|
68
|
+
help="(Legacy) Use --backend-url instead.",
|
|
69
|
+
)
|
|
70
|
+
def worker(
|
|
71
|
+
automation_type: str,
|
|
72
|
+
worker_id: str | None,
|
|
73
|
+
master_url: str,
|
|
74
|
+
worker_token: str,
|
|
75
|
+
backend_url: str | None,
|
|
76
|
+
redis_url: str | None,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""Start a worker process pinned to one automation type."""
|
|
79
|
+
from fleet.worker import Agent
|
|
80
|
+
|
|
81
|
+
url = backend_url or redis_url or "redis://localhost:6379/0"
|
|
82
|
+
wid = worker_id or socket.gethostname()
|
|
83
|
+
agent = Agent(
|
|
84
|
+
automation_type=automation_type,
|
|
85
|
+
worker_id=wid,
|
|
86
|
+
master_url=master_url,
|
|
87
|
+
worker_token=worker_token,
|
|
88
|
+
redis_url=url,
|
|
89
|
+
)
|
|
90
|
+
try:
|
|
91
|
+
asyncio.run(agent.run())
|
|
92
|
+
except KeyboardInterrupt:
|
|
93
|
+
sys.exit(0)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _list_plugins() -> None:
|
|
97
|
+
from fleet.core.automation import load_entry_points
|
|
98
|
+
from fleet.core.proxy import load_provider_entry_points
|
|
99
|
+
|
|
100
|
+
reg = load_entry_points()
|
|
101
|
+
if reg:
|
|
102
|
+
click.echo("automations:")
|
|
103
|
+
for name, cls in reg.items():
|
|
104
|
+
kind = cls.__mro__[1].__name__
|
|
105
|
+
click.echo(f" {name:30} {cls.__module__}.{cls.__name__} ({kind})")
|
|
106
|
+
else:
|
|
107
|
+
click.echo("automations: (none installed)")
|
|
108
|
+
providers = load_provider_entry_points()
|
|
109
|
+
if providers:
|
|
110
|
+
click.echo("proxy providers:")
|
|
111
|
+
for name in providers:
|
|
112
|
+
click.echo(f" {name}")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@cli.command()
|
|
116
|
+
def plugins() -> None:
|
|
117
|
+
"""List installed automations + proxy providers discovered via entry-points."""
|
|
118
|
+
_list_plugins()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@cli.command(name="info")
|
|
122
|
+
def info_alias() -> None:
|
|
123
|
+
"""Alias of `plugins`. Kept for backwards compatibility."""
|
|
124
|
+
_list_plugins()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@cli.command()
|
|
128
|
+
@click.argument("automation_type")
|
|
129
|
+
@click.option("--master-url", envvar="MASTER_URL", required=True)
|
|
130
|
+
@click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", required=True)
|
|
131
|
+
@click.option("--n", default=10, show_default=True, help="entries per pop")
|
|
132
|
+
@click.option("--interval", default=1.0, show_default=True, help="seconds between polls")
|
|
133
|
+
@click.option("--peek", is_flag=True, help="non-destructive read instead of pop")
|
|
134
|
+
def tail(
|
|
135
|
+
automation_type: str,
|
|
136
|
+
master_url: str,
|
|
137
|
+
admin_token: str,
|
|
138
|
+
n: int,
|
|
139
|
+
interval: float,
|
|
140
|
+
peek: bool,
|
|
141
|
+
) -> None:
|
|
142
|
+
"""Stream the output of an automation type to stdout, one JSON object per line."""
|
|
143
|
+
import httpx
|
|
144
|
+
|
|
145
|
+
headers = {"Authorization": f"Bearer {admin_token}"}
|
|
146
|
+
op = "peek" if peek else "pop"
|
|
147
|
+
base = master_url.rstrip("/")
|
|
148
|
+
try:
|
|
149
|
+
while True:
|
|
150
|
+
try:
|
|
151
|
+
if peek:
|
|
152
|
+
url = f"{base}/api/v1/automations/{automation_type}/output/peek?n={n}"
|
|
153
|
+
r = httpx.get(url, headers=headers, timeout=10)
|
|
154
|
+
else:
|
|
155
|
+
url = f"{base}/api/v1/automations/{automation_type}/output/pop"
|
|
156
|
+
r = httpx.post(url, headers=headers, json={"n": n}, timeout=10)
|
|
157
|
+
r.raise_for_status()
|
|
158
|
+
for env in r.json():
|
|
159
|
+
click.echo(json.dumps(env, default=str))
|
|
160
|
+
except Exception as e:
|
|
161
|
+
click.echo(f"# {op} failed: {e}", err=True)
|
|
162
|
+
import time as _t
|
|
163
|
+
_t.sleep(max(0.1, interval))
|
|
164
|
+
except KeyboardInterrupt:
|
|
165
|
+
sys.exit(0)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@cli.command()
|
|
169
|
+
@click.argument("automation_type")
|
|
170
|
+
@click.argument("payload") # JSON string or @path
|
|
171
|
+
@click.option("--master-url", envvar="MASTER_URL", required=True)
|
|
172
|
+
@click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", required=True)
|
|
173
|
+
@click.option("--task-id", default=None, help="override the framework's UUID")
|
|
174
|
+
@click.option("--max-attempts", default=None, type=int, help="override per-task retry budget")
|
|
175
|
+
def submit(
|
|
176
|
+
automation_type: str,
|
|
177
|
+
payload: str,
|
|
178
|
+
master_url: str,
|
|
179
|
+
admin_token: str,
|
|
180
|
+
task_id: str | None,
|
|
181
|
+
max_attempts: int | None,
|
|
182
|
+
) -> None:
|
|
183
|
+
"""Submit a single task to a BatchAutomation.
|
|
184
|
+
|
|
185
|
+
PAYLOAD is a JSON string or @path/to/file.json. Validated against the
|
|
186
|
+
automation's TaskPayload schema server-side.
|
|
187
|
+
"""
|
|
188
|
+
import httpx
|
|
189
|
+
|
|
190
|
+
body_text = payload
|
|
191
|
+
if payload.startswith("@"):
|
|
192
|
+
body_text = open(payload[1:]).read()
|
|
193
|
+
try:
|
|
194
|
+
parsed = json.loads(body_text)
|
|
195
|
+
except json.JSONDecodeError as e:
|
|
196
|
+
click.echo(f"payload is not valid JSON: {e}", err=True)
|
|
197
|
+
sys.exit(2)
|
|
198
|
+
req: dict = {"payload": parsed}
|
|
199
|
+
if task_id:
|
|
200
|
+
req["task_id"] = task_id
|
|
201
|
+
if max_attempts is not None:
|
|
202
|
+
req["max_attempts"] = max_attempts
|
|
203
|
+
url = master_url.rstrip("/") + f"/api/v1/automations/{automation_type}/tasks"
|
|
204
|
+
r = httpx.post(url, headers={"Authorization": f"Bearer {admin_token}"}, json=req, timeout=10)
|
|
205
|
+
if r.status_code >= 400:
|
|
206
|
+
click.echo(f"submit failed ({r.status_code}): {r.text}", err=True)
|
|
207
|
+
sys.exit(1)
|
|
208
|
+
click.echo(json.dumps(r.json()))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@cli.command()
|
|
212
|
+
@click.option("--master-url", envvar="MASTER_URL", default=None)
|
|
213
|
+
@click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", default=None)
|
|
214
|
+
@click.option("--worker-token", envvar="FLEET_WORKER_TOKEN", default=None)
|
|
215
|
+
@click.option("--backend-url", envvar="FLEET_BACKEND_URL", default=None)
|
|
216
|
+
def doctor(
|
|
217
|
+
master_url: str | None,
|
|
218
|
+
admin_token: str | None,
|
|
219
|
+
worker_token: str | None,
|
|
220
|
+
backend_url: str | None,
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Pre-flight sanity checks: env tokens, plugins, backend, master reachable."""
|
|
223
|
+
asyncio.run(_doctor(master_url, admin_token, worker_token, backend_url))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
async def _doctor(master_url, admin_token, worker_token, backend_url) -> None:
|
|
227
|
+
import httpx
|
|
228
|
+
|
|
229
|
+
ok = True
|
|
230
|
+
|
|
231
|
+
def _check(label: str, condition: bool, detail: str = "") -> None:
|
|
232
|
+
nonlocal ok
|
|
233
|
+
mark = "✓" if condition else "✗"
|
|
234
|
+
ok = ok and condition
|
|
235
|
+
click.echo(f"{mark} {label}" + (f" — {detail}" if detail else ""))
|
|
236
|
+
|
|
237
|
+
_check("FLEET_ADMIN_TOKEN set", bool(admin_token))
|
|
238
|
+
_check("FLEET_WORKER_TOKEN set", bool(worker_token))
|
|
239
|
+
|
|
240
|
+
from fleet.core.automation import load_entry_points
|
|
241
|
+
reg = load_entry_points()
|
|
242
|
+
_check("automations discovered", bool(reg), f"{len(reg)} found")
|
|
243
|
+
for name in reg:
|
|
244
|
+
click.echo(f" {name}")
|
|
245
|
+
|
|
246
|
+
if backend_url:
|
|
247
|
+
try:
|
|
248
|
+
from fleet.core.backend import RedisBackend, backend_from_url
|
|
249
|
+
be = backend_from_url(backend_url)
|
|
250
|
+
_check("backend constructs", True, type(be).__name__)
|
|
251
|
+
if isinstance(be, RedisBackend):
|
|
252
|
+
try:
|
|
253
|
+
pong = await be.r.ping()
|
|
254
|
+
_check("redis ping", bool(pong))
|
|
255
|
+
info = await be.r.info("persistence")
|
|
256
|
+
aof = info.get(b"aof_enabled", info.get("aof_enabled"))
|
|
257
|
+
aof_on = str(aof).strip() in ("1", "True", "true")
|
|
258
|
+
_check(
|
|
259
|
+
"redis AOF on", aof_on,
|
|
260
|
+
"" if aof_on else "data lives only in RAM; configs/pools die on restart",
|
|
261
|
+
)
|
|
262
|
+
except Exception as e:
|
|
263
|
+
_check("redis reachable", False, str(e))
|
|
264
|
+
finally:
|
|
265
|
+
await be.aclose()
|
|
266
|
+
except Exception as e:
|
|
267
|
+
_check("backend constructs", False, str(e))
|
|
268
|
+
else:
|
|
269
|
+
click.echo("· backend-url not set (skipping backend checks)")
|
|
270
|
+
|
|
271
|
+
if master_url and admin_token:
|
|
272
|
+
try:
|
|
273
|
+
async with httpx.AsyncClient(timeout=3) as c:
|
|
274
|
+
r = await c.get(master_url.rstrip("/") + "/healthz")
|
|
275
|
+
_check("master /healthz reachable", r.status_code == 200, f"HTTP {r.status_code}")
|
|
276
|
+
r = await c.get(
|
|
277
|
+
master_url.rstrip("/") + "/api/v1/automations",
|
|
278
|
+
headers={"Authorization": f"Bearer {admin_token}"},
|
|
279
|
+
)
|
|
280
|
+
_check("admin token accepted", r.status_code == 200, f"HTTP {r.status_code}")
|
|
281
|
+
except Exception as e:
|
|
282
|
+
_check("master reachable", False, str(e))
|
|
283
|
+
else:
|
|
284
|
+
click.echo("· master-url or admin-token not set (skipping master checks)")
|
|
285
|
+
|
|
286
|
+
sys.exit(0 if ok else 1)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
if __name__ == "__main__":
|
|
290
|
+
cli()
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from fleet.core.automation import (
|
|
2
|
+
BaseAutomation,
|
|
3
|
+
BatchAutomation,
|
|
4
|
+
ContinuousAutomation,
|
|
5
|
+
Task,
|
|
6
|
+
catalog_doc,
|
|
7
|
+
get_registry,
|
|
8
|
+
load_entry_points,
|
|
9
|
+
register,
|
|
10
|
+
)
|
|
11
|
+
from fleet.core.backend import Backend, RedisBackend
|
|
12
|
+
from fleet.core.config import BaseConfig
|
|
13
|
+
from fleet.core.context import Context
|
|
14
|
+
from fleet.core.contract import Pool, Queue, Stream
|
|
15
|
+
from fleet.core.country_presets import known_presets, resolve_countries
|
|
16
|
+
from fleet.core.primitives import (
|
|
17
|
+
KV,
|
|
18
|
+
Counter,
|
|
19
|
+
Lock,
|
|
20
|
+
PoolFilter,
|
|
21
|
+
PoolHandle,
|
|
22
|
+
PoolItem,
|
|
23
|
+
QueueHandle,
|
|
24
|
+
StreamReader,
|
|
25
|
+
)
|
|
26
|
+
from fleet.core.proxy import (
|
|
27
|
+
ProxyHandle,
|
|
28
|
+
ProxyProvider,
|
|
29
|
+
ProxySession,
|
|
30
|
+
build_provider,
|
|
31
|
+
get_provider_registry,
|
|
32
|
+
load_provider_entry_points,
|
|
33
|
+
register_provider,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"Backend",
|
|
38
|
+
"BaseAutomation",
|
|
39
|
+
"BaseConfig",
|
|
40
|
+
"BatchAutomation",
|
|
41
|
+
"ContinuousAutomation",
|
|
42
|
+
"Context",
|
|
43
|
+
"Counter",
|
|
44
|
+
"KV",
|
|
45
|
+
"Lock",
|
|
46
|
+
"Pool",
|
|
47
|
+
"PoolFilter",
|
|
48
|
+
"PoolHandle",
|
|
49
|
+
"PoolItem",
|
|
50
|
+
"ProxyHandle",
|
|
51
|
+
"ProxyProvider",
|
|
52
|
+
"ProxySession",
|
|
53
|
+
"Queue",
|
|
54
|
+
"QueueHandle",
|
|
55
|
+
"RedisBackend",
|
|
56
|
+
"Stream",
|
|
57
|
+
"StreamReader",
|
|
58
|
+
"Task",
|
|
59
|
+
"build_provider",
|
|
60
|
+
"catalog_doc",
|
|
61
|
+
"get_provider_registry",
|
|
62
|
+
"get_registry",
|
|
63
|
+
"known_presets",
|
|
64
|
+
"load_entry_points",
|
|
65
|
+
"load_provider_entry_points",
|
|
66
|
+
"register",
|
|
67
|
+
"register_provider",
|
|
68
|
+
"resolve_countries",
|
|
69
|
+
]
|