fleet-framework 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. fleet_framework-0.1.0/LICENSE +21 -0
  2. fleet_framework-0.1.0/PKG-INFO +160 -0
  3. fleet_framework-0.1.0/README.md +124 -0
  4. fleet_framework-0.1.0/fleet/__init__.py +1 -0
  5. fleet_framework-0.1.0/fleet/cli.py +290 -0
  6. fleet_framework-0.1.0/fleet/core/__init__.py +69 -0
  7. fleet_framework-0.1.0/fleet/core/automation.py +125 -0
  8. fleet_framework-0.1.0/fleet/core/backend.py +736 -0
  9. fleet_framework-0.1.0/fleet/core/config.py +38 -0
  10. fleet_framework-0.1.0/fleet/core/context.py +102 -0
  11. fleet_framework-0.1.0/fleet/core/contract.py +87 -0
  12. fleet_framework-0.1.0/fleet/core/country_presets.py +50 -0
  13. fleet_framework-0.1.0/fleet/core/events.py +55 -0
  14. fleet_framework-0.1.0/fleet/core/logging.py +97 -0
  15. fleet_framework-0.1.0/fleet/core/memory_backend.py +492 -0
  16. fleet_framework-0.1.0/fleet/core/metrics.py +61 -0
  17. fleet_framework-0.1.0/fleet/core/otel.py +97 -0
  18. fleet_framework-0.1.0/fleet/core/primitives.py +310 -0
  19. fleet_framework-0.1.0/fleet/core/protocol.py +171 -0
  20. fleet_framework-0.1.0/fleet/core/proxy.py +166 -0
  21. fleet_framework-0.1.0/fleet/core/reconcile.py +75 -0
  22. fleet_framework-0.1.0/fleet/core/sqlite_backend.py +1117 -0
  23. fleet_framework-0.1.0/fleet/core/store.py +104 -0
  24. fleet_framework-0.1.0/fleet/master/__init__.py +3 -0
  25. fleet_framework-0.1.0/fleet/master/api.py +324 -0
  26. fleet_framework-0.1.0/fleet/master/app.py +105 -0
  27. fleet_framework-0.1.0/fleet/master/auth.py +132 -0
  28. fleet_framework-0.1.0/fleet/master/broadcaster.py +37 -0
  29. fleet_framework-0.1.0/fleet/master/dashboard/__init__.py +4 -0
  30. fleet_framework-0.1.0/fleet/master/dashboard/router.py +36 -0
  31. fleet_framework-0.1.0/fleet/master/dashboard/static/style.css +97 -0
  32. fleet_framework-0.1.0/fleet/master/dashboard/templates/index.html +372 -0
  33. fleet_framework-0.1.0/fleet/master/metrics_route.py +141 -0
  34. fleet_framework-0.1.0/fleet/master/ratelimit.py +55 -0
  35. fleet_framework-0.1.0/fleet/master/ws_router.py +142 -0
  36. fleet_framework-0.1.0/fleet/worker/__init__.py +3 -0
  37. fleet_framework-0.1.0/fleet/worker/agent.py +173 -0
  38. fleet_framework-0.1.0/fleet/worker/reconcile_loop.py +246 -0
  39. fleet_framework-0.1.0/fleet/worker/slot_runner.py +256 -0
  40. fleet_framework-0.1.0/fleet/worker/ws_client.py +164 -0
  41. fleet_framework-0.1.0/fleet_browser/__init__.py +21 -0
  42. fleet_framework-0.1.0/fleet_browser/browser.py +277 -0
  43. fleet_framework-0.1.0/fleet_browser/cert.py +68 -0
  44. fleet_framework-0.1.0/fleet_browser/fingerprint.py +327 -0
  45. fleet_framework-0.1.0/fleet_browser/humanizer.py +157 -0
  46. fleet_framework-0.1.0/fleet_browser/pool.py +241 -0
  47. fleet_framework-0.1.0/fleet_browser/proxy_extension.py +122 -0
  48. fleet_framework-0.1.0/fleet_browser/solver.py +51 -0
  49. fleet_framework-0.1.0/fleet_browser/stealth.py +80 -0
  50. fleet_framework-0.1.0/fleet_cloudflare/__init__.py +22 -0
  51. fleet_framework-0.1.0/fleet_cloudflare/bypasser.py +168 -0
  52. fleet_framework-0.1.0/fleet_cloudflare/harvest.py +266 -0
  53. fleet_framework-0.1.0/fleet_cloudflare/replay.py +82 -0
  54. fleet_framework-0.1.0/fleet_cloudflare/solver.py +28 -0
  55. fleet_framework-0.1.0/fleet_content/__init__.py +24 -0
  56. fleet_framework-0.1.0/fleet_content/automation.py +43 -0
  57. fleet_framework-0.1.0/fleet_content/contracts.py +76 -0
  58. fleet_framework-0.1.0/fleet_detect/__init__.py +26 -0
  59. fleet_framework-0.1.0/fleet_detect/contracts.py +67 -0
  60. fleet_framework-0.1.0/fleet_detect/detect.py +126 -0
  61. fleet_framework-0.1.0/fleet_framework.egg-info/PKG-INFO +160 -0
  62. fleet_framework-0.1.0/fleet_framework.egg-info/SOURCES.txt +88 -0
  63. fleet_framework-0.1.0/fleet_framework.egg-info/dependency_links.txt +1 -0
  64. fleet_framework-0.1.0/fleet_framework.egg-info/entry_points.txt +9 -0
  65. fleet_framework-0.1.0/fleet_framework.egg-info/requires.txt +28 -0
  66. fleet_framework-0.1.0/fleet_framework.egg-info/top_level.txt +14 -0
  67. fleet_framework-0.1.0/fleet_headers/__init__.py +28 -0
  68. fleet_framework-0.1.0/fleet_headers/profiles.py +131 -0
  69. fleet_framework-0.1.0/fleet_jobs/__init__.py +28 -0
  70. fleet_framework-0.1.0/fleet_jobs/automation.py +34 -0
  71. fleet_framework-0.1.0/fleet_jobs/contracts.py +143 -0
  72. fleet_framework-0.1.0/fleet_marketplace/__init__.py +33 -0
  73. fleet_framework-0.1.0/fleet_marketplace/automation.py +32 -0
  74. fleet_framework-0.1.0/fleet_marketplace/contracts.py +151 -0
  75. fleet_framework-0.1.0/fleet_news/__init__.py +21 -0
  76. fleet_framework-0.1.0/fleet_news/automation.py +51 -0
  77. fleet_framework-0.1.0/fleet_news/contracts.py +59 -0
  78. fleet_framework-0.1.0/fleet_place/__init__.py +33 -0
  79. fleet_framework-0.1.0/fleet_place/automation.py +37 -0
  80. fleet_framework-0.1.0/fleet_place/contracts.py +156 -0
  81. fleet_framework-0.1.0/fleet_provider_dataimpulse/__init__.py +82 -0
  82. fleet_framework-0.1.0/fleet_provider_evomi/__init__.py +76 -0
  83. fleet_framework-0.1.0/fleet_serp/__init__.py +30 -0
  84. fleet_framework-0.1.0/fleet_serp/automation.py +47 -0
  85. fleet_framework-0.1.0/fleet_serp/contracts.py +100 -0
  86. fleet_framework-0.1.0/fleet_social/__init__.py +34 -0
  87. fleet_framework-0.1.0/fleet_social/automation.py +44 -0
  88. fleet_framework-0.1.0/fleet_social/contracts.py +172 -0
  89. fleet_framework-0.1.0/pyproject.toml +97 -0
  90. fleet_framework-0.1.0/setup.cfg +4 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 fleet contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,160 @@
1
+ Metadata-Version: 2.4
2
+ Name: fleet-framework
3
+ Version: 0.1.0
4
+ Summary: generic distributed-automation framework — master/worker, browser pool, anti-bot helpers, and abstract automation contracts (SERP, content, news, place, marketplace, jobs, social)
5
+ Author: Sarper Avci
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/sarperavci/fleet
8
+ Project-URL: Repository, https://github.com/sarperavci/fleet
9
+ Requires-Python: >=3.11
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: fastapi>=0.115.0
13
+ Requires-Dist: uvicorn[standard]>=0.30.0
14
+ Requires-Dist: pydantic>=2.7.0
15
+ Requires-Dist: websockets>=12.0
16
+ Requires-Dist: httpx>=0.27.0
17
+ Requires-Dist: redis>=5.0.0
18
+ Requires-Dist: jinja2>=3.1.0
19
+ Requires-Dist: psutil>=5.9.0
20
+ Requires-Dist: click>=8.1.0
21
+ Provides-Extra: browser
22
+ Requires-Dist: DrissionPage>=4.1.0; extra == "browser"
23
+ Requires-Dist: cryptography>=42.0.0; extra == "browser"
24
+ Provides-Extra: cloudflare
25
+ Requires-Dist: fleet-framework[browser]; extra == "cloudflare"
26
+ Provides-Extra: otel
27
+ Requires-Dist: opentelemetry-api>=1.27.0; extra == "otel"
28
+ Requires-Dist: opentelemetry-sdk>=1.27.0; extra == "otel"
29
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27.0; extra == "otel"
30
+ Provides-Extra: test
31
+ Requires-Dist: pytest>=8.0; extra == "test"
32
+ Requires-Dist: pytest-asyncio>=0.23; extra == "test"
33
+ Provides-Extra: all
34
+ Requires-Dist: fleet-framework[browser,cloudflare,otel]; extra == "all"
35
+ Dynamic: license-file
36
+
37
+ # Fleet
38
+
39
+ A framework for distributed automation fleets. One master, many workers, typed output streams between them. Born out of a production CAPTCHA-solving farm, generalized to fit any "N machines each doing M parallel things" workload.
40
+
41
+ ## What it is
42
+
43
+ Two pip packages. `fleet-core` is the framework: master, worker shell, reconcile loop, Redis-backed store, output streams, event bus, a small dashboard. `fleet-browser` is an optional add-on that ships a battle-tested Chromium pool with fingerprint rotation, proxy auth, and orphan-process cleanup.
44
+
45
+ You write a Python class subclassing `ContinuousAutomation` or `BatchAutomation` and a Pydantic config. Ship it as a pip package with one entry-point line. Install it on the master and on each worker host and the framework wires up everything else: config push, validation, slot lifecycle, recycle, heal, stats, dashboard, stream-based inter-automation comms.
46
+
47
+ ## What's already in the box
48
+
49
+ - Master process: REST + WebSocket API, auto-discovered plugins, dashboard, Redis-backed state.
50
+ - Worker process: pulls config, runs N concurrent slots, reconciles to desired state, heals on failure.
51
+ - Reconcile loop: pure-functional diff, smallest-converging-plan semantics, slot recycle, self-heal.
52
+ - Typed output streams: ZSET-backed, bounded by length and TTL, one per automation type, pop or peek over REST or in-process.
53
+ - Event bus: Redis pubsub for lightweight signals.
54
+ - Auth: bearer tokens, admin vs worker, constant-time comparison. WebSocket handshake validated before accept.
55
+ - Hardening from the original farm: tini in the worker container reaps zombies, `psutil`-based process-tree cleanup catches orphans, recycle's heal step closes the "zero-slot lockout" failure mode.
56
+
57
+ ## What you write
58
+
59
+ Roughly 30 lines per automation. Here's a fully working example:
60
+
61
+ ```python
62
+ import asyncio
63
+ import httpx
64
+ from fleet.core import BaseConfig, ContinuousAutomation, register
65
+
66
+ class PingerConfig(BaseConfig):
67
+ url: str
68
+ interval_seconds: float = 5.0
69
+
70
+ @register("pinger")
71
+ class Pinger(ContinuousAutomation[PingerConfig]):
72
+ Config = PingerConfig
73
+
74
+ async def run_slot(self, ctx):
75
+ async with httpx.AsyncClient(proxy=ctx.proxy) as client:
76
+ while not ctx.shutdown.is_set():
77
+ r = await client.get(ctx.config.url)
78
+ await ctx.emit({"status": r.status_code})
79
+ await asyncio.sleep(ctx.config.interval_seconds)
80
+ ```
81
+
82
+ Plus one line in your `pyproject.toml`:
83
+
84
+ ```toml
85
+ [project.entry-points."fleet.automations"]
86
+ pinger = "my_package:Pinger"
87
+ ```
88
+
89
+ ## Quickstart
90
+
91
+ ```bash
92
+ git clone <repo-url> fleet && cd fleet
93
+ python3 -m venv .venv && source .venv/bin/activate
94
+ pip install -e packages/fleet-core
95
+ pip install -e examples/hello-world
96
+
97
+ # in one terminal:
98
+ redis-server --port 6379 --appendonly yes
99
+ # in another:
100
+ FLEET_ADMIN_TOKEN=admin FLEET_WORKER_TOKEN=worker fleet master
101
+ # in another:
102
+ MASTER_URL=http://localhost:8000 FLEET_WORKER_TOKEN=worker fleet worker --type hello-world --id w1
103
+ # in another:
104
+ curl -X PATCH http://localhost:8000/api/v1/automations/hello-world/workers/w1/config \
105
+ -H "Authorization: Bearer admin" -H "Content-Type: application/json" \
106
+ -d '{"enabled": true, "slots": 2}'
107
+ curl -X POST http://localhost:8000/api/v1/automations/hello-world/output/pop \
108
+ -H "Authorization: Bearer admin" -H "Content-Type: application/json" \
109
+ -d '{"n": 5}'
110
+ ```
111
+
112
+ Walkthrough with explanations: [docs/getting-started/quickstart.md](docs/getting-started/quickstart.md).
113
+
114
+ ## Repo layout
115
+
116
+ ```
117
+ fleet/
118
+ ├── packages/
119
+ │ ├── fleet-core/ master, worker, reconcile, store, streams, event bus, dashboard, CLI
120
+ │ └── fleet-browser/ optional Chromium pool, fingerprint, humanizer, proxy auth, cert
121
+ ├── examples/
122
+ │ ├── hello-world/ minimal ContinuousAutomation (emit greetings on a timer)
123
+ │ └── echo-consumer/ consumes another automation's output stream
124
+ └── docs/ GitBook-format documentation (also lives on the `docs` branch)
125
+ ```
126
+
127
+ ## Documentation
128
+
129
+ Full docs in [`docs/`](docs/), rendered as a GitBook from the `docs` branch.
130
+
131
+ - [Introduction](docs/README.md)
132
+ - [Installation](docs/getting-started/installation.md)
133
+ - [Quickstart](docs/getting-started/quickstart.md)
134
+ - [Your first automation](docs/getting-started/first-automation.md)
135
+ - [Architecture](docs/concepts/architecture.md)
136
+ - [Writing a continuous automation](docs/guides/continuous-automation.md)
137
+ - [Writing a batch automation](docs/guides/batch-automation.md)
138
+ - [Making two automations talk](docs/guides/inter-automation.md)
139
+ - [Browser-based automations](docs/guides/browser-automations.md)
140
+ - [Deployment](docs/guides/deployment.md)
141
+ - [REST API](docs/reference/rest-api.md)
142
+ - [WebSocket protocol](docs/reference/ws-protocol.md)
143
+ - [Authentication](docs/operations/auth.md)
144
+ - [Troubleshooting](docs/operations/troubleshooting.md)
145
+
146
+ ## Origins
147
+
148
+ Fleet started as the control plane for a Cloudflare Turnstile-solving farm. That farm hit every interesting failure mode in distributed automation: zombie subprocess accumulation, master state wipe on Redis restart, gen-counter regression, slot-recycle drops with no heal, and config drift across heterogeneous workers. Each of those has a fix baked into the framework.
149
+
150
+ The framework is what the original farm should have been from day one. It's deliberately small. If you want a scheduler, an orchestrator, or a service mesh, this isn't it. If you want a thin layer that turns "N hosts doing repetitive work" into a manageable system, it is.
151
+
152
+ ## Status
153
+
154
+ Pre-alpha. The public API will change. No PyPI release yet — install from source. Production deployments are running on it but with a lot of manual operational knowledge to back them up. Watch the changelog before relying on anything sensitive.
155
+
156
+ See [ROADMAP.md](ROADMAP.md) for the path to v0.1.
157
+
158
+ ## License
159
+
160
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,124 @@
1
+ # Fleet
2
+
3
+ A framework for distributed automation fleets. One master, many workers, typed output streams between them. Born out of a production CAPTCHA-solving farm, generalized to fit any "N machines each doing M parallel things" workload.
4
+
5
+ ## What it is
6
+
7
+ Two pip packages. `fleet-core` is the framework: master, worker shell, reconcile loop, Redis-backed store, output streams, event bus, a small dashboard. `fleet-browser` is an optional add-on that ships a battle-tested Chromium pool with fingerprint rotation, proxy auth, and orphan-process cleanup.
8
+
9
+ You write a Python class subclassing `ContinuousAutomation` or `BatchAutomation` and a Pydantic config. Ship it as a pip package with one entry-point line. Install it on the master and on each worker host and the framework wires up everything else: config push, validation, slot lifecycle, recycle, heal, stats, dashboard, stream-based inter-automation comms.
10
+
11
+ ## What's already in the box
12
+
13
+ - Master process: REST + WebSocket API, auto-discovered plugins, dashboard, Redis-backed state.
14
+ - Worker process: pulls config, runs N concurrent slots, reconciles to desired state, heals on failure.
15
+ - Reconcile loop: pure-functional diff, smallest-converging-plan semantics, slot recycle, self-heal.
16
+ - Typed output streams: ZSET-backed, bounded by length and TTL, one per automation type, pop or peek over REST or in-process.
17
+ - Event bus: Redis pubsub for lightweight signals.
18
+ - Auth: bearer tokens, admin vs worker, constant-time comparison. WebSocket handshake validated before accept.
19
+ - Hardening from the original farm: tini in the worker container reaps zombies, `psutil`-based process-tree cleanup catches orphans, recycle's heal step closes the "zero-slot lockout" failure mode.
20
+
21
+ ## What you write
22
+
23
+ Roughly 30 lines per automation. Here's a fully working example:
24
+
25
+ ```python
26
+ import asyncio
27
+ import httpx
28
+ from fleet.core import BaseConfig, ContinuousAutomation, register
29
+
30
+ class PingerConfig(BaseConfig):
31
+ url: str
32
+ interval_seconds: float = 5.0
33
+
34
+ @register("pinger")
35
+ class Pinger(ContinuousAutomation[PingerConfig]):
36
+ Config = PingerConfig
37
+
38
+ async def run_slot(self, ctx):
39
+ async with httpx.AsyncClient(proxy=ctx.proxy) as client:
40
+ while not ctx.shutdown.is_set():
41
+ r = await client.get(ctx.config.url)
42
+ await ctx.emit({"status": r.status_code})
43
+ await asyncio.sleep(ctx.config.interval_seconds)
44
+ ```
45
+
46
+ Plus one line in your `pyproject.toml`:
47
+
48
+ ```toml
49
+ [project.entry-points."fleet.automations"]
50
+ pinger = "my_package:Pinger"
51
+ ```
52
+
53
+ ## Quickstart
54
+
55
+ ```bash
56
+ git clone <repo-url> fleet && cd fleet
57
+ python3 -m venv .venv && source .venv/bin/activate
58
+ pip install -e packages/fleet-core
59
+ pip install -e examples/hello-world
60
+
61
+ # in one terminal:
62
+ redis-server --port 6379 --appendonly yes
63
+ # in another:
64
+ FLEET_ADMIN_TOKEN=admin FLEET_WORKER_TOKEN=worker fleet master
65
+ # in another:
66
+ MASTER_URL=http://localhost:8000 FLEET_WORKER_TOKEN=worker fleet worker --type hello-world --id w1
67
+ # in another:
68
+ curl -X PATCH http://localhost:8000/api/v1/automations/hello-world/workers/w1/config \
69
+ -H "Authorization: Bearer admin" -H "Content-Type: application/json" \
70
+ -d '{"enabled": true, "slots": 2}'
71
+ curl -X POST http://localhost:8000/api/v1/automations/hello-world/output/pop \
72
+ -H "Authorization: Bearer admin" -H "Content-Type: application/json" \
73
+ -d '{"n": 5}'
74
+ ```
75
+
76
+ Walkthrough with explanations: [docs/getting-started/quickstart.md](docs/getting-started/quickstart.md).
77
+
78
+ ## Repo layout
79
+
80
+ ```
81
+ fleet/
82
+ ├── packages/
83
+ │ ├── fleet-core/ master, worker, reconcile, store, streams, event bus, dashboard, CLI
84
+ │ └── fleet-browser/ optional Chromium pool, fingerprint, humanizer, proxy auth, cert
85
+ ├── examples/
86
+ │ ├── hello-world/ minimal ContinuousAutomation (emit greetings on a timer)
87
+ │ └── echo-consumer/ consumes another automation's output stream
88
+ └── docs/ GitBook-format documentation (also lives on the `docs` branch)
89
+ ```
90
+
91
+ ## Documentation
92
+
93
+ Full docs in [`docs/`](docs/), rendered as a GitBook from the `docs` branch.
94
+
95
+ - [Introduction](docs/README.md)
96
+ - [Installation](docs/getting-started/installation.md)
97
+ - [Quickstart](docs/getting-started/quickstart.md)
98
+ - [Your first automation](docs/getting-started/first-automation.md)
99
+ - [Architecture](docs/concepts/architecture.md)
100
+ - [Writing a continuous automation](docs/guides/continuous-automation.md)
101
+ - [Writing a batch automation](docs/guides/batch-automation.md)
102
+ - [Making two automations talk](docs/guides/inter-automation.md)
103
+ - [Browser-based automations](docs/guides/browser-automations.md)
104
+ - [Deployment](docs/guides/deployment.md)
105
+ - [REST API](docs/reference/rest-api.md)
106
+ - [WebSocket protocol](docs/reference/ws-protocol.md)
107
+ - [Authentication](docs/operations/auth.md)
108
+ - [Troubleshooting](docs/operations/troubleshooting.md)
109
+
110
+ ## Origins
111
+
112
+ Fleet started as the control plane for a Cloudflare Turnstile-solving farm. That farm hit every interesting failure mode in distributed automation: zombie subprocess accumulation, master state wipe on Redis restart, gen-counter regression, slot-recycle drops with no heal, and config drift across heterogeneous workers. Each of those has a fix baked into the framework.
113
+
114
+ The framework is what the original farm should have been from day one. It's deliberately small. If you want a scheduler, an orchestrator, or a service mesh, this isn't it. If you want a thin layer that turns "N hosts doing repetitive work" into a manageable system, it is.
115
+
116
+ ## Status
117
+
118
+ Pre-alpha. The public API will change. No PyPI release yet — install from source. Production deployments are running on it but with a lot of manual operational knowledge to back them up. Watch the changelog before relying on anything sensitive.
119
+
120
+ See [ROADMAP.md](ROADMAP.md) for the path to v0.1.
121
+
122
+ ## License
123
+
124
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1 @@
1
+ # namespace marker
@@ -0,0 +1,290 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import json
5
+ import os
6
+ import socket
7
+ import sys
8
+
9
+ import click
10
+ import uvicorn
11
+
12
+ from fleet.core.logging import configure_logging
13
+
14
+ configure_logging()
15
+
16
+
17
+ @click.group()
18
+ def cli() -> None:
19
+ # fleet — master + worker CLI.
20
+ pass
21
+
22
+
23
+ @cli.command()
24
+ @click.option("--host", default="0.0.0.0", show_default=True)
25
+ @click.option("--port", default=8000, show_default=True)
26
+ @click.option(
27
+ "--backend-url",
28
+ envvar="FLEET_BACKEND_URL",
29
+ default=None,
30
+ show_default=True,
31
+ help="Backend URL: redis://..., memory://, or any registered scheme. "
32
+ "Defaults to $FLEET_BACKEND_URL, then $REDIS_URL, then redis://localhost:6379/0.",
33
+ )
34
+ @click.option(
35
+ "--redis-url",
36
+ envvar="REDIS_URL",
37
+ default=None,
38
+ show_default=True,
39
+ help="(Legacy) Use --backend-url instead.",
40
+ )
41
+ def master(host: str, port: int, backend_url: str | None, redis_url: str | None) -> None:
42
+ """Start the master process. Binds REST + WS + dashboard."""
43
+ from fleet.master import create_app
44
+
45
+ url = backend_url or redis_url or "redis://localhost:6379/0"
46
+ if redis_url:
47
+ os.environ.setdefault("REDIS_URL", redis_url)
48
+ os.environ.setdefault("FLEET_BACKEND_URL", url)
49
+ app = create_app(url)
50
+ uvicorn.run(app, host=host, port=port, log_level=os.environ.get("LOG_LEVEL", "info").lower())
51
+
52
+
53
+ @cli.command()
54
+ @click.option("--type", "automation_type", required=True, help="automation type name (entry-point key)")
55
+ @click.option("--id", "worker_id", default=None, help="worker id (defaults to hostname)")
56
+ @click.option("--master-url", envvar="MASTER_URL", required=True)
57
+ @click.option("--worker-token", envvar="FLEET_WORKER_TOKEN", required=True)
58
+ @click.option(
59
+ "--backend-url",
60
+ envvar="FLEET_BACKEND_URL",
61
+ default=None,
62
+ help="Backend URL; falls back to --redis-url then redis://localhost:6379/0.",
63
+ )
64
+ @click.option(
65
+ "--redis-url",
66
+ envvar="REDIS_URL",
67
+ default=None,
68
+ help="(Legacy) Use --backend-url instead.",
69
+ )
70
+ def worker(
71
+ automation_type: str,
72
+ worker_id: str | None,
73
+ master_url: str,
74
+ worker_token: str,
75
+ backend_url: str | None,
76
+ redis_url: str | None,
77
+ ) -> None:
78
+ """Start a worker process pinned to one automation type."""
79
+ from fleet.worker import Agent
80
+
81
+ url = backend_url or redis_url or "redis://localhost:6379/0"
82
+ wid = worker_id or socket.gethostname()
83
+ agent = Agent(
84
+ automation_type=automation_type,
85
+ worker_id=wid,
86
+ master_url=master_url,
87
+ worker_token=worker_token,
88
+ redis_url=url,
89
+ )
90
+ try:
91
+ asyncio.run(agent.run())
92
+ except KeyboardInterrupt:
93
+ sys.exit(0)
94
+
95
+
96
+ def _list_plugins() -> None:
97
+ from fleet.core.automation import load_entry_points
98
+ from fleet.core.proxy import load_provider_entry_points
99
+
100
+ reg = load_entry_points()
101
+ if reg:
102
+ click.echo("automations:")
103
+ for name, cls in reg.items():
104
+ kind = cls.__mro__[1].__name__
105
+ click.echo(f" {name:30} {cls.__module__}.{cls.__name__} ({kind})")
106
+ else:
107
+ click.echo("automations: (none installed)")
108
+ providers = load_provider_entry_points()
109
+ if providers:
110
+ click.echo("proxy providers:")
111
+ for name in providers:
112
+ click.echo(f" {name}")
113
+
114
+
115
+ @cli.command()
116
+ def plugins() -> None:
117
+ """List installed automations + proxy providers discovered via entry-points."""
118
+ _list_plugins()
119
+
120
+
121
+ @cli.command(name="info")
122
+ def info_alias() -> None:
123
+ """Alias of `plugins`. Kept for backwards compatibility."""
124
+ _list_plugins()
125
+
126
+
127
+ @cli.command()
128
+ @click.argument("automation_type")
129
+ @click.option("--master-url", envvar="MASTER_URL", required=True)
130
+ @click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", required=True)
131
+ @click.option("--n", default=10, show_default=True, help="entries per pop")
132
+ @click.option("--interval", default=1.0, show_default=True, help="seconds between polls")
133
+ @click.option("--peek", is_flag=True, help="non-destructive read instead of pop")
134
+ def tail(
135
+ automation_type: str,
136
+ master_url: str,
137
+ admin_token: str,
138
+ n: int,
139
+ interval: float,
140
+ peek: bool,
141
+ ) -> None:
142
+ """Stream the output of an automation type to stdout, one JSON object per line."""
143
+ import httpx
144
+
145
+ headers = {"Authorization": f"Bearer {admin_token}"}
146
+ op = "peek" if peek else "pop"
147
+ base = master_url.rstrip("/")
148
+ try:
149
+ while True:
150
+ try:
151
+ if peek:
152
+ url = f"{base}/api/v1/automations/{automation_type}/output/peek?n={n}"
153
+ r = httpx.get(url, headers=headers, timeout=10)
154
+ else:
155
+ url = f"{base}/api/v1/automations/{automation_type}/output/pop"
156
+ r = httpx.post(url, headers=headers, json={"n": n}, timeout=10)
157
+ r.raise_for_status()
158
+ for env in r.json():
159
+ click.echo(json.dumps(env, default=str))
160
+ except Exception as e:
161
+ click.echo(f"# {op} failed: {e}", err=True)
162
+ import time as _t
163
+ _t.sleep(max(0.1, interval))
164
+ except KeyboardInterrupt:
165
+ sys.exit(0)
166
+
167
+
168
+ @cli.command()
169
+ @click.argument("automation_type")
170
+ @click.argument("payload") # JSON string or @path
171
+ @click.option("--master-url", envvar="MASTER_URL", required=True)
172
+ @click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", required=True)
173
+ @click.option("--task-id", default=None, help="override the framework's UUID")
174
+ @click.option("--max-attempts", default=None, type=int, help="override per-task retry budget")
175
+ def submit(
176
+ automation_type: str,
177
+ payload: str,
178
+ master_url: str,
179
+ admin_token: str,
180
+ task_id: str | None,
181
+ max_attempts: int | None,
182
+ ) -> None:
183
+ """Submit a single task to a BatchAutomation.
184
+
185
+ PAYLOAD is a JSON string or @path/to/file.json. Validated against the
186
+ automation's TaskPayload schema server-side.
187
+ """
188
+ import httpx
189
+
190
+ body_text = payload
191
+ if payload.startswith("@"):
192
+ body_text = open(payload[1:]).read()
193
+ try:
194
+ parsed = json.loads(body_text)
195
+ except json.JSONDecodeError as e:
196
+ click.echo(f"payload is not valid JSON: {e}", err=True)
197
+ sys.exit(2)
198
+ req: dict = {"payload": parsed}
199
+ if task_id:
200
+ req["task_id"] = task_id
201
+ if max_attempts is not None:
202
+ req["max_attempts"] = max_attempts
203
+ url = master_url.rstrip("/") + f"/api/v1/automations/{automation_type}/tasks"
204
+ r = httpx.post(url, headers={"Authorization": f"Bearer {admin_token}"}, json=req, timeout=10)
205
+ if r.status_code >= 400:
206
+ click.echo(f"submit failed ({r.status_code}): {r.text}", err=True)
207
+ sys.exit(1)
208
+ click.echo(json.dumps(r.json()))
209
+
210
+
211
+ @cli.command()
212
+ @click.option("--master-url", envvar="MASTER_URL", default=None)
213
+ @click.option("--admin-token", envvar="FLEET_ADMIN_TOKEN", default=None)
214
+ @click.option("--worker-token", envvar="FLEET_WORKER_TOKEN", default=None)
215
+ @click.option("--backend-url", envvar="FLEET_BACKEND_URL", default=None)
216
+ def doctor(
217
+ master_url: str | None,
218
+ admin_token: str | None,
219
+ worker_token: str | None,
220
+ backend_url: str | None,
221
+ ) -> None:
222
+ """Pre-flight sanity checks: env tokens, plugins, backend, master reachable."""
223
+ asyncio.run(_doctor(master_url, admin_token, worker_token, backend_url))
224
+
225
+
226
+ async def _doctor(master_url, admin_token, worker_token, backend_url) -> None:
227
+ import httpx
228
+
229
+ ok = True
230
+
231
+ def _check(label: str, condition: bool, detail: str = "") -> None:
232
+ nonlocal ok
233
+ mark = "✓" if condition else "✗"
234
+ ok = ok and condition
235
+ click.echo(f"{mark} {label}" + (f" — {detail}" if detail else ""))
236
+
237
+ _check("FLEET_ADMIN_TOKEN set", bool(admin_token))
238
+ _check("FLEET_WORKER_TOKEN set", bool(worker_token))
239
+
240
+ from fleet.core.automation import load_entry_points
241
+ reg = load_entry_points()
242
+ _check("automations discovered", bool(reg), f"{len(reg)} found")
243
+ for name in reg:
244
+ click.echo(f" {name}")
245
+
246
+ if backend_url:
247
+ try:
248
+ from fleet.core.backend import RedisBackend, backend_from_url
249
+ be = backend_from_url(backend_url)
250
+ _check("backend constructs", True, type(be).__name__)
251
+ if isinstance(be, RedisBackend):
252
+ try:
253
+ pong = await be.r.ping()
254
+ _check("redis ping", bool(pong))
255
+ info = await be.r.info("persistence")
256
+ aof = info.get(b"aof_enabled", info.get("aof_enabled"))
257
+ aof_on = str(aof).strip() in ("1", "True", "true")
258
+ _check(
259
+ "redis AOF on", aof_on,
260
+ "" if aof_on else "data lives only in RAM; configs/pools die on restart",
261
+ )
262
+ except Exception as e:
263
+ _check("redis reachable", False, str(e))
264
+ finally:
265
+ await be.aclose()
266
+ except Exception as e:
267
+ _check("backend constructs", False, str(e))
268
+ else:
269
+ click.echo("· backend-url not set (skipping backend checks)")
270
+
271
+ if master_url and admin_token:
272
+ try:
273
+ async with httpx.AsyncClient(timeout=3) as c:
274
+ r = await c.get(master_url.rstrip("/") + "/healthz")
275
+ _check("master /healthz reachable", r.status_code == 200, f"HTTP {r.status_code}")
276
+ r = await c.get(
277
+ master_url.rstrip("/") + "/api/v1/automations",
278
+ headers={"Authorization": f"Bearer {admin_token}"},
279
+ )
280
+ _check("admin token accepted", r.status_code == 200, f"HTTP {r.status_code}")
281
+ except Exception as e:
282
+ _check("master reachable", False, str(e))
283
+ else:
284
+ click.echo("· master-url or admin-token not set (skipping master checks)")
285
+
286
+ sys.exit(0 if ok else 1)
287
+
288
+
289
+ if __name__ == "__main__":
290
+ cli()
@@ -0,0 +1,69 @@
1
+ from fleet.core.automation import (
2
+ BaseAutomation,
3
+ BatchAutomation,
4
+ ContinuousAutomation,
5
+ Task,
6
+ catalog_doc,
7
+ get_registry,
8
+ load_entry_points,
9
+ register,
10
+ )
11
+ from fleet.core.backend import Backend, RedisBackend
12
+ from fleet.core.config import BaseConfig
13
+ from fleet.core.context import Context
14
+ from fleet.core.contract import Pool, Queue, Stream
15
+ from fleet.core.country_presets import known_presets, resolve_countries
16
+ from fleet.core.primitives import (
17
+ KV,
18
+ Counter,
19
+ Lock,
20
+ PoolFilter,
21
+ PoolHandle,
22
+ PoolItem,
23
+ QueueHandle,
24
+ StreamReader,
25
+ )
26
+ from fleet.core.proxy import (
27
+ ProxyHandle,
28
+ ProxyProvider,
29
+ ProxySession,
30
+ build_provider,
31
+ get_provider_registry,
32
+ load_provider_entry_points,
33
+ register_provider,
34
+ )
35
+
36
+ __all__ = [
37
+ "Backend",
38
+ "BaseAutomation",
39
+ "BaseConfig",
40
+ "BatchAutomation",
41
+ "ContinuousAutomation",
42
+ "Context",
43
+ "Counter",
44
+ "KV",
45
+ "Lock",
46
+ "Pool",
47
+ "PoolFilter",
48
+ "PoolHandle",
49
+ "PoolItem",
50
+ "ProxyHandle",
51
+ "ProxyProvider",
52
+ "ProxySession",
53
+ "Queue",
54
+ "QueueHandle",
55
+ "RedisBackend",
56
+ "Stream",
57
+ "StreamReader",
58
+ "Task",
59
+ "build_provider",
60
+ "catalog_doc",
61
+ "get_provider_registry",
62
+ "get_registry",
63
+ "known_presets",
64
+ "load_entry_points",
65
+ "load_provider_entry_points",
66
+ "register",
67
+ "register_provider",
68
+ "resolve_countries",
69
+ ]