avtomatika 1.0b7__py3-none-any.whl → 1.0b9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- avtomatika/api/handlers.py +3 -255
- avtomatika/api/routes.py +42 -63
- avtomatika/app_keys.py +2 -0
- avtomatika/config.py +18 -0
- avtomatika/constants.py +2 -26
- avtomatika/data_types.py +4 -23
- avtomatika/dispatcher.py +9 -26
- avtomatika/engine.py +127 -6
- avtomatika/executor.py +53 -25
- avtomatika/health_checker.py +23 -5
- avtomatika/history/base.py +60 -6
- avtomatika/history/noop.py +18 -7
- avtomatika/history/postgres.py +8 -6
- avtomatika/history/sqlite.py +7 -5
- avtomatika/metrics.py +1 -1
- avtomatika/reputation.py +46 -40
- avtomatika/s3.py +379 -0
- avtomatika/security.py +56 -74
- avtomatika/services/__init__.py +0 -0
- avtomatika/services/worker_service.py +266 -0
- avtomatika/storage/base.py +55 -4
- avtomatika/storage/memory.py +56 -7
- avtomatika/storage/redis.py +214 -251
- avtomatika/utils/webhook_sender.py +44 -2
- avtomatika/watcher.py +35 -35
- avtomatika/ws_manager.py +10 -9
- {avtomatika-1.0b7.dist-info → avtomatika-1.0b9.dist-info}/METADATA +81 -7
- avtomatika-1.0b9.dist-info/RECORD +48 -0
- {avtomatika-1.0b7.dist-info → avtomatika-1.0b9.dist-info}/WHEEL +1 -1
- avtomatika-1.0b7.dist-info/RECORD +0 -45
- {avtomatika-1.0b7.dist-info → avtomatika-1.0b9.dist-info}/licenses/LICENSE +0 -0
- {avtomatika-1.0b7.dist-info → avtomatika-1.0b9.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from asyncio import sleep
|
|
1
|
+
from asyncio import CancelledError, Queue, QueueFull, create_task, sleep
|
|
2
|
+
from contextlib import suppress
|
|
2
3
|
from dataclasses import asdict, dataclass
|
|
3
4
|
from logging import getLogger
|
|
4
5
|
from typing import Any
|
|
@@ -22,8 +23,49 @@ class WebhookSender:
|
|
|
22
23
|
self.session = session
|
|
23
24
|
self.timeout = ClientTimeout(total=10)
|
|
24
25
|
self.max_retries = 3
|
|
26
|
+
self._queue: Queue[tuple[str, WebhookPayload]] = Queue(maxsize=1000)
|
|
27
|
+
self._worker_task = None
|
|
25
28
|
|
|
26
|
-
|
|
29
|
+
def start(self) -> None:
|
|
30
|
+
if not self._worker_task:
|
|
31
|
+
self._worker_task = create_task(self._worker())
|
|
32
|
+
logger.info("WebhookSender background worker started.")
|
|
33
|
+
|
|
34
|
+
async def stop(self) -> None:
|
|
35
|
+
if self._worker_task:
|
|
36
|
+
self._worker_task.cancel()
|
|
37
|
+
with suppress(CancelledError):
|
|
38
|
+
await self._worker_task
|
|
39
|
+
self._worker_task = None
|
|
40
|
+
logger.info("WebhookSender background worker stopped.")
|
|
41
|
+
|
|
42
|
+
async def send(self, url: str, payload: WebhookPayload) -> None:
|
|
43
|
+
"""
|
|
44
|
+
Queues a webhook to be sent. Non-blocking.
|
|
45
|
+
Drops the message if the queue is full to prevent backpressure.
|
|
46
|
+
"""
|
|
47
|
+
try:
|
|
48
|
+
self._queue.put_nowait((url, payload))
|
|
49
|
+
except QueueFull:
|
|
50
|
+
logger.error(
|
|
51
|
+
f"Webhook queue is full! Dropping webhook for job {payload.job_id} to {url}. "
|
|
52
|
+
"Consider increasing queue size or checking external service latency."
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
async def _worker(self) -> None:
|
|
56
|
+
while True:
|
|
57
|
+
try:
|
|
58
|
+
url, payload = await self._queue.get()
|
|
59
|
+
try:
|
|
60
|
+
await self._send_single(url, payload)
|
|
61
|
+
except Exception as e:
|
|
62
|
+
logger.exception(f"Unexpected error in webhook worker: {e}")
|
|
63
|
+
finally:
|
|
64
|
+
self._queue.task_done()
|
|
65
|
+
except CancelledError:
|
|
66
|
+
break
|
|
67
|
+
|
|
68
|
+
async def _send_single(self, url: str, payload: WebhookPayload) -> bool:
|
|
27
69
|
"""
|
|
28
70
|
Sends a webhook payload to the specified URL with retries.
|
|
29
71
|
Returns True if successful, False otherwise.
|
avtomatika/watcher.py
CHANGED
|
@@ -3,6 +3,8 @@ from logging import getLogger
|
|
|
3
3
|
from typing import TYPE_CHECKING
|
|
4
4
|
from uuid import uuid4
|
|
5
5
|
|
|
6
|
+
from .constants import JOB_STATUS_FAILED, JOB_STATUS_WAITING_FOR_WORKER
|
|
7
|
+
|
|
6
8
|
if TYPE_CHECKING:
|
|
7
9
|
from .engine import OrchestratorEngine
|
|
8
10
|
|
|
@@ -26,46 +28,43 @@ class Watcher:
|
|
|
26
28
|
self._running = True
|
|
27
29
|
while self._running:
|
|
28
30
|
try:
|
|
29
|
-
await sleep(self.watch_interval_seconds)
|
|
30
|
-
|
|
31
31
|
# Attempt to acquire distributed lock
|
|
32
32
|
# We set TTL slightly longer than the expected execution time (60s)
|
|
33
|
-
if
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
if await self.storage.acquire_lock("global_watcher_lock", self._instance_id, 60):
|
|
34
|
+
try:
|
|
35
|
+
logger.debug("Watcher running check for timed out jobs...")
|
|
36
|
+
timed_out_job_ids = await self.storage.get_timed_out_jobs(limit=100)
|
|
36
37
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
for job_id in timed_out_job_ids:
|
|
39
|
+
logger.warning(f"Job {job_id} timed out. Moving to failed state.")
|
|
40
|
+
try:
|
|
41
|
+
# Get the latest version to avoid overwriting
|
|
42
|
+
job_state = await self.storage.get_job_state(job_id)
|
|
43
|
+
if job_state and job_state["status"] == JOB_STATUS_WAITING_FOR_WORKER:
|
|
44
|
+
job_state["status"] = JOB_STATUS_FAILED
|
|
45
|
+
job_state["error_message"] = "Worker task timed out."
|
|
46
|
+
await self.storage.save_job_state(job_id, job_state)
|
|
40
47
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
try:
|
|
44
|
-
# Get the latest version to avoid overwriting
|
|
45
|
-
job_state = await self.storage.get_job_state(job_id)
|
|
46
|
-
if job_state and job_state["status"] == "waiting_for_worker":
|
|
47
|
-
job_state["status"] = "failed"
|
|
48
|
-
job_state["error_message"] = "Worker task timed out."
|
|
49
|
-
await self.storage.save_job_state(job_id, job_state)
|
|
48
|
+
# Increment the metric
|
|
49
|
+
from . import metrics
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
51
|
+
metrics.jobs_failed_total.inc(
|
|
52
|
+
{
|
|
53
|
+
metrics.LABEL_BLUEPRINT: job_state.get(
|
|
54
|
+
"blueprint_name",
|
|
55
|
+
"unknown",
|
|
56
|
+
),
|
|
57
|
+
},
|
|
58
|
+
)
|
|
59
|
+
except Exception:
|
|
60
|
+
logger.exception(
|
|
61
|
+
f"Failed to update state for timed out job {job_id}",
|
|
61
62
|
)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
# Always release the lock so we (or others) can run next time
|
|
68
|
-
await self.storage.release_lock("global_watcher_lock", self._instance_id)
|
|
63
|
+
finally:
|
|
64
|
+
# Always release the lock so we (or others) can run next time
|
|
65
|
+
await self.storage.release_lock("global_watcher_lock", self._instance_id)
|
|
66
|
+
else:
|
|
67
|
+
logger.debug("Watcher lock held by another instance. Skipping check.")
|
|
69
68
|
|
|
70
69
|
except CancelledError:
|
|
71
70
|
logger.info("Watcher received cancellation request.")
|
|
@@ -73,7 +72,8 @@ class Watcher:
|
|
|
73
72
|
except Exception:
|
|
74
73
|
logger.exception("Error in Watcher main loop.")
|
|
75
74
|
|
|
76
|
-
|
|
75
|
+
# Sleep at the end of iteration
|
|
76
|
+
await sleep(self.watch_interval_seconds)
|
|
77
77
|
|
|
78
78
|
def stop(self):
|
|
79
79
|
"""Stops the watcher."""
|
avtomatika/ws_manager.py
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
from asyncio import Lock
|
|
2
2
|
from logging import getLogger
|
|
3
|
+
from typing import Any
|
|
3
4
|
|
|
4
5
|
from aiohttp import web
|
|
5
6
|
|
|
7
|
+
from .constants import MSG_TYPE_PROGRESS
|
|
8
|
+
|
|
6
9
|
logger = getLogger(__name__)
|
|
7
10
|
|
|
8
11
|
|
|
9
12
|
class WebSocketManager:
|
|
10
13
|
"""Manages active WebSocket connections from workers."""
|
|
11
14
|
|
|
12
|
-
def __init__(self):
|
|
15
|
+
def __init__(self) -> None:
|
|
13
16
|
self._connections: dict[str, web.WebSocketResponse] = {}
|
|
14
17
|
self._lock = Lock()
|
|
15
18
|
|
|
16
|
-
async def register(self, worker_id: str, ws: web.WebSocketResponse):
|
|
19
|
+
async def register(self, worker_id: str, ws: web.WebSocketResponse) -> None:
|
|
17
20
|
"""Registers a new WebSocket connection for a worker."""
|
|
18
21
|
async with self._lock:
|
|
19
22
|
if worker_id in self._connections:
|
|
@@ -22,14 +25,14 @@ class WebSocketManager:
|
|
|
22
25
|
self._connections[worker_id] = ws
|
|
23
26
|
logger.info(f"WebSocket connection registered for worker {worker_id}.")
|
|
24
27
|
|
|
25
|
-
async def unregister(self, worker_id: str):
|
|
28
|
+
async def unregister(self, worker_id: str) -> None:
|
|
26
29
|
"""Unregisters a WebSocket connection."""
|
|
27
30
|
async with self._lock:
|
|
28
31
|
if worker_id in self._connections:
|
|
29
32
|
del self._connections[worker_id]
|
|
30
33
|
logger.info(f"WebSocket connection for worker {worker_id} unregistered.")
|
|
31
34
|
|
|
32
|
-
async def send_command(self, worker_id: str, command: dict):
|
|
35
|
+
async def send_command(self, worker_id: str, command: dict[str, Any]) -> bool:
|
|
33
36
|
"""Sends a JSON command to a specific worker."""
|
|
34
37
|
async with self._lock:
|
|
35
38
|
connection = self._connections.get(worker_id)
|
|
@@ -46,12 +49,10 @@ class WebSocketManager:
|
|
|
46
49
|
return False
|
|
47
50
|
|
|
48
51
|
@staticmethod
|
|
49
|
-
async def handle_message(worker_id: str, message: dict):
|
|
52
|
+
async def handle_message(worker_id: str, message: dict[str, Any]) -> None:
|
|
50
53
|
"""Handles an incoming message from a worker."""
|
|
51
54
|
event_type = message.get("event")
|
|
52
|
-
if event_type ==
|
|
53
|
-
# In a real application, you'd likely forward this to a history store
|
|
54
|
-
# or a pub/sub system for real-time UI updates.
|
|
55
|
+
if event_type == MSG_TYPE_PROGRESS:
|
|
55
56
|
logger.info(
|
|
56
57
|
f"Received progress update from worker {worker_id} for job {message.get('job_id')}: "
|
|
57
58
|
f"{message.get('progress', 0) * 100:.0f}% - {message.get('message', '')}"
|
|
@@ -59,7 +60,7 @@ class WebSocketManager:
|
|
|
59
60
|
else:
|
|
60
61
|
logger.debug(f"Received unhandled event from worker {worker_id}: {event_type}")
|
|
61
62
|
|
|
62
|
-
async def close_all(self):
|
|
63
|
+
async def close_all(self) -> None:
|
|
63
64
|
"""Closes all active WebSocket connections."""
|
|
64
65
|
async with self._lock:
|
|
65
66
|
logger.info(f"Closing {len(self._connections)} active WebSocket connections...")
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: avtomatika
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.0b9
|
|
4
4
|
Summary: A state-machine based orchestrator for long-running AI and other jobs.
|
|
5
|
+
Author-email: Dmitrii Gagarin <madgagarin@gmail.com>
|
|
5
6
|
Project-URL: Homepage, https://github.com/avtomatika-ai/avtomatika
|
|
6
7
|
Project-URL: Bug Tracker, https://github.com/avtomatika-ai/avtomatika/issues
|
|
8
|
+
Keywords: orchestrator,state-machine,workflow,distributed,ai,llm,rxon,hln
|
|
7
9
|
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
8
11
|
Classifier: Programming Language :: Python :: 3
|
|
9
12
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
13
|
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Typing :: Typed
|
|
11
15
|
Requires-Python: >=3.11
|
|
12
16
|
Description-Content-Type: text/markdown
|
|
13
17
|
License-File: LICENSE
|
|
18
|
+
Requires-Dist: rxon
|
|
14
19
|
Requires-Dist: aiohttp~=3.12
|
|
15
20
|
Requires-Dist: python-json-logger~=4.0
|
|
16
21
|
Requires-Dist: graphviz~=0.21
|
|
@@ -20,6 +25,9 @@ Requires-Dist: msgpack~=1.1
|
|
|
20
25
|
Requires-Dist: orjson~=3.11
|
|
21
26
|
Provides-Extra: redis
|
|
22
27
|
Requires-Dist: redis~=7.1; extra == "redis"
|
|
28
|
+
Provides-Extra: s3
|
|
29
|
+
Requires-Dist: obstore>=0.2; extra == "s3"
|
|
30
|
+
Requires-Dist: aiofiles~=25.1; extra == "s3"
|
|
23
31
|
Provides-Extra: history
|
|
24
32
|
Requires-Dist: aiosqlite~=0.22; extra == "history"
|
|
25
33
|
Requires-Dist: asyncpg~=0.30; extra == "history"
|
|
@@ -37,14 +45,22 @@ Requires-Dist: pytest-mock~=3.14; extra == "test"
|
|
|
37
45
|
Requires-Dist: aioresponses~=0.7; extra == "test"
|
|
38
46
|
Requires-Dist: backports.zstd~=1.2; extra == "test"
|
|
39
47
|
Requires-Dist: opentelemetry-instrumentation-aiohttp-client; extra == "test"
|
|
48
|
+
Requires-Dist: obstore>=0.2; extra == "test"
|
|
49
|
+
Requires-Dist: aiofiles~=25.1; extra == "test"
|
|
40
50
|
Provides-Extra: all
|
|
41
51
|
Requires-Dist: avtomatika[redis]; extra == "all"
|
|
42
52
|
Requires-Dist: avtomatika[history]; extra == "all"
|
|
43
53
|
Requires-Dist: avtomatika[telemetry]; extra == "all"
|
|
54
|
+
Requires-Dist: avtomatika[s3]; extra == "all"
|
|
44
55
|
Dynamic: license-file
|
|
45
56
|
|
|
46
57
|
# Avtomatika Orchestrator
|
|
47
58
|
|
|
59
|
+
[](https://opensource.org/licenses/MIT)
|
|
60
|
+
[](https://www.python.org/downloads/release/python-3110/)
|
|
61
|
+
[](https://github.com/avtomatika-ai/avtomatika/actions/workflows/ci.yml)
|
|
62
|
+
[](https://github.com/astral-sh/ruff)
|
|
63
|
+
|
|
48
64
|
Avtomatika is a powerful, state-driven engine for managing complex asynchronous workflows in Python. It provides a robust framework for building scalable and resilient applications by separating process logic from execution logic.
|
|
49
65
|
|
|
50
66
|
This document serves as a comprehensive guide for developers looking to build pipelines (blueprints) and embed the Orchestrator into their applications.
|
|
@@ -60,6 +76,7 @@ This document serves as a comprehensive guide for developers looking to build pi
|
|
|
60
76
|
- [Parallel Execution and Aggregation (Fan-out/Fan-in)](#parallel-execution-and-aggregation-fan-outfan-in)
|
|
61
77
|
- [Dependency Injection (DataStore)](#dependency-injection-datastore)
|
|
62
78
|
- [Native Scheduler](#native-scheduler)
|
|
79
|
+
- [S3 Payload Offloading](#s3-payload-offloading)
|
|
63
80
|
- [Webhook Notifications](#webhook-notifications)
|
|
64
81
|
- [Production Configuration](#production-configuration)
|
|
65
82
|
- [Fault Tolerance](#fault-tolerance)
|
|
@@ -81,8 +98,9 @@ The project is based on a simple yet powerful architectural pattern that separat
|
|
|
81
98
|
|
|
82
99
|
Avtomatika is part of a larger ecosystem:
|
|
83
100
|
|
|
101
|
+
* **[Avtomatika Protocol](https://github.com/avtomatika-ai/rxon)**: Shared package containing protocol definitions, data models, and utilities ensuring consistency across all components.
|
|
84
102
|
* **[Avtomatika Worker SDK](https://github.com/avtomatika-ai/avtomatika-worker)**: The official Python SDK for building workers that connect to this engine.
|
|
85
|
-
* **[
|
|
103
|
+
* **[HLN Protocol](https://github.com/avtomatika-ai/hln)**: The architectural specification and manifesto behind the system (Hierarchical Logic Network).
|
|
86
104
|
* **[Full Example](https://github.com/avtomatika-ai/avtomatika-full-example)**: A complete reference project demonstrating the engine and workers in action.
|
|
87
105
|
|
|
88
106
|
## Installation
|
|
@@ -107,6 +125,11 @@ Avtomatika is part of a larger ecosystem:
|
|
|
107
125
|
pip install "avtomatika[telemetry]"
|
|
108
126
|
```
|
|
109
127
|
|
|
128
|
+
* **Install with S3 support (Payload Offloading):**
|
|
129
|
+
```bash
|
|
130
|
+
pip install "avtomatika[s3]"
|
|
131
|
+
```
|
|
132
|
+
|
|
110
133
|
* **Install all dependencies, including for testing:**
|
|
111
134
|
```bash
|
|
112
135
|
pip install "avtomatika[all,test]"
|
|
@@ -128,7 +151,7 @@ storage = MemoryStorage()
|
|
|
128
151
|
config = Config() # Loads configuration from environment variables
|
|
129
152
|
|
|
130
153
|
# Explicitly set tokens for this example
|
|
131
|
-
# Client token must be sent in the 'X-
|
|
154
|
+
# Client token must be sent in the 'X-Client-Token' header.
|
|
132
155
|
config.CLIENT_TOKEN = "my-secret-client-token"
|
|
133
156
|
# Worker token must be sent in the 'X-Worker-Token' header.
|
|
134
157
|
config.GLOBAL_WORKER_TOKEN = "my-secret-worker-token"
|
|
@@ -250,6 +273,25 @@ async def publish_handler_old_style(context):
|
|
|
250
273
|
print(f"Job {context.job_id}: Publishing video at {output_path} ({duration}s).")
|
|
251
274
|
context.actions.transition_to("complete")
|
|
252
275
|
```
|
|
276
|
+
## Key Concepts: JobContext and Actions
|
|
277
|
+
|
|
278
|
+
### High Performance Architecture
|
|
279
|
+
|
|
280
|
+
Avtomatika is engineered for high-load environments with thousands of concurrent workers.
|
|
281
|
+
|
|
282
|
+
* **O(1) Dispatcher**: Uses advanced Redis Set intersections to find suitable workers instantly.
|
|
283
|
+
* **Zero Trust Security**:
|
|
284
|
+
* **mTLS (Mutual TLS)**: Mutual authentication between Orchestrator and Workers using certificates.
|
|
285
|
+
* **STS (Security Token Service)**: Token rotation mechanism with short-lived access tokens.
|
|
286
|
+
* **Identity Extraction**: Automatically maps Certificate Common Name (CN) to Worker ID.
|
|
287
|
+
* **Data Integrity**:
|
|
288
|
+
* **End-to-End Validation**: Automatic verification of file size and ETag (hash) during S3 transfers.
|
|
289
|
+
* **Audit Trail**: File metadata is logged in history for full traceability.
|
|
290
|
+
* **Protocol Layer**: Built on top of `rxon`, a strict contract defining interactions, ensuring forward compatibility and allowing transport evolution (e.g., to gRPC).
|
|
291
|
+
* **Non-Blocking I/O**:
|
|
292
|
+
* **Webhooks**: Sent via a bounded background queue.
|
|
293
|
+
* **S3 Streaming**: Constant memory usage regardless of file size.
|
|
294
|
+
|
|
253
295
|
## Blueprint Cookbook: Key Features
|
|
254
296
|
|
|
255
297
|
### 1. Conditional Transitions (`.when()`)
|
|
@@ -365,7 +407,30 @@ daily_at = "02:00"
|
|
|
365
407
|
|
|
366
408
|
The orchestrator can send asynchronous notifications to an external system when a job completes, fails, or is quarantined. This eliminates the need for clients to constantly poll the API for status updates.
|
|
367
409
|
|
|
368
|
-
|
|
410
|
+
### 7. S3 Payload Offloading
|
|
411
|
+
|
|
412
|
+
Orchestrator provides first-class support for handling large files via S3-compatible storage, powered by the high-performance `obstore` library (Rust bindings).
|
|
413
|
+
|
|
414
|
+
* **Memory Safe (Streaming)**: Uses streaming for uploads and downloads, allowing processing of files larger than available RAM without OOM errors.
|
|
415
|
+
* **Managed Mode**: The Orchestrator manages file lifecycle (automatic cleanup of S3 objects and local temporary files on job completion).
|
|
416
|
+
* **Dependency Injection**: Use the `task_files` argument in your handlers to easily read/write data.
|
|
417
|
+
* **Directory Support**: Supports recursive download and upload of entire directories.
|
|
418
|
+
|
|
419
|
+
```python
|
|
420
|
+
@bp.handler_for("process_data")
|
|
421
|
+
async def process_data(task_files, actions):
|
|
422
|
+
# Streaming download of a large file
|
|
423
|
+
local_path = await task_files.download("large_dataset.csv")
|
|
424
|
+
|
|
425
|
+
# ... process data ...
|
|
426
|
+
|
|
427
|
+
# Upload results
|
|
428
|
+
await task_files.write_json("results.json", {"status": "done"})
|
|
429
|
+
|
|
430
|
+
actions.transition_to("finished")
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
## Production Configuration
|
|
369
434
|
* **Events:**
|
|
370
435
|
* `job_finished`: The job reached a final success state.
|
|
371
436
|
* `job_failed`: The job failed (e.g., due to an error or invalid input).
|
|
@@ -477,13 +542,18 @@ By default, the engine uses in-memory storage. For production, you must configur
|
|
|
477
542
|
|
|
478
543
|
The orchestrator uses tokens to authenticate API requests.
|
|
479
544
|
|
|
480
|
-
* **Client Authentication**: All API clients must provide a token in the `X-
|
|
545
|
+
* **Client Authentication**: All API clients must provide a token in the `X-Client-Token` header. The orchestrator validates this token against client configurations.
|
|
481
546
|
* **Worker Authentication**: Workers must provide a token in the `X-Worker-Token` header.
|
|
482
547
|
* `GLOBAL_WORKER_TOKEN`: You can set a global token for all workers using this environment variable. For development and testing, it defaults to `"secure-worker-token"`.
|
|
483
548
|
* **Individual Tokens**: For production, it is recommended to define individual tokens for each worker in a separate configuration file and provide its path via the `WORKERS_CONFIG_PATH` environment variable. Tokens from this file are stored in a hashed format for security.
|
|
484
549
|
|
|
485
550
|
> **Note on Dynamic Reloading:** The worker configuration file can be reloaded without restarting the orchestrator by sending an authenticated `POST` request to the `/api/v1/admin/reload-workers` endpoint. This allows for dynamic updates of worker tokens.
|
|
486
551
|
|
|
552
|
+
### Pure Holon Mode
|
|
553
|
+
For high-security environments or when operating as a Compound Holon within an HLN, you can disable the public client API.
|
|
554
|
+
* **Enable/Disable**: Set `ENABLE_CLIENT_API="false"` (default: `true`).
|
|
555
|
+
* **Effect**: The Orchestrator will stop listening on `/api/v1/jobs/...`. It will only accept tasks via the Worker Protocol (RXON) from its parent.
|
|
556
|
+
|
|
487
557
|
### Observability
|
|
488
558
|
|
|
489
559
|
When installed with the telemetry dependency, the system automatically provides:
|
|
@@ -495,7 +565,11 @@ When installed with the telemetry dependency, the system automatically provides:
|
|
|
495
565
|
### Setup Environment
|
|
496
566
|
|
|
497
567
|
* Clone the repository.
|
|
498
|
-
*
|
|
568
|
+
* **For local development**, install the protocol package first:
|
|
569
|
+
```bash
|
|
570
|
+
pip install -e ../rxon
|
|
571
|
+
```
|
|
572
|
+
* Then install the engine in editable mode with all dependencies:
|
|
499
573
|
```bash
|
|
500
574
|
pip install -e ".[all,test]"
|
|
501
575
|
```
|
|
@@ -513,7 +587,7 @@ When installed with the telemetry dependency, the system automatically provides:
|
|
|
513
587
|
|
|
514
588
|
To run the `avtomatika` test suite:
|
|
515
589
|
```bash
|
|
516
|
-
pytest
|
|
590
|
+
pytest tests/
|
|
517
591
|
```
|
|
518
592
|
|
|
519
593
|
### Interactive API Documentation
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
avtomatika/__init__.py,sha256=D5r3L-H06uxsY_wgfh7u9YR29QvZMer1BlvzjW9Umfo,701
|
|
2
|
+
avtomatika/api.html,sha256=RLx-D1uFCSAXIf_2WgFlSTWrWPcmonNYM-9oNanKXBg,32835
|
|
3
|
+
avtomatika/app_keys.py,sha256=Zd2TaGPduzyEFJgdPvgSH1skdBx2mX-Prj1ma9fAXRo,1275
|
|
4
|
+
avtomatika/blueprint.py,sha256=AJ8hZkzvqkdu9aPmlJeFzuSj57L6Xm41KS9kBMGB4Vg,11827
|
|
5
|
+
avtomatika/client_config_loader.py,sha256=zVVHZlxSqZUaNpZ4zoU0T1CFYXdxy-3vKSmPcaFuHSY,2772
|
|
6
|
+
avtomatika/compression.py,sha256=bhA1kw4YrCR3I3kdquZSY0fAzCrRrjtz55uepzLUDKI,2498
|
|
7
|
+
avtomatika/config.py,sha256=27ov8BNbiUpkZ1sjtx3pifRavwcxJ_zUgIdkL_pgqv8,3595
|
|
8
|
+
avtomatika/constants.py,sha256=ohZ34TdM07fD0kJLgoKGkReWSCgBWQ_yOitJpwxtDTk,169
|
|
9
|
+
avtomatika/context.py,sha256=T6Ux4Fb1DwWRGTpMNeukM51MQDQbGk2HS6Cwpc0dc1s,4248
|
|
10
|
+
avtomatika/data_types.py,sha256=D_IUzMW8zMz-_MaqVp9MG53rG37Cb3McyRZuIXxvdlE,1108
|
|
11
|
+
avtomatika/datastore.py,sha256=gJjhZ5kxjF8pmbbPQb_qu3HPUpfy2c6T75KZ-smb_zg,545
|
|
12
|
+
avtomatika/dispatcher.py,sha256=eWuGTBxUpWddpdJkij-UZn0I7BJgqCNg_5Y33ohG2NM,8666
|
|
13
|
+
avtomatika/engine.py,sha256=Nhm-Pni-riPGyPM4SAig6PzyvPHe-VtYD1p4qcE0h4U,20487
|
|
14
|
+
avtomatika/executor.py,sha256=X5AU7hWflH8rSYKxl_wh2RhdYhpyktynmK8mcfJgT-8,24218
|
|
15
|
+
avtomatika/health_checker.py,sha256=jXYSH4BPeZ4LCxSZV4uXM4BZhGJYgpoAOWQXE8yojLo,2078
|
|
16
|
+
avtomatika/logging_config.py,sha256=Zb6f9Nri9WVWhlpuBg6Lpi5SWRLGIUmS8Dc3xD1Gg0g,2993
|
|
17
|
+
avtomatika/metrics.py,sha256=tiksK1fFSOMlz8zFu6GT19JTduvxMTNlLu0QFrTHoQI,1866
|
|
18
|
+
avtomatika/py.typed,sha256=CT_L7gw2MLcQY-X0vs-xB5Vr0wzvGo7GuQYPI_qwJE8,65
|
|
19
|
+
avtomatika/quota.py,sha256=DNcaL6k0J1REeP8sVqbY9FprY_3BSr2SxM2Vf4mEqdw,1612
|
|
20
|
+
avtomatika/ratelimit.py,sha256=hFGW5oN9G6_W_jnHmopXW8bRjjzlvanY19MLghsNLE8,1306
|
|
21
|
+
avtomatika/reputation.py,sha256=pK-x9FrPN2Oc2gtPa1AZJHlhvkd7xlRe4orxM2auJJc,3979
|
|
22
|
+
avtomatika/s3.py,sha256=I0fDw5I44RJAqSv4tREvwHp2cxB0mGY_l2cVZWpe3As,14110
|
|
23
|
+
avtomatika/scheduler.py,sha256=F5Kv5Rx34nDd0mE5jxjwpjRg8duDZBEr91N5Y6CNR24,4231
|
|
24
|
+
avtomatika/scheduler_config_loader.py,sha256=F6mLM8yPRgG4bMHV_WnXX7UOrXD8fCXJT30bbEuQ2mk,1311
|
|
25
|
+
avtomatika/security.py,sha256=eENEUc0OsHm6wN2H-ckGmiaV9qrZSbYsHFCWyYb3aLs,3271
|
|
26
|
+
avtomatika/telemetry.py,sha256=ZBt1_xJ36PzDSz-zdCXeNp58NiezUgbqvMctTG25PT0,2352
|
|
27
|
+
avtomatika/watcher.py,sha256=IKBqJ_r52ya0wiH8Gb0qFRMC8DFsusdRzPHjruWvFh4,3558
|
|
28
|
+
avtomatika/worker_config_loader.py,sha256=n0j8gfuJDacWONr8744RsHTCWpc_1ZTRMC-rJZh6P6A,2249
|
|
29
|
+
avtomatika/ws_manager.py,sha256=Qw0buPba5wCJKLP6jZ3QDyYPyG-EEEfL36ytzLqX0LA,3048
|
|
30
|
+
avtomatika/api/handlers.py,sha256=1t4qG-Mw6yf4UMViFkLMwO5Dfuk3VVxq1is77wBkzAc,11686
|
|
31
|
+
avtomatika/api/routes.py,sha256=MrtcRNjybxODmKhab0FCzgZGPRcfznwpFtDCdgh8RT4,3937
|
|
32
|
+
avtomatika/history/base.py,sha256=RsCvCkHK1teHjXSk9ZHVEtpQlIjz8kWsfKYHVnapf6c,3848
|
|
33
|
+
avtomatika/history/noop.py,sha256=hLzt0RblsrKUtoyQNauOni6jCi-IYCWEPsiR0vh7tho,1226
|
|
34
|
+
avtomatika/history/postgres.py,sha256=T0XpDurnh48pPI-2JhB285GdNIexNkCSu8ExhLJzcxc,9538
|
|
35
|
+
avtomatika/history/sqlite.py,sha256=txWax9RVzBQzIZuU-SjHnEXEzBmGzIjqzoVsK2oyiAQ,9252
|
|
36
|
+
avtomatika/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
37
|
+
avtomatika/services/worker_service.py,sha256=Xei_5DPqWO8wl_X7koFoAMnkxRgRs9ADC-H4-fVGteI,11406
|
|
38
|
+
avtomatika/storage/__init__.py,sha256=mGRj_40dWZ7R7uYbqC6gCsUWCKHAbZz4ZVIhYg5dT_E,262
|
|
39
|
+
avtomatika/storage/base.py,sha256=Tb_4fF0Vr10cgoXepA-1YUSgi27qYKQ7Qz1Y87XiRII,13375
|
|
40
|
+
avtomatika/storage/memory.py,sha256=amXIur3FU1H37s2Xs1IqIXxqPlrERxVH8obXnu3Ojvo,14535
|
|
41
|
+
avtomatika/storage/redis.py,sha256=nqqE4-r3tTLvQkObnDONONVX1TUxdy6nyq8RAt-qIT4,20138
|
|
42
|
+
avtomatika/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
|
+
avtomatika/utils/webhook_sender.py,sha256=4Ud2Tz18yV4kzfGQyDO2by8s2EIEwHkv1vtIUis9kVQ,3606
|
|
44
|
+
avtomatika-1.0b9.dist-info/licenses/LICENSE,sha256=tqCjw9Y1vbU-hLcWi__7wQstLbt2T1XWPdbQYqCxuWY,1072
|
|
45
|
+
avtomatika-1.0b9.dist-info/METADATA,sha256=_yNyRuwGvnUKM7BrVKTxvA9RbSWy6s0_jowc1tF9ykY,28169
|
|
46
|
+
avtomatika-1.0b9.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
47
|
+
avtomatika-1.0b9.dist-info/top_level.txt,sha256=gLDWhA_wxHj0I6fG5X8vw9fE0HSN4hTE2dEJzeVS2x8,11
|
|
48
|
+
avtomatika-1.0b9.dist-info/RECORD,,
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
avtomatika/__init__.py,sha256=D5r3L-H06uxsY_wgfh7u9YR29QvZMer1BlvzjW9Umfo,701
|
|
2
|
-
avtomatika/api.html,sha256=RLx-D1uFCSAXIf_2WgFlSTWrWPcmonNYM-9oNanKXBg,32835
|
|
3
|
-
avtomatika/app_keys.py,sha256=hwg5IBYUsCe-tCru61z3a4-lTw98JL41SXWCmIM_YHc,1161
|
|
4
|
-
avtomatika/blueprint.py,sha256=AJ8hZkzvqkdu9aPmlJeFzuSj57L6Xm41KS9kBMGB4Vg,11827
|
|
5
|
-
avtomatika/client_config_loader.py,sha256=zVVHZlxSqZUaNpZ4zoU0T1CFYXdxy-3vKSmPcaFuHSY,2772
|
|
6
|
-
avtomatika/compression.py,sha256=bhA1kw4YrCR3I3kdquZSY0fAzCrRrjtz55uepzLUDKI,2498
|
|
7
|
-
avtomatika/config.py,sha256=Tc-vpaQS11i_JTa1pQjGuQD3R5Kj9sIf6gGJGjItBBo,2487
|
|
8
|
-
avtomatika/constants.py,sha256=WL58Nh-EY6baM9Ur_tR9merwPRGb41_klUG2V-yUUaA,963
|
|
9
|
-
avtomatika/context.py,sha256=T6Ux4Fb1DwWRGTpMNeukM51MQDQbGk2HS6Cwpc0dc1s,4248
|
|
10
|
-
avtomatika/data_types.py,sha256=odbkraTYhOo1j3ETIX8NJOy8yIwTdQ_i3juATsjroas,1424
|
|
11
|
-
avtomatika/datastore.py,sha256=gJjhZ5kxjF8pmbbPQb_qu3HPUpfy2c6T75KZ-smb_zg,545
|
|
12
|
-
avtomatika/dispatcher.py,sha256=RrUPvDnCEwAdqxrGpUUcvQZOxG8Ir1E3m9yJjkQeEig,9635
|
|
13
|
-
avtomatika/engine.py,sha256=LZBN4Z50cEITHcrzoYqPS-Rjw4WdPSKRuq-iaBipEbE,15475
|
|
14
|
-
avtomatika/executor.py,sha256=iR9NqS9IffoW692Cq5PKM0P5lmIikmQ65vx8b-D9IZQ,23098
|
|
15
|
-
avtomatika/health_checker.py,sha256=WXwvRJ-3cZC2Udc_ogsyIQp7VzcvJjq_IaqzkTdE0TE,1265
|
|
16
|
-
avtomatika/logging_config.py,sha256=Zb6f9Nri9WVWhlpuBg6Lpi5SWRLGIUmS8Dc3xD1Gg0g,2993
|
|
17
|
-
avtomatika/metrics.py,sha256=7XDhr_xMJ9JpElpZmBG7R0ml7AMdAp9UYp_W-i7tyLg,1858
|
|
18
|
-
avtomatika/py.typed,sha256=CT_L7gw2MLcQY-X0vs-xB5Vr0wzvGo7GuQYPI_qwJE8,65
|
|
19
|
-
avtomatika/quota.py,sha256=DNcaL6k0J1REeP8sVqbY9FprY_3BSr2SxM2Vf4mEqdw,1612
|
|
20
|
-
avtomatika/ratelimit.py,sha256=hFGW5oN9G6_W_jnHmopXW8bRjjzlvanY19MLghsNLE8,1306
|
|
21
|
-
avtomatika/reputation.py,sha256=IHcaIAILWZftPPmXj5En28OSDNK7U8ivQ-w30zIF8fk,3748
|
|
22
|
-
avtomatika/scheduler.py,sha256=F5Kv5Rx34nDd0mE5jxjwpjRg8duDZBEr91N5Y6CNR24,4231
|
|
23
|
-
avtomatika/scheduler_config_loader.py,sha256=F6mLM8yPRgG4bMHV_WnXX7UOrXD8fCXJT30bbEuQ2mk,1311
|
|
24
|
-
avtomatika/security.py,sha256=kkU68YmLWq1ClMUdEW98pS9WsEwHinHoZcdMoPm63Uk,4417
|
|
25
|
-
avtomatika/telemetry.py,sha256=ZBt1_xJ36PzDSz-zdCXeNp58NiezUgbqvMctTG25PT0,2352
|
|
26
|
-
avtomatika/watcher.py,sha256=WAYTjhVmXyqWZfWfQm-iFDVZFBYpNRfwgGDxa_LCnAI,3354
|
|
27
|
-
avtomatika/worker_config_loader.py,sha256=n0j8gfuJDacWONr8744RsHTCWpc_1ZTRMC-rJZh6P6A,2249
|
|
28
|
-
avtomatika/ws_manager.py,sha256=pi5xe0ivsCjRZw08ri5N-gAChMH2I2YPLpl3E2tP89k,3057
|
|
29
|
-
avtomatika/api/handlers.py,sha256=MO6QkbT001jj4latUXGT0hGOmQf_6TkRkmwx19OcXeQ,22176
|
|
30
|
-
avtomatika/api/routes.py,sha256=vSwj2jJlmftZrjrctt-mNYLF23CfCtlUfaMoZzNOqCk,4895
|
|
31
|
-
avtomatika/history/base.py,sha256=Gfw0Gb4Mt9wQrMlYLugZwey_6-cDej5OUctiMTCWg7Q,1668
|
|
32
|
-
avtomatika/history/noop.py,sha256=ETVtPiTfkaMpzhGD8c0_4Iu6pWD89dnPrrRrSIjmc8s,970
|
|
33
|
-
avtomatika/history/postgres.py,sha256=vtW4LMW7Vli5MjcGYY3ez667-C8Cq3I7kIHrcEgSYps,9409
|
|
34
|
-
avtomatika/history/sqlite.py,sha256=Blc9ckvzoDaMRStXyfJOzMAdU_t2JcwtQtVdPgnr6s0,9131
|
|
35
|
-
avtomatika/storage/__init__.py,sha256=mGRj_40dWZ7R7uYbqC6gCsUWCKHAbZz4ZVIhYg5dT_E,262
|
|
36
|
-
avtomatika/storage/base.py,sha256=hW7XFhe6CQDP69q5NPSkUzEInIFxDR1-AyRPZNPEDEc,11424
|
|
37
|
-
avtomatika/storage/memory.py,sha256=BYox_3v1qRqfJyyjkjxXj5OmyGBu0EZqz5BWKZy6YsU,12561
|
|
38
|
-
avtomatika/storage/redis.py,sha256=opOhqBL_uCsNXcMD_W_tJU-8wzDUSjBJWEsXrwP2_YM,21035
|
|
39
|
-
avtomatika/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
40
|
-
avtomatika/utils/webhook_sender.py,sha256=sbX6HEED_TvH3MBxFlVrAqP-n68y0q5quYty5b8Z4D8,1941
|
|
41
|
-
avtomatika-1.0b7.dist-info/licenses/LICENSE,sha256=tqCjw9Y1vbU-hLcWi__7wQstLbt2T1XWPdbQYqCxuWY,1072
|
|
42
|
-
avtomatika-1.0b7.dist-info/METADATA,sha256=FVUEaG5_NWobxU5_FDG2l5RmdhvVN5t1quZXQ0GsSpw,24215
|
|
43
|
-
avtomatika-1.0b7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
44
|
-
avtomatika-1.0b7.dist-info/top_level.txt,sha256=gLDWhA_wxHj0I6fG5X8vw9fE0HSN4hTE2dEJzeVS2x8,11
|
|
45
|
-
avtomatika-1.0b7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|