avtomatika 1.0b7__tar.gz → 1.0b9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. {avtomatika-1.0b7 → avtomatika-1.0b9}/PKG-INFO +81 -7
  2. avtomatika-1.0b7/src/avtomatika.egg-info/PKG-INFO → avtomatika-1.0b9/README.md +70 -52
  3. {avtomatika-1.0b7 → avtomatika-1.0b9}/pyproject.toml +13 -2
  4. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/api/handlers.py +3 -255
  5. avtomatika-1.0b9/src/avtomatika/api/routes.py +97 -0
  6. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/app_keys.py +2 -0
  7. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/config.py +18 -0
  8. avtomatika-1.0b9/src/avtomatika/constants.py +6 -0
  9. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/data_types.py +4 -23
  10. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/dispatcher.py +9 -26
  11. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/engine.py +127 -6
  12. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/executor.py +53 -25
  13. avtomatika-1.0b9/src/avtomatika/health_checker.py +57 -0
  14. avtomatika-1.0b9/src/avtomatika/history/base.py +105 -0
  15. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/history/noop.py +18 -7
  16. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/history/postgres.py +8 -6
  17. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/history/sqlite.py +7 -5
  18. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/metrics.py +1 -1
  19. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/reputation.py +46 -40
  20. avtomatika-1.0b9/src/avtomatika/s3.py +379 -0
  21. avtomatika-1.0b9/src/avtomatika/security.py +96 -0
  22. avtomatika-1.0b9/src/avtomatika/services/worker_service.py +266 -0
  23. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/storage/base.py +55 -4
  24. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/storage/memory.py +56 -7
  25. avtomatika-1.0b9/src/avtomatika/storage/redis.py +473 -0
  26. avtomatika-1.0b9/src/avtomatika/utils/__init__.py +0 -0
  27. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/utils/webhook_sender.py +44 -2
  28. avtomatika-1.0b9/src/avtomatika/watcher.py +80 -0
  29. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/ws_manager.py +10 -9
  30. avtomatika-1.0b7/README.md → avtomatika-1.0b9/src/avtomatika.egg-info/PKG-INFO +126 -7
  31. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika.egg-info/SOURCES.txt +11 -0
  32. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika.egg-info/requires.txt +8 -0
  33. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_dispatcher.py +31 -44
  34. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_dispatcher_extended.py +7 -3
  35. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_engine.py +41 -5
  36. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_error_handling.py +34 -18
  37. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_executor.py +3 -0
  38. avtomatika-1.0b9/tests/test_handlers.py +236 -0
  39. avtomatika-1.0b9/tests/test_handlers_sts.py +87 -0
  40. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_history.py +11 -6
  41. avtomatika-1.0b9/tests/test_horizontal_scaling.py +179 -0
  42. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_integration.py +18 -17
  43. avtomatika-1.0b9/tests/test_mtls.py +194 -0
  44. avtomatika-1.0b9/tests/test_postgres_history.py +84 -0
  45. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_reputation.py +2 -2
  46. avtomatika-1.0b9/tests/test_rxon_handler.py +59 -0
  47. avtomatika-1.0b9/tests/test_s3.py +276 -0
  48. avtomatika-1.0b9/tests/test_s3_metadata.py +138 -0
  49. avtomatika-1.0b9/tests/test_sts.py +188 -0
  50. avtomatika-1.0b9/tests/test_validation_integration.py +48 -0
  51. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_webhook_sender.py +23 -3
  52. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_ws_manager.py +19 -16
  53. avtomatika-1.0b7/src/avtomatika/api/routes.py +0 -118
  54. avtomatika-1.0b7/src/avtomatika/constants.py +0 -30
  55. avtomatika-1.0b7/src/avtomatika/health_checker.py +0 -39
  56. avtomatika-1.0b7/src/avtomatika/history/base.py +0 -51
  57. avtomatika-1.0b7/src/avtomatika/security.py +0 -114
  58. avtomatika-1.0b7/src/avtomatika/storage/redis.py +0 -510
  59. avtomatika-1.0b7/src/avtomatika/watcher.py +0 -80
  60. avtomatika-1.0b7/tests/test_handlers.py +0 -451
  61. avtomatika-1.0b7/tests/test_postgres_history.py +0 -107
  62. {avtomatika-1.0b7 → avtomatika-1.0b9}/LICENSE +0 -0
  63. {avtomatika-1.0b7 → avtomatika-1.0b9}/setup.cfg +0 -0
  64. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/__init__.py +0 -0
  65. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/api.html +0 -0
  66. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/blueprint.py +0 -0
  67. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/client_config_loader.py +0 -0
  68. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/compression.py +0 -0
  69. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/context.py +0 -0
  70. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/datastore.py +0 -0
  71. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/logging_config.py +0 -0
  72. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/py.typed +0 -0
  73. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/quota.py +0 -0
  74. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/ratelimit.py +0 -0
  75. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/scheduler.py +0 -0
  76. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/scheduler_config_loader.py +0 -0
  77. {avtomatika-1.0b7/src/avtomatika/utils → avtomatika-1.0b9/src/avtomatika/services}/__init__.py +0 -0
  78. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/storage/__init__.py +0 -0
  79. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/telemetry.py +0 -0
  80. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika/worker_config_loader.py +0 -0
  81. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika.egg-info/dependency_links.txt +0 -0
  82. {avtomatika-1.0b7 → avtomatika-1.0b9}/src/avtomatika.egg-info/top_level.txt +0 -0
  83. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_blueprint_conditions.py +0 -0
  84. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_blueprint_integrity.py +0 -0
  85. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_blueprints.py +0 -0
  86. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_client_config_loader.py +0 -0
  87. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_compression.py +0 -0
  88. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_config_validation.py +0 -0
  89. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_context.py +0 -0
  90. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_health_checker.py +0 -0
  91. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_logging_config.py +0 -0
  92. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_memory_locking.py +0 -0
  93. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_memory_storage.py +0 -0
  94. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_metrics.py +0 -0
  95. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_noop_history.py +0 -0
  96. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_optimization.py +0 -0
  97. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_ratelimit.py +0 -0
  98. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_redis_locking.py +0 -0
  99. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_redis_storage.py +0 -0
  100. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_scheduler.py +0 -0
  101. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_telemetry.py +0 -0
  102. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_watcher.py +0 -0
  103. {avtomatika-1.0b7 → avtomatika-1.0b9}/tests/test_worker_config_loader.py +0 -0
@@ -1,16 +1,21 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: avtomatika
3
- Version: 1.0b7
3
+ Version: 1.0b9
4
4
  Summary: A state-machine based orchestrator for long-running AI and other jobs.
5
+ Author-email: Dmitrii Gagarin <madgagarin@gmail.com>
5
6
  Project-URL: Homepage, https://github.com/avtomatika-ai/avtomatika
6
7
  Project-URL: Bug Tracker, https://github.com/avtomatika-ai/avtomatika/issues
8
+ Keywords: orchestrator,state-machine,workflow,distributed,ai,llm,rxon,hln
7
9
  Classifier: Development Status :: 4 - Beta
10
+ Classifier: Intended Audience :: Developers
8
11
  Classifier: Programming Language :: Python :: 3
9
12
  Classifier: License :: OSI Approved :: MIT License
10
13
  Classifier: Operating System :: OS Independent
14
+ Classifier: Typing :: Typed
11
15
  Requires-Python: >=3.11
12
16
  Description-Content-Type: text/markdown
13
17
  License-File: LICENSE
18
+ Requires-Dist: rxon
14
19
  Requires-Dist: aiohttp~=3.12
15
20
  Requires-Dist: python-json-logger~=4.0
16
21
  Requires-Dist: graphviz~=0.21
@@ -20,6 +25,9 @@ Requires-Dist: msgpack~=1.1
20
25
  Requires-Dist: orjson~=3.11
21
26
  Provides-Extra: redis
22
27
  Requires-Dist: redis~=7.1; extra == "redis"
28
+ Provides-Extra: s3
29
+ Requires-Dist: obstore>=0.2; extra == "s3"
30
+ Requires-Dist: aiofiles~=25.1; extra == "s3"
23
31
  Provides-Extra: history
24
32
  Requires-Dist: aiosqlite~=0.22; extra == "history"
25
33
  Requires-Dist: asyncpg~=0.30; extra == "history"
@@ -37,14 +45,22 @@ Requires-Dist: pytest-mock~=3.14; extra == "test"
37
45
  Requires-Dist: aioresponses~=0.7; extra == "test"
38
46
  Requires-Dist: backports.zstd~=1.2; extra == "test"
39
47
  Requires-Dist: opentelemetry-instrumentation-aiohttp-client; extra == "test"
48
+ Requires-Dist: obstore>=0.2; extra == "test"
49
+ Requires-Dist: aiofiles~=25.1; extra == "test"
40
50
  Provides-Extra: all
41
51
  Requires-Dist: avtomatika[redis]; extra == "all"
42
52
  Requires-Dist: avtomatika[history]; extra == "all"
43
53
  Requires-Dist: avtomatika[telemetry]; extra == "all"
54
+ Requires-Dist: avtomatika[s3]; extra == "all"
44
55
  Dynamic: license-file
45
56
 
46
57
  # Avtomatika Orchestrator
47
58
 
59
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
60
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/release/python-3110/)
61
+ [![Tests](https://github.com/avtomatika-ai/avtomatika/actions/workflows/ci.yml/badge.svg)](https://github.com/avtomatika-ai/avtomatika/actions/workflows/ci.yml)
62
+ [![Code Style: Ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
63
+
48
64
  Avtomatika is a powerful, state-driven engine for managing complex asynchronous workflows in Python. It provides a robust framework for building scalable and resilient applications by separating process logic from execution logic.
49
65
 
50
66
  This document serves as a comprehensive guide for developers looking to build pipelines (blueprints) and embed the Orchestrator into their applications.
@@ -60,6 +76,7 @@ This document serves as a comprehensive guide for developers looking to build pi
60
76
  - [Parallel Execution and Aggregation (Fan-out/Fan-in)](#parallel-execution-and-aggregation-fan-outfan-in)
61
77
  - [Dependency Injection (DataStore)](#dependency-injection-datastore)
62
78
  - [Native Scheduler](#native-scheduler)
79
+ - [S3 Payload Offloading](#s3-payload-offloading)
63
80
  - [Webhook Notifications](#webhook-notifications)
64
81
  - [Production Configuration](#production-configuration)
65
82
  - [Fault Tolerance](#fault-tolerance)
@@ -81,8 +98,9 @@ The project is based on a simple yet powerful architectural pattern that separat
81
98
 
82
99
  Avtomatika is part of a larger ecosystem:
83
100
 
101
+ * **[Avtomatika Protocol](https://github.com/avtomatika-ai/rxon)**: Shared package containing protocol definitions, data models, and utilities ensuring consistency across all components.
84
102
  * **[Avtomatika Worker SDK](https://github.com/avtomatika-ai/avtomatika-worker)**: The official Python SDK for building workers that connect to this engine.
85
- * **[RCA Protocol](https://github.com/avtomatika-ai/rca)**: The architectural specification and manifesto behind the system.
103
+ * **[HLN Protocol](https://github.com/avtomatika-ai/hln)**: The architectural specification and manifesto behind the system (Hierarchical Logic Network).
86
104
  * **[Full Example](https://github.com/avtomatika-ai/avtomatika-full-example)**: A complete reference project demonstrating the engine and workers in action.
87
105
 
88
106
  ## Installation
@@ -107,6 +125,11 @@ Avtomatika is part of a larger ecosystem:
107
125
  pip install "avtomatika[telemetry]"
108
126
  ```
109
127
 
128
+ * **Install with S3 support (Payload Offloading):**
129
+ ```bash
130
+ pip install "avtomatika[s3]"
131
+ ```
132
+
110
133
  * **Install all dependencies, including for testing:**
111
134
  ```bash
112
135
  pip install "avtomatika[all,test]"
@@ -128,7 +151,7 @@ storage = MemoryStorage()
128
151
  config = Config() # Loads configuration from environment variables
129
152
 
130
153
  # Explicitly set tokens for this example
131
- # Client token must be sent in the 'X-Avtomatika-Token' header.
154
+ # Client token must be sent in the 'X-Client-Token' header.
132
155
  config.CLIENT_TOKEN = "my-secret-client-token"
133
156
  # Worker token must be sent in the 'X-Worker-Token' header.
134
157
  config.GLOBAL_WORKER_TOKEN = "my-secret-worker-token"
@@ -250,6 +273,25 @@ async def publish_handler_old_style(context):
250
273
  print(f"Job {context.job_id}: Publishing video at {output_path} ({duration}s).")
251
274
  context.actions.transition_to("complete")
252
275
  ```
276
+ ## Key Concepts: JobContext and Actions
277
+
278
+ ### High Performance Architecture
279
+
280
+ Avtomatika is engineered for high-load environments with thousands of concurrent workers.
281
+
282
+ * **O(1) Dispatcher**: Uses advanced Redis Set intersections to find suitable workers instantly.
283
+ * **Zero Trust Security**:
284
+ * **mTLS (Mutual TLS)**: Mutual authentication between Orchestrator and Workers using certificates.
285
+ * **STS (Security Token Service)**: Token rotation mechanism with short-lived access tokens.
286
+ * **Identity Extraction**: Automatically maps Certificate Common Name (CN) to Worker ID.
287
+ * **Data Integrity**:
288
+ * **End-to-End Validation**: Automatic verification of file size and ETag (hash) during S3 transfers.
289
+ * **Audit Trail**: File metadata is logged in history for full traceability.
290
+ * **Protocol Layer**: Built on top of `rxon`, a strict contract defining interactions, ensuring forward compatibility and allowing transport evolution (e.g., to gRPC).
291
+ * **Non-Blocking I/O**:
292
+ * **Webhooks**: Sent via a bounded background queue.
293
+ * **S3 Streaming**: Constant memory usage regardless of file size.
294
+
253
295
  ## Blueprint Cookbook: Key Features
254
296
 
255
297
  ### 1. Conditional Transitions (`.when()`)
@@ -365,7 +407,30 @@ daily_at = "02:00"
365
407
 
366
408
  The orchestrator can send asynchronous notifications to an external system when a job completes, fails, or is quarantined. This eliminates the need for clients to constantly poll the API for status updates.
367
409
 
368
- * **Usage:** Pass a `webhook_url` in the request body when creating a job.
410
+ ### 7. S3 Payload Offloading
411
+
412
+ Orchestrator provides first-class support for handling large files via S3-compatible storage, powered by the high-performance `obstore` library (Rust bindings).
413
+
414
+ * **Memory Safe (Streaming)**: Uses streaming for uploads and downloads, allowing processing of files larger than available RAM without OOM errors.
415
+ * **Managed Mode**: The Orchestrator manages file lifecycle (automatic cleanup of S3 objects and local temporary files on job completion).
416
+ * **Dependency Injection**: Use the `task_files` argument in your handlers to easily read/write data.
417
+ * **Directory Support**: Supports recursive download and upload of entire directories.
418
+
419
+ ```python
420
+ @bp.handler_for("process_data")
421
+ async def process_data(task_files, actions):
422
+ # Streaming download of a large file
423
+ local_path = await task_files.download("large_dataset.csv")
424
+
425
+ # ... process data ...
426
+
427
+ # Upload results
428
+ await task_files.write_json("results.json", {"status": "done"})
429
+
430
+ actions.transition_to("finished")
431
+ ```
432
+
433
+ ## Production Configuration
369
434
  * **Events:**
370
435
  * `job_finished`: The job reached a final success state.
371
436
  * `job_failed`: The job failed (e.g., due to an error or invalid input).
@@ -477,13 +542,18 @@ By default, the engine uses in-memory storage. For production, you must configur
477
542
 
478
543
  The orchestrator uses tokens to authenticate API requests.
479
544
 
480
- * **Client Authentication**: All API clients must provide a token in the `X-Avtomatika-Token` header. The orchestrator validates this token against client configurations.
545
+ * **Client Authentication**: All API clients must provide a token in the `X-Client-Token` header. The orchestrator validates this token against client configurations.
481
546
  * **Worker Authentication**: Workers must provide a token in the `X-Worker-Token` header.
482
547
  * `GLOBAL_WORKER_TOKEN`: You can set a global token for all workers using this environment variable. For development and testing, it defaults to `"secure-worker-token"`.
483
548
  * **Individual Tokens**: For production, it is recommended to define individual tokens for each worker in a separate configuration file and provide its path via the `WORKERS_CONFIG_PATH` environment variable. Tokens from this file are stored in a hashed format for security.
484
549
 
485
550
  > **Note on Dynamic Reloading:** The worker configuration file can be reloaded without restarting the orchestrator by sending an authenticated `POST` request to the `/api/v1/admin/reload-workers` endpoint. This allows for dynamic updates of worker tokens.
486
551
 
552
+ ### Pure Holon Mode
553
+ For high-security environments or when operating as a Compound Holon within an HLN, you can disable the public client API.
554
+ * **Enable/Disable**: Set `ENABLE_CLIENT_API="false"` (default: `true`).
555
+ * **Effect**: The Orchestrator will stop listening on `/api/v1/jobs/...`. It will only accept tasks via the Worker Protocol (RXON) from its parent.
556
+
487
557
  ### Observability
488
558
 
489
559
  When installed with the telemetry dependency, the system automatically provides:
@@ -495,7 +565,11 @@ When installed with the telemetry dependency, the system automatically provides:
495
565
  ### Setup Environment
496
566
 
497
567
  * Clone the repository.
498
- * Install the package in editable mode with all dependencies:
568
+ * **For local development**, install the protocol package first:
569
+ ```bash
570
+ pip install -e ../rxon
571
+ ```
572
+ * Then install the engine in editable mode with all dependencies:
499
573
  ```bash
500
574
  pip install -e ".[all,test]"
501
575
  ```
@@ -513,7 +587,7 @@ When installed with the telemetry dependency, the system automatically provides:
513
587
 
514
588
  To run the `avtomatika` test suite:
515
589
  ```bash
516
- pytest avtomatika/tests/
590
+ pytest tests/
517
591
  ```
518
592
 
519
593
  ### Interactive API Documentation
@@ -1,50 +1,10 @@
1
- Metadata-Version: 2.4
2
- Name: avtomatika
3
- Version: 1.0b7
4
- Summary: A state-machine based orchestrator for long-running AI and other jobs.
5
- Project-URL: Homepage, https://github.com/avtomatika-ai/avtomatika
6
- Project-URL: Bug Tracker, https://github.com/avtomatika-ai/avtomatika/issues
7
- Classifier: Development Status :: 4 - Beta
8
- Classifier: Programming Language :: Python :: 3
9
- Classifier: License :: OSI Approved :: MIT License
10
- Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.11
12
- Description-Content-Type: text/markdown
13
- License-File: LICENSE
14
- Requires-Dist: aiohttp~=3.12
15
- Requires-Dist: python-json-logger~=4.0
16
- Requires-Dist: graphviz~=0.21
17
- Requires-Dist: zstandard~=0.24
18
- Requires-Dist: aioprometheus~=23.12
19
- Requires-Dist: msgpack~=1.1
20
- Requires-Dist: orjson~=3.11
21
- Provides-Extra: redis
22
- Requires-Dist: redis~=7.1; extra == "redis"
23
- Provides-Extra: history
24
- Requires-Dist: aiosqlite~=0.22; extra == "history"
25
- Requires-Dist: asyncpg~=0.30; extra == "history"
26
- Provides-Extra: telemetry
27
- Requires-Dist: opentelemetry-api~=1.39; extra == "telemetry"
28
- Requires-Dist: opentelemetry-sdk~=1.39; extra == "telemetry"
29
- Requires-Dist: opentelemetry-exporter-otlp~=1.39; extra == "telemetry"
30
- Requires-Dist: opentelemetry-instrumentation-aiohttp-client~=0.59b0; extra == "telemetry"
31
- Provides-Extra: test
32
- Requires-Dist: pytest~=9.0; extra == "test"
33
- Requires-Dist: pytest-asyncio~=1.1; extra == "test"
34
- Requires-Dist: fakeredis~=2.33; extra == "test"
35
- Requires-Dist: pytest-aiohttp~=1.1; extra == "test"
36
- Requires-Dist: pytest-mock~=3.14; extra == "test"
37
- Requires-Dist: aioresponses~=0.7; extra == "test"
38
- Requires-Dist: backports.zstd~=1.2; extra == "test"
39
- Requires-Dist: opentelemetry-instrumentation-aiohttp-client; extra == "test"
40
- Provides-Extra: all
41
- Requires-Dist: avtomatika[redis]; extra == "all"
42
- Requires-Dist: avtomatika[history]; extra == "all"
43
- Requires-Dist: avtomatika[telemetry]; extra == "all"
44
- Dynamic: license-file
45
-
46
1
  # Avtomatika Orchestrator
47
2
 
3
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
4
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/release/python-3110/)
5
+ [![Tests](https://github.com/avtomatika-ai/avtomatika/actions/workflows/ci.yml/badge.svg)](https://github.com/avtomatika-ai/avtomatika/actions/workflows/ci.yml)
6
+ [![Code Style: Ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
7
+
48
8
  Avtomatika is a powerful, state-driven engine for managing complex asynchronous workflows in Python. It provides a robust framework for building scalable and resilient applications by separating process logic from execution logic.
49
9
 
50
10
  This document serves as a comprehensive guide for developers looking to build pipelines (blueprints) and embed the Orchestrator into their applications.
@@ -60,6 +20,7 @@ This document serves as a comprehensive guide for developers looking to build pi
60
20
  - [Parallel Execution and Aggregation (Fan-out/Fan-in)](#parallel-execution-and-aggregation-fan-outfan-in)
61
21
  - [Dependency Injection (DataStore)](#dependency-injection-datastore)
62
22
  - [Native Scheduler](#native-scheduler)
23
+ - [S3 Payload Offloading](#s3-payload-offloading)
63
24
  - [Webhook Notifications](#webhook-notifications)
64
25
  - [Production Configuration](#production-configuration)
65
26
  - [Fault Tolerance](#fault-tolerance)
@@ -81,8 +42,9 @@ The project is based on a simple yet powerful architectural pattern that separat
81
42
 
82
43
  Avtomatika is part of a larger ecosystem:
83
44
 
45
+ * **[Avtomatika Protocol](https://github.com/avtomatika-ai/rxon)**: Shared package containing protocol definitions, data models, and utilities ensuring consistency across all components.
84
46
  * **[Avtomatika Worker SDK](https://github.com/avtomatika-ai/avtomatika-worker)**: The official Python SDK for building workers that connect to this engine.
85
- * **[RCA Protocol](https://github.com/avtomatika-ai/rca)**: The architectural specification and manifesto behind the system.
47
+ * **[HLN Protocol](https://github.com/avtomatika-ai/hln)**: The architectural specification and manifesto behind the system (Hierarchical Logic Network).
86
48
  * **[Full Example](https://github.com/avtomatika-ai/avtomatika-full-example)**: A complete reference project demonstrating the engine and workers in action.
87
49
 
88
50
  ## Installation
@@ -107,6 +69,11 @@ Avtomatika is part of a larger ecosystem:
107
69
  pip install "avtomatika[telemetry]"
108
70
  ```
109
71
 
72
+ * **Install with S3 support (Payload Offloading):**
73
+ ```bash
74
+ pip install "avtomatika[s3]"
75
+ ```
76
+
110
77
  * **Install all dependencies, including for testing:**
111
78
  ```bash
112
79
  pip install "avtomatika[all,test]"
@@ -128,7 +95,7 @@ storage = MemoryStorage()
128
95
  config = Config() # Loads configuration from environment variables
129
96
 
130
97
  # Explicitly set tokens for this example
131
- # Client token must be sent in the 'X-Avtomatika-Token' header.
98
+ # Client token must be sent in the 'X-Client-Token' header.
132
99
  config.CLIENT_TOKEN = "my-secret-client-token"
133
100
  # Worker token must be sent in the 'X-Worker-Token' header.
134
101
  config.GLOBAL_WORKER_TOKEN = "my-secret-worker-token"
@@ -250,6 +217,25 @@ async def publish_handler_old_style(context):
250
217
  print(f"Job {context.job_id}: Publishing video at {output_path} ({duration}s).")
251
218
  context.actions.transition_to("complete")
252
219
  ```
220
+ ## Key Concepts: JobContext and Actions
221
+
222
+ ### High Performance Architecture
223
+
224
+ Avtomatika is engineered for high-load environments with thousands of concurrent workers.
225
+
226
+ * **O(1) Dispatcher**: Uses advanced Redis Set intersections to find suitable workers instantly.
227
+ * **Zero Trust Security**:
228
+ * **mTLS (Mutual TLS)**: Mutual authentication between Orchestrator and Workers using certificates.
229
+ * **STS (Security Token Service)**: Token rotation mechanism with short-lived access tokens.
230
+ * **Identity Extraction**: Automatically maps Certificate Common Name (CN) to Worker ID.
231
+ * **Data Integrity**:
232
+ * **End-to-End Validation**: Automatic verification of file size and ETag (hash) during S3 transfers.
233
+ * **Audit Trail**: File metadata is logged in history for full traceability.
234
+ * **Protocol Layer**: Built on top of `rxon`, a strict contract defining interactions, ensuring forward compatibility and allowing transport evolution (e.g., to gRPC).
235
+ * **Non-Blocking I/O**:
236
+ * **Webhooks**: Sent via a bounded background queue.
237
+ * **S3 Streaming**: Constant memory usage regardless of file size.
238
+
253
239
  ## Blueprint Cookbook: Key Features
254
240
 
255
241
  ### 1. Conditional Transitions (`.when()`)
@@ -365,7 +351,30 @@ daily_at = "02:00"
365
351
 
366
352
  The orchestrator can send asynchronous notifications to an external system when a job completes, fails, or is quarantined. This eliminates the need for clients to constantly poll the API for status updates.
367
353
 
368
- * **Usage:** Pass a `webhook_url` in the request body when creating a job.
354
+ ### 7. S3 Payload Offloading
355
+
356
+ Orchestrator provides first-class support for handling large files via S3-compatible storage, powered by the high-performance `obstore` library (Rust bindings).
357
+
358
+ * **Memory Safe (Streaming)**: Uses streaming for uploads and downloads, allowing processing of files larger than available RAM without OOM errors.
359
+ * **Managed Mode**: The Orchestrator manages file lifecycle (automatic cleanup of S3 objects and local temporary files on job completion).
360
+ * **Dependency Injection**: Use the `task_files` argument in your handlers to easily read/write data.
361
+ * **Directory Support**: Supports recursive download and upload of entire directories.
362
+
363
+ ```python
364
+ @bp.handler_for("process_data")
365
+ async def process_data(task_files, actions):
366
+ # Streaming download of a large file
367
+ local_path = await task_files.download("large_dataset.csv")
368
+
369
+ # ... process data ...
370
+
371
+ # Upload results
372
+ await task_files.write_json("results.json", {"status": "done"})
373
+
374
+ actions.transition_to("finished")
375
+ ```
376
+
377
+ ## Production Configuration
369
378
  * **Events:**
370
379
  * `job_finished`: The job reached a final success state.
371
380
  * `job_failed`: The job failed (e.g., due to an error or invalid input).
@@ -477,13 +486,18 @@ By default, the engine uses in-memory storage. For production, you must configur
477
486
 
478
487
  The orchestrator uses tokens to authenticate API requests.
479
488
 
480
- * **Client Authentication**: All API clients must provide a token in the `X-Avtomatika-Token` header. The orchestrator validates this token against client configurations.
489
+ * **Client Authentication**: All API clients must provide a token in the `X-Client-Token` header. The orchestrator validates this token against client configurations.
481
490
  * **Worker Authentication**: Workers must provide a token in the `X-Worker-Token` header.
482
491
  * `GLOBAL_WORKER_TOKEN`: You can set a global token for all workers using this environment variable. For development and testing, it defaults to `"secure-worker-token"`.
483
492
  * **Individual Tokens**: For production, it is recommended to define individual tokens for each worker in a separate configuration file and provide its path via the `WORKERS_CONFIG_PATH` environment variable. Tokens from this file are stored in a hashed format for security.
484
493
 
485
494
  > **Note on Dynamic Reloading:** The worker configuration file can be reloaded without restarting the orchestrator by sending an authenticated `POST` request to the `/api/v1/admin/reload-workers` endpoint. This allows for dynamic updates of worker tokens.
486
495
 
496
+ ### Pure Holon Mode
497
+ For high-security environments or when operating as a Compound Holon within an HLN, you can disable the public client API.
498
+ * **Enable/Disable**: Set `ENABLE_CLIENT_API="false"` (default: `true`).
499
+ * **Effect**: The Orchestrator will stop listening on `/api/v1/jobs/...`. It will only accept tasks via the Worker Protocol (RXON) from its parent.
500
+
487
501
  ### Observability
488
502
 
489
503
  When installed with the telemetry dependency, the system automatically provides:
@@ -495,7 +509,11 @@ When installed with the telemetry dependency, the system automatically provides:
495
509
  ### Setup Environment
496
510
 
497
511
  * Clone the repository.
498
- * Install the package in editable mode with all dependencies:
512
+ * **For local development**, install the protocol package first:
513
+ ```bash
514
+ pip install -e ../rxon
515
+ ```
516
+ * Then install the engine in editable mode with all dependencies:
499
517
  ```bash
500
518
  pip install -e ".[all,test]"
501
519
  ```
@@ -513,7 +531,7 @@ When installed with the telemetry dependency, the system automatically provides:
513
531
 
514
532
  To run the `avtomatika` test suite:
515
533
  ```bash
516
- pytest avtomatika/tests/
534
+ pytest tests/
517
535
  ```
518
536
 
519
537
  ### Interactive API Documentation
@@ -533,4 +551,4 @@ For a deeper dive into the system, please refer to the following documents:
533
551
  - [**Architecture Guide**](https://github.com/avtomatika-ai/avtomatika/blob/main/docs/architecture.md): A detailed overview of the system components and their interactions.
534
552
  - [**API Reference**](https://github.com/avtomatika-ai/avtomatika/blob/main/docs/api_reference.md): Full specification of the HTTP API.
535
553
  - [**Deployment Guide**](https://github.com/avtomatika-ai/avtomatika/blob/main/docs/deployment.md): Instructions for deploying with Gunicorn/Uvicorn and NGINX.
536
- - [**Cookbook**](https://github.com/avtomatika-ai/avtomatika/blob/main/docs/cookbook/README.md): Examples and best practices for creating blueprints.
554
+ - [**Cookbook**](https://github.com/avtomatika-ai/avtomatika/blob/main/docs/cookbook/README.md): Examples and best practices for creating blueprints.
@@ -4,17 +4,24 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "avtomatika"
7
- version = "1.0b7"
7
+ version = "1.0b9"
8
8
  description = "A state-machine based orchestrator for long-running AI and other jobs."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
11
+ authors = [
12
+ {name = "Dmitrii Gagarin", email = "madgagarin@gmail.com"},
13
+ ]
14
+ keywords = ["orchestrator", "state-machine", "workflow", "distributed", "ai", "llm", "rxon", "hln"]
11
15
  classifiers = [
12
16
  "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Developers",
13
18
  "Programming Language :: Python :: 3",
14
19
  "License :: OSI Approved :: MIT License",
15
20
  "Operating System :: OS Independent",
21
+ "Typing :: Typed",
16
22
  ]
17
23
  dependencies = [
24
+ "rxon",
18
25
  "aiohttp~=3.12",
19
26
  "python-json-logger~=4.0",
20
27
  "graphviz~=0.21",
@@ -26,6 +33,7 @@ dependencies = [
26
33
 
27
34
  [project.optional-dependencies]
28
35
  redis = ["redis~=7.1"]
36
+ s3 = ["obstore>=0.2", "aiofiles~=25.1"]
29
37
  history = ["aiosqlite~=0.22", "asyncpg~=0.30"]
30
38
  telemetry = [
31
39
  "opentelemetry-api~=1.39",
@@ -42,11 +50,14 @@ test = [
42
50
  "aioresponses~=0.7",
43
51
  "backports.zstd~=1.2",
44
52
  "opentelemetry-instrumentation-aiohttp-client",
53
+ "obstore>=0.2",
54
+ "aiofiles~=25.1",
45
55
  ]
46
56
  all = [
47
57
  "avtomatika[redis]",
48
58
  "avtomatika[history]",
49
59
  "avtomatika[telemetry]",
60
+ "avtomatika[s3]",
50
61
  ]
51
62
 
52
63
  [project.urls]
@@ -54,7 +65,7 @@ all = [
54
65
  "Bug Tracker" = "https://github.com/avtomatika-ai/avtomatika/issues"
55
66
 
56
67
  [tool.setuptools.package-data]
57
- "avtomatika" = ["api.html"]
68
+ "avtomatika" = ["api.html", "py.typed"]
58
69
 
59
70
  [tool.setuptools.packages.find]
60
71
  where = ["src"]