avtomatika 1.0b6__tar.gz → 1.0b8__tar.gz

{avtomatika-1.0b6/src/avtomatika.egg-info → avtomatika-1.0b8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avtomatika
-Version: 1.0b6
+Version: 1.0b8
 Summary: A state-machine based orchestrator for long-running AI and other jobs.
 Project-URL: Homepage, https://github.com/avtomatika-ai/avtomatika
 Project-URL: Bug Tracker, https://github.com/avtomatika-ai/avtomatika/issues
@@ -12,7 +12,6 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp~=3.12
-Requires-Dist: aiocache~=0.12
 Requires-Dist: python-json-logger~=4.0
 Requires-Dist: graphviz~=0.21
 Requires-Dist: zstandard~=0.24
@@ -21,6 +20,9 @@ Requires-Dist: msgpack~=1.1
 Requires-Dist: orjson~=3.11
 Provides-Extra: redis
 Requires-Dist: redis~=7.1; extra == "redis"
+Provides-Extra: s3
+Requires-Dist: obstore>=0.2; extra == "s3"
+Requires-Dist: aiofiles~=23.2; extra == "s3"
 Provides-Extra: history
 Requires-Dist: aiosqlite~=0.22; extra == "history"
 Requires-Dist: asyncpg~=0.30; extra == "history"
@@ -38,10 +40,13 @@ Requires-Dist: pytest-mock~=3.14; extra == "test"
 Requires-Dist: aioresponses~=0.7; extra == "test"
 Requires-Dist: backports.zstd~=1.2; extra == "test"
 Requires-Dist: opentelemetry-instrumentation-aiohttp-client; extra == "test"
+Requires-Dist: obstore>=0.2; extra == "test"
+Requires-Dist: aiofiles~=23.2; extra == "test"
 Provides-Extra: all
 Requires-Dist: avtomatika[redis]; extra == "all"
 Requires-Dist: avtomatika[history]; extra == "all"
 Requires-Dist: avtomatika[telemetry]; extra == "all"
+Requires-Dist: avtomatika[s3]; extra == "all"
 Dynamic: license-file
 
 # Avtomatika Orchestrator
@@ -61,6 +66,8 @@ This document serves as a comprehensive guide for developers looking to build pi
   - [Parallel Execution and Aggregation (Fan-out/Fan-in)](#parallel-execution-and-aggregation-fan-outfan-in)
   - [Dependency Injection (DataStore)](#dependency-injection-datastore)
   - [Native Scheduler](#native-scheduler)
+  - [S3 Payload Offloading](#s3-payload-offloading)
+  - [Webhook Notifications](#webhook-notifications)
 - [Production Configuration](#production-configuration)
   - [Fault Tolerance](#fault-tolerance)
   - [Storage Backend](#storage-backend)
@@ -107,6 +114,11 @@ Avtomatika is part of a larger ecosystem:
     pip install "avtomatika[telemetry]"
     ```
 
+*   **Install with S3 support (Payload Offloading):**
+    ```bash
+    pip install "avtomatika[s3]"
+    ```
+
 *   **Install all dependencies, including for testing:**
     ```bash
     pip install "avtomatika[all,test]"
@@ -157,7 +169,13 @@ async def end_handler(context):
 engine = OrchestratorEngine(storage, config)
 engine.register_blueprint(my_blueprint)
 
-# 4. Define the main entrypoint to run the server
+# 4. Accessing Components (Optional)
+# You can access the internal aiohttp app and core components using AppKeys
+# from avtomatika.app_keys import ENGINE_KEY, DISPATCHER_KEY
+# app = engine.app
+# dispatcher = app[DISPATCHER_KEY]
+
+# 5. Define the main entrypoint to run the server
 async def main():
     await engine.start()
     
@@ -244,6 +262,19 @@ async def publish_handler_old_style(context):
     print(f"Job {context.job_id}: Publishing video at {output_path} ({duration}s).")
     context.actions.transition_to("complete")
 ```
+## Key Concepts: JobContext and Actions
+
+### High Performance Architecture
+
+Avtomatika is engineered for high-load environments with thousands of concurrent workers.
+
+*   **O(1) Dispatcher**: Uses advanced Redis Set intersections to find suitable workers instantly, regardless of the cluster size. No O(N) scanning.
+*   **Non-Blocking I/O**:
+    *   **Webhooks**: Sent via a bounded background queue to prevent backpressure.
+    *   **History Logging**: Writes to SQL databases are buffered and asynchronous, ensuring the main execution loop never blocks.
+    *   **Redis Streams**: Uses blocking reads to eliminate busy-waiting and reduce CPU usage.
+*   **Memory Safety**: S3 file transfers use streaming to handle multi-gigabyte files with constant, low RAM usage.
+
 ## Blueprint Cookbook: Key Features
 
 ### 1. Conditional Transitions (`.when()`)
@@ -355,6 +386,63 @@ blueprint = "backup_flow"
 daily_at = "02:00"
 ```
 
+### 6. Webhook Notifications
+
+The orchestrator can send asynchronous notifications to an external system when a job completes, fails, or is quarantined. This eliminates the need for clients to constantly poll the API for status updates.
+
+### 7. S3 Payload Offloading
+
+Orchestrator provides first-class support for handling large files via S3-compatible storage, powered by the high-performance `obstore` library (Rust bindings).
+
+*   **Memory Safe (Streaming)**: Uses streaming for uploads and downloads, allowing processing of files larger than available RAM without OOM errors.
+*   **Managed Mode**: The Orchestrator manages file lifecycle (automatic cleanup of S3 objects and local temporary files on job completion).
+*   **Dependency Injection**: Use the `task_files` argument in your handlers to easily read/write data.
+*   **Directory Support**: Supports recursive download and upload of entire directories.
+
+```python
+@bp.handler_for("process_data")
+async def process_data(task_files, actions):
+    # Streaming download of a large file
+    local_path = await task_files.download("large_dataset.csv")
+    
+    # ... process data ...
+    
+    # Upload results
+    await task_files.write_json("results.json", {"status": "done"})
+    
+    actions.transition_to("finished")
+```
+
+## Production Configuration
+*   **Events:**
+    *   `job_finished`: The job reached a final success state.
+    *   `job_failed`: The job failed (e.g., due to an error or invalid input).
+    *   `job_quarantined`: The job was moved to quarantine after repeated failures.
+
+**Example Request:**
+```json
+POST /api/v1/jobs/my_flow
+{
+    "initial_data": {
+        "video_url": "..."
+    },
+    "webhook_url": "https://my-app.com/webhooks/avtomatika"
+}
+```
+
+**Example Webhook Payload:**
+```json
+{
+    "event": "job_finished",
+    "job_id": "123e4567-e89b-12d3-a456-426614174000",
+    "status": "finished",
+    "result": {
+        "output_path": "/videos/result.mp4"
+    },
+    "error": null
+}
+```
+
 ## Production Configuration
 
 The orchestrator's behavior can be configured through environment variables. Additionally, any configuration parameter loaded from environment variables can be programmatically overridden in your application code after the `Config` object has been initialized. This provides flexibility for different deployment and testing scenarios.

{avtomatika-1.0b6 → avtomatika-1.0b8}/README.md

@@ -15,6 +15,8 @@ This document serves as a comprehensive guide for developers looking to build pi
   - [Parallel Execution and Aggregation (Fan-out/Fan-in)](#parallel-execution-and-aggregation-fan-outfan-in)
   - [Dependency Injection (DataStore)](#dependency-injection-datastore)
   - [Native Scheduler](#native-scheduler)
+  - [S3 Payload Offloading](#s3-payload-offloading)
+  - [Webhook Notifications](#webhook-notifications)
 - [Production Configuration](#production-configuration)
   - [Fault Tolerance](#fault-tolerance)
   - [Storage Backend](#storage-backend)
@@ -61,6 +63,11 @@ Avtomatika is part of a larger ecosystem:
     pip install "avtomatika[telemetry]"
     ```
 
+*   **Install with S3 support (Payload Offloading):**
+    ```bash
+    pip install "avtomatika[s3]"
+    ```
+
 *   **Install all dependencies, including for testing:**
     ```bash
     pip install "avtomatika[all,test]"
@@ -111,7 +118,13 @@ async def end_handler(context):
 engine = OrchestratorEngine(storage, config)
 engine.register_blueprint(my_blueprint)
 
-# 4. Define the main entrypoint to run the server
+# 4. Accessing Components (Optional)
+# You can access the internal aiohttp app and core components using AppKeys
+# from avtomatika.app_keys import ENGINE_KEY, DISPATCHER_KEY
+# app = engine.app
+# dispatcher = app[DISPATCHER_KEY]
+
+# 5. Define the main entrypoint to run the server
 async def main():
     await engine.start()
     
@@ -198,6 +211,19 @@ async def publish_handler_old_style(context):
     print(f"Job {context.job_id}: Publishing video at {output_path} ({duration}s).")
     context.actions.transition_to("complete")
 ```
+## Key Concepts: JobContext and Actions
+
+### High Performance Architecture
+
+Avtomatika is engineered for high-load environments with thousands of concurrent workers.
+
+*   **O(1) Dispatcher**: Uses advanced Redis Set intersections to find suitable workers instantly, regardless of the cluster size. No O(N) scanning.
+*   **Non-Blocking I/O**:
+    *   **Webhooks**: Sent via a bounded background queue to prevent backpressure.
+    *   **History Logging**: Writes to SQL databases are buffered and asynchronous, ensuring the main execution loop never blocks.
+    *   **Redis Streams**: Uses blocking reads to eliminate busy-waiting and reduce CPU usage.
+*   **Memory Safety**: S3 file transfers use streaming to handle multi-gigabyte files with constant, low RAM usage.
+
 ## Blueprint Cookbook: Key Features
 
 ### 1. Conditional Transitions (`.when()`)
@@ -309,6 +335,63 @@ blueprint = "backup_flow"
 daily_at = "02:00"
 ```
 
+### 6. Webhook Notifications
+
+The orchestrator can send asynchronous notifications to an external system when a job completes, fails, or is quarantined. This eliminates the need for clients to constantly poll the API for status updates.
+
+### 7. S3 Payload Offloading
+
+Orchestrator provides first-class support for handling large files via S3-compatible storage, powered by the high-performance `obstore` library (Rust bindings).
+
+*   **Memory Safe (Streaming)**: Uses streaming for uploads and downloads, allowing processing of files larger than available RAM without OOM errors.
+*   **Managed Mode**: The Orchestrator manages file lifecycle (automatic cleanup of S3 objects and local temporary files on job completion).
+*   **Dependency Injection**: Use the `task_files` argument in your handlers to easily read/write data.
+*   **Directory Support**: Supports recursive download and upload of entire directories.
+
+```python
+@bp.handler_for("process_data")
+async def process_data(task_files, actions):
+    # Streaming download of a large file
+    local_path = await task_files.download("large_dataset.csv")
+    
+    # ... process data ...
+    
+    # Upload results
+    await task_files.write_json("results.json", {"status": "done"})
+    
+    actions.transition_to("finished")
+```
+
+## Production Configuration
+*   **Events:**
+    *   `job_finished`: The job reached a final success state.
+    *   `job_failed`: The job failed (e.g., due to an error or invalid input).
+    *   `job_quarantined`: The job was moved to quarantine after repeated failures.
+
+**Example Request:**
+```json
+POST /api/v1/jobs/my_flow
+{
+    "initial_data": {
+        "video_url": "..."
+    },
+    "webhook_url": "https://my-app.com/webhooks/avtomatika"
+}
+```
+
+**Example Webhook Payload:**
+```json
+{
+    "event": "job_finished",
+    "job_id": "123e4567-e89b-12d3-a456-426614174000",
+    "status": "finished",
+    "result": {
+        "output_path": "/videos/result.mp4"
+    },
+    "error": null
+}
+```
+
 ## Production Configuration
 
 The orchestrator's behavior can be configured through environment variables. Additionally, any configuration parameter loaded from environment variables can be programmatically overridden in your application code after the `Config` object has been initialized. This provides flexibility for different deployment and testing scenarios.

{avtomatika-1.0b6 → avtomatika-1.0b8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "avtomatika"
-version = "1.0b6"
+version = "1.0b8"
 description = "A state-machine based orchestrator for long-running AI and other jobs."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -16,7 +16,6 @@ classifiers = [
 ]
 dependencies = [
     "aiohttp~=3.12",
-    "aiocache~=0.12",
     "python-json-logger~=4.0",
     "graphviz~=0.21",
     "zstandard~=0.24",
@@ -27,6 +26,7 @@ dependencies = [
 
 [project.optional-dependencies]
 redis = ["redis~=7.1"]
+s3 = ["obstore>=0.2", "aiofiles~=23.2"]
 history = ["aiosqlite~=0.22", "asyncpg~=0.30"]
 telemetry = [
     "opentelemetry-api~=1.39",
@@ -43,11 +43,14 @@ test = [
     "aioresponses~=0.7",
     "backports.zstd~=1.2",
     "opentelemetry-instrumentation-aiohttp-client",
+    "obstore>=0.2",
+    "aiofiles~=23.2",
 ]
 all = [
     "avtomatika[redis]",
     "avtomatika[history]",
     "avtomatika[telemetry]",
+    "avtomatika[s3]",
 ]
 
 [project.urls]