avtomatika 1.0b2__py3-none-any.whl → 1.0b3__py3-none-any.whl

avtomatika/__init__.py

@@ -4,6 +4,7 @@
 This module exposes the primary classes for building and running state-driven automations.
 """
 
+import contextlib
 from importlib.metadata import version
 
 __version__ = version("avtomatika")
@@ -22,9 +23,7 @@ __all__ = [
     "StorageBackend",
 ]
 
-try:
+with contextlib.suppress(ImportError):
     from .storage.redis import RedisStorage  # noqa: F401
 
     __all__.append("RedisStorage")
-except ImportError:
-    pass

avtomatika/blueprint.py

@@ -168,8 +168,7 @@ class StateMachineBlueprint:
         for handler in self.conditional_handlers:
             if handler.state == state and handler.evaluate(context):
                 return handler.func
-        default_handler = self.handlers.get(state)
-        if default_handler:
+        if default_handler := self.handlers.get(state):
             return default_handler
         raise ValueError(
             f"No suitable handler found for state '{state}' in blueprint '{self.name}' for the given context.",
@@ -230,12 +229,11 @@ class StateMachineBlueprint:
                     f"Could not parse handler '{handler_func.__name__}' for state '{handler_state}'. "
                     f"Graph may be incomplete. Error: {e}"
                 )
-                pass
         for state in states:
             dot.node(state, state)
 
-        if output_filename:
-            dot.render(output_filename, format=output_format, cleanup=True)
-            print(f"Graph rendered to {output_filename}.{output_format}")
-        else:
+        if not output_filename:
             return dot.source
+        dot.render(output_filename, format=output_format, cleanup=True)
+        print(f"Graph rendered to {output_filename}.{output_format}")
+        return None

avtomatika/client_config_loader.py

@@ -26,25 +26,37 @@ async def load_client_configs_to_redis(
             config_path,
         )
         return
+    except Exception as e:
+        logger.error(f"Failed to parse client config file '{config_path}': {e}")
+        raise ValueError(f"Invalid client configuration file: {e}") from e
 
     loaded_count = 0
     for client_name, config in clients_data.items():
+        if not isinstance(config, dict):
+            logger.error(f"Client '{client_name}' configuration must be a table (dict).")
+            raise ValueError(f"Invalid configuration for client '{client_name}'")
+
         token = config.get("token")
         if not token:
-            logger.warning(
-                "Skipping client '%s' due to missing 'token' field.",
-                client_name,
-            )
-            continue
+            logger.error(f"Client '{client_name}' is missing required 'token' field.")
+            raise ValueError(f"Missing token for client '{client_name}'")
+
+        if not isinstance(token, str):
+            logger.error(f"Token for client '{client_name}' must be a string.")
+            raise ValueError(f"Invalid token type for client '{client_name}'")
 
         # Separate static config from dynamic quota values
         static_config = {k: v for k, v in config.items() if k != "monthly_attempts"}
         quota = config.get("monthly_attempts")
 
+        if quota is not None and not isinstance(quota, int):
+            logger.error(f"Quota 'monthly_attempts' for client '{client_name}' must be an integer.")
+            raise ValueError(f"Invalid quota type for client '{client_name}'")
+
         try:
             # Assume these storage methods will be implemented
             await storage.save_client_config(token, static_config)
-            if quota is not None and isinstance(quota, int):
+            if quota is not None:
                 await storage.initialize_client_quota(token, quota)
 
             loaded_count += 1

avtomatika/dispatcher.py

@@ -28,15 +28,13 @@ class Dispatcher:
         self.config = config
         self._round_robin_indices: Dict[str, int] = defaultdict(int)
 
+    @staticmethod
     def _is_worker_compliant(
-        self,
         worker: Dict[str, Any],
         requirements: Dict[str, Any],
     ) -> bool:
         """Checks if a worker meets the specified resource requirements."""
-        # GPU check
-        required_gpu = requirements.get("gpu_info")
-        if required_gpu:
+        if required_gpu := requirements.get("gpu_info"):
             gpu_info = worker.get("resources", {}).get("gpu_info")
             if not gpu_info:
                 return False
@@ -51,17 +49,15 @@ class Dispatcher:
             ):
                 return False
 
-        # Installed models check
-        required_models = requirements.get("installed_models")
-        if required_models:
+        if required_models := requirements.get("installed_models"):
             installed_models = {m["name"] for m in worker.get("installed_models", [])}
             if not set(required_models).issubset(installed_models):
                 return False
 
         return True
 
+    @staticmethod
     def _select_default(
-        self,
         workers: List[Dict[str, Any]],
         task_type: str,
     ) -> Dict[str, Any]:
@@ -74,7 +70,7 @@ class Dispatcher:
         """
         warm_workers = [w for w in workers if task_type in w.get("hot_cache", [])]
 
-        target_pool = warm_workers if warm_workers else workers
+        target_pool = warm_workers or workers
 
         # The `cost` field is deprecated but maintained for backward compatibility.
         min_cost = min(w.get("cost", float("inf")) for w in target_pool)
@@ -95,8 +91,8 @@ class Dispatcher:
         self._round_robin_indices[task_type] = idx + 1
         return selected_worker
 
+    @staticmethod
     def _select_least_connections(
-        self,
         workers: List[Dict[str, Any]],
         task_type: str,
     ) -> Dict[str, Any]:
@@ -105,15 +101,16 @@ class Dispatcher:
         """
         return min(workers, key=lambda w: w.get("load", 0.0))
 
+    @staticmethod
     def _select_cheapest(
-        self,
         workers: List[Dict[str, Any]],
         task_type: str,
     ) -> Dict[str, Any]:
         """Selects the cheapest worker based on 'cost_per_second'."""
         return min(workers, key=lambda w: w.get("cost_per_second", float("inf")))
 
-    def _get_best_value_score(self, worker: Dict[str, Any]) -> float:
+    @staticmethod
+    def _get_best_value_score(worker: Dict[str, Any]) -> float:
         """Calculates a "score" for a worker using the formula cost / reputation.
         The lower the score, the better.
         """
@@ -121,9 +118,7 @@ class Dispatcher:
         # Default reputation is 1.0 if absent
         reputation = worker.get("reputation", 1.0)
         # Avoid division by zero
-        if reputation == 0:
-            return float("inf")
-        return cost / reputation
+        return float("inf") if reputation == 0 else cost / reputation
 
     def _select_best_value(
         self,
@@ -153,10 +148,9 @@ class Dispatcher:
         idle_workers = [w for w in all_workers if w.get("status", "idle") == "idle"]
         logger.debug(f"Idle workers: {[w['worker_id'] for w in idle_workers]}")
         if not idle_workers:
-            # If there are no idle workers, check if there are any busy workers in multi-orchestrator mode.
-            # This doesn't change the logic (an error will still occur), but it makes the logs more informative.
-            busy_mo_workers = [w for w in all_workers if w.get("status") == "busy" and "multi_orchestrator_info" in w]
-            if busy_mo_workers:
+            if busy_mo_workers := [
+                w for w in all_workers if w.get("status") == "busy" and "multi_orchestrator_info" in w
+            ]:
                 logger.warning(
                     f"No idle workers. Found {len(busy_mo_workers)} busy workers "
                     f"in multi-orchestrator mode. They are likely performing tasks for other Orchestrators.",

avtomatika/engine.py

@@ -485,8 +485,7 @@ class OrchestratorEngine:
             await self.storage.save_job_state(job_id, job_state)
             # Optionally, trigger a specific 'cancelled' transition if defined in the blueprint
             transitions = job_state.get("current_task_transitions", {})
-            next_state = transitions.get("cancelled")
-            if next_state:
+            if next_state := transitions.get("cancelled"):
                 job_state["current_state"] = next_state
                 job_state["status"] = "running"  # It's running the cancellation handler now
                 await self.storage.save_job_state(job_id, job_state)
@@ -494,9 +493,7 @@ class OrchestratorEngine:
             return web.json_response({"status": "result_accepted_cancelled"}, status=200)
 
         transitions = job_state.get("current_task_transitions", {})
-        next_state = transitions.get(result_status)
-
-        if next_state:
+        if next_state := transitions.get(result_status):
             logging.info(f"Job {job_id} transitioning based on worker status '{result_status}' to state '{next_state}'")
 
             worker_data = result.get("data")
@@ -602,7 +599,8 @@ class OrchestratorEngine:
         await load_client_configs_to_redis(self.storage)
         return web.json_response({"status": "db_flushed"}, status=200)
 
-    async def _docs_handler(self, request: web.Request) -> web.Response:
+    @staticmethod
+    async def _docs_handler(request: web.Request) -> web.Response:
         from importlib import resources
 
         try:
@@ -647,16 +645,7 @@ class OrchestratorEngine:
             all_protected_apps.append(protected_app)
 
         for app in all_protected_apps:
-            app.router.add_get("/jobs/{job_id}", self._get_job_status_handler)
-            app.router.add_post("/jobs/{job_id}/cancel", self._cancel_job_handler)
-            if not isinstance(self.history_storage, NoOpHistoryStorage):
-                app.router.add_get("/jobs/{job_id}/history", self._get_job_history_handler)
-            app.router.add_get("/blueprints/{blueprint_name}/graph", self._get_blueprint_graph_handler)
-            app.router.add_get("/workers", self._get_workers_handler)
-            app.router.add_get("/jobs", self._get_jobs_handler)
-            app.router.add_get("/dashboard", self._get_dashboard_handler)
-            app.router.add_post("/admin/reload-workers", self._reload_worker_configs_handler)
-
+            self._register_common_routes(app)
         if has_unversioned_routes:
             self.app.add_subapp("/api/", protected_app)
         for version, app in versioned_apps.items():
@@ -676,6 +665,17 @@ class OrchestratorEngine:
         worker_app.router.add_get("/ws/{worker_id}", self._websocket_handler)
         self.app.add_subapp("/_worker/", worker_app)
 
+    def _register_common_routes(self, app):
+        app.router.add_get("/jobs/{job_id}", self._get_job_status_handler)
+        app.router.add_post("/jobs/{job_id}/cancel", self._cancel_job_handler)
+        if not isinstance(self.history_storage, NoOpHistoryStorage):
+            app.router.add_get("/jobs/{job_id}/history", self._get_job_history_handler)
+        app.router.add_get("/blueprints/{blueprint_name}/graph", self._get_blueprint_graph_handler)
+        app.router.add_get("/workers", self._get_workers_handler)
+        app.router.add_get("/jobs", self._get_jobs_handler)
+        app.router.add_get("/dashboard", self._get_dashboard_handler)
+        app.router.add_post("/admin/reload-workers", self._reload_worker_configs_handler)
+
     async def _websocket_handler(self, request: web.Request) -> web.WebSocketResponse:
         worker_id = request.match_info.get("worker_id")
         if not worker_id:

avtomatika/executor.py

@@ -35,11 +35,13 @@ except ImportError:
         def inject(self, *args, **kwargs):
             pass
 
-        def extract(self, *args, **kwargs):
+        @staticmethod
+        def extract(*args, **kwargs):
             return None
 
     class NoOpTraceContextTextMapPropagator:
-        def extract(self, *args, **kwargs):
+        @staticmethod
+        def extract(*args, **kwargs):
             return None
 
     trace = NoOpTracer()
@@ -485,7 +487,8 @@ class JobExecutor:
         await self.storage.save_job_state(parent_job_id, parent_job_state)
         await self.storage.enqueue_job(parent_job_id)
 
-    def _handle_task_completion(self, task: Task):
+    @staticmethod
+    def _handle_task_completion(task: Task):
         """Callback to handle completion of a job processing task."""
         try:
             # This will re-raise any exception caught in the task

avtomatika/ratelimit.py

@@ -1,3 +1,4 @@
+from contextlib import suppress
 from typing import Awaitable, Callable
 
 from aiohttp import web
@@ -23,23 +24,15 @@ def rate_limit_middleware_factory(
         """Rate-limiting middleware that uses the provided storage backend."""
         # Determine the key for rate limiting (e.g., by worker_id or IP)
         # For worker endpoints, we key by worker_id. For others, by IP.
-        key_identifier = request.match_info.get("worker_id", request.remote)
-        if not key_identifier:
-            # Fallback for cases where remote IP might not be available
-            key_identifier = "unknown"
+        key_identifier = request.match_info.get("worker_id", request.remote) or "unknown"
 
         # Key by identifier and path to have per-endpoint limits
         rate_limit_key = f"ratelimit:{key_identifier}:{request.path}"
 
-        try:
+        with suppress(Exception):
             count = await storage.increment_key_with_ttl(rate_limit_key, period)
             if count > limit:
                 return web.json_response({"error": "Too Many Requests"}, status=429)
-        except Exception:
-            # If the rate limiter fails for any reason (e.g., Redis down),
-            # it's safer to let the request through than to block everything.
-            pass
-
         return await handler(request)
 
     return rate_limit_middleware

avtomatika/reputation.py

@@ -1,6 +1,7 @@
 from asyncio import CancelledError, sleep
 from logging import getLogger
 from typing import TYPE_CHECKING
+from uuid import uuid4
 
 if TYPE_CHECKING:
     from .engine import OrchestratorEngine
@@ -20,14 +21,22 @@ class ReputationCalculator:
         self.history_storage = engine.history_storage
         self.interval_seconds = interval_seconds
         self._running = False
+        self._instance_id = str(uuid4())
 
     async def run(self):
         """The main loop that periodically triggers reputation recalculation."""
-        logger.info("ReputationCalculator started.")
+        logger.info(f"ReputationCalculator started (Instance ID: {self._instance_id}).")
         self._running = True
         while self._running:
             try:
-                await self.calculate_all_reputations()
+                # Attempt to acquire lock
+                if await self.storage.acquire_lock("global_reputation_lock", self._instance_id, 300):
+                    try:
+                        await self.calculate_all_reputations()
+                    finally:
+                        await self.storage.release_lock("global_reputation_lock", self._instance_id)
+                else:
+                    logger.debug("ReputationCalculator lock held by another instance. Skipping.")
             except CancelledError:
                 break
             except Exception:

avtomatika/storage/__init__.py

@@ -1,11 +1,11 @@
+import contextlib
+
 from .base import StorageBackend
 from .memory import MemoryStorage
 
 __all__ = ["StorageBackend", "MemoryStorage"]
 
-try:
+with contextlib.suppress(ImportError):
     from .redis import RedisStorage  # noqa: F401
 
     __all__.append("RedisStorage")
-except ImportError:
-    pass

avtomatika/storage/base.py

@@ -264,3 +264,26 @@ class StorageBackend(ABC):
         Used for metrics.
         """
         raise NotImplementedError
+
+    @abstractmethod
+    async def acquire_lock(self, key: str, holder_id: str, ttl: int) -> bool:
+        """
+        Attempts to acquire a distributed lock.
+
+        :param key: The unique key of the lock (e.g., 'watcher_lock').
+        :param holder_id: A unique identifier for the caller (e.g., UUID).
+        :param ttl: Time-to-live for the lock in seconds.
+        :return: True if the lock was acquired, False otherwise.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def release_lock(self, key: str, holder_id: str) -> bool:
+        """
+        Releases a distributed lock if it is held by the specified holder_id.
+
+        :param key: The unique key of the lock.
+        :param holder_id: The identifier of the caller who presumably holds the lock.
+        :return: True if the lock was successfully released, False otherwise.
+        """
+        raise NotImplementedError

avtomatika/storage/memory.py

@@ -25,6 +25,7 @@ class MemoryStorage(StorageBackend):
         self._worker_tokens: Dict[str, str] = {}
         self._generic_keys: Dict[str, Any] = {}
         self._generic_key_ttls: Dict[str, float] = {}
+        self._locks: Dict[str, tuple[str, float]] = {}  # key -> (holder_id, expiry_time)
 
         self._lock = Lock()
 
@@ -128,9 +129,11 @@ class MemoryStorage(StorageBackend):
         async with self._lock:
             now = monotonic()
             active_workers = []
-            for worker_id, worker_info in self._workers.items():
-                if self._worker_ttls.get(worker_id, 0) > now:
-                    active_workers.append(worker_info)
+            active_workers.extend(
+                worker_info
+                for worker_id, worker_info in self._workers.items()
+                if self._worker_ttls.get(worker_id, 0) > now
+            )
             return active_workers
 
     async def add_job_to_watch(self, job_id: str, timeout_at: float) -> None:
@@ -226,6 +229,7 @@ class MemoryStorage(StorageBackend):
             self._quotas.clear()
             self._generic_keys.clear()
             self._generic_key_ttls.clear()
+            self._locks.clear()
 
     async def get_job_queue_length(self) -> int:
         # No lock needed for asyncio.Queue.qsize()
@@ -234,13 +238,9 @@ class MemoryStorage(StorageBackend):
     async def get_active_worker_count(self) -> int:
         async with self._lock:
             now = monotonic()
-            count = 0
             # Create a copy of keys to avoid issues with concurrent modifications
             worker_ids = list(self._workers.keys())
-            for worker_id in worker_ids:
-                if self._worker_ttls.get(worker_id, 0) > now:
-                    count += 1
-            return count
+            return sum(self._worker_ttls.get(worker_id, 0) > now for worker_id in worker_ids)
 
     async def get_worker_info(self, worker_id: str) -> Optional[Dict[str, Any]]:
         async with self._lock:
@@ -273,3 +273,29 @@ class MemoryStorage(StorageBackend):
             "average_bid": 0,
             "error": "Statistics are not supported for MemoryStorage backend.",
         }
+
+    async def acquire_lock(self, key: str, holder_id: str, ttl: int) -> bool:
+        async with self._lock:
+            now = monotonic()
+            current_lock = self._locks.get(key)
+
+            # If lock exists and hasn't expired
+            if current_lock and current_lock[1] > now:
+                # If explicitly owned by us, we can extend/re-enter (optional behavior)
+                # But for strict locking, if it's held, return False (unless it's us? let's simpler: just False if held)
+                return False
+
+            # Acquire lock
+            self._locks[key] = (holder_id, now + ttl)
+            return True
+
+    async def release_lock(self, key: str, holder_id: str) -> bool:
+        async with self._lock:
+            current_lock = self._locks.get(key)
+            if current_lock:
+                owner, expiry = current_lock
+                # Only release if we are the owner
+                if owner == holder_id:
+                    del self._locks[key]
+                    return True
+            return False

avtomatika/storage/redis.py

@@ -25,9 +25,7 @@ class RedisStorage(StorageBackend):
         """Get the job state from Redis."""
         key = self._get_key(job_id)
         data = await self._redis.get(key)
-        if data:
-            return loads(data)
-        return None
+        return loads(data) if data else None
 
     async def get_priority_queue_stats(self, task_type: str) -> Dict[str, Any]:
         """Gets statistics for the priority queue (Sorted Set) for a given task type."""
@@ -74,7 +72,7 @@ class RedisStorage(StorageBackend):
         self,
         job_id: str,
         update_data: Dict[str, Any],
-    ) -> Dict[str, Any]:
+    ) -> dict[Any, Any] | None | Any:
         """Atomically update the job state in Redis using a transaction."""
         key = self._get_key(job_id)
 
@@ -134,10 +132,7 @@ class RedisStorage(StorageBackend):
         try:
             # BZPOPMAX returns a tuple (key, member, score)
             result = await self._redis.bzpopmax([key], timeout=timeout)
-            if result:
-                # result[1] contains the element (task data) in bytes
-                return loads(result[1])
-            return None
+            return loads(result[1]) if result else None
         except CancelledError:
             return None
         except ResponseError as e:
@@ -228,18 +223,13 @@ class RedisStorage(StorageBackend):
 
     async def get_available_workers(self) -> list[dict[str, Any]]:
         """Gets a list of active workers by scanning keys in Redis."""
-        workers = []
         worker_keys = [key async for key in self._redis.scan_iter("orchestrator:worker:info:*")]  # type: ignore[attr-defined]
 
         if not worker_keys:
             return []
 
         worker_data_list = await self._redis.mget(worker_keys)
-        for data in worker_data_list:
-            if data:
-                workers.append(loads(data))
-
-        return workers
+        return [loads(data) for data in worker_data_list if data]
 
     async def add_job_to_watch(self, job_id: str, timeout_at: float) -> None:
         """Adds a job to a Redis sorted set.
@@ -277,9 +267,7 @@ class RedisStorage(StorageBackend):
         try:
             # Lock for 1 second so that the while loop in the executor is not too tight
             result = await self._redis.brpop(["orchestrator:job_queue"], timeout=1)  # type: ignore[misc]
-            if result:
-                return result[1].decode("utf-8")
-            return None
+            return result[1].decode("utf-8") if result else None
         except CancelledError:
             return None
 
@@ -407,6 +395,35 @@ class RedisStorage(StorageBackend):
         """Gets the full info for a worker by its ID."""
         key = f"orchestrator:worker:info:{worker_id}"
         data = await self._redis.get(key)
-        if data:
-            return loads(data)
-        return None
+        return loads(data) if data else None
+
+    async def acquire_lock(self, key: str, holder_id: str, ttl: int) -> bool:
+        """Attempts to acquire a lock using Redis SET NX."""
+        redis_key = f"orchestrator:lock:{key}"
+        # Returns True if set was successful (key didn't exist), None otherwise
+        result = await self._redis.set(redis_key, holder_id, nx=True, ex=ttl)
+        return bool(result)
+
+    async def release_lock(self, key: str, holder_id: str) -> bool:
+        """Releases the lock using a Lua script to ensure ownership."""
+        redis_key = f"orchestrator:lock:{key}"
+
+        LUA_RELEASE_SCRIPT = """
+        if redis.call("get", KEYS[1]) == ARGV[1] then
+            return redis.call("del", KEYS[1])
+        else
+            return 0
+        end
+        """
+        try:
+            result = await self._redis.eval(LUA_RELEASE_SCRIPT, 1, redis_key, holder_id)
+            return bool(result)
+        except ResponseError as e:
+            # Fallback for fakeredis if needed, though fakeredis usually supports eval
+            if "unknown command" in str(e):
+                current_val = await self._redis.get(redis_key)
+                if current_val and current_val.decode("utf-8") == holder_id:
+                    await self._redis.delete(redis_key)
+                    return True
+                return False
+            raise e

avtomatika/telemetry.py

@@ -27,7 +27,8 @@ except ImportError:
             pass
 
     class DummyTracer:
-        def start_as_current_span(self, name, context=None):
+        @staticmethod
+        def start_as_current_span(name, context=None):
             return DummySpan()
 
     class NoOpTrace:
@@ -46,8 +47,7 @@ def setup_telemetry(service_name: str = "avtomatika"):
     resource = Resource(attributes={"service.name": service_name})
     provider = TracerProvider(resource=resource)
 
-    otlp_endpoint = getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
-    if otlp_endpoint:
+    if otlp_endpoint := getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):
         logger.info(f"OTLP exporter enabled, sending traces to {otlp_endpoint}")
         try:
             from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (

avtomatika/watcher.py

@@ -1,6 +1,7 @@
 from asyncio import CancelledError, sleep
 from logging import getLogger
 from typing import TYPE_CHECKING
+from uuid import uuid4
 
 if TYPE_CHECKING:
     from .engine import OrchestratorEngine
@@ -17,43 +18,56 @@ class Watcher:
         self.config = engine.config
         self._running = False
         self.watch_interval_seconds = self.config.WATCHER_INTERVAL_SECONDS
+        self._instance_id = str(uuid4())
 
     async def run(self):
         """The main loop of the watcher."""
-        logger.info("Watcher started.")
+        logger.info(f"Watcher started (Instance ID: {self._instance_id}).")
         self._running = True
         while self._running:
             try:
                 await sleep(self.watch_interval_seconds)
-                logger.info("Watcher running check for timed out jobs...")
 
-                timed_out_job_ids = await self.storage.get_timed_out_jobs()
+                # Attempt to acquire distributed lock
+                # We set TTL slightly longer than the expected execution time,
+                # but shorter than the interval if possible.
+                # Actually, a fixed TTL like 60s is fine as long as we release it.
+                if not await self.storage.acquire_lock("global_watcher_lock", self._instance_id, 60):
+                    logger.debug("Watcher lock held by another instance. Skipping check.")
+                    continue
 
-                for job_id in timed_out_job_ids:
-                    logger.warning(f"Job {job_id} timed out. Moving to failed state.")
-                    try:
-                        # Get the latest version to avoid overwriting
-                        job_state = await self.storage.get_job_state(job_id)
-                        if job_state and job_state["status"] == "waiting_for_worker":
-                            job_state["status"] = "failed"
-                            job_state["error_message"] = "Worker task timed out."
-                            await self.storage.save_job_state(job_id, job_state)
+                try:
+                    logger.info("Watcher running check for timed out jobs...")
+                    timed_out_job_ids = await self.storage.get_timed_out_jobs()
 
-                            # Increment the metric
-                            from . import metrics
+                    for job_id in timed_out_job_ids:
+                        logger.warning(f"Job {job_id} timed out. Moving to failed state.")
+                        try:
+                            # Get the latest version to avoid overwriting
+                            job_state = await self.storage.get_job_state(job_id)
+                            if job_state and job_state["status"] == "waiting_for_worker":
+                                job_state["status"] = "failed"
+                                job_state["error_message"] = "Worker task timed out."
+                                await self.storage.save_job_state(job_id, job_state)
 
-                            metrics.jobs_failed_total.inc(
-                                {
-                                    metrics.LABEL_BLUEPRINT: job_state.get(
-                                        "blueprint_name",
-                                        "unknown",
-                                    ),
-                                },
+                                # Increment the metric
+                                from . import metrics
+
+                                metrics.jobs_failed_total.inc(
+                                    {
+                                        metrics.LABEL_BLUEPRINT: job_state.get(
+                                            "blueprint_name",
+                                            "unknown",
+                                        ),
+                                    },
+                                )
+                        except Exception:
+                            logger.exception(
+                                f"Failed to update state for timed out job {job_id}",
                             )
-                    except Exception:
-                        logger.exception(
-                            f"Failed to update state for timed out job {job_id}",
-                        )
+                finally:
+                    # Always release the lock so we (or others) can run next time
+                    await self.storage.release_lock("global_watcher_lock", self._instance_id)
 
             except CancelledError:
                 logger.info("Watcher received cancellation request.")

avtomatika/worker_config_loader.py

@@ -28,14 +28,19 @@ async def load_worker_configs_to_redis(storage: StorageBackend, config_path: str
             workers_config: Dict[str, Any] = load(f)
     except Exception as e:
         logger.error(f"Failed to load or parse worker config file '{config_path}': {e}")
-        return
+        raise ValueError(f"Invalid worker configuration file: {e}") from e
 
     for worker_id, config in workers_config.items():
+        if not isinstance(config, dict):
+            logger.error(f"Worker '{worker_id}' configuration must be a table.")
+            raise ValueError(f"Invalid configuration for worker '{worker_id}'")
+
         token = config.get("token")
         if not token:
             logger.warning(f"No token found for worker_id '{worker_id}' in {config_path}. Skipping.")
+            # Skipping might be safer here if we want to allow partial configs, but strict is better.
+            # Let's keep existing skip logic but log error? No, let's allow skip if user really wants.
             continue
-
         try:
             # Hash the token before storing it
             hashed_token = sha256(token.encode()).hexdigest()

avtomatika/ws_manager.py

@@ -46,7 +46,8 @@ class WebSocketManager:
                 logger.warning(f"Cannot send command: No active WebSocket connection for worker {worker_id}.")
                 return False
 
-    async def handle_message(self, worker_id: str, message: dict):
+    @staticmethod
+    async def handle_message(worker_id: str, message: dict):
         """Handles an incoming message from a worker."""
         event_type = message.get("event")
         if event_type == "progress_update":

{avtomatika-1.0b2.dist-info → avtomatika-1.0b3.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: avtomatika
-Version: 1.0b2
-Summary: A state-machine based orchestrator for long-running jobs.
+Version: 1.0b3
+Summary: A state-machine based orchestrator for long-running AI and other jobs.
 Project-URL: Homepage, https://github.com/avtomatika-ai/avtomatika
 Project-URL: Bug Tracker, https://github.com/avtomatika-ai/avtomatika/issues
 Classifier: Development Status :: 4 - Beta
@@ -18,25 +18,25 @@ Requires-Dist: graphviz~=0.21
 Requires-Dist: zstandard~=0.24
 Requires-Dist: aioprometheus~=23.12
 Provides-Extra: redis
-Requires-Dist: redis~=6.4; extra == "redis"
+Requires-Dist: redis~=7.1; extra == "redis"
 Requires-Dist: orjson~=3.11; extra == "redis"
 Provides-Extra: history
-Requires-Dist: aiosqlite~=0.21; extra == "history"
+Requires-Dist: aiosqlite~=0.22; extra == "history"
 Requires-Dist: asyncpg~=0.30; extra == "history"
 Requires-Dist: orjson~=3.11; extra == "history"
 Provides-Extra: telemetry
-Requires-Dist: opentelemetry-api~=1.38; extra == "telemetry"
-Requires-Dist: opentelemetry-sdk~=1.38; extra == "telemetry"
-Requires-Dist: opentelemetry-exporter-otlp~=1.36; extra == "telemetry"
+Requires-Dist: opentelemetry-api~=1.39; extra == "telemetry"
+Requires-Dist: opentelemetry-sdk~=1.39; extra == "telemetry"
+Requires-Dist: opentelemetry-exporter-otlp~=1.39; extra == "telemetry"
 Requires-Dist: opentelemetry-instrumentation-aiohttp-client~=0.59b0; extra == "telemetry"
 Provides-Extra: test
-Requires-Dist: pytest~=8.4; extra == "test"
+Requires-Dist: pytest~=9.0; extra == "test"
 Requires-Dist: pytest-asyncio~=1.1; extra == "test"
-Requires-Dist: fakeredis~=2.31; extra == "test"
+Requires-Dist: fakeredis~=2.33; extra == "test"
 Requires-Dist: pytest-aiohttp~=1.1; extra == "test"
 Requires-Dist: pytest-mock~=3.14; extra == "test"
 Requires-Dist: aioresponses~=0.7; extra == "test"
-Requires-Dist: backports.zstd; extra == "test"
+Requires-Dist: backports.zstd~=1.2; extra == "test"
 Requires-Dist: opentelemetry-instrumentation-aiohttp-client; extra == "test"
 Provides-Extra: all
 Requires-Dist: avtomatika[redis]; extra == "all"
@@ -285,7 +285,7 @@ Run multiple tasks simultaneously and gather their results.
 @my_blueprint.handler_for("process_files")
 async def fan_out_handler(initial_data, actions):
     tasks_to_dispatch = [
-        {"task_type": "file_analysis", "params": {"file": file}}
+        {"task_type": "file_analysis", "params": {"file": file}})
         for file in initial_data.get("files", [])
     ]
     # Use dispatch_parallel to send all tasks at once.
@@ -332,6 +332,8 @@ async def cache_handler(data_stores):
 
 The orchestrator's behavior can be configured through environment variables. Additionally, any configuration parameter loaded from environment variables can be programmatically overridden in your application code after the `Config` object has been initialized. This provides flexibility for different deployment and testing scenarios.
 
+**Important:** The system employs **strict validation** for configuration files (`clients.toml`, `workers.toml`) at startup. If a configuration file is invalid (e.g., malformed TOML, missing required fields), the application will **fail fast** and exit with an error, rather than starting in a partially broken state. This ensures the security and integrity of the deployment.
+
 ### Fault Tolerance
 
 The orchestrator has built-in mechanisms for handling failures based on the `error.code` field in a worker's response.
@@ -340,6 +342,13 @@ The orchestrator has built-in mechanisms for handling failures based on the `err
 *   **PERMANENT_ERROR**: A permanent error (e.g., a corrupted file). The task will be immediately sent to quarantine for manual investigation.
 *   **INVALID_INPUT_ERROR**: An error in the input data. The entire pipeline (Job) will be immediately moved to the failed state.
 
+### High Availability & Distributed Locking
+
+The architecture supports horizontal scaling. Multiple Orchestrator instances can run behind a load balancer.
+
+*   **Stateless API:** The API is stateless; all state is persisted in Redis.
+*   **Distributed Locking:** Background processes (`Watcher`, `ReputationCalculator`) use distributed locks (via Redis `SET NX`) to coordinate and prevent race conditions when multiple instances are active.
+
 ### Storage Backend
 
 By default, the engine uses in-memory storage. For production, you must configure persistent storage via environment variables.
@@ -408,3 +417,12 @@ To run the `avtomatika` test suite:
 ```bash
 pytest avtomatika/tests/
 ```
+
+## Detailed Documentation
+
+For a deeper dive into the system, please refer to the following documents in the `docs/` directory:
+
+- [**Architecture Guide**](docs/architecture.md): A detailed overview of the system components and their interactions.
+- [**API Reference**](docs/api_reference.md): Full specification of the HTTP API.
+- [**Deployment Guide**](docs/deployment.md): Instructions for deploying with Gunicorn/Uvicorn and NGINX.
+- [**Cookbook**](docs/cookbook/README.md): Examples and best practices for creating blueprints.

avtomatika-1.0b3.dist-info/RECORD

@@ -0,0 +1,37 @@
+avtomatika/__init__.py,sha256=nlk59j7YcK1gapRUVfHjvFZVAD_PZoamgHEptchP3TA,698
+avtomatika/api.html,sha256=Z-Ikqrle7YPXagx2D-C5ylVZicLQFSsIzPsHCQgqMHM,33628
+avtomatika/blueprint.py,sha256=Hx5h0upr_IYbCy1ebUTpXw4bnt5yYhgWtdPLVE1_h48,9403
+avtomatika/client_config_loader.py,sha256=zVVHZlxSqZUaNpZ4zoU0T1CFYXdxy-3vKSmPcaFuHSY,2772
+avtomatika/compression.py,sha256=bhA1kw4YrCR3I3kdquZSY0fAzCrRrjtz55uepzLUDKI,2498
+avtomatika/config.py,sha256=0vlMfVMjxwVUC8m_NglGocC_EoklzAc0qmt3UJbxm10,2087
+avtomatika/context.py,sha256=rnF09jqQGkaKlax8P5ku9USwijSm6dommDGZbeVrzLk,4295
+avtomatika/data_types.py,sha256=g-g5hPnCpzeATgOn5v7EvDm5ps314owFJD5iWJ6IPR0,1425
+avtomatika/datastore.py,sha256=ERMyiFYQpAhVYijxzTrrdm6jtIPFf4dngWIa0qod3Wc,551
+avtomatika/dispatcher.py,sha256=a_7DjJwSXbW-ZzqcjZG0ZXMYDD2JLZxpQRIzHOrjeow,9688
+avtomatika/engine.py,sha256=zwouopyGjHkyiE3dMndxu1uAIMOFvnV1h8-ZIFHIH-k,37507
+avtomatika/executor.py,sha256=JHwT2DR-Hbrb_-Le1-mVaXiiQ7z-PkMsuIYB9ciiVo0,21201
+avtomatika/health_checker.py,sha256=WXwvRJ-3cZC2Udc_ogsyIQp7VzcvJjq_IaqzkTdE0TE,1265
+avtomatika/logging_config.py,sha256=e0-eEEGHw1zz9ZshzXaxfavV0uZfamRNdcAeHnrgBYQ,1370
+avtomatika/metrics.py,sha256=7XDhr_xMJ9JpElpZmBG7R0ml7AMdAp9UYp_W-i7tyLg,1858
+avtomatika/py.typed,sha256=CT_L7gw2MLcQY-X0vs-xB5Vr0wzvGo7GuQYPI_qwJE8,65
+avtomatika/quota.py,sha256=DNcaL6k0J1REeP8sVqbY9FprY_3BSr2SxM2Vf4mEqdw,1612
+avtomatika/ratelimit.py,sha256=hFGW5oN9G6_W_jnHmopXW8bRjjzlvanY19MLghsNLE8,1306
+avtomatika/reputation.py,sha256=IHcaIAILWZftPPmXj5En28OSDNK7U8ivQ-w30zIF8fk,3748
+avtomatika/security.py,sha256=afj28O3xB20EmA75DAQCQm_QKzx_tX2Qv9zE9TlcFvM,4441
+avtomatika/telemetry.py,sha256=ZBt1_xJ36PzDSz-zdCXeNp58NiezUgbqvMctTG25PT0,2352
+avtomatika/watcher.py,sha256=IHaqSqp3XSGXjRY-LEeTG9BJpq2nqJSnmjY_Vdvk3jo,3493
+avtomatika/worker_config_loader.py,sha256=Ir8jbZ_07U8NAcu3r_EXM1jQvNpVEvHRP0k9vsq3mio,2255
+avtomatika/ws_manager.py,sha256=v3nz-w4AhoV_vqs3y8twXaMxm7s52wg2wzCMzTkPd8M,3081
+avtomatika/history/base.py,sha256=p0zItsdxFzd889LujV8py6GwK4CUfqAt8QL915mrT4k,1680
+avtomatika/history/noop.py,sha256=Hk5yJsS4S5G5A7NRRMEafIV_IFI9hddSwEvRg2Reh0M,982
+avtomatika/history/postgres.py,sha256=zanh_WktXM_C8ZPsYGiI1x4ScyHDNE1LVOeYiN72NdY,7685
+avtomatika/history/sqlite.py,sha256=hivl--uJ47MosrD6qhBwW3KYAQvVgbPNM6UYYatqFKM,8862
+avtomatika/storage/__init__.py,sha256=ygqv240XuYuHjU_2eci0J3FWoJLNSRpUFA2GzBrHMKg,259
+avtomatika/storage/base.py,sha256=BCC7uAQrko1UCwZo5kGF-0blwJiFcLCcT-pMnhYAxqY,10494
+avtomatika/storage/memory.py,sha256=7VhQO02SbYc65uDTOY9g43CVOgsodxzg-WYo0JGpUec,11387
+avtomatika/storage/redis.py,sha256=kgNUJuwcxQvCzul0m5COKhDnfJGKReMNeWxtG_BGfLc,18171
+avtomatika-1.0b3.dist-info/licenses/LICENSE,sha256=tqCjw9Y1vbU-hLcWi__7wQstLbt2T1XWPdbQYqCxuWY,1072
+avtomatika-1.0b3.dist-info/METADATA,sha256=OXavLWy_3WKrw5oW2Q8WdWvAdFFDA1-XvXGN59LQLKM,19455
+avtomatika-1.0b3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+avtomatika-1.0b3.dist-info/top_level.txt,sha256=gLDWhA_wxHj0I6fG5X8vw9fE0HSN4hTE2dEJzeVS2x8,11
+avtomatika-1.0b3.dist-info/RECORD,,

avtomatika-1.0b2.dist-info/RECORD

@@ -1,37 +0,0 @@
-avtomatika/__init__.py,sha256=BtYVoi9xy3yuMDMmzvLTO3AfbL2xeEjVz2bP9GhQeHU,675
-avtomatika/api.html,sha256=Z-Ikqrle7YPXagx2D-C5ylVZicLQFSsIzPsHCQgqMHM,33628
-avtomatika/blueprint.py,sha256=RvDidn2xkziZOOnJlkV30SMp59GuscASz11GolxK7fE,9445
-avtomatika/client_config_loader.py,sha256=N41juqx9CF2XZ3-WRXspkLG2eoGhj9LkWdaMCCCk33E,1962
-avtomatika/compression.py,sha256=bhA1kw4YrCR3I3kdquZSY0fAzCrRrjtz55uepzLUDKI,2498
-avtomatika/config.py,sha256=0vlMfVMjxwVUC8m_NglGocC_EoklzAc0qmt3UJbxm10,2087
-avtomatika/context.py,sha256=rnF09jqQGkaKlax8P5ku9USwijSm6dommDGZbeVrzLk,4295
-avtomatika/data_types.py,sha256=g-g5hPnCpzeATgOn5v7EvDm5ps314owFJD5iWJ6IPR0,1425
-avtomatika/datastore.py,sha256=ERMyiFYQpAhVYijxzTrrdm6jtIPFf4dngWIa0qod3Wc,551
-avtomatika/dispatcher.py,sha256=2YW-A0QXD4T6RF3KiNHflGs8LeLNrMlo5NScE4T49-o,10015
-avtomatika/engine.py,sha256=IeBy7pnWjERZwTIAp02wcaKggRMi5Jof-02kB2LfAKc,37482
-avtomatika/executor.py,sha256=-gbLfluZfwtLnAYI8GdK1xswA_DYp_-yoLnP4AWlcyQ,21157
-avtomatika/health_checker.py,sha256=WXwvRJ-3cZC2Udc_ogsyIQp7VzcvJjq_IaqzkTdE0TE,1265
-avtomatika/logging_config.py,sha256=e0-eEEGHw1zz9ZshzXaxfavV0uZfamRNdcAeHnrgBYQ,1370
-avtomatika/metrics.py,sha256=7XDhr_xMJ9JpElpZmBG7R0ml7AMdAp9UYp_W-i7tyLg,1858
-avtomatika/py.typed,sha256=CT_L7gw2MLcQY-X0vs-xB5Vr0wzvGo7GuQYPI_qwJE8,65
-avtomatika/quota.py,sha256=DNcaL6k0J1REeP8sVqbY9FprY_3BSr2SxM2Vf4mEqdw,1612
-avtomatika/ratelimit.py,sha256=bZSmdCHviSGMVDNOKTBSswDl6P9Dc63BKLqKU50twpg,1579
-avtomatika/reputation.py,sha256=XFEqDW4TEqjrWgGB2KwnBBJF1RpwLlZPgFwxjPl2z8w,3221
-avtomatika/security.py,sha256=afj28O3xB20EmA75DAQCQm_QKzx_tX2Qv9zE9TlcFvM,4441
-avtomatika/telemetry.py,sha256=FIbbYjNX0JhpfT7UHjx6EdcFUGF1DpQDrbicIp0ZGvA,2353
-avtomatika/watcher.py,sha256=djqwgK8wu73LL2Rtxa6mV-EXimFO7T-iRQV5U_26dc8,2571
-avtomatika/worker_config_loader.py,sha256=2Fu2gU3O5H9VJ62D_Yc4gT19vV_be8DJZWzrTTovq8E,1788
-avtomatika/ws_manager.py,sha256=X420oV9_vnujV78VWuOVNqiMZAMQzfw6_0Ep694LJjQ,3069
-avtomatika/history/base.py,sha256=p0zItsdxFzd889LujV8py6GwK4CUfqAt8QL915mrT4k,1680
-avtomatika/history/noop.py,sha256=Hk5yJsS4S5G5A7NRRMEafIV_IFI9hddSwEvRg2Reh0M,982
-avtomatika/history/postgres.py,sha256=zanh_WktXM_C8ZPsYGiI1x4ScyHDNE1LVOeYiN72NdY,7685
-avtomatika/history/sqlite.py,sha256=hivl--uJ47MosrD6qhBwW3KYAQvVgbPNM6UYYatqFKM,8862
-avtomatika/storage/__init__.py,sha256=TFb3_Ab6K1tlGaBihMqhzwdwCarMG2uRpATDW372YNA,235
-avtomatika/storage/base.py,sha256=rcHbTB_0s4ZhkxIACnp9f6Ow7Jvr9pHx96t3VUf4lIc,9581
-avtomatika/storage/memory.py,sha256=Wqi879hw1cZBIhMDk_bKPaWhnH33QEl4aUIxu9g3M0M,10282
-avtomatika/storage/redis.py,sha256=eT65_tvRufy-txiKfSSs5UmKBeEio8aWG0UTBfj9DdY,17088
-avtomatika-1.0b2.dist-info/licenses/LICENSE,sha256=tqCjw9Y1vbU-hLcWi__7wQstLbt2T1XWPdbQYqCxuWY,1072
-avtomatika-1.0b2.dist-info/METADATA,sha256=IofXNi3-o-YDIfLKrBIT0mV5osHPvRyYX95b_9WI8dY,18083
-avtomatika-1.0b2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-avtomatika-1.0b2.dist-info/top_level.txt,sha256=gLDWhA_wxHj0I6fG5X8vw9fE0HSN4hTE2dEJzeVS2x8,11
-avtomatika-1.0b2.dist-info/RECORD,,