avtomatika 1.0b5__py3-none-any.whl → 1.0b7__py3-none-any.whl

avtomatika/executor.py

@@ -1,5 +1,4 @@
 from asyncio import CancelledError, Task, create_task, sleep
-from inspect import signature
 from logging import getLogger
 from time import monotonic
 from types import SimpleNamespace
@@ -167,17 +166,17 @@ class JobExecutor:
                         handler = blueprint.find_handler(context.current_state, context)
 
                     # Build arguments for the handler dynamically.
-                    handler_signature = signature(handler)
-                    params_to_inject = {}
+                    param_names = blueprint.get_handler_params(handler)
+                    params_to_inject: dict[str, Any] = {}
 
-                    if "context" in handler_signature.parameters:
+                    if "context" in param_names:
                         params_to_inject["context"] = context
-                        if "actions" in handler_signature.parameters:
+                        if "actions" in param_names:
                             params_to_inject["actions"] = action_factory
                     else:
                         # New injection logic with prioritized lookup.
                         context_as_dict = context._asdict()
-                        for param_name in handler_signature.parameters:
+                        for param_name in param_names:
                             # Look in JobContext fields first.
                             if param_name in context_as_dict:
                                 params_to_inject[param_name] = context_as_dict[param_name]
@@ -232,7 +231,7 @@ class JobExecutor:
         job_state: dict[str, Any],
         next_state: str,
         duration_ms: int,
-    ):
+    ) -> None:
         job_id = job_state["id"]
         previous_state = job_state["current_state"]
         logger.info(f"Job {job_id} transitioning from {previous_state} to {next_state}")
@@ -260,13 +259,19 @@ class JobExecutor:
         else:
             logger.info(f"Job {job_id} reached terminal state {next_state}")
             await self._check_and_resume_parent(job_state)
+            # Send webhook for finished/failed jobs
+            event_type = "job_finished" if next_state == "finished" else "job_failed"
+            # Since _check_and_resume_parent is for sub-jobs, we only send webhook if it's a top-level job
+            # or if the user explicitly requested it for sub-jobs (by providing webhook_url).
+            # The current logic stores webhook_url in job_state, so we just check it.
+            await self.engine.send_job_webhook(job_state, event_type)
 
     async def _handle_dispatch(
         self,
         job_state: dict[str, Any],
         task_info: dict[str, Any],
         duration_ms: int,
-    ):
+    ) -> None:
         job_id = job_state["id"]
         current_state = job_state["current_state"]
 
@@ -302,7 +307,6 @@ class JobExecutor:
             await self.storage.save_job_state(job_id, job_state)
             await self.storage.add_job_to_watch(job_id, timeout_at)
 
-            # Now, dispatch the task
             await self.dispatcher.dispatch(job_state, task_info)
 
     async def _handle_run_blueprint(
@@ -310,7 +314,7 @@ class JobExecutor:
         parent_job_state: dict[str, Any],
         sub_blueprint_info: dict[str, Any],
         duration_ms: int,
-    ):
+    ) -> None:
         parent_job_id = parent_job_state["id"]
         child_job_id = str(uuid4())
 
@@ -350,7 +354,7 @@ class JobExecutor:
         job_state: dict[str, Any],
         parallel_info: dict[str, Any],
         duration_ms: int,
-    ):
+    ) -> None:
         job_id = job_state["id"]
         tasks_to_dispatch = parallel_info["tasks"]
         aggregate_into = parallel_info["aggregate_into"]
@@ -398,7 +402,7 @@ class JobExecutor:
         job_state: dict[str, Any],
         error: Exception,
         duration_ms: int,
-    ):
+    ) -> None:
         """Handles failures that occur *during the execution of a handler*.
 
         This is different from a task failure reported by a worker. This logic
@@ -447,13 +451,14 @@ class JobExecutor:
             await self.storage.quarantine_job(job_id)
             # If this quarantined job was a sub-job, we must now resume its parent.
             await self._check_and_resume_parent(job_state)
+            await self.engine.send_job_webhook(job_state, "job_quarantined")
             from . import metrics
 
             metrics.jobs_failed_total.inc(
                 {metrics.LABEL_BLUEPRINT: job_state.get("blueprint_name", "unknown")},
             )
 
-    async def _check_and_resume_parent(self, child_job_state: dict[str, Any]):
+    async def _check_and_resume_parent(self, child_job_state: dict[str, Any]) -> None:
         """Checks if a completed job was a sub-job. If so, it resumes the parent
         job, passing the success/failure outcome of the child.
         """
@@ -493,7 +498,7 @@ class JobExecutor:
         await self.storage.enqueue_job(parent_job_id)
 
     @staticmethod
-    def _handle_task_completion(task: Task):
+    def _handle_task_completion(task: Task) -> None:
         """Callback to handle completion of a job processing task."""
         try:
             # This will re-raise any exception caught in the task
@@ -505,7 +510,7 @@ class JobExecutor:
             # Log any other exceptions that occurred in the task.
             logger.exception("Unhandled exception in job processing task")
 
-    async def run(self):
+    async def run(self) -> None:
         import asyncio
 
         logger.info("JobExecutor started.")
@@ -536,5 +541,5 @@ class JobExecutor:
                 await sleep(1)
         logger.info("JobExecutor stopped.")
 
-    def stop(self):
+    def stop(self) -> None:
         self._running = False

avtomatika/history/postgres.py

@@ -1,9 +1,13 @@
 from abc import ABC
+from contextlib import suppress
+from datetime import datetime
 from logging import getLogger
 from typing import Any
 from uuid import uuid4
+from zoneinfo import ZoneInfo
 
-from asyncpg import Pool, PostgresError, create_pool  # type: ignore[import-untyped]
+from asyncpg import Connection, Pool, PostgresError, create_pool  # type: ignore[import-untyped]
+from orjson import dumps, loads
 
 from .base import HistoryStorageBase
 
@@ -41,14 +45,24 @@ CREATE_JOB_ID_INDEX_PG = "CREATE INDEX IF NOT EXISTS idx_job_id ON job_history(j
 class PostgresHistoryStorage(HistoryStorageBase, ABC):
     """Implementation of the history store based on asyncpg for PostgreSQL."""
 
-    def __init__(self, dsn: str):
+    def __init__(self, dsn: str, tz_name: str = "UTC"):
         self._dsn = dsn
         self._pool: Pool | None = None
+        self.tz_name = tz_name
+        self.tz = ZoneInfo(tz_name)
+
+    async def _setup_connection(self, conn: Connection):
+        """Configures the connection session with the correct timezone."""
+        try:
+            await conn.execute(f"SET TIME ZONE '{self.tz_name}'")
+        except PostgresError as e:
+            logger.error(f"Failed to set timezone '{self.tz_name}' for PG connection: {e}")
 
     async def initialize(self):
         """Initializes the connection pool to PostgreSQL and creates tables."""
         try:
-            self._pool = await create_pool(dsn=self._dsn)
+            # We use init parameter to configure each new connection in the pool
+            self._pool = await create_pool(dsn=self._dsn, init=self._setup_connection)
             if not self._pool:
                 raise RuntimeError("Failed to create a connection pool.")
 
@@ -56,7 +70,7 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
                 await conn.execute(CREATE_JOB_HISTORY_TABLE_PG)
                 await conn.execute(CREATE_WORKER_HISTORY_TABLE_PG)
                 await conn.execute(CREATE_JOB_ID_INDEX_PG)
-            logger.info("PostgreSQL history storage initialized.")
+            logger.info(f"PostgreSQL history storage initialized (TZ={self.tz_name}).")
         except (PostgresError, OSError) as e:
             logger.error(f"Failed to initialize PostgreSQL history storage: {e}")
             raise
@@ -74,14 +88,20 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
 
         query = """
             INSERT INTO job_history (
-                event_id, job_id, state, event_type, duration_ms,
+                event_id, job_id, timestamp, state, event_type, duration_ms,
                 previous_state, next_state, worker_id, attempt_number,
                 context_snapshot
-            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
         """
+        now = datetime.now(self.tz)
+
+        context_snapshot = event_data.get("context_snapshot")
+        context_snapshot_json = dumps(context_snapshot).decode("utf-8") if context_snapshot else None
+
         params = (
             uuid4(),
             event_data.get("job_id"),
+            now,
             event_data.get("state"),
             event_data.get("event_type"),
             event_data.get("duration_ms"),
@@ -89,7 +109,7 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
             event_data.get("next_state"),
             event_data.get("worker_id"),
             event_data.get("attempt_number"),
-            event_data.get("context_snapshot"),
+            context_snapshot_json,
         )
         try:
             async with self._pool.acquire() as conn:
@@ -104,14 +124,20 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
 
         query = """
             INSERT INTO worker_history (
-                event_id, worker_id, event_type, worker_info_snapshot
-            ) VALUES ($1, $2, $3, $4)
+                event_id, worker_id, timestamp, event_type, worker_info_snapshot
+            ) VALUES ($1, $2, $3, $4, $5)
         """
+        now = datetime.now(self.tz)
+
+        worker_info = event_data.get("worker_info_snapshot")
+        worker_info_json = dumps(worker_info).decode("utf-8") if worker_info else None
+
         params = (
             uuid4(),
             event_data.get("worker_id"),
+            now,
             event_data.get("event_type"),
-            event_data.get("worker_info_snapshot"),
+            worker_info_json,
         )
         try:
             async with self._pool.acquire() as conn:
@@ -119,6 +145,23 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
         except PostgresError as e:
             logger.error(f"Failed to log worker event to PostgreSQL: {e}")
 
+    def _format_row(self, row: dict[str, Any]) -> dict[str, Any]:
+        """Helper to format a row from DB: convert timestamp to local TZ and decode JSON."""
+        item = dict(row)
+
+        if isinstance(item.get("context_snapshot"), str):
+            with suppress(Exception):
+                item["context_snapshot"] = loads(item["context_snapshot"])
+
+        if isinstance(item.get("worker_info_snapshot"), str):
+            with suppress(Exception):
+                item["worker_info_snapshot"] = loads(item["worker_info_snapshot"])
+
+        if "timestamp" in item and isinstance(item["timestamp"], datetime):
+            item["timestamp"] = item["timestamp"].astimezone(self.tz)
+
+        return item
+
     async def get_job_history(self, job_id: str) -> list[dict[str, Any]]:
         """Gets the full history for the specified job from PostgreSQL."""
         if not self._pool:
@@ -128,7 +171,7 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
         try:
             async with self._pool.acquire() as conn:
                 rows = await conn.fetch(query, job_id)
-                return [dict(row) for row in rows]
+                return [self._format_row(row) for row in rows]
         except PostgresError as e:
             logger.error(
                 f"Failed to get job history for job_id {job_id} from PostgreSQL: {e}",
@@ -154,7 +197,7 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
         try:
             async with self._pool.acquire() as conn:
                 rows = await conn.fetch(query, limit, offset)
-                return [dict(row) for row in rows]
+                return [self._format_row(row) for row in rows]
         except PostgresError as e:
             logger.error(f"Failed to get jobs list from PostgreSQL: {e}")
             return []
@@ -206,7 +249,7 @@ class PostgresHistoryStorage(HistoryStorageBase, ABC):
         try:
             async with self._pool.acquire() as conn:
                 rows = await conn.fetch(query, worker_id, since_days)
-                return [dict(row) for row in rows]
+                return [self._format_row(row) for row in rows]
         except PostgresError as e:
             logger.error(f"Failed to get worker history for worker_id {worker_id} from PostgreSQL: {e}")
             return []

avtomatika/history/sqlite.py

@@ -1,6 +1,10 @@
+from contextlib import suppress
+from datetime import datetime
 from logging import getLogger
+from time import time
 from typing import Any
 from uuid import uuid4
+from zoneinfo import ZoneInfo
 
 from aiosqlite import Connection, Error, Row, connect
 from orjson import dumps, loads
@@ -13,7 +17,7 @@ CREATE_JOB_HISTORY_TABLE = """
 CREATE TABLE IF NOT EXISTS job_history (
     event_id TEXT PRIMARY KEY,
     job_id TEXT NOT NULL,
-    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+    timestamp REAL NOT NULL,
     state TEXT,
     event_type TEXT NOT NULL,
     duration_ms INTEGER,
@@ -29,7 +33,7 @@ CREATE_WORKER_HISTORY_TABLE = """
 CREATE TABLE IF NOT EXISTS worker_history (
     event_id TEXT PRIMARY KEY,
     worker_id TEXT NOT NULL,
-    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+    timestamp REAL NOT NULL,
     event_type TEXT NOT NULL,
     worker_info_snapshot TEXT
 );
@@ -39,11 +43,15 @@ CREATE_JOB_ID_INDEX = "CREATE INDEX IF NOT EXISTS idx_job_id ON job_history(job_
 
 
 class SQLiteHistoryStorage(HistoryStorageBase):
-    """Implementation of the history store based on aiosqlite."""
+    """Implementation of the history store based on aiosqlite.
+    Stores timestamps as Unix time (UTC) for correct sorting,
+    and converts them to the configured timezone upon retrieval.
+    """
 
-    def __init__(self, db_path: str):
+    def __init__(self, db_path: str, tz_name: str = "UTC"):
         self._db_path = db_path
         self._conn: Connection | None = None
+        self.tz = ZoneInfo(tz_name)
 
     async def initialize(self):
         """Initializes the database connection and creates tables if they don't exist."""
@@ -66,6 +74,23 @@ class SQLiteHistoryStorage(HistoryStorageBase):
             await self._conn.close()
             logger.info("SQLite history storage connection closed.")
 
+    def _format_row(self, row: dict[str, Any]) -> dict[str, Any]:
+        """Helper to format a row from DB: decode JSON and convert timestamp."""
+        item = dict(row)
+
+        if item.get("context_snapshot"):
+            with suppress(Exception):
+                item["context_snapshot"] = loads(item["context_snapshot"])
+
+        if item.get("worker_info_snapshot"):
+            with suppress(Exception):
+                item["worker_info_snapshot"] = loads(item["worker_info_snapshot"])
+
+        if "timestamp" in item and isinstance(item["timestamp"], (int, float)):
+            item["timestamp"] = datetime.fromtimestamp(item["timestamp"], self.tz)
+
+        return item
+
     async def log_job_event(self, event_data: dict[str, Any]):
         """Logs a job lifecycle event to the job_history table."""
         if not self._conn:
@@ -73,14 +98,20 @@ class SQLiteHistoryStorage(HistoryStorageBase):
 
         query = """
             INSERT INTO job_history (
-                event_id, job_id, state, event_type, duration_ms,
+                event_id, job_id, timestamp, state, event_type, duration_ms,
                 previous_state, next_state, worker_id, attempt_number,
                 context_snapshot
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         """
+        now_ts = time()
+
+        context_snapshot = event_data.get("context_snapshot")
+        context_snapshot_json = dumps(context_snapshot).decode("utf-8") if context_snapshot else None
+
         params = (
             str(uuid4()),
             event_data.get("job_id"),
+            now_ts,
             event_data.get("state"),
             event_data.get("event_type"),
             event_data.get("duration_ms"),
@@ -88,7 +119,7 @@ class SQLiteHistoryStorage(HistoryStorageBase):
             event_data.get("next_state"),
             event_data.get("worker_id"),
             event_data.get("attempt_number"),
-            (dumps(event_data.get("context_snapshot")) if event_data.get("context_snapshot") else None),
+            context_snapshot_json,
         )
 
         try:
@@ -104,14 +135,20 @@ class SQLiteHistoryStorage(HistoryStorageBase):
 
         query = """
             INSERT INTO worker_history (
-                event_id, worker_id, event_type, worker_info_snapshot
-            ) VALUES (?, ?, ?, ?)
+                event_id, worker_id, timestamp, event_type, worker_info_snapshot
+            ) VALUES (?, ?, ?, ?, ?)
         """
+        now_ts = time()
+
+        worker_info = event_data.get("worker_info_snapshot")
+        worker_info_json = dumps(worker_info).decode("utf-8") if worker_info else None
+
         params = (
             str(uuid4()),
             event_data.get("worker_id"),
+            now_ts,
             event_data.get("event_type"),
-            (dumps(event_data.get("worker_info_snapshot")) if event_data.get("worker_info_snapshot") else None),
+            worker_info_json,
         )
 
         try:
@@ -130,14 +167,7 @@ class SQLiteHistoryStorage(HistoryStorageBase):
             self._conn.row_factory = Row
             async with self._conn.execute(query, (job_id,)) as cursor:
                 rows = await cursor.fetchall()
-                history = []
-                for row in rows:
-                    item = dict(row)
-                    # Deserialize the JSON string back into a dict
-                    if item.get("context_snapshot"):
-                        item["context_snapshot"] = loads(item["context_snapshot"])
-                    history.append(item)
-                return history
+                return [self._format_row(row) for row in rows]
         except Error as e:
             logger.error(f"Failed to get job history for job_id {job_id}: {e}")
             return []
@@ -163,13 +193,7 @@ class SQLiteHistoryStorage(HistoryStorageBase):
             self._conn.row_factory = Row
             async with self._conn.execute(query, (limit, offset)) as cursor:
                 rows = await cursor.fetchall()
-                jobs = []
-                for row in rows:
-                    item = dict(row)
-                    if item.get("context_snapshot"):
-                        item["context_snapshot"] = loads(item["context_snapshot"])
-                    jobs.append(item)
-                return jobs
+                return [self._format_row(row) for row in rows]
         except Error as e:
             logger.error(f"Failed to get jobs list: {e}")
             return []
@@ -214,23 +238,19 @@ class SQLiteHistoryStorage(HistoryStorageBase):
         if not self._conn:
             raise RuntimeError("History storage is not initialized.")
 
+        threshold_ts = time() - (since_days * 86400)
+
         query = """
             SELECT * FROM job_history
             WHERE worker_id = ?
-            AND timestamp >= date('now', '-' || ? || ' days')
+            AND timestamp >= ?
             ORDER BY timestamp DESC
         """
         try:
             self._conn.row_factory = Row
-            async with self._conn.execute(query, (worker_id, since_days)) as cursor:
+            async with self._conn.execute(query, (worker_id, threshold_ts)) as cursor:
                 rows = await cursor.fetchall()
-                history = []
-                for row in rows:
-                    item = dict(row)
-                    if item.get("context_snapshot"):
-                        item["context_snapshot"] = loads(item["context_snapshot"])
-                    history.append(item)
-                return history
+                return [self._format_row(row) for row in rows]
         except Error as e:
             logger.error(f"Failed to get worker history for worker_id {worker_id}: {e}")
             return []

avtomatika/logging_config.py

@@ -1,10 +1,50 @@
+from datetime import datetime
 from logging import DEBUG, Formatter, StreamHandler, getLogger
 from sys import stdout
+from zoneinfo import ZoneInfo
 
 from pythonjsonlogger import json
 
 
-def setup_logging(log_level: str = "INFO", log_format: str = "json"):
+class TimezoneFormatter(Formatter):
+    """Formatter that respects a custom timezone."""
+
+    def __init__(self, fmt=None, datefmt=None, style="%", validate=True, *, tz_name="UTC"):
+        super().__init__(fmt, datefmt, style, validate)
+        self.tz = ZoneInfo(tz_name)
+
+    def converter(self, timestamp):
+        return datetime.fromtimestamp(timestamp, self.tz)
+
+    def formatTime(self, record, datefmt=None):
+        dt = self.converter(record.created)
+        if datefmt:
+            s = dt.strftime(datefmt)
+        else:
+            try:
+                s = dt.isoformat(timespec="milliseconds")
+            except TypeError:
+                s = dt.isoformat()
+        return s
+
+
+class TimezoneJsonFormatter(json.JsonFormatter):
+    """JSON Formatter that respects a custom timezone."""
+
+    def __init__(self, *args, tz_name="UTC", **kwargs):
+        super().__init__(*args, **kwargs)
+        self.tz = ZoneInfo(tz_name)
+
+    def formatTime(self, record, datefmt=None):
+        # Override formatTime to use timezone-aware datetime
+        dt = datetime.fromtimestamp(record.created, self.tz)
+        if datefmt:
+            return dt.strftime(datefmt)
+        # Return ISO format with offset
+        return dt.isoformat()
+
+
+def setup_logging(log_level: str = "INFO", log_format: str = "json", tz_name: str = "UTC"):
     """Configures structured logging for the entire application."""
     logger = getLogger("avtomatika")
     logger.setLevel(log_level)
@@ -13,13 +53,15 @@ def setup_logging(log_level: str = "INFO", log_format: str = "json"):
     formatter: Formatter
     if log_format.lower() == "json":
         # Formatter for JSON logs
-        formatter = json.JsonFormatter(
+        formatter = TimezoneJsonFormatter(
             "%(asctime)s %(name)s %(levelname)s %(message)s %(pathname)s %(lineno)d",
+            tz_name=tz_name,
         )
     else:
         # Standard text formatter
-        formatter = Formatter(
+        formatter = TimezoneFormatter(
             "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            tz_name=tz_name,
         )
 
     handler.setFormatter(formatter)
@@ -30,12 +72,21 @@ def setup_logging(log_level: str = "INFO", log_format: str = "json"):
 
     # Configure the root logger to see logs from libraries (aiohttp, etc.)
     root_logger = getLogger()
-    # Set the root logger level so as not to filter messages
-    # for child loggers prematurely.
     root_logger.setLevel(DEBUG)
+
     if not root_logger.handlers:
         root_handler = StreamHandler(stdout)
-        root_handler.setFormatter(
-            Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
+        root_formatter = TimezoneFormatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            tz_name=tz_name,
         )
+        root_handler.setFormatter(root_formatter)
         root_logger.addHandler(root_handler)
+    else:
+        for h in root_logger.handlers:
+            h.setFormatter(
+                TimezoneFormatter(
+                    "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                    tz_name=tz_name,
+                )
+            )

avtomatika/scheduler.py

@@ -0,0 +1,119 @@
+from asyncio import CancelledError, sleep
+from datetime import datetime
+from logging import getLogger
+from typing import TYPE_CHECKING
+from zoneinfo import ZoneInfo
+
+from .scheduler_config_loader import ScheduledJobConfig, load_schedules_from_file
+
+if TYPE_CHECKING:
+    from .engine import OrchestratorEngine
+
+logger = getLogger(__name__)
+
+
+class Scheduler:
+    def __init__(self, engine: "OrchestratorEngine"):
+        self.engine = engine
+        self.config = engine.config
+        self.storage = engine.storage
+        self._running = False
+        self.schedules: list[ScheduledJobConfig] = []
+        self.timezone = ZoneInfo(self.config.TZ)
+
+    def load_config(self) -> None:
+        if not self.config.SCHEDULES_CONFIG_PATH:
+            logger.info("No SCHEDULES_CONFIG_PATH set. Scheduler will not run any jobs.")
+            return
+
+        try:
+            self.schedules = load_schedules_from_file(self.config.SCHEDULES_CONFIG_PATH)
+            logger.info(f"Loaded {len(self.schedules)} scheduled jobs.")
+        except Exception as e:
+            logger.error(f"Failed to load schedules config: {e}")
+
+    async def run(self) -> None:
+        self.load_config()
+        if not self.schedules:
+            logger.info("No schedules found. Scheduler loop will not start.")
+            return
+
+        logger.info("Scheduler started.")
+        self._running = True
+
+        while self._running:
+            try:
+                now_utc = datetime.now(ZoneInfo("UTC"))
+                now_tz = now_utc.astimezone(self.timezone)
+
+                for job in self.schedules:
+                    await self._process_job(job, now_tz)
+
+                await sleep(1)
+
+            except CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error in scheduler loop: {e}", exc_info=True)
+                await sleep(30)
+
+        logger.info("Scheduler stopped.")
+
+    def stop(self) -> None:
+        self._running = False
+
+    async def _process_job(self, job: ScheduledJobConfig, now_tz: datetime) -> None:
+        if job.interval_seconds:
+            await self._process_interval_job(job, now_tz)
+        else:
+            await self._process_calendar_job(job, now_tz)
+
+    async def _process_interval_job(self, job: ScheduledJobConfig, now_tz: datetime) -> None:
+        last_run_key = f"scheduler:last_run:{job.name}"
+        last_run_ts = await self.storage.get_str(last_run_key)
+
+        now_ts = now_tz.timestamp()
+
+        if last_run_ts and job.interval_seconds is not None and now_ts - float(last_run_ts) < job.interval_seconds:
+            return
+
+        lock_key = f"scheduler:lock:interval:{job.name}"
+        if await self.storage.set_nx_ttl(lock_key, "locked", ttl=5):
+            try:
+                await self._trigger_job(job)
+                await self.storage.set_str(last_run_key, str(now_ts))
+            except Exception as e:
+                logger.error(f"Failed to trigger interval job {job.name}: {e}")
+
+    async def _process_calendar_job(self, job: ScheduledJobConfig, now_tz: datetime) -> None:
+        target_time_str = job.daily_at or job.time
+        if not target_time_str:
+            return
+
+        current_time_str = now_tz.strftime("%H:%M")
+
+        if current_time_str != target_time_str:
+            return
+
+        if job.weekly_days:
+            current_day_str = now_tz.strftime("%a").lower()
+            if current_day_str not in [d.lower() for d in job.weekly_days]:
+                return
+
+        if job.monthly_dates and now_tz.day not in job.monthly_dates:
+            return
+
+        date_str = now_tz.strftime("%Y-%m-%d")
+        lock_key = f"scheduler:lock:{job.name}:{date_str}"
+
+        if await self.storage.set_nx_ttl(lock_key, "locked", ttl=86400):
+            logger.info(f"Triggering scheduled job {job.name}")
+            await self._trigger_job(job)
+
+    async def _trigger_job(self, job: ScheduledJobConfig) -> None:
+        try:
+            await self.engine.create_background_job(
+                blueprint_name=job.blueprint, initial_data=job.input_data, source=f"scheduler:{job.name}"
+            )
+        except Exception as e:
+            logger.error(f"Failed to create background job {job.name}: {e}")