python-platform 0.1.1__py3-none-any.whl

python_platform/background_jobs/pgqueuer/runtime.py

@@ -0,0 +1,479 @@
+"""PgQueuer 1.3 runtime ownership、registration 与 graceful recovery policy。"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import Any, cast
+
+from pgqueuer import PgQueuer, RetryRequested
+from pgqueuer.adapters.drivers.asyncpg import AsyncpgDriver, AsyncpgPoolDriver
+from pgqueuer.adapters.persistence.qb import (
+    DBSettings,
+    QueryBuilderEnvironment,
+    QueryQueueBuilder,
+    QuerySchedulerBuilder,
+)
+from pgqueuer.adapters.persistence.queries import Queries
+from pgqueuer.core.executors import (
+    DatabaseRetryEntrypointExecutor,
+    EntrypointExecutorParameters,
+)
+from pgqueuer.domain.models import CronExpressionEntrypoint
+from pgqueuer.domain.types import QueueExecutionMode
+from pgqueuer.models import Channel, Context, Job, Schedule
+from pydantic import BaseModel
+
+from ...background_execution.lifecycle import (
+    _BackgroundRuntimeFailure,
+    _TaskExecution,
+)
+from ...options import Options
+from ..catalog import BackgroundJobCatalog
+from ..contracts import BackgroundJobDefinition, BackgroundJobSchedule
+from ..errors import (
+    BackgroundJobDefinitionError,
+    BackgroundJobExecutionError,
+    BackgroundJobPayloadError,
+)
+from ..execution import _BackgroundJobExecutor, _encode_scheduled_envelope
+from .options import PgQueuerBackgroundJobOptions
+from .supervision import (
+    _PgQueuerManagerBinding,
+    _PgQueuerManagerError,
+    _PgQueuerManagerSupervisor,
+)
+
+
+class _PgQueuerRuntimeError(RuntimeError):
+    def __init__(self, *, operation: str, error_type: str) -> None:
+        self.operation = operation
+        self.error_type = error_type
+        super().__init__(f"PgQueuer {operation} failed with {error_type}")
+
+
+class _ReadinessAwareAsyncpgPoolDriver(AsyncpgPoolDriver):
+    """以 PgQueuer 已有 listener bootstrap 作为 executor ready authority。"""
+
+    def __init__(self, pool: Any, ready: asyncio.Event) -> None:
+        super().__init__(pool)
+        self._python_platform_ready = ready
+
+    async def add_listener(
+        self,
+        channel: str,
+        callback: Callable[[str | bytes | bytearray], None],
+    ) -> None:
+        await super().add_listener(channel, callback)
+        self._python_platform_ready.set()
+
+
+class _ReadinessAwareQueries(Queries):
+    """以 PgQueuer 已有 schedule insert 作为 scheduler ready authority。"""
+
+    def __init__(
+        self,
+        driver: _ReadinessAwareAsyncpgPoolDriver,
+        *,
+        qbe: QueryBuilderEnvironment,
+        qbq: QueryQueueBuilder,
+        qbs: QuerySchedulerBuilder,
+        ready: asyncio.Event,
+    ) -> None:
+        super().__init__(driver, qbe=qbe, qbq=qbq, qbs=qbs)
+        self._python_platform_ready = ready
+
+    async def insert_schedule(
+        self,
+        schedules: dict[CronExpressionEntrypoint, timedelta],
+    ) -> None:
+        await super().insert_schedule(schedules)
+        self._python_platform_ready.set()
+
+
+@dataclass
+class _ShutdownAwareRetryExecutor(DatabaseRetryEntrypointExecutor):
+    deadline_cancellation: asyncio.Event = field(default_factory=asyncio.Event)
+
+    async def execute(self, job: Job, context: Context) -> None:
+        try:
+            await super().execute(job, context)
+        except RetryRequested as retry:
+            # Envelope/version/schema failures are permanent facts；重试只会重复同一失败。
+            permanent_failure = retry.__cause__
+            if isinstance(
+                permanent_failure,
+                (BackgroundJobDefinitionError, BackgroundJobPayloadError),
+            ):
+                raise permanent_failure from None
+            raise
+        except asyncio.CancelledError:
+            if self.deadline_cancellation.is_set():
+                raise RetryRequested(
+                    delay=timedelta(0),
+                    reason="application shutdown deadline",
+                ) from None
+            raise
+
+
+class _PgQueuerRuntime:
+    __slots__ = (
+        "_active_deliveries",
+        "_catalog",
+        "_cleanup_failure",
+        "_deadline_cancellation",
+        "_executor",
+        "_executor_ready",
+        "_job_name",
+        "_options",
+        "_pgqueuer",
+        "_pool",
+        "_scheduler_ready",
+        "_stop_event",
+        "_supervisor",
+    )
+
+    def __init__(
+        self,
+        pool: Any,
+        options: Options[PgQueuerBackgroundJobOptions],
+        catalog: BackgroundJobCatalog,
+        executor: _BackgroundJobExecutor,
+        job_name: str,
+    ) -> None:
+        self._options = options.value
+        self._catalog = catalog
+        self._executor = executor
+        self._job_name = job_name
+        self._pool = pool
+        self._pgqueuer: PgQueuer | None = None
+        self._cleanup_failure: BaseException | None = None
+        self._active_deliveries: set[asyncio.Task[object]] = set()
+        self._stop_event = asyncio.Event()
+        self._deadline_cancellation = asyncio.Event()
+        self._executor_ready = asyncio.Event()
+        self._scheduler_ready = asyncio.Event()
+        self._supervisor = _PgQueuerManagerSupervisor()
+
+    @property
+    def binding(self) -> _TaskExecution:
+        return _TaskExecution(
+            start=self.start,
+            stop_claiming=self.stop_claiming,
+            wait=self.wait,
+            wait_for_failure=self.wait_for_failure,
+            cancel_for_deadline=self.cancel_for_deadline,
+        )
+
+    async def initialize(self) -> None:
+        if self._cleanup_failure is not None:
+            raise self._cleanup_failure
+        self._stop_event.clear()
+        self._deadline_cancellation.clear()
+        self._executor_ready.clear()
+        self._scheduler_ready.clear()
+        self._supervisor = _PgQueuerManagerSupervisor()
+        try:
+            queries = _queries_for_pool(
+                self._pool,
+                self._options,
+                executor_ready=self._executor_ready,
+                scheduler_ready=self._scheduler_ready,
+            )
+            settings = queries.qbe.settings
+            pgqueuer = PgQueuer(
+                connection=queries.driver,
+                channel=Channel(settings.channel),
+                queries=queries,
+            )
+            self._pgqueuer = pgqueuer
+            self._register_entrypoints(pgqueuer)
+            self._register_schedules(pgqueuer)
+            # scheduler 也会 enqueue；任一执行角色启动前都必须验证完整 queue schema。
+            await pgqueuer.qm.verify_structure()
+            if self._options.scheduler_enabled and not await queries.has_table(
+                settings.schedules_table
+            ):
+                raise RuntimeError("required PgQueuer schedule table is missing")
+        except BaseException as original_error:
+            await self._cleanup_failed_start(original_error)
+            if not isinstance(original_error, Exception):
+                raise
+            raise _PgQueuerRuntimeError(
+                operation="startup verification",
+                error_type=type(original_error).__name__,
+            ) from original_error
+
+    async def start(self) -> None:
+        if self._cleanup_failure is not None:
+            raise self._cleanup_failure
+        if self._pgqueuer is None:
+            await self.initialize()
+        pgqueuer = self._require_pgqueuer()
+        bindings: list[_PgQueuerManagerBinding] = []
+        if self._options.executor_enabled:
+
+            async def run_executor() -> None:
+                await pgqueuer.qm.run(
+                    dequeue_timeout=self._options.poll_interval,
+                    batch_size=self._options.batch_size,
+                    mode=QueueExecutionMode.continuous,
+                    max_concurrent_tasks=self._options.max_concurrent_jobs,
+                    heartbeat_timeout=self._options.heartbeat_timeout,
+                    log_aggregation_interval=self._options.log_aggregation_interval,
+                    shutdown_on_listener_failure=True,
+                )
+
+            bindings.append(
+                _PgQueuerManagerBinding(
+                    component="pgqueuer-executor",
+                    task_name="python-platform-pgqueuer-executor",
+                    ready=self._executor_ready,
+                    run=run_executor,
+                )
+            )
+        if self._options.scheduler_enabled:
+
+            async def run_scheduler() -> None:
+                await pgqueuer.sm.run()
+
+            bindings.append(
+                _PgQueuerManagerBinding(
+                    component="pgqueuer-scheduler",
+                    task_name="python-platform-pgqueuer-scheduler",
+                    ready=self._scheduler_ready,
+                    run=run_scheduler,
+                )
+            )
+        try:
+            await self._supervisor.start(tuple(bindings))
+        except BaseException as original_error:
+            await self._cleanup_failed_start(original_error)
+            if not isinstance(original_error, _PgQueuerManagerError):
+                raise
+            raise _PgQueuerRuntimeError(
+                operation=f"{original_error.component} bootstrap",
+                error_type=original_error.error_type,
+            ) from original_error
+
+    def stop_claiming(self) -> None:
+        self._supervisor.begin_stop()
+        self._stop_event.set()
+        pgqueuer = self._pgqueuer
+        if pgqueuer is not None:
+            pgqueuer.shutdown.set()
+
+    async def wait(self) -> None:
+        try:
+            await self._supervisor.wait()
+        finally:
+            await self.close()
+
+    async def wait_for_failure(self) -> _BackgroundRuntimeFailure:
+        failure = await self._supervisor.wait_for_failure()
+        return _BackgroundRuntimeFailure(
+            component=failure.component,
+            error_type=failure.error_type,
+            _source=failure._source,
+        )
+
+    def cancel_for_deadline(self) -> None:
+        self._deadline_cancellation.set()
+        for task in tuple(self._active_deliveries):
+            task.cancel()
+        self._supervisor.cancel()
+
+    async def close(self) -> None:
+        self.stop_claiming()
+        try:
+            await self._supervisor.close()
+        except BaseException as error:
+            if self._cleanup_failure is None:
+                self._cleanup_failure = error
+        if self._cleanup_failure is not None:
+            raise self._cleanup_failure
+        self._pgqueuer = None
+
+    async def _cleanup_failed_start(self, original_error: BaseException) -> None:
+        try:
+            await self.close()
+        except BaseException as cleanup_error:
+            # 只有本逻辑 Job 的所有 manager 成功收口才能替换实例；共享 pool 仍由 Module 拥有。
+            self._cleanup_failure = BaseExceptionGroup(
+                "PgQueuer startup and cleanup failed", (original_error, cleanup_error)
+            )
+            raise self._cleanup_failure from None
+
+    def _register_entrypoints(self, pgqueuer: PgQueuer) -> None:
+        for definition in self._catalog.definitions:
+            if definition.name != self._job_name:
+                continue
+            executor_factory = self._executor_factory(definition)
+
+            @pgqueuer.entrypoint(
+                _entrypoint_name(definition.name, definition.version),
+                on_failure="hold",
+                executor_factory=executor_factory,
+            )
+            async def execute(
+                job: Job,
+                *,
+                _definition: BackgroundJobDefinition[BaseModel] = definition,
+                _runtime: _PgQueuerRuntime = self,
+            ) -> None:
+                await _runtime._execute_job(job, _definition)
+
+    def _register_schedules(self, pgqueuer: PgQueuer) -> None:
+        if not self._options.scheduler_enabled:
+            return
+        for schedule in self._catalog.schedules:
+            if schedule.job_name != self._job_name:
+                continue
+            definition = self._catalog.get(schedule.job_name, schedule.job_version)
+            assert definition is not None
+            envelope = _encode_scheduled_envelope(definition, schedule.payload)
+            target_entrypoint = _entrypoint_name(definition.name, definition.version)
+
+            @pgqueuer.schedule(
+                _schedule_entrypoint_name(schedule),
+                schedule.cron,
+                clean_old=False,
+            )
+            async def enqueue_scheduled(
+                _schedule: Schedule,
+                *,
+                _envelope: bytes = envelope,
+                _entrypoint: str = target_entrypoint,
+                _runtime: _PgQueuerRuntime = self,
+            ) -> None:
+                queries = _runtime._require_queries()
+                await queries.enqueue(_entrypoint, _envelope)
+
+    def _executor_factory(
+        self,
+        definition: BackgroundJobDefinition[BaseModel],
+    ) -> Callable[[EntrypointExecutorParameters], _ShutdownAwareRetryExecutor]:
+        policy = definition.retry_policy
+
+        def create(parameters: EntrypointExecutorParameters) -> _ShutdownAwareRetryExecutor:
+            return _ShutdownAwareRetryExecutor(
+                parameters=parameters,
+                max_attempts=policy.max_attempts,
+                initial_delay=policy.initial_delay,
+                max_delay=policy.max_delay,
+                backoff_multiplier=policy.backoff_multiplier,
+                deadline_cancellation=self._deadline_cancellation,
+            )
+
+        return create
+
+    async def _execute_job(
+        self,
+        job: Job,
+        definition: BackgroundJobDefinition[BaseModel],
+    ) -> None:
+        task = asyncio.current_task()
+        if task is None:
+            raise RuntimeError("PgQueuer delivery requires an asyncio Task")
+        owned_task = cast(asyncio.Task[object], task)
+        self._active_deliveries.add(owned_task)
+        try:
+            execution_error: BackgroundJobExecutionError | None = None
+            try:
+                await self._executor.execute(
+                    job.payload,
+                    job_id=str(job.id),
+                    attempt=job.attempts,
+                    scheduled_at=job.execute_after,
+                    cancellation_event=self._stop_event,
+                    connection_name=self._options.connection_name,
+                )
+            except (
+                BackgroundJobDefinitionError,
+                BackgroundJobExecutionError,
+                BackgroundJobPayloadError,
+            ):
+                raise
+            except Exception as original_error:
+                # PgQueuer 会持久化 traceback/reason；边界前只允许结构化类型事实离开 handler。
+                execution_error = BackgroundJobExecutionError(
+                    job=definition.name,
+                    version=definition.version,
+                    error_type=type(original_error).__name__,
+                )
+            if execution_error is not None:
+                # 离开 except suite 后再抛出，确保 wrapper 连隐式 __context__ 也不保留。
+                raise execution_error from None
+        finally:
+            self._active_deliveries.discard(owned_task)
+
+    def _require_pgqueuer(self) -> PgQueuer:
+        if self._pgqueuer is None:
+            raise RuntimeError("PgQueuer runtime is not initialized")
+        return self._pgqueuer
+
+    def _require_queries(self) -> Queries:
+        pgqueuer = self._require_pgqueuer()
+        queries = pgqueuer.queries
+        if not isinstance(queries, Queries):
+            raise RuntimeError("PgQueuer runtime does not own Queries")
+        return queries
+
+
+def _entrypoint_name(name: str, version: int) -> str:
+    return f"{name}.v{version}"
+
+
+def _schedule_entrypoint_name(schedule: BackgroundJobSchedule) -> str:
+    return f"schedule.{schedule.name}"
+
+
+def _database_settings(options: PgQueuerBackgroundJobOptions) -> DBSettings:
+    prefix = options.prefix
+    # 每个字段都显式传入，防止 PgQueuer BaseSettings 从 process environment 建立第二权威。
+    return DBSettings(
+        prefix=prefix,
+        db_schema=options.schema_name,
+        channel=f"{prefix}ch_pgqueuer",
+        function=f"{prefix}fn_pgqueuer_changed",
+        statistics_table=f"{prefix}pgqueuer_statistics",
+        queue_status_type=f"{prefix}pgqueuer_status",
+        queue_table=f"{prefix}pgqueuer",
+        queue_table_log=f"{prefix}pgqueuer_log",
+        trigger=f"{prefix}tg_pgqueuer_changed",
+        schedules_table=f"{prefix}pgqueuer_schedules",
+        durability="durable",
+        widen_id=True,
+    )
+
+
+def _queries_for_pool(
+    pool: Any,
+    options: PgQueuerBackgroundJobOptions,
+    *,
+    executor_ready: asyncio.Event,
+    scheduler_ready: asyncio.Event,
+) -> Queries:
+    settings = _database_settings(options)
+    return _ReadinessAwareQueries(
+        _ReadinessAwareAsyncpgPoolDriver(pool, executor_ready),
+        qbe=QueryBuilderEnvironment(settings=settings),
+        qbq=QueryQueueBuilder(settings=settings),
+        qbs=QuerySchedulerBuilder(settings=settings),
+        ready=scheduler_ready,
+    )
+
+
+def _queries_for_connection(
+    connection: Any,
+    options: PgQueuerBackgroundJobOptions,
+) -> Queries:
+    settings = _database_settings(options)
+    return Queries(
+        AsyncpgDriver(connection),
+        qbe=QueryBuilderEnvironment(settings=settings),
+        qbq=QueryQueueBuilder(settings=settings),
+        qbs=QuerySchedulerBuilder(settings=settings),
+    )

python_platform/background_jobs/pgqueuer/sql/pgqueuer_1_3_2_install.sql

@@ -0,0 +1,118 @@
+CREATE SCHEMA IF NOT EXISTS python_platform_jobs;
+    CREATE TYPE python_platform_jobs.pgqueuer_status AS ENUM ('queued', 'picked', 'successful', 'exception', 'canceled', 'deleted', 'failed');
+    CREATE  TABLE python_platform_jobs.pgqueuer (
+        id BIGSERIAL PRIMARY KEY,
+        priority INT NOT NULL,
+        queue_manager_id UUID,
+        created TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        updated TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        heartbeat TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        execute_after TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        status python_platform_jobs.pgqueuer_status NOT NULL,
+        entrypoint TEXT NOT NULL,
+        dedupe_key TEXT,
+        payload BYTEA,
+        headers JSONB,
+        attempts INT NOT NULL DEFAULT 0
+    );
+    CREATE INDEX pgqueuer_priority_id_id1_idx ON python_platform_jobs.pgqueuer (priority ASC, id DESC)
+        INCLUDE (id) WHERE status = 'queued';
+    CREATE INDEX pgqueuer_updated_id_id1_idx ON python_platform_jobs.pgqueuer (updated ASC, id DESC)
+        INCLUDE (id) WHERE status = 'picked';
+    CREATE INDEX pgqueuer_queue_manager_id_idx ON python_platform_jobs.pgqueuer (queue_manager_id)
+        WHERE queue_manager_id IS NOT NULL;
+    CREATE INDEX pgqueuer_ep_prio_id_idx ON python_platform_jobs.pgqueuer (entrypoint, priority DESC, id ASC)
+        WHERE status = 'queued';
+    CREATE INDEX pgqueuer_ep_ea_idx ON python_platform_jobs.pgqueuer (entrypoint, execute_after)
+        WHERE status = 'queued';
+    CREATE UNIQUE INDEX IF NOT EXISTS pgqueuer_unique_dedupe_key ON
+        python_platform_jobs.pgqueuer (dedupe_key) WHERE ((status IN ('queued', 'picked') AND dedupe_key IS NOT NULL));
+
+    CREATE  TABLE python_platform_jobs.pgqueuer_log (
+        id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+        created TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        job_id BIGINT NOT NULL,
+        status python_platform_jobs.pgqueuer_status NOT NULL,
+        priority INT NOT NULL,
+        entrypoint TEXT NOT NULL,
+        traceback JSONB DEFAULT NULL,
+        aggregated BOOLEAN DEFAULT FALSE
+    );
+    CREATE INDEX pgqueuer_log_not_aggregated ON python_platform_jobs.pgqueuer_log ((1)) WHERE not aggregated;
+    CREATE INDEX pgqueuer_log_created ON python_platform_jobs.pgqueuer_log (created);
+    CREATE INDEX pgqueuer_log_status ON python_platform_jobs.pgqueuer_log (status);
+    CREATE INDEX pgqueuer_log_job_id_status ON python_platform_jobs.pgqueuer_log (job_id, created DESC);
+
+    CREATE  TABLE python_platform_jobs.pgqueuer_statistics (
+        id BIGSERIAL PRIMARY KEY,
+        created TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT DATE_TRUNC('sec', NOW() at time zone 'UTC'),
+        count BIGINT NOT NULL,
+        priority INT NOT NULL,
+        status python_platform_jobs.pgqueuer_status NOT NULL,
+        entrypoint TEXT NOT NULL
+    );
+    CREATE UNIQUE INDEX pgqueuer_statistics_unique_count ON python_platform_jobs.pgqueuer_statistics (
+        priority,
+        DATE_TRUNC('sec', created at time zone 'UTC'),
+        status,
+        entrypoint
+    );
+
+    CREATE  TABLE python_platform_jobs.pgqueuer_schedules (
+        id BIGSERIAL PRIMARY KEY,
+        expression TEXT NOT NULL, -- Crontab-like schedule definition (e.g., '* * * * *')
+        entrypoint TEXT NOT NULL,
+        heartbeat TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        created TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        updated TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        next_run TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
+        last_run TIMESTAMP WITH TIME ZONE,
+        status python_platform_jobs.pgqueuer_status DEFAULT 'queued',
+        UNIQUE (expression, entrypoint)
+    );
+
+    CREATE FUNCTION python_platform_jobs.fn_pgqueuer_changed() RETURNS TRIGGER AS $$
+    DECLARE
+        to_emit BOOLEAN := false;  -- Flag to decide whether to emit a notification
+    BEGIN
+        -- Check operation type and set the emit flag accordingly
+        IF TG_OP = 'UPDATE' AND OLD IS DISTINCT FROM NEW THEN
+            to_emit := true;
+        ELSIF TG_OP = 'DELETE' THEN
+            to_emit := true;
+        ELSIF TG_OP = 'INSERT' THEN
+            to_emit := true;
+        ELSIF TG_OP = 'TRUNCATE' THEN
+            to_emit := true;
+        END IF;
+
+        -- Perform notification if the emit flag is set
+        IF to_emit THEN
+            PERFORM pg_notify(
+                'ch_pgqueuer',
+                json_build_object(
+                    'channel', 'ch_pgqueuer',
+                    'operation', lower(TG_OP),
+                    'sent_at', NOW(),
+                    'table', TG_TABLE_NAME,
+                    'type', 'table_changed_event'
+                )::text
+            );
+        END IF;
+
+        -- Return appropriate value based on the operation
+        IF TG_OP IN ('INSERT', 'UPDATE') THEN
+            RETURN NEW;
+        ELSIF TG_OP = 'DELETE' THEN
+            RETURN OLD;
+        ELSE
+            RETURN NULL; -- For TRUNCATE and other non-row-specific contexts
+        END IF;
+
+    END;
+    $$ LANGUAGE plpgsql;
+
+    CREATE TRIGGER tg_pgqueuer_changed
+    AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON python_platform_jobs.pgqueuer
+    EXECUTE FUNCTION python_platform_jobs.fn_pgqueuer_changed();
+