python-platform 0.1.1__py3-none-any.whl

python_platform/background_jobs/execution.py

@@ -0,0 +1,201 @@
+"""Durable delivery envelope 与现有 REQUEST/ACTION pipeline 的 provider-neutral bridge。"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+from uuid import uuid4
+
+from dishka import AsyncContainer, Scope
+from pydantic import BaseModel, ValidationError
+
+from ..application_services.invocation import _InvocationRuntime
+from ..authorization import CurrentUser
+from ..invocation import BackgroundSystemCaller
+from ..observability.context import invocation_span
+from .catalog import BackgroundJobCatalog
+from .contracts import BackgroundJobContext, BackgroundJobDefinition
+from .errors import BackgroundJobDefinitionError, BackgroundJobPayloadError
+
+
+class _BackgroundJobExecutor:
+    """Queue provider 只交付稳定 facts；事务、scope 与 event 顺序仍由 framework owner 执行。"""
+
+    __slots__ = ("_caller", "_catalog", "_container", "_runtime")
+
+    def __init__(
+        self,
+        container: AsyncContainer,
+        runtime: _InvocationRuntime,
+        catalog: BackgroundJobCatalog,
+        caller: BackgroundSystemCaller,
+    ) -> None:
+        self._container = container
+        self._runtime = runtime
+        self._catalog = catalog
+        self._caller = caller
+
+    async def execute(
+        self,
+        payload: bytes | None,
+        *,
+        job_id: str,
+        attempt: int,
+        scheduled_at: datetime,
+        cancellation_event: asyncio.Event,
+        connection_name: str,
+    ) -> None:
+        name, version, raw_payload, trace_context = _decode_envelope(payload)
+        definition = self._catalog.get(name, version)
+        if definition is None:
+            raise BackgroundJobDefinitionError(
+                job=name,
+                reason=f"version {version} is not registered in this Application",
+            )
+        try:
+            validated = definition.payload_type.model_validate(raw_payload)
+        except ValidationError:
+            raise BackgroundJobPayloadError(
+                job=name,
+                version=version,
+                reason="delivery payload failed schema validation",
+            ) from None
+        context = BackgroundJobContext(
+            job_id=job_id,
+            name=name,
+            version=version,
+            attempt=attempt,
+            scheduled_at=scheduled_at,
+            system_caller=self._caller,
+            _cancellation_event=cancellation_event,
+        )
+
+        async def run() -> object:
+            async with self._container(scope=Scope.REQUEST) as request_container:
+                from ..invocation.interception import _EntryInterceptionCatalog
+
+                entries = await request_container.get(_EntryInterceptionCatalog)
+                return await entries.invoke(
+                    self._runtime,
+                    request_container,
+                    definition.handler_type,
+                    "execute",
+                    (validated, context),
+                    connection_name=connection_name,
+                )
+
+        async def execute_with_ownership() -> None:
+            # lease/caller 必须由实际执行 Task 持有，且在业务及清理完成后才释放。
+            async with self._runtime.outer_lease():
+                token = self._runtime.bind_caller(CurrentUser.anonymous(), str(uuid4()))
+                try:
+                    with invocation_span(
+                        self._runtime.tracer, definition.name, carrier=trace_context
+                    ):
+                        await run()
+                finally:
+                    self._runtime.reset_caller(token)
+
+        await asyncio.wait_for(
+            execute_with_ownership(),
+            timeout=definition.timeout.total_seconds(),
+        )
+
+
+def _encode_envelope(
+    definition: BackgroundJobDefinition[BaseModel],
+    payload: BaseModel,
+    *,
+    trace_context: Mapping[str, str] | None = None,
+) -> bytes:
+    try:
+        validated = definition.payload_type.model_validate(payload)
+    except ValidationError:
+        raise BackgroundJobPayloadError(
+            job=definition.name,
+            version=definition.version,
+            reason="enqueue payload failed schema validation",
+        ) from None
+    return json.dumps(
+        {
+            "name": definition.name,
+            "version": definition.version,
+            "payload": validated.model_dump(mode="json"),
+            **({"trace_context": dict(trace_context)} if trace_context else {}),
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    ).encode()
+
+
+def _encode_scheduled_envelope(
+    definition: BackgroundJobDefinition[BaseModel],
+    canonical_payload: bytes,
+) -> bytes:
+    try:
+        payload = json.loads(canonical_payload)
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        raise AssertionError("Catalog must contain canonical UTF-8 JSON payload") from None
+    return json.dumps(
+        {
+            "name": definition.name,
+            "version": definition.version,
+            "payload": payload,
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    ).encode()
+
+
+def _decode_envelope(payload: bytes | None) -> tuple[str, int, object, Mapping[str, str]]:
+    if payload is None:
+        raise BackgroundJobPayloadError(
+            job="<unknown>",
+            version=0,
+            reason="delivery envelope is missing",
+        )
+    try:
+        candidate: Any = json.loads(payload)
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        raise BackgroundJobPayloadError(
+            job="<unknown>",
+            version=0,
+            reason="delivery envelope must be UTF-8 JSON",
+        ) from None
+    required_keys = {"name", "version", "payload"}
+    if not isinstance(candidate, dict) or not (
+        required_keys <= set(candidate) <= required_keys | {"trace_context"}
+    ):
+        raise BackgroundJobPayloadError(
+            job="<unknown>",
+            version=0,
+            reason="delivery envelope shape is invalid",
+        )
+    name = candidate["name"]
+    version = candidate["version"]
+    if not isinstance(name, str) or not name or name != name.strip():
+        raise BackgroundJobPayloadError(
+            job="<unknown>",
+            version=0,
+            reason="delivery envelope identity is invalid",
+        )
+    if not isinstance(version, int) or isinstance(version, bool) or version < 1:
+        raise BackgroundJobPayloadError(
+            job=name,
+            version=0,
+            reason="delivery envelope version is invalid",
+        )
+    trace_context = candidate.get("trace_context", {})
+    if not isinstance(trace_context, dict) or any(
+        key != "traceparent" or not isinstance(value, str) or len(value) > 512
+        for key, value in trace_context.items()
+    ):
+        raise BackgroundJobPayloadError(
+            job=name, version=version, reason="delivery trace context shape is invalid"
+        )
+    return name, version, candidate["payload"], trace_context

python_platform/background_jobs/pgqueuer/UPSTREAM_LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 janbjorge
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_platform/background_jobs/pgqueuer/__init__.py

@@ -0,0 +1,6 @@
+"""Optional PgQueuer/PostgreSQL durable background job provider。"""
+
+from .module import PgQueuerBackgroundJobsModule
+from .options import PgQueuerBackgroundJobOptions
+
+__all__ = ("PgQueuerBackgroundJobOptions", "PgQueuerBackgroundJobsModule")

python_platform/background_jobs/pgqueuer/enqueue.py

@@ -0,0 +1,82 @@
+"""Active SQLAlchemy ACTION 内的 transactional PgQueuer enqueue implementation。"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any
+
+import asyncpg  # type: ignore[import-untyped]
+from pydantic import BaseModel
+
+from ...application_services.invocation import _InvocationRuntime
+from ...observability.context import capture_trace_context
+from ...options import Options
+from ...sqlalchemy import SqlAlchemySessionProvider
+from ..catalog import BackgroundJobCatalog
+from ..contracts import BackgroundJobDefinition
+from ..errors import BackgroundJobDefinitionError
+from ..execution import _encode_envelope
+from .options import PgQueuerBackgroundJobOptions
+from .runtime import _entrypoint_name, _queries_for_connection
+
+
+class _PgQueuerBackgroundJobEnqueuer:
+    __slots__ = ("_catalog", "_options", "_provider", "_runtime")
+
+    def __init__(
+        self,
+        provider: SqlAlchemySessionProvider,
+        catalog: BackgroundJobCatalog,
+        options: Options[PgQueuerBackgroundJobOptions],
+        runtime: _InvocationRuntime,
+    ) -> None:
+        self._provider = provider
+        self._catalog = catalog
+        self._options = options.value
+        self._runtime = runtime
+
+    async def enqueue(
+        self,
+        definition: BackgroundJobDefinition[BaseModel],
+        payload: BaseModel,
+        *,
+        priority: int = 0,
+        delay: timedelta | None = None,
+    ) -> str:
+        if not self._catalog.owns(definition):
+            raise BackgroundJobDefinitionError(
+                job=getattr(definition, "name", "<unknown>"),
+                reason="definition is not owned by this Application catalog",
+            )
+        if (
+            not isinstance(priority, int)
+            or isinstance(priority, bool)
+            or not -(2**31) <= priority < 2**31
+        ):
+            raise ValueError("priority must be a PostgreSQL int32 value")
+        if delay is not None and (not isinstance(delay, timedelta) or delay.total_seconds() < 0):
+            raise ValueError("delay must be a non-negative timedelta or None")
+        work = self._provider._manager.require_current(transactional=True)
+        if work.connection_name != self._options.connection_name:
+            raise RuntimeError("transactional enqueue connection does not match current UnitOfWork")
+        session = await self._provider.get_session()
+        connection = await session.connection()
+        raw = await connection.get_raw_connection()
+        driver_connection: Any = raw.driver_connection
+        if not isinstance(driver_connection, asyncpg.Connection):
+            raise TypeError("SQLAlchemy connection does not expose an asyncpg connection")
+        queries = _queries_for_connection(
+            driver_connection,
+            self._options,
+        )
+        identifiers = await queries.enqueue(
+            _entrypoint_name(definition.name, definition.version),
+            _encode_envelope(
+                definition, payload, trace_context=capture_trace_context(self._runtime.tracer)
+            ),
+            priority,
+            delay,
+        )
+        if len(identifiers) != 1:
+            raise RuntimeError("PgQueuer enqueue did not return one job identifier")
+        return str(identifiers[0])

python_platform/background_jobs/pgqueuer/migrations/0001_pgqueuer_1_3_2.py

@@ -0,0 +1,63 @@
+"""Install the reviewed PgQueuer 1.3.2 schema-aware SQL snapshot。"""
+
+from __future__ import annotations
+
+import hashlib
+from importlib import resources
+from typing import Any
+
+from alembic import context, op
+
+revision = "python_platform_pgqueuer_0001"
+down_revision = None
+branch_labels = ("python_platform_background_jobs_pgqueuer",)
+depends_on = None
+
+_EXPECTED_SHA256 = "2c937aa4c42f780ece67df2739760e10f3dbcc6e299862e1cbf1ccdbaa0baf06"
+_SCHEMA = "python_platform_jobs"
+
+
+def upgrade() -> None:
+    sql = _install_sql()
+    _execute_driver_sql(sql)
+
+
+def downgrade() -> None:
+    config = context.get_context().config
+    if config is None:
+        raise RuntimeError("PgQueuer downgrade requires an Alembic Config")
+    if config.attributes.get("background_execution_disabled") is not True:
+        raise RuntimeError("PgQueuer downgrade requires disabled background execution")
+    active = (
+        op.get_bind()
+        .exec_driver_sql(
+            "SELECT count(*) FROM python_platform_jobs.pgqueuer "
+            "WHERE status IN ('queued', 'picked')"
+        )
+        .scalar_one()
+    )
+    if active:
+        raise RuntimeError("PgQueuer downgrade refuses queued or picked jobs")
+    op.get_bind().exec_driver_sql(f"DROP SCHEMA {_SCHEMA} CASCADE")
+
+
+def _install_sql() -> str:
+    sql = (
+        resources.files("python_platform.background_jobs.pgqueuer")
+        .joinpath("sql")
+        .joinpath("pgqueuer_1_3_2_install.sql")
+        .read_text(encoding="utf-8")
+    )
+    # Repository text files retain one POSIX terminator；fingerprint 仍绑定 upstream
+    # build_install_query() 的逐字输出，不能吞掉其有意义的尾随空格。
+    generated_sql = sql.removesuffix("\n")
+    if hashlib.sha256(generated_sql.encode()).hexdigest() != _EXPECTED_SHA256:
+        raise RuntimeError("PgQueuer install SQL fingerprint changed")
+    return generated_sql
+
+
+def _execute_driver_sql(sql: str) -> None:
+    # asyncpg 不接受 prepared multi-statement；复用 Alembic 当前 transaction 的 raw driver。
+    proxied: Any = op.get_bind().connection
+    adapted: Any = proxied.dbapi_connection
+    adapted.await_(adapted.driver_connection.execute(sql))

python_platform/background_jobs/pgqueuer/migrations/__init__.py

@@ -0,0 +1 @@
+"""PgQueuer provider Alembic revision package。"""

python_platform/background_jobs/pgqueuer/module.py

@@ -0,0 +1,102 @@
+"""Host-selected PgQueuer durable job provider Module。"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import asyncpg  # type: ignore[import-untyped]
+from dishka import Provider, Scope
+
+from ...background_jobs import BackgroundJobCatalog
+from ...background_workers.runtime import _BackgroundCoordinator
+from ...lifecycle import ConfigureContext, InitializeContext, ShutdownContext
+from ...modularity import AppModule
+from ...options import Options
+from ...sqlalchemy import (
+    SqlAlchemyMigrationRegistration,
+    SqlAlchemyPersistenceModule,
+    contributes_sqlalchemy_migrations,
+)
+from ...sqlalchemy.options import ConnectionStringsOptions
+from ..contracts import BackgroundJobEnqueuer
+from ..execution import _BackgroundJobExecutor
+from .enqueue import _PgQueuerBackgroundJobEnqueuer
+from .options import PgQueuerBackgroundJobOptions
+from .runtime import _PgQueuerRuntime
+
+
+@contributes_sqlalchemy_migrations(
+    SqlAlchemyMigrationRegistration(
+        migrations_package="python_platform.background_jobs.pgqueuer.migrations",
+        branch_label="python_platform_background_jobs_pgqueuer",
+        owned_schemas=("python_platform_jobs",),
+    )
+)
+class PgQueuerBackgroundJobsModule(AppModule):
+    dependencies = (SqlAlchemyPersistenceModule,)
+
+    def __init__(self) -> None:
+        self._runtimes: list[_PgQueuerRuntime] = []
+        self._pool: Any | None = None
+
+    def configure(self, context: ConfigureContext) -> None:
+        context.configure(PgQueuerBackgroundJobOptions, section="pgqueuer_background_jobs")
+        provider = Provider()
+        provider.provide(
+            _BackgroundJobExecutor,
+            provides=_BackgroundJobExecutor,
+            scope=Scope.APP,
+        )
+        provider.provide(
+            _PgQueuerBackgroundJobEnqueuer,
+            provides=BackgroundJobEnqueuer,
+            scope=Scope.ACTION,
+        )
+        context.services.contribute(provider, reason="PgQueuer durable background job provider")
+
+    async def initialize(self, context: InitializeContext) -> None:
+        coordinator = await context.container.get(_BackgroundCoordinator)
+        names = coordinator.local_job_names
+        if not names:
+            return
+        connections = context.options.get(ConnectionStringsOptions)
+        options = await context.container.get(Options[PgQueuerBackgroundJobOptions])
+        catalog = await context.container.get(BackgroundJobCatalog)
+        executor = await context.container.get(_BackgroundJobExecutor)
+        if options.value.executor_enabled and options.value.pool_max_size <= len(names):
+            raise ValueError("PgQueuer pool must have room for each local job listener and queries")
+        self._pool = await asyncpg.create_pool(
+            dsn=connections._url_for(options.value.connection_name).replace(
+                "postgresql+asyncpg://", "postgresql://", 1
+            ),
+            min_size=options.value.pool_min_size,
+            max_size=options.value.pool_max_size,
+        )
+        for name in names:
+            # 每个逻辑 Job 原生 manager 只注册其版本／Cron。独立 stop 不影响共享进程其他任务，
+            # 不动态改写原生 registry，不删除或抢领已暂停任务的队列。
+            runtime = _PgQueuerRuntime(
+                self._pool,
+                options,
+                catalog,
+                executor,
+                name,
+            )
+            self._runtimes.append(runtime)
+            coordinator.bind_durable(name, runtime.binding)
+            await runtime.initialize()
+
+    async def shutdown(self, context: ShutdownContext) -> None:
+        outcomes = await asyncio.gather(
+            *(runtime.close() for runtime in self._runtimes), return_exceptions=True
+        )
+        failures = [outcome for outcome in outcomes if isinstance(outcome, BaseException)]
+        if self._pool is not None:
+            try:
+                await self._pool.close()
+            except BaseException as error:
+                failures.append(error)
+            self._pool = None
+        if failures:
+            raise BaseExceptionGroup("PgQueuer cleanup failed", tuple(failures))

python_platform/background_jobs/pgqueuer/options.py

@@ -0,0 +1,70 @@
+"""PgQueuer provider options；连接字符串仍由 consuming Host 的 SQLAlchemy options 拥有。"""
+
+from __future__ import annotations
+
+import re
+from datetime import timedelta
+
+from pydantic import ConfigDict, Field, field_validator, model_validator
+
+from ...options import BaseOptions
+from ...sqlalchemy.options import _validate_connection_name
+
+_POSTGRES_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+class PgQueuerBackgroundJobOptions(BaseOptions):
+    model_config = ConfigDict(extra="forbid")
+
+    connection_name: str = "default"
+    schema_name: str = "python_platform_jobs"
+    prefix: str = ""
+    executor_enabled: bool = True
+    scheduler_enabled: bool = True
+    batch_size: int = Field(default=10, ge=1)
+    max_concurrent_jobs: int = Field(default=20, ge=2)
+    poll_interval: timedelta = Field(default=timedelta(seconds=1), gt=timedelta(0))
+    heartbeat_timeout: timedelta = Field(default=timedelta(seconds=30), gt=timedelta(0))
+    log_aggregation_interval: timedelta = Field(
+        default=timedelta(seconds=30),
+        ge=timedelta(0),
+    )
+    pool_min_size: int = Field(default=1, ge=0)
+    pool_max_size: int = Field(default=10, ge=2)
+
+    @field_validator("connection_name")
+    @classmethod
+    def _connection_name_is_valid(cls, value: str) -> str:
+        _validate_connection_name(value)
+        if value != "default":
+            raise ValueError("Phase 5 PgQueuer migration owns only the default connection")
+        return value
+
+    @field_validator("schema_name")
+    @classmethod
+    def _schema_is_valid(cls, value: str) -> str:
+        if not _POSTGRES_IDENTIFIER.fullmatch(value):
+            raise ValueError("schema_name must be one unquoted PostgreSQL identifier")
+        normalized = value.lower()
+        if normalized != "python_platform_jobs":
+            raise ValueError("Phase 5 PgQueuer migration owns only schema python_platform_jobs")
+        return normalized
+
+    @field_validator("prefix")
+    @classmethod
+    def _prefix_is_valid(cls, value: str) -> str:
+        if value and not _POSTGRES_IDENTIFIER.fullmatch(value):
+            raise ValueError("prefix must be an unquoted PostgreSQL identifier prefix")
+        if value:
+            raise ValueError("Phase 5 PgQueuer migration owns only the empty prefix")
+        return value
+
+    @model_validator(mode="after")
+    def _runtime_shape_is_valid(self) -> PgQueuerBackgroundJobOptions:
+        if not self.executor_enabled and not self.scheduler_enabled:
+            raise ValueError("executor_enabled or scheduler_enabled must be true")
+        if self.max_concurrent_jobs < self.batch_size * 2:
+            raise ValueError("max_concurrent_jobs must be at least twice batch_size")
+        if self.pool_min_size > self.pool_max_size:
+            raise ValueError("pool_min_size cannot exceed pool_max_size")
+        return self