pgrelay 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alembic.ini +37 -0
- migrations/env.py +61 -0
- migrations/versions/0001_initial_schema.py +147 -0
- pgrelay/__init__.py +3 -0
- pgrelay/__main__.py +12 -0
- pgrelay/api/__init__.py +1 -0
- pgrelay/api/app.py +98 -0
- pgrelay/api/dependencies.py +62 -0
- pgrelay/api/routers/__init__.py +1 -0
- pgrelay/api/routers/health.py +28 -0
- pgrelay/api/routers/jobs.py +107 -0
- pgrelay/api/routers/queues.py +52 -0
- pgrelay/api/routers/stats.py +22 -0
- pgrelay/api/routers/workers.py +21 -0
- pgrelay/cli/__init__.py +1 -0
- pgrelay/cli/app.py +18 -0
- pgrelay/cli/commands_api.py +15 -0
- pgrelay/cli/commands_doctor.py +76 -0
- pgrelay/cli/commands_jobs.py +106 -0
- pgrelay/cli/commands_migrate.py +24 -0
- pgrelay/cli/commands_worker.py +29 -0
- pgrelay/config/__init__.py +1 -0
- pgrelay/config/settings.py +111 -0
- pgrelay/constants.py +42 -0
- pgrelay/db/__init__.py +1 -0
- pgrelay/db/base.py +7 -0
- pgrelay/db/migrations.py +37 -0
- pgrelay/db/models.py +147 -0
- pgrelay/db/session.py +43 -0
- pgrelay/errors.py +70 -0
- pgrelay/observability/__init__.py +1 -0
- pgrelay/observability/logging.py +32 -0
- pgrelay/observability/metrics.py +16 -0
- pgrelay/py.typed +1 -0
- pgrelay/repositories/__init__.py +1 -0
- pgrelay/repositories/attempts.py +63 -0
- pgrelay/repositories/job_rows.py +125 -0
- pgrelay/repositories/job_sql.py +205 -0
- pgrelay/repositories/job_state.py +203 -0
- pgrelay/repositories/jobs.py +200 -0
- pgrelay/repositories/protocols.py +175 -0
- pgrelay/repositories/queues.py +111 -0
- pgrelay/repositories/stats.py +161 -0
- pgrelay/repositories/workers.py +89 -0
- pgrelay/schemas/__init__.py +1 -0
- pgrelay/schemas/api_errors.py +19 -0
- pgrelay/schemas/enqueue.py +52 -0
- pgrelay/schemas/jobs.py +119 -0
- pgrelay/schemas/queues.py +24 -0
- pgrelay/schemas/stats.py +20 -0
- pgrelay/schemas/workers.py +18 -0
- pgrelay/sdk/__init__.py +1 -0
- pgrelay/sdk/client.py +107 -0
- pgrelay/sdk/result.py +15 -0
- pgrelay/security/__init__.py +1 -0
- pgrelay/security/auth.py +51 -0
- pgrelay/services/__init__.py +1 -0
- pgrelay/services/enqueue.py +78 -0
- pgrelay/services/jobs.py +105 -0
- pgrelay/services/purge.py +56 -0
- pgrelay/services/queues.py +54 -0
- pgrelay/services/stats.py +28 -0
- pgrelay/services/workers.py +19 -0
- pgrelay/utils/__init__.py +1 -0
- pgrelay/utils/ids.py +16 -0
- pgrelay/utils/json.py +20 -0
- pgrelay/utils/redaction.py +35 -0
- pgrelay/utils/validation.py +85 -0
- pgrelay/worker/__init__.py +1 -0
- pgrelay/worker/backoff.py +14 -0
- pgrelay/worker/dispatcher.py +51 -0
- pgrelay/worker/handlers.py +46 -0
- pgrelay/worker/heartbeat.py +35 -0
- pgrelay/worker/http_executor.py +220 -0
- pgrelay/worker/python_executor.py +55 -0
- pgrelay/worker/recovery.py +19 -0
- pgrelay/worker/runner.py +316 -0
- pgrelay/worker/signals.py +13 -0
- pgrelay-0.1.0.dist-info/METADATA +254 -0
- pgrelay-0.1.0.dist-info/RECORD +83 -0
- pgrelay-0.1.0.dist-info/WHEEL +4 -0
- pgrelay-0.1.0.dist-info/entry_points.txt +3 -0
- pgrelay-0.1.0.dist-info/licenses/LICENSE +21 -0
alembic.ini
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[alembic]
|
|
2
|
+
script_location = migrations
|
|
3
|
+
version_table = pgrelay_alembic_version
|
|
4
|
+
prepend_sys_path = .
|
|
5
|
+
|
|
6
|
+
[loggers]
|
|
7
|
+
keys = root,sqlalchemy,alembic
|
|
8
|
+
|
|
9
|
+
[handlers]
|
|
10
|
+
keys = console
|
|
11
|
+
|
|
12
|
+
[formatters]
|
|
13
|
+
keys = generic
|
|
14
|
+
|
|
15
|
+
[logger_root]
|
|
16
|
+
level = WARN
|
|
17
|
+
handlers = console
|
|
18
|
+
qualname =
|
|
19
|
+
|
|
20
|
+
[logger_sqlalchemy]
|
|
21
|
+
level = WARN
|
|
22
|
+
handlers =
|
|
23
|
+
qualname = sqlalchemy.engine
|
|
24
|
+
|
|
25
|
+
[logger_alembic]
|
|
26
|
+
level = INFO
|
|
27
|
+
handlers =
|
|
28
|
+
qualname = alembic
|
|
29
|
+
|
|
30
|
+
[handler_console]
|
|
31
|
+
class = StreamHandler
|
|
32
|
+
args = (sys.stderr,)
|
|
33
|
+
level = NOTSET
|
|
34
|
+
formatter = generic
|
|
35
|
+
|
|
36
|
+
[formatter_generic]
|
|
37
|
+
format = %(levelname)-5.5s [%(name)s] %(message)s
|
migrations/env.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Alembic environment for PgRelay."""
|
|
2
|
+
|
|
3
|
+
from logging.config import fileConfig
|
|
4
|
+
|
|
5
|
+
from alembic import context
|
|
6
|
+
from sqlalchemy import engine_from_config, pool
|
|
7
|
+
from sqlalchemy.engine import make_url
|
|
8
|
+
|
|
9
|
+
from pgrelay.config.settings import load_settings
|
|
10
|
+
from pgrelay.db import models
|
|
11
|
+
from pgrelay.db.base import Base
|
|
12
|
+
|
|
13
|
+
config = context.config
|
|
14
|
+
|
|
15
|
+
if config.config_file_name is not None:
|
|
16
|
+
fileConfig(config.config_file_name)
|
|
17
|
+
|
|
18
|
+
database_url = config.get_main_option("sqlalchemy.url") or load_settings().database_url
|
|
19
|
+
url = make_url(database_url).set(drivername="postgresql+psycopg")
|
|
20
|
+
config.set_main_option("sqlalchemy.url", url.render_as_string(hide_password=False))
|
|
21
|
+
target_metadata = Base.metadata
|
|
22
|
+
loaded_models = (models.PgRelayAttempt, models.PgRelayJob, models.PgRelayQueue, models.PgRelayWorker)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def run_migrations_offline() -> None:
|
|
26
|
+
"""Run migrations without creating an engine."""
|
|
27
|
+
context.configure(
|
|
28
|
+
url=url.render_as_string(hide_password=False),
|
|
29
|
+
target_metadata=target_metadata,
|
|
30
|
+
literal_binds=True,
|
|
31
|
+
dialect_opts={"paramstyle": "named"},
|
|
32
|
+
version_table="pgrelay_alembic_version",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
with context.begin_transaction():
|
|
36
|
+
context.run_migrations()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def run_migrations_online() -> None:
|
|
40
|
+
"""Run migrations with a sync Alembic engine."""
|
|
41
|
+
connectable = engine_from_config(
|
|
42
|
+
config.get_section(config.config_ini_section, {}),
|
|
43
|
+
prefix="sqlalchemy.",
|
|
44
|
+
poolclass=pool.NullPool,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
with connectable.connect() as connection:
|
|
48
|
+
context.configure(
|
|
49
|
+
connection=connection,
|
|
50
|
+
target_metadata=target_metadata,
|
|
51
|
+
version_table="pgrelay_alembic_version",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
with context.begin_transaction():
|
|
55
|
+
context.run_migrations()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if context.is_offline_mode():
|
|
59
|
+
run_migrations_offline()
|
|
60
|
+
else:
|
|
61
|
+
run_migrations_online()
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Initial PgRelay schema."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
|
|
5
|
+
import sqlalchemy as sa
|
|
6
|
+
from alembic import op
|
|
7
|
+
from sqlalchemy.dialects import postgresql
|
|
8
|
+
|
|
9
|
+
revision: str = "0001_initial_schema"
|
|
10
|
+
down_revision: str | None = None
|
|
11
|
+
branch_labels: str | Sequence[str] | None = None
|
|
12
|
+
depends_on: str | Sequence[str] | None = None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def upgrade() -> None:
|
|
16
|
+
"""Create PgRelay tables and indexes."""
|
|
17
|
+
op.create_table(
|
|
18
|
+
"pgrelay_queue",
|
|
19
|
+
sa.Column("name", sa.Text(), primary_key=True),
|
|
20
|
+
sa.Column("paused", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
|
21
|
+
sa.Column("concurrency_limit", sa.Integer(), nullable=False, server_default=sa.text("8")),
|
|
22
|
+
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
23
|
+
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
24
|
+
sa.CheckConstraint("concurrency_limit > 0", name="ck_pgrelay_queue_concurrency_limit"),
|
|
25
|
+
)
|
|
26
|
+
op.create_table(
|
|
27
|
+
"pgrelay_job",
|
|
28
|
+
sa.Column("id", postgresql.UUID(), primary_key=True, server_default=sa.text("gen_random_uuid()")),
|
|
29
|
+
sa.Column("queue_name", sa.Text(), sa.ForeignKey("pgrelay_queue.name"), nullable=False),
|
|
30
|
+
sa.Column("kind", sa.Text(), nullable=False),
|
|
31
|
+
sa.Column("name", sa.Text(), nullable=False),
|
|
32
|
+
sa.Column("payload", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")),
|
|
33
|
+
sa.Column("headers", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")),
|
|
34
|
+
sa.Column("metadata", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")),
|
|
35
|
+
sa.Column("status", sa.Text(), nullable=False, server_default=sa.text("'pending'")),
|
|
36
|
+
sa.Column("priority", sa.Integer(), nullable=False, server_default=sa.text("0")),
|
|
37
|
+
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default=sa.text("10")),
|
|
38
|
+
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default=sa.text("0")),
|
|
39
|
+
sa.Column("available_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
40
|
+
sa.Column("timeout_seconds", sa.Integer(), nullable=False, server_default=sa.text("30")),
|
|
41
|
+
sa.Column("idempotency_key", sa.Text(), nullable=True),
|
|
42
|
+
sa.Column("dedupe_key", sa.Text(), nullable=True),
|
|
43
|
+
sa.Column(
|
|
44
|
+
"replayed_from_job_id",
|
|
45
|
+
postgresql.UUID(),
|
|
46
|
+
sa.ForeignKey("pgrelay_job.id", ondelete="SET NULL"),
|
|
47
|
+
nullable=True,
|
|
48
|
+
),
|
|
49
|
+
sa.Column("locked_by", sa.Text(), nullable=True),
|
|
50
|
+
sa.Column("locked_until", sa.DateTime(timezone=True), nullable=True),
|
|
51
|
+
sa.Column("last_error_type", sa.Text(), nullable=True),
|
|
52
|
+
sa.Column("last_error_message", sa.Text(), nullable=True),
|
|
53
|
+
sa.Column("last_response_status", sa.Integer(), nullable=True),
|
|
54
|
+
sa.Column("trace_id", sa.Text(), nullable=True),
|
|
55
|
+
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
56
|
+
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
57
|
+
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
|
|
58
|
+
sa.CheckConstraint("kind IN ('http', 'handler')", name="ck_pgrelay_job_kind"),
|
|
59
|
+
sa.CheckConstraint(
|
|
60
|
+
"status IN ('pending', 'leased', 'succeeded', 'dead_letter', 'cancelled')",
|
|
61
|
+
name="ck_pgrelay_job_status",
|
|
62
|
+
),
|
|
63
|
+
sa.CheckConstraint("max_attempts > 0", name="ck_pgrelay_job_max_attempts"),
|
|
64
|
+
)
|
|
65
|
+
op.create_table(
|
|
66
|
+
"pgrelay_attempt",
|
|
67
|
+
sa.Column("id", postgresql.UUID(), primary_key=True, server_default=sa.text("gen_random_uuid()")),
|
|
68
|
+
sa.Column(
|
|
69
|
+
"job_id",
|
|
70
|
+
postgresql.UUID(),
|
|
71
|
+
sa.ForeignKey("pgrelay_job.id", ondelete="CASCADE"),
|
|
72
|
+
nullable=False,
|
|
73
|
+
),
|
|
74
|
+
sa.Column("attempt_number", sa.Integer(), nullable=False),
|
|
75
|
+
sa.Column("worker_id", sa.Text(), nullable=False),
|
|
76
|
+
sa.Column("status", sa.Text(), nullable=False),
|
|
77
|
+
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
|
78
|
+
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
79
|
+
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default=sa.text("0")),
|
|
80
|
+
sa.Column("error_type", sa.Text(), nullable=True),
|
|
81
|
+
sa.Column("error_message", sa.Text(), nullable=True),
|
|
82
|
+
sa.Column("response_status", sa.Integer(), nullable=True),
|
|
83
|
+
sa.Column("response_body_preview", sa.String(length=2000), nullable=True),
|
|
84
|
+
sa.CheckConstraint("attempt_number > 0", name="ck_pgrelay_attempt_number"),
|
|
85
|
+
sa.CheckConstraint(
|
|
86
|
+
"status IN ('succeeded', 'failed', 'timeout', 'lease_expired')",
|
|
87
|
+
name="ck_pgrelay_attempt_status",
|
|
88
|
+
),
|
|
89
|
+
)
|
|
90
|
+
op.create_table(
|
|
91
|
+
"pgrelay_worker",
|
|
92
|
+
sa.Column("worker_id", sa.Text(), primary_key=True),
|
|
93
|
+
sa.Column("queues", postgresql.ARRAY(sa.Text()), nullable=False),
|
|
94
|
+
sa.Column("hostname", sa.Text(), nullable=False),
|
|
95
|
+
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
96
|
+
sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
|
|
97
|
+
)
|
|
98
|
+
op.create_index(
|
|
99
|
+
"idx_pgrelay_job_claim",
|
|
100
|
+
"pgrelay_job",
|
|
101
|
+
["queue_name", sa.literal_column("priority DESC"), "available_at", "created_at"],
|
|
102
|
+
postgresql_where=sa.text("status = 'pending'"),
|
|
103
|
+
)
|
|
104
|
+
op.create_index(
|
|
105
|
+
"idx_pgrelay_job_locked_expired",
|
|
106
|
+
"pgrelay_job",
|
|
107
|
+
["locked_until"],
|
|
108
|
+
postgresql_where=sa.text("status = 'leased'"),
|
|
109
|
+
)
|
|
110
|
+
op.create_index("idx_pgrelay_job_status_created", "pgrelay_job", ["status", "created_at"])
|
|
111
|
+
op.create_index(
|
|
112
|
+
"idx_pgrelay_job_dedupe_key",
|
|
113
|
+
"pgrelay_job",
|
|
114
|
+
["dedupe_key"],
|
|
115
|
+
postgresql_where=sa.text("dedupe_key IS NOT NULL"),
|
|
116
|
+
)
|
|
117
|
+
op.create_index(
|
|
118
|
+
"uq_pgrelay_job_queue_idempotency_key",
|
|
119
|
+
"pgrelay_job",
|
|
120
|
+
["queue_name", "idempotency_key"],
|
|
121
|
+
unique=True,
|
|
122
|
+
postgresql_where=sa.text("idempotency_key IS NOT NULL"),
|
|
123
|
+
)
|
|
124
|
+
op.create_index(
|
|
125
|
+
"idx_pgrelay_job_replayed_from",
|
|
126
|
+
"pgrelay_job",
|
|
127
|
+
["replayed_from_job_id"],
|
|
128
|
+
postgresql_where=sa.text("replayed_from_job_id IS NOT NULL"),
|
|
129
|
+
)
|
|
130
|
+
op.create_index("idx_pgrelay_attempt_job_started", "pgrelay_attempt", ["job_id", "started_at"])
|
|
131
|
+
op.create_index("idx_pgrelay_worker_last_heartbeat", "pgrelay_worker", ["last_heartbeat_at"])
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def downgrade() -> None:
|
|
135
|
+
"""Drop PgRelay tables and indexes."""
|
|
136
|
+
op.drop_index("idx_pgrelay_worker_last_heartbeat", table_name="pgrelay_worker")
|
|
137
|
+
op.drop_index("idx_pgrelay_attempt_job_started", table_name="pgrelay_attempt")
|
|
138
|
+
op.drop_index("idx_pgrelay_job_replayed_from", table_name="pgrelay_job")
|
|
139
|
+
op.drop_index("uq_pgrelay_job_queue_idempotency_key", table_name="pgrelay_job")
|
|
140
|
+
op.drop_index("idx_pgrelay_job_dedupe_key", table_name="pgrelay_job")
|
|
141
|
+
op.drop_index("idx_pgrelay_job_status_created", table_name="pgrelay_job")
|
|
142
|
+
op.drop_index("idx_pgrelay_job_locked_expired", table_name="pgrelay_job")
|
|
143
|
+
op.drop_index("idx_pgrelay_job_claim", table_name="pgrelay_job")
|
|
144
|
+
op.drop_table("pgrelay_worker")
|
|
145
|
+
op.drop_table("pgrelay_attempt")
|
|
146
|
+
op.drop_table("pgrelay_job")
|
|
147
|
+
op.drop_table("pgrelay_queue")
|
pgrelay/__init__.py
ADDED
pgrelay/__main__.py
ADDED
pgrelay/api/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""API package."""
|
pgrelay/api/app.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""FastAPI application factory."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncGenerator
|
|
4
|
+
from contextlib import asynccontextmanager
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from fastapi import FastAPI, HTTPException, Request
|
|
8
|
+
from fastapi.encoders import jsonable_encoder
|
|
9
|
+
from fastapi.exceptions import RequestValidationError
|
|
10
|
+
from fastapi.responses import JSONResponse
|
|
11
|
+
|
|
12
|
+
from pgrelay.api.routers import health, jobs, queues, stats, workers
|
|
13
|
+
from pgrelay.config.settings import Settings, load_settings
|
|
14
|
+
from pgrelay.db.session import create_engine, create_session_factory
|
|
15
|
+
from pgrelay.errors import ERROR_STATUS_CODES, PgRelayError
|
|
16
|
+
from pgrelay.observability.logging import setup_logging
|
|
17
|
+
from pgrelay.repositories.attempts import AttemptRepository
|
|
18
|
+
from pgrelay.repositories.jobs import JobRepository
|
|
19
|
+
from pgrelay.repositories.queues import QueueRepository
|
|
20
|
+
from pgrelay.repositories.stats import StatsRepository
|
|
21
|
+
from pgrelay.repositories.workers import WorkerRepository
|
|
22
|
+
from pgrelay.schemas.api_errors import ApiError, ApiErrorResponse
|
|
23
|
+
from pgrelay.services.enqueue import EnqueueService
|
|
24
|
+
from pgrelay.services.jobs import JobService
|
|
25
|
+
from pgrelay.services.queues import QueueService
|
|
26
|
+
from pgrelay.services.stats import StatsService
|
|
27
|
+
from pgrelay.services.workers import WorkerService
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def create_app(settings: Settings | None = None) -> FastAPI:
|
|
31
|
+
"""Create a configured FastAPI app."""
|
|
32
|
+
|
|
33
|
+
@asynccontextmanager
|
|
34
|
+
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|
35
|
+
runtime_settings = settings or load_settings()
|
|
36
|
+
runtime_settings.validate_runtime()
|
|
37
|
+
setup_logging(runtime_settings.log_level)
|
|
38
|
+
engine = create_engine(runtime_settings)
|
|
39
|
+
session_factory = create_session_factory(engine)
|
|
40
|
+
queue_repository = QueueRepository()
|
|
41
|
+
job_repository = JobRepository(queue_repository)
|
|
42
|
+
attempt_repository = AttemptRepository()
|
|
43
|
+
worker_repository = WorkerRepository()
|
|
44
|
+
stats_repository = StatsRepository()
|
|
45
|
+
app.state.settings = runtime_settings
|
|
46
|
+
app.state.engine = engine
|
|
47
|
+
app.state.session_factory = session_factory
|
|
48
|
+
app.state.enqueue_service = EnqueueService(job_repository, runtime_settings)
|
|
49
|
+
app.state.job_service = JobService(job_repository, attempt_repository)
|
|
50
|
+
app.state.queue_service = QueueService(queue_repository)
|
|
51
|
+
app.state.worker_service = WorkerService(worker_repository)
|
|
52
|
+
app.state.stats_service = StatsService(stats_repository)
|
|
53
|
+
try:
|
|
54
|
+
yield
|
|
55
|
+
finally:
|
|
56
|
+
await engine.dispose()
|
|
57
|
+
|
|
58
|
+
app = FastAPI(title="PgRelay Admin API", version="0.1.0", lifespan=lifespan)
|
|
59
|
+
app.add_exception_handler(PgRelayError, _pgrelay_error_handler) # type: ignore[arg-type]
|
|
60
|
+
app.add_exception_handler(RequestValidationError, _request_validation_handler) # type: ignore[arg-type]
|
|
61
|
+
app.add_exception_handler(HTTPException, _http_exception_handler) # type: ignore[arg-type]
|
|
62
|
+
app.add_exception_handler(Exception, _unexpected_exception_handler)
|
|
63
|
+
app.include_router(health.router)
|
|
64
|
+
app.include_router(jobs.router)
|
|
65
|
+
app.include_router(queues.router)
|
|
66
|
+
app.include_router(stats.router)
|
|
67
|
+
app.include_router(workers.router)
|
|
68
|
+
return app
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _error_response(code: str, message: str, details: dict[str, Any] | None, status_code: int) -> JSONResponse:
|
|
72
|
+
body = ApiErrorResponse(error=ApiError(code=code, message=message, details=details))
|
|
73
|
+
return JSONResponse(status_code=status_code, content=jsonable_encoder(body))
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
async def _pgrelay_error_handler(_request: Request, exc: PgRelayError) -> JSONResponse:
|
|
77
|
+
"""Handle PgRelay domain errors."""
|
|
78
|
+
code = exc.error_code
|
|
79
|
+
return _error_response(code, str(exc), None, ERROR_STATUS_CODES.get(code, 500))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
async def _request_validation_handler(_request: Request, exc: RequestValidationError) -> JSONResponse:
|
|
83
|
+
"""Handle FastAPI request validation errors."""
|
|
84
|
+
return _error_response(
|
|
85
|
+
"validation_error", "Request validation failed", {"errors": jsonable_encoder(exc.errors())}, 422
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
async def _http_exception_handler(_request: Request, exc: HTTPException) -> JSONResponse:
|
|
90
|
+
"""Handle HTTP exceptions with the common envelope."""
|
|
91
|
+
if isinstance(exc.detail, dict) and "error" in exc.detail:
|
|
92
|
+
return JSONResponse(status_code=exc.status_code, content=exc.detail)
|
|
93
|
+
return _error_response("http_error", str(exc.detail), None, exc.status_code)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def _unexpected_exception_handler(_request: Request, exc: Exception) -> JSONResponse:
|
|
97
|
+
"""Handle unexpected errors."""
|
|
98
|
+
return _error_response("pgrelay_error", "Internal server error", {"type": type(exc).__name__}, 500)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""FastAPI dependency helpers."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncGenerator
|
|
4
|
+
from typing import cast
|
|
5
|
+
|
|
6
|
+
from fastapi import Header, Request
|
|
7
|
+
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
8
|
+
|
|
9
|
+
from pgrelay.config.settings import Settings
|
|
10
|
+
from pgrelay.security.auth import require_api_token
|
|
11
|
+
from pgrelay.services.enqueue import EnqueueService
|
|
12
|
+
from pgrelay.services.jobs import JobService
|
|
13
|
+
from pgrelay.services.queues import QueueService
|
|
14
|
+
from pgrelay.services.stats import StatsService
|
|
15
|
+
from pgrelay.services.workers import WorkerService
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_settings(request: Request) -> Settings:
|
|
19
|
+
"""Return application settings."""
|
|
20
|
+
return cast(Settings, request.app.state.settings)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
async def get_session(request: Request) -> AsyncGenerator[AsyncSession, None]:
|
|
24
|
+
"""Yield an API database session."""
|
|
25
|
+
session_factory: async_sessionmaker[AsyncSession] = request.app.state.session_factory
|
|
26
|
+
async with session_factory() as session:
|
|
27
|
+
try:
|
|
28
|
+
yield session
|
|
29
|
+
await session.commit()
|
|
30
|
+
except Exception:
|
|
31
|
+
await session.rollback()
|
|
32
|
+
raise
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_enqueue_service(request: Request) -> EnqueueService:
|
|
36
|
+
"""Return enqueue service."""
|
|
37
|
+
return cast(EnqueueService, request.app.state.enqueue_service)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_job_service(request: Request) -> JobService:
|
|
41
|
+
"""Return job service."""
|
|
42
|
+
return cast(JobService, request.app.state.job_service)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_queue_service(request: Request) -> QueueService:
|
|
46
|
+
"""Return queue service."""
|
|
47
|
+
return cast(QueueService, request.app.state.queue_service)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def get_stats_service(request: Request) -> StatsService:
|
|
51
|
+
"""Return stats service."""
|
|
52
|
+
return cast(StatsService, request.app.state.stats_service)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def get_worker_service(request: Request) -> WorkerService:
|
|
56
|
+
"""Return worker service."""
|
|
57
|
+
return cast(WorkerService, request.app.state.worker_service)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
async def require_auth(request: Request, authorization: str | None = Header(default=None)) -> None:
|
|
61
|
+
"""Require API authentication for admin endpoints."""
|
|
62
|
+
await require_api_token(get_settings(request), authorization)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""API routers package."""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Health endpoints."""
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Request
|
|
4
|
+
from sqlalchemy import text
|
|
5
|
+
from sqlalchemy.exc import SQLAlchemyError
|
|
6
|
+
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
7
|
+
|
|
8
|
+
from pgrelay.errors import DatabaseUnavailableError
|
|
9
|
+
|
|
10
|
+
router = APIRouter(tags=["health"])
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@router.get("/healthz")
|
|
14
|
+
async def healthz() -> dict[str, str]:
|
|
15
|
+
"""Return process liveness."""
|
|
16
|
+
return {"status": "ok"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@router.get("/readyz")
|
|
20
|
+
async def readyz(request: Request) -> dict[str, str]:
|
|
21
|
+
"""Return database readiness."""
|
|
22
|
+
session_factory: async_sessionmaker[AsyncSession] = request.app.state.session_factory
|
|
23
|
+
try:
|
|
24
|
+
async with session_factory() as session:
|
|
25
|
+
await session.execute(text("SELECT 1"))
|
|
26
|
+
except SQLAlchemyError as exc:
|
|
27
|
+
raise DatabaseUnavailableError("Database unavailable") from exc
|
|
28
|
+
return {"status": "ready", "database": "ok"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Job admin endpoints."""
|
|
2
|
+
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
|
|
5
|
+
from fastapi import APIRouter, Depends, Query, Response, status
|
|
6
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
7
|
+
|
|
8
|
+
from pgrelay.api.dependencies import get_enqueue_service, get_job_service, get_session, require_auth
|
|
9
|
+
from pgrelay.schemas.enqueue import EnqueueJobRequest
|
|
10
|
+
from pgrelay.schemas.jobs import (
|
|
11
|
+
AttemptResponse,
|
|
12
|
+
CancelJobResponse,
|
|
13
|
+
JobListResponse,
|
|
14
|
+
JobResponse,
|
|
15
|
+
ReplayJobRequest,
|
|
16
|
+
ReplayJobResponse,
|
|
17
|
+
)
|
|
18
|
+
from pgrelay.sdk.result import EnqueueResult
|
|
19
|
+
from pgrelay.services.enqueue import EnqueueService
|
|
20
|
+
from pgrelay.services.jobs import JobService
|
|
21
|
+
|
|
22
|
+
router = APIRouter(prefix="/v1/jobs", tags=["jobs"], dependencies=[Depends(require_auth)])
|
|
23
|
+
SESSION_DEPENDENCY = Depends(get_session)
|
|
24
|
+
ENQUEUE_SERVICE_DEPENDENCY = Depends(get_enqueue_service)
|
|
25
|
+
JOB_SERVICE_DEPENDENCY = Depends(get_job_service)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@router.post("", response_model=EnqueueResult, status_code=status.HTTP_201_CREATED)
|
|
29
|
+
async def enqueue_job(
|
|
30
|
+
request: EnqueueJobRequest,
|
|
31
|
+
response: Response,
|
|
32
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
33
|
+
service: EnqueueService = ENQUEUE_SERVICE_DEPENDENCY,
|
|
34
|
+
) -> EnqueueResult:
|
|
35
|
+
"""Enqueue a job through the admin API."""
|
|
36
|
+
result = await service.enqueue(session, request)
|
|
37
|
+
if not result.created:
|
|
38
|
+
response.status_code = status.HTTP_200_OK
|
|
39
|
+
return result
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@router.get("", response_model=JobListResponse)
|
|
43
|
+
async def list_jobs(
|
|
44
|
+
status_filter: str | None = Query(default=None, alias="status"),
|
|
45
|
+
queue_name: str | None = None,
|
|
46
|
+
kind: str | None = None,
|
|
47
|
+
name: str | None = None,
|
|
48
|
+
dedupe_key: str | None = None,
|
|
49
|
+
limit: int = Query(default=50, ge=1, le=200),
|
|
50
|
+
offset: int = Query(default=0, ge=0),
|
|
51
|
+
include_total: bool = False,
|
|
52
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
53
|
+
service: JobService = JOB_SERVICE_DEPENDENCY,
|
|
54
|
+
) -> JobListResponse:
|
|
55
|
+
"""List jobs without payload fields."""
|
|
56
|
+
return await service.list_jobs(
|
|
57
|
+
session,
|
|
58
|
+
status=status_filter,
|
|
59
|
+
queue_name=queue_name,
|
|
60
|
+
kind=kind,
|
|
61
|
+
name=name,
|
|
62
|
+
dedupe_key=dedupe_key,
|
|
63
|
+
limit=limit,
|
|
64
|
+
offset=offset,
|
|
65
|
+
include_total=include_total,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@router.get("/{job_id}", response_model=JobResponse)
|
|
70
|
+
async def get_job(
|
|
71
|
+
job_id: UUID,
|
|
72
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
73
|
+
service: JobService = JOB_SERVICE_DEPENDENCY,
|
|
74
|
+
) -> JobResponse:
|
|
75
|
+
"""Return job detail."""
|
|
76
|
+
return await service.detail(session, job_id=job_id)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@router.get("/{job_id}/attempts", response_model=list[AttemptResponse])
|
|
80
|
+
async def get_attempts(
|
|
81
|
+
job_id: UUID,
|
|
82
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
83
|
+
service: JobService = JOB_SERVICE_DEPENDENCY,
|
|
84
|
+
) -> list[AttemptResponse]:
|
|
85
|
+
"""Return attempts for a job."""
|
|
86
|
+
return await service.attempts_for_job(session, job_id=job_id)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@router.post("/{job_id}/replay", response_model=ReplayJobResponse, status_code=status.HTTP_201_CREATED)
|
|
90
|
+
async def replay_job(
|
|
91
|
+
job_id: UUID,
|
|
92
|
+
request: ReplayJobRequest,
|
|
93
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
94
|
+
service: JobService = JOB_SERVICE_DEPENDENCY,
|
|
95
|
+
) -> ReplayJobResponse:
|
|
96
|
+
"""Replay a job."""
|
|
97
|
+
return await service.replay(session, job_id=job_id, force=request.force)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@router.post("/{job_id}/cancel", response_model=CancelJobResponse)
|
|
101
|
+
async def cancel_job(
|
|
102
|
+
job_id: UUID,
|
|
103
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
104
|
+
service: JobService = JOB_SERVICE_DEPENDENCY,
|
|
105
|
+
) -> CancelJobResponse:
|
|
106
|
+
"""Cancel a pending job."""
|
|
107
|
+
return await service.cancel(session, job_id=job_id)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Queue admin endpoints."""
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Depends
|
|
4
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
5
|
+
|
|
6
|
+
from pgrelay.api.dependencies import get_queue_service, get_session, require_auth
|
|
7
|
+
from pgrelay.schemas.queues import QueueResponse, QueueUpdateRequest
|
|
8
|
+
from pgrelay.services.queues import QueueService
|
|
9
|
+
|
|
10
|
+
router = APIRouter(prefix="/v1/queues", tags=["queues"], dependencies=[Depends(require_auth)])
|
|
11
|
+
SESSION_DEPENDENCY = Depends(get_session)
|
|
12
|
+
QUEUE_SERVICE_DEPENDENCY = Depends(get_queue_service)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@router.get("", response_model=list[QueueResponse])
|
|
16
|
+
async def list_queues(
|
|
17
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
18
|
+
service: QueueService = QUEUE_SERVICE_DEPENDENCY,
|
|
19
|
+
) -> list[QueueResponse]:
|
|
20
|
+
"""List queues."""
|
|
21
|
+
return await service.list_queues(session)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@router.put("/{queue_name}", response_model=QueueResponse)
|
|
25
|
+
async def update_queue(
|
|
26
|
+
queue_name: str,
|
|
27
|
+
request: QueueUpdateRequest,
|
|
28
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
29
|
+
service: QueueService = QUEUE_SERVICE_DEPENDENCY,
|
|
30
|
+
) -> QueueResponse:
|
|
31
|
+
"""Create or update a queue."""
|
|
32
|
+
return await service.upsert_or_update(session, queue_name=queue_name, request=request)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@router.post("/{queue_name}/pause", response_model=QueueResponse)
|
|
36
|
+
async def pause_queue(
|
|
37
|
+
queue_name: str,
|
|
38
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
39
|
+
service: QueueService = QUEUE_SERVICE_DEPENDENCY,
|
|
40
|
+
) -> QueueResponse:
|
|
41
|
+
"""Pause a queue."""
|
|
42
|
+
return await service.pause(session, queue_name=queue_name)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@router.post("/{queue_name}/resume", response_model=QueueResponse)
|
|
46
|
+
async def resume_queue(
|
|
47
|
+
queue_name: str,
|
|
48
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
49
|
+
service: QueueService = QUEUE_SERVICE_DEPENDENCY,
|
|
50
|
+
) -> QueueResponse:
|
|
51
|
+
"""Resume a queue."""
|
|
52
|
+
return await service.resume(session, queue_name=queue_name)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Stats endpoints."""
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Depends
|
|
4
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
5
|
+
|
|
6
|
+
from pgrelay.api.dependencies import get_session, get_stats_service, require_auth
|
|
7
|
+
from pgrelay.schemas.stats import StatsResponse
|
|
8
|
+
from pgrelay.services.stats import StatsService
|
|
9
|
+
|
|
10
|
+
router = APIRouter(prefix="/v1/stats", tags=["stats"], dependencies=[Depends(require_auth)])
|
|
11
|
+
SESSION_DEPENDENCY = Depends(get_session)
|
|
12
|
+
STATS_SERVICE_DEPENDENCY = Depends(get_stats_service)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@router.get("", response_model=StatsResponse)
|
|
16
|
+
async def get_stats(
|
|
17
|
+
approximate: bool = False,
|
|
18
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
19
|
+
service: StatsService = STATS_SERVICE_DEPENDENCY,
|
|
20
|
+
) -> StatsResponse:
|
|
21
|
+
"""Return PgRelay stats."""
|
|
22
|
+
return await service.get_stats(session, approximate=approximate)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Worker endpoints."""
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Depends
|
|
4
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
5
|
+
|
|
6
|
+
from pgrelay.api.dependencies import get_session, get_worker_service, require_auth
|
|
7
|
+
from pgrelay.schemas.workers import WorkerResponse
|
|
8
|
+
from pgrelay.services.workers import WorkerService
|
|
9
|
+
|
|
10
|
+
router = APIRouter(prefix="/v1/workers", tags=["workers"], dependencies=[Depends(require_auth)])
|
|
11
|
+
SESSION_DEPENDENCY = Depends(get_session)
|
|
12
|
+
WORKER_SERVICE_DEPENDENCY = Depends(get_worker_service)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@router.get("", response_model=list[WorkerResponse])
|
|
16
|
+
async def list_workers(
|
|
17
|
+
session: AsyncSession = SESSION_DEPENDENCY,
|
|
18
|
+
service: WorkerService = WORKER_SERVICE_DEPENDENCY,
|
|
19
|
+
) -> list[WorkerResponse]:
|
|
20
|
+
"""List worker heartbeats."""
|
|
21
|
+
return await service.list_workers(session)
|
pgrelay/cli/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""CLI package."""
|