python-pq 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pq/__init__.py +1 -1
- pq/client.py +95 -16
- pq/migrations/versions/20261007T120000Z_d4e5f6a7b8c9_add_task_requeue.py +43 -0
- pq/models.py +31 -0
- pq/worker.py +101 -63
- {python_pq-0.8.0.dist-info → python_pq-0.9.0.dist-info}/METADATA +14 -1
- {python_pq-0.8.0.dist-info → python_pq-0.9.0.dist-info}/RECORD +10 -9
- {python_pq-0.8.0.dist-info → python_pq-0.9.0.dist-info}/WHEEL +0 -0
- {python_pq-0.8.0.dist-info → python_pq-0.9.0.dist-info}/entry_points.txt +0 -0
- {python_pq-0.8.0.dist-info → python_pq-0.9.0.dist-info}/licenses/LICENSE +0 -0
pq/__init__.py
CHANGED
pq/client.py
CHANGED
|
@@ -13,9 +13,9 @@ from croniter.croniter import CroniterBadCronError
|
|
|
13
13
|
from loguru import logger
|
|
14
14
|
import sqlalchemy as sa
|
|
15
15
|
from sqlalchemy import create_engine, delete, func, select, update
|
|
16
|
-
from sqlalchemy.dialects.postgresql import insert
|
|
16
|
+
from sqlalchemy.dialects.postgresql import JSONB, insert
|
|
17
17
|
from sqlalchemy.engine import Engine
|
|
18
|
-
from sqlalchemy.orm import sessionmaker
|
|
18
|
+
from sqlalchemy.orm import InstrumentedAttribute, sessionmaker
|
|
19
19
|
|
|
20
20
|
from pq.models import Base, Periodic, Task, TaskStatus
|
|
21
21
|
from pq.priority import Priority
|
|
@@ -230,6 +230,16 @@ class PQ:
|
|
|
230
230
|
Behaves like enqueue(), but on conflict for client_id, updates all fields.
|
|
231
231
|
Status resets to PENDING, attempts to 0, and timestamps are cleared.
|
|
232
232
|
|
|
233
|
+
If the existing task is RUNNING, it is not changed. The new version
|
|
234
|
+
is stored on the row and runs after the current run ends (the worker
|
|
235
|
+
or the stale reaper re-queues the row with it). A second upsert while
|
|
236
|
+
the row is still RUNNING replaces the stored version, so only the
|
|
237
|
+
latest one runs. One ``client_id`` does not run twice at the same
|
|
238
|
+
time, as long as the stale reaper's timeout is longer than the
|
|
239
|
+
longest run. If the worker running the task died, the stored version
|
|
240
|
+
runs only after the stale reaper re-queues the row: with the reaper
|
|
241
|
+
disabled (``stale_task_timeout=None``) it never runs.
|
|
242
|
+
|
|
233
243
|
Args:
|
|
234
244
|
task: Callable function to execute.
|
|
235
245
|
*args: Positional arguments to pass to the handler.
|
|
@@ -255,6 +265,41 @@ class PQ:
|
|
|
255
265
|
if run_at is None:
|
|
256
266
|
run_at = datetime.now(UTC)
|
|
257
267
|
|
|
268
|
+
# In ``ON CONFLICT DO UPDATE``, ``Task.<column>`` refers to the
|
|
269
|
+
# existing row. A RUNNING row keeps every column as it is (the run
|
|
270
|
+
# in progress keeps its payload, ``started_at``, ``attempts`` and
|
|
271
|
+
# ``max_runtime_seconds``); the new version is parked in
|
|
272
|
+
# ``requeue`` and applied when the run ends. Any other status gets
|
|
273
|
+
# the plain overwrite. All in one statement, so the decision is
|
|
274
|
+
# made under the row lock that ``ON CONFLICT`` takes.
|
|
275
|
+
running = Task.status == TaskStatus.RUNNING
|
|
276
|
+
# ``run_at`` is added in SQL as ``timestamptz``, so Postgres
|
|
277
|
+
# serializes it as ISO 8601 with an offset. A naive ``run_at`` is
|
|
278
|
+
# then read in this session's time zone, the same as on the direct
|
|
279
|
+
# path, and not later in the session of whoever applies the version.
|
|
280
|
+
requeue = sa.literal(
|
|
281
|
+
{
|
|
282
|
+
"name": name,
|
|
283
|
+
"payload": payload,
|
|
284
|
+
"priority": int(priority),
|
|
285
|
+
"max_runtime_seconds": max_runtime_seconds,
|
|
286
|
+
},
|
|
287
|
+
JSONB,
|
|
288
|
+
).op("||", return_type=JSONB)(
|
|
289
|
+
sa.func.jsonb_build_object(
|
|
290
|
+
"run_at", sa.cast(sa.literal(run_at), sa.DateTime(timezone=True))
|
|
291
|
+
)
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def unless_running[T](
|
|
295
|
+
value: T, column: InstrumentedAttribute[T]
|
|
296
|
+
) -> sa.ColumnElement[T]:
|
|
297
|
+
# ``sa.literal(..., column.type)`` binds the value through the
|
|
298
|
+
# column's type. Required for the ``task_status`` enum, which
|
|
299
|
+
# stores member names (``'PENDING'``): a bare enum member in a
|
|
300
|
+
# ``CASE`` would bind ``'pending'`` and fail.
|
|
301
|
+
return sa.case((running, column), else_=sa.literal(value, column.type))
|
|
302
|
+
|
|
258
303
|
stmt = (
|
|
259
304
|
insert(Task)
|
|
260
305
|
.values(
|
|
@@ -269,16 +314,19 @@ class PQ:
|
|
|
269
314
|
.on_conflict_do_update(
|
|
270
315
|
index_elements=["client_id"],
|
|
271
316
|
set_={
|
|
272
|
-
"name": name,
|
|
273
|
-
"payload": payload,
|
|
274
|
-
"priority": priority,
|
|
275
|
-
"status": TaskStatus.PENDING,
|
|
276
|
-
"run_at": run_at,
|
|
277
|
-
"max_runtime_seconds":
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
"
|
|
281
|
-
"
|
|
317
|
+
"name": unless_running(name, Task.name),
|
|
318
|
+
"payload": unless_running(payload, Task.payload),
|
|
319
|
+
"priority": unless_running(int(priority), Task.priority),
|
|
320
|
+
"status": unless_running(TaskStatus.PENDING, Task.status),
|
|
321
|
+
"run_at": unless_running(run_at, Task.run_at),
|
|
322
|
+
"max_runtime_seconds": unless_running(
|
|
323
|
+
max_runtime_seconds, Task.max_runtime_seconds
|
|
324
|
+
),
|
|
325
|
+
"attempts": unless_running(0, Task.attempts),
|
|
326
|
+
"started_at": unless_running(None, Task.started_at),
|
|
327
|
+
"completed_at": unless_running(None, Task.completed_at),
|
|
328
|
+
"error": unless_running(None, Task.error),
|
|
329
|
+
"requeue": sa.case((running, requeue), else_=sa.null()),
|
|
282
330
|
},
|
|
283
331
|
)
|
|
284
332
|
.returning(Task.id)
|
|
@@ -582,13 +630,17 @@ class PQ:
|
|
|
582
630
|
return result.rowcount
|
|
583
631
|
|
|
584
632
|
def reap_stale_tasks(self, threshold: timedelta) -> int:
|
|
585
|
-
"""Mark stale RUNNING tasks as FAILED.
|
|
633
|
+
"""Mark stale RUNNING tasks as FAILED, or re-queue them.
|
|
586
634
|
|
|
587
635
|
When a worker dies mid-execution (e.g. pod restart, OOM on the worker
|
|
588
636
|
process), in-flight tasks stay RUNNING forever because no parent
|
|
589
637
|
process remains to update their status. This method detects those
|
|
590
638
|
orphaned rows and transitions them to FAILED.
|
|
591
639
|
|
|
640
|
+
A stale row that holds a version parked by ``upsert()`` (see
|
|
641
|
+
``Task.requeue``) is re-queued as PENDING with that version instead
|
|
642
|
+
of being failed, so the parked version is not lost.
|
|
643
|
+
|
|
592
644
|
Tasks enqueued with a per-task ``max_runtime_seconds`` override get a
|
|
593
645
|
proportionally larger reaper window: the effective threshold for
|
|
594
646
|
a row is ``max(threshold, max_runtime_seconds * 2)``. This avoids
|
|
@@ -605,7 +657,7 @@ class PQ:
|
|
|
605
657
|
``timedelta(seconds=max_runtime_seconds * 2)``.
|
|
606
658
|
|
|
607
659
|
Returns:
|
|
608
|
-
Number of tasks reaped.
|
|
660
|
+
Number of tasks reaped (failed plus re-queued).
|
|
609
661
|
"""
|
|
610
662
|
now = datetime.now(UTC)
|
|
611
663
|
threshold_seconds = threshold.total_seconds()
|
|
@@ -649,6 +701,7 @@ class PQ:
|
|
|
649
701
|
.where(
|
|
650
702
|
Task.status == TaskStatus.RUNNING,
|
|
651
703
|
Task.started_at < stale_cutoff,
|
|
704
|
+
Task.requeue.is_(None),
|
|
652
705
|
)
|
|
653
706
|
.values(
|
|
654
707
|
status=TaskStatus.FAILED,
|
|
@@ -669,7 +722,31 @@ class PQ:
|
|
|
669
722
|
f" started_at={started_at},"
|
|
670
723
|
f" max_runtime_seconds={max_runtime_seconds})"
|
|
671
724
|
)
|
|
672
|
-
|
|
725
|
+
|
|
726
|
+
# Stale rows with a parked upsert: re-queue them with the
|
|
727
|
+
# parked version. ``FOR UPDATE`` serializes with a concurrent
|
|
728
|
+
# ``upsert()`` (which may replace ``requeue``) and with the
|
|
729
|
+
# worker's own end-of-run update.
|
|
730
|
+
requeued = list(
|
|
731
|
+
session.execute(
|
|
732
|
+
select(Task)
|
|
733
|
+
.where(
|
|
734
|
+
Task.status == TaskStatus.RUNNING,
|
|
735
|
+
Task.started_at < stale_cutoff,
|
|
736
|
+
Task.requeue.is_not(None),
|
|
737
|
+
)
|
|
738
|
+
.with_for_update()
|
|
739
|
+
).scalars()
|
|
740
|
+
)
|
|
741
|
+
for task in requeued:
|
|
742
|
+
logger.warning(
|
|
743
|
+
f"Reaped stale task '{task.name}' (id={task.id},"
|
|
744
|
+
f" started_at={task.started_at},"
|
|
745
|
+
f" max_runtime_seconds={task.max_runtime_seconds});"
|
|
746
|
+
" re-queued by a parked upsert"
|
|
747
|
+
)
|
|
748
|
+
task.apply_requeue()
|
|
749
|
+
return len(reaped) + len(requeued)
|
|
673
750
|
|
|
674
751
|
def run_worker(
|
|
675
752
|
self,
|
|
@@ -687,7 +764,9 @@ class PQ:
|
|
|
687
764
|
On SIGTERM or SIGINT the worker stops claiming new tasks, waits up
|
|
688
765
|
to ``drain_timeout`` seconds for in-flight tasks to finish, then
|
|
689
766
|
kills any still-running task and marks its row FAILED with an
|
|
690
|
-
explicit shutdown error. Interrupted tasks are not re-queued
|
|
767
|
+
explicit shutdown error. Interrupted tasks are not re-queued,
|
|
768
|
+
unless an ``upsert()`` stored a new version while they ran: then the
|
|
769
|
+
row is re-queued with that version.
|
|
691
770
|
|
|
692
771
|
Args:
|
|
693
772
|
concurrency: Maximum number of tasks to process simultaneously.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""add requeue to pq_tasks
|
|
2
|
+
|
|
3
|
+
Revision ID: d4e5f6a7b8c9
|
|
4
|
+
Revises: aee3e8e7e647
|
|
5
|
+
Create Date: 2026-10-07 12:00:00 Z
|
|
6
|
+
|
|
7
|
+
Holds the next version of a one-off task when ``upsert()`` hits a row
|
|
8
|
+
that is RUNNING. The current run keeps its row unchanged; whoever ends
|
|
9
|
+
the run (worker or stale reaper) re-queues the row from this column
|
|
10
|
+
instead of closing it. This makes sure one ``client_id`` never runs
|
|
11
|
+
twice at the same time (ricwo/pq#27).
|
|
12
|
+
|
|
13
|
+
Nullable without a default, so on PostgreSQL 11+ ``ADD COLUMN`` is a
|
|
14
|
+
catalog-only change (no table rewrite). Existing rows get NULL, which
|
|
15
|
+
means "no parked version" — the previous behaviour.
|
|
16
|
+
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from typing import Sequence, Union
|
|
20
|
+
|
|
21
|
+
from alembic import op
|
|
22
|
+
import sqlalchemy as sa
|
|
23
|
+
from sqlalchemy.dialects import postgresql
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# revision identifiers, used by Alembic.
|
|
27
|
+
revision: str = "d4e5f6a7b8c9"
|
|
28
|
+
down_revision: Union[str, Sequence[str], None] = "aee3e8e7e647"
|
|
29
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
30
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def upgrade() -> None:
|
|
34
|
+
"""Add the nullable ``requeue`` column to ``pq_tasks``."""
|
|
35
|
+
op.add_column(
|
|
36
|
+
"pq_tasks",
|
|
37
|
+
sa.Column("requeue", postgresql.JSONB(), nullable=True),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def downgrade() -> None:
|
|
42
|
+
"""Remove the ``requeue`` column from ``pq_tasks``."""
|
|
43
|
+
op.drop_column("pq_tasks", "requeue")
|
pq/models.py
CHANGED
|
@@ -79,6 +79,37 @@ class Task(Base):
|
|
|
79
79
|
# ``max_runtime_seconds * 2``). Useful for occasionally-long tasks
|
|
80
80
|
# in a fleet whose default is sized for typical short work.
|
|
81
81
|
max_runtime_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
82
|
+
# Next version of this task, parked by an ``upsert()`` that arrived while
|
|
83
|
+
# the row was RUNNING: name, payload, priority, run_at (ISO 8601) and
|
|
84
|
+
# max_runtime_seconds. Whoever ends the current run (worker or reaper)
|
|
85
|
+
# re-queues the row with it instead of closing the row. NULL when no
|
|
86
|
+
# upsert is parked (the common case). ``none_as_null`` makes the ORM
|
|
87
|
+
# write SQL NULL for ``None`` (plain ``JSONB`` writes JSON ``null``,
|
|
88
|
+
# which ``IS NULL`` does not match).
|
|
89
|
+
requeue: Mapped[dict[str, Any] | None] = mapped_column(
|
|
90
|
+
JSONB(none_as_null=True), nullable=True
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def apply_requeue(self) -> None:
|
|
94
|
+
"""Turn this RUNNING row into a fresh PENDING task from ``requeue``.
|
|
95
|
+
|
|
96
|
+
The caller must hold a row lock (``FOR UPDATE``) and guarantee
|
|
97
|
+
that ``requeue`` is not None.
|
|
98
|
+
"""
|
|
99
|
+
spec = self.requeue
|
|
100
|
+
if spec is None:
|
|
101
|
+
raise ValueError(f"Task id={self.id} has no parked requeue version")
|
|
102
|
+
self.name = spec["name"]
|
|
103
|
+
self.payload = spec["payload"]
|
|
104
|
+
self.priority = spec["priority"]
|
|
105
|
+
self.run_at = datetime.fromisoformat(spec["run_at"])
|
|
106
|
+
self.max_runtime_seconds = spec["max_runtime_seconds"]
|
|
107
|
+
self.status = TaskStatus.PENDING
|
|
108
|
+
self.attempts = 0
|
|
109
|
+
self.started_at = None
|
|
110
|
+
self.completed_at = None
|
|
111
|
+
self.error = None
|
|
112
|
+
self.requeue = None
|
|
82
113
|
|
|
83
114
|
|
|
84
115
|
class Periodic(Base):
|
pq/worker.py
CHANGED
|
@@ -123,6 +123,9 @@ class _ChildSlot:
|
|
|
123
123
|
name: str
|
|
124
124
|
start_time: float
|
|
125
125
|
is_periodic: bool
|
|
126
|
+
# ``started_at`` written by the claim of a one-off run. Identifies this
|
|
127
|
+
# run on the row (see ``_finish_one_off``). None for periodic slots.
|
|
128
|
+
started_at: datetime | None = None
|
|
126
129
|
periodic_max_concurrent: int | None = None
|
|
127
130
|
|
|
128
131
|
|
|
@@ -569,7 +572,8 @@ def _maybe_reap_stale(
|
|
|
569
572
|
|
|
570
573
|
Args:
|
|
571
574
|
pq: PQ client instance.
|
|
572
|
-
stale_task_timeout: RUNNING tasks older than this are marked FAILED
|
|
575
|
+
stale_task_timeout: RUNNING tasks older than this are marked FAILED
|
|
576
|
+
(or re-queued, if an upsert parked a new version on them).
|
|
573
577
|
``None`` disables reaping.
|
|
574
578
|
reaper_interval: Seconds between reaper checks.
|
|
575
579
|
last_reap: Mutable list containing last reap timestamp.
|
|
@@ -581,9 +585,15 @@ def _maybe_reap_stale(
|
|
|
581
585
|
if now - last_reap[0] < reaper_interval:
|
|
582
586
|
return
|
|
583
587
|
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
588
|
+
# A reaper error must not stop the worker loop (in concurrent mode that
|
|
589
|
+
# would also skip the drain of in-flight children). Retry next interval.
|
|
590
|
+
try:
|
|
591
|
+
reaped = pq.reap_stale_tasks(stale_task_timeout)
|
|
592
|
+
except Exception as e:
|
|
593
|
+
logger.error(f"Error reaping stale tasks: {e}")
|
|
594
|
+
else:
|
|
595
|
+
if reaped:
|
|
596
|
+
logger.info(f"Reaped {reaped} stale RUNNING task(s)")
|
|
587
597
|
|
|
588
598
|
last_reap[0] = now
|
|
589
599
|
|
|
@@ -612,7 +622,9 @@ def run_worker(
|
|
|
612
622
|
still-running task children and marks their rows FAILED with an explicit
|
|
613
623
|
shutdown error. Interrupted tasks are NOT re-queued — pq's at-most-once
|
|
614
624
|
semantics are preserved; applications that need redelivery must
|
|
615
|
-
re-enqueue such tasks themselves.
|
|
625
|
+
re-enqueue such tasks themselves. The one exception: if an ``upsert()``
|
|
626
|
+
stored a new version on the row while it ran, the row is re-queued with
|
|
627
|
+
that new version (the interrupted run itself is not retried).
|
|
616
628
|
|
|
617
629
|
Args:
|
|
618
630
|
pq: PQ client instance.
|
|
@@ -625,9 +637,12 @@ def run_worker(
|
|
|
625
637
|
retention_days: Days to keep completed/failed tasks. Default: 7.
|
|
626
638
|
Set to 0 to disable automatic cleanup.
|
|
627
639
|
cleanup_interval: Seconds between cleanup runs. Default: 3600 (1 hour).
|
|
628
|
-
stale_task_timeout: Mark RUNNING tasks older than this as FAILED
|
|
640
|
+
stale_task_timeout: Mark RUNNING tasks older than this as FAILED
|
|
641
|
+
(or re-queue them, if an upsert parked a new version on them).
|
|
629
642
|
Catches orphaned tasks whose worker died mid-execution.
|
|
630
|
-
Default: 1 hour. Set to ``None`` to disable.
|
|
643
|
+
Default: 1 hour. Set to ``None`` to disable. Must be longer
|
|
644
|
+
than the longest run: a live run that is reaped can overlap
|
|
645
|
+
with the version re-queued after it.
|
|
631
646
|
drain_timeout: Seconds to wait for in-flight tasks when shutting
|
|
632
647
|
down on SIGTERM/SIGINT. Default: 20. Must be set below the
|
|
633
648
|
orchestrator's termination grace period, with headroom for the
|
|
@@ -756,6 +771,66 @@ def run_worker_once(
|
|
|
756
771
|
return False
|
|
757
772
|
|
|
758
773
|
|
|
774
|
+
def _finish_one_off(
|
|
775
|
+
pq: PQ,
|
|
776
|
+
task_id: int,
|
|
777
|
+
started_at: datetime | None,
|
|
778
|
+
name: str,
|
|
779
|
+
status: TaskStatus,
|
|
780
|
+
error_msg: str | None,
|
|
781
|
+
) -> None:
|
|
782
|
+
"""Record the end of a one-off run.
|
|
783
|
+
|
|
784
|
+
Only touches the row if it is still RUNNING *with this run's*
|
|
785
|
+
``started_at`` (the value written by its claim). So a reaper's FAILED
|
|
786
|
+
verdict (or a cancel) is not overwritten, and neither is a later run
|
|
787
|
+
of the same row: if the reaper re-queued the row while this run was
|
|
788
|
+
still alive and another worker claimed it, ``started_at`` no longer
|
|
789
|
+
matches. ``None`` never matches (a RUNNING row always has
|
|
790
|
+
``started_at``), so the row is left alone. If an ``upsert()`` parked a
|
|
791
|
+
new version on the row while it ran (``Task.requeue``), the row is
|
|
792
|
+
re-queued as PENDING with that version instead of being closed. The
|
|
793
|
+
outcome of this run is then only in the worker log, the same as when
|
|
794
|
+
an upsert overwrites a COMPLETED or FAILED row.
|
|
795
|
+
|
|
796
|
+
``FOR UPDATE`` serializes this read-then-write with a concurrent
|
|
797
|
+
``upsert()``, whose ``ON CONFLICT DO UPDATE`` locks the same row: a
|
|
798
|
+
parked version is either seen here, or the upsert runs after this
|
|
799
|
+
commit and hits a non-RUNNING row, which it overwrites directly.
|
|
800
|
+
"""
|
|
801
|
+
try:
|
|
802
|
+
with pq.session() as session:
|
|
803
|
+
task = session.execute(
|
|
804
|
+
select(Task)
|
|
805
|
+
.where(
|
|
806
|
+
Task.id == task_id,
|
|
807
|
+
Task.status == TaskStatus.RUNNING,
|
|
808
|
+
Task.started_at == started_at,
|
|
809
|
+
)
|
|
810
|
+
.with_for_update()
|
|
811
|
+
).scalar_one_or_none()
|
|
812
|
+
if task is None:
|
|
813
|
+
logger.warning(
|
|
814
|
+
f"Task '{name}' (id={task_id}) was no longer RUNNING"
|
|
815
|
+
" for this run when the worker tried to record its result"
|
|
816
|
+
" — likely reaped or canceled"
|
|
817
|
+
)
|
|
818
|
+
return
|
|
819
|
+
if task.requeue is not None:
|
|
820
|
+
logger.info(
|
|
821
|
+
f"Task '{name}' (id={task_id}) ended as {status.value};"
|
|
822
|
+
" re-queued with the version upserted while it ran"
|
|
823
|
+
)
|
|
824
|
+
task.apply_requeue()
|
|
825
|
+
return
|
|
826
|
+
task.status = status
|
|
827
|
+
task.completed_at = datetime.now(UTC)
|
|
828
|
+
if error_msg:
|
|
829
|
+
task.error = error_msg
|
|
830
|
+
except Exception as e:
|
|
831
|
+
logger.error(f"Error updating task status: {e}")
|
|
832
|
+
|
|
833
|
+
|
|
759
834
|
def _process_one_off_task(
|
|
760
835
|
pq: PQ,
|
|
761
836
|
*,
|
|
@@ -798,9 +873,11 @@ def _process_one_off_task(
|
|
|
798
873
|
if task is None:
|
|
799
874
|
return False
|
|
800
875
|
|
|
801
|
-
# Mark as running
|
|
876
|
+
# Mark as running. ``started_at`` also identifies this run when
|
|
877
|
+
# Phase 3 records its result (see ``_finish_one_off``).
|
|
878
|
+
started_at = datetime.now(UTC)
|
|
802
879
|
task.status = TaskStatus.RUNNING
|
|
803
|
-
task.started_at =
|
|
880
|
+
task.started_at = started_at
|
|
804
881
|
task.attempts += 1
|
|
805
882
|
|
|
806
883
|
# Get task data for execution (before session closes)
|
|
@@ -869,27 +946,9 @@ def _process_one_off_task(
|
|
|
869
946
|
elapsed = time.perf_counter() - start
|
|
870
947
|
|
|
871
948
|
# Phase 3: Update task status (guarded — only if still RUNNING, so we
|
|
872
|
-
# don't overwrite a reaper's FAILED verdict for an orphaned task
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
values: dict[str, object] = {
|
|
876
|
-
"status": status,
|
|
877
|
-
"completed_at": datetime.now(UTC),
|
|
878
|
-
}
|
|
879
|
-
if error_msg:
|
|
880
|
-
values["error"] = error_msg
|
|
881
|
-
result = session.execute(
|
|
882
|
-
update(Task)
|
|
883
|
-
.where(Task.id == task_id, Task.status == TaskStatus.RUNNING)
|
|
884
|
-
.values(**values)
|
|
885
|
-
)
|
|
886
|
-
if result.rowcount == 0:
|
|
887
|
-
logger.warning(
|
|
888
|
-
f"Task '{name}' (id={task_id}) was no longer RUNNING"
|
|
889
|
-
" when Phase 3 tried to update — likely reaped"
|
|
890
|
-
)
|
|
891
|
-
except Exception as e:
|
|
892
|
-
logger.error(f"Error updating task status: {e}")
|
|
949
|
+
# don't overwrite a reaper's FAILED verdict for an orphaned task; and
|
|
950
|
+
# re-queued instead of closed if an upsert was parked on the row)
|
|
951
|
+
_finish_one_off(pq, task_id, started_at, name, status, error_msg)
|
|
893
952
|
|
|
894
953
|
# Log result
|
|
895
954
|
if status == TaskStatus.COMPLETED:
|
|
@@ -1092,8 +1151,9 @@ def _claim_and_fork_one_off(
|
|
|
1092
1151
|
if task is None:
|
|
1093
1152
|
return None
|
|
1094
1153
|
|
|
1154
|
+
started_at = datetime.now(UTC)
|
|
1095
1155
|
task.status = TaskStatus.RUNNING
|
|
1096
|
-
task.started_at =
|
|
1156
|
+
task.started_at = started_at
|
|
1097
1157
|
task.attempts += 1
|
|
1098
1158
|
|
|
1099
1159
|
name = task.name
|
|
@@ -1128,15 +1188,7 @@ def _claim_and_fork_one_off(
|
|
|
1128
1188
|
)
|
|
1129
1189
|
except Exception as e:
|
|
1130
1190
|
logger.error(f"Error starting task '{name}': {e}")
|
|
1131
|
-
|
|
1132
|
-
with pq.session() as session:
|
|
1133
|
-
t = session.get(Task, task_id)
|
|
1134
|
-
if t:
|
|
1135
|
-
t.status = TaskStatus.FAILED
|
|
1136
|
-
t.completed_at = datetime.now(UTC)
|
|
1137
|
-
t.error = str(e)
|
|
1138
|
-
except Exception as update_err:
|
|
1139
|
-
logger.error(f"Error updating task status: {update_err}")
|
|
1191
|
+
_finish_one_off(pq, task_id, started_at, name, TaskStatus.FAILED, str(e))
|
|
1140
1192
|
return None
|
|
1141
1193
|
|
|
1142
1194
|
return _ChildSlot(
|
|
@@ -1146,6 +1198,7 @@ def _claim_and_fork_one_off(
|
|
|
1146
1198
|
name=name,
|
|
1147
1199
|
start_time=time.perf_counter(),
|
|
1148
1200
|
is_periodic=False,
|
|
1201
|
+
started_at=started_at,
|
|
1149
1202
|
)
|
|
1150
1203
|
|
|
1151
1204
|
|
|
@@ -1342,29 +1395,14 @@ def _reap_and_update(pq: PQ, slot: _ChildSlot) -> None:
|
|
|
1342
1395
|
if result.exit_kind == "timeout":
|
|
1343
1396
|
error_msg = f"Timed out after {elapsed:.3f} s"
|
|
1344
1397
|
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
row_result = session.execute(
|
|
1354
|
-
update(Task)
|
|
1355
|
-
.where(
|
|
1356
|
-
Task.id == slot.task_id,
|
|
1357
|
-
Task.status == TaskStatus.RUNNING,
|
|
1358
|
-
)
|
|
1359
|
-
.values(**values)
|
|
1360
|
-
)
|
|
1361
|
-
if row_result.rowcount == 0:
|
|
1362
|
-
logger.warning(
|
|
1363
|
-
f"Task '{slot.name}' (id={slot.task_id}) was no longer"
|
|
1364
|
-
" RUNNING when reap tried to update — likely reaped"
|
|
1365
|
-
)
|
|
1366
|
-
except Exception as e:
|
|
1367
|
-
logger.error(f"Error updating task status: {e}")
|
|
1398
|
+
_finish_one_off(
|
|
1399
|
+
pq,
|
|
1400
|
+
slot.task_id,
|
|
1401
|
+
slot.started_at,
|
|
1402
|
+
slot.name,
|
|
1403
|
+
result.task_status,
|
|
1404
|
+
error_msg,
|
|
1405
|
+
)
|
|
1368
1406
|
|
|
1369
1407
|
if result.task_status == TaskStatus.COMPLETED:
|
|
1370
1408
|
logger.debug(f"Task '{slot.name}' completed in {elapsed:.3f} s")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-pq
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Postgres-backed job queue for Python
|
|
5
5
|
Keywords: postgres,job-queue,task-queue,background-jobs,worker
|
|
6
6
|
Author: ricwo
|
|
@@ -143,6 +143,19 @@ pq.upsert(send_email, to="a@b.com", client_id="welcome-email")
|
|
|
143
143
|
pq.upsert(send_email, to="new@b.com", client_id="welcome-email")
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
+
If the task is running, the new version is queued and runs after the current run ends, so one `client_id` does not run twice at the same time. This lets a task schedule its own continuation:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
def backfill(offset: int) -> None:
|
|
150
|
+
more = process_rows(offset, limit=1000)
|
|
151
|
+
if more:
|
|
152
|
+
with PQ(DATABASE_URL) as client:
|
|
153
|
+
# Runs after this run ends, never next to it
|
|
154
|
+
client.upsert(backfill, offset=offset + 1000, client_id="backfill")
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
If the worker running the task dies, the queued version runs after the stale-task reaper picks up the row (`stale_task_timeout`, default 1 hour). Keep `stale_task_timeout` longer than your longest run: if the reaper picks up a run that is still alive, the queued version can start next to it.
|
|
158
|
+
|
|
146
159
|
## Periodic Tasks
|
|
147
160
|
|
|
148
161
|
### Intervals
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
pq/__init__.py,sha256=
|
|
2
|
-
pq/client.py,sha256=
|
|
1
|
+
pq/__init__.py,sha256=O1rJ8uMMxIZnXoK4SwLuxJitPd_yzj--RK8O828QnNg,398
|
|
2
|
+
pq/client.py,sha256=6YrRWsOHE-2-iylFra5suZ3GfnHrw5OCF3d33oVrytU,32473
|
|
3
3
|
pq/config.py,sha256=U7dW-LVHy4feLUDQAmxKSF8bavtwxMY4thYCNMR7nDc,293
|
|
4
4
|
pq/logging.py,sha256=41kft5saLMLzzhHSstx27MSAHdFcd02_jxNGm8c-aoI,789
|
|
5
5
|
pq/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
|
|
@@ -12,14 +12,15 @@ pq/migrations/versions/20260205T120000Z_a1b2c3d4e5f6_add_max_concurrent.py,sha25
|
|
|
12
12
|
pq/migrations/versions/20260205T180000Z_b7c8d9e0f1a2_add_periodic_key.py,sha256=W_sXMji-s6QObxxghDZf4nIeROWclc3QLpFaOIzDtH8,1137
|
|
13
13
|
pq/migrations/versions/20260217T120000Z_c3d4e5f6a7b8_add_periodic_active.py,sha256=o5Wft7OkeXg6bAbLKwVSuwoIPBscuJr0ERcA1pVu_lI,758
|
|
14
14
|
pq/migrations/versions/20260527T170136Z_aee3e8e7e647_add_task_max_runtime_seconds.py,sha256=DUAdNbVCR83eeZKIvqRg7nyT-fEU_HS7pR81yM5CMew,1708
|
|
15
|
+
pq/migrations/versions/20261007T120000Z_d4e5f6a7b8c9_add_task_requeue.py,sha256=pX4NoYY4dOXfJq22AxlQmlYu1N7ucTUJe-xHbgA8Myo,1322
|
|
15
16
|
pq/migrations/versions/__init__.py,sha256=HtcCnjmL67QdG8QvzZXVOii80Q1cKNkuzNT3emD11oU,26
|
|
16
|
-
pq/models.py,sha256=
|
|
17
|
+
pq/models.py,sha256=Ayilyk20Pp-kFibC-G2FgXZZ0g-C_XbmQlYnMSUBNg8,6366
|
|
17
18
|
pq/priority.py,sha256=edqpQeGiJrxcmpJ5MZCvwTD9F6sIZbRoI1mx_LmUxAs,264
|
|
18
19
|
pq/registry.py,sha256=1RFfB-dXXM2qKwLF6XeYeUm9buca1_fQvKQDnpfsXKs,2306
|
|
19
20
|
pq/serialization.py,sha256=ZspG9qO4YYi-xCrtPGCg8yrzqRI2CGYnO_Ewjh8lE1E,3291
|
|
20
|
-
pq/worker.py,sha256=
|
|
21
|
-
python_pq-0.
|
|
22
|
-
python_pq-0.
|
|
23
|
-
python_pq-0.
|
|
24
|
-
python_pq-0.
|
|
25
|
-
python_pq-0.
|
|
21
|
+
pq/worker.py,sha256=LxRyOF2DzKS20fNkCTRrULr7Q6HnwPvfAMx4oxN84A4,51537
|
|
22
|
+
python_pq-0.9.0.dist-info/licenses/LICENSE,sha256=Bx9dUtjCpaluWGalMqkI3SGkpPxx_AYdJdmV5ptCM4g,1062
|
|
23
|
+
python_pq-0.9.0.dist-info/WHEEL,sha256=eh7sammvW2TypMMMGKgsM83HyA_3qQ5Lgg3ynoecH3M,79
|
|
24
|
+
python_pq-0.9.0.dist-info/entry_points.txt,sha256=fpK4uA3gQaFpcDmdy5GFbP4VP7gaJcn4H4GJyMzRIM8,36
|
|
25
|
+
python_pq-0.9.0.dist-info/METADATA,sha256=GHQxdQieuDtKhpcLdW8WLiSXW_FTimVuYqkINNXUzLs,10912
|
|
26
|
+
python_pq-0.9.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|