pg-boss 12.26.4 → 12.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -5
- package/dist/attorney.d.ts +2 -1
- package/dist/attorney.d.ts.map +1 -1
- package/dist/attorney.js +80 -7
- package/dist/cli.js +2 -2
- package/dist/contractor.d.ts +2 -1
- package/dist/contractor.d.ts.map +1 -1
- package/dist/contractor.js +44 -2
- package/dist/drifter.d.ts +2 -0
- package/dist/drifter.d.ts.map +1 -1
- package/dist/drifter.js +53 -8
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +11 -3
- package/dist/migrationStore.d.ts +1 -1
- package/dist/migrationStore.d.ts.map +1 -1
- package/dist/migrationStore.js +187 -11
- package/dist/plans.d.ts +3 -2
- package/dist/plans.d.ts.map +1 -1
- package/dist/plans.js +126 -25
- package/dist/schema.json +14 -2
- package/dist/timekeeper.d.ts +1 -0
- package/dist/timekeeper.d.ts.map +1 -1
- package/dist/timekeeper.js +65 -5
- package/dist/tools.d.ts +32 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +40 -1
- package/dist/types.d.ts +53 -10
- package/dist/types.d.ts.map +1 -1
- package/package.json +14 -13
package/dist/plans.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { indexKeysRaw, indexPredicateRaw, displayIndexDefinition, extractFunctionBody, normalizeFunctionBody } from "./drifter.js";
|
|
1
|
+
import { indexKeysRaw, indexIncludeRaw, indexPredicateRaw, displayIndexDefinition, extractFunctionBody, normalizeFunctionBody } from "./drifter.js";
|
|
2
2
|
import schemaManifest from './schema.json' with { type: 'json' };
|
|
3
|
+
import { normalizeSchemaName, resolveSchemaName } from "./tools.js";
|
|
3
4
|
export const PG_ERROR = {
|
|
4
5
|
divisionByZero: '22012'
|
|
5
6
|
};
|
|
@@ -400,6 +401,7 @@ function createTableJobCommon(schema) {
|
|
|
400
401
|
SELECT ${schema}.job_table_run($cmd$${createIndexJobPolicyStately(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
401
402
|
SELECT ${schema}.job_table_run($cmd$${createIndexJobPolicyExclusive(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
402
403
|
SELECT ${schema}.job_table_run($cmd$${createIndexJobPolicyKeyStrictFifo(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
404
|
+
SELECT ${schema}.job_table_run($cmd$${createIndexJobPolicyKeyStrictFifoHeads(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
403
405
|
SELECT ${schema}.job_table_run($cmd$${createCheckConstraintKeyStrictFifo(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
404
406
|
SELECT ${schema}.job_table_run($cmd$${createIndexJobThrottle(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
405
407
|
SELECT ${schema}.job_table_run($cmd$${createIndexJobFetch(schema)}$cmd$, '${COMMON_JOB_TABLE}');
|
|
@@ -419,6 +421,7 @@ function createTableJobIndexes(schema, noDeferrableConstraints = false, noCoveri
|
|
|
419
421
|
${createIndexJobPolicyStately(schema)};
|
|
420
422
|
${createIndexJobPolicyExclusive(schema)};
|
|
421
423
|
${createIndexJobPolicyKeyStrictFifo(schema)};
|
|
424
|
+
${createIndexJobPolicyKeyStrictFifoHeads(schema, noCoveringIndex)};
|
|
422
425
|
${createCheckConstraintKeyStrictFifo(schema)};
|
|
423
426
|
${createIndexJobThrottle(schema)};
|
|
424
427
|
${createIndexJobFetch(schema, noCoveringIndex)};
|
|
@@ -549,6 +552,7 @@ function createQueueFunction(schema, noPartitioning = false) {
|
|
|
549
552
|
EXECUTE ${schema}.job_table_format($cmd$${createIndexJobPolicyExclusive(schema)}$cmd$, tablename);
|
|
550
553
|
ELSIF options->>'policy' = '${QUEUE_POLICIES.key_strict_fifo}' THEN
|
|
551
554
|
EXECUTE ${schema}.job_table_format($cmd$${createIndexJobPolicyKeyStrictFifo(schema)}$cmd$, tablename);
|
|
555
|
+
EXECUTE ${schema}.job_table_format($cmd$${createIndexJobPolicyKeyStrictFifoHeads(schema)}$cmd$, tablename);
|
|
552
556
|
EXECUTE ${schema}.job_table_format($cmd$${createCheckConstraintKeyStrictFifo(schema)}$cmd$, tablename);
|
|
553
557
|
END IF;
|
|
554
558
|
|
|
@@ -608,8 +612,11 @@ export function createQueue(schema, name, options, noAdvisoryLocks) {
|
|
|
608
612
|
// channel human-recognizable in pg_stat_activity; 24 hex chars leaves ample headroom under
|
|
609
613
|
// the 63-byte identifier limit. Channels are already scoped to a single database, so unlike
|
|
610
614
|
// advisoryLock there is no need to mix in current_database().
|
|
615
|
+
//
|
|
616
|
+
// normalizeSchemaName, not resolveSchemaName: the channel is never matched against the catalog, so
|
|
617
|
+
// it only has to agree across instances on the same schema. See the note on the helper.
|
|
611
618
|
export function notifyChannelSql(schema) {
|
|
612
|
-
return `('pgboss_' || left(encode(sha224('${schema}'::bytea), 'hex'), 24))`;
|
|
619
|
+
return `('pgboss_' || left(encode(sha224('${normalizeSchemaName(schema)}'::bytea), 'hex'), 24))`;
|
|
613
620
|
}
|
|
614
621
|
// Parameter-less statement that wakes workers on a notify-enabled queue. Embedded into
|
|
615
622
|
// flow batches so it commits in the same transaction as the inserts.
|
|
@@ -657,6 +664,21 @@ function createIndexJobPolicyExclusive(schema) {
|
|
|
657
664
|
function createIndexJobPolicyKeyStrictFifo(schema) {
|
|
658
665
|
return `CREATE UNIQUE INDEX job_i8 ON ${schema}.job (name, singleton_key) WHERE state IN ('${JOB_STATES.active}', '${JOB_STATES.retry}', '${JOB_STATES.failed}') AND policy = '${QUEUE_POLICIES.key_strict_fifo}'`;
|
|
659
666
|
}
|
|
667
|
+
function createIndexJobPolicyKeyStrictFifoHeads(schema, noCoveringIndex = false) {
|
|
668
|
+
// Column order mirrors the strict_fifo_heads ORDER BY (state DESC puts a retry job ahead
|
|
669
|
+
// of created siblings) so the DISTINCT ON head-per-key scan stays an ordered index scan.
|
|
670
|
+
//
|
|
671
|
+
// INCLUDE (start_after) is load-bearing, not a covering-index habit: `policy` and `blocked`
|
|
672
|
+
// are satisfied by the partial predicate, so start_after is the only remaining filter in the
|
|
673
|
+
// heads CTE. Without it every index entry needs a heap fetch and the scan degrades from an
|
|
674
|
+
// Index Only Scan to an Index Scan (measured at 200k queued rows / 2,000 keys: 47ms -> 111ms,
|
|
675
|
+
// ~400k extra buffer hits). It costs no HOT updates that job_i5 doesn't already cost, since
|
|
676
|
+
// job_i5 indexes start_after too, and it only widens a partial index that key_strict_fifo
|
|
677
|
+
// rows enter. Backends without covering indexes (the CockroachDB profile) fall back to the
|
|
678
|
+
// narrow form and pay the heap fetches; they take a different fetch path anyway (noSkipLocked).
|
|
679
|
+
const include = noCoveringIndex ? '' : ' INCLUDE (start_after)';
|
|
680
|
+
return `CREATE INDEX job_i10 ON ${schema}.job (name, singleton_key, state DESC, created_on, id)${include} WHERE state < '${JOB_STATES.active}' AND NOT blocked AND policy = '${QUEUE_POLICIES.key_strict_fifo}'`;
|
|
681
|
+
}
|
|
660
682
|
function createCheckConstraintKeyStrictFifo(schema) {
|
|
661
683
|
return `ALTER TABLE ${schema}.job ADD CONSTRAINT job_key_strict_fifo_singleton_key_check CHECK (NOT (policy = '${QUEUE_POLICIES.key_strict_fifo}' AND singleton_key IS NULL))`;
|
|
662
684
|
}
|
|
@@ -705,7 +727,7 @@ function trySetQueueTimestamp(schema, queues, column, seconds) {
|
|
|
705
727
|
values: [queues]
|
|
706
728
|
};
|
|
707
729
|
}
|
|
708
|
-
export function updateQueue(schema
|
|
730
|
+
export function updateQueue(schema) {
|
|
709
731
|
return `
|
|
710
732
|
WITH options as (SELECT $2::jsonb as data)
|
|
711
733
|
UPDATE ${schema}.queue SET
|
|
@@ -723,9 +745,9 @@ export function updateQueue(schema, { deadLetter } = {}) {
|
|
|
723
745
|
THEN (o.data->>'heartbeatSeconds')::int
|
|
724
746
|
ELSE heartbeat_seconds END,
|
|
725
747
|
notify = COALESCE((o.data->>'notify')::bool, notify),
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
748
|
+
dead_letter = CASE WHEN jsonb_exists(o.data, 'deadLetter')
|
|
749
|
+
THEN o.data->>'deadLetter'
|
|
750
|
+
ELSE dead_letter END,
|
|
729
751
|
updated_on = now()
|
|
730
752
|
FROM options o
|
|
731
753
|
WHERE name = $1
|
|
@@ -918,7 +940,7 @@ export function ensureQueueStatsPartitions(schema) {
|
|
|
918
940
|
IF NOT EXISTS (
|
|
919
941
|
SELECT 1 FROM pg_class c
|
|
920
942
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
921
|
-
WHERE n.nspname = '${schema}' AND c.relname = part_name
|
|
943
|
+
WHERE n.nspname = '${resolveSchemaName(schema)}' AND c.relname = part_name
|
|
922
944
|
) THEN
|
|
923
945
|
EXECUTE format(
|
|
924
946
|
'CREATE TABLE ${schema}.%I PARTITION OF ${schema}.queue_stats FOR VALUES FROM (%L) TO (%L)',
|
|
@@ -947,7 +969,7 @@ export function dropOldQueueStatsPartitions(schema, days) {
|
|
|
947
969
|
JOIN pg_class p ON p.oid = i.inhparent
|
|
948
970
|
JOIN pg_class c ON c.oid = i.inhrelid
|
|
949
971
|
JOIN pg_namespace n ON n.oid = p.relnamespace
|
|
950
|
-
WHERE n.nspname = '${schema}' AND p.relname = 'queue_stats'
|
|
972
|
+
WHERE n.nspname = '${resolveSchemaName(schema)}' AND p.relname = 'queue_stats'
|
|
951
973
|
LOOP
|
|
952
974
|
suffix := substring(r.relname FROM 'queue_stats_(.*)$');
|
|
953
975
|
IF suffix ~ '^[0-9]{8}$' THEN
|
|
@@ -1088,8 +1110,23 @@ export function setVersion(schema, version) {
|
|
|
1088
1110
|
export function versionTableExists(schema) {
|
|
1089
1111
|
return `SELECT to_regclass('${schema}.version') as name`;
|
|
1090
1112
|
}
|
|
1113
|
+
// Installed pg-boss schemas whose name differs from the configured one by case alone. Postgres
|
|
1114
|
+
// folds a bare name and stores a quoted one verbatim, so `MySchema` and `"MySchema"` are two
|
|
1115
|
+
// schemas that look nearly identical in config. Used on the install path to tell a caller who
|
|
1116
|
+
// mis-spelled the quoting that their data is next door, rather than silently installing a second,
|
|
1117
|
+
// empty schema beside the populated one.
|
|
1118
|
+
export function getSchemaCaseVariants(schema) {
|
|
1119
|
+
const resolved = resolveSchemaName(schema).replace(SINGLE_QUOTE_REGEX, "''");
|
|
1120
|
+
return `
|
|
1121
|
+
SELECT n.nspname as name
|
|
1122
|
+
FROM pg_namespace n
|
|
1123
|
+
JOIN pg_class c ON c.relnamespace = n.oid AND c.relname = 'version' AND c.relkind IN ('r', 'p')
|
|
1124
|
+
WHERE lower(n.nspname) = lower('${resolved}') AND n.nspname <> '${resolved}'
|
|
1125
|
+
ORDER BY n.nspname
|
|
1126
|
+
`;
|
|
1127
|
+
}
|
|
1091
1128
|
export function getPartitionedQueueTables(schema) {
|
|
1092
|
-
return `SELECT table_name FROM ${schema}.queue WHERE partition = true`;
|
|
1129
|
+
return `SELECT table_name, policy FROM ${schema}.queue WHERE partition = true`;
|
|
1093
1130
|
}
|
|
1094
1131
|
export function insertVersion(schema, version) {
|
|
1095
1132
|
return `INSERT INTO ${schema}.version(version) VALUES ('${version}')`;
|
|
@@ -1168,6 +1205,7 @@ function buildFetchParams(options) {
|
|
|
1168
1205
|
*/
|
|
1169
1206
|
export function fetchNextJob(options, noSkipLocked = false) {
|
|
1170
1207
|
const { schema, table, name, policy, limit, includeMetadata, priority = true, orderByCreatedOn = true, ignoreStartAfter = false, groupConcurrency, minPriority, maxPriority } = options;
|
|
1208
|
+
const keyStrictFifo = policy === QUEUE_POLICIES.key_strict_fifo;
|
|
1171
1209
|
const singletonFetch = limit > 1 && (policy === QUEUE_POLICIES.singleton || policy === QUEUE_POLICIES.stately);
|
|
1172
1210
|
const hasIgnoreSingletons = options.ignoreSingletons != null && options.ignoreSingletons.length > 0;
|
|
1173
1211
|
const hasIgnoreGroups = options.ignoreGroups != null && options.ignoreGroups.length > 0;
|
|
@@ -1228,6 +1266,23 @@ export function fetchNextJob(options, noSkipLocked = false) {
|
|
|
1228
1266
|
: `(j.group_id IS NULL
|
|
1229
1267
|
OR ${activeGroupCountExpression} < ${groupLimit})`
|
|
1230
1268
|
: '';
|
|
1269
|
+
// state DESC (retry > created in the enum) makes a retry job its key's head. job_i8
|
|
1270
|
+
// guarantees at most one job per key in active/retry/failed, and the NOT EXISTS blocker
|
|
1271
|
+
// below rejects every sibling of a retry job — so if created_on picked the head, an older
|
|
1272
|
+
// deferred job whose start_after has since arrived would claim the head slot while being
|
|
1273
|
+
// unfetchable, and the retry job (fetchable but not the head) would deadlock the key.
|
|
1274
|
+
const strictFifoHeadsCte = keyStrictFifo
|
|
1275
|
+
? `strict_fifo_heads AS MATERIALIZED (
|
|
1276
|
+
SELECT DISTINCT ON (h.singleton_key) h.id
|
|
1277
|
+
FROM ${schema}.${table} h
|
|
1278
|
+
WHERE h.name = '${name}'
|
|
1279
|
+
AND h.state < '${JOB_STATES.active}'
|
|
1280
|
+
AND NOT h.blocked
|
|
1281
|
+
AND h.policy = '${QUEUE_POLICIES.key_strict_fifo}'
|
|
1282
|
+
${!ignoreStartAfter ? 'AND h.start_after <= now()' : ''}
|
|
1283
|
+
ORDER BY h.singleton_key, h.state DESC, h.created_on, h.id
|
|
1284
|
+
), `
|
|
1285
|
+
: '';
|
|
1231
1286
|
const whereConditions = [
|
|
1232
1287
|
`j.name = '${name}'`,
|
|
1233
1288
|
`j.state < '${JOB_STATES.active}'`,
|
|
@@ -1238,6 +1293,18 @@ export function fetchNextJob(options, noSkipLocked = false) {
|
|
|
1238
1293
|
// timestamp, so `<` would leave freshly-inserted jobs invisible until the clock ticks.
|
|
1239
1294
|
// NOTIFY gating already uses `start_after <= now()` for the same reason.
|
|
1240
1295
|
!ignoreStartAfter ? 'j.start_after <= now()' : '',
|
|
1296
|
+
keyStrictFifo ? 'j.id IN (SELECT id FROM strict_fifo_heads)' : '',
|
|
1297
|
+
keyStrictFifo
|
|
1298
|
+
? `NOT EXISTS (
|
|
1299
|
+
SELECT 1
|
|
1300
|
+
FROM ${schema}.${table} b
|
|
1301
|
+
WHERE b.name = j.name
|
|
1302
|
+
AND b.singleton_key = j.singleton_key
|
|
1303
|
+
AND b.state IN ('${JOB_STATES.active}', '${JOB_STATES.retry}', '${JOB_STATES.failed}')
|
|
1304
|
+
AND b.policy = '${QUEUE_POLICIES.key_strict_fifo}'
|
|
1305
|
+
AND b.id <> j.id
|
|
1306
|
+
)`
|
|
1307
|
+
: '',
|
|
1241
1308
|
hasIgnoreSingletons ? `COALESCE(j.singleton_key, '') <> ALL(${params.ignoreSingletonsParam})` : '',
|
|
1242
1309
|
hasIgnoreGroups ? `(j.group_id IS NULL OR j.group_id <> ALL(${params.ignoreGroupsParam}))` : '',
|
|
1243
1310
|
hasMinPriority ? `j.priority >= ${params.minPriorityParam}` : '',
|
|
@@ -1296,6 +1363,7 @@ export function fetchNextJob(options, noSkipLocked = false) {
|
|
|
1296
1363
|
return {
|
|
1297
1364
|
text: `
|
|
1298
1365
|
WITH
|
|
1366
|
+
${strictFifoHeadsCte}
|
|
1299
1367
|
${activeGroupCountMapCte}
|
|
1300
1368
|
${nextCte}
|
|
1301
1369
|
${singletonCte}
|
|
@@ -1443,6 +1511,26 @@ export function restoreJobs(schema, table) {
|
|
|
1443
1511
|
AND id = ANY($2::uuid[])
|
|
1444
1512
|
`;
|
|
1445
1513
|
}
|
|
1514
|
+
// A `startAfter` string is either an absolute date time or a delay expressed as a Postgres
|
|
1515
|
+
// interval. A trailing 'Z' has always marked a date time; a leading ISO 8601 calendar date
|
|
1516
|
+
// (YYYY-MM-DD) marks one as well, which is what lets the other 8601 zone designators through
|
|
1517
|
+
// ('+00:00', '+05:30', '-08:00') along with zone-less and date-only strings — all of which
|
|
1518
|
+
// used to reach the interval cast and fail as 'invalid input syntax for type interval'.
|
|
1519
|
+
//
|
|
1520
|
+
// Recognition is only ever widened, so anything that resolves as an interval today still
|
|
1521
|
+
// does: bare seconds ('0', '300'), phrases ('5 minutes'), ISO 8601 durations ('PT1H') and
|
|
1522
|
+
// the year-month form ('2027-01', which Postgres reads as 2027 years 1 mon) carry neither
|
|
1523
|
+
// mark. A date time with an explicit offset resolves to that exact instant.
|
|
1524
|
+
//
|
|
1525
|
+
// A zone-less date time would otherwise be cast in the database session's TimeZone, so the
|
|
1526
|
+
// public entry points pin it to UTC before it gets here (Attorney.pinZonelessDateTime). This
|
|
1527
|
+
// cast is still session-TZ dependent for anything that reaches it unpinned, which is why the
|
|
1528
|
+
// pin lives at the boundary rather than in this expression: a caller may legitimately pass a
|
|
1529
|
+
// form Postgres resolves itself ('2027-01-01 08:00:00 America/New_York'), and rewriting those
|
|
1530
|
+
// in SQL would mean re-implementing timestamp parsing in a regex.
|
|
1531
|
+
function isDateTimeString(expression) {
|
|
1532
|
+
return `(right(${expression}, 1) = 'Z' OR ${expression} ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}')`;
|
|
1533
|
+
}
|
|
1446
1534
|
export function insertJobs(schema, { table, name, returnId = true, notify = false }) {
|
|
1447
1535
|
// When notify is enabled we always RETURN start_after so the wrapper below can gate
|
|
1448
1536
|
// the NOTIFY on immediate availability, regardless of whether the caller wants ids.
|
|
@@ -1501,7 +1589,7 @@ export function insertJobs(schema, { table, name, returnId = true, notify = fals
|
|
|
1501
1589
|
FROM (
|
|
1502
1590
|
SELECT *,
|
|
1503
1591
|
CASE
|
|
1504
|
-
WHEN
|
|
1592
|
+
WHEN ${isDateTimeString('"startAfter"')} THEN CAST("startAfter" as timestamp with time zone)
|
|
1505
1593
|
ELSE now() + CAST(COALESCE("startAfter",'0') as interval)
|
|
1506
1594
|
END as start_after
|
|
1507
1595
|
FROM json_to_recordset($1::json) as x (
|
|
@@ -1775,10 +1863,11 @@ function failJobsBody(schema, table, where, output, forceTerminal = false) {
|
|
|
1775
1863
|
SELECT * FROM failed_jobs
|
|
1776
1864
|
),
|
|
1777
1865
|
dlq_jobs as (
|
|
1778
|
-
INSERT INTO ${schema}.job (name, data, output, retry_limit, retry_backoff, retry_delay, keep_until, deletion_seconds,
|
|
1779
|
-
expire_seconds, source_name, source_id, source_created_on, source_retry_count, singleton_key, heartbeat_seconds)
|
|
1866
|
+
INSERT INTO ${schema}.job (name, priority, data, output, retry_limit, retry_backoff, retry_delay, keep_until, deletion_seconds,
|
|
1867
|
+
expire_seconds, source_name, source_id, source_created_on, source_retry_count, singleton_key, group_id, group_tier, heartbeat_seconds)
|
|
1780
1868
|
SELECT
|
|
1781
1869
|
r.dead_letter,
|
|
1870
|
+
r.priority,
|
|
1782
1871
|
r.data,
|
|
1783
1872
|
r.output,
|
|
1784
1873
|
q.retry_limit,
|
|
@@ -1792,7 +1881,9 @@ function failJobsBody(schema, table, where, output, forceTerminal = false) {
|
|
|
1792
1881
|
r.created_on,
|
|
1793
1882
|
r.retry_count,
|
|
1794
1883
|
r.singleton_key,
|
|
1795
|
-
r.
|
|
1884
|
+
r.group_id,
|
|
1885
|
+
r.group_tier,
|
|
1886
|
+
q.heartbeat_seconds
|
|
1796
1887
|
FROM results r
|
|
1797
1888
|
JOIN ${schema}.queue q ON q.name = r.dead_letter
|
|
1798
1889
|
WHERE state = '${JOB_STATES.failed}'
|
|
@@ -1987,9 +2078,10 @@ export function insertRetryJob(schema, table) {
|
|
|
1987
2078
|
export function insertDeadLetterJob(schema) {
|
|
1988
2079
|
return `
|
|
1989
2080
|
INSERT INTO ${schema}.job (name, data, output, retry_limit, retry_backoff, retry_delay, keep_until, deletion_seconds,
|
|
1990
|
-
expire_seconds, source_name, source_id, source_created_on, source_retry_count, singleton_key, heartbeat_seconds
|
|
2081
|
+
expire_seconds, source_name, source_id, source_created_on, source_retry_count, singleton_key, heartbeat_seconds,
|
|
2082
|
+
priority, group_id, group_tier)
|
|
1991
2083
|
SELECT $1, $2, $3, q.retry_limit, q.retry_backoff, q.retry_delay, now() + q.retention_seconds * interval '1s', q.deletion_seconds,
|
|
1992
|
-
q.expire_seconds, $4, $5, $6, $7, $8, $9
|
|
2084
|
+
q.expire_seconds, $4, $5, $6, $7, $8, q.heartbeat_seconds, $9, $10, $11
|
|
1993
2085
|
FROM ${schema}.queue q WHERE q.name = $1
|
|
1994
2086
|
`;
|
|
1995
2087
|
}
|
|
@@ -1998,7 +2090,11 @@ export function insertDeadLetterJob(schema) {
|
|
|
1998
2090
|
// oldest-first, capped at $4. The JOIN in `candidates` only matches jobs whose destination queue
|
|
1999
2091
|
// exists, so legacy/orphaned jobs (NULL source_name, no override) are never deleted — they stay
|
|
2000
2092
|
// in the DLQ rather than being lost. Re-created jobs get a new id, `created` state, retry_count 0,
|
|
2001
|
-
// cleared output, NULL source_*, and
|
|
2093
|
+
// cleared output, NULL source_*, and every queue-config column (retry/retention/policy/expiry/
|
|
2094
|
+
// heartbeat/dead_letter) from the destination queue as it is configured now — per-job overrides
|
|
2095
|
+
// from the original send() are not preserved, since the DLQ copy never stored them. `dead_letter`
|
|
2096
|
+
// is the same value send() falls back to, so a second terminal failure re-enters the DLQ.
|
|
2097
|
+
// Job-identity columns (priority, singleton_key, group_id, group_tier) are carried over instead.
|
|
2002
2098
|
export function redriveJobs(schema, table) {
|
|
2003
2099
|
return `
|
|
2004
2100
|
WITH candidates AS (
|
|
@@ -2020,11 +2116,12 @@ export function redriveJobs(schema, table) {
|
|
|
2020
2116
|
ins AS (
|
|
2021
2117
|
INSERT INTO ${schema}.job
|
|
2022
2118
|
(name, data, priority, retry_limit, retry_backoff, retry_delay, retry_delay_max,
|
|
2023
|
-
expire_seconds, keep_until, deletion_seconds, policy, singleton_key,
|
|
2119
|
+
expire_seconds, keep_until, deletion_seconds, policy, singleton_key, group_id, group_tier,
|
|
2120
|
+
heartbeat_seconds, dead_letter)
|
|
2024
2121
|
SELECT COALESCE($2, m.source_name), m.data, m.priority, q.retry_limit, q.retry_backoff,
|
|
2025
2122
|
q.retry_delay, q.retry_delay_max, q.expire_seconds,
|
|
2026
2123
|
now() + q.retention_seconds * interval '1s', q.deletion_seconds, q.policy,
|
|
2027
|
-
m.singleton_key, m.heartbeat_seconds
|
|
2124
|
+
m.singleton_key, m.group_id, m.group_tier, q.heartbeat_seconds, q.dead_letter
|
|
2028
2125
|
FROM moved m JOIN ${schema}.queue q ON q.name = COALESCE($2, m.source_name)
|
|
2029
2126
|
-- A destination queue's short/stately policy can still collide on (name, singleton_key)
|
|
2030
2127
|
-- if two redriven jobs share a key (job_i1/job_i3); dropping just that row here, matching
|
|
@@ -2080,11 +2177,11 @@ export function updateJob(schema, table, name, by, match, notify = false) {
|
|
|
2080
2177
|
const ordering = (by === 'singletonKey' && match !== 'all')
|
|
2081
2178
|
? `ORDER BY job.created_on ${match === 'oldest' ? 'ASC' : 'DESC'} LIMIT 1`
|
|
2082
2179
|
: '';
|
|
2083
|
-
// Resolve the incoming startAfter the same way insertJobs does (absolute
|
|
2180
|
+
// Resolve the incoming startAfter the same way insertJobs does (absolute date time vs.
|
|
2084
2181
|
// relative interval), falling back to the row's current start_after when not supplied.
|
|
2085
2182
|
const resolvedStartAfter = `
|
|
2086
2183
|
CASE WHEN jsonb_exists(o.data, 'startAfter')
|
|
2087
|
-
THEN CASE WHEN
|
|
2184
|
+
THEN CASE WHEN ${isDateTimeString("o.data->>'startAfter'")}
|
|
2088
2185
|
THEN (o.data->>'startAfter')::timestamptz
|
|
2089
2186
|
ELSE now() + CAST(o.data->>'startAfter' AS interval) END
|
|
2090
2187
|
ELSE job.start_after END`;
|
|
@@ -2278,9 +2375,12 @@ export function locked(schema, query, key, noAdvisoryLocks) {
|
|
|
2278
2375
|
const statements = Array.isArray(query) ? query : [query];
|
|
2279
2376
|
return transaction(noAdvisoryLocks ? statements : [advisoryLock(schema, key), ...statements]);
|
|
2280
2377
|
}
|
|
2378
|
+
// normalizeSchemaName, not resolveSchemaName: the key is opaque to postgres and never compared
|
|
2379
|
+
// against the catalog, so it only has to agree across instances on the same schema. See the note
|
|
2380
|
+
// on the helper.
|
|
2281
2381
|
function advisoryLock(schema, key) {
|
|
2282
2382
|
return `SELECT pg_advisory_xact_lock(
|
|
2283
|
-
('x' || encode(sha224((current_database() || '.pgboss.${schema}${key || ''}')::bytea), 'hex'))::bit(64)::bigint
|
|
2383
|
+
('x' || encode(sha224((current_database() || '.pgboss.${normalizeSchemaName(schema)}${key || ''}')::bytea), 'hex'))::bit(64)::bigint
|
|
2284
2384
|
)`;
|
|
2285
2385
|
}
|
|
2286
2386
|
export function assertMigration(schema, version) {
|
|
@@ -2437,7 +2537,7 @@ export function getNextBamCommand(schema, { useLiveness = false } = {}) {
|
|
|
2437
2537
|
AND l.granted
|
|
2438
2538
|
AND l.mode = 'ShareUpdateExclusiveLock'
|
|
2439
2539
|
AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())
|
|
2440
|
-
AND l.relation = to_regclass(quote_ident('${schema}') || '.' || quote_ident(${tableCol}))
|
|
2540
|
+
AND l.relation = to_regclass(quote_ident('${resolveSchemaName(schema)}') || '.' || quote_ident(${tableCol}))
|
|
2441
2541
|
)`;
|
|
2442
2542
|
const stale = (startedCol, tableCol) => `(
|
|
2443
2543
|
${startedCol} < now() - interval '${BAM_LIVENESS_GRACE_SECONDS} seconds'
|
|
@@ -2510,7 +2610,7 @@ export function bamHealProbe(schema, command) {
|
|
|
2510
2610
|
FROM pg_class c
|
|
2511
2611
|
JOIN pg_index i ON i.indexrelid = c.oid
|
|
2512
2612
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
2513
|
-
WHERE n.nspname = '${schema.replace(SINGLE_QUOTE_REGEX, "''")}' AND c.relname = '${match[1].replace(SINGLE_QUOTE_REGEX, "''")}'
|
|
2613
|
+
WHERE n.nspname = '${resolveSchemaName(schema).replace(SINGLE_QUOTE_REGEX, "''")}' AND c.relname = '${match[1].replace(SINGLE_QUOTE_REGEX, "''")}'
|
|
2514
2614
|
`;
|
|
2515
2615
|
}
|
|
2516
2616
|
export function setBamCompleted(schema, id) {
|
|
@@ -2577,7 +2677,8 @@ const POLICY_JOB_INDEXES = {
|
|
|
2577
2677
|
2: QUEUE_POLICIES.singleton,
|
|
2578
2678
|
3: QUEUE_POLICIES.stately,
|
|
2579
2679
|
6: QUEUE_POLICIES.exclusive,
|
|
2580
|
-
8: QUEUE_POLICIES.key_strict_fifo
|
|
2680
|
+
8: QUEUE_POLICIES.key_strict_fifo,
|
|
2681
|
+
10: QUEUE_POLICIES.key_strict_fifo
|
|
2581
2682
|
};
|
|
2582
2683
|
// job_iN indexes with no policy gate — created on every job table regardless of policy
|
|
2583
2684
|
// (throttle i4, fetch i5, group-concurrency i7, blocking i9).
|
|
@@ -2677,7 +2778,7 @@ export function expectedManagedFunctions(schema, partitioned) {
|
|
|
2677
2778
|
export function expectedManagedIndexes(schema, partitioned, partitions = []) {
|
|
2678
2779
|
const managed = (name, table, indexdef) => {
|
|
2679
2780
|
const def = applyManifestSchema(indexdef, schema);
|
|
2680
|
-
return { name, table, keys: indexKeysRaw(def), predicate: indexPredicateRaw(def), definition: displayIndexDefinition(def) };
|
|
2781
|
+
return { name, table, keys: indexKeysRaw(def), include: indexIncludeRaw(def), predicate: indexPredicateRaw(def), definition: displayIndexDefinition(def) };
|
|
2681
2782
|
};
|
|
2682
2783
|
const jobTable = partitioned ? COMMON_JOB_TABLE : 'job';
|
|
2683
2784
|
const out = [];
|
package/dist/schema.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_generated": "Generated by scripts/gen-manifest.ts — do not edit by hand. Run `npm run gen:manifest`.",
|
|
3
|
-
"version":
|
|
3
|
+
"version": 38,
|
|
4
4
|
"schemaToken": "{{schema}}",
|
|
5
5
|
"partitioned": {
|
|
6
6
|
"tables": [
|
|
@@ -1060,6 +1060,12 @@
|
|
|
1060
1060
|
"valid": true,
|
|
1061
1061
|
"def": "CREATE UNIQUE INDEX job_common_i1 ON {{schema}}.job_common USING btree (name, COALESCE(singleton_key, ''::text)) WHERE ((state = 'created'::{{schema}}.job_state) AND (policy = 'short'::text))"
|
|
1062
1062
|
},
|
|
1063
|
+
{
|
|
1064
|
+
"name": "job_common_i10",
|
|
1065
|
+
"table": "job_common",
|
|
1066
|
+
"valid": true,
|
|
1067
|
+
"def": "CREATE INDEX job_common_i10 ON {{schema}}.job_common USING btree (name, singleton_key, state DESC, created_on, id) INCLUDE (start_after) WHERE ((state < 'active'::{{schema}}.job_state) AND (NOT blocked) AND (policy = 'key_strict_fifo'::text))"
|
|
1068
|
+
},
|
|
1063
1069
|
{
|
|
1064
1070
|
"name": "job_common_i2",
|
|
1065
1071
|
"table": "job_common",
|
|
@@ -1130,7 +1136,7 @@
|
|
|
1130
1136
|
"functions": [
|
|
1131
1137
|
{
|
|
1132
1138
|
"name": "create_queue",
|
|
1133
|
-
"def": "CREATE OR REPLACE FUNCTION {{schema}}.create_queue(queue_name text, options jsonb)\n RETURNS void\n LANGUAGE plpgsql\nAS $function$\n DECLARE\n tablename varchar := CASE WHEN options->>'partition' = 'true'\n THEN 'j' || encode(sha224(queue_name::bytea), 'hex')\n ELSE 'job_common'\n END;\n queue_created_on timestamptz;\n BEGIN\n\n WITH q as (\n INSERT INTO {{schema}}.queue (\n name,\n policy,\n retry_limit,\n retry_delay,\n retry_backoff,\n retry_delay_max,\n expire_seconds,\n retention_seconds,\n deletion_seconds,\n warning_queued,\n dead_letter,\n partition,\n table_name,\n heartbeat_seconds,\n notify\n )\n VALUES (\n queue_name,\n options->>'policy',\n COALESCE((options->>'retryLimit')::int, 2),\n COALESCE((options->>'retryDelay')::int, 0),\n COALESCE((options->>'retryBackoff')::bool, false),\n (options->>'retryDelayMax')::int,\n COALESCE((options->>'expireInSeconds')::int, 900),\n COALESCE((options->>'retentionSeconds')::int, 1209600),\n COALESCE((options->>'deleteAfterSeconds')::int, 604800),\n COALESCE((options->>'warningQueueSize')::int, 0),\n options->>'deadLetter',\n COALESCE((options->>'partition')::bool, false),\n tablename,\n (options->>'heartbeatSeconds')::int,\n COALESCE((options->>'notify')::bool, false)\n )\n ON CONFLICT DO NOTHING\n RETURNING created_on\n )\n SELECT created_on into queue_created_on from q;\n\n IF queue_created_on IS NULL OR options->>'partition' IS DISTINCT FROM 'true' THEN\n RETURN;\n END IF;\n\n EXECUTE format('CREATE TABLE {{schema}}.%I (LIKE {{schema}}.job INCLUDING DEFAULTS)', tablename);\n\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD PRIMARY KEY (name, id)$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES {{schema}}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES {{schema}}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, tablename);\n\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i5 ON {{schema}}.job (name, start_after) WHERE state < 'active' AND NOT blocked$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i4 ON {{schema}}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> 'cancelled' AND singleton_on IS NOT NULL$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i7 ON {{schema}}.job (name, group_id) WHERE state = 'active' AND group_id IS NOT NULL$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i9 ON {{schema}}.job (name, id) WHERE blocking AND state = 'completed'$cmd$, tablename);\n\n IF options->>'policy' = 'short' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i1 ON {{schema}}.job (name, COALESCE(singleton_key, '')) WHERE state = 'created' AND policy = 'short'$cmd$, tablename);\n ELSIF options->>'policy' = 'singleton' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i2 ON {{schema}}.job (name, COALESCE(singleton_key, '')) WHERE state = 'active' AND policy = 'singleton'$cmd$, tablename);\n ELSIF options->>'policy' = 'stately' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i3 ON {{schema}}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'stately'$cmd$, tablename);\n ELSIF options->>'policy' = 'exclusive' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i6 ON {{schema}}.job (name, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'exclusive'$cmd$, tablename);\n ELSIF options->>'policy' = 'key_strict_fifo' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i8 ON {{schema}}.job (name, singleton_key) WHERE state IN ('active', 'retry', 'failed') AND policy = 'key_strict_fifo'$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD CONSTRAINT job_key_strict_fifo_singleton_key_check CHECK (NOT (policy = 'key_strict_fifo' AND singleton_key IS NULL))$cmd$, tablename);\n END IF;\n\n EXECUTE format('ALTER TABLE {{schema}}.%I ADD CONSTRAINT cjc CHECK (name=%L)', tablename, queue_name);\n EXECUTE format('ALTER TABLE {{schema}}.job ATTACH PARTITION {{schema}}.%I FOR VALUES IN (%L)', tablename, queue_name);\n END;\n $function$\n"
|
|
1139
|
+
"def": "CREATE OR REPLACE FUNCTION {{schema}}.create_queue(queue_name text, options jsonb)\n RETURNS void\n LANGUAGE plpgsql\nAS $function$\n DECLARE\n tablename varchar := CASE WHEN options->>'partition' = 'true'\n THEN 'j' || encode(sha224(queue_name::bytea), 'hex')\n ELSE 'job_common'\n END;\n queue_created_on timestamptz;\n BEGIN\n\n WITH q as (\n INSERT INTO {{schema}}.queue (\n name,\n policy,\n retry_limit,\n retry_delay,\n retry_backoff,\n retry_delay_max,\n expire_seconds,\n retention_seconds,\n deletion_seconds,\n warning_queued,\n dead_letter,\n partition,\n table_name,\n heartbeat_seconds,\n notify\n )\n VALUES (\n queue_name,\n options->>'policy',\n COALESCE((options->>'retryLimit')::int, 2),\n COALESCE((options->>'retryDelay')::int, 0),\n COALESCE((options->>'retryBackoff')::bool, false),\n (options->>'retryDelayMax')::int,\n COALESCE((options->>'expireInSeconds')::int, 900),\n COALESCE((options->>'retentionSeconds')::int, 1209600),\n COALESCE((options->>'deleteAfterSeconds')::int, 604800),\n COALESCE((options->>'warningQueueSize')::int, 0),\n options->>'deadLetter',\n COALESCE((options->>'partition')::bool, false),\n tablename,\n (options->>'heartbeatSeconds')::int,\n COALESCE((options->>'notify')::bool, false)\n )\n ON CONFLICT DO NOTHING\n RETURNING created_on\n )\n SELECT created_on into queue_created_on from q;\n\n IF queue_created_on IS NULL OR options->>'partition' IS DISTINCT FROM 'true' THEN\n RETURN;\n END IF;\n\n EXECUTE format('CREATE TABLE {{schema}}.%I (LIKE {{schema}}.job INCLUDING DEFAULTS)', tablename);\n\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD PRIMARY KEY (name, id)$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD CONSTRAINT q_fkey FOREIGN KEY (name) REFERENCES {{schema}}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD CONSTRAINT dlq_fkey FOREIGN KEY (dead_letter) REFERENCES {{schema}}.queue (name) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED$cmd$, tablename);\n\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i5 ON {{schema}}.job (name, start_after) WHERE state < 'active' AND NOT blocked$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i4 ON {{schema}}.job (name, singleton_on, COALESCE(singleton_key, '')) WHERE state <> 'cancelled' AND singleton_on IS NOT NULL$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i7 ON {{schema}}.job (name, group_id) WHERE state = 'active' AND group_id IS NOT NULL$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i9 ON {{schema}}.job (name, id) WHERE blocking AND state = 'completed'$cmd$, tablename);\n\n IF options->>'policy' = 'short' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i1 ON {{schema}}.job (name, COALESCE(singleton_key, '')) WHERE state = 'created' AND policy = 'short'$cmd$, tablename);\n ELSIF options->>'policy' = 'singleton' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i2 ON {{schema}}.job (name, COALESCE(singleton_key, '')) WHERE state = 'active' AND policy = 'singleton'$cmd$, tablename);\n ELSIF options->>'policy' = 'stately' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i3 ON {{schema}}.job (name, state, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'stately'$cmd$, tablename);\n ELSIF options->>'policy' = 'exclusive' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i6 ON {{schema}}.job (name, COALESCE(singleton_key, '')) WHERE state <= 'active' AND policy = 'exclusive'$cmd$, tablename);\n ELSIF options->>'policy' = 'key_strict_fifo' THEN\n EXECUTE {{schema}}.job_table_format($cmd$CREATE UNIQUE INDEX job_i8 ON {{schema}}.job (name, singleton_key) WHERE state IN ('active', 'retry', 'failed') AND policy = 'key_strict_fifo'$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$CREATE INDEX job_i10 ON {{schema}}.job (name, singleton_key, state DESC, created_on, id) INCLUDE (start_after) WHERE state < 'active' AND NOT blocked AND policy = 'key_strict_fifo'$cmd$, tablename);\n EXECUTE {{schema}}.job_table_format($cmd$ALTER TABLE {{schema}}.job ADD CONSTRAINT job_key_strict_fifo_singleton_key_check CHECK (NOT (policy = 'key_strict_fifo' AND singleton_key IS NULL))$cmd$, tablename);\n END IF;\n\n EXECUTE format('ALTER TABLE {{schema}}.%I ADD CONSTRAINT cjc CHECK (name=%L)', tablename, queue_name);\n EXECUTE format('ALTER TABLE {{schema}}.job ATTACH PARTITION {{schema}}.%I FOR VALUES IN (%L)', tablename, queue_name);\n END;\n $function$\n"
|
|
1134
1140
|
},
|
|
1135
1141
|
{
|
|
1136
1142
|
"name": "delete_queue",
|
|
@@ -1986,6 +1992,12 @@
|
|
|
1986
1992
|
"valid": true,
|
|
1987
1993
|
"def": "CREATE UNIQUE INDEX job_i1 ON {{schema}}.job USING btree (name, COALESCE(singleton_key, ''::text)) WHERE ((state = 'created'::{{schema}}.job_state) AND (policy = 'short'::text))"
|
|
1988
1994
|
},
|
|
1995
|
+
{
|
|
1996
|
+
"name": "job_i10",
|
|
1997
|
+
"table": "job",
|
|
1998
|
+
"valid": true,
|
|
1999
|
+
"def": "CREATE INDEX job_i10 ON {{schema}}.job USING btree (name, singleton_key, state DESC, created_on, id) INCLUDE (start_after) WHERE ((state < 'active'::{{schema}}.job_state) AND (NOT blocked) AND (policy = 'key_strict_fifo'::text))"
|
|
2000
|
+
},
|
|
1989
2001
|
{
|
|
1990
2002
|
"name": "job_i2",
|
|
1991
2003
|
"table": "job",
|
package/dist/timekeeper.d.ts
CHANGED
package/dist/timekeeper.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"timekeeper.d.ts","sourceRoot":"","sources":["../src/timekeeper.ts"],"names":[],"mappings":"AACA,OAAO,YAAY,MAAM,aAAa,CAAA;AAGtC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAGvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAGnC,eAAO,MAAM,MAAM;;CAElB,CAAA;
|
|
1
|
+
{"version":3,"file":"timekeeper.d.ts","sourceRoot":"","sources":["../src/timekeeper.ts"],"names":[],"mappings":"AACA,OAAO,YAAY,MAAM,aAAa,CAAA;AAGtC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAGvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAGnC,eAAO,MAAM,MAAM;;CAElB,CAAA;AAwCD,cAAM,UAAW,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;IAChE,EAAE,EAAE,KAAK,CAAC,SAAS,CAAA;IACnB,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAA;IACxC,OAAO,EAAE,OAAO,CAAA;IAEhB,OAAO,CAAC,OAAO,CAAO;IACtB,OAAO,CAAC,mBAAmB,CAAmC;IAC9D,OAAO,CAAC,mBAAmB,CAAmC;IAC9D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,aAAa,CAAQ;IAS7B,OAAO,CAAC,eAAe,CAAoB;IAE3C,SAAS,SAAI;IACb,MAAM;;;;MAAS;gBAEF,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAQ5F,IAAI,YAAY,IAAK,OAAO,CAE3B;IAED,OAAO,KAAK,cAAc,GASzB;IAEK,KAAK;IAqBL,IAAI;IAwBJ,cAAc;IAyCd,MAAM;IA0BN,IAAI;IA8CV,YAAY,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM;YAYxB,QAAQ;IAWhB,YAAY,CAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;IAiBrE,QAAQ,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBzG,UAAU,CAAE,IAAI,EAAE,MAAM,EAAE,GAAG,SAAK,GAAG,OAAO,CAAC,IAAI,CAAC;CAIzD;AAED,eAAe,UAAU,CAAA"}
|
package/dist/timekeeper.js
CHANGED
|
@@ -19,8 +19,30 @@ const WARNINGS = {
|
|
|
19
19
|
}
|
|
20
20
|
};
|
|
21
21
|
const WARNING_TYPES = {
|
|
22
|
-
CLOCK_SKEW: 'clock_skew'
|
|
22
|
+
CLOCK_SKEW: 'clock_skew',
|
|
23
|
+
INVALID_SCHEDULE: 'invalid_schedule'
|
|
23
24
|
};
|
|
25
|
+
/**
|
|
26
|
+
* Asserts that `tz` is a time zone cron evaluation can actually use.
|
|
27
|
+
*
|
|
28
|
+
* cron-parser validates `tz` lazily: parsing without a reference date never constructs a CronDate,
|
|
29
|
+
* so every string is accepted and a bad zone only surfaces later, when a date is computed, as an
|
|
30
|
+
* opaque "CronDate: unhandled timestamp". Passing a reference date here forces that construction so
|
|
31
|
+
* a typo like 'America/New_Yrok' is rejected by schedule() rather than persisted to the schedule
|
|
32
|
+
* table. Deliberately reuses cron-parser rather than an independent Intl check, so what schedule()
|
|
33
|
+
* accepts is exactly what the cron pass can evaluate.
|
|
34
|
+
*
|
|
35
|
+
* The caller validates the cron expression first, so a failure here is attributable to the zone.
|
|
36
|
+
*/
|
|
37
|
+
function assertTimezone(tz) {
|
|
38
|
+
try {
|
|
39
|
+
CronExpressionParser.parse('* * * * *', { tz, strict: false, currentDate: new Date() });
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// Quoted so an empty string renders as `""` rather than a dangling colon
|
|
43
|
+
throw new Error(`Unknown or unsupported time zone: "${tz}"`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
24
46
|
class Timekeeper extends EventEmitter {
|
|
25
47
|
db;
|
|
26
48
|
config;
|
|
@@ -30,6 +52,14 @@ class Timekeeper extends EventEmitter {
|
|
|
30
52
|
skewMonitorInterval;
|
|
31
53
|
timekeeping;
|
|
32
54
|
_checkingSkew = false;
|
|
55
|
+
// Rows already warned about, keyed on (name, key, cron, timezone). Unlike every other warning
|
|
56
|
+
// type, an unusable schedule never heals on its own: clock skew converges, a backlog drains, a
|
|
57
|
+
// slow query is a one-off, but a bad row sits there until a human edits it. Warning every pass
|
|
58
|
+
// would persist a row every cronMonitorIntervalSeconds forever, and warningRetentionDays has no
|
|
59
|
+
// default, so a single typo could grow the warning table without bound. Rebuilt each pass from
|
|
60
|
+
// the rows still broken, so a fixed or deleted schedule drops out and would warn again if it
|
|
61
|
+
// came back.
|
|
62
|
+
warnedSchedules = new Set();
|
|
33
63
|
clockSkew = 0;
|
|
34
64
|
events = EVENTS;
|
|
35
65
|
constructor(db, manager, config) {
|
|
@@ -53,6 +83,8 @@ class Timekeeper extends EventEmitter {
|
|
|
53
83
|
}
|
|
54
84
|
async start() {
|
|
55
85
|
this.stopped = false;
|
|
86
|
+
// A restart should re-surface a row nobody has fixed yet
|
|
87
|
+
this.warnedSchedules.clear();
|
|
56
88
|
await this.cacheClockSkew();
|
|
57
89
|
await this.manager.createQueue(QUEUES.SEND_IT);
|
|
58
90
|
const options = {
|
|
@@ -134,9 +166,32 @@ class Timekeeper extends EventEmitter {
|
|
|
134
166
|
}
|
|
135
167
|
async cron() {
|
|
136
168
|
const schedules = await this.getSchedules();
|
|
137
|
-
const scheduled =
|
|
138
|
-
|
|
139
|
-
|
|
169
|
+
const scheduled = [];
|
|
170
|
+
const stillBroken = new Set();
|
|
171
|
+
for (const { name, key, data, options, cron, timezone } of schedules) {
|
|
172
|
+
let due;
|
|
173
|
+
try {
|
|
174
|
+
due = this.shouldSendIt(cron, timezone);
|
|
175
|
+
}
|
|
176
|
+
catch (err) {
|
|
177
|
+
// Evaluating one row must not decide the fate of the others. schedule() now rejects an
|
|
178
|
+
// unusable time zone, but a row written by an earlier release — or straight into the table —
|
|
179
|
+
// still throws here. This was a single filter() over every schedule, so one such row
|
|
180
|
+
// propagated out of cron() and silently stopped scheduling for every queue in the
|
|
181
|
+
// deployment, on every pass, until someone found the row. Skip it and warn instead, naming
|
|
182
|
+
// the schedule so it is actually fixable.
|
|
183
|
+
const warned = JSON.stringify([name, key, cron, timezone]);
|
|
184
|
+
stillBroken.add(warned);
|
|
185
|
+
if (!this.warnedSchedules.has(warned)) {
|
|
186
|
+
await emitAndPersistWarning(this.warningContext, WARNING_TYPES.INVALID_SCHEDULE, `Warning: schedule for queue "${name}" (key "${key}") could not be evaluated and was skipped: ${err.message}`, { queue: name, key, cron, timezone });
|
|
187
|
+
}
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (due) {
|
|
191
|
+
scheduled.push({ data: { name, data, options }, singletonKey: `${name}__${key}`, singletonSeconds: 60 });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
this.warnedSchedules = stillBroken;
|
|
140
195
|
if (scheduled.length > 0 && !this.stopped) {
|
|
141
196
|
await this.manager.insert(QUEUES.SEND_IT, scheduled);
|
|
142
197
|
}
|
|
@@ -173,7 +228,12 @@ class Timekeeper extends EventEmitter {
|
|
|
173
228
|
}
|
|
174
229
|
async schedule(name, cron, data, options = {}) {
|
|
175
230
|
const { tz = 'UTC', key = '', ...rest } = options;
|
|
176
|
-
|
|
231
|
+
// Expression first, so a bad expression reports as one rather than as a time zone problem. The
|
|
232
|
+
// check is deliberately run against UTC rather than the supplied tz: it only works today
|
|
233
|
+
// because cron-parser is lazy about an unusable zone, and if that ever changes this call would
|
|
234
|
+
// throw the opaque "CronDate: unhandled timestamp" that assertTimezone exists to replace.
|
|
235
|
+
CronExpressionParser.parse(cron, { tz: 'UTC', strict: false });
|
|
236
|
+
assertTimezone(tz);
|
|
177
237
|
Attorney.checkSendArgs([name, data, { ...rest }]);
|
|
178
238
|
Attorney.assertKey(key);
|
|
179
239
|
try {
|
package/dist/tools.d.ts
CHANGED
|
@@ -18,5 +18,36 @@ export interface AbortablePromise<T> extends Promise<T> {
|
|
|
18
18
|
}
|
|
19
19
|
declare function delay(ms: number, error?: string, abortController?: AbortController): AbortablePromise<void>;
|
|
20
20
|
declare function resolveWithinSeconds<T>(promise: Promise<T>, seconds: number, message?: string, abortController?: AbortController): Promise<T | void>;
|
|
21
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Resolves a configured schema string to the name postgres actually stores in the catalog.
|
|
23
|
+
*
|
|
24
|
+
* the `schema` option is interpolated verbatim into identifier positions, so a caller may quote it
|
|
25
|
+
* to reach names that are not legal bare identifiers, e.g. `'"My-Schema"'`. The catalog holds the
|
|
26
|
+
* *resolved* name, so any comparison against `pg_namespace.nspname` (and anything else reading a
|
|
27
|
+
* name back out of postgres) needs this rather than the raw config value.
|
|
28
|
+
*
|
|
29
|
+
* A quoted name resolves to its contents verbatim; a bare one is folded to lower case, as postgres
|
|
30
|
+
* does on the way in. So `"MySchema"` and `MySchema` are different schemas, the latter stored as
|
|
31
|
+
* `myschema`.
|
|
32
|
+
*
|
|
33
|
+
* Do NOT use this to derive the notify channel or the advisory lock - see normalizeSchemaName.
|
|
34
|
+
*/
|
|
35
|
+
declare function resolveSchemaName(schema: string): string;
|
|
36
|
+
/**
|
|
37
|
+
* Canonicalizes a configured schema string for the values pg-boss *derives* from it rather than
|
|
38
|
+
* looks up: the notify channel name and the advisory lock key.
|
|
39
|
+
*
|
|
40
|
+
* Those are hashes of a string, never compared against the catalog - the channel literal is
|
|
41
|
+
* generated from the same expression for both LISTEN and NOTIFY, and the advisory key is opaque to
|
|
42
|
+
* postgres. So the only requirement is that every instance pointed at the same physical schema
|
|
43
|
+
* agrees, which means the *sole* transform needed is collapsing quoting that carries no meaning:
|
|
44
|
+
* `'"pgboss"'` and `'pgboss'` are one schema and must not end up on separate channels.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately NOT resolveSchemaName. Folding a bare `MySchema` to `myschema` would also be
|
|
47
|
+
* "correct", but it changes the channel and lock key for a name that has always been legal, so a
|
|
48
|
+
* rolling upgrade would leave old and new instances failing to coordinate. Leaving bare names
|
|
49
|
+
* untouched keeps the derived values byte-identical to every prior release.
|
|
50
|
+
*/
|
|
51
|
+
declare function normalizeSchemaName(schema: string): string;
|
|
52
|
+
export { delay, normalizeSchemaName, resolveSchemaName, resolveWithinSeconds, unwrapSQLResult };
|
|
22
53
|
//# sourceMappingURL=tools.d.ts.map
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAEA;;;;;;;EAOE;AACF,iBAAS,eAAe,CAAE,MAAM,EAAE;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,EAAE,GAAG,GAAG,EAAE,GAAG;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,CAQ9F;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC,CAAE,SAAQ,OAAO,CAAC,CAAC,CAAC;IACrD,KAAK,EAAE,MAAM,IAAI,CAAA;CAClB;AAED,iBAAS,KAAK,CAAE,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,eAAe,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAsBrG;AAED,iBAAe,oBAAoB,CAAC,CAAC,EAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAapJ;AAED,OAAO,EACL,KAAK,EACL,oBAAoB,EACpB,eAAe,EAChB,CAAA"}
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAEA;;;;;;;EAOE;AACF,iBAAS,eAAe,CAAE,MAAM,EAAE;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,EAAE,GAAG,GAAG,EAAE,GAAG;IAAE,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,CAQ9F;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC,CAAE,SAAQ,OAAO,CAAC,CAAC,CAAC;IACrD,KAAK,EAAE,MAAM,IAAI,CAAA;CAClB;AAED,iBAAS,KAAK,CAAE,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,eAAe,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAsBrG;AAED,iBAAe,oBAAoB,CAAC,CAAC,EAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAapJ;AAQD;;;;;;;;;;;;;GAaG;AACH,iBAAS,iBAAiB,CAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAElD;AAED;;;;;;;;;;;;;;GAcG;AACH,iBAAS,mBAAmB,CAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,OAAO,EACL,KAAK,EACL,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EAChB,CAAA"}
|
package/dist/tools.js
CHANGED
|
@@ -48,4 +48,43 @@ async function resolveWithinSeconds(promise, seconds, message, abortController)
|
|
|
48
48
|
}
|
|
49
49
|
return result;
|
|
50
50
|
}
|
|
51
|
-
|
|
51
|
+
// a quoted name whose contents are already a legal bare identifier *and* already lower case, so
|
|
52
|
+
// postgres would resolve the two spellings to the same schema.
|
|
53
|
+
const REDUNDANTLY_QUOTED_SCHEMA_REGEX = /^"[a-z_][a-z0-9_]*"$/;
|
|
54
|
+
const isQuoted = (schema) => schema.startsWith('"') && schema.endsWith('"') && schema.length > 1;
|
|
55
|
+
/**
|
|
56
|
+
* Resolves a configured schema string to the name postgres actually stores in the catalog.
|
|
57
|
+
*
|
|
58
|
+
* the `schema` option is interpolated verbatim into identifier positions, so a caller may quote it
|
|
59
|
+
* to reach names that are not legal bare identifiers, e.g. `'"My-Schema"'`. The catalog holds the
|
|
60
|
+
* *resolved* name, so any comparison against `pg_namespace.nspname` (and anything else reading a
|
|
61
|
+
* name back out of postgres) needs this rather than the raw config value.
|
|
62
|
+
*
|
|
63
|
+
* A quoted name resolves to its contents verbatim; a bare one is folded to lower case, as postgres
|
|
64
|
+
* does on the way in. So `"MySchema"` and `MySchema` are different schemas, the latter stored as
|
|
65
|
+
* `myschema`.
|
|
66
|
+
*
|
|
67
|
+
* Do NOT use this to derive the notify channel or the advisory lock - see normalizeSchemaName.
|
|
68
|
+
*/
|
|
69
|
+
function resolveSchemaName(schema) {
|
|
70
|
+
return isQuoted(schema) ? schema.slice(1, -1) : schema.toLowerCase();
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Canonicalizes a configured schema string for the values pg-boss *derives* from it rather than
|
|
74
|
+
* looks up: the notify channel name and the advisory lock key.
|
|
75
|
+
*
|
|
76
|
+
* Those are hashes of a string, never compared against the catalog - the channel literal is
|
|
77
|
+
* generated from the same expression for both LISTEN and NOTIFY, and the advisory key is opaque to
|
|
78
|
+
* postgres. So the only requirement is that every instance pointed at the same physical schema
|
|
79
|
+
* agrees, which means the *sole* transform needed is collapsing quoting that carries no meaning:
|
|
80
|
+
* `'"pgboss"'` and `'pgboss'` are one schema and must not end up on separate channels.
|
|
81
|
+
*
|
|
82
|
+
* Deliberately NOT resolveSchemaName. Folding a bare `MySchema` to `myschema` would also be
|
|
83
|
+
* "correct", but it changes the channel and lock key for a name that has always been legal, so a
|
|
84
|
+
* rolling upgrade would leave old and new instances failing to coordinate. Leaving bare names
|
|
85
|
+
* untouched keeps the derived values byte-identical to every prior release.
|
|
86
|
+
*/
|
|
87
|
+
function normalizeSchemaName(schema) {
|
|
88
|
+
return REDUNDANTLY_QUOTED_SCHEMA_REGEX.test(schema) ? schema.slice(1, -1) : schema;
|
|
89
|
+
}
|
|
90
|
+
export { delay, normalizeSchemaName, resolveSchemaName, resolveWithinSeconds, unwrapSQLResult };
|