pg-boss 12.29.0 → 12.30.0

package/dist/attorney.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"attorney.d.ts","sourceRoot":"","sources":["../src/attorney.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,KAAK,MAAM,YAAY,CAAA;AAExC,QAAA,MAAM,MAAM;;;;CAIX,CAAA;AA+ED,iBAAS,iBAAiB,CAAE,MAAM,GAAE,GAAQ,QAc3C;AAgBD,iBAAS,mBAAmB,CAAE,KAAK,EAAE,MAAM,UAE1C;AAED,iBAAS,aAAa,CAAE,IAAI,EAAE,GAAG,GAAG,KAAK,CAAC,OAAO,CAgDhD;AAOD,iBAAS,eAAe,CAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAc,EAAE;;CAAK,GAAG,KAAK,CAAC,OAAO,CA4D3E;AAED,iBAAS,mBAAmB,CAAE,MAAM,EAAE,GAAG,QAOxC;AAED,iBAAS,gBAAgB,CAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,QA2D/C;AA2GD,iBAAS,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG;IAClD,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAA;IAClC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;CACjC,CA8BA;AAED,iBAAS,cAAc,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAQlD;AAED,iBAAS,SAAS,CAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,kBAAkB,GAAG,KAAK,CAAC,0BAA0B,CAyB9F;AA4FD,iBAAS,wBAAwB,CAAE,IAAI,EAAE,MAAM,QAiC9C;AAED,iBAAS,eAAe,CAAE,IAAI,EAAE,MAAM,QAIrC;AAED,iBAAS,SAAS,CAAE,GAAG,EAAE,MAAM,QAI9B;AAuLD,OAAO,EACL,SAAS,EACT,wBAAwB,EACxB,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,MAAM,EACN,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,EAClB,CAAA"}
+{"version":3,"file":"attorney.d.ts","sourceRoot":"","sources":["../src/attorney.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,KAAK,MAAM,YAAY,CAAA;AAExC,QAAA,MAAM,MAAM;;;;CAIX,CAAA;AAkFD,iBAAS,iBAAiB,CAAE,MAAM,GAAE,GAAQ,QAc3C;AAgBD,iBAAS,mBAAmB,CAAE,KAAK,EAAE,MAAM,UAE1C;AAED,iBAAS,aAAa,CAAE,IAAI,EAAE,GAAG,GAAG,KAAK,CAAC,OAAO,CAgDhD;AAOD,iBAAS,eAAe,CAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAc,EAAE;;CAAK,GAAG,KAAK,CAAC,OAAO,CA4D3E;AAED,iBAAS,mBAAmB,CAAE,MAAM,EAAE,GAAG,QAOxC;AAED,iBAAS,gBAAgB,CAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,QA2D/C;AA2GD,iBAAS,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG;IAClD,OAAO,EAAE,KAAK,CAAC,mBAAmB,CAAA;IAClC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;CACjC,CA8BA;AAED,iBAAS,cAAc,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAQlD;AAED,iBAAS,SAAS,CAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC,kBAAkB,GAAG,KAAK,CAAC,0BAA0B,CAyB9F;AA4FD,iBAAS,wBAAwB,CAAE,IAAI,EAAE,MAAM,QAiC9C;AAED,iBAAS,eAAe,CAAE,IAAI,EAAE,MAAM,QAIrC;AAED,iBAAS,SAAS,CAAE,GAAG,EAAE,MAAM,QAI9B;AA+LD,OAAO,EACL,SAAS,EACT,wBAAwB,EACxB,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,MAAM,EACN,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,EAClB,CAAA"}

package/dist/attorney.js

@@ -17,7 +17,8 @@ const COMPATIBILITY_FLAGS = [
     'noCoveringIndexes',
     'noListenNotify',
     'noIndexProgressView',
-    'noReindex'
+    'noReindex',
+    'noMonitorVacuum'
 ];
 // The single source of truth for backend presets, mirrored by test/testHelper.ts.
 const BACKEND_PROFILES = {
@@ -39,7 +40,8 @@ const BACKEND_PROFILES = {
             // CONCURRENTLY, "unimplemented: this syntax" without — and the bloat check itself cannot run:
             // there is no pg_relation_size(), and reltuples / relpages is an "unsupported binary
             // operator: <float4> / <int4>".
-            noReindex: true
+            noReindex: true,
+            noMonitorVacuum: true
         }
     },
     yugabytedb: {
@@ -53,7 +55,8 @@ const BACKEND_PROFILES = {
             // "REINDEX not supported yet" with or without CONCURRENTLY, and DocDB compaction owns
             // reclamation. The bloat check runs but can never match: relpages and pg_relation_size() are
             // 0 for every relation.
-            noReindex: true
+            noReindex: true,
+            noMonitorVacuum: true
         }
     },
     // No noIndexProgressView: pg-boss keeps its tables coordinator-local (it never calls
@@ -570,6 +573,7 @@ function applyOpsConfig(config) {
     }
     config.queueStatRetentionDays = config.queueStatRetentionDays || 7;
     validateReindexConfig(config);
+    validateVacuumMonitorConfig(config);
 }
 function validateReindexConfig(config) {
     assert(!('reindex' in config) || typeof config.reindex === 'boolean' || (typeof config.reindex === 'object' && config.reindex !== null), 'configuration assert: reindex must be a boolean or an options object');
@@ -587,6 +591,11 @@ function validateReindexConfig(config) {
     config.reindexIntervalSeconds = config.reindexIntervalSeconds || POLICY.MAX_EXPIRATION_HOURS * 60 * 60;
     assert(config.reindexIntervalSeconds / 60 / 60 <= POLICY.MAX_EXPIRATION_HOURS, `configuration assert: reindexIntervalSeconds cannot exceed ${POLICY.MAX_EXPIRATION_HOURS} hours`);
 }
+function validateVacuumMonitorConfig(config) {
+    // A switch, not a threshold: the check derives its trigger from the server's own autovacuum
+    // settings and from measured dead tuples, so there is no number here for a caller to get wrong.
+    assert(!('monitorVacuum' in config) || typeof config.monitorVacuum === 'boolean', 'configuration assert: monitorVacuum must be a boolean');
+}
 function validateDeletionConfig(config) {
     assert(!('deleteAfterSeconds' in config) || config.deleteAfterSeconds >= 0, 'configuration assert: deleteAfterSeconds must be at least 0 (0 disables deletion)');
 }

package/dist/boss.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"boss.d.ts","sourceRoot":"","sources":["../src/boss.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AACtC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAGvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AA0BnC,cAAM,IAAK,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAoB1D,MAAM;;;MAAS;gBAGb,EAAE,EAAE,KAAK,CAAC,SAAS,EACnB,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAa1C,IAAI,WAAW,IAAK,OAAO,CAE1B;IAEK,KAAK;IAWL,IAAI;IA8FJ,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,gBAAgB;IAsSvF;;;;OAIG;IACG,kBAAkB,CAAE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAwB7E;AAED,eAAe,IAAI,CAAA"}
+{"version":3,"file":"boss.d.ts","sourceRoot":"","sources":["../src/boss.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AACtC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AAGvC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AAgHnC,cAAM,IAAK,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAsC1D,MAAM;;;MAAS;gBAGb,EAAE,EAAE,KAAK,CAAC,SAAS,EACnB,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAa1C,IAAI,WAAW,IAAK,OAAO,CAE1B;IAEK,KAAK;IAWL,IAAI;IA8FJ,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,gBAAgB;IAwqBvF;;;;OAIG;IACG,kBAAkB,CAAE,OAAO,CAAC,EAAE,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAwB7E;AAED,eAAe,IAAI,CAAA"}

package/dist/boss.js

@@ -10,13 +10,62 @@ const events = {
 // Default thresholds and warning messages
 const WARNINGS = {
     SLOW_QUERY: { seconds: 30, message: 'Warning: slow query. Your queues and/or database server should be reviewed' },
-    LARGE_QUEUE: { size: 10_000, message: 'Warning: large queue backlog. Your queue should be reviewed' }
+    LARGE_QUEUE: { size: 10_000, message: 'Warning: large queue backlog. Your queue should be reviewed' },
+    // No threshold constant: the trigger is measured, not configured. See #checkVacuum.
+    XMIN_HORIZON: { message: 'Warning: the database transaction horizon is pinned, so completed jobs cannot be cleaned up' },
+    AUTOVACUUM_DISABLED: { message: 'Warning: autovacuum is disabled, so nothing is reclaiming completed jobs' },
+    // Deliberately weaker than XMIN_HORIZON, and the difference is the evidence behind each. That one
+    // asserts the horizon is pinned and that jobs cannot be reclaimed, having watched a vacuum run and
+    // reclaim nothing across two passes. This one has a stopwatch and nothing else. The aggregate does
+    // pin the horizon while it runs - that much is definitional - but whether any reclamation was
+    // actually lost to it is not measured here, so this reports the exposure and the action taken and
+    // leaves the diagnosis to XMIN_HORIZON, which will fire on its own if the harm is real.
+    MONITOR_BACKOFF: { message: 'Warning: queue monitoring spent long enough scanning the job table to risk holding back autovacuum, so the next stats refresh is deferred' }
 };
 const WARNING_TYPES = {
     SLOW_QUERY: 'slow_query',
     QUEUE_BACKLOG: 'queue_backlog',
-    INDEX_BLOAT: 'index_bloat'
+    INDEX_BLOAT: 'index_bloat',
+    XMIN_HORIZON: 'xmin_horizon',
+    AUTOVACUUM_DISABLED: 'autovacuum_disabled',
+    MONITOR_BACKOFF: 'monitor_backoff'
 };
+// Which holder class each source in the horizon query represents, phrased so the warning names
+// something an operator can go and look at.
+const XMIN_HOLDERS = {
+    backends: 'a backend holding an open transaction',
+    slots: 'a replication slot',
+    slotsCatalog: 'a replication slot (catalog xmin)',
+    standbys: 'a standby with hot_standby_feedback enabled',
+    prepared: 'a prepared transaction'
+};
+// Name the holder as specifically as the catalog allowed. Falls back to the holder class when the
+// source is not a backend (a replication slot has no pid to report) or when no backend row came
+// back, so the warning never loses the description it had before.
+function describeXminHolder(source, row) {
+    const holder = source === 'backends' ? row.backendHolder : null;
+    if (!holder)
+        return XMIN_HOLDERS[source];
+    const app = holder.applicationName || null;
+    const where = [
+        app && `application_name '${app}'`,
+        holder.userName && `role ${holder.userName}`,
+        `pid ${holder.pid}`
+    ].filter(Boolean).join(', ');
+    const open = holder.xactSeconds === null || holder.xactSeconds === undefined
+        ? 'for an unreadable length of time'
+        : `for ${holder.xactSeconds}s`;
+    // Three outcomes, not two, because an empty application_name cannot be classified either way and
+    // saying "another application" would be asserting something unmeasured. Comparison is against
+    // this connection's own value rather than the literal 'pgboss': an adapter-supplied pool carries
+    // whatever the host app set, so "same application_name as us" is the strongest true claim.
+    const who = !app
+        ? 'a backend with no application_name'
+        : app === row.selfApplicationName
+            ? "this application's own connection"
+            : 'another application';
+    return `${who} (${where}) has held a transaction open ${open}`;
+}
 // SQLSTATE 25001 (active_sql_transaction): "REINDEX CONCURRENTLY cannot run inside a transaction
 // block". Raised when a user-supplied adapter wraps executeSql in a transaction — a property of the
 // adapter, not of this pass, so it disables rebuilds for the life of the instance rather than
@@ -41,6 +90,24 @@ class Boss extends EventEmitter {
     // Local rate limit for passes that only report bloat, which deliberately leave the shared interval
     // claim to whichever instance can act on it.
     #detectOnly = 0;
+    // Latched while a job table is over its autovacuum budget, so a sustained pin warns once rather
+    // than every pass. Cleared when every table is back under budget, so a later episode warns again.
+    #warnedXminHorizon = false;
+    // Same latching for the sibling diagnosis: garbage growing on a table nothing vacuums at all.
+    #warnedAutovacuumDisabled = false;
+    // Per job table: the dead-tuple count and vacuum timestamp last seen while the table was over its
+    // autovacuum budget. Two observations are what turn "there is garbage" into "a vacuum ran and
+    // reclaimed none of it" - see #checkXminHorizon.
+    #garbageEvidence = new Map();
+    // Sources the connected role could not read. Narrowed once on first failure rather than retried
+    // every pass; an empty-but-unreadable source must never be reported as healthy.
+    #xminHorizonSources = null;
+    // Seconds this supervise pass held the MVCC horizon inside the queue-stats aggregate, as the
+    // server measured it, summed across every table and every 100-name chunk. The sum, not the worst
+    // single query: chunking and partitioned queues both multiply the aggregate within one pass (each
+    // chunk re-reads the whole heap - measured at 384,632 buffers for 2 of 20 queue names), and it is
+    // the total contiguous pin that starves autovacuum, not any one scan.
+    #statsElapsedSeconds = 0;
     events = events;
     constructor(db, manager, config) {
         super();
@@ -176,6 +243,12 @@ class Boss extends EventEmitter {
         }
         if (this.#stopping)
             return;
+        // Immediately after the aggregates it measures, not at the head of the next pass. Deferring the
+        // write left a whole supervise interval between a slow aggregate finishing and the gate closing
+        // behind it, and a getQueueStats({ force: true }) landing in that window ran exactly the scan
+        // the backoff was about to prevent.
+        await this.#applyMonitorBackoff();
+        await this.#checkVacuum();
         await this.#maintainWarnings();
         await this.#maintainQueueStats();
         // Last in the pass: a rebuild is DDL that can run for seconds, so nothing time-sensitive
@@ -191,21 +264,44 @@ class Boss extends EventEmitter {
             return;
         if (rows.length) {
             const queues = rows.map((q) => q.name);
-            const cacheStatsSql = plans.cacheQueueStats(this.#config.schema, table, queues, this.#config.noAdvisoryLocks);
-            const { rows: rowsCacheStats } = await this.#executeQuery(cacheStatsSql);
-            if (this.#config.persistQueueStats) {
-                const insertSql = plans.insertQueueStats(this.#config.schema, queues, this.#config.noAdvisoryLocks);
-                await this.#executeQuery(insertSql);
+            // The vacuum-safety backoff defers the stats aggregate, not the whole pass. Job expiry and
+            // heartbeat failure below are narrow indexed updates that pin nothing, and gating them on the
+            // horizon would turn a vacuum-safety valve into a job-expiry outage lasting two naptimes at
+            // minimum. See the note on plans.trySetQueueMonitorTime. The flag is a property of the
+            // statement, not of a queue, so every row carries the same value.
+            const refreshStats = rows[0].refreshStats !== false;
+            if (refreshStats) {
+                const cacheStatsSql = plans.cacheQueueStats(this.#config.schema, table, queues, this.#config.noAdvisoryLocks);
+                // The pin this pass cost, taken from the server's own clock (see the pinSeconds column in
+                // cacheQueueStats) and not from a stopwatch around the call - that would count pool wait,
+                // network and event-loop lag, none of which hold the horizon. The client measurement stays as
+                // a fallback for a backend or adapter that returns no rows to read it from.
+                const statsStarted = Date.now();
+                const { rows: rowsCacheStats } = await this.#executeQuery(cacheStatsSql);
+                const pinned = rowsCacheStats.reduce((max, row) => Math.max(max, Number(row.pinSeconds) || 0), 0);
+                // Only when the aggregate actually ran. An empty result means another instance holds the
+                // stats try-lock and this statement returned without scanning anything, so the elapsed
+                // client time is pool and network latency - exactly the measurement the server-side
+                // pinSeconds column exists to avoid backing off on.
+                if (rowsCacheStats.length) {
+                    this.#statsElapsedSeconds += pinned || (Date.now() - statsStarted) / 1000;
+                }
+                if (this.#config.persistQueueStats) {
+                    const insertSql = plans.insertQueueStats(this.#config.schema, queues, this.#config.noAdvisoryLocks);
+                    await this.#executeQuery(insertSql);
+                }
+                if (this.#stopping)
+                    return;
+                // Coerce with Number(): CockroachDB returns these integer columns as strings, so a bare `>`
+                // would compare lexicographically ("100" > "9" === false) and silently miss the backlog. On
+                // standard Postgres these are already numbers, so Number() is a no-op.
+                const warnings = rowsCacheStats.filter(i => Number(i.queuedCount) > (Number(i.warningQueueSize) || this.#largeQueueSize));
+                for (const warning of warnings) {
+                    await emitAndPersistWarning(this.#warningContext, WARNING_TYPES.QUEUE_BACKLOG, WARNINGS.LARGE_QUEUE.message, warning);
+                }
             }
             if (this.#stopping)
                 return;
-            // Coerce with Number(): CockroachDB returns these integer columns as strings, so a bare `>`
-            // would compare lexicographically ("100" > "9" === false) and silently miss the backlog. On
-            // standard Postgres these are already numbers, so Number() is a no-op.
-            const warnings = rowsCacheStats.filter(i => Number(i.queuedCount) > (Number(i.warningQueueSize) || this.#largeQueueSize));
-            for (const warning of warnings) {
-                await emitAndPersistWarning(this.#warningContext, WARNING_TYPES.QUEUE_BACKLOG, WARNINGS.LARGE_QUEUE.message, warning);
-            }
             // CockroachDB rejects the multi-mutation failJobs() CTE these use, so under noMultiMutationCte
             // route expiry through the manager's split select/delete/re-insert variants instead.
             if (this.#config.noMultiMutationCte) {
@@ -241,6 +337,317 @@ class Boss extends EventEmitter {
             await this.#executeQuery(depSql);
         }
     }
+    /**
+     * Defer the next queue-stats aggregate when this pass spent long enough inside it to threaten
+     * autovacuum, and tell the operator that it happened.
+     *
+     * The aggregate in cacheQueueStats is a whole-heap scan - the queue-name filter does not reduce
+     * the I/O, only the number of rows fed to the aggregate transitions - and for its whole duration
+     * it advertises a snapshot Postgres will not vacuum past. Measured on a 3 GB / 10M-row job table:
+     * 0.19 s at 1M rows, 2.1 s with parallel workers and 3.6 s without at 10M, and 24-75 s for the
+     * same 3 GB read cold from network storage. Against that, a 200,000-row delete stayed
+     * "dead but not yet removable" for exactly as long as one such snapshot was held open, and was
+     * removed by the next vacuum after it closed.
+     *
+     * What is measured is the server's own transaction duration, not the latency of the call: a pool
+     * with no free connection makes executeSql slow without pinning anything (a 1 ms transaction timed
+     * at 1.97 s behind one busy connection), and backing off on that would defer monitoring for pool
+     * contention and then tell the operator to go and shrink their job table.
+     *
+     * So the danger is not the aggregate; it is the aggregate's duty cycle. Three things push it up
+     * without anyone changing a default - a slow enough scan that a pass outruns the interval, the
+     * 100-name chunking that makes one pass re-read the heap once per chunk, and one aggregate per
+     * table on a partitioned deployment - and one thing pushes it up deliberately, which is tuning
+     * monitorIntervalSeconds down. All four land in the same place, and #statsElapsedSeconds measures
+     * all four the same way.
+     *
+     * The threshold and the length both come from the server's own autovacuum_naptime rather than
+     * from a constant here; see plans.setMonitorBackoff for why the free window has to be longer than
+     * a naptime and why two of them is the right number. The statement returns a row only when it
+     * engaged, so this warns exactly when monitoring is actually being held back.
+     *
+     * PostgreSQL-only, and off with the rest of the vacuum subsystem: distributed backends have no
+     * autovacuum_naptime to read and reclaim on their own schedule, and an operator who set
+     * monitorVacuum: false has taken vacuum health into their own hands.
+     */
+    async #applyMonitorBackoff() {
+        const elapsed = this.#config.__test__monitor_stats_seconds ?? this.#statsElapsedSeconds;
+        this.#statsElapsedSeconds = 0;
+        if (!elapsed)
+            return;
+        if (this.#config.monitorVacuum === false || this.#config.noMonitorVacuum)
+            return;
+        const { rows } = await this.#executeQuery(plans.setMonitorBackoff(this.#config.schema, elapsed));
+        const engaged = rows.at(0);
+        if (!engaged)
+            return;
+        await emitAndPersistWarning(this.#warningContext, WARNING_TYPES.MONITOR_BACKOFF, `${WARNINGS.MONITOR_BACKOFF.message}: ${elapsed.toFixed(1)}s of aggregate against an ` +
+            `autovacuum_naptime of ${Number(engaged.naptimeSeconds)}s. ` +
+            'Shrink the job table (retention, deleteAfterSeconds) or partition its busiest queues.', {
+            elapsedSeconds: elapsed,
+            naptimeSeconds: Number(engaged.naptimeSeconds),
+            backoffSeconds: Number(engaged.backoffSeconds),
+            backoffUntil: engaged.backoffUntil
+        });
+    }
+    /**
+     * Warn when the queues are producing garbage that nothing is reclaiming. Every documented
+     * Postgres-queue collapse has this as its precondition, and until now pg-boss reported only the
+     * symptom (`queue_backlog`) and never the cause: a backlog caused by too few workers and one
+     * caused by unreclaimed garbage look identical from the queue's own counters, and have opposite
+     * fixes.
+     *
+     * One measurement, two diagnoses, because the fixes are completely different:
+     *
+     *   `xmin_horizon`        vacuum runs and reclaims nothing - something is pinning the MVCC
+     *                         horizon, and the fix is to find and release the holder.
+     *   `autovacuum_disabled` nothing runs at all - autovacuum is turned off on the table, and the
+     *                         fix is to turn it back on or vacuum on a schedule that keeps up.
+     *
+     * The trigger is the harm itself, read from `pg_stat_user_tables`, not a transaction-count proxy.
+     * A measured comparison of the two (scripts/xmin-horizon-experiment.js) is what settled this: the
+     * transaction count that corresponds to a given amount of unreclaimable garbage moves by ~9x with
+     * producer batching alone, and without bound with unrelated traffic on the same cluster, because
+     * `age()` is cluster-global. No constant survives that. Dead tuples do not move: the same 40,000
+     * jobs left the same 120,000 unreclaimable rows across both producer shapes.
+     *
+     * Both diagnoses share the first condition - a job table past the point Postgres itself would
+     * vacuum it (`autovacuum_vacuum_threshold + scale_factor * live`, per-table storage parameters
+     * honoured) - and then split on what two consecutive observations show:
+     *
+     *   a vacuum ran and the dead-tuple count did not fall  -> the horizon is pinned
+     *   no vacuum ran and the dead-tuple count grew         -> nothing is vacuuming this table
+     *
+     * Either way it takes two passes to establish, so the first pass after a problem starts never
+     * warns. That is the intended cost of not guessing: one supervise interval of delay buys a signal
+     * that means what it says. It is also what keeps an operator who has turned autovacuum off
+     * deliberately and vacuums on their own schedule quiet - their manual vacuum both moves the
+     * timestamp and drops the count, so neither branch matches.
+     *
+     * PostgreSQL-only: CockroachDB and YugabyteDB reclaim on their engine's own schedule rather than
+     * from the oldest live snapshot, and their profiles set noMonitorVacuum.
+     */
+    async #checkVacuum() {
+        if (this.#stopping)
+            return;
+        if (this.#config.monitorVacuum === false || this.#config.noMonitorVacuum)
+            return;
+        const { rows } = await this.#executeQuery(plans.getJobTableGarbage(this.#config.schema));
+        if (this.#stopping)
+            return;
+        // Vacuum ran and reclaimed nothing, versus nothing ran at all.
+        const stuck = [];
+        const unvacuumed = [];
+        // Tracked separately so one table sitting permanently over budget with autovacuum off cannot
+        // pin the other diagnosis's latch and suppress a later, unrelated episode.
+        let vacuumedOverBudget = false;
+        let disabledOverBudget = false;
+        // Partitioned queues come and go, so drop evidence for tables the query no longer returns
+        // rather than holding a row per table this instance has ever seen.
+        const present = new Set(rows.map(row => row.name));
+        for (const name of this.#garbageEvidence.keys()) {
+            if (!present.has(name))
+                this.#garbageEvidence.delete(name);
+        }
+        for (const row of rows) {
+            const name = row.name;
+            const liveTuples = Number(row.liveTuples);
+            const deadTuples = Number(row.deadTuples);
+            const budget = Number(row.threshold) + Number(row.scaleFactor) * liveTuples;
+            const disabled = row.autovacuumEnabled === false;
+            // Compared only against the previous observation of the same table, so any monotonic clock
+            // works; a table never vacuumed reads as 0 and stays there, which is exactly the state the
+            // autovacuum_disabled branch is looking for.
+            const lastVacuum = row.lastVacuum ? new Date(row.lastVacuum).getTime() : 0;
+            if (deadTuples <= budget) {
+                this.#garbageEvidence.delete(name);
+                continue;
+            }
+            if (disabled) {
+                disabledOverBudget = true;
+            }
+            else {
+                vacuumedOverBudget = true;
+            }
+            const previous = this.#garbageEvidence.get(name);
+            this.#garbageEvidence.set(name, { deadTuples, lastVacuum });
+            if (!previous)
+                continue;
+            const garbage = {
+                name,
+                liveTuples,
+                deadTuples,
+                budget,
+                // Null rather than 0 when the table has never been vacuumed at all - "never" and "just now"
+                // are opposite readings and must not share a value.
+                vacuumAgeSeconds: row.vacuumAgeSeconds === null ? null : Number(row.vacuumAgeSeconds),
+                lastVacuum
+            };
+            if (disabled) {
+                // Nothing ran between the two observations and the pile grew.
+                if (lastVacuum === previous.lastVacuum && deadTuples > previous.deadTuples) {
+                    unvacuumed.push(garbage);
+                }
+            }
+            else if (lastVacuum > previous.lastVacuum && deadTuples >= previous.deadTuples) {
+                // A vacuum ran between the two observations and left at least as much garbage as it found.
+                stuck.push(garbage);
+            }
+        }
+        if (!vacuumedOverBudget)
+            this.#warnedXminHorizon = false;
+        if (!disabledOverBudget)
+            this.#warnedAutovacuumDisabled = false;
+        await this.#warnAutovacuumDisabled(unvacuumed);
+        await this.#warnXminHorizon(stuck);
+    }
+    /**
+     * Garbage piling up on a table nothing vacuums. No horizon lookup: there is no vacuum here that
+     * could have failed, so there is no holder to blame, and naming one would send the operator after
+     * the wrong thing entirely. The fix is autovacuum, not a stuck transaction.
+     */
+    async #warnAutovacuumDisabled(unvacuumed) {
+        if (!unvacuumed.length || this.#warnedAutovacuumDisabled || this.#stopping)
+            return;
+        this.#warnedAutovacuumDisabled = true;
+        const worst = unvacuumed.reduce((a, b) => (b.deadTuples > a.deadTuples ? b : a));
+        const since = worst.vacuumAgeSeconds === null
+            ? 'has never been vacuumed'
+            : `was last vacuumed ${worst.vacuumAgeSeconds}s ago`;
+        await emitAndPersistWarning(this.#warningContext, WARNING_TYPES.AUTOVACUUM_DISABLED, `${WARNINGS.AUTOVACUUM_DISABLED.message}: ${worst.name} has autovacuum_enabled = false, holds ` +
+            `${worst.deadTuples} dead rows and is still growing, and ${since} ` +
+            `(Postgres would vacuum this table at ${Math.round(worst.budget)})`, {
+            table: worst.name,
+            tables: unvacuumed.map(t => t.name),
+            liveTuples: worst.liveTuples,
+            deadTuples: worst.deadTuples,
+            budget: Math.round(worst.budget),
+            vacuumAgeSeconds: worst.vacuumAgeSeconds
+        });
+    }
+    /** Garbage that survived a vacuum, attributed to whatever is holding the horizon back. */
+    async #warnXminHorizon(stuck) {
+        if (!stuck.length || this.#warnedXminHorizon || this.#stopping)
+            return;
+        // The vacuum a holder has to predate is the most recent one that failed, so the newest wins.
+        const lastVacuum = new Date(Math.max(...stuck.map(t => t.lastVacuum)));
+        const horizon = await this.#readXminHorizon(lastVacuum);
+        if (!horizon || this.#stopping)
+            return;
+        const holder = this.#attributeXminHorizon(horizon.row);
+        // Garbage that survived a vacuum with no holder old enough to explain it is a real problem, but
+        // not this one - staying quiet is what keeps the warning worth acting on.
+        if (!holder)
+            return;
+        this.#warnedXminHorizon = true;
+        const worst = stuck.reduce((a, b) => (b.deadTuples > a.deadTuples ? b : a));
+        const seconds = Number(horizon.row.oldestTransactionSeconds);
+        const backend = holder.source === 'backends' ? horizon.row.backendHolder ?? null : null;
+        const opaque = Number(horizon.row.opaqueBackends) || 0;
+        // Only worth saying when the holder could not be named: with a name in hand the operator has
+        // what they need, and the grant is beside the point.
+        const opaqueNote = opaque && !backend
+            ? ` ${opaque} backend(s) could not be inspected by this role; GRANT pg_read_all_stats to name them.`
+            : '';
+        await emitAndPersistWarning(this.#warningContext, WARNING_TYPES.XMIN_HORIZON, `${WARNINGS.XMIN_HORIZON.message}: ${describeXminHolder(holder.source, horizon.row)}, ` +
+            `holding it ${holder.age} transactions back. ` +
+            `${worst.name} has ${worst.deadTuples} dead rows that a vacuum ${worst.vacuumAgeSeconds}s ago could not reclaim ` +
+            `(Postgres vacuums this table at ${Math.round(worst.budget)}).${opaqueNote}`, {
+            source: holder.source,
+            holder: describeXminHolder(holder.source, horizon.row),
+            holderClass: XMIN_HOLDERS[holder.source],
+            // Null unless the holder is a backend this role could read a row for. `self` says whether it
+            // shares this connection's application_name — pg-boss pinning its own horizon and an
+            // external reporting tool pinning it have opposite fixes.
+            holderPid: backend?.pid ?? null,
+            holderApplicationName: backend?.applicationName ?? null,
+            holderUserName: backend?.userName ?? null,
+            holderState: backend?.state ?? null,
+            holderTransactionSeconds: backend?.xactSeconds ?? null,
+            self: backend ? backend.applicationName === horizon.row.selfApplicationName : null,
+            // Backends whose transaction this role is not allowed to time. Non-zero means the picture is
+            // partial, whether or not a holder was named.
+            opaqueBackends: opaque,
+            transactions: holder.age,
+            table: worst.name,
+            liveTuples: worst.liveTuples,
+            deadTuples: worst.deadTuples,
+            budget: Math.round(worst.budget),
+            vacuumAgeSeconds: worst.vacuumAgeSeconds,
+            tables: stuck.map(t => t.name),
+            oldestTransactionSeconds: Number.isFinite(seconds) ? seconds : null,
+            // Named so a partial answer is never mistaken for a clean one.
+            unreadableSources: horizon.unreadable
+        });
+    }
+    /**
+     * The widest holder in the row. Every source the query returns has already been qualified — the
+     * backends column is filtered server-side to transactions that predate the failed vacuum, and a
+     * slot, standby or prepared transaction advertises an xmin only while something is genuinely
+     * stuck — so this is a straight maximum. The horizon is pinned to the oldest of them, which makes
+     * the widest the one worth naming.
+     */
+    #attributeXminHorizon(row) {
+        let worst = null;
+        // Iterates the known holder classes, not the row's columns: the row also carries
+        // oldestTransactionSeconds, which is informational and would otherwise read as a holder named
+        // after a column with an age of 0.
+        for (const source of Object.keys(XMIN_HOLDERS)) {
+            const value = row[source];
+            const age = value === undefined || value === null ? NaN : Number(value);
+            if (Number.isFinite(age) && (!worst || age > worst.age)) {
+                worst = { source, age };
+            }
+        }
+        return worst;
+    }
+    /**
+     * Read the horizon, narrowing to the sources this role can actually read. Called only when a
+     * table already shows unreclaimable garbage, so an installation with a healthy horizon never
+     * pays for it.
+     */
+    async #readXminHorizon(lastVacuum) {
+        const attempted = this.#xminHorizonSources ?? plans.XMIN_HORIZON_QUERY_SOURCES;
+        // Narrowing has already found that this role can read none of them, and the check said so once.
+        // Without this the empty list would be handed back to getXminHorizon, which cannot build a
+        // statement with no columns, and every later pass would re-fail, re-probe and re-warn.
+        if (!attempted.length)
+            return null;
+        try {
+            const { rows } = await this.#executeQuery(plans.getXminHorizon(lastVacuum, attempted));
+            const unreadable = plans.XMIN_HORIZON_QUERY_SOURCES.filter(name => !attempted.includes(name));
+            return { row: (rows[0] ?? {}), unreadable };
+        }
+        catch (err) {
+            // A role that cannot read one of the catalogs fails the whole statement. Narrow to the
+            // sources that do work rather than losing the check entirely - but remember what was dropped,
+            // because "no rows" from an unreadable view must not read as "horizon is healthy".
+            const readable = await this.#narrowXminHorizonSources(attempted, lastVacuum);
+            this.#xminHorizonSources = readable;
+            if (!readable.length) {
+                this.emit(events.warning, {
+                    message: 'Unable to read the transaction horizon; the xmin_horizon check is disabled for this instance.',
+                    data: { type: WARNING_TYPES.XMIN_HORIZON, error: err?.message }
+                });
+            }
+            return null;
+        }
+    }
+    // Probe each source alone to find which ones this role can actually read. Runs only after the
+    // combined query has already failed, so the cost is paid once per instance, not per pass.
+    async #narrowXminHorizonSources(attempted, lastVacuum) {
+        const readable = [];
+        for (const source of attempted) {
+            try {
+                await this.#executeQuery(plans.getXminHorizon(lastVacuum, [source]));
+                readable.push(source);
+            }
+            catch {
+                // Unreadable for this role; reported alongside the warning as unreadableSources.
+            }
+        }
+        return readable;
+    }
     // DDL runs outside the slow-query timer. A REINDEX is expected to take seconds — routing it
     // through #executeQuery would emit a bogus slow_query warning on every rebuild.
     async #executeDdl(sql) {

package/dist/manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../src/manager.ts"],"names":[],"mappings":"AAEA,OAAO,YAAY,MAAM,aAAa,CAAA;AAGtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAA;AAC7B,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AAEzC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAG7C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAU,KAAK,eAAe,EAAE,MAAM,UAAU,CAAA;AAwEvD,cAAM,OAAQ,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAC7D,MAAM;;;MAAS;IACf,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG;QAAE,MAAM,CAAC,EAAE,KAAK,CAAA;KAAE,CAAC,GAAG,EAAE,CAAA;IAC/C,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAA;IACxC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B,OAAO,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B,kBAAkB,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;IAC9C,WAAW,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;IACvC,UAAU,EAAE,UAAU,GAAG,SAAS,CAAA;IAClC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,CAAA;IAChD,sBAAsB,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;gBAM5B,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAe1E,MAAM,CAAC,CAAC,GAAG,MAAM,EAAG,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC;IAYrD,UAAU,IAAK,IAAI;IAyYb,KAAK;IAkBL,aAAa,CAAE,EAAE,IAAY,EAAE;;KAAK;IAUpC,aAAa,CAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;IAwBxD,IAAI;IAkBJ,OAAO;IAUb,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACjF,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAuH1J,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,OAAO;IAWf,UAAU,CAAE,OAAO,GAAE;QAAE,eAAe,CAAC,EAAE,OAAO,CAAA;KAAO;IAUvD,kBAAkB,IAAK,OAAO;IAIxB,OAAO,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,cAA+B,GAAG,OAAO,CAAC,IAAI,CAAC;IAoC3F,YAAY,CAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAarC,WAAW,CAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAUhC,mBAAmB,IAAK,IAAI;IAQtB,SAAS,CAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtD,WAAW,CAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9D,OAAO,CAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBlF,IAAI,CAAE,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IACrD,IAAI,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAO/F,SAAS,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IASvI,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAW3I,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAwD3I,SAAS,CAAE,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAgFhE,MAAM,CAAE,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IACpE,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IA0BpH,MAAM,CAAE,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IACpE,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IA6D9G,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,EACvB,OAAO,GAAE,KAAK,CAAC,aAAkB;IAiG7B,IAAI,CAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA0G1G,qBAAqB,CAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM;IAiBpE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,YAAY,GAAG;QAAE,eAAe,EAAE,IAAI,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;IACpH,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAiD5E,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,oBAAoB;IAU5B,OAAO,CAAC,kBAAkB;IAQpB,QAAQ,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB;YAsBhG,iBAAiB;YAQjB,mBAAmB;IAQ3B,IAAI,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;YAmBpF,eAAe;IA2BvB,4BAA4B,CAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAK/E,8BAA8B,CAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IASjF,0BAA0B,CAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YA6BpE,qBAAqB;YAmBrB,kBAAkB;IA6E1B,SAAS,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUrF,OAAO,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,cAAmB,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB3E,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUlF,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUlF,OAAO,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IASnF,KAAK,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUjF,KAAK,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC;IAUlH,WAAW,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAO;IAsBtF,cAAc,CAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAehD,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IAuBnE,WAAW,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,kBAAuB;IA6BjE,QAAQ,CAAE,IAAI,EAAE,MAAM;IAMtB,WAAW,CAAE,IAAI,EAAE,MAAM;IAiBzB,gBAAgB,CAAE,IAAI,EAAE,MAAM;IAO9B,gBAAgB,CAAE,IAAI,EAAE,MAAM;IAO9B,aAAa,CAAE,IAAI,CAAC,EAAE,MAAM;IA4B5B,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;IAgGhG,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IA4BxH,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;IA0BnG,eAAe,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAQjH,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAQrH,OAAO,CAAC,QAAQ;CAWjB;AAED,eAAe,OAAO,CAAA"}
+{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../src/manager.ts"],"names":[],"mappings":"AAEA,OAAO,YAAY,MAAM,aAAa,CAAA;AAGtC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAA;AAC7B,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AAEzC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAG7C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAU,KAAK,eAAe,EAAE,MAAM,UAAU,CAAA;AAiFvD,cAAM,OAAQ,SAAQ,YAAa,YAAW,KAAK,CAAC,WAAW;;IAC7D,MAAM;;;MAAS;IAGf,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG;QAAE,MAAM,CAAC,EAAE,KAAK,CAAA;KAAE,CAAC,GAAG,EAAE,CAAA;IAC/C,MAAM,EAAE,KAAK,CAAC,0BAA0B,CAAA;IACxC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B,OAAO,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B,kBAAkB,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;IAC9C,WAAW,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;IACvC,UAAU,EAAE,UAAU,GAAG,SAAS,CAAA;IAClC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,CAAA;IAChD,sBAAsB,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;gBAM5B,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,0BAA0B;IAe1E,MAAM,CAAC,CAAC,GAAG,MAAM,EAAG,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC;IAYrD,UAAU,IAAK,IAAI;IAyYb,KAAK;IAkBL,aAAa,CAAE,EAAE,IAAY,EAAE;;KAAK;IAUpC,aAAa,CAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC;IAwBxD,IAAI;IAkBJ,OAAO;IAUb,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACjF,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAuH1J,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,OAAO;IAWf,UAAU,CAAE,OAAO,GAAE;QAAE,eAAe,CAAC,EAAE,OAAO,CAAA;KAAO;IAUvD,kBAAkB,IAAK,OAAO;IAIxB,OAAO,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,cAA+B,GAAG,OAAO,CAAC,IAAI,CAAC;IAoC3F,YAAY,CAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAarC,WAAW,CAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAUhC,mBAAmB,IAAK,IAAI;IAQtB,SAAS,CAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtD,WAAW,CAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9D,OAAO,CAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBlF,IAAI,CAAE,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IACrD,IAAI,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAO/F,SAAS,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IASvI,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAW3I,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAwD3I,SAAS,CAAE,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAgFhE,MAAM,CAAE,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IACpE,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IA0BpH,MAAM,CAAE,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IACpE,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC;IA6D9G,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,EACvB,OAAO,GAAE,KAAK,CAAC,aAAkB;IAiG7B,IAAI,CAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA0G1G,qBAAqB,CAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM;IAsCpE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,YAAY,GAAG;QAAE,eAAe,EAAE,IAAI,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;IACpH,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAmD5E,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,oBAAoB;IAU5B,OAAO,CAAC,kBAAkB;IAQpB,QAAQ,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB;YAsBhG,iBAAiB;YAQjB,mBAAmB;IAQ3B,IAAI,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;YAmBpF,eAAe;IA2BvB,4BAA4B,CAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAK/E,8BAA8B,CAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IASjF,0BAA0B,CAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YA6BpE,qBAAqB;YAmBrB,kBAAkB;IA6E1B,SAAS,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUrF,OAAO,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,cAAmB,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB3E,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUlF,MAAM,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUlF,OAAO,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IASnF,KAAK,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB;IAUjF,KAAK,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC;IAUlH,WAAW,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAO;IAsBtF,cAAc,CAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAehD,SAAS,CAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IAuBnE,WAAW,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,kBAAuB;IA6BjE,QAAQ,CAAE,IAAI,EAAE,MAAM;IAMtB,WAAW,CAAE,IAAI,EAAE,MAAM;IAiBzB,gBAAgB,CAAE,IAAI,EAAE,MAAM;IAO9B,gBAAgB,CAAE,IAAI,EAAE,MAAM;IAO9B,aAAa,CAAE,IAAI,CAAC,EAAE,MAAM;IA4B5B,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;IAqHhG,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IA4BxH,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,eAAoB,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;IA0BnG,eAAe,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAQjH,aAAa,CAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,KAAK,CAAC,iBAAsB,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;IAQrH,OAAO,CAAC,QAAQ;CAWjB;AAED,eAAe,OAAO,CAAA"}