pixivflow 2.19.2 → 2.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/scheduler-runtime.js +12 -2
- package/dist/delivery/DeliveryLedgerPort.d.ts +14 -0
- package/dist/delivery/DeliveryLedgerPort.js +43 -0
- package/dist/download/DownloadManager.d.ts +5 -0
- package/dist/download/DownloadManager.js +12 -2
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +27 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +105 -7
- package/dist/download/handlers/NovelTargetHandler.d.ts +27 -1
- package/dist/download/handlers/NovelTargetHandler.js +107 -3
- package/dist/package.json +1 -1
- package/dist/pixiv-client/TargetSearchRunner.js +24 -8
- package/dist/scheduler/SlotCoordinator.d.ts +74 -1
- package/dist/scheduler/SlotCoordinator.js +108 -1
- package/dist/scheduler/WorkIdentity.d.ts +49 -0
- package/dist/scheduler/WorkIdentity.js +31 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts +9 -0
- package/dist/storage/repositories/DeliveryRepository.js +16 -0
- package/dist/storage/repositories/OutboxRepository.d.ts +9 -0
- package/dist/storage/repositories/OutboxRepository.js +17 -0
- package/dist/storage/repositories/SlotRepository.d.ts +27 -0
- package/dist/storage/repositories/SlotRepository.js +45 -0
- package/dist/topic/TopicPipeline.js +4 -0
- package/dist/utils/errors.d.ts +9 -0
- package/dist/utils/errors.js +14 -1
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
|
@@ -57,6 +57,7 @@ const DeliveryDispatcher_1 = require("../delivery/DeliveryDispatcher");
|
|
|
57
57
|
const token_maintenance_1 = require("../utils/token-maintenance");
|
|
58
58
|
const schedules_1 = require("../scheduler/schedules");
|
|
59
59
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
60
|
+
const DeliveryLedgerPort_1 = require("../delivery/DeliveryLedgerPort");
|
|
60
61
|
const OutboxWorker_1 = require("../delivery/OutboxWorker");
|
|
61
62
|
const LegacyOutboxMigration_1 = require("../delivery/LegacyOutboxMigration");
|
|
62
63
|
const NotificationPolicy_1 = require("../notification/NotificationPolicy");
|
|
@@ -276,7 +277,7 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
276
277
|
});
|
|
277
278
|
return;
|
|
278
279
|
}
|
|
279
|
-
const coordinator = new SlotCoordinator_1.SlotCoordinator(database);
|
|
280
|
+
const coordinator = new SlotCoordinator_1.SlotCoordinator(database, (0, DeliveryLedgerPort_1.createDeliveryLedgerPort)(database));
|
|
280
281
|
// Ad-hoc/manual execution (run-once / explicit refetch) runs the download
|
|
281
282
|
// plan WITHOUT a scheduled Slot: it can never mark a scheduled occurrence
|
|
282
283
|
// complete or be resumed as one. Scheduled runs (cron/http/catchup) always
|
|
@@ -383,7 +384,14 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
383
384
|
// re-runs a finished cell — that is what prevents a second post). Membership
|
|
384
385
|
// comes from the materialized snapshot, so a config reload cannot add cells.
|
|
385
386
|
const pending = slotCtx ? coordinator.pendingTargets(slotCtx.slotId, targets) : targets.map((target) => ({ target, cell: null }));
|
|
386
|
-
|
|
387
|
+
const selected = onlyTarget ? pending.filter((p) => p.target.id === onlyTarget) : pending;
|
|
388
|
+
// Hand each cell its durable identity. Without this the handler cannot tell a
|
|
389
|
+
// first selection from a recovery, and would re-rank on resume — silently
|
|
390
|
+
// re-pointing the logical item at a different work than the one it owns.
|
|
391
|
+
const targetExecutionContexts = slotCtx
|
|
392
|
+
? coordinator.executionContextsFor(slotCtx.slotId, selected)
|
|
393
|
+
: undefined;
|
|
394
|
+
let runTargets = selected.map((p) => p.target);
|
|
387
395
|
if (slotCtx && runTargets.length === 0) {
|
|
388
396
|
logger_1.logger.info('All slot cells already complete', { slot: slotCtx.slotId });
|
|
389
397
|
coordinator.finish(slotCtx, schedule, targets);
|
|
@@ -424,6 +432,8 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
424
432
|
});
|
|
425
433
|
}
|
|
426
434
|
const downloadManager = new DownloadManager_1.DownloadManager(scopedConfig, pixivClient, database, fileService);
|
|
435
|
+
if (targetExecutionContexts)
|
|
436
|
+
downloadManager.setTargetExecutionContexts(targetExecutionContexts);
|
|
427
437
|
if (options.excludedWorkIds)
|
|
428
438
|
downloadManager.setProcessedWorkIds(options.excludedWorkIds);
|
|
429
439
|
activeDownloadManager = downloadManager;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Database } from '../storage/Database';
|
|
2
|
+
import { SchedulerDeliveryPort } from '../scheduler/SlotCoordinator';
|
|
3
|
+
/**
|
|
4
|
+
* Answers the slot FSM's only delivery question — "does this cell still have an
|
|
5
|
+
* owner downstream?" — from the durable delivery ledger.
|
|
6
|
+
*
|
|
7
|
+
* The distinction matters after a crash: a `delivery_pending` cell whose intent
|
|
8
|
+
* is still live must be left alone (the OutboxWorker retries THAT work to a
|
|
9
|
+
* terminal ACK), while one whose intent is terminally failed must be failed
|
|
10
|
+
* explicitly. Both are read from committed rows, so the answer is the same
|
|
11
|
+
* before and after a restart — which is exactly what a recovery decision needs.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createDeliveryLedgerPort(database: Database): SchedulerDeliveryPort;
|
|
14
|
+
//# sourceMappingURL=DeliveryLedgerPort.d.ts.map
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createDeliveryLedgerPort = createDeliveryLedgerPort;
|
|
4
|
+
/**
|
|
5
|
+
* Answers the slot FSM's only delivery question — "does this cell still have an
|
|
6
|
+
* owner downstream?" — from the durable delivery ledger.
|
|
7
|
+
*
|
|
8
|
+
* The distinction matters after a crash: a `delivery_pending` cell whose intent
|
|
9
|
+
* is still live must be left alone (the OutboxWorker retries THAT work to a
|
|
10
|
+
* terminal ACK), while one whose intent is terminally failed must be failed
|
|
11
|
+
* explicitly. Both are read from committed rows, so the answer is the same
|
|
12
|
+
* before and after a restart — which is exactly what a recovery decision needs.
|
|
13
|
+
*/
|
|
14
|
+
function createDeliveryLedgerPort(database) {
|
|
15
|
+
return {
|
|
16
|
+
stateFor({ deliveryTarget, slotId, targetId }) {
|
|
17
|
+
const rows = database.deliveries.listForCell(deliveryTarget, slotId, targetId);
|
|
18
|
+
if (rows.length === 0)
|
|
19
|
+
return { kind: 'unknown' };
|
|
20
|
+
const confirmed = rows.find((r) => r.status === 'delivered' || r.status === 'duplicate');
|
|
21
|
+
if (confirmed) {
|
|
22
|
+
return { kind: 'confirmed', workId: confirmed.pixivId, workType: confirmed.workType };
|
|
23
|
+
}
|
|
24
|
+
const pending = rows.filter((r) => r.status === 'pending');
|
|
25
|
+
// Still actionable = the outbox is going to try again. Its retry budget is
|
|
26
|
+
// the only thing that may converge this delivery, so keep hands off.
|
|
27
|
+
if (pending.some((r) => database.outbox.hasActionableDelivery(r.id))) {
|
|
28
|
+
return { kind: 'live' };
|
|
29
|
+
}
|
|
30
|
+
// Either the outbox exhausted its retries (dead/failed) or the intent was
|
|
31
|
+
// never queued. Nobody will converge it, and re-running selection would
|
|
32
|
+
// re-point the logical item at a different work — so fail it as-is.
|
|
33
|
+
if (pending.length > 0 || rows.some((r) => r.status === 'failed')) {
|
|
34
|
+
return {
|
|
35
|
+
kind: 'lost',
|
|
36
|
+
reason: 'delivery intent reached a terminal failure without an ACK; failing the cell instead of re-selecting a different work',
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
return { kind: 'unknown' };
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=DeliveryLedgerPort.js.map
|
|
@@ -5,6 +5,7 @@ import { IDatabase } from '../interfaces/IDatabase';
|
|
|
5
5
|
import { IFileService } from '../interfaces/IFileService';
|
|
6
6
|
import { DeliveryService } from '../delivery/DeliveryService';
|
|
7
7
|
import { TargetOutcome } from '../scheduler/TargetOutcome';
|
|
8
|
+
import { TargetExecutionContext } from '../scheduler/WorkIdentity';
|
|
8
9
|
/**
|
|
9
10
|
* Download Manager with Concurrency Control
|
|
10
11
|
*
|
|
@@ -76,6 +77,10 @@ export declare class DownloadManager implements IDownloadManager {
|
|
|
76
77
|
slotName: string;
|
|
77
78
|
slotDate: string;
|
|
78
79
|
};
|
|
80
|
+
/** Per-cell work identity for THIS run, keyed by target id (scheduled runs only). */
|
|
81
|
+
private targetExecutionContexts;
|
|
82
|
+
/** Publish the durable cell identity each handler must honour (scheduled runs). */
|
|
83
|
+
setTargetExecutionContexts(contexts: Map<string, TargetExecutionContext>): void;
|
|
79
84
|
/**
|
|
80
85
|
* Request cooperative cancellation of the current run. In-flight item
|
|
81
86
|
* finishes; no further targets/items are started. runAllTargets() will
|
|
@@ -86,6 +86,12 @@ class DownloadManager {
|
|
|
86
86
|
}
|
|
87
87
|
/** Slot context propagated to delivery templates/outcomes for scheduled runs. */
|
|
88
88
|
slotContext;
|
|
89
|
+
/** Per-cell work identity for THIS run, keyed by target id (scheduled runs only). */
|
|
90
|
+
targetExecutionContexts = new Map();
|
|
91
|
+
/** Publish the durable cell identity each handler must honour (scheduled runs). */
|
|
92
|
+
setTargetExecutionContexts(contexts) {
|
|
93
|
+
this.targetExecutionContexts = contexts;
|
|
94
|
+
}
|
|
89
95
|
/**
|
|
90
96
|
* Request cooperative cancellation of the current run. In-flight item
|
|
91
97
|
* finishes; no further targets/items are started. runAllTargets() will
|
|
@@ -216,11 +222,15 @@ class DownloadManager {
|
|
|
216
222
|
}
|
|
217
223
|
}
|
|
218
224
|
async dispatchTarget(target) {
|
|
225
|
+
// The cell identity must reach the handler or it cannot tell a first
|
|
226
|
+
// selection from a recovery: it would re-rank and silently re-point the
|
|
227
|
+
// logical item at a different work.
|
|
228
|
+
const execution = target.id ? this.targetExecutionContexts.get(target.id) : undefined;
|
|
219
229
|
switch (target.type) {
|
|
220
230
|
case 'illustration':
|
|
221
|
-
return await this.illustrationHandler.handle(target);
|
|
231
|
+
return await this.illustrationHandler.handle(target, execution);
|
|
222
232
|
case 'novel':
|
|
223
|
-
return await this.novelHandler.handle(target);
|
|
233
|
+
return await this.novelHandler.handle(target, execution);
|
|
224
234
|
default:
|
|
225
235
|
logger_1.logger.warn(`Unsupported target type ${target.type}`);
|
|
226
236
|
return { kind: 'failed', retryable: false, error: `unsupported target type ${target.type}` };
|
|
@@ -6,6 +6,7 @@ import { IllustrationDownloader } from '../IllustrationDownloader';
|
|
|
6
6
|
import { DownloadPipeline } from '../pipeline/DownloadPipeline';
|
|
7
7
|
import { DeliveryService } from '../../delivery/DeliveryService';
|
|
8
8
|
import { TargetOutcome } from '../../scheduler/TargetOutcome';
|
|
9
|
+
import { TargetExecutionContext } from '../../scheduler/WorkIdentity';
|
|
9
10
|
import type { TopicPipelineFactory } from '../../topic/createTopicPipeline';
|
|
10
11
|
export declare class IllustrationTargetHandler {
|
|
11
12
|
private readonly client;
|
|
@@ -17,8 +18,14 @@ export declare class IllustrationTargetHandler {
|
|
|
17
18
|
private readonly deliveryService?;
|
|
18
19
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
19
20
|
private outcomes;
|
|
21
|
+
/**
|
|
22
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
23
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
24
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
25
|
+
*/
|
|
26
|
+
private execution;
|
|
20
27
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, illustrationDownloader: IllustrationDownloader, pipeline: DownloadPipeline, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
21
|
-
handle(target: TargetConfig): Promise<TargetOutcome>;
|
|
28
|
+
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
22
29
|
/** Reduce the outcomes collected while processing one target to one cell result. */
|
|
23
30
|
private summarize;
|
|
24
31
|
private classifyError;
|
|
@@ -35,6 +42,25 @@ export declare class IllustrationTargetHandler {
|
|
|
35
42
|
private handleUserIllustrations;
|
|
36
43
|
private sortByPopularityAndLog;
|
|
37
44
|
private logError;
|
|
45
|
+
/**
|
|
46
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
47
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
48
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
49
|
+
* never be filtered out of its own recovery.
|
|
50
|
+
*
|
|
51
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
52
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
53
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
54
|
+
*/
|
|
55
|
+
private recoverLockedWork;
|
|
56
|
+
/**
|
|
57
|
+
* Process ONE candidate work for this cell.
|
|
58
|
+
*
|
|
59
|
+
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
60
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
61
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
62
|
+
* backfill still works for a cell that has committed to nothing.
|
|
63
|
+
*/
|
|
38
64
|
private downloadAndDeliver;
|
|
39
65
|
/**
|
|
40
66
|
* Turn a downloaded artifact into the target's business outcome. In cache
|
|
@@ -5,6 +5,7 @@ const logger_1 = require("../../logger");
|
|
|
5
5
|
const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
|
|
6
6
|
const errors_1 = require("../../utils/errors");
|
|
7
7
|
const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
8
|
+
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
8
9
|
const target_label_1 = require("../../utils/target-label");
|
|
9
10
|
class IllustrationTargetHandler {
|
|
10
11
|
client;
|
|
@@ -16,6 +17,12 @@ class IllustrationTargetHandler {
|
|
|
16
17
|
deliveryService;
|
|
17
18
|
/** Outcomes produced during this handle() call (deliveries + terminal non-matches). */
|
|
18
19
|
outcomes = [];
|
|
20
|
+
/**
|
|
21
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
23
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
24
|
+
*/
|
|
25
|
+
execution = null;
|
|
19
26
|
constructor(client, database, rankingService, illustrationDownloader, pipeline, topicPipelineFactory, deliveryService) {
|
|
20
27
|
this.client = client;
|
|
21
28
|
this.database = database;
|
|
@@ -25,8 +32,17 @@ class IllustrationTargetHandler {
|
|
|
25
32
|
this.topicPipelineFactory = topicPipelineFactory;
|
|
26
33
|
this.deliveryService = deliveryService;
|
|
27
34
|
}
|
|
28
|
-
async handle(target) {
|
|
35
|
+
async handle(target, execution) {
|
|
29
36
|
this.outcomes = [];
|
|
37
|
+
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
38
|
+
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
39
|
+
// Crash/shutdown recovery is not an intentional second run: running the
|
|
40
|
+
// candidate pipeline here would re-rank and could bind this logical item to a
|
|
41
|
+
// different work than the one it already committed to.
|
|
42
|
+
if (this.execution?.lockedWorkId) {
|
|
43
|
+
await this.recoverLockedWork(target, this.execution.lockedWorkId);
|
|
44
|
+
return this.summarize(target);
|
|
45
|
+
}
|
|
30
46
|
if (target.illustId) {
|
|
31
47
|
await this.handleSingleIllustration(target);
|
|
32
48
|
return this.summarize(target);
|
|
@@ -361,14 +377,96 @@ class IllustrationTargetHandler {
|
|
|
361
377
|
stack: error instanceof Error ? error.stack : undefined,
|
|
362
378
|
});
|
|
363
379
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
380
|
+
/**
|
|
381
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
382
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
383
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
384
|
+
* never be filtered out of its own recovery.
|
|
385
|
+
*
|
|
386
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
387
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
388
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
389
|
+
*/
|
|
390
|
+
async recoverLockedWork(target, lockedWorkId) {
|
|
391
|
+
const displayTag = (0, target_label_1.getTargetLabel)(target);
|
|
392
|
+
logger_1.logger.info(`Recovering locked work ${lockedWorkId} for ${displayTag}; candidate selection is skipped`, {
|
|
393
|
+
slotId: this.execution?.slotId,
|
|
394
|
+
targetId: this.execution?.targetId,
|
|
395
|
+
lockedWorkId,
|
|
369
396
|
});
|
|
370
|
-
|
|
397
|
+
const illustId = Number(lockedWorkId);
|
|
398
|
+
if (!Number.isFinite(illustId)) {
|
|
399
|
+
this.outcomes.push({
|
|
400
|
+
kind: 'failed',
|
|
401
|
+
retryable: false,
|
|
402
|
+
error: `LOCKED_WORK_UNAVAILABLE: cell work id "${lockedWorkId}" is not a valid illustration id`,
|
|
403
|
+
});
|
|
371
404
|
return;
|
|
405
|
+
}
|
|
406
|
+
try {
|
|
407
|
+
const detail = await this.client.getIllustration(illustId);
|
|
408
|
+
await this.downloadAndDeliver(detail, displayTag, target);
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
412
|
+
const retryable = (0, errors_1.isRetryableNetworkError)(error);
|
|
413
|
+
this.logError(error, `Failed to recover locked illustration ${lockedWorkId}`);
|
|
414
|
+
this.outcomes.push({
|
|
415
|
+
kind: 'failed',
|
|
416
|
+
retryable,
|
|
417
|
+
error: retryable
|
|
418
|
+
? `locked work ${lockedWorkId} could not be fetched (will retry the SAME work): ${message}`
|
|
419
|
+
: `LOCKED_WORK_UNAVAILABLE: locked work ${lockedWorkId} is no longer fetchable: ${message}`,
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Process ONE candidate work for this cell.
|
|
425
|
+
*
|
|
426
|
+
* The cell is bound to `illust` BEFORE any side effect: it is the binding, not
|
|
427
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
428
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
429
|
+
* backfill still works for a cell that has committed to nothing.
|
|
430
|
+
*/
|
|
431
|
+
async downloadAndDeliver(illust, tag, target) {
|
|
432
|
+
const execution = this.execution;
|
|
433
|
+
const workId = String(illust.id);
|
|
434
|
+
// Recovery continues a work the cell already bound; it must never be released.
|
|
435
|
+
const recovering = Boolean(execution?.lockedWorkId);
|
|
436
|
+
if (execution && !recovering) {
|
|
437
|
+
const binding = execution.bind(workId, 'illustration');
|
|
438
|
+
if (!binding.won) {
|
|
439
|
+
// Another writer elected a different work for this cell. First selection
|
|
440
|
+
// is authoritative: never process a work the cell does not own.
|
|
441
|
+
logger_1.logger.warn(`Cell is bound to work ${binding.workId}; declining to select ${workId}`, {
|
|
442
|
+
slotId: execution.slotId,
|
|
443
|
+
targetId: execution.targetId,
|
|
444
|
+
boundWorkId: binding.workId,
|
|
445
|
+
});
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
let artifact = null;
|
|
450
|
+
try {
|
|
451
|
+
artifact = await this.illustrationDownloader.downloadIllustration(illust, tag, {
|
|
452
|
+
aiMetadataCheck: target.aiMetadataCheck === true,
|
|
453
|
+
maxPageCount: target.maxPageCount,
|
|
454
|
+
includeDeliveryPreviews: Boolean(target.storageMode === 'cache' && target.delivery?.target?.trim()),
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
// Nothing was persisted, so the cell may still pick another candidate.
|
|
459
|
+
if (execution && !recovering)
|
|
460
|
+
execution.release(workId);
|
|
461
|
+
throw error;
|
|
462
|
+
}
|
|
463
|
+
if (!artifact) {
|
|
464
|
+
if (execution && !recovering)
|
|
465
|
+
execution.release(workId);
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
// Committed: the artifact is durable and this work now defines the cell. Even
|
|
469
|
+
// if enqueue throws below, recovery must resume THIS work — never release.
|
|
372
470
|
this.recordArtifactOutcome(artifact, target);
|
|
373
471
|
}
|
|
374
472
|
/**
|
|
@@ -6,6 +6,7 @@ import { DownloadPipeline } from '../pipeline/DownloadPipeline';
|
|
|
6
6
|
import { NovelDownloader } from '../NovelDownloader';
|
|
7
7
|
import { DeliveryService } from '../../delivery/DeliveryService';
|
|
8
8
|
import { TargetOutcome } from '../../scheduler/TargetOutcome';
|
|
9
|
+
import { TargetExecutionContext } from '../../scheduler/WorkIdentity';
|
|
9
10
|
import type { TopicPipelineFactory } from '../../topic/createTopicPipeline';
|
|
10
11
|
export declare class NovelTargetHandler {
|
|
11
12
|
private readonly client;
|
|
@@ -16,8 +17,14 @@ export declare class NovelTargetHandler {
|
|
|
16
17
|
private readonly topicPipelineFactory?;
|
|
17
18
|
private readonly deliveryService?;
|
|
18
19
|
private outcomes;
|
|
20
|
+
/**
|
|
21
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
22
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
23
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
24
|
+
*/
|
|
25
|
+
private execution;
|
|
19
26
|
constructor(client: IPixivClient, database: IDatabase, rankingService: RankingService, pipeline: DownloadPipeline, novelDownloader: NovelDownloader, topicPipelineFactory?: TopicPipelineFactory | undefined, deliveryService?: DeliveryService | undefined);
|
|
20
|
-
handle(target: TargetConfig): Promise<TargetOutcome>;
|
|
27
|
+
handle(target: TargetConfig, execution?: TargetExecutionContext): Promise<TargetOutcome>;
|
|
21
28
|
private summarize;
|
|
22
29
|
private classifyError;
|
|
23
30
|
private fetchNovels;
|
|
@@ -34,6 +41,25 @@ export declare class NovelTargetHandler {
|
|
|
34
41
|
private handleUserNovels;
|
|
35
42
|
private sortByPopularityAndLog;
|
|
36
43
|
private logError;
|
|
44
|
+
/**
|
|
45
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
46
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
47
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
48
|
+
* never be filtered out of its own recovery.
|
|
49
|
+
*
|
|
50
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
51
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
52
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
53
|
+
*/
|
|
54
|
+
private recoverLockedWork;
|
|
55
|
+
/**
|
|
56
|
+
* Process ONE candidate work for this cell.
|
|
57
|
+
*
|
|
58
|
+
* The cell is bound to `novel` BEFORE any side effect: it is the binding, not
|
|
59
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
60
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
61
|
+
* backfill still works for a cell that has committed to nothing.
|
|
62
|
+
*/
|
|
37
63
|
private downloadAndDeliver;
|
|
38
64
|
}
|
|
39
65
|
//# sourceMappingURL=NovelTargetHandler.d.ts.map
|
|
@@ -5,6 +5,7 @@ const logger_1 = require("../../logger");
|
|
|
5
5
|
const errors_1 = require("../../utils/errors");
|
|
6
6
|
const pixiv_date_utils_1 = require("../../utils/pixiv-date-utils");
|
|
7
7
|
const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
8
|
+
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
8
9
|
const target_label_1 = require("../../utils/target-label");
|
|
9
10
|
class NovelTargetHandler {
|
|
10
11
|
client;
|
|
@@ -15,6 +16,12 @@ class NovelTargetHandler {
|
|
|
15
16
|
topicPipelineFactory;
|
|
16
17
|
deliveryService;
|
|
17
18
|
outcomes = [];
|
|
19
|
+
/**
|
|
20
|
+
* Cell identity for this handle() call. Set only for a single-work cell of a
|
|
21
|
+
* scheduled occurrence; null for ad-hoc runs and N-works-per-run targets, which
|
|
22
|
+
* have no single (slotId,targetId) -> workId identity to honour.
|
|
23
|
+
*/
|
|
24
|
+
execution = null;
|
|
18
25
|
constructor(client, database, rankingService, pipeline, novelDownloader, topicPipelineFactory, deliveryService) {
|
|
19
26
|
this.client = client;
|
|
20
27
|
this.database = database;
|
|
@@ -24,8 +31,17 @@ class NovelTargetHandler {
|
|
|
24
31
|
this.topicPipelineFactory = topicPipelineFactory;
|
|
25
32
|
this.deliveryService = deliveryService;
|
|
26
33
|
}
|
|
27
|
-
async handle(target) {
|
|
34
|
+
async handle(target, execution) {
|
|
28
35
|
this.outcomes = [];
|
|
36
|
+
this.execution = execution && (0, WorkIdentity_1.isSingleWorkCell)(target) ? execution : null;
|
|
37
|
+
// A cell that already owns a work is in RECOVERY, not in a new selection.
|
|
38
|
+
// Crash/shutdown recovery is not an intentional second run: running the
|
|
39
|
+
// candidate pipeline here would re-rank and could bind this logical item to a
|
|
40
|
+
// different work than the one it already committed to.
|
|
41
|
+
if (this.execution?.lockedWorkId) {
|
|
42
|
+
await this.recoverLockedWork(target, this.execution.lockedWorkId);
|
|
43
|
+
return this.summarize();
|
|
44
|
+
}
|
|
29
45
|
if (target.novelId !== undefined && target.novelId !== null && target.novelId !== '') {
|
|
30
46
|
const novelNum = typeof target.novelId === 'number' ? target.novelId : Number(target.novelId);
|
|
31
47
|
if (!Number.isFinite(novelNum)) {
|
|
@@ -410,10 +426,98 @@ class NovelTargetHandler {
|
|
|
410
426
|
stack: error instanceof Error ? error.stack : undefined,
|
|
411
427
|
});
|
|
412
428
|
}
|
|
429
|
+
/**
|
|
430
|
+
* Continue the work this cell ALREADY owns. Selection is deliberately skipped:
|
|
431
|
+
* no search, no ranking, no topic expansion, no backfill pool, and no
|
|
432
|
+
* "already downloaded / already delivered" exclusion — the cell's own work must
|
|
433
|
+
* never be filtered out of its own recovery.
|
|
434
|
+
*
|
|
435
|
+
* A locked work that is permanently gone is a terminal failure of THIS logical
|
|
436
|
+
* item (`LOCKED_WORK_UNAVAILABLE`). It is never silently replaced by another
|
|
437
|
+
* candidate; that would mutate the item's identity behind the operator's back.
|
|
438
|
+
*/
|
|
439
|
+
async recoverLockedWork(target, lockedWorkId) {
|
|
440
|
+
const displayTag = (0, target_label_1.getTargetLabel)(target);
|
|
441
|
+
logger_1.logger.info(`Recovering locked work ${lockedWorkId} for ${displayTag}; candidate selection is skipped`, {
|
|
442
|
+
slotId: this.execution?.slotId,
|
|
443
|
+
targetId: this.execution?.targetId,
|
|
444
|
+
lockedWorkId,
|
|
445
|
+
});
|
|
446
|
+
const novelId = Number(lockedWorkId);
|
|
447
|
+
if (!Number.isFinite(novelId)) {
|
|
448
|
+
this.outcomes.push({
|
|
449
|
+
kind: 'failed',
|
|
450
|
+
retryable: false,
|
|
451
|
+
error: `LOCKED_WORK_UNAVAILABLE: cell work id "${lockedWorkId}" is not a valid novel id`,
|
|
452
|
+
});
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
try {
|
|
456
|
+
const detail = await this.client.getNovelDetail(novelId);
|
|
457
|
+
const novel = {
|
|
458
|
+
id: detail.id,
|
|
459
|
+
title: detail.title,
|
|
460
|
+
user: detail.user,
|
|
461
|
+
create_date: detail.create_date,
|
|
462
|
+
};
|
|
463
|
+
await this.downloadAndDeliver(novel, `novel-${novelId}`, target);
|
|
464
|
+
}
|
|
465
|
+
catch (error) {
|
|
466
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
467
|
+
const retryable = (0, errors_1.isRetryableNetworkError)(error);
|
|
468
|
+
this.logError(error, `Failed to recover locked novel ${lockedWorkId}`);
|
|
469
|
+
this.outcomes.push({
|
|
470
|
+
kind: 'failed',
|
|
471
|
+
retryable,
|
|
472
|
+
error: retryable
|
|
473
|
+
? `locked work ${lockedWorkId} could not be fetched (will retry the SAME work): ${message}`
|
|
474
|
+
: `LOCKED_WORK_UNAVAILABLE: locked work ${lockedWorkId} is no longer fetchable: ${message}`,
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Process ONE candidate work for this cell.
|
|
480
|
+
*
|
|
481
|
+
* The cell is bound to `novel` BEFORE any side effect: it is the binding, not
|
|
482
|
+
* the candidate list, that decides what a later recovery resumes. The binding
|
|
483
|
+
* is only rolled back when the attempt produced no artifact at all, so in-run
|
|
484
|
+
* backfill still works for a cell that has committed to nothing.
|
|
485
|
+
*/
|
|
413
486
|
async downloadAndDeliver(novel, tag, target) {
|
|
414
|
-
const
|
|
415
|
-
|
|
487
|
+
const execution = this.execution;
|
|
488
|
+
const workId = String(novel.id);
|
|
489
|
+
// Recovery continues a work the cell already bound; it must never be released.
|
|
490
|
+
const recovering = Boolean(execution?.lockedWorkId);
|
|
491
|
+
if (execution && !recovering) {
|
|
492
|
+
const binding = execution.bind(workId, 'novel');
|
|
493
|
+
if (!binding.won) {
|
|
494
|
+
// Another writer elected a different work for this cell. First selection
|
|
495
|
+
// is authoritative: never process a work the cell does not own.
|
|
496
|
+
logger_1.logger.warn(`Cell is bound to work ${binding.workId}; declining to select ${workId}`, {
|
|
497
|
+
slotId: execution.slotId,
|
|
498
|
+
targetId: execution.targetId,
|
|
499
|
+
boundWorkId: binding.workId,
|
|
500
|
+
});
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
let artifact;
|
|
505
|
+
try {
|
|
506
|
+
artifact = await this.novelDownloader.download(novel, tag, target);
|
|
507
|
+
}
|
|
508
|
+
catch (error) {
|
|
509
|
+
// Nothing was persisted, so the cell may still pick another candidate.
|
|
510
|
+
if (execution && !recovering)
|
|
511
|
+
execution.release(workId);
|
|
512
|
+
throw error;
|
|
513
|
+
}
|
|
514
|
+
if (!artifact) {
|
|
515
|
+
if (execution && !recovering)
|
|
516
|
+
execution.release(workId);
|
|
416
517
|
return;
|
|
518
|
+
}
|
|
519
|
+
// Committed: the artifact is durable and this work now defines the cell. Even
|
|
520
|
+
// if the delivery below throws, recovery must resume THIS work — never release.
|
|
417
521
|
const isDelivery = target.storageMode === 'cache' && target.delivery?.target?.trim();
|
|
418
522
|
if (!isDelivery || !this.deliveryService) {
|
|
419
523
|
this.outcomes.push({ kind: 'stored', workId: artifact.pixivId, workType: artifact.type });
|
package/dist/package.json
CHANGED
|
@@ -56,10 +56,10 @@ class TargetSearchRunner {
|
|
|
56
56
|
return target.limit ? sorted.slice(0, target.limit) : sorted;
|
|
57
57
|
}
|
|
58
58
|
async searchSingleIllust(target, tag, requestDelayMs, signal) {
|
|
59
|
-
return this.searchWithPagination(target, tag, requestDelayMs, (options) => this.kit.illustrations.searchPage(options), (t, g) => (0, query_mapper_1.mapTargetToIllustQuery)({ ...t, tag: g }), signal);
|
|
59
|
+
return this.searchWithPagination(target, tag, requestDelayMs, (options, cursor) => this.kit.illustrations.searchPage(options, cursor), (t, g) => (0, query_mapper_1.mapTargetToIllustQuery)({ ...t, tag: g }), signal);
|
|
60
60
|
}
|
|
61
61
|
async searchSingleNovel(target, tag, requestDelayMs, signal) {
|
|
62
|
-
return this.searchWithPagination(target, tag, requestDelayMs, (options) => this.kit.novels.searchPage(options), (t, g) => (0, query_mapper_1.mapTargetToNovelQuery)({ ...t, tag: g }), signal);
|
|
62
|
+
return this.searchWithPagination(target, tag, requestDelayMs, (options, cursor) => this.kit.novels.searchPage(options, cursor), (t, g) => (0, query_mapper_1.mapTargetToNovelQuery)({ ...t, tag: g }), signal);
|
|
63
63
|
}
|
|
64
64
|
/**
|
|
65
65
|
* Date-aware pagination. Mirrors the legacy SearchService semantics:
|
|
@@ -96,23 +96,39 @@ class TargetSearchRunner {
|
|
|
96
96
|
let cursor = null;
|
|
97
97
|
let pageCount = 0;
|
|
98
98
|
let shouldStop = false;
|
|
99
|
+
// Progress invariant: every iteration must terminate, advance to a cursor we
|
|
100
|
+
// have never fetched, or fail explicitly. A cursor that repeats (same `next`
|
|
101
|
+
// twice, or an A -> B -> A cycle) proves the upstream adapter is not making
|
|
102
|
+
// progress; re-requesting would spin forever, so we refuse instead.
|
|
103
|
+
const seenCursors = new Set();
|
|
99
104
|
while ((!fetchLimit || results.length < fetchLimit) && !shouldStop) {
|
|
100
105
|
(0, errors_1.throwIfAborted)(signal, 'search cancelled');
|
|
101
106
|
pageCount++;
|
|
102
107
|
// Dates are intentionally filtered CLIENT-side below (legacy behavior;
|
|
103
108
|
// PixivFlow needs the early-stop walk on create_date).
|
|
109
|
+
//
|
|
110
|
+
// The cursor must travel as searchPage's SECOND POSITIONAL argument: the
|
|
111
|
+
// kit reads `options.cursor` only inside its own `search()`. Putting it in
|
|
112
|
+
// the options object left the cursor permanently null, so page 1 (the
|
|
113
|
+
// newest works) was re-fetched on every iteration and a search that found
|
|
114
|
+
// no in-range work on that page never terminated.
|
|
104
115
|
const page = await fetchOne({
|
|
105
116
|
word: base.word,
|
|
106
117
|
sort: base.sort,
|
|
107
118
|
searchTarget: base.searchTarget,
|
|
108
119
|
includeR18: base.includeR18,
|
|
109
|
-
cursor,
|
|
110
|
-
// Threaded into the kit transport, which combines it with its per-request
|
|
111
|
-
// timeout and honours it in retry back-off. Without this the pager could
|
|
112
|
-
// stay blocked in a single hung request for the rest of the run.
|
|
113
120
|
signal,
|
|
114
|
-
});
|
|
115
|
-
|
|
121
|
+
}, cursor);
|
|
122
|
+
const nextCursor = page.next;
|
|
123
|
+
if (nextCursor !== null) {
|
|
124
|
+
if (nextCursor === cursor || seenCursors.has(nextCursor)) {
|
|
125
|
+
throw new errors_1.PaginationError(`Search pager for tag "${tag}" did not advance: page ${pageCount} returned a cursor that ` +
|
|
126
|
+
(nextCursor === cursor ? 'equals the one just fetched' : 'was already fetched') +
|
|
127
|
+
`. Refusing to re-request it indefinitely.`);
|
|
128
|
+
}
|
|
129
|
+
seenCursors.add(nextCursor);
|
|
130
|
+
}
|
|
131
|
+
cursor = nextCursor;
|
|
116
132
|
for (const item of page.items) {
|
|
117
133
|
const decision = this.filterItemByDate(item, target, startDate, endDate, sortMode);
|
|
118
134
|
if (decision.shouldStop) {
|
|
@@ -3,6 +3,7 @@ import { Database } from '../storage/Database';
|
|
|
3
3
|
import { CellStatus, SlotItemRecord, SlotRecord, SlotStatus } from '../storage/repositories/SlotRepository';
|
|
4
4
|
import { TriggerSource } from './OccurrenceResolver';
|
|
5
5
|
import { TargetOutcome } from './TargetOutcome';
|
|
6
|
+
import { TargetExecutionContext, WorkBinding } from './WorkIdentity';
|
|
6
7
|
/**
|
|
7
8
|
* Execution-lease TTL and heartbeat cadence.
|
|
8
9
|
*
|
|
@@ -54,6 +55,40 @@ export interface SlotResolveResult {
|
|
|
54
55
|
error?: string;
|
|
55
56
|
status?: number;
|
|
56
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* What a `delivery_pending` cell still owes downstream, read from the durable
|
|
60
|
+
* delivery ledger.
|
|
61
|
+
*
|
|
62
|
+
* - `live` : a non-terminal intent still owns the cell — the outbox is
|
|
63
|
+
* retrying THAT work toward a terminal ACK.
|
|
64
|
+
* - `confirmed` : the ACK already landed but the cell promotion was lost.
|
|
65
|
+
* - `lost` : the intent is terminally failed; nobody will converge it.
|
|
66
|
+
* - `unknown` : no delivery fact for this cell (not a delivery cell).
|
|
67
|
+
*/
|
|
68
|
+
export type CellDeliveryState = {
|
|
69
|
+
kind: 'live';
|
|
70
|
+
} | {
|
|
71
|
+
kind: 'confirmed';
|
|
72
|
+
workId: string;
|
|
73
|
+
workType: string;
|
|
74
|
+
} | {
|
|
75
|
+
kind: 'lost';
|
|
76
|
+
reason: string;
|
|
77
|
+
} | {
|
|
78
|
+
kind: 'unknown';
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* Port for the delivery ledger, injected by the runtime. It lets the slot FSM
|
|
82
|
+
* hand a `delivery_pending` cell to whoever owns its delivery without this class
|
|
83
|
+
* learning anything about TelePost, bots or any hosting platform (see below).
|
|
84
|
+
*/
|
|
85
|
+
export interface SchedulerDeliveryPort {
|
|
86
|
+
stateFor(input: {
|
|
87
|
+
deliveryTarget: string;
|
|
88
|
+
slotId: string;
|
|
89
|
+
targetId: string;
|
|
90
|
+
}): CellDeliveryState;
|
|
91
|
+
}
|
|
57
92
|
/**
|
|
58
93
|
* Owns the Schedule Slot ledger for one scheduler run. A Slot is one durable
|
|
59
94
|
* execution occurrence of a Schedule (NOT a morning/evening row). It ensures
|
|
@@ -66,7 +101,8 @@ export interface SlotResolveResult {
|
|
|
66
101
|
*/
|
|
67
102
|
export declare class SlotCoordinator {
|
|
68
103
|
private readonly database;
|
|
69
|
-
|
|
104
|
+
private readonly delivery?;
|
|
105
|
+
constructor(database: Database, delivery?: SchedulerDeliveryPort | undefined);
|
|
70
106
|
/**
|
|
71
107
|
* Resolve + validate the canonical occurrence for a trigger. Uses ONLY the
|
|
72
108
|
* schedule's cron + timezone and the trigger instant (never a client-supplied
|
|
@@ -103,8 +139,33 @@ export declare class SlotCoordinator {
|
|
|
103
139
|
target: TargetConfig;
|
|
104
140
|
cell: SlotItemRecord;
|
|
105
141
|
}[];
|
|
142
|
+
/**
|
|
143
|
+
* Build the per-target execution contexts for one run, so every handler
|
|
144
|
+
* receives the cell's durable identity instead of just a TargetConfig.
|
|
145
|
+
*
|
|
146
|
+
* This is the boundary where the identity used to be dropped (`pending` was
|
|
147
|
+
* reduced to `p.target`), which is what let a resume re-rank and re-point the
|
|
148
|
+
* logical item at a different work.
|
|
149
|
+
*/
|
|
150
|
+
executionContextsFor(slotId: string, entries: {
|
|
151
|
+
target: TargetConfig;
|
|
152
|
+
cell: SlotItemRecord | null;
|
|
153
|
+
}[]): Map<string, TargetExecutionContext>;
|
|
106
154
|
/** Lock the selected work for a cell (first selection wins; retries keep it). */
|
|
107
155
|
lockWork(slotId: string, targetId: string, workId: string, workType: string): void;
|
|
156
|
+
/**
|
|
157
|
+
* CAS-bind a cell to the work a handler is about to process. Returns the
|
|
158
|
+
* authoritative binding: when another worker already elected a different work,
|
|
159
|
+
* `won` is false and the caller MUST continue with the returned `workId`
|
|
160
|
+
* instead of its own candidate.
|
|
161
|
+
*/
|
|
162
|
+
lockWorkCas(slotId: string, targetId: string, workId: string, workType: string): WorkBinding;
|
|
163
|
+
/**
|
|
164
|
+
* Release a provisional binding whose work produced no local artifact, so the
|
|
165
|
+
* next candidate of an UNBOUND cell can be tried. Refused once anything was
|
|
166
|
+
* committed (see SlotRepository.releaseCellWork).
|
|
167
|
+
*/
|
|
168
|
+
releaseWorkCas(slotId: string, targetId: string, workId: string): void;
|
|
108
169
|
/** Record a cell's terminal state from the download/delivery outcome. */
|
|
109
170
|
markCell(slotId: string, targetId: string, status: CellStatus, error?: string): void;
|
|
110
171
|
/**
|
|
@@ -116,6 +177,18 @@ export declare class SlotCoordinator {
|
|
|
116
177
|
applyOutcome(slotId: string, targetId: string, outcome: TargetOutcome): void;
|
|
117
178
|
/** Promote a delivery_pending cell to submitted from a confirmed ACK. */
|
|
118
179
|
markDelivered(slotId: string, targetId: string, workId: string, workType: string): void;
|
|
180
|
+
/**
|
|
181
|
+
* Decide what a `delivery_pending` cell owes, from the durable delivery
|
|
182
|
+
* ledger, and converge the FSM accordingly. Returns true when the scheduler
|
|
183
|
+
* must NOT run the target handler for it.
|
|
184
|
+
*
|
|
185
|
+
* Once an intent is durable the outbox owns the delivery: re-running selection
|
|
186
|
+
* could only produce a DIFFERENT work for the same logical item (and a second
|
|
187
|
+
* delivery, since the delivery idempotency key is work-scoped). A terminally
|
|
188
|
+
* failed delivery is converged explicitly — never "repaired" by picking
|
|
189
|
+
* another work behind the operator's back.
|
|
190
|
+
*/
|
|
191
|
+
private settlePendingDelivery;
|
|
119
192
|
private safeTransition;
|
|
120
193
|
/** Cross-process execution lease. Duplicate triggers converge, never parallel-run. */
|
|
121
194
|
claimRunLease(slotId: string, owner: string, leaseMs: number): boolean;
|
|
@@ -4,6 +4,7 @@ exports.SlotCoordinator = exports.SLOT_HEARTBEAT_MS = exports.SLOT_LEASE_TTL_MS
|
|
|
4
4
|
exports.timezoneForSchedule = timezoneForSchedule;
|
|
5
5
|
const logger_1 = require("../logger");
|
|
6
6
|
const OccurrenceResolver_1 = require("./OccurrenceResolver");
|
|
7
|
+
const WorkIdentity_1 = require("./WorkIdentity");
|
|
7
8
|
/**
|
|
8
9
|
* Execution-lease TTL and heartbeat cadence.
|
|
9
10
|
*
|
|
@@ -17,6 +18,13 @@ const OccurrenceResolver_1 = require("./OccurrenceResolver");
|
|
|
17
18
|
*/
|
|
18
19
|
exports.SLOT_LEASE_TTL_MS = 3 * 60 * 1000;
|
|
19
20
|
exports.SLOT_HEARTBEAT_MS = 30 * 1000;
|
|
21
|
+
/** The delivery channel a scheduled cell publishes to (null = download-only). */
|
|
22
|
+
function deliveryTargetOf(target) {
|
|
23
|
+
if (target.storageMode !== 'cache')
|
|
24
|
+
return null;
|
|
25
|
+
const name = target.delivery?.target;
|
|
26
|
+
return typeof name === 'string' && name.trim() ? name.trim() : null;
|
|
27
|
+
}
|
|
20
28
|
/**
|
|
21
29
|
* Owns the Schedule Slot ledger for one scheduler run. A Slot is one durable
|
|
22
30
|
* execution occurrence of a Schedule (NOT a morning/evening row). It ensures
|
|
@@ -29,8 +37,10 @@ exports.SLOT_HEARTBEAT_MS = 30 * 1000;
|
|
|
29
37
|
*/
|
|
30
38
|
class SlotCoordinator {
|
|
31
39
|
database;
|
|
32
|
-
|
|
40
|
+
delivery;
|
|
41
|
+
constructor(database, delivery) {
|
|
33
42
|
this.database = database;
|
|
43
|
+
this.delivery = delivery;
|
|
34
44
|
}
|
|
35
45
|
/**
|
|
36
46
|
* Resolve + validate the canonical occurrence for a trigger. Uses ONLY the
|
|
@@ -133,15 +143,65 @@ class SlotCoordinator {
|
|
|
133
143
|
continue;
|
|
134
144
|
if (cell.status === 'submitted' || cell.status === 'no_candidate')
|
|
135
145
|
continue;
|
|
146
|
+
// A cell whose work already has a durable delivery intent is NOT the
|
|
147
|
+
// scheduler's to re-run: the OutboxWorker retries the SAME work to a
|
|
148
|
+
// terminal ACK. Re-selecting here is what let recovery stop pointing at the
|
|
149
|
+
// cell's own work (or enqueue a second one), so it is delegated instead.
|
|
150
|
+
if (cell.status === 'delivery_pending' && (0, WorkIdentity_1.isSingleWorkCell)(target) && this.settlePendingDelivery(slotId, target)) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
136
153
|
out.push({ target, cell });
|
|
137
154
|
}
|
|
138
155
|
return out;
|
|
139
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* Build the per-target execution contexts for one run, so every handler
|
|
159
|
+
* receives the cell's durable identity instead of just a TargetConfig.
|
|
160
|
+
*
|
|
161
|
+
* This is the boundary where the identity used to be dropped (`pending` was
|
|
162
|
+
* reduced to `p.target`), which is what let a resume re-rank and re-point the
|
|
163
|
+
* logical item at a different work.
|
|
164
|
+
*/
|
|
165
|
+
executionContextsFor(slotId, entries) {
|
|
166
|
+
const contexts = new Map();
|
|
167
|
+
for (const { target, cell } of entries) {
|
|
168
|
+
if (!target.id || !cell)
|
|
169
|
+
continue;
|
|
170
|
+
const targetId = target.id;
|
|
171
|
+
contexts.set(targetId, {
|
|
172
|
+
slotId,
|
|
173
|
+
targetId,
|
|
174
|
+
lockedWorkId: cell.workId,
|
|
175
|
+
bind: (workId, workType) => this.lockWorkCas(slotId, targetId, workId, workType),
|
|
176
|
+
release: (workId) => this.releaseWorkCas(slotId, targetId, workId),
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
return contexts;
|
|
180
|
+
}
|
|
140
181
|
/** Lock the selected work for a cell (first selection wins; retries keep it). */
|
|
141
182
|
lockWork(slotId, targetId, workId, workType) {
|
|
142
183
|
this.database.slots.ensureCell(slotId, targetId, workType);
|
|
143
184
|
this.database.slots.lockCellWork(slotId, targetId, workId, workType);
|
|
144
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* CAS-bind a cell to the work a handler is about to process. Returns the
|
|
188
|
+
* authoritative binding: when another worker already elected a different work,
|
|
189
|
+
* `won` is false and the caller MUST continue with the returned `workId`
|
|
190
|
+
* instead of its own candidate.
|
|
191
|
+
*/
|
|
192
|
+
lockWorkCas(slotId, targetId, workId, workType) {
|
|
193
|
+
this.database.slots.ensureCell(slotId, targetId, workType);
|
|
194
|
+
const { cell, won } = this.database.slots.tryLockCellWork(slotId, targetId, workId, workType);
|
|
195
|
+
return { workId: cell.workId ?? workId, won };
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Release a provisional binding whose work produced no local artifact, so the
|
|
199
|
+
* next candidate of an UNBOUND cell can be tried. Refused once anything was
|
|
200
|
+
* committed (see SlotRepository.releaseCellWork).
|
|
201
|
+
*/
|
|
202
|
+
releaseWorkCas(slotId, targetId, workId) {
|
|
203
|
+
this.database.slots.releaseCellWork(slotId, targetId, workId);
|
|
204
|
+
}
|
|
145
205
|
/** Record a cell's terminal state from the download/delivery outcome. */
|
|
146
206
|
markCell(slotId, targetId, status, error) {
|
|
147
207
|
const cell = this.database.slots.getCell(slotId, targetId);
|
|
@@ -203,6 +263,53 @@ class SlotCoordinator {
|
|
|
203
263
|
this.database.slots.lockCellWork(slotId, targetId, workId, workType);
|
|
204
264
|
this.safeTransition(slotId, targetId, 'submitted');
|
|
205
265
|
}
|
|
266
|
+
/**
|
|
267
|
+
* Decide what a `delivery_pending` cell owes, from the durable delivery
|
|
268
|
+
* ledger, and converge the FSM accordingly. Returns true when the scheduler
|
|
269
|
+
* must NOT run the target handler for it.
|
|
270
|
+
*
|
|
271
|
+
* Once an intent is durable the outbox owns the delivery: re-running selection
|
|
272
|
+
* could only produce a DIFFERENT work for the same logical item (and a second
|
|
273
|
+
* delivery, since the delivery idempotency key is work-scoped). A terminally
|
|
274
|
+
* failed delivery is converged explicitly — never "repaired" by picking
|
|
275
|
+
* another work behind the operator's back.
|
|
276
|
+
*/
|
|
277
|
+
settlePendingDelivery(slotId, target) {
|
|
278
|
+
const deliveryTarget = deliveryTargetOf(target);
|
|
279
|
+
if (!deliveryTarget || !this.delivery || !target.id)
|
|
280
|
+
return false;
|
|
281
|
+
const state = this.delivery.stateFor({ deliveryTarget, slotId, targetId: target.id });
|
|
282
|
+
switch (state.kind) {
|
|
283
|
+
case 'live':
|
|
284
|
+
logger_1.logger.info('Cell delivery still owned by the outbox; selection not re-run', {
|
|
285
|
+
slot: slotId,
|
|
286
|
+
target: target.id,
|
|
287
|
+
deliveryTarget,
|
|
288
|
+
});
|
|
289
|
+
return true;
|
|
290
|
+
case 'confirmed':
|
|
291
|
+
// The ACK landed but the promotion was lost (crash between the delivery
|
|
292
|
+
// ledger write and the cell transition). Heal from the ledger — the work
|
|
293
|
+
// IS delivered, so re-selecting would post a second one.
|
|
294
|
+
logger_1.logger.info('Cell delivery already confirmed; promoting the cell from the ledger', {
|
|
295
|
+
slot: slotId,
|
|
296
|
+
target: target.id,
|
|
297
|
+
workId: state.workId,
|
|
298
|
+
});
|
|
299
|
+
this.markDelivered(slotId, target.id, state.workId, state.workType);
|
|
300
|
+
return true;
|
|
301
|
+
case 'lost':
|
|
302
|
+
logger_1.logger.warn('Cell delivery failed terminally; failing the cell instead of re-selecting', {
|
|
303
|
+
slot: slotId,
|
|
304
|
+
target: target.id,
|
|
305
|
+
reason: state.reason,
|
|
306
|
+
});
|
|
307
|
+
this.applyOutcome(slotId, target.id, { kind: 'failed', retryable: false, error: state.reason });
|
|
308
|
+
return true;
|
|
309
|
+
case 'unknown':
|
|
310
|
+
return false;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
206
313
|
safeTransition(slotId, targetId, next, error) {
|
|
207
314
|
try {
|
|
208
315
|
this.database.slots.transitionCell(slotId, targetId, next, error);
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { TargetConfig } from '../config';
|
|
2
|
+
/**
|
|
3
|
+
* Work identity for a scheduled cell.
|
|
4
|
+
*
|
|
5
|
+
* Invariant: (slotId, targetId) -> ONE stable workId. The first selection wins,
|
|
6
|
+
* and every later crash / shutdown / lease recovery continues with THAT work.
|
|
7
|
+
* Re-running candidate selection on resume silently re-points the logical item
|
|
8
|
+
* at a different work (A -> B): the item then reports success while A sits
|
|
9
|
+
* downloaded-but-never-delivered, and if A's delivery intent was already
|
|
10
|
+
* durable both A and B get posted under different idempotency identities.
|
|
11
|
+
*/
|
|
12
|
+
export interface WorkBinding {
|
|
13
|
+
/** Authoritative work id for the cell after the attempt (the CAS winner). */
|
|
14
|
+
workId: string;
|
|
15
|
+
/** True when this caller established — or already held — the binding. */
|
|
16
|
+
won: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Per-cell execution context handed to a target handler, so the cell's identity
|
|
20
|
+
* travels with the work instead of being dropped at the dispatch boundary.
|
|
21
|
+
*
|
|
22
|
+
* `lockedWorkId` is an authoritative INPUT: when it is set, the handler must
|
|
23
|
+
* continue exactly that work and must not run ranking / topic selection /
|
|
24
|
+
* backfill / already-seen filtering. `bind()` is called for a candidate BEFORE
|
|
25
|
+
* any of its side effects, so a crash during the download can never resume the
|
|
26
|
+
* cell onto a different work.
|
|
27
|
+
*/
|
|
28
|
+
export interface TargetExecutionContext {
|
|
29
|
+
readonly slotId: string;
|
|
30
|
+
readonly targetId: string;
|
|
31
|
+
/** The work this logical item is already bound to; null for a fresh cell. */
|
|
32
|
+
readonly lockedWorkId: string | null;
|
|
33
|
+
/** CAS-bind the cell to `workId`; returns the authoritative binding. */
|
|
34
|
+
bind(workId: string, workType: string): WorkBinding;
|
|
35
|
+
/** Roll back a provisional binding for a work that produced no artifact. */
|
|
36
|
+
release(workId: string): void;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* True when one cell of this target selects at most ONE work per run — the only
|
|
40
|
+
* shape the (slotId, targetId) -> workId invariant can hold for.
|
|
41
|
+
*
|
|
42
|
+
* A target that intentionally pulls N works per run (limit > 1, a novel series,
|
|
43
|
+
* a user feed) owns N works inside a single cell, so pinning that cell to one
|
|
44
|
+
* locked id on resume would silently shrink the run to a single work. Those keep
|
|
45
|
+
* per-run candidate selection; the work-identity contract covers single-work
|
|
46
|
+
* cells, which is what a scheduled "one post per slot" target is.
|
|
47
|
+
*/
|
|
48
|
+
export declare function isSingleWorkCell(target: TargetConfig): boolean;
|
|
49
|
+
//# sourceMappingURL=WorkIdentity.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isSingleWorkCell = isSingleWorkCell;
|
|
4
|
+
/**
|
|
5
|
+
* True when one cell of this target selects at most ONE work per run — the only
|
|
6
|
+
* shape the (slotId, targetId) -> workId invariant can hold for.
|
|
7
|
+
*
|
|
8
|
+
* A target that intentionally pulls N works per run (limit > 1, a novel series,
|
|
9
|
+
* a user feed) owns N works inside a single cell, so pinning that cell to one
|
|
10
|
+
* locked id on resume would silently shrink the run to a single work. Those keep
|
|
11
|
+
* per-run candidate selection; the work-identity contract covers single-work
|
|
12
|
+
* cells, which is what a scheduled "one post per slot" target is.
|
|
13
|
+
*/
|
|
14
|
+
function isSingleWorkCell(target) {
|
|
15
|
+
// An explicit limit is the operator's own answer to "how many per run?".
|
|
16
|
+
if (typeof target.limit === 'number' && target.limit > 0)
|
|
17
|
+
return target.limit === 1;
|
|
18
|
+
// Pinned single-work targets.
|
|
19
|
+
if (target.type === 'novel' && target.novelId !== undefined && target.novelId !== null)
|
|
20
|
+
return true;
|
|
21
|
+
if (target.type !== 'novel' && target.illustId)
|
|
22
|
+
return true;
|
|
23
|
+
// N-work containers never collapse into a single identity.
|
|
24
|
+
if (target.userId)
|
|
25
|
+
return false;
|
|
26
|
+
if (target.type === 'novel' && target.seriesId)
|
|
27
|
+
return false;
|
|
28
|
+
// Search/ranking default to ten works per run; a topic run defaults to one.
|
|
29
|
+
return target.mode === 'topic';
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=WorkIdentity.js.map
|
|
@@ -43,6 +43,15 @@ export declare class DeliveryRepository extends BaseRepository {
|
|
|
43
43
|
};
|
|
44
44
|
getById(id: string): DeliveryRow | null;
|
|
45
45
|
getByIdempotencyKey(key: string): DeliveryRow | null;
|
|
46
|
+
/**
|
|
47
|
+
* Every delivery intent this ledger recorded for ONE slot cell, oldest first.
|
|
48
|
+
*
|
|
49
|
+
* Scoped by (slot_id, target_id) on purpose: the dedup key above is per WORK,
|
|
50
|
+
* so it structurally cannot answer "does this logical item still owe a
|
|
51
|
+
* delivery?" — which is the question crash recovery has to ask before it is
|
|
52
|
+
* allowed to re-run selection for a `delivery_pending` cell.
|
|
53
|
+
*/
|
|
54
|
+
listForCell(deliveryTarget: string, slotId: string, targetId: string): DeliveryRow[];
|
|
46
55
|
/**
|
|
47
56
|
* True when this exact work is already CONFIRMED (delivered, or an attested
|
|
48
57
|
* historical duplicate) for this target. Pending/failed intents do not block
|
|
@@ -39,6 +39,22 @@ class DeliveryRepository extends BaseRepository_1.BaseRepository {
|
|
|
39
39
|
const row = this.db.prepare(`SELECT * FROM deliveries WHERE idempotency_key = ?`).get(key);
|
|
40
40
|
return row ? this.toRow(row) : null;
|
|
41
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Every delivery intent this ledger recorded for ONE slot cell, oldest first.
|
|
44
|
+
*
|
|
45
|
+
* Scoped by (slot_id, target_id) on purpose: the dedup key above is per WORK,
|
|
46
|
+
* so it structurally cannot answer "does this logical item still owe a
|
|
47
|
+
* delivery?" — which is the question crash recovery has to ask before it is
|
|
48
|
+
* allowed to re-run selection for a `delivery_pending` cell.
|
|
49
|
+
*/
|
|
50
|
+
listForCell(deliveryTarget, slotId, targetId) {
|
|
51
|
+
const rows = this.db
|
|
52
|
+
.prepare(`SELECT * FROM deliveries
|
|
53
|
+
WHERE delivery_target = ? AND slot_id = ? AND target_id = ?
|
|
54
|
+
ORDER BY created_at ASC`)
|
|
55
|
+
.all(deliveryTarget, slotId, targetId);
|
|
56
|
+
return rows.map((r) => this.toRow(r));
|
|
57
|
+
}
|
|
42
58
|
/**
|
|
43
59
|
* True when this exact work is already CONFIRMED (delivered, or an attested
|
|
44
60
|
* historical duplicate) for this target. Pending/failed intents do not block
|
|
@@ -72,6 +72,15 @@ export declare class OutboxRepository extends BaseRepository {
|
|
|
72
72
|
enqueue(input: NewOutboxItem, now?: number): OutboxRow;
|
|
73
73
|
get(id: string): OutboxRow | null;
|
|
74
74
|
getByKey(kind: OutboxKind, key: string): OutboxRow | null;
|
|
75
|
+
/**
|
|
76
|
+
* True when this delivery intent still has a row the worker can act on
|
|
77
|
+
* (pending / claimed / waiting to retry).
|
|
78
|
+
*
|
|
79
|
+
* This is the "the outbox still owns it" test. A delivery whose outbox row is
|
|
80
|
+
* done/dead/cancelled is NOT actionable: whatever happened downstream is
|
|
81
|
+
* terminal, so nobody is going to converge that delivery by retrying it.
|
|
82
|
+
*/
|
|
83
|
+
hasActionableDelivery(deliveryId: string): boolean;
|
|
75
84
|
list(status?: OutboxStatus, limit?: number): OutboxRow[];
|
|
76
85
|
/**
|
|
77
86
|
* Claim up to `limit` due rows for `owner`. Due = pending/retry_wait whose
|
|
@@ -53,6 +53,23 @@ class OutboxRepository extends BaseRepository_1.BaseRepository {
|
|
|
53
53
|
.get(kind, key);
|
|
54
54
|
return row ? this.toRow(row) : null;
|
|
55
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* True when this delivery intent still has a row the worker can act on
|
|
58
|
+
* (pending / claimed / waiting to retry).
|
|
59
|
+
*
|
|
60
|
+
* This is the "the outbox still owns it" test. A delivery whose outbox row is
|
|
61
|
+
* done/dead/cancelled is NOT actionable: whatever happened downstream is
|
|
62
|
+
* terminal, so nobody is going to converge that delivery by retrying it.
|
|
63
|
+
*/
|
|
64
|
+
hasActionableDelivery(deliveryId) {
|
|
65
|
+
const row = this.db
|
|
66
|
+
.prepare(`SELECT 1 FROM outbox
|
|
67
|
+
WHERE delivery_id = ? AND kind = 'delivery'
|
|
68
|
+
AND status IN ('pending','processing','retry_wait')
|
|
69
|
+
LIMIT 1`)
|
|
70
|
+
.get(deliveryId);
|
|
71
|
+
return Boolean(row);
|
|
72
|
+
}
|
|
56
73
|
list(status, limit = 100) {
|
|
57
74
|
const bounded = Math.max(1, Math.min(Math.trunc(limit), 500));
|
|
58
75
|
const rows = status
|
|
@@ -92,6 +92,33 @@ export declare class SlotRepository extends BaseRepository {
|
|
|
92
92
|
* candidate replacement is a separate explicit operator action (clearCellWork).
|
|
93
93
|
*/
|
|
94
94
|
lockCellWork(slotId: string, targetId: string, workId: string, workType: string): SlotItemRecord;
|
|
95
|
+
/**
|
|
96
|
+
* Compare-and-set the cell's work binding, with real concurrency semantics.
|
|
97
|
+
*
|
|
98
|
+
* `lockCellWork` above only stops *this* process from overwriting an existing
|
|
99
|
+
* id (COALESCE); it cannot elect a winner between two callers that both read
|
|
100
|
+
* "no binding". This one can: the write is guarded by `work_id IS NULL OR
|
|
101
|
+
* work_id = @workId`, so if two workers select A and B at the same instant
|
|
102
|
+
* exactly one UPDATE lands. The returned record is authoritative — a losing
|
|
103
|
+
* caller MUST continue with `cell.workId`, never with its own candidate, or
|
|
104
|
+
* the cell would process a work it does not own.
|
|
105
|
+
*/
|
|
106
|
+
tryLockCellWork(slotId: string, targetId: string, workId: string, workType: string): {
|
|
107
|
+
cell: SlotItemRecord;
|
|
108
|
+
won: boolean;
|
|
109
|
+
};
|
|
110
|
+
/**
|
|
111
|
+
* Roll back a PROVISIONAL binding taken for a work that produced no local
|
|
112
|
+
* artifact (a candidate that failed while the cell was still unbound).
|
|
113
|
+
*
|
|
114
|
+
* Deliberately narrow: it only clears the row while it still holds exactly
|
|
115
|
+
* `workId` AND is still `selected`. Once the cell moved on (an artifact was
|
|
116
|
+
* persisted -> delivery_pending/submitted/..., or another writer rebound it)
|
|
117
|
+
* the release is refused, so it can never erase a committed work identity.
|
|
118
|
+
* Re-selecting is only ever allowed while nothing was committed — that is what
|
|
119
|
+
* keeps (slotId,targetId) -> workId stable.
|
|
120
|
+
*/
|
|
121
|
+
releaseCellWork(slotId: string, targetId: string, workId: string): boolean;
|
|
95
122
|
/** Explicit operator action: forget the locked work so a re-run picks another candidate. */
|
|
96
123
|
clearCellWork(slotId: string, targetId: string): void;
|
|
97
124
|
setCellStatus(slotId: string, targetId: string, status: CellStatus, error?: string): void;
|
|
@@ -137,6 +137,51 @@ class SlotRepository extends BaseRepository_1.BaseRepository {
|
|
|
137
137
|
.run({ slotId, targetId, workId, workType });
|
|
138
138
|
return this.getCell(slotId, targetId);
|
|
139
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* Compare-and-set the cell's work binding, with real concurrency semantics.
|
|
142
|
+
*
|
|
143
|
+
* `lockCellWork` above only stops *this* process from overwriting an existing
|
|
144
|
+
* id (COALESCE); it cannot elect a winner between two callers that both read
|
|
145
|
+
* "no binding". This one can: the write is guarded by `work_id IS NULL OR
|
|
146
|
+
* work_id = @workId`, so if two workers select A and B at the same instant
|
|
147
|
+
* exactly one UPDATE lands. The returned record is authoritative — a losing
|
|
148
|
+
* caller MUST continue with `cell.workId`, never with its own candidate, or
|
|
149
|
+
* the cell would process a work it does not own.
|
|
150
|
+
*/
|
|
151
|
+
tryLockCellWork(slotId, targetId, workId, workType) {
|
|
152
|
+
this.db
|
|
153
|
+
.prepare(`UPDATE schedule_slot_items
|
|
154
|
+
SET work_id = @workId,
|
|
155
|
+
work_type = CASE WHEN work_id IS NULL THEN @workType ELSE work_type END,
|
|
156
|
+
status = CASE WHEN status = 'pending' THEN 'selected' ELSE status END,
|
|
157
|
+
attempt_count = CASE WHEN work_id IS NULL THEN attempt_count + 1 ELSE attempt_count END,
|
|
158
|
+
updated_at = CURRENT_TIMESTAMP
|
|
159
|
+
WHERE slot_id = @slotId AND target_id = @targetId
|
|
160
|
+
AND (work_id IS NULL OR work_id = @workId)`)
|
|
161
|
+
.run({ slotId, targetId, workId, workType });
|
|
162
|
+
const cell = this.getCell(slotId, targetId);
|
|
163
|
+
return { cell, won: cell.workId === workId };
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Roll back a PROVISIONAL binding taken for a work that produced no local
|
|
167
|
+
* artifact (a candidate that failed while the cell was still unbound).
|
|
168
|
+
*
|
|
169
|
+
* Deliberately narrow: it only clears the row while it still holds exactly
|
|
170
|
+
* `workId` AND is still `selected`. Once the cell moved on (an artifact was
|
|
171
|
+
* persisted -> delivery_pending/submitted/..., or another writer rebound it)
|
|
172
|
+
* the release is refused, so it can never erase a committed work identity.
|
|
173
|
+
* Re-selecting is only ever allowed while nothing was committed — that is what
|
|
174
|
+
* keeps (slotId,targetId) -> workId stable.
|
|
175
|
+
*/
|
|
176
|
+
releaseCellWork(slotId, targetId, workId) {
|
|
177
|
+
const info = this.db
|
|
178
|
+
.prepare(`UPDATE schedule_slot_items
|
|
179
|
+
SET work_id = NULL, status = 'pending', updated_at = CURRENT_TIMESTAMP
|
|
180
|
+
WHERE slot_id = @slotId AND target_id = @targetId
|
|
181
|
+
AND work_id = @workId AND status = 'selected'`)
|
|
182
|
+
.run({ slotId, targetId, workId });
|
|
183
|
+
return info.changes > 0;
|
|
184
|
+
}
|
|
140
185
|
/** Explicit operator action: forget the locked work so a re-run picks another candidate. */
|
|
141
186
|
clearCellWork(slotId, targetId) {
|
|
142
187
|
this.db
|
|
@@ -123,6 +123,10 @@ class TopicPipeline {
|
|
|
123
123
|
// A cancellation must not be degraded into "no results for this tag": that
|
|
124
124
|
// would silently continue the cancelled run across the remaining tags.
|
|
125
125
|
(0, errors_1.rethrowIfCancelled)(error, this.signal);
|
|
126
|
+
// A broken pager contract must not degrade either: reporting "no works
|
|
127
|
+
// today" would hide the failure and let the run claim success.
|
|
128
|
+
if (error instanceof errors_1.PaginationError)
|
|
129
|
+
throw error;
|
|
126
130
|
logger_1.logger.warn('[TopicCollector] search failed tag=' + tag + ' type=' + contentType + ': ' + (error instanceof Error ? error.message : String(error)));
|
|
127
131
|
return [];
|
|
128
132
|
}
|
package/dist/utils/errors.d.ts
CHANGED
|
@@ -33,6 +33,15 @@ export declare class NetworkError extends PixivFlowError {
|
|
|
33
33
|
waitTime?: number;
|
|
34
34
|
});
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* The pagination adapter returned a cursor it had already handed back (the same
|
|
38
|
+
* `next` twice, or a cycle). Re-requesting would repeat the previous page
|
|
39
|
+
* forever, which is exactly the Attempt 2 fallback-day failure, so the pager
|
|
40
|
+
* fails deterministically instead of spinning.
|
|
41
|
+
*/
|
|
42
|
+
export declare class PaginationError extends PixivFlowError {
|
|
43
|
+
constructor(message: string, cause?: Error);
|
|
44
|
+
}
|
|
36
45
|
export declare class DownloadError extends PixivFlowError {
|
|
37
46
|
readonly itemId?: number | undefined;
|
|
38
47
|
readonly itemType?: "illustration" | "novel" | undefined;
|
package/dist/utils/errors.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Provides consistent error types and handling patterns across the application
|
|
5
5
|
*/
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.ErrorRecoveryStrategy = exports.HelpRequest = exports.VersionRequest = exports.OperationCancelledError = exports.DatabaseError = exports.PendingDeliveryError = exports.DownloadError = exports.NetworkError = exports.AuthenticationError = exports.ConfigError = exports.PixivFlowError = exports.PixivTimeoutError = exports.PixivRateLimitError = exports.PixivNotFoundError = exports.PixivNetworkError = exports.PixivHttpError = exports.PixivError = void 0;
|
|
7
|
+
exports.ErrorRecoveryStrategy = exports.HelpRequest = exports.VersionRequest = exports.OperationCancelledError = exports.DatabaseError = exports.PendingDeliveryError = exports.DownloadError = exports.PaginationError = exports.NetworkError = exports.AuthenticationError = exports.ConfigError = exports.PixivFlowError = exports.PixivTimeoutError = exports.PixivRateLimitError = exports.PixivNotFoundError = exports.PixivNetworkError = exports.PixivHttpError = exports.PixivError = void 0;
|
|
8
8
|
exports.isPixivKitError = isPixivKitError;
|
|
9
9
|
exports.toNetworkError = toNetworkError;
|
|
10
10
|
exports.isOperationCancelled = isOperationCancelled;
|
|
@@ -92,6 +92,19 @@ class NetworkError extends PixivFlowError {
|
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
exports.NetworkError = NetworkError;
|
|
95
|
+
/**
|
|
96
|
+
* The pagination adapter returned a cursor it had already handed back (the same
|
|
97
|
+
* `next` twice, or a cycle). Re-requesting would repeat the previous page
|
|
98
|
+
* forever, which is exactly the Attempt 2 fallback-day failure, so the pager
|
|
99
|
+
* fails deterministically instead of spinning.
|
|
100
|
+
*/
|
|
101
|
+
class PaginationError extends PixivFlowError {
|
|
102
|
+
constructor(message, cause) {
|
|
103
|
+
super(message, 'PAGINATION_ERROR', undefined, cause);
|
|
104
|
+
this.name = 'PaginationError';
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
exports.PaginationError = PaginationError;
|
|
95
108
|
class DownloadError extends PixivFlowError {
|
|
96
109
|
itemId;
|
|
97
110
|
itemType;
|
package/dist/version.js
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BUILD = void 0;
|
|
4
4
|
// GENERATED by scripts/write-version.js — do not edit manually.
|
|
5
|
-
exports.BUILD = { version: '2.19.
|
|
5
|
+
exports.BUILD = { version: '2.19.3', commit: '699f027bb45a' };
|
|
6
6
|
//# sourceMappingURL=version.js.map
|
package/dist/webui/package.json
CHANGED
package/package.json
CHANGED