pixivflow 2.22.1 → 2.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/SchedulerCommand.js +12 -3
- package/dist/commands/scheduler-runtime.d.ts +29 -0
- package/dist/commands/scheduler-runtime.js +54 -1
- package/dist/download/FileService.d.ts +10 -0
- package/dist/download/FileService.js +6 -0
- package/dist/download/NovelDownloader.js +56 -0
- package/dist/download/exec/DownloadExecutor.js +11 -2
- package/dist/download/handlers/IllustrationTargetHandler.js +15 -0
- package/dist/download/handlers/NovelTargetHandler.js +12 -2
- package/dist/download/novelMarkers.d.ts +54 -0
- package/dist/download/novelMarkers.js +122 -0
- package/dist/interfaces/IDatabase.d.ts +11 -0
- package/dist/interfaces/IFileService.d.ts +4 -0
- package/dist/logger.d.ts +24 -0
- package/dist/logger.js +88 -11
- package/dist/notification/NotificationPolicy.js +18 -5
- package/dist/observability/classify.d.ts +7 -0
- package/dist/observability/classify.js +51 -0
- package/dist/observability/context.d.ts +10 -0
- package/dist/observability/context.js +22 -0
- package/dist/observability/index.d.ts +7 -0
- package/dist/observability/index.js +36 -0
- package/dist/observability/types.d.ts +23 -0
- package/dist/observability/types.js +3 -0
- package/dist/package.json +1 -1
- package/dist/scheduler/RecoveryOutcome.d.ts +24 -0
- package/dist/scheduler/RecoveryOutcome.js +44 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts +4 -0
- package/dist/scheduler/SlotBusinessStatus.d.ts +32 -0
- package/dist/scheduler/SlotBusinessStatus.js +43 -0
- package/dist/scheduler/SlotCoordinator.d.ts +7 -0
- package/dist/scheduler/SlotCoordinator.js +13 -0
- package/dist/storage/Database.d.ts +4 -0
- package/dist/storage/Database.js +7 -0
- package/dist/storage/DatabaseMigration.js +21 -0
- package/dist/storage/repositories/SystemErrorRepository.d.ts +19 -0
- package/dist/storage/repositories/SystemErrorRepository.js +92 -0
- package/dist/version.js +1 -1
- package/dist/webui/package.json +1 -1
- package/dist/webui/routes/admin-logs.d.ts +3 -0
- package/dist/webui/routes/admin-logs.js +9 -0
- package/dist/webui/routes/handlers/admin-logs-handlers.d.ts +6 -0
- package/dist/webui/routes/handlers/admin-logs-handlers.js +106 -0
- package/dist/webui/routes/handlers/system-errors-handlers.d.ts +6 -0
- package/dist/webui/routes/handlers/system-errors-handlers.js +59 -0
- package/dist/webui/routes/system-errors.d.ts +3 -0
- package/dist/webui/routes/system-errors.js +9 -0
- package/dist/webui/server/server-routes.js +5 -0
- package/node_modules/@redtidev/pixiv-client/dist/app-api/novels.d.ts.map +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/app-api/novels.js +18 -2
- package/node_modules/@redtidev/pixiv-client/dist/app-api/novels.js.map +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/index.d.ts +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/index.d.ts.map +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/index.js.map +1 -1
- package/node_modules/@redtidev/pixiv-client/dist/models/index.d.ts +40 -4
- package/node_modules/@redtidev/pixiv-client/dist/models/index.d.ts.map +1 -1
- package/node_modules/@redtidev/pixiv-client/src/app-api/novels.ts +28 -2
- package/node_modules/@redtidev/pixiv-client/src/index.ts +3 -0
- package/node_modules/@redtidev/pixiv-client/src/models/index.ts +34 -4
- package/package.json +2 -2
|
@@ -12,6 +12,7 @@ const ScheduleTriggerServer_1 = require("../scheduler/ScheduleTriggerServer");
|
|
|
12
12
|
const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
|
|
13
13
|
const schedules_1 = require("../scheduler/schedules");
|
|
14
14
|
const scheduler_runtime_1 = require("./scheduler-runtime");
|
|
15
|
+
const RecoveryOutcome_1 = require("../scheduler/RecoveryOutcome");
|
|
15
16
|
const SchedulerIdleLifecycle_1 = require("./SchedulerIdleLifecycle");
|
|
16
17
|
/**
|
|
17
18
|
* Decide whether the authenticated HTTP trigger server should be mounted. It is
|
|
@@ -263,9 +264,17 @@ class SchedulerCommand extends Command_1.BaseCommand {
|
|
|
263
264
|
recoverStatus: (targetId, requestId) => {
|
|
264
265
|
const slot = runtime.database.slots.findRecoverySlot(requestId, targetId);
|
|
265
266
|
const cell = slot && runtime.database.slots.getCell(slot.id, targetId);
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
267
|
+
if (!slot || !cell)
|
|
268
|
+
return null;
|
|
269
|
+
const businessState = (0, RecoveryOutcome_1.recoveryOutcomeFor)(cell);
|
|
270
|
+
return {
|
|
271
|
+
requestId,
|
|
272
|
+
slotId: slot.id,
|
|
273
|
+
state: cell.status,
|
|
274
|
+
slotStatus: slot.status,
|
|
275
|
+
business_state: businessState,
|
|
276
|
+
message: (0, RecoveryOutcome_1.recoveryUserMessage)(businessState),
|
|
277
|
+
};
|
|
269
278
|
},
|
|
270
279
|
status: (scheduleId) => {
|
|
271
280
|
const cfg = resolveConfig();
|
|
@@ -119,6 +119,35 @@ export interface SchedulerRuntime {
|
|
|
119
119
|
close(): void;
|
|
120
120
|
}
|
|
121
121
|
export declare function notifyScheduleFailure(config: StandaloneConfig, database: Database, schedule: ScheduleConfig, failure: JobFailure): Promise<void>;
|
|
122
|
+
/**
|
|
123
|
+
* A `no_candidate` scan that attempted nothing and whose only candidate skips
|
|
124
|
+
* are `duplicate` is a TRUE dead-end: every candidate the scan surfaced was
|
|
125
|
+
* already delivered, and nothing was attempted to be widened. Advancing the
|
|
126
|
+
* fallback stage for this would re-fetch the SAME stale duplicate pool with a
|
|
127
|
+
* wider bound and burn the whole scheduler budget again (the production bug
|
|
128
|
+
* that turned an empty day on bot1 into a 30-minute OperationCancelledError).
|
|
129
|
+
* Terminalise it immediately instead of entering the fallback re-run loop.
|
|
130
|
+
*/
|
|
131
|
+
export interface CandidateExhaustionDiagnostic {
|
|
132
|
+
stage: 'candidate_selection';
|
|
133
|
+
result: 'no_candidate';
|
|
134
|
+
reason: 'duplicate_exhausted' | 'filter_exhausted' | 'no_candidate';
|
|
135
|
+
searched: number;
|
|
136
|
+
duplicates: number;
|
|
137
|
+
filtered: number;
|
|
138
|
+
attempted: number;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Structured diagnostic for a terminal `no_candidate`: what the scan surfaced
|
|
142
|
+
* (`searched`), how many were already-delivered duplicates, how many were
|
|
143
|
+
* filtered by the run's own rules, and how many were actually attempted.
|
|
144
|
+
* Emitted as a JSON log so a Mini App can show "当天候选均已投稿" instead of
|
|
145
|
+
* treating the slot as a system failure.
|
|
146
|
+
*/
|
|
147
|
+
export declare function candidateExhaustionDiagnostic(outcome: Extract<TargetOutcome, {
|
|
148
|
+
kind: 'no_candidate';
|
|
149
|
+
}>): CandidateExhaustionDiagnostic | null;
|
|
150
|
+
export declare function isDuplicateOnlyDeadEnd(outcome: TargetOutcome): boolean;
|
|
122
151
|
/**
|
|
123
152
|
* Expanded, still-bounded candidate scan bound for one fallback stage
|
|
124
153
|
* (§schedule-recovery). Stage 0 is the primary pass; stage N scans
|
|
@@ -45,6 +45,8 @@ exports.EXECUTION_MODES = void 0;
|
|
|
45
45
|
exports.withDeliveryMode = withDeliveryMode;
|
|
46
46
|
exports.shouldTerminaliseAbortedSlot = shouldTerminaliseAbortedSlot;
|
|
47
47
|
exports.notifyScheduleFailure = notifyScheduleFailure;
|
|
48
|
+
exports.candidateExhaustionDiagnostic = candidateExhaustionDiagnostic;
|
|
49
|
+
exports.isDuplicateOnlyDeadEnd = isDuplicateOnlyDeadEnd;
|
|
48
50
|
exports.fallbackScanLimit = fallbackScanLimit;
|
|
49
51
|
exports.createSchedulerRuntime = createSchedulerRuntime;
|
|
50
52
|
exports.runWithTimeout = runWithTimeout;
|
|
@@ -201,6 +203,44 @@ function buildProxyUrl(network) {
|
|
|
201
203
|
const auth = proxy.username ? `${proxy.username}:${proxy.password ?? ''}@` : '';
|
|
202
204
|
return `${protocol}://${auth}${proxy.host}:${proxy.port}`;
|
|
203
205
|
}
|
|
206
|
+
/**
|
|
207
|
+
* Structured diagnostic for a terminal `no_candidate`: what the scan surfaced
|
|
208
|
+
* (`searched`), how many were already-delivered duplicates, how many were
|
|
209
|
+
* filtered by the run's own rules, and how many were actually attempted.
|
|
210
|
+
* Emitted as a JSON log so a Mini App can show "当天候选均已投稿" instead of
|
|
211
|
+
* treating the slot as a system failure.
|
|
212
|
+
*/
|
|
213
|
+
function candidateExhaustionDiagnostic(outcome) {
|
|
214
|
+
const scan = outcome.scan;
|
|
215
|
+
if (!scan)
|
|
216
|
+
return null;
|
|
217
|
+
const skipped = scan.skipped;
|
|
218
|
+
const duplicates = skipped.filter((s) => s.code === 'duplicate').length;
|
|
219
|
+
const filtered = skipped.filter((s) => s.code !== 'duplicate' && s.code !== 'unavailable').length;
|
|
220
|
+
const reason = isDuplicateOnlyDeadEnd(outcome)
|
|
221
|
+
? 'duplicate_exhausted'
|
|
222
|
+
: filtered > 0
|
|
223
|
+
? 'filter_exhausted'
|
|
224
|
+
: 'no_candidate';
|
|
225
|
+
return {
|
|
226
|
+
stage: 'candidate_selection',
|
|
227
|
+
result: 'no_candidate',
|
|
228
|
+
reason,
|
|
229
|
+
searched: skipped.length,
|
|
230
|
+
duplicates,
|
|
231
|
+
filtered,
|
|
232
|
+
attempted: scan.attempted,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
function isDuplicateOnlyDeadEnd(outcome) {
|
|
236
|
+
if (outcome.kind !== 'no_candidate' || !outcome.scan)
|
|
237
|
+
return false;
|
|
238
|
+
const s = outcome.scan;
|
|
239
|
+
return (s.attempted === 0 &&
|
|
240
|
+
s.outages.length === 0 &&
|
|
241
|
+
s.skipped.length > 0 &&
|
|
242
|
+
s.skipped.every((skip) => skip.code === 'duplicate'));
|
|
243
|
+
}
|
|
204
244
|
/**
|
|
205
245
|
* Expanded, still-bounded candidate scan bound for one fallback stage
|
|
206
246
|
* (§schedule-recovery). Stage 0 is the primary pass; stage N scans
|
|
@@ -536,13 +576,26 @@ async function createSchedulerRuntime(configPathArg) {
|
|
|
536
576
|
// an exhausted occurrence reports the true cause — never a generic
|
|
537
577
|
// "target did not complete".
|
|
538
578
|
if (!scheduleSlot.manualRequestId &&
|
|
539
|
-
(outcome.kind === '
|
|
579
|
+
(outcome.kind === 'duplicate' ||
|
|
580
|
+
(outcome.kind === 'no_candidate' && !isDuplicateOnlyDeadEnd(outcome))) &&
|
|
540
581
|
coordinator.cellFallbackStage(scheduleSlot.slotId, target.id) < maxFallbackStages - 1) {
|
|
541
582
|
coordinator.advanceFallback(scheduleSlot.slotId, target.id, outcome.kind === 'duplicate'
|
|
542
583
|
? `duplicate candidates (stage ${coordinator.cellFallbackStage(scheduleSlot.slotId, target.id)})`
|
|
543
584
|
: outcome.reason ?? 'no eligible candidate', maxFallbackStages);
|
|
544
585
|
return;
|
|
545
586
|
}
|
|
587
|
+
if (outcome.kind === 'no_candidate') {
|
|
588
|
+
const diag = candidateExhaustionDiagnostic(outcome);
|
|
589
|
+
if (diag) {
|
|
590
|
+
logger_1.logger.info('candidate_exhaustion', {
|
|
591
|
+
schedule_id: scheduleSlot.scheduleId,
|
|
592
|
+
slot_id: scheduleSlot.slotId,
|
|
593
|
+
bot_id: scheduleSlot.scheduleId?.split('-')[0],
|
|
594
|
+
target_id: target.id,
|
|
595
|
+
...diag,
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
}
|
|
546
599
|
coordinator.applyOutcome(scheduleSlot.slotId, target.id, outcome);
|
|
547
600
|
notificationPolicy.noteOutcome(scheduleSlot.slotId, scheduleSlot, schedule, target, outcome);
|
|
548
601
|
if (scheduleSlot.manualRequestId && (outcome.kind === 'no_candidate' || outcome.kind === 'duplicate' ||
|
|
@@ -31,12 +31,22 @@ export interface PixivMetadata {
|
|
|
31
31
|
name: string;
|
|
32
32
|
is_chinese: boolean;
|
|
33
33
|
};
|
|
34
|
+
assets?: Array<{
|
|
35
|
+
marker: string;
|
|
36
|
+
kind: 'uploadedimage' | 'pixivimage';
|
|
37
|
+
sourceId: string;
|
|
38
|
+
url?: string;
|
|
39
|
+
localPath?: string;
|
|
40
|
+
status: 'pending' | 'downloaded' | 'failed' | 'unavailable';
|
|
41
|
+
failureReason?: string;
|
|
42
|
+
}>;
|
|
34
43
|
}
|
|
35
44
|
export declare class FileService implements IFileService {
|
|
36
45
|
private readonly storage;
|
|
37
46
|
constructor(storage: StorageConfig);
|
|
38
47
|
initialise(): Promise<void>;
|
|
39
48
|
saveImage(buffer: ArrayBuffer, fileName: string, metadata?: FileMetadata): Promise<string>;
|
|
49
|
+
saveBinary(buffer: ArrayBuffer, fileName: string, directory: string): Promise<string>;
|
|
40
50
|
saveText(content: string, fileName: string, metadata?: FileMetadata): Promise<string>;
|
|
41
51
|
private static readonly MAX_FILENAME_BYTES;
|
|
42
52
|
sanitizeFileName(name: string): string;
|
|
@@ -27,6 +27,12 @@ class FileService {
|
|
|
27
27
|
await node_fs_1.promises.writeFile(uniquePath, Buffer.from(buffer));
|
|
28
28
|
return uniquePath;
|
|
29
29
|
}
|
|
30
|
+
async saveBinary(buffer, fileName, directory) {
|
|
31
|
+
await (0, fs_1.ensureDir)(directory);
|
|
32
|
+
const uniquePath = await this.findUniquePath(directory, fileName);
|
|
33
|
+
await node_fs_1.promises.writeFile(uniquePath, Buffer.from(buffer));
|
|
34
|
+
return uniquePath;
|
|
35
|
+
}
|
|
30
36
|
async saveText(content, fileName, metadata) {
|
|
31
37
|
const baseDirectory = this.storage.novelDirectory ?? this.storage.downloadDirectory;
|
|
32
38
|
const organizationMode = this.storage.novelOrganization ?? 'flat';
|
|
@@ -35,7 +35,9 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.NovelDownloader = void 0;
|
|
37
37
|
const logger_1 = require("../logger");
|
|
38
|
+
const node_path_1 = require("node:path");
|
|
38
39
|
const language_detection_1 = require("../utils/language-detection");
|
|
40
|
+
const novelMarkers_1 = require("./novelMarkers");
|
|
39
41
|
const LANGUAGE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
40
42
|
class NovelDownloader {
|
|
41
43
|
client;
|
|
@@ -156,6 +158,33 @@ class NovelDownloader {
|
|
|
156
158
|
date: detail.create_date ? new Date(detail.create_date) : new Date(),
|
|
157
159
|
};
|
|
158
160
|
const filePath = await this.fileService.saveText(content, fileName, metadata);
|
|
161
|
+
const assets = typeof textResponse === 'string'
|
|
162
|
+
? []
|
|
163
|
+
: (0, novelMarkers_1.extractNovelAssets)(text, textResponse);
|
|
164
|
+
const hadImages = assets.length > 0;
|
|
165
|
+
const pendingAssets = assets.filter((a) => Boolean(a.url));
|
|
166
|
+
if (pendingAssets.length) {
|
|
167
|
+
const imagesDir = (0, node_path_1.join)((0, node_path_1.dirname)(filePath), 'images');
|
|
168
|
+
for (const asset of pendingAssets) {
|
|
169
|
+
try {
|
|
170
|
+
const buffer = await this.client.downloadImage(asset.url);
|
|
171
|
+
asset.localPath = await this.fileService.saveBinary(buffer, novelAssetFileName(asset), imagesDir);
|
|
172
|
+
asset.status = 'downloaded';
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
asset.status = 'failed';
|
|
176
|
+
asset.failureReason = error instanceof Error ? error.message : String(error);
|
|
177
|
+
logger_1.logger.warn(`Failed to download novel inline image ${asset.sourceId} for novel ${detail.id}`, {
|
|
178
|
+
novelId: detail.id,
|
|
179
|
+
sourceId: asset.sourceId,
|
|
180
|
+
reason: asset.failureReason,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
if (hadImages) {
|
|
185
|
+
logger_1.logger.info(`Novel ${detail.id} inline images: ${assets.filter((a) => a.status === 'downloaded').length}/${assets.length} downloaded`, { novelId: detail.id });
|
|
186
|
+
}
|
|
187
|
+
}
|
|
159
188
|
const pixivMetadata = {
|
|
160
189
|
pixiv_id: detail.id,
|
|
161
190
|
title: detail.title,
|
|
@@ -181,6 +210,19 @@ class NovelDownloader {
|
|
|
181
210
|
},
|
|
182
211
|
}
|
|
183
212
|
: {}),
|
|
213
|
+
...(assets.length
|
|
214
|
+
? {
|
|
215
|
+
assets: assets.map((a) => ({
|
|
216
|
+
marker: a.marker,
|
|
217
|
+
kind: a.kind,
|
|
218
|
+
sourceId: a.sourceId,
|
|
219
|
+
url: a.url,
|
|
220
|
+
localPath: a.localPath,
|
|
221
|
+
status: a.status,
|
|
222
|
+
failureReason: a.failureReason,
|
|
223
|
+
})),
|
|
224
|
+
}
|
|
225
|
+
: {}),
|
|
184
226
|
};
|
|
185
227
|
let metadataPath;
|
|
186
228
|
try {
|
|
@@ -222,4 +264,18 @@ class NovelDownloader {
|
|
|
222
264
|
}
|
|
223
265
|
}
|
|
224
266
|
exports.NovelDownloader = NovelDownloader;
|
|
267
|
+
function novelAssetFileName(asset) {
|
|
268
|
+
let ext = '';
|
|
269
|
+
try {
|
|
270
|
+
const pathname = new URL(asset.url).pathname;
|
|
271
|
+
const last = pathname.split('/').pop() || '';
|
|
272
|
+
const dot = last.lastIndexOf('.');
|
|
273
|
+
if (dot >= 0)
|
|
274
|
+
ext = last.slice(dot);
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
// fall through: no extension
|
|
278
|
+
}
|
|
279
|
+
return `${asset.sourceId}${ext}` || asset.sourceId;
|
|
280
|
+
}
|
|
225
281
|
//# sourceMappingURL=NovelDownloader.js.map
|
|
@@ -26,8 +26,11 @@ class DownloadExecutor {
|
|
|
26
26
|
itemId: item?.id,
|
|
27
27
|
...(contextProvider?.(item, index) ?? {}),
|
|
28
28
|
};
|
|
29
|
+
const itemId = item?.id ?? item?.pixivId ?? String(item);
|
|
30
|
+
logger_1.logger.info('download_started', { pixiv_id: itemId, attempt, stage: 'download' });
|
|
29
31
|
try {
|
|
30
32
|
const res = await task(item, index);
|
|
33
|
+
logger_1.logger.info('download_completed', { pixiv_id: itemId, attempt, stage: 'download' });
|
|
31
34
|
results[index] = res;
|
|
32
35
|
break;
|
|
33
36
|
}
|
|
@@ -35,21 +38,27 @@ class DownloadExecutor {
|
|
|
35
38
|
onError?.(error, item, index, attempt);
|
|
36
39
|
const decision = recovery.decide(error, effectiveContext);
|
|
37
40
|
onDecision?.(decision, { item, index, attempt, error });
|
|
41
|
+
const status = error?.status;
|
|
42
|
+
const errMsg = error instanceof Error ? error.message : String(error);
|
|
38
43
|
if (decision.action === 'skip') {
|
|
39
|
-
logger_1.logger.warn(
|
|
44
|
+
logger_1.logger.warn(status ? `download_http_failed` : `download_failed`, {
|
|
45
|
+
pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, error: errMsg,
|
|
46
|
+
});
|
|
40
47
|
break;
|
|
41
48
|
}
|
|
42
49
|
if (decision.action === 'fail') {
|
|
50
|
+
logger_1.logger.error(`download_failed`, { pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, error: errMsg });
|
|
43
51
|
throw error instanceof Error ? error : new Error(String(error));
|
|
44
52
|
}
|
|
45
53
|
// retry/backoff
|
|
46
54
|
const delayMs = decision.delayMs ?? 0;
|
|
55
|
+
logger_1.logger.warn(`download_retry`, { pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, retry_delay_ms: delayMs, error: errMsg });
|
|
47
56
|
if (delayMs > 0) {
|
|
48
57
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
49
58
|
}
|
|
50
59
|
attempt += 1;
|
|
51
60
|
if (attempt > (decision.maxAttempts ?? maxAttempts)) {
|
|
52
|
-
logger_1.logger.error(`
|
|
61
|
+
logger_1.logger.error(`download_failed`, { pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, error: errMsg });
|
|
53
62
|
throw error instanceof Error ? error : new Error(String(error));
|
|
54
63
|
}
|
|
55
64
|
}
|
|
@@ -8,6 +8,8 @@ const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
|
8
8
|
const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
|
|
9
9
|
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
10
10
|
const target_label_1 = require("../../utils/target-label");
|
|
11
|
+
const observability_1 = require("../../observability");
|
|
12
|
+
const context_1 = require("../../observability/context");
|
|
11
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
12
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
13
15
|
class IllustrationTargetHandler {
|
|
@@ -159,6 +161,13 @@ class IllustrationTargetHandler {
|
|
|
159
161
|
classifyError(error, displayTag, mode, target) {
|
|
160
162
|
const message = error instanceof Error ? error.message : String(error);
|
|
161
163
|
const scan = this.scan ?? undefined;
|
|
164
|
+
const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
|
|
165
|
+
(0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)(target, this.execution, {
|
|
166
|
+
message,
|
|
167
|
+
stage: 'pixiv_download',
|
|
168
|
+
...cls,
|
|
169
|
+
trace_id: this.execution?.slotId,
|
|
170
|
+
}), cls);
|
|
162
171
|
// A hard job-level outage is named as such and is never recorded as a
|
|
163
172
|
// no-candidate business outcome, whatever its message happens to look like.
|
|
164
173
|
const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
|
|
@@ -166,6 +175,8 @@ class IllustrationTargetHandler {
|
|
|
166
175
|
logger_1.logger.error(`Illustration ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
|
|
167
176
|
error: message,
|
|
168
177
|
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
178
|
+
error_type: cls.error_type,
|
|
179
|
+
retryable: cls.retryable,
|
|
169
180
|
...(outage ? { jobLevelOutage: outage } : {}),
|
|
170
181
|
});
|
|
171
182
|
if (outage) {
|
|
@@ -508,9 +519,13 @@ class IllustrationTargetHandler {
|
|
|
508
519
|
if (endpoint) {
|
|
509
520
|
errorMessage = `${errorMessage} [URL: ${endpoint}]`;
|
|
510
521
|
}
|
|
522
|
+
const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
|
|
523
|
+
(0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)({ type: 'illustration' }, this.execution, { message: errorMessage, stage: 'pixiv_download', ...cls, trace_id: this.execution?.slotId }), cls);
|
|
511
524
|
logger_1.logger.error(message, {
|
|
512
525
|
error: errorMessage,
|
|
513
526
|
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
527
|
+
error_type: cls.error_type,
|
|
528
|
+
retryable: cls.retryable,
|
|
514
529
|
stack: error instanceof Error ? error.stack : undefined,
|
|
515
530
|
});
|
|
516
531
|
}
|
|
@@ -8,6 +8,8 @@ const pixiv_utils_1 = require("../../utils/pixiv-utils");
|
|
|
8
8
|
const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
|
|
9
9
|
const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
|
|
10
10
|
const target_label_1 = require("../../utils/target-label");
|
|
11
|
+
const observability_1 = require("../../observability");
|
|
12
|
+
const context_1 = require("../../observability/context");
|
|
11
13
|
const DownloadPlanner_1 = require("../plan/DownloadPlanner");
|
|
12
14
|
const deliveryContext_1 = require("./deliveryContext");
|
|
13
15
|
class NovelTargetHandler {
|
|
@@ -109,7 +111,7 @@ class NovelTargetHandler {
|
|
|
109
111
|
return this.summarize();
|
|
110
112
|
}
|
|
111
113
|
catch (error) {
|
|
112
|
-
return this.classifyError(error, displayTag, mode);
|
|
114
|
+
return this.classifyError(error, displayTag, mode, target);
|
|
113
115
|
}
|
|
114
116
|
}
|
|
115
117
|
/**
|
|
@@ -165,15 +167,19 @@ class NovelTargetHandler {
|
|
|
165
167
|
...(scan ? { scan } : {}),
|
|
166
168
|
};
|
|
167
169
|
}
|
|
168
|
-
classifyError(error, displayTag, mode) {
|
|
170
|
+
classifyError(error, displayTag, mode, target) {
|
|
169
171
|
const message = error instanceof Error ? error.message : String(error);
|
|
170
172
|
const scan = this.scan ?? undefined;
|
|
173
|
+
const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
|
|
174
|
+
(0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)(target, this.execution, { message, stage: 'pixiv_download', ...cls, trace_id: this.execution?.slotId }), cls);
|
|
171
175
|
// A hard job-level outage is named as such and is never recorded as a
|
|
172
176
|
// no-candidate business outcome, whatever its message happens to look like.
|
|
173
177
|
const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
|
|
174
178
|
this.database.logExecution(displayTag, 'novel', 'failed', message);
|
|
175
179
|
logger_1.logger.error(`Novel ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
|
|
176
180
|
error: message,
|
|
181
|
+
error_type: cls.error_type,
|
|
182
|
+
retryable: cls.retryable,
|
|
177
183
|
...(outage ? { jobLevelOutage: outage } : {}),
|
|
178
184
|
});
|
|
179
185
|
if (outage) {
|
|
@@ -535,9 +541,13 @@ class NovelTargetHandler {
|
|
|
535
541
|
if (endpoint) {
|
|
536
542
|
errorMessage = `${errorMessage} [URL: ${endpoint}]`;
|
|
537
543
|
}
|
|
544
|
+
const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
|
|
545
|
+
(0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)({ type: 'novel' }, this.execution, { message: errorMessage, stage: 'pixiv_download', ...cls, trace_id: this.execution?.slotId }), cls);
|
|
538
546
|
logger_1.logger.error(message, {
|
|
539
547
|
error: errorMessage,
|
|
540
548
|
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
549
|
+
error_type: cls.error_type,
|
|
550
|
+
retryable: cls.retryable,
|
|
541
551
|
stack: error instanceof Error ? error.stack : undefined,
|
|
542
552
|
});
|
|
543
553
|
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { PixivNovelTextResponse } from '@redtidev/pixiv-client';
|
|
2
|
+
/**
|
|
3
|
+
* Single-pass novel text tokenizer, ported from PixEz
|
|
4
|
+
* (Notsfsssf/pixez-flutter lib/page/novel/viewer/image_text.dart,
|
|
5
|
+
* NovelSpansGenerator.buildSpans). Same endpoint + same extraction path as
|
|
6
|
+
* PixivFlow's webview fallback; keeps the `[[...]]` double-bracket close rule so
|
|
7
|
+
* URLs/etc. inside jumpuri/ruby markers are not truncated early.
|
|
8
|
+
*/
|
|
9
|
+
export type NovelMarker = {
|
|
10
|
+
type: 'text';
|
|
11
|
+
value: string;
|
|
12
|
+
} | {
|
|
13
|
+
type: 'newpage';
|
|
14
|
+
raw: string;
|
|
15
|
+
} | {
|
|
16
|
+
type: 'chapter';
|
|
17
|
+
raw: string;
|
|
18
|
+
title: string;
|
|
19
|
+
} | {
|
|
20
|
+
type: 'pixivimage';
|
|
21
|
+
raw: string;
|
|
22
|
+
key: string;
|
|
23
|
+
} | {
|
|
24
|
+
type: 'uploadedimage';
|
|
25
|
+
raw: string;
|
|
26
|
+
key: string;
|
|
27
|
+
} | {
|
|
28
|
+
type: 'jumpuri';
|
|
29
|
+
raw: string;
|
|
30
|
+
url?: string;
|
|
31
|
+
} | {
|
|
32
|
+
type: 'ruby';
|
|
33
|
+
raw: string;
|
|
34
|
+
reading: string;
|
|
35
|
+
};
|
|
36
|
+
export interface NovelAsset {
|
|
37
|
+
marker: string;
|
|
38
|
+
kind: 'uploadedimage' | 'pixivimage';
|
|
39
|
+
sourceId: string;
|
|
40
|
+
url?: string;
|
|
41
|
+
localPath?: string;
|
|
42
|
+
status: 'pending' | 'downloaded' | 'failed' | 'unavailable';
|
|
43
|
+
failureReason?: string;
|
|
44
|
+
}
|
|
45
|
+
export declare function scanNovelMarkers(source: string): NovelMarker[];
|
|
46
|
+
/**
|
|
47
|
+
* Build the ordered set of inline image assets referenced by the novel text.
|
|
48
|
+
* `unavailable` = the source is missing in the webview payload (deleted/no
|
|
49
|
+
* permission) so it will never succeed; `pending` = resolvable + downloadable.
|
|
50
|
+
*/
|
|
51
|
+
export declare function extractNovelAssets(text: string, response?: Pick<PixivNovelTextResponse, 'images' | 'illusts'>): NovelAsset[];
|
|
52
|
+
/** Minimal self-check for the scanner's double-bracket handling. */
|
|
53
|
+
export declare function demo(): void;
|
|
54
|
+
//# sourceMappingURL=novelMarkers.d.ts.map
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scanNovelMarkers = scanNovelMarkers;
|
|
4
|
+
exports.extractNovelAssets = extractNovelAssets;
|
|
5
|
+
exports.demo = demo;
|
|
6
|
+
const linkRegex = /https?:\/\/\S+/;
|
|
7
|
+
function parseMarker(span) {
|
|
8
|
+
if (span.startsWith('[newpage]'))
|
|
9
|
+
return { type: 'newpage', raw: span };
|
|
10
|
+
if (span.startsWith('[chapter:')) {
|
|
11
|
+
return { type: 'chapter', raw: span, title: span.slice('[chapter:'.length, -1) };
|
|
12
|
+
}
|
|
13
|
+
if (span.startsWith('[pixivimage:')) {
|
|
14
|
+
return { type: 'pixivimage', raw: span, key: span.slice('[pixivimage:'.length, -1) };
|
|
15
|
+
}
|
|
16
|
+
if (span.startsWith('[uploadedimage:')) {
|
|
17
|
+
return { type: 'uploadedimage', raw: span, key: span.slice('[uploadedimage:'.length, -1) };
|
|
18
|
+
}
|
|
19
|
+
if (span.startsWith('[[jumpuri:')) {
|
|
20
|
+
const body = span.slice('[[jumpuri:'.length, span.endsWith(']]') ? -2 : -1);
|
|
21
|
+
return { type: 'jumpuri', raw: span, url: body.match(linkRegex)?.[0] };
|
|
22
|
+
}
|
|
23
|
+
if (span.startsWith('[[rb:')) {
|
|
24
|
+
const reading = span.slice('[[rb:'.length, span.endsWith(']]') ? -2 : -1);
|
|
25
|
+
return { type: 'ruby', raw: span, reading };
|
|
26
|
+
}
|
|
27
|
+
return { type: 'text', value: span };
|
|
28
|
+
}
|
|
29
|
+
function scanNovelMarkers(source) {
|
|
30
|
+
const result = [];
|
|
31
|
+
let now = '';
|
|
32
|
+
for (const ch of source) {
|
|
33
|
+
if (ch === '[') {
|
|
34
|
+
if (!now) {
|
|
35
|
+
now = ch;
|
|
36
|
+
}
|
|
37
|
+
else if (now === '[') {
|
|
38
|
+
now += ch;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
result.push(parseMarker(now));
|
|
42
|
+
now = ch;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
else if (ch === ']') {
|
|
46
|
+
if (now.startsWith('[[')) {
|
|
47
|
+
if (now.endsWith(']')) {
|
|
48
|
+
now += ch;
|
|
49
|
+
result.push(parseMarker(now));
|
|
50
|
+
now = '';
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
now += ch;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
now += ch;
|
|
58
|
+
result.push(parseMarker(now));
|
|
59
|
+
now = '';
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
now += ch;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
if (now)
|
|
67
|
+
result.push(parseMarker(now));
|
|
68
|
+
return result;
|
|
69
|
+
}
|
|
70
|
+
function uploadedImageUrl(img) {
|
|
71
|
+
if (!img?.urls)
|
|
72
|
+
return undefined;
|
|
73
|
+
return img.urls.original ?? img.urls.the1200X1200 ?? img.urls.the480Mw ?? img.urls.the240Mw ?? img.urls.the128X128;
|
|
74
|
+
}
|
|
75
|
+
function illustImageUrl(ref) {
|
|
76
|
+
if (!ref?.illust?.images)
|
|
77
|
+
return undefined;
|
|
78
|
+
return ref.illust.images.original ?? ref.illust.images.medium ?? ref.illust.images.small;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Build the ordered set of inline image assets referenced by the novel text.
|
|
82
|
+
* `unavailable` = the source is missing in the webview payload (deleted/no
|
|
83
|
+
* permission) so it will never succeed; `pending` = resolvable + downloadable.
|
|
84
|
+
*/
|
|
85
|
+
function extractNovelAssets(text, response) {
|
|
86
|
+
const assets = [];
|
|
87
|
+
for (const marker of scanNovelMarkers(text)) {
|
|
88
|
+
if (marker.type === 'uploadedimage') {
|
|
89
|
+
const url = uploadedImageUrl(response?.images?.[marker.key]);
|
|
90
|
+
assets.push({
|
|
91
|
+
marker: marker.raw,
|
|
92
|
+
kind: 'uploadedimage',
|
|
93
|
+
sourceId: marker.key,
|
|
94
|
+
url,
|
|
95
|
+
status: url ? 'pending' : 'unavailable',
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
else if (marker.type === 'pixivimage') {
|
|
99
|
+
const url = illustImageUrl(response?.illusts?.[marker.key]);
|
|
100
|
+
assets.push({
|
|
101
|
+
marker: marker.raw,
|
|
102
|
+
kind: 'pixivimage',
|
|
103
|
+
sourceId: marker.key,
|
|
104
|
+
url,
|
|
105
|
+
status: url ? 'pending' : 'unavailable',
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return assets;
|
|
110
|
+
}
|
|
111
|
+
/** Minimal self-check for the scanner's double-bracket handling. */
|
|
112
|
+
function demo() {
|
|
113
|
+
const { assert } = require('node:assert');
|
|
114
|
+
for (const s of [
|
|
115
|
+
'before [uploadedimage:11] after',
|
|
116
|
+
'[[jumpuri:Title > https://example.com/a]] tail',
|
|
117
|
+
'mixed [pixivimage:12551-1] and [[rb:漢字>かな]] done',
|
|
118
|
+
]) {
|
|
119
|
+
assert(Array.isArray(scanNovelMarkers(s)) && scanNovelMarkers(s).length > 0, 'markers parsed');
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
//# sourceMappingURL=novelMarkers.js.map
|
|
@@ -112,5 +112,16 @@ export interface IDatabase {
|
|
|
112
112
|
* Update file path in database
|
|
113
113
|
*/
|
|
114
114
|
updateFilePath(pixivId: string, type: 'illustration' | 'novel', oldPath: string, newPath: string): number;
|
|
115
|
+
/**
|
|
116
|
+
* Durable system-error ledger (observability; never throws).
|
|
117
|
+
*/
|
|
118
|
+
readonly systemErrors: {
|
|
119
|
+
record(input: import('../observability/types').SystemErrorInput): void;
|
|
120
|
+
list(opts?: Record<string, unknown>): import('../observability/types').SystemErrorRow[];
|
|
121
|
+
markResolved(id: number): {
|
|
122
|
+
changes: number;
|
|
123
|
+
};
|
|
124
|
+
countSince(botId: string | null, hours: number): number;
|
|
125
|
+
};
|
|
115
126
|
}
|
|
116
127
|
//# sourceMappingURL=IDatabase.d.ts.map
|
|
@@ -16,6 +16,10 @@ export interface IFileService {
|
|
|
16
16
|
* Save text file (novel)
|
|
17
17
|
*/
|
|
18
18
|
saveText(content: string, fileName: string, metadata?: FileMetadata): Promise<string>;
|
|
19
|
+
/**
|
|
20
|
+
* Save a binary asset into an explicit directory (e.g. novel inline images).
|
|
21
|
+
*/
|
|
22
|
+
saveBinary(buffer: ArrayBuffer, fileName: string, directory: string): Promise<string>;
|
|
19
23
|
/**
|
|
20
24
|
* Sanitize file name
|
|
21
25
|
*/
|
package/dist/logger.d.ts
CHANGED
|
@@ -1,18 +1,42 @@
|
|
|
1
1
|
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
2
|
+
export type LogFormat = 'text' | 'json';
|
|
3
|
+
export interface LogContext {
|
|
4
|
+
service?: string;
|
|
5
|
+
component?: string;
|
|
6
|
+
bot_id?: string;
|
|
7
|
+
schedule_id?: string;
|
|
8
|
+
slot_id?: string;
|
|
9
|
+
pixiv_id?: string;
|
|
10
|
+
stage?: string;
|
|
11
|
+
trace_id?: string;
|
|
12
|
+
[key: string]: unknown;
|
|
13
|
+
}
|
|
2
14
|
interface LogMeta {
|
|
3
15
|
[key: string]: unknown;
|
|
4
16
|
}
|
|
5
17
|
declare class Logger {
|
|
6
18
|
private static readonly levelOrder;
|
|
7
19
|
private threshold;
|
|
20
|
+
private format;
|
|
8
21
|
private logPath;
|
|
9
22
|
setLevel(level: LogLevel): void;
|
|
23
|
+
setFormat(format: LogFormat): void;
|
|
24
|
+
/** Run `fn` with a log context that is merged into every log line emitted inside. */
|
|
25
|
+
runWithContext<T>(ctx: LogContext, fn: () => T): T;
|
|
10
26
|
setLogPath(path: string): void;
|
|
11
27
|
debug(message: string, meta?: LogMeta): void;
|
|
12
28
|
info(message: string, meta?: LogMeta): void;
|
|
13
29
|
warn(message: string, meta?: LogMeta): void;
|
|
14
30
|
error(message: string, meta?: LogMeta): void;
|
|
15
31
|
private write;
|
|
32
|
+
/**
|
|
33
|
+
* Rotate the current log file when it exceeds the size cap:
|
|
34
|
+
* rename to <name>-<ts>.log.gz, then prune archived logs older than
|
|
35
|
+
* retention days. Run inside write (low-traffic scheduler, acceptable;
|
|
36
|
+
* use a background job if this ever becomes a high-rate service).
|
|
37
|
+
*/
|
|
38
|
+
private rotateIfNeeded;
|
|
39
|
+
private pruneArchives;
|
|
16
40
|
}
|
|
17
41
|
export declare const logger: Logger;
|
|
18
42
|
export {};
|