pixivflow 2.22.1 → 2.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/commands/SchedulerCommand.js +12 -3
  2. package/dist/commands/scheduler-runtime.d.ts +29 -0
  3. package/dist/commands/scheduler-runtime.js +54 -1
  4. package/dist/download/FileService.d.ts +10 -0
  5. package/dist/download/FileService.js +6 -0
  6. package/dist/download/NovelDownloader.js +56 -0
  7. package/dist/download/exec/DownloadExecutor.js +11 -2
  8. package/dist/download/handlers/IllustrationTargetHandler.js +15 -0
  9. package/dist/download/handlers/NovelTargetHandler.js +12 -2
  10. package/dist/download/novelMarkers.d.ts +54 -0
  11. package/dist/download/novelMarkers.js +122 -0
  12. package/dist/interfaces/IDatabase.d.ts +11 -0
  13. package/dist/interfaces/IFileService.d.ts +4 -0
  14. package/dist/logger.d.ts +24 -0
  15. package/dist/logger.js +88 -11
  16. package/dist/notification/NotificationPolicy.js +18 -5
  17. package/dist/observability/classify.d.ts +7 -0
  18. package/dist/observability/classify.js +51 -0
  19. package/dist/observability/context.d.ts +10 -0
  20. package/dist/observability/context.js +22 -0
  21. package/dist/observability/index.d.ts +7 -0
  22. package/dist/observability/index.js +36 -0
  23. package/dist/observability/types.d.ts +23 -0
  24. package/dist/observability/types.js +3 -0
  25. package/dist/package.json +1 -1
  26. package/dist/scheduler/RecoveryOutcome.d.ts +24 -0
  27. package/dist/scheduler/RecoveryOutcome.js +44 -0
  28. package/dist/scheduler/ScheduleTriggerServer.d.ts +4 -0
  29. package/dist/scheduler/SlotBusinessStatus.d.ts +32 -0
  30. package/dist/scheduler/SlotBusinessStatus.js +43 -0
  31. package/dist/scheduler/SlotCoordinator.d.ts +7 -0
  32. package/dist/scheduler/SlotCoordinator.js +13 -0
  33. package/dist/storage/Database.d.ts +4 -0
  34. package/dist/storage/Database.js +7 -0
  35. package/dist/storage/DatabaseMigration.js +21 -0
  36. package/dist/storage/repositories/SystemErrorRepository.d.ts +19 -0
  37. package/dist/storage/repositories/SystemErrorRepository.js +92 -0
  38. package/dist/version.js +1 -1
  39. package/dist/webui/package.json +1 -1
  40. package/dist/webui/routes/admin-logs.d.ts +3 -0
  41. package/dist/webui/routes/admin-logs.js +9 -0
  42. package/dist/webui/routes/handlers/admin-logs-handlers.d.ts +6 -0
  43. package/dist/webui/routes/handlers/admin-logs-handlers.js +106 -0
  44. package/dist/webui/routes/handlers/system-errors-handlers.d.ts +6 -0
  45. package/dist/webui/routes/handlers/system-errors-handlers.js +59 -0
  46. package/dist/webui/routes/system-errors.d.ts +3 -0
  47. package/dist/webui/routes/system-errors.js +9 -0
  48. package/dist/webui/server/server-routes.js +5 -0
  49. package/node_modules/@redtidev/pixiv-client/dist/app-api/novels.d.ts.map +1 -1
  50. package/node_modules/@redtidev/pixiv-client/dist/app-api/novels.js +18 -2
  51. package/node_modules/@redtidev/pixiv-client/dist/app-api/novels.js.map +1 -1
  52. package/node_modules/@redtidev/pixiv-client/dist/index.d.ts +1 -1
  53. package/node_modules/@redtidev/pixiv-client/dist/index.d.ts.map +1 -1
  54. package/node_modules/@redtidev/pixiv-client/dist/index.js.map +1 -1
  55. package/node_modules/@redtidev/pixiv-client/dist/models/index.d.ts +40 -4
  56. package/node_modules/@redtidev/pixiv-client/dist/models/index.d.ts.map +1 -1
  57. package/node_modules/@redtidev/pixiv-client/src/app-api/novels.ts +28 -2
  58. package/node_modules/@redtidev/pixiv-client/src/index.ts +3 -0
  59. package/node_modules/@redtidev/pixiv-client/src/models/index.ts +34 -4
  60. package/package.json +2 -2
@@ -12,6 +12,7 @@ const ScheduleTriggerServer_1 = require("../scheduler/ScheduleTriggerServer");
12
12
  const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
13
13
  const schedules_1 = require("../scheduler/schedules");
14
14
  const scheduler_runtime_1 = require("./scheduler-runtime");
15
+ const RecoveryOutcome_1 = require("../scheduler/RecoveryOutcome");
15
16
  const SchedulerIdleLifecycle_1 = require("./SchedulerIdleLifecycle");
16
17
  /**
17
18
  * Decide whether the authenticated HTTP trigger server should be mounted. It is
@@ -263,9 +264,17 @@ class SchedulerCommand extends Command_1.BaseCommand {
263
264
  recoverStatus: (targetId, requestId) => {
264
265
  const slot = runtime.database.slots.findRecoverySlot(requestId, targetId);
265
266
  const cell = slot && runtime.database.slots.getCell(slot.id, targetId);
266
- return slot && cell
267
- ? { requestId, slotId: slot.id, state: cell.status, slotStatus: slot.status }
268
- : null;
267
+ if (!slot || !cell)
268
+ return null;
269
+ const businessState = (0, RecoveryOutcome_1.recoveryOutcomeFor)(cell);
270
+ return {
271
+ requestId,
272
+ slotId: slot.id,
273
+ state: cell.status,
274
+ slotStatus: slot.status,
275
+ business_state: businessState,
276
+ message: (0, RecoveryOutcome_1.recoveryUserMessage)(businessState),
277
+ };
269
278
  },
270
279
  status: (scheduleId) => {
271
280
  const cfg = resolveConfig();
@@ -119,6 +119,35 @@ export interface SchedulerRuntime {
119
119
  close(): void;
120
120
  }
121
121
  export declare function notifyScheduleFailure(config: StandaloneConfig, database: Database, schedule: ScheduleConfig, failure: JobFailure): Promise<void>;
122
+ /**
123
+ * A `no_candidate` scan that attempted nothing and whose only candidate skips
124
+ * are `duplicate` is a TRUE dead-end: every candidate the scan surfaced was
125
+ * already delivered, and nothing was attempted to be widened. Advancing the
126
+ * fallback stage for this would re-fetch the SAME stale duplicate pool with a
127
+ * wider bound and burn the whole scheduler budget again (the production bug
128
+ * that turned an empty day on bot1 into a 30-minute OperationCancelledError).
129
+ * Terminalise it immediately instead of entering the fallback re-run loop.
130
+ */
131
+ export interface CandidateExhaustionDiagnostic {
132
+ stage: 'candidate_selection';
133
+ result: 'no_candidate';
134
+ reason: 'duplicate_exhausted' | 'filter_exhausted' | 'no_candidate';
135
+ searched: number;
136
+ duplicates: number;
137
+ filtered: number;
138
+ attempted: number;
139
+ }
140
+ /**
141
+ * Structured diagnostic for a terminal `no_candidate`: what the scan surfaced
142
+ * (`searched`), how many were already-delivered duplicates, how many were
143
+ * filtered by the run's own rules, and how many were actually attempted.
144
+ * Emitted as a JSON log so a Mini App can show "当天候选均已投稿" instead of
145
+ * treating the slot as a system failure.
146
+ */
147
+ export declare function candidateExhaustionDiagnostic(outcome: Extract<TargetOutcome, {
148
+ kind: 'no_candidate';
149
+ }>): CandidateExhaustionDiagnostic | null;
150
+ export declare function isDuplicateOnlyDeadEnd(outcome: TargetOutcome): boolean;
122
151
  /**
123
152
  * Expanded, still-bounded candidate scan bound for one fallback stage
124
153
  * (§schedule-recovery). Stage 0 is the primary pass; stage N scans
@@ -45,6 +45,8 @@ exports.EXECUTION_MODES = void 0;
45
45
  exports.withDeliveryMode = withDeliveryMode;
46
46
  exports.shouldTerminaliseAbortedSlot = shouldTerminaliseAbortedSlot;
47
47
  exports.notifyScheduleFailure = notifyScheduleFailure;
48
+ exports.candidateExhaustionDiagnostic = candidateExhaustionDiagnostic;
49
+ exports.isDuplicateOnlyDeadEnd = isDuplicateOnlyDeadEnd;
48
50
  exports.fallbackScanLimit = fallbackScanLimit;
49
51
  exports.createSchedulerRuntime = createSchedulerRuntime;
50
52
  exports.runWithTimeout = runWithTimeout;
@@ -201,6 +203,44 @@ function buildProxyUrl(network) {
201
203
  const auth = proxy.username ? `${proxy.username}:${proxy.password ?? ''}@` : '';
202
204
  return `${protocol}://${auth}${proxy.host}:${proxy.port}`;
203
205
  }
206
+ /**
207
+ * Structured diagnostic for a terminal `no_candidate`: what the scan surfaced
208
+ * (`searched`), how many were already-delivered duplicates, how many were
209
+ * filtered by the run's own rules, and how many were actually attempted.
210
+ * Emitted as a JSON log so a Mini App can show "当天候选均已投稿" instead of
211
+ * treating the slot as a system failure.
212
+ */
213
+ function candidateExhaustionDiagnostic(outcome) {
214
+ const scan = outcome.scan;
215
+ if (!scan)
216
+ return null;
217
+ const skipped = scan.skipped;
218
+ const duplicates = skipped.filter((s) => s.code === 'duplicate').length;
219
+ const filtered = skipped.filter((s) => s.code !== 'duplicate' && s.code !== 'unavailable').length;
220
+ const reason = isDuplicateOnlyDeadEnd(outcome)
221
+ ? 'duplicate_exhausted'
222
+ : filtered > 0
223
+ ? 'filter_exhausted'
224
+ : 'no_candidate';
225
+ return {
226
+ stage: 'candidate_selection',
227
+ result: 'no_candidate',
228
+ reason,
229
+ searched: skipped.length,
230
+ duplicates,
231
+ filtered,
232
+ attempted: scan.attempted,
233
+ };
234
+ }
235
+ function isDuplicateOnlyDeadEnd(outcome) {
236
+ if (outcome.kind !== 'no_candidate' || !outcome.scan)
237
+ return false;
238
+ const s = outcome.scan;
239
+ return (s.attempted === 0 &&
240
+ s.outages.length === 0 &&
241
+ s.skipped.length > 0 &&
242
+ s.skipped.every((skip) => skip.code === 'duplicate'));
243
+ }
204
244
  /**
205
245
  * Expanded, still-bounded candidate scan bound for one fallback stage
206
246
  * (§schedule-recovery). Stage 0 is the primary pass; stage N scans
@@ -536,13 +576,26 @@ async function createSchedulerRuntime(configPathArg) {
536
576
  // an exhausted occurrence reports the true cause — never a generic
537
577
  // "target did not complete".
538
578
  if (!scheduleSlot.manualRequestId &&
539
- (outcome.kind === 'no_candidate' || outcome.kind === 'duplicate') &&
579
+ (outcome.kind === 'duplicate' ||
580
+ (outcome.kind === 'no_candidate' && !isDuplicateOnlyDeadEnd(outcome))) &&
540
581
  coordinator.cellFallbackStage(scheduleSlot.slotId, target.id) < maxFallbackStages - 1) {
541
582
  coordinator.advanceFallback(scheduleSlot.slotId, target.id, outcome.kind === 'duplicate'
542
583
  ? `duplicate candidates (stage ${coordinator.cellFallbackStage(scheduleSlot.slotId, target.id)})`
543
584
  : outcome.reason ?? 'no eligible candidate', maxFallbackStages);
544
585
  return;
545
586
  }
587
+ if (outcome.kind === 'no_candidate') {
588
+ const diag = candidateExhaustionDiagnostic(outcome);
589
+ if (diag) {
590
+ logger_1.logger.info('candidate_exhaustion', {
591
+ schedule_id: scheduleSlot.scheduleId,
592
+ slot_id: scheduleSlot.slotId,
593
+ bot_id: scheduleSlot.scheduleId?.split('-')[0],
594
+ target_id: target.id,
595
+ ...diag,
596
+ });
597
+ }
598
+ }
546
599
  coordinator.applyOutcome(scheduleSlot.slotId, target.id, outcome);
547
600
  notificationPolicy.noteOutcome(scheduleSlot.slotId, scheduleSlot, schedule, target, outcome);
548
601
  if (scheduleSlot.manualRequestId && (outcome.kind === 'no_candidate' || outcome.kind === 'duplicate' ||
@@ -31,12 +31,22 @@ export interface PixivMetadata {
31
31
  name: string;
32
32
  is_chinese: boolean;
33
33
  };
34
+ assets?: Array<{
35
+ marker: string;
36
+ kind: 'uploadedimage' | 'pixivimage';
37
+ sourceId: string;
38
+ url?: string;
39
+ localPath?: string;
40
+ status: 'pending' | 'downloaded' | 'failed' | 'unavailable';
41
+ failureReason?: string;
42
+ }>;
34
43
  }
35
44
  export declare class FileService implements IFileService {
36
45
  private readonly storage;
37
46
  constructor(storage: StorageConfig);
38
47
  initialise(): Promise<void>;
39
48
  saveImage(buffer: ArrayBuffer, fileName: string, metadata?: FileMetadata): Promise<string>;
49
+ saveBinary(buffer: ArrayBuffer, fileName: string, directory: string): Promise<string>;
40
50
  saveText(content: string, fileName: string, metadata?: FileMetadata): Promise<string>;
41
51
  private static readonly MAX_FILENAME_BYTES;
42
52
  sanitizeFileName(name: string): string;
@@ -27,6 +27,12 @@ class FileService {
27
27
  await node_fs_1.promises.writeFile(uniquePath, Buffer.from(buffer));
28
28
  return uniquePath;
29
29
  }
30
+ async saveBinary(buffer, fileName, directory) {
31
+ await (0, fs_1.ensureDir)(directory);
32
+ const uniquePath = await this.findUniquePath(directory, fileName);
33
+ await node_fs_1.promises.writeFile(uniquePath, Buffer.from(buffer));
34
+ return uniquePath;
35
+ }
30
36
  async saveText(content, fileName, metadata) {
31
37
  const baseDirectory = this.storage.novelDirectory ?? this.storage.downloadDirectory;
32
38
  const organizationMode = this.storage.novelOrganization ?? 'flat';
@@ -35,7 +35,9 @@ var __importStar = (this && this.__importStar) || (function () {
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.NovelDownloader = void 0;
37
37
  const logger_1 = require("../logger");
38
+ const node_path_1 = require("node:path");
38
39
  const language_detection_1 = require("../utils/language-detection");
40
+ const novelMarkers_1 = require("./novelMarkers");
39
41
  const LANGUAGE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
40
42
  class NovelDownloader {
41
43
  client;
@@ -156,6 +158,33 @@ class NovelDownloader {
156
158
  date: detail.create_date ? new Date(detail.create_date) : new Date(),
157
159
  };
158
160
  const filePath = await this.fileService.saveText(content, fileName, metadata);
161
+ const assets = typeof textResponse === 'string'
162
+ ? []
163
+ : (0, novelMarkers_1.extractNovelAssets)(text, textResponse);
164
+ const hadImages = assets.length > 0;
165
+ const pendingAssets = assets.filter((a) => Boolean(a.url));
166
+ if (pendingAssets.length) {
167
+ const imagesDir = (0, node_path_1.join)((0, node_path_1.dirname)(filePath), 'images');
168
+ for (const asset of pendingAssets) {
169
+ try {
170
+ const buffer = await this.client.downloadImage(asset.url);
171
+ asset.localPath = await this.fileService.saveBinary(buffer, novelAssetFileName(asset), imagesDir);
172
+ asset.status = 'downloaded';
173
+ }
174
+ catch (error) {
175
+ asset.status = 'failed';
176
+ asset.failureReason = error instanceof Error ? error.message : String(error);
177
+ logger_1.logger.warn(`Failed to download novel inline image ${asset.sourceId} for novel ${detail.id}`, {
178
+ novelId: detail.id,
179
+ sourceId: asset.sourceId,
180
+ reason: asset.failureReason,
181
+ });
182
+ }
183
+ }
184
+ if (hadImages) {
185
+ logger_1.logger.info(`Novel ${detail.id} inline images: ${assets.filter((a) => a.status === 'downloaded').length}/${assets.length} downloaded`, { novelId: detail.id });
186
+ }
187
+ }
159
188
  const pixivMetadata = {
160
189
  pixiv_id: detail.id,
161
190
  title: detail.title,
@@ -181,6 +210,19 @@ class NovelDownloader {
181
210
  },
182
211
  }
183
212
  : {}),
213
+ ...(assets.length
214
+ ? {
215
+ assets: assets.map((a) => ({
216
+ marker: a.marker,
217
+ kind: a.kind,
218
+ sourceId: a.sourceId,
219
+ url: a.url,
220
+ localPath: a.localPath,
221
+ status: a.status,
222
+ failureReason: a.failureReason,
223
+ })),
224
+ }
225
+ : {}),
184
226
  };
185
227
  let metadataPath;
186
228
  try {
@@ -222,4 +264,18 @@ class NovelDownloader {
222
264
  }
223
265
  }
224
266
  exports.NovelDownloader = NovelDownloader;
267
+ function novelAssetFileName(asset) {
268
+ let ext = '';
269
+ try {
270
+ const pathname = new URL(asset.url).pathname;
271
+ const last = pathname.split('/').pop() || '';
272
+ const dot = last.lastIndexOf('.');
273
+ if (dot >= 0)
274
+ ext = last.slice(dot);
275
+ }
276
+ catch {
277
+ // fall through: no extension
278
+ }
279
+ return `${asset.sourceId}${ext}` || asset.sourceId;
280
+ }
225
281
  //# sourceMappingURL=NovelDownloader.js.map
@@ -26,8 +26,11 @@ class DownloadExecutor {
26
26
  itemId: item?.id,
27
27
  ...(contextProvider?.(item, index) ?? {}),
28
28
  };
29
+ const itemId = item?.id ?? item?.pixivId ?? String(item);
30
+ logger_1.logger.info('download_started', { pixiv_id: itemId, attempt, stage: 'download' });
29
31
  try {
30
32
  const res = await task(item, index);
33
+ logger_1.logger.info('download_completed', { pixiv_id: itemId, attempt, stage: 'download' });
31
34
  results[index] = res;
32
35
  break;
33
36
  }
@@ -35,21 +38,27 @@ class DownloadExecutor {
35
38
  onError?.(error, item, index, attempt);
36
39
  const decision = recovery.decide(error, effectiveContext);
37
40
  onDecision?.(decision, { item, index, attempt, error });
41
+ const status = error?.status;
42
+ const errMsg = error instanceof Error ? error.message : String(error);
38
43
  if (decision.action === 'skip') {
39
- logger_1.logger.warn(`Skipping item at index ${index} after attempt ${attempt}${decision.reason ? `: ${decision.reason}` : ''}`);
44
+ logger_1.logger.warn(status ? `download_http_failed` : `download_failed`, {
45
+ pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, error: errMsg,
46
+ });
40
47
  break;
41
48
  }
42
49
  if (decision.action === 'fail') {
50
+ logger_1.logger.error(`download_failed`, { pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, error: errMsg });
43
51
  throw error instanceof Error ? error : new Error(String(error));
44
52
  }
45
53
  // retry/backoff
46
54
  const delayMs = decision.delayMs ?? 0;
55
+ logger_1.logger.warn(`download_retry`, { pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, retry_delay_ms: delayMs, error: errMsg });
47
56
  if (delayMs > 0) {
48
57
  await new Promise((resolve) => setTimeout(resolve, delayMs));
49
58
  }
50
59
  attempt += 1;
51
60
  if (attempt > (decision.maxAttempts ?? maxAttempts)) {
52
- logger_1.logger.error(`Max attempts reached for item at index ${index}; failing`);
61
+ logger_1.logger.error(`download_failed`, { pixiv_id: itemId, attempt, stage: 'download', http_status: status ?? null, error: errMsg });
53
62
  throw error instanceof Error ? error : new Error(String(error));
54
63
  }
55
64
  }
@@ -8,6 +8,8 @@ const pixiv_utils_1 = require("../../utils/pixiv-utils");
8
8
  const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
9
9
  const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
10
10
  const target_label_1 = require("../../utils/target-label");
11
+ const observability_1 = require("../../observability");
12
+ const context_1 = require("../../observability/context");
11
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
12
14
  const deliveryContext_1 = require("./deliveryContext");
13
15
  class IllustrationTargetHandler {
@@ -159,6 +161,13 @@ class IllustrationTargetHandler {
159
161
  classifyError(error, displayTag, mode, target) {
160
162
  const message = error instanceof Error ? error.message : String(error);
161
163
  const scan = this.scan ?? undefined;
164
+ const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
165
+ (0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)(target, this.execution, {
166
+ message,
167
+ stage: 'pixiv_download',
168
+ ...cls,
169
+ trace_id: this.execution?.slotId,
170
+ }), cls);
162
171
  // A hard job-level outage is named as such and is never recorded as a
163
172
  // no-candidate business outcome, whatever its message happens to look like.
164
173
  const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
@@ -166,6 +175,8 @@ class IllustrationTargetHandler {
166
175
  logger_1.logger.error(`Illustration ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
167
176
  error: message,
168
177
  errorType: error instanceof Error ? error.constructor.name : typeof error,
178
+ error_type: cls.error_type,
179
+ retryable: cls.retryable,
169
180
  ...(outage ? { jobLevelOutage: outage } : {}),
170
181
  });
171
182
  if (outage) {
@@ -508,9 +519,13 @@ class IllustrationTargetHandler {
508
519
  if (endpoint) {
509
520
  errorMessage = `${errorMessage} [URL: ${endpoint}]`;
510
521
  }
522
+ const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
523
+ (0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)({ type: 'illustration' }, this.execution, { message: errorMessage, stage: 'pixiv_download', ...cls, trace_id: this.execution?.slotId }), cls);
511
524
  logger_1.logger.error(message, {
512
525
  error: errorMessage,
513
526
  errorType: error instanceof Error ? error.constructor.name : typeof error,
527
+ error_type: cls.error_type,
528
+ retryable: cls.retryable,
514
529
  stack: error instanceof Error ? error.stack : undefined,
515
530
  });
516
531
  }
@@ -8,6 +8,8 @@ const pixiv_utils_1 = require("../../utils/pixiv-utils");
8
8
  const TargetOutcome_1 = require("../../scheduler/TargetOutcome");
9
9
  const WorkIdentity_1 = require("../../scheduler/WorkIdentity");
10
10
  const target_label_1 = require("../../utils/target-label");
11
+ const observability_1 = require("../../observability");
12
+ const context_1 = require("../../observability/context");
11
13
  const DownloadPlanner_1 = require("../plan/DownloadPlanner");
12
14
  const deliveryContext_1 = require("./deliveryContext");
13
15
  class NovelTargetHandler {
@@ -109,7 +111,7 @@ class NovelTargetHandler {
109
111
  return this.summarize();
110
112
  }
111
113
  catch (error) {
112
- return this.classifyError(error, displayTag, mode);
114
+ return this.classifyError(error, displayTag, mode, target);
113
115
  }
114
116
  }
115
117
  /**
@@ -165,15 +167,19 @@ class NovelTargetHandler {
165
167
  ...(scan ? { scan } : {}),
166
168
  };
167
169
  }
168
- classifyError(error, displayTag, mode) {
170
+ classifyError(error, displayTag, mode, target) {
169
171
  const message = error instanceof Error ? error.message : String(error);
170
172
  const scan = this.scan ?? undefined;
173
+ const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
174
+ (0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)(target, this.execution, { message, stage: 'pixiv_download', ...cls, trace_id: this.execution?.slotId }), cls);
171
175
  // A hard job-level outage is named as such and is never recorded as a
172
176
  // no-candidate business outcome, whatever its message happens to look like.
173
177
  const outage = (0, TargetOutcome_1.classifyJobLevelOutage)(error);
174
178
  this.database.logExecution(displayTag, 'novel', 'failed', message);
175
179
  logger_1.logger.error(`Novel ${mode === 'ranking' ? 'ranking' : 'tag'} ${displayTag} failed`, {
176
180
  error: message,
181
+ error_type: cls.error_type,
182
+ retryable: cls.retryable,
177
183
  ...(outage ? { jobLevelOutage: outage } : {}),
178
184
  });
179
185
  if (outage) {
@@ -535,9 +541,13 @@ class NovelTargetHandler {
535
541
  if (endpoint) {
536
542
  errorMessage = `${errorMessage} [URL: ${endpoint}]`;
537
543
  }
544
+ const cls = (0, observability_1.classifySystemError)(error, error.status, 'pixiv_download');
545
+ (0, observability_1.recordSystemError)(this.database, (0, context_1.targetErrorContext)({ type: 'novel' }, this.execution, { message: errorMessage, stage: 'pixiv_download', ...cls, trace_id: this.execution?.slotId }), cls);
538
546
  logger_1.logger.error(message, {
539
547
  error: errorMessage,
540
548
  errorType: error instanceof Error ? error.constructor.name : typeof error,
549
+ error_type: cls.error_type,
550
+ retryable: cls.retryable,
541
551
  stack: error instanceof Error ? error.stack : undefined,
542
552
  });
543
553
  }
@@ -0,0 +1,54 @@
1
+ import type { PixivNovelTextResponse } from '@redtidev/pixiv-client';
2
+ /**
3
+ * Single-pass novel text tokenizer, ported from PixEz
4
+ * (Notsfsssf/pixez-flutter lib/page/novel/viewer/image_text.dart,
5
+ * NovelSpansGenerator.buildSpans). Same endpoint + same extraction path as
6
+ * PixivFlow's webview fallback; keeps the `[[...]]` double-bracket close rule so
7
+ * URLs/etc. inside jumpuri/ruby markers are not truncated early.
8
+ */
9
+ export type NovelMarker = {
10
+ type: 'text';
11
+ value: string;
12
+ } | {
13
+ type: 'newpage';
14
+ raw: string;
15
+ } | {
16
+ type: 'chapter';
17
+ raw: string;
18
+ title: string;
19
+ } | {
20
+ type: 'pixivimage';
21
+ raw: string;
22
+ key: string;
23
+ } | {
24
+ type: 'uploadedimage';
25
+ raw: string;
26
+ key: string;
27
+ } | {
28
+ type: 'jumpuri';
29
+ raw: string;
30
+ url?: string;
31
+ } | {
32
+ type: 'ruby';
33
+ raw: string;
34
+ reading: string;
35
+ };
36
+ export interface NovelAsset {
37
+ marker: string;
38
+ kind: 'uploadedimage' | 'pixivimage';
39
+ sourceId: string;
40
+ url?: string;
41
+ localPath?: string;
42
+ status: 'pending' | 'downloaded' | 'failed' | 'unavailable';
43
+ failureReason?: string;
44
+ }
45
+ export declare function scanNovelMarkers(source: string): NovelMarker[];
46
+ /**
47
+ * Build the ordered set of inline image assets referenced by the novel text.
48
+ * `unavailable` = the source is missing in the webview payload (deleted/no
49
+ * permission) so it will never succeed; `pending` = resolvable + downloadable.
50
+ */
51
+ export declare function extractNovelAssets(text: string, response?: Pick<PixivNovelTextResponse, 'images' | 'illusts'>): NovelAsset[];
52
+ /** Minimal self-check for the scanner's double-bracket handling. */
53
+ export declare function demo(): void;
54
+ //# sourceMappingURL=novelMarkers.d.ts.map
@@ -0,0 +1,122 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.scanNovelMarkers = scanNovelMarkers;
4
+ exports.extractNovelAssets = extractNovelAssets;
5
+ exports.demo = demo;
6
+ const linkRegex = /https?:\/\/\S+/;
7
+ function parseMarker(span) {
8
+ if (span.startsWith('[newpage]'))
9
+ return { type: 'newpage', raw: span };
10
+ if (span.startsWith('[chapter:')) {
11
+ return { type: 'chapter', raw: span, title: span.slice('[chapter:'.length, -1) };
12
+ }
13
+ if (span.startsWith('[pixivimage:')) {
14
+ return { type: 'pixivimage', raw: span, key: span.slice('[pixivimage:'.length, -1) };
15
+ }
16
+ if (span.startsWith('[uploadedimage:')) {
17
+ return { type: 'uploadedimage', raw: span, key: span.slice('[uploadedimage:'.length, -1) };
18
+ }
19
+ if (span.startsWith('[[jumpuri:')) {
20
+ const body = span.slice('[[jumpuri:'.length, span.endsWith(']]') ? -2 : -1);
21
+ return { type: 'jumpuri', raw: span, url: body.match(linkRegex)?.[0] };
22
+ }
23
+ if (span.startsWith('[[rb:')) {
24
+ const reading = span.slice('[[rb:'.length, span.endsWith(']]') ? -2 : -1);
25
+ return { type: 'ruby', raw: span, reading };
26
+ }
27
+ return { type: 'text', value: span };
28
+ }
29
+ function scanNovelMarkers(source) {
30
+ const result = [];
31
+ let now = '';
32
+ for (const ch of source) {
33
+ if (ch === '[') {
34
+ if (!now) {
35
+ now = ch;
36
+ }
37
+ else if (now === '[') {
38
+ now += ch;
39
+ }
40
+ else {
41
+ result.push(parseMarker(now));
42
+ now = ch;
43
+ }
44
+ }
45
+ else if (ch === ']') {
46
+ if (now.startsWith('[[')) {
47
+ if (now.endsWith(']')) {
48
+ now += ch;
49
+ result.push(parseMarker(now));
50
+ now = '';
51
+ }
52
+ else {
53
+ now += ch;
54
+ }
55
+ }
56
+ else {
57
+ now += ch;
58
+ result.push(parseMarker(now));
59
+ now = '';
60
+ }
61
+ }
62
+ else {
63
+ now += ch;
64
+ }
65
+ }
66
+ if (now)
67
+ result.push(parseMarker(now));
68
+ return result;
69
+ }
70
+ function uploadedImageUrl(img) {
71
+ if (!img?.urls)
72
+ return undefined;
73
+ return img.urls.original ?? img.urls.the1200X1200 ?? img.urls.the480Mw ?? img.urls.the240Mw ?? img.urls.the128X128;
74
+ }
75
+ function illustImageUrl(ref) {
76
+ if (!ref?.illust?.images)
77
+ return undefined;
78
+ return ref.illust.images.original ?? ref.illust.images.medium ?? ref.illust.images.small;
79
+ }
80
+ /**
81
+ * Build the ordered set of inline image assets referenced by the novel text.
82
+ * `unavailable` = the source is missing in the webview payload (deleted/no
83
+ * permission) so it will never succeed; `pending` = resolvable + downloadable.
84
+ */
85
+ function extractNovelAssets(text, response) {
86
+ const assets = [];
87
+ for (const marker of scanNovelMarkers(text)) {
88
+ if (marker.type === 'uploadedimage') {
89
+ const url = uploadedImageUrl(response?.images?.[marker.key]);
90
+ assets.push({
91
+ marker: marker.raw,
92
+ kind: 'uploadedimage',
93
+ sourceId: marker.key,
94
+ url,
95
+ status: url ? 'pending' : 'unavailable',
96
+ });
97
+ }
98
+ else if (marker.type === 'pixivimage') {
99
+ const url = illustImageUrl(response?.illusts?.[marker.key]);
100
+ assets.push({
101
+ marker: marker.raw,
102
+ kind: 'pixivimage',
103
+ sourceId: marker.key,
104
+ url,
105
+ status: url ? 'pending' : 'unavailable',
106
+ });
107
+ }
108
+ }
109
+ return assets;
110
+ }
111
+ /** Minimal self-check for the scanner's double-bracket handling. */
112
+ function demo() {
113
+ const { assert } = require('node:assert');
114
+ for (const s of [
115
+ 'before [uploadedimage:11] after',
116
+ '[[jumpuri:Title > https://example.com/a]] tail',
117
+ 'mixed [pixivimage:12551-1] and [[rb:漢字>かな]] done',
118
+ ]) {
119
+ assert(Array.isArray(scanNovelMarkers(s)) && scanNovelMarkers(s).length > 0, 'markers parsed');
120
+ }
121
+ }
122
+ //# sourceMappingURL=novelMarkers.js.map
@@ -112,5 +112,16 @@ export interface IDatabase {
112
112
  * Update file path in database
113
113
  */
114
114
  updateFilePath(pixivId: string, type: 'illustration' | 'novel', oldPath: string, newPath: string): number;
115
+ /**
116
+ * Durable system-error ledger (observability; never throws).
117
+ */
118
+ readonly systemErrors: {
119
+ record(input: import('../observability/types').SystemErrorInput): void;
120
+ list(opts?: Record<string, unknown>): import('../observability/types').SystemErrorRow[];
121
+ markResolved(id: number): {
122
+ changes: number;
123
+ };
124
+ countSince(botId: string | null, hours: number): number;
125
+ };
115
126
  }
116
127
  //# sourceMappingURL=IDatabase.d.ts.map
@@ -16,6 +16,10 @@ export interface IFileService {
16
16
  * Save text file (novel)
17
17
  */
18
18
  saveText(content: string, fileName: string, metadata?: FileMetadata): Promise<string>;
19
+ /**
20
+ * Save a binary asset into an explicit directory (e.g. novel inline images).
21
+ */
22
+ saveBinary(buffer: ArrayBuffer, fileName: string, directory: string): Promise<string>;
19
23
  /**
20
24
  * Sanitize file name
21
25
  */
package/dist/logger.d.ts CHANGED
@@ -1,18 +1,42 @@
1
1
  export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
2
+ export type LogFormat = 'text' | 'json';
3
+ export interface LogContext {
4
+ service?: string;
5
+ component?: string;
6
+ bot_id?: string;
7
+ schedule_id?: string;
8
+ slot_id?: string;
9
+ pixiv_id?: string;
10
+ stage?: string;
11
+ trace_id?: string;
12
+ [key: string]: unknown;
13
+ }
2
14
  interface LogMeta {
3
15
  [key: string]: unknown;
4
16
  }
5
17
  declare class Logger {
6
18
  private static readonly levelOrder;
7
19
  private threshold;
20
+ private format;
8
21
  private logPath;
9
22
  setLevel(level: LogLevel): void;
23
+ setFormat(format: LogFormat): void;
24
+ /** Run `fn` with a log context that is merged into every log line emitted inside. */
25
+ runWithContext<T>(ctx: LogContext, fn: () => T): T;
10
26
  setLogPath(path: string): void;
11
27
  debug(message: string, meta?: LogMeta): void;
12
28
  info(message: string, meta?: LogMeta): void;
13
29
  warn(message: string, meta?: LogMeta): void;
14
30
  error(message: string, meta?: LogMeta): void;
15
31
  private write;
32
+ /**
33
+ * Rotate the current log file when it exceeds the size cap:
34
+ * rename to <name>-<ts>.log.gz, then prune archived logs older than
35
+ * retention days. Run inside write (low-traffic scheduler, acceptable;
36
+ * use a background job if this ever becomes a high-rate service).
37
+ */
38
+ private rotateIfNeeded;
39
+ private pruneArchives;
16
40
  }
17
41
  export declare const logger: Logger;
18
42
  export {};