pixivflow 2.12.3 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -4
- package/dist/commands/DoctorCommand.d.ts +20 -0
- package/dist/commands/DoctorCommand.d.ts.map +1 -0
- package/dist/commands/DoctorCommand.js +107 -0
- package/dist/commands/DoctorCommand.js.map +1 -0
- package/dist/commands/ReconcileCommand.d.ts +24 -0
- package/dist/commands/ReconcileCommand.d.ts.map +1 -0
- package/dist/commands/ReconcileCommand.js +70 -0
- package/dist/commands/ReconcileCommand.js.map +1 -0
- package/dist/commands/SchedulerCommand.d.ts.map +1 -1
- package/dist/commands/SchedulerCommand.js +5 -0
- package/dist/commands/SchedulerCommand.js.map +1 -1
- package/dist/commands/SchedulerRunOnceCommand.d.ts.map +1 -1
- package/dist/commands/SchedulerRunOnceCommand.js +6 -0
- package/dist/commands/SchedulerRunOnceCommand.js.map +1 -1
- package/dist/commands/index.d.ts +3 -1
- package/dist/commands/index.d.ts.map +1 -1
- package/dist/commands/index.js +7 -1
- package/dist/commands/index.js.map +1 -1
- package/dist/commands/scheduler-runtime.d.ts +10 -1
- package/dist/commands/scheduler-runtime.d.ts.map +1 -1
- package/dist/commands/scheduler-runtime.js +157 -47
- package/dist/commands/scheduler-runtime.js.map +1 -1
- package/dist/config/types.d.ts +10 -1
- package/dist/config/types.d.ts.map +1 -1
- package/dist/delivery/DeliveryAck.d.ts +70 -0
- package/dist/delivery/DeliveryAck.d.ts.map +1 -0
- package/dist/delivery/DeliveryAck.js +72 -0
- package/dist/delivery/DeliveryAck.js.map +1 -0
- package/dist/delivery/DeliveryDispatcher.js +1 -1
- package/dist/delivery/DeliveryDispatcher.js.map +1 -1
- package/dist/delivery/DeliveryService.d.ts +74 -0
- package/dist/delivery/DeliveryService.d.ts.map +1 -0
- package/dist/delivery/DeliveryService.js +207 -0
- package/dist/delivery/DeliveryService.js.map +1 -0
- package/dist/delivery/HttpMultipartDelivery.d.ts +10 -1
- package/dist/delivery/HttpMultipartDelivery.d.ts.map +1 -1
- package/dist/delivery/HttpMultipartDelivery.js +40 -65
- package/dist/delivery/HttpMultipartDelivery.js.map +1 -1
- package/dist/delivery/LegacyOutboxMigration.d.ts +21 -0
- package/dist/delivery/LegacyOutboxMigration.d.ts.map +1 -0
- package/dist/delivery/LegacyOutboxMigration.js +125 -0
- package/dist/delivery/LegacyOutboxMigration.js.map +1 -0
- package/dist/delivery/OutboxWorker.d.ts +69 -0
- package/dist/delivery/OutboxWorker.d.ts.map +1 -0
- package/dist/delivery/OutboxWorker.js +194 -0
- package/dist/delivery/OutboxWorker.js.map +1 -0
- package/dist/delivery/types.d.ts +11 -0
- package/dist/delivery/types.d.ts.map +1 -1
- package/dist/download/DownloadManager.d.ts +17 -12
- package/dist/download/DownloadManager.d.ts.map +1 -1
- package/dist/download/DownloadManager.js +31 -53
- package/dist/download/DownloadManager.js.map +1 -1
- package/dist/download/NovelDownloader.d.ts +3 -1
- package/dist/download/NovelDownloader.d.ts.map +1 -1
- package/dist/download/NovelDownloader.js +30 -2
- package/dist/download/NovelDownloader.js.map +1 -1
- package/dist/download/handlers/IllustrationTargetHandler.d.ts +18 -8
- package/dist/download/handlers/IllustrationTargetHandler.d.ts.map +1 -1
- package/dist/download/handlers/IllustrationTargetHandler.js +100 -74
- package/dist/download/handlers/IllustrationTargetHandler.js.map +1 -1
- package/dist/download/handlers/NovelTargetHandler.d.ts +8 -8
- package/dist/download/handlers/NovelTargetHandler.d.ts.map +1 -1
- package/dist/download/handlers/NovelTargetHandler.js +64 -83
- package/dist/download/handlers/NovelTargetHandler.js.map +1 -1
- package/dist/download/plan/DownloadPlanner.d.ts +6 -1
- package/dist/download/plan/DownloadPlanner.d.ts.map +1 -1
- package/dist/download/plan/DownloadPlanner.js +28 -5
- package/dist/download/plan/DownloadPlanner.js.map +1 -1
- package/dist/notification/NotificationPolicy.d.ts +38 -0
- package/dist/notification/NotificationPolicy.d.ts.map +1 -0
- package/dist/notification/NotificationPolicy.js +89 -0
- package/dist/notification/NotificationPolicy.js.map +1 -0
- package/dist/package.json +1 -1
- package/dist/pixiv/PixivClient.d.ts +4 -0
- package/dist/pixiv/PixivClient.d.ts.map +1 -1
- package/dist/pixiv/PixivClient.js +15 -2
- package/dist/pixiv/PixivClient.js.map +1 -1
- package/dist/pixiv/RateLimitCoordinator.d.ts +30 -0
- package/dist/pixiv/RateLimitCoordinator.d.ts.map +1 -0
- package/dist/pixiv/RateLimitCoordinator.js +66 -0
- package/dist/pixiv/RateLimitCoordinator.js.map +1 -0
- package/dist/pixiv/client/PixivApiCore.d.ts +5 -0
- package/dist/pixiv/client/PixivApiCore.d.ts.map +1 -1
- package/dist/pixiv/client/PixivApiCore.js +31 -9
- package/dist/pixiv/client/PixivApiCore.js.map +1 -1
- package/dist/scheduler/ScheduleTriggerServer.d.ts +7 -0
- package/dist/scheduler/ScheduleTriggerServer.d.ts.map +1 -1
- package/dist/scheduler/ScheduleTriggerServer.js +17 -0
- package/dist/scheduler/ScheduleTriggerServer.js.map +1 -1
- package/dist/scheduler/SlotCoordinator.d.ts +15 -0
- package/dist/scheduler/SlotCoordinator.d.ts.map +1 -1
- package/dist/scheduler/SlotCoordinator.js +74 -0
- package/dist/scheduler/SlotCoordinator.js.map +1 -1
- package/dist/scheduler/SlotStateMachine.d.ts +32 -0
- package/dist/scheduler/SlotStateMachine.d.ts.map +1 -0
- package/dist/scheduler/SlotStateMachine.js +65 -0
- package/dist/scheduler/SlotStateMachine.js.map +1 -0
- package/dist/scheduler/TargetOutcome.d.ts +63 -0
- package/dist/scheduler/TargetOutcome.d.ts.map +1 -0
- package/dist/scheduler/TargetOutcome.js +28 -0
- package/dist/scheduler/TargetOutcome.js.map +1 -0
- package/dist/storage/Database.d.ts +16 -0
- package/dist/storage/Database.d.ts.map +1 -1
- package/dist/storage/Database.js +29 -0
- package/dist/storage/Database.js.map +1 -1
- package/dist/storage/DatabaseMigration.d.ts.map +1 -1
- package/dist/storage/DatabaseMigration.js +66 -0
- package/dist/storage/DatabaseMigration.js.map +1 -1
- package/dist/storage/repositories/DeliveryRepository.d.ts +73 -0
- package/dist/storage/repositories/DeliveryRepository.d.ts.map +1 -0
- package/dist/storage/repositories/DeliveryRepository.js +148 -0
- package/dist/storage/repositories/DeliveryRepository.js.map +1 -0
- package/dist/storage/repositories/MetadataRepository.d.ts +27 -0
- package/dist/storage/repositories/MetadataRepository.d.ts.map +1 -0
- package/dist/storage/repositories/MetadataRepository.js +58 -0
- package/dist/storage/repositories/MetadataRepository.js.map +1 -0
- package/dist/storage/repositories/OutboxRepository.d.ts +70 -0
- package/dist/storage/repositories/OutboxRepository.d.ts.map +1 -0
- package/dist/storage/repositories/OutboxRepository.js +187 -0
- package/dist/storage/repositories/OutboxRepository.js.map +1 -0
- package/dist/storage/repositories/SlotRepository.d.ts +26 -1
- package/dist/storage/repositories/SlotRepository.d.ts.map +1 -1
- package/dist/storage/repositories/SlotRepository.js +76 -2
- package/dist/storage/repositories/SlotRepository.js.map +1 -1
- package/dist/webui/package.json +1 -1
- package/package.json +1 -1
- package/dist/delivery/DeliveryOutbox.d.ts +0 -43
- package/dist/delivery/DeliveryOutbox.d.ts.map +0 -1
- package/dist/delivery/DeliveryOutbox.js +0 -293
- package/dist/delivery/DeliveryOutbox.js.map +0 -1
|
@@ -96,6 +96,9 @@ class DatabaseMigration {
|
|
|
96
96
|
trigger_source TEXT,
|
|
97
97
|
slot_date TEXT NOT NULL DEFAULT '',
|
|
98
98
|
slot_name TEXT NOT NULL DEFAULT '',
|
|
99
|
+
lease_owner TEXT,
|
|
100
|
+
lease_until INTEGER,
|
|
101
|
+
heartbeat_at INTEGER,
|
|
99
102
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
100
103
|
started_at DATETIME,
|
|
101
104
|
completed_at DATETIME,
|
|
@@ -117,6 +120,61 @@ class DatabaseMigration {
|
|
|
117
120
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
118
121
|
completed_at DATETIME,
|
|
119
122
|
UNIQUE(slot_id, target_id)
|
|
123
|
+
)`,
|
|
124
|
+
// Delivery ledger: confirmed downstream submissions. This is separate
|
|
125
|
+
// from downloads (local file facts). The natural idempotency key makes
|
|
126
|
+
// ACK-lost retries converge to one row; (target, type, pixiv_id) marks
|
|
127
|
+
// which works are already CONFIRMED delivered to which target.
|
|
128
|
+
`CREATE TABLE IF NOT EXISTS deliveries (
|
|
129
|
+
id TEXT PRIMARY KEY,
|
|
130
|
+
delivery_target TEXT NOT NULL,
|
|
131
|
+
work_type TEXT NOT NULL,
|
|
132
|
+
pixiv_id TEXT NOT NULL,
|
|
133
|
+
slot_id TEXT,
|
|
134
|
+
target_id TEXT,
|
|
135
|
+
idempotency_key TEXT NOT NULL UNIQUE,
|
|
136
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
137
|
+
remote_id TEXT,
|
|
138
|
+
remote_status TEXT,
|
|
139
|
+
reuse_reason TEXT,
|
|
140
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
141
|
+
last_error TEXT,
|
|
142
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
143
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
144
|
+
delivered_at DATETIME
|
|
145
|
+
)`,
|
|
146
|
+
// Durable transactional outbox: every external side effect (content
|
|
147
|
+
// delivery or notification) originates from one row here. The worker
|
|
148
|
+
// leases due rows; crashes leave an expired lease that a restart claims.
|
|
149
|
+
`CREATE TABLE IF NOT EXISTS outbox (
|
|
150
|
+
id TEXT PRIMARY KEY,
|
|
151
|
+
kind TEXT NOT NULL,
|
|
152
|
+
idempotency_key TEXT,
|
|
153
|
+
delivery_id TEXT,
|
|
154
|
+
delivery_target TEXT NOT NULL,
|
|
155
|
+
payload_json TEXT NOT NULL,
|
|
156
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
157
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
158
|
+
max_attempts INTEGER NOT NULL DEFAULT 12,
|
|
159
|
+
next_attempt_at INTEGER NOT NULL,
|
|
160
|
+
lease_owner TEXT,
|
|
161
|
+
lease_until INTEGER,
|
|
162
|
+
last_error TEXT,
|
|
163
|
+
created_at INTEGER NOT NULL,
|
|
164
|
+
updated_at INTEGER NOT NULL,
|
|
165
|
+
completed_at INTEGER
|
|
166
|
+
)`,
|
|
167
|
+
// Lightweight Pixiv metadata cache (novel language, rating) to cut
|
|
168
|
+
// repeated detail/full-text requests and 429 amplification.
|
|
169
|
+
`CREATE TABLE IF NOT EXISTS pixiv_metadata (
|
|
170
|
+
pixiv_id TEXT NOT NULL,
|
|
171
|
+
work_type TEXT NOT NULL,
|
|
172
|
+
language TEXT,
|
|
173
|
+
x_restrict INTEGER,
|
|
174
|
+
published_at TEXT,
|
|
175
|
+
title TEXT,
|
|
176
|
+
checked_at INTEGER NOT NULL,
|
|
177
|
+
PRIMARY KEY (pixiv_id, work_type)
|
|
120
178
|
)`,
|
|
121
179
|
];
|
|
122
180
|
// Phase 1: create tables (idempotent). Must run before any PRAGMA-based
|
|
@@ -139,6 +197,9 @@ class DatabaseMigration {
|
|
|
139
197
|
occurrence_label: `ALTER TABLE schedule_slots ADD COLUMN occurrence_label TEXT NOT NULL DEFAULT ''`,
|
|
140
198
|
timezone: `ALTER TABLE schedule_slots ADD COLUMN timezone TEXT NOT NULL DEFAULT 'UTC'`,
|
|
141
199
|
target_ids: 'ALTER TABLE schedule_slots ADD COLUMN target_ids TEXT',
|
|
200
|
+
lease_owner: 'ALTER TABLE schedule_slots ADD COLUMN lease_owner TEXT',
|
|
201
|
+
lease_until: 'ALTER TABLE schedule_slots ADD COLUMN lease_until INTEGER',
|
|
202
|
+
heartbeat_at: 'ALTER TABLE schedule_slots ADD COLUMN heartbeat_at INTEGER',
|
|
142
203
|
};
|
|
143
204
|
const columnAlters = [];
|
|
144
205
|
for (const [col, sql] of Object.entries(slotColumnMigrations)) {
|
|
@@ -160,6 +221,11 @@ class DatabaseMigration {
|
|
|
160
221
|
`CREATE INDEX IF NOT EXISTS idx_slots_schedule_occ ON schedule_slots(schedule_id, occurrence_at DESC)`,
|
|
161
222
|
`CREATE INDEX IF NOT EXISTS idx_slot_items_slot ON schedule_slot_items(slot_id)`,
|
|
162
223
|
`CREATE INDEX IF NOT EXISTS idx_slot_items_work ON schedule_slot_items(work_id, work_type)`,
|
|
224
|
+
`CREATE INDEX IF NOT EXISTS idx_deliveries_dedupe ON deliveries(delivery_target, work_type, pixiv_id, status)`,
|
|
225
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_outbox_key ON outbox(kind, idempotency_key) WHERE idempotency_key IS NOT NULL`,
|
|
226
|
+
`CREATE INDEX IF NOT EXISTS idx_outbox_due ON outbox(status, next_attempt_at)`,
|
|
227
|
+
`CREATE INDEX IF NOT EXISTS idx_outbox_delivery ON outbox(delivery_id)`,
|
|
228
|
+
`CREATE INDEX IF NOT EXISTS idx_slots_lease ON schedule_slots(lease_until)`,
|
|
163
229
|
];
|
|
164
230
|
const postMigration = this.db.transaction((stmts) => {
|
|
165
231
|
for (const sql of stmts) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DatabaseMigration.js","sourceRoot":"","sources":["../../src/storage/DatabaseMigration.ts"],"names":[],"mappings":";;;AACA,4CAAgD;AAChD,sCAAmC;AAEnC;;GAEG;AACH,MAAa,iBAAiB;IACC;IAA7B,YAA6B,EAA2B;QAA3B,OAAE,GAAF,EAAE,CAAyB;IAAG,CAAC;IAE5D;;OAEG;IACI,OAAO;QACZ,IAAI,CAAC;YACH,MAAM,UAAU,GAAG;gBACjB;;;;YAII;gBACJ;;;;;;;;;;;YAWI;gBACJ;;;;;;;YAOI;gBACJ;;;;;;;;;;YAUI;gBACJ;;;;;;;YAOI;gBACJ;;;;;;;;;;;;;YAaI;gBACJ,uEAAuE;gBACvE,sEAAsE;gBACtE,oEAAoE;gBACpE,EAAE;gBACF,sEAAsE;gBACtE,uEAAuE;gBACvE,oEAAoE;gBACpE,qEAAqE;gBACrE,6DAA6D;gBAC7D
|
|
1
|
+
{"version":3,"file":"DatabaseMigration.js","sourceRoot":"","sources":["../../src/storage/DatabaseMigration.ts"],"names":[],"mappings":";;;AACA,4CAAgD;AAChD,sCAAmC;AAEnC;;GAEG;AACH,MAAa,iBAAiB;IACC;IAA7B,YAA6B,EAA2B;QAA3B,OAAE,GAAF,EAAE,CAAyB;IAAG,CAAC;IAE5D;;OAEG;IACI,OAAO;QACZ,IAAI,CAAC;YACH,MAAM,UAAU,GAAG;gBACjB;;;;YAII;gBACJ;;;;;;;;;;;YAWI;gBACJ;;;;;;;YAOI;gBACJ;;;;;;;;;;YAUI;gBACJ;;;;;;;YAOI;gBACJ;;;;;;;;;;;;;YAaI;gBACJ,uEAAuE;gBACvE,sEAAsE;gBACtE,oEAAoE;gBACpE,EAAE;gBACF,sEAAsE;gBACtE,uEAAuE;gBACvE,oEAAoE;gBACpE,qEAAqE;gBACrE,6DAA6D;gBAC7D;;;;;;;;;;;;;;;;;;;YAmBI;gBACJ,sEAAsE;gBACtE,sEAAsE;gBACtE,mEAAmE;gBACnE;;;;;;;;;;;;;YAaI;gBACJ,sEAAsE;gBACtE,uEAAuE;gBACvE,uEAAuE;gBACvE,+DAA+D;gBAC/D;;;;;;;;;;;;;;;;;YAiBI;gBACJ,oEAAoE;gBACpE,qEAAqE;gBACrE,yEAAyE;gBACzE;;;;;;;;;;;;;;;;;YAiBI;gBACJ,mEAAmE;gBACnE,4DAA4D;gBAC5D;;;;;;;;;YASI;aACL,CAAC;YAEF,wEAAwE;YACxE,2EAA2E;YAC3E,2DAA2D;YAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAe,EAAE,EAAE;gBAC3D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;gBAC7B,CAAC;YACH,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,UAAU,CAAC,CAAC;YAEzB,uEAAuE;YACvE,0EAA0E;YAC1E,2EAA2E;YAC3E,+DAA+D;YAC/D,MAAM,QAAQ,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,EAA8B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5H,MAAM,oBAAoB,GAA2B;gBACnD,aAAa,EAAE,6DAA6D;gBAC5E,eAAe,EAAE,gFAAgF;gBACjG,gBAAgB,EAAE,iFAAiF;gBACnG,QAAQ,EAAE,4EAA4E;gBACtF,UAAU,EAAE,uDAAuD;gBACnE,WAAW,EAAE,wDAAwD;gBACrE,WAAW,EAAE,2DAA2D;gBACxE,YAAY,EAAE,4DAA4D;aAC3E,CAAC;YACF,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBAC9D,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;oBAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtD,CAAC;YAED,8CAA8C;YAC9C,MAAM,OAAO,GAAG;gBACd,qFAAqF;gBACrF,gEAAgE;gBAChE,oFAAoF;gBACpF,mFAAmF;gBACnF,sGAAsG;gBACtG,4FAA4F;gBAC5F,wFAAwF;gBACxF,8EAA8E;gBAC9E,4EAA4E;gBAC5E,oFAAoF;gBACpF,sGAAsG;gBACtG,gFAAgF;gBAChF,2FAA2F;gBAC3F,8GAA8G;gBAC9G,qHAAqH;gBACrH,8EAA8E;gBAC9E,uEAAuE;gBACvE,2EAA2E;aAC5E,CAAC;YAEF,MAAM,aAAa,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAe,EAAE,EAAE;gBAC5D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;gBAC7B,CAAC;YACH,CAAC,CAAC,CAAC;YACH,aAAa,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;YAE7C,6DAA6D;YAC7D,IAAI,CAAC;gBACH,uDAAuD;gBACvD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,EAA6B,CAAC;gBACxG,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;gBAE1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACvB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,EAAE,CAAC;oBAC3F,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sFAAsF,CAAC,CAAC,GAAG,EAAE,CAAC;gBAChH,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,2CAA2C;gBAC3C,oEAAoE;gBACpE,eAAM,CAAC,IAAI,CAAC,oDAAoD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,CAAC;YAED,yEAAyE;YACzE,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAA6B,CAAC;gBAC9G,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,CAAC;oBACvD,IAAI,CAAC,EAAE,CAAC,OAAO,CACb,yFAAyF,CAC1F,CAAC,GAAG,EAAE,CAAC;gBACV,CAAC;gBACD,IAAI,CAAC,EAAE,CAAC,OAAO,CACb,qHAAqH,CACtH,CAAC,GAAG,EAAE,CAAC;YACV,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,eAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,0EAA0E;YAC1E,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,GAAG,EAA6B,CAAC;gBACtG,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3B,2EAA2E;oBAC3E,8BAA8B;oBAC9B,eAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,eAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,sBAAa,CACrB,mCAAmC,EACnC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAtRD,8CAsRC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { BaseRepository } from './BaseRepository';
|
|
2
|
+
export type DeliveryStatus = 'pending' | 'delivered' | 'duplicate' | 'failed';
|
|
3
|
+
export interface DeliveryRow {
|
|
4
|
+
id: string;
|
|
5
|
+
deliveryTarget: string;
|
|
6
|
+
workType: string;
|
|
7
|
+
pixivId: string;
|
|
8
|
+
slotId: string | null;
|
|
9
|
+
targetId: string | null;
|
|
10
|
+
idempotencyKey: string;
|
|
11
|
+
status: DeliveryStatus;
|
|
12
|
+
remoteId: string | null;
|
|
13
|
+
remoteStatus: string | null;
|
|
14
|
+
reuseReason: string | null;
|
|
15
|
+
attempts: number;
|
|
16
|
+
lastError: string | null;
|
|
17
|
+
createdAt: string;
|
|
18
|
+
updatedAt: string;
|
|
19
|
+
deliveredAt: string | null;
|
|
20
|
+
}
|
|
21
|
+
export interface NewDelivery {
|
|
22
|
+
id: string;
|
|
23
|
+
deliveryTarget: string;
|
|
24
|
+
workType: string;
|
|
25
|
+
pixivId: string;
|
|
26
|
+
slotId?: string | null;
|
|
27
|
+
targetId?: string | null;
|
|
28
|
+
idempotencyKey: string;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The delivery ledger. Answers "has this work actually been CONFIRMED delivered
|
|
32
|
+
* to this target?" — distinct from the downloads table ("was the file cached
|
|
33
|
+
* locally?"). downloaded != delivered.
|
|
34
|
+
*
|
|
35
|
+
* Dedup scope is (delivery_target, work_type, pixiv_id): bot1 having posted a
|
|
36
|
+
* work does not forbid bot2 posting it.
|
|
37
|
+
*/
|
|
38
|
+
export declare class DeliveryRepository extends BaseRepository {
|
|
39
|
+
/** Insert a pending intent. No-op if the idempotency key already exists. */
|
|
40
|
+
insertIntent(input: NewDelivery): {
|
|
41
|
+
row: DeliveryRow;
|
|
42
|
+
created: boolean;
|
|
43
|
+
};
|
|
44
|
+
getById(id: string): DeliveryRow | null;
|
|
45
|
+
getByIdempotencyKey(key: string): DeliveryRow | null;
|
|
46
|
+
/**
|
|
47
|
+
* True when this exact work is already CONFIRMED (delivered, or an attested
|
|
48
|
+
* historical duplicate) for this target. Pending/failed intents do not block
|
|
49
|
+
* selection — they are retried, not treated as a delivered fact.
|
|
50
|
+
*/
|
|
51
|
+
isDelivered(deliveryTarget: string, workType: string, pixivId: string): boolean;
|
|
52
|
+
/** Batch form for candidate-pipeline pre-lock dedupe. */
|
|
53
|
+
deliveredIds(deliveryTarget: string, workType: string, pixivIds: string[]): Set<string>;
|
|
54
|
+
recordAck(id: string, ack: {
|
|
55
|
+
status: DeliveryStatus;
|
|
56
|
+
remoteId?: string;
|
|
57
|
+
remoteStatus?: string;
|
|
58
|
+
reuseReason?: string;
|
|
59
|
+
error?: string;
|
|
60
|
+
}): void;
|
|
61
|
+
/** Backfill a delivery fact discovered during downstream reconciliation. */
|
|
62
|
+
backfill(input: NewDelivery & {
|
|
63
|
+
remoteId?: string;
|
|
64
|
+
remoteStatus?: string;
|
|
65
|
+
}): {
|
|
66
|
+
row: DeliveryRow;
|
|
67
|
+
created: boolean;
|
|
68
|
+
};
|
|
69
|
+
pending(limit?: number): DeliveryRow[];
|
|
70
|
+
countByStatus(): Record<DeliveryStatus, number>;
|
|
71
|
+
private toRow;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=DeliveryRepository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DeliveryRepository.d.ts","sourceRoot":"","sources":["../../../src/storage/repositories/DeliveryRepository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE9E,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,cAAc,CAAC;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;GAOG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IACpD,4EAA4E;IAC5E,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG;QAAE,GAAG,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAsBxE,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAKvC,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAKpD;;;;OAIG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO;IAY/E,yDAAyD;IACzD,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IAmBvF,SAAS,CACP,EAAE,EAAE,MAAM,EACV,GAAG,EAAE;QAAE,MAAM,EAAE,cAAc,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC9G,IAAI;IAwBP,4EAA4E;IAC5E,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,GAAG,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAYnH,OAAO,CAAC,KAAK,SAAM,GAAG,WAAW,EAAE;IAOnC,aAAa,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC;IAS/C,OAAO,CAAC,KAAK;CAoBd"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DeliveryRepository = void 0;
|
|
4
|
+
const BaseRepository_1 = require("./BaseRepository");
|
|
5
|
+
/**
|
|
6
|
+
* The delivery ledger. Answers "has this work actually been CONFIRMED delivered
|
|
7
|
+
* to this target?" — distinct from the downloads table ("was the file cached
|
|
8
|
+
* locally?"). downloaded != delivered.
|
|
9
|
+
*
|
|
10
|
+
* Dedup scope is (delivery_target, work_type, pixiv_id): bot1 having posted a
|
|
11
|
+
* work does not forbid bot2 posting it.
|
|
12
|
+
*/
|
|
13
|
+
class DeliveryRepository extends BaseRepository_1.BaseRepository {
|
|
14
|
+
/** Insert a pending intent. No-op if the idempotency key already exists. */
|
|
15
|
+
insertIntent(input) {
|
|
16
|
+
const info = this.db
|
|
17
|
+
.prepare(`INSERT INTO deliveries
|
|
18
|
+
(id, delivery_target, work_type, pixiv_id, slot_id, target_id, idempotency_key, status)
|
|
19
|
+
VALUES
|
|
20
|
+
(@id, @deliveryTarget, @workType, @pixivId, @slotId, @targetId, @idempotencyKey, 'pending')
|
|
21
|
+
ON CONFLICT(idempotency_key) DO NOTHING`)
|
|
22
|
+
.run({
|
|
23
|
+
id: input.id,
|
|
24
|
+
deliveryTarget: input.deliveryTarget,
|
|
25
|
+
workType: input.workType,
|
|
26
|
+
pixivId: input.pixivId,
|
|
27
|
+
slotId: input.slotId ?? null,
|
|
28
|
+
targetId: input.targetId ?? null,
|
|
29
|
+
idempotencyKey: input.idempotencyKey,
|
|
30
|
+
});
|
|
31
|
+
const created = info.changes > 0;
|
|
32
|
+
return { row: created ? this.getById(input.id) : this.getByIdempotencyKey(input.idempotencyKey), created };
|
|
33
|
+
}
|
|
34
|
+
getById(id) {
|
|
35
|
+
const row = this.db.prepare(`SELECT * FROM deliveries WHERE id = ?`).get(id);
|
|
36
|
+
return row ? this.toRow(row) : null;
|
|
37
|
+
}
|
|
38
|
+
getByIdempotencyKey(key) {
|
|
39
|
+
const row = this.db.prepare(`SELECT * FROM deliveries WHERE idempotency_key = ?`).get(key);
|
|
40
|
+
return row ? this.toRow(row) : null;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* True when this exact work is already CONFIRMED (delivered, or an attested
|
|
44
|
+
* historical duplicate) for this target. Pending/failed intents do not block
|
|
45
|
+
* selection — they are retried, not treated as a delivered fact.
|
|
46
|
+
*/
|
|
47
|
+
isDelivered(deliveryTarget, workType, pixivId) {
|
|
48
|
+
const row = this.db
|
|
49
|
+
.prepare(`SELECT 1 FROM deliveries
|
|
50
|
+
WHERE delivery_target = ? AND work_type = ? AND pixiv_id = ?
|
|
51
|
+
AND status IN ('delivered','duplicate')
|
|
52
|
+
LIMIT 1`)
|
|
53
|
+
.get(deliveryTarget, workType, String(pixivId));
|
|
54
|
+
return Boolean(row);
|
|
55
|
+
}
|
|
56
|
+
/** Batch form for candidate-pipeline pre-lock dedupe. */
|
|
57
|
+
deliveredIds(deliveryTarget, workType, pixivIds) {
|
|
58
|
+
const out = new Set();
|
|
59
|
+
if (pixivIds.length === 0)
|
|
60
|
+
return out;
|
|
61
|
+
const CHUNK = 400;
|
|
62
|
+
for (let i = 0; i < pixivIds.length; i += CHUNK) {
|
|
63
|
+
const slice = pixivIds.slice(i, i + CHUNK);
|
|
64
|
+
const placeholders = slice.map(() => '?').join(',');
|
|
65
|
+
const rows = this.db
|
|
66
|
+
.prepare(`SELECT DISTINCT pixiv_id FROM deliveries
|
|
67
|
+
WHERE delivery_target = ? AND work_type = ? AND status IN ('delivered','duplicate')
|
|
68
|
+
AND pixiv_id IN (${placeholders})`)
|
|
69
|
+
.all(deliveryTarget, workType, ...slice);
|
|
70
|
+
for (const r of rows)
|
|
71
|
+
out.add(r.pixiv_id);
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
recordAck(id, ack) {
|
|
76
|
+
const terminal = ack.status === 'delivered' || ack.status === 'duplicate' || ack.status === 'failed';
|
|
77
|
+
const sets = [
|
|
78
|
+
'status = @status',
|
|
79
|
+
'remote_id = COALESCE(@remoteId, remote_id)',
|
|
80
|
+
'remote_status = @remoteStatus',
|
|
81
|
+
'reuse_reason = @reuseReason',
|
|
82
|
+
'last_error = @error',
|
|
83
|
+
'attempts = attempts + 1',
|
|
84
|
+
'updated_at = CURRENT_TIMESTAMP',
|
|
85
|
+
];
|
|
86
|
+
if (terminal)
|
|
87
|
+
sets.push(`delivered_at = COALESCE(delivered_at, CASE WHEN @status IN ('delivered','duplicate') THEN CURRENT_TIMESTAMP ELSE NULL END)`);
|
|
88
|
+
this.db
|
|
89
|
+
.prepare(`UPDATE deliveries SET ${sets.join(', ')} WHERE id = @id`)
|
|
90
|
+
.run({
|
|
91
|
+
id,
|
|
92
|
+
status: ack.status,
|
|
93
|
+
remoteId: ack.remoteId ?? null,
|
|
94
|
+
remoteStatus: ack.remoteStatus ?? null,
|
|
95
|
+
reuseReason: ack.reuseReason ?? null,
|
|
96
|
+
error: ack.error ?? null,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
/** Backfill a delivery fact discovered during downstream reconciliation. */
|
|
100
|
+
backfill(input) {
|
|
101
|
+
const { row, created } = this.insertIntent(input);
|
|
102
|
+
if (created || row.status !== 'delivered') {
|
|
103
|
+
this.recordAck(input.id, {
|
|
104
|
+
status: 'delivered',
|
|
105
|
+
remoteId: input.remoteId,
|
|
106
|
+
remoteStatus: input.remoteStatus ?? 'reconciled',
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
return { row: this.getById(input.id), created };
|
|
110
|
+
}
|
|
111
|
+
pending(limit = 200) {
|
|
112
|
+
const rows = this.db
|
|
113
|
+
.prepare(`SELECT * FROM deliveries WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?`)
|
|
114
|
+
.all(limit);
|
|
115
|
+
return rows.map((r) => this.toRow(r));
|
|
116
|
+
}
|
|
117
|
+
countByStatus() {
|
|
118
|
+
const rows = this.db
|
|
119
|
+
.prepare(`SELECT status, COUNT(*) AS n FROM deliveries GROUP BY status`)
|
|
120
|
+
.all();
|
|
121
|
+
const out = { pending: 0, delivered: 0, duplicate: 0, failed: 0 };
|
|
122
|
+
for (const r of rows)
|
|
123
|
+
out[r.status] = r.n;
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
toRow(row) {
|
|
127
|
+
return {
|
|
128
|
+
id: row.id,
|
|
129
|
+
deliveryTarget: row.delivery_target,
|
|
130
|
+
workType: row.work_type,
|
|
131
|
+
pixivId: row.pixiv_id,
|
|
132
|
+
slotId: row.slot_id,
|
|
133
|
+
targetId: row.target_id,
|
|
134
|
+
idempotencyKey: row.idempotency_key,
|
|
135
|
+
status: row.status,
|
|
136
|
+
remoteId: row.remote_id,
|
|
137
|
+
remoteStatus: row.remote_status,
|
|
138
|
+
reuseReason: row.reuse_reason,
|
|
139
|
+
attempts: row.attempts,
|
|
140
|
+
lastError: row.last_error,
|
|
141
|
+
createdAt: row.created_at,
|
|
142
|
+
updatedAt: row.updated_at,
|
|
143
|
+
deliveredAt: row.delivered_at,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
exports.DeliveryRepository = DeliveryRepository;
|
|
148
|
+
//# sourceMappingURL=DeliveryRepository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DeliveryRepository.js","sourceRoot":"","sources":["../../../src/storage/repositories/DeliveryRepository.ts"],"names":[],"mappings":";;;AAAA,qDAAkD;AAiClD;;;;;;;GAOG;AACH,MAAa,kBAAmB,SAAQ,+BAAc;IACpD,4EAA4E;IAC5E,YAAY,CAAC,KAAkB;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;iDAIyC,CAC1C;aACA,GAAG,CAAC;YACH,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,IAAI;YAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;YAChC,cAAc,EAAE,KAAK,CAAC,cAAc;SACrC,CAAC,CAAC;QACL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;QACjC,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,cAAc,CAAE,EAAE,OAAO,EAAE,CAAC;IAC/G,CAAC;IAED,OAAO,CAAC,EAAU;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAQ,CAAC;QACpF,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtC,CAAC;IAED,mBAAmB,CAAC,GAAW;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,GAAG,CAAQ,CAAC;QAClG,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtC,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,cAAsB,EAAE,QAAgB,EAAE,OAAe;QACnE,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CACN;;;iBAGS,CACV;aACA,GAAG,CAAC,cAAc,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,yDAAyD;IACzD,YAAY,CAAC,cAAsB,EAAE,QAAgB,EAAE,QAAkB;QACvE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,CAAC;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC3C,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;iBACjB,OAAO,CACN;;gCAEsB,YAAY,GAAG,CACtC;iBACA,GAAG,CAAC,cAAc,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAgC,CAAC;YAC1E,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,SAAS,CACP,EAAU,EACV,GAA+G;QAE/G,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC;QACrG,MAAM,IAAI,GAAG;YACX,kBAAkB;YAClB,4CAA4C;YAC5C,+BAA+B;YAC/B,6BAA6B;YAC7B,qBAAqB;YACrB,yBAAyB;YACzB,gCAAgC;SACjC,CAAC;QACF,IAAI,QAAQ;YAAE,IAAI,CAAC,IAAI,CAAC,4HAA4H,CAAC,CAAC;QACtJ,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,yBAAyB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC;aAClE,GAAG,CAAC;YACH,EAAE;YACF,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;YAC9B,YAAY,EAAE,GAAG,CAAC,YAAY,IAAI,IAAI;YACtC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;YACpC,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI;SACzB,CAAC,CAAC;IACP,CAAC;IAED,4EAA4E;IAC5E,QAAQ,CAAC,KAAiE;QACxE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAClD,IAAI,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE;gBACvB,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,YAAY;aACjD,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAE,EAAE,OAAO,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,CAAC,KAAK,GAAG,GAAG;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,mFAAmF,CAAC;aAC5F,GAAG,CAAC,KAAK,CAAU,CAAC;QACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,aAAa;QACX,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,8DAA8D,CAAC;aACvE,GAAG,EAAkD,CAAC;QACzD,MAAM,GAAG,GAAmC,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAClG,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,GAAQ;QACpB,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,cAAc,EAAE,GAAG,CAAC,eAAe;YACnC,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,OAAO,EAAE,GAAG,CAAC,QAAQ;YACrB,MAAM,EAAE,GAAG,CAAC,OAAO;YACnB,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,cAAc,EAAE,GAAG,CAAC,eAAe;YACnC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,YAAY,EAAE,GAAG,CAAC,aAAa;YAC/B,WAAW,EAAE,GAAG,CAAC,YAAY;YAC7B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,WAAW,EAAE,GAAG,CAAC,YAAY;SAC9B,CAAC;IACJ,CAAC;CACF;AAnJD,gDAmJC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { BaseRepository } from './BaseRepository';
|
|
2
|
+
export interface PixivMetadata {
|
|
3
|
+
pixivId: string;
|
|
4
|
+
workType: string;
|
|
5
|
+
language: string | null;
|
|
6
|
+
xRestrict: number | null;
|
|
7
|
+
publishedAt: string | null;
|
|
8
|
+
title: string | null;
|
|
9
|
+
checkedAt: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Lightweight SQLite metadata cache. Novel language filtering otherwise issues
|
|
13
|
+
* a full detail + full-text request for every candidate on every fallback run;
|
|
14
|
+
* caching the classified result bounds request volume and reduces 429 storms.
|
|
15
|
+
* Entries expire after a TTL so fresh classifications still happen eventually.
|
|
16
|
+
*/
|
|
17
|
+
export declare class MetadataRepository extends BaseRepository {
|
|
18
|
+
get(pixivId: string, workType: string): PixivMetadata | null;
|
|
19
|
+
fresh(pixivId: string, workType: string, ttlMs: number, now?: number): PixivMetadata | null;
|
|
20
|
+
put(pixivId: string, workType: string, data: {
|
|
21
|
+
language?: string | null;
|
|
22
|
+
xRestrict?: number | null;
|
|
23
|
+
publishedAt?: string | null;
|
|
24
|
+
title?: string | null;
|
|
25
|
+
}, now?: number): void;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=MetadataRepository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MetadataRepository.d.ts","sourceRoot":"","sources":["../../../src/storage/repositories/MetadataRepository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,qBAAa,kBAAmB,SAAQ,cAAc;IACpD,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAgB5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,aAAa,GAAG,IAAI;IAOvG,GAAG,CACD,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,EACjH,GAAG,GAAE,MAAmB,GACvB,IAAI;CAsBR"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MetadataRepository = void 0;
|
|
4
|
+
const BaseRepository_1 = require("./BaseRepository");
|
|
5
|
+
/**
|
|
6
|
+
* Lightweight SQLite metadata cache. Novel language filtering otherwise issues
|
|
7
|
+
* a full detail + full-text request for every candidate on every fallback run;
|
|
8
|
+
* caching the classified result bounds request volume and reduces 429 storms.
|
|
9
|
+
* Entries expire after a TTL so fresh classifications still happen eventually.
|
|
10
|
+
*/
|
|
11
|
+
class MetadataRepository extends BaseRepository_1.BaseRepository {
|
|
12
|
+
get(pixivId, workType) {
|
|
13
|
+
const row = this.db
|
|
14
|
+
.prepare(`SELECT * FROM pixiv_metadata WHERE pixiv_id = ? AND work_type = ?`)
|
|
15
|
+
.get(String(pixivId), workType);
|
|
16
|
+
if (!row)
|
|
17
|
+
return null;
|
|
18
|
+
return {
|
|
19
|
+
pixivId: row.pixiv_id,
|
|
20
|
+
workType: row.work_type,
|
|
21
|
+
language: row.language,
|
|
22
|
+
xRestrict: row.x_restrict,
|
|
23
|
+
publishedAt: row.published_at,
|
|
24
|
+
title: row.title,
|
|
25
|
+
checkedAt: row.checked_at,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
fresh(pixivId, workType, ttlMs, now = Date.now()) {
|
|
29
|
+
const row = this.get(pixivId, workType);
|
|
30
|
+
if (!row)
|
|
31
|
+
return null;
|
|
32
|
+
if (now - row.checkedAt > ttlMs)
|
|
33
|
+
return null;
|
|
34
|
+
return row;
|
|
35
|
+
}
|
|
36
|
+
put(pixivId, workType, data, now = Date.now()) {
|
|
37
|
+
this.db
|
|
38
|
+
.prepare(`INSERT INTO pixiv_metadata (pixiv_id, work_type, language, x_restrict, published_at, title, checked_at)
|
|
39
|
+
VALUES (@pixivId, @workType, @language, @xRestrict, @publishedAt, @title, @now)
|
|
40
|
+
ON CONFLICT(pixiv_id, work_type) DO UPDATE SET
|
|
41
|
+
language=excluded.language,
|
|
42
|
+
x_restrict=excluded.x_restrict,
|
|
43
|
+
published_at=excluded.published_at,
|
|
44
|
+
title=excluded.title,
|
|
45
|
+
checked_at=excluded.checked_at`)
|
|
46
|
+
.run({
|
|
47
|
+
pixivId: String(pixivId),
|
|
48
|
+
workType,
|
|
49
|
+
language: data.language ?? null,
|
|
50
|
+
xRestrict: data.xRestrict ?? null,
|
|
51
|
+
publishedAt: data.publishedAt ?? null,
|
|
52
|
+
title: data.title ?? null,
|
|
53
|
+
now,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.MetadataRepository = MetadataRepository;
|
|
58
|
+
//# sourceMappingURL=MetadataRepository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MetadataRepository.js","sourceRoot":"","sources":["../../../src/storage/repositories/MetadataRepository.ts"],"names":[],"mappings":";;;AAAA,qDAAkD;AAYlD;;;;;GAKG;AACH,MAAa,kBAAmB,SAAQ,+BAAc;IACpD,GAAG,CAAC,OAAe,EAAE,QAAgB;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,mEAAmE,CAAC;aAC5E,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAQ,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,QAAQ;YACrB,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,WAAW,EAAE,GAAG,CAAC,YAAY;YAC7B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,GAAG,CAAC,UAAU;SAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,QAAgB,EAAE,KAAa,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;QAC9E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,IAAI,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC;QAC7C,OAAO,GAAG,CAAC;IACb,CAAC;IAED,GAAG,CACD,OAAe,EACf,QAAgB,EAChB,IAAiH,EACjH,MAAc,IAAI,CAAC,GAAG,EAAE;QAExB,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;;;;;0CAOkC,CACnC;aACA,GAAG,CAAC;YACH,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;YACxB,QAAQ;YACR,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;YACjC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;YACzB,GAAG;SACJ,CAAC,CAAC;IACP,CAAC;CACF;AAnDD,gDAmDC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { BaseRepository } from './BaseRepository';
|
|
2
|
+
export type OutboxKind = 'delivery' | 'notification';
|
|
3
|
+
export type OutboxStatus = 'pending' | 'processing' | 'retry_wait' | 'done' | 'dead';
|
|
4
|
+
export interface OutboxRow {
|
|
5
|
+
id: string;
|
|
6
|
+
kind: OutboxKind;
|
|
7
|
+
idempotencyKey: string | null;
|
|
8
|
+
deliveryId: string | null;
|
|
9
|
+
deliveryTarget: string;
|
|
10
|
+
payloadJson: string;
|
|
11
|
+
status: OutboxStatus;
|
|
12
|
+
attempts: number;
|
|
13
|
+
maxAttempts: number;
|
|
14
|
+
nextAttemptAt: number;
|
|
15
|
+
leaseOwner: string | null;
|
|
16
|
+
leaseUntil: number | null;
|
|
17
|
+
lastError: string | null;
|
|
18
|
+
createdAt: number;
|
|
19
|
+
updatedAt: number;
|
|
20
|
+
completedAt: number | null;
|
|
21
|
+
}
|
|
22
|
+
export interface NewOutboxItem {
|
|
23
|
+
kind: OutboxKind;
|
|
24
|
+
deliveryTarget: string;
|
|
25
|
+
idempotencyKey?: string | null;
|
|
26
|
+
deliveryId?: string | null;
|
|
27
|
+
payload: unknown;
|
|
28
|
+
/** Initial due time (epoch ms). Default now. */
|
|
29
|
+
dueAt?: number;
|
|
30
|
+
maxAttempts?: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* SQLite transactional outbox. One durable row per intended external side
|
|
34
|
+
* effect (a content delivery or a notification). A single OutboxWorker claims
|
|
35
|
+
* due rows with a short lease, performs the effect, and marks them done. Rows
|
|
36
|
+
* survive process kill / machine stop; a restart simply resumes due rows.
|
|
37
|
+
*/
|
|
38
|
+
export declare class OutboxRepository extends BaseRepository {
|
|
39
|
+
static owner(): string;
|
|
40
|
+
/** Insert an intent. Returns the row. Idempotent on (kind, idempotency_key). */
|
|
41
|
+
enqueue(input: NewOutboxItem, now?: number): OutboxRow;
|
|
42
|
+
get(id: string): OutboxRow | null;
|
|
43
|
+
getByKey(kind: OutboxKind, key: string): OutboxRow | null;
|
|
44
|
+
/**
|
|
45
|
+
* Claim up to `limit` due rows for `owner`. Due = pending/retry_wait whose
|
|
46
|
+
* next_attempt_at has passed, OR processing rows whose lease expired (a
|
|
47
|
+
* crashed prior worker). The claim is a compare-and-set inside one
|
|
48
|
+
* statement, so two workers/processes never grab the same row.
|
|
49
|
+
*/
|
|
50
|
+
claimDue(owner: string, leaseMs: number, limit: number, now?: number): OutboxRow[];
|
|
51
|
+
markDone(id: string, now?: number): void;
|
|
52
|
+
/** Schedule a later retry, or flip to dead once attempts are exhausted. */
|
|
53
|
+
markRetry(id: string, nextAt: number, error: string, now?: number): OutboxStatus;
|
|
54
|
+
/** Release a processing row back to pending without counting a retry. */
|
|
55
|
+
release(id: string, now?: number): void;
|
|
56
|
+
/** Reopen a dead row for one more attempt (operator / reconcile action). */
|
|
57
|
+
requeue(id: string, dueAt?: number): void;
|
|
58
|
+
counts(now?: number): {
|
|
59
|
+
pending: number;
|
|
60
|
+
retryWait: number;
|
|
61
|
+
processing: number;
|
|
62
|
+
dead: number;
|
|
63
|
+
done24h: number;
|
|
64
|
+
oldestPendingMs: number | null;
|
|
65
|
+
};
|
|
66
|
+
/** Rows still non-terminal (used by reconcile/doctor). */
|
|
67
|
+
staleProcessing(now?: number): OutboxRow[];
|
|
68
|
+
private toRow;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=OutboxRepository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"OutboxRepository.d.ts","sourceRoot":"","sources":["../../../src/storage/repositories/OutboxRepository.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,cAAc,CAAC;AACrD,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,YAAY,GAAG,YAAY,GAAG,MAAM,GAAG,MAAM,CAAC;AAErF,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,cAAc;IAClD,MAAM,CAAC,KAAK,IAAI,MAAM;IAItB,gFAAgF;IAChF,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,GAAE,MAAmB,GAAG,SAAS;IAgClE,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAKjC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAOzD;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,SAAS,EAAE;IAiC9F,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,IAAI;IAWpD,2EAA2E;IAC3E,SAAS,CACP,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,GAAG,GAAE,MAAmB,GACvB,YAAY;IA0Bf,yEAAyE;IACzE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,IAAI;IASnD,4EAA4E;IAC5E,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,GAAE,MAAmB,GAAG,IAAI;IAUrD,MAAM,CAAC,GAAG,GAAE,MAAmB,GAAG;QAChC,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC;IAqBD,0DAA0D;IAC1D,eAAe,CAAC,GAAG,GAAE,MAAmB,GAAG,SAAS,EAAE;IAOtD,OAAO,CAAC,KAAK;CAoBd"}
|