scenri 0.9.3 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/{chunk-Y5L3KLX5.js → chunk-3ZRMKVBB.js} +13 -6
- package/dist/{cli-SX46UQO6.js → cli-MQO66C5O.js} +3 -3
- package/dist/index.js +2 -2
- package/dist/serve.js +188 -56
- package/dist/{src-F3KJZNZT.js → src-3ITTV2UE.js} +3 -3
- package/package.json +1 -1
- package/studio-dist/assets/{index-eKmAemHO.js → index-uiX1mW43.js} +1 -1
- package/studio-dist/index.html +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.9.4](https://github.com/tonygorb/Scenri/compare/v0.9.3...v0.9.4) (2026-09-14)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* a rate-limited store is not an empty one ([1eef6eb](https://github.com/tonygorb/Scenri/commit/1eef6eb7471e1d2de97ccedcafba7962a5d393ad))
|
|
9
|
+
* **catalog:** a rate-limited store is not an empty one ([2e7f45a](https://github.com/tonygorb/Scenri/commit/2e7f45a07adcb202722fd07972d4e5d63bbda944))
|
|
10
|
+
* **catalog:** keep an AVIF picture, and say why a page was refused ([7e08137](https://github.com/tonygorb/Scenri/commit/7e08137b543d6b491e91cc6c7f4de9aa67be4b9e))
|
|
11
|
+
* **catalog:** keep the enum out of the error list, and cut what nothing produces ([1201832](https://github.com/tonygorb/Scenri/commit/1201832ac7df892506647224e790dbbaea080092))
|
|
12
|
+
* **catalog:** stop waiting out a store that has made up its mind ([8f6751a](https://github.com/tonygorb/Scenri/commit/8f6751a6462b6e44b06b08a6b2dc2aa12ad456d1))
|
|
13
|
+
|
|
3
14
|
## [0.9.3](https://github.com/tonygorb/Scenri/compare/v0.9.2...v0.9.3) (2026-09-14)
|
|
4
15
|
|
|
5
16
|
|
|
@@ -551,7 +551,7 @@ function openDb(homeDir) {
|
|
|
551
551
|
function createImageStore(homeDir) {
|
|
552
552
|
const dir = join(homeDir, "images");
|
|
553
553
|
mkdirSync(dir, { recursive: true, mode: 448 });
|
|
554
|
-
const EXTS = ["png", "jpg", "jpeg", "webp", "gif", "avif"];
|
|
554
|
+
const EXTS = ["png", "jpg", "jpeg", "webp", "gif", "avif", "heif", "heic"];
|
|
555
555
|
const fileFor = (hash, ext = "png") => join(dir, `${hash}.${ext}`);
|
|
556
556
|
const resolve = (hash) => {
|
|
557
557
|
for (const ext of EXTS) {
|
|
@@ -1737,11 +1737,18 @@ function productImportMethods(db) {
|
|
|
1737
1737
|
).run(sourceId);
|
|
1738
1738
|
return r2.changes;
|
|
1739
1739
|
}
|
|
1740
|
-
|
|
1740
|
+
db.exec("CREATE TEMP TABLE IF NOT EXISTS seen_keys (k TEXT PRIMARY KEY)");
|
|
1741
|
+
db.exec("DELETE FROM seen_keys");
|
|
1742
|
+
const add = db.prepare("INSERT OR IGNORE INTO seen_keys (k) VALUES (?)");
|
|
1743
|
+
db.transaction((keys) => {
|
|
1744
|
+
for (const k of keys) add.run(k);
|
|
1745
|
+
})(seenExternalKeys);
|
|
1741
1746
|
const r = db.prepare(
|
|
1742
1747
|
`UPDATE catalog_products SET status='unavailable', updated_at=datetime('now')
|
|
1743
|
-
WHERE source_id=? AND status='active'
|
|
1744
|
-
|
|
1748
|
+
WHERE source_id=? AND status='active'
|
|
1749
|
+
AND external_key NOT IN (SELECT k FROM seen_keys)`
|
|
1750
|
+
).run(sourceId);
|
|
1751
|
+
db.exec("DELETE FROM seen_keys");
|
|
1745
1752
|
return r.changes;
|
|
1746
1753
|
}
|
|
1747
1754
|
};
|
|
@@ -2025,5 +2032,5 @@ function createCore(homeDir = defaultHome()) {
|
|
|
2025
2032
|
}
|
|
2026
2033
|
|
|
2027
2034
|
export { ASPECT_TOLERANCE, BUDGET_EXHAUSTED, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, SCHEMA_VERSION, STEM_MIN, SchemaTooNewError, SpendCapError, TRIGRAM_MIN, budgetSize, createCatalogStore, createCore, createStore, defaultHome, fold, ftsMatch, matchesQuery, ratioLabel, searchTerms, termMatches, uniqueProjectSlug, uniqueSetSlug, uniqueSlug };
|
|
2028
|
-
//# sourceMappingURL=chunk-
|
|
2029
|
-
//# sourceMappingURL=chunk-
|
|
2035
|
+
//# sourceMappingURL=chunk-3ZRMKVBB.js.map
|
|
2036
|
+
//# sourceMappingURL=chunk-3ZRMKVBB.js.map
|
|
@@ -463,7 +463,7 @@ async function addOrRepair(ownEntry, say, over) {
|
|
|
463
463
|
}
|
|
464
464
|
async function rememberDecline(home) {
|
|
465
465
|
try {
|
|
466
|
-
const { createCore } = await import('./src-
|
|
466
|
+
const { createCore } = await import('./src-3ITTV2UE.js');
|
|
467
467
|
const core = createCore(home);
|
|
468
468
|
try {
|
|
469
469
|
core.store.setSetting("desktop.prompt", "declined");
|
|
@@ -530,5 +530,5 @@ function tail(path, bytes = 4096) {
|
|
|
530
530
|
}
|
|
531
531
|
|
|
532
532
|
export { addToDesktop, assetsDirFor, installDeps, runDesktopCommand, runExecFile, runOpenCommand };
|
|
533
|
-
//# sourceMappingURL=cli-
|
|
534
|
-
//# sourceMappingURL=cli-
|
|
533
|
+
//# sourceMappingURL=cli-MQO66C5O.js.map
|
|
534
|
+
//# sourceMappingURL=cli-MQO66C5O.js.map
|
package/dist/index.js
CHANGED
|
@@ -86,12 +86,12 @@ try {
|
|
|
86
86
|
break;
|
|
87
87
|
}
|
|
88
88
|
case "desktop": {
|
|
89
|
-
const { runDesktopCommand } = await import('./cli-
|
|
89
|
+
const { runDesktopCommand } = await import('./cli-MQO66C5O.js');
|
|
90
90
|
process.exit(await runDesktopCommand(command, fileURLToPath(import.meta.url)));
|
|
91
91
|
break;
|
|
92
92
|
}
|
|
93
93
|
case "open": {
|
|
94
|
-
const { runOpenCommand } = await import('./cli-
|
|
94
|
+
const { runOpenCommand } = await import('./cli-MQO66C5O.js');
|
|
95
95
|
process.exit(await runOpenCommand(fileURLToPath(import.meta.url)));
|
|
96
96
|
break;
|
|
97
97
|
}
|
package/dist/serve.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createUpdateChecker, classify, isReleaseTriplet, findNpm, stageVersion } from './chunk-FYQ5BAFA.js';
|
|
2
2
|
import { shouldAdoptRunning, anotherScenriLines, portBusyLines } from './chunk-CMLJGPYC.js';
|
|
3
|
-
import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-
|
|
3
|
+
import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-3ZRMKVBB.js';
|
|
4
4
|
import { detectInstallKind } from './chunk-PAIAXRAC.js';
|
|
5
5
|
export { detectInstallKind } from './chunk-PAIAXRAC.js';
|
|
6
6
|
import { readMeta, repoSlug } from './chunk-Y3ZPBPLP.js';
|
|
@@ -4887,6 +4887,57 @@ async function inspectMark(buf, toPng2) {
|
|
|
4887
4887
|
};
|
|
4888
4888
|
}
|
|
4889
4889
|
|
|
4890
|
+
// ../catalog/src/failures.ts
|
|
4891
|
+
function pageFailure(status) {
|
|
4892
|
+
if (status === 429 || status === 503) return "RATE_LIMITED";
|
|
4893
|
+
if (status === 401 || status === 403 || status === 405) return "PAGE_BLOCKED";
|
|
4894
|
+
if (status === 404 || status === 410) return "PAGE_NOT_FOUND";
|
|
4895
|
+
return "PAGE_UNREADABLE";
|
|
4896
|
+
}
|
|
4897
|
+
function imageFailure(status) {
|
|
4898
|
+
if (status === 429 || status === 503) return "RATE_LIMITED";
|
|
4899
|
+
if (status === 401 || status === 403) return "IMAGE_FORBIDDEN";
|
|
4900
|
+
if (status === 404 || status === 410) return "IMAGE_NOT_FOUND";
|
|
4901
|
+
return "DOWNLOAD_FAILED";
|
|
4902
|
+
}
|
|
4903
|
+
function thrownFailure(err, aborted = false) {
|
|
4904
|
+
if (aborted) return "ABORTED";
|
|
4905
|
+
const m = String(err?.message ?? err).toLowerCase();
|
|
4906
|
+
if (m.includes("abort") || m.includes("timed out") || m.includes("timeout")) return "IMAGE_TIMEOUT";
|
|
4907
|
+
return "DOWNLOAD_FAILED";
|
|
4908
|
+
}
|
|
4909
|
+
function tally(into, reason, n = 1) {
|
|
4910
|
+
into[reason] = (into[reason] ?? 0) + n;
|
|
4911
|
+
}
|
|
4912
|
+
function leadingReason(t) {
|
|
4913
|
+
const rows = Object.entries(t);
|
|
4914
|
+
if (!rows.length) return null;
|
|
4915
|
+
return rows.sort((a, b) => b[1] - a[1])[0][0];
|
|
4916
|
+
}
|
|
4917
|
+
function summarise(t, saved, asked) {
|
|
4918
|
+
const reason = leadingReason(t);
|
|
4919
|
+
if (!reason) return null;
|
|
4920
|
+
const none = saved === 0;
|
|
4921
|
+
switch (reason) {
|
|
4922
|
+
case "RATE_LIMITED":
|
|
4923
|
+
return none ? "The store asked us to slow down, so nothing could be read. Waiting a few minutes and trying again usually works." : `The store asked us to slow down partway, so ${asked - saved} of ${asked} products were skipped. Trying again picks up the rest.`;
|
|
4924
|
+
case "PAGE_BLOCKED":
|
|
4925
|
+
return none ? "This store would not let us read its product pages." : `${asked - saved} of ${asked} product pages would not open.`;
|
|
4926
|
+
case "PAGE_NOT_FOUND":
|
|
4927
|
+
return `${asked - saved} of ${asked} products are no longer on the site.`;
|
|
4928
|
+
case "IMAGE_FORBIDDEN":
|
|
4929
|
+
case "IMAGE_NOT_FOUND":
|
|
4930
|
+
case "IMAGE_TIMEOUT":
|
|
4931
|
+
case "IMAGE_EMPTY":
|
|
4932
|
+
case "DOWNLOAD_FAILED":
|
|
4933
|
+
return none ? "We found the products, but none of their pictures could be downloaded." : "Some pictures could not be downloaded. The products they belong to are still here.";
|
|
4934
|
+
case "ABORTED":
|
|
4935
|
+
return null;
|
|
4936
|
+
default:
|
|
4937
|
+
return none ? "The products on this site could not be read." : null;
|
|
4938
|
+
}
|
|
4939
|
+
}
|
|
4940
|
+
|
|
4890
4941
|
// ../catalog/src/url.ts
|
|
4891
4942
|
function normalizeStoreUrl(input) {
|
|
4892
4943
|
let raw = input.trim();
|
|
@@ -5105,12 +5156,49 @@ async function readBounded2(res, maxBytes) {
|
|
|
5105
5156
|
function sleep4(ms) {
|
|
5106
5157
|
return new Promise((r) => setTimeout(r, ms));
|
|
5107
5158
|
}
|
|
5159
|
+
var cooledUntil = /* @__PURE__ */ new Map();
|
|
5160
|
+
var hostOf = (url) => {
|
|
5161
|
+
try {
|
|
5162
|
+
return new URL(url).host;
|
|
5163
|
+
} catch {
|
|
5164
|
+
return "";
|
|
5165
|
+
}
|
|
5166
|
+
};
|
|
5167
|
+
async function waitForHost(host, signal) {
|
|
5168
|
+
if (!host) return;
|
|
5169
|
+
for (; ; ) {
|
|
5170
|
+
const left = (cooledUntil.get(host) ?? 0) - Date.now();
|
|
5171
|
+
if (left <= 0 || signal?.aborted) return;
|
|
5172
|
+
await sleep4(Math.min(left, 250));
|
|
5173
|
+
}
|
|
5174
|
+
}
|
|
5175
|
+
function coolHost(host, ms) {
|
|
5176
|
+
if (!host || ms <= 0) return;
|
|
5177
|
+
const until = Date.now() + ms;
|
|
5178
|
+
if (until > (cooledUntil.get(host) ?? 0)) cooledUntil.set(host, until);
|
|
5179
|
+
}
|
|
5180
|
+
var RETRY_AFTER_CAP_MS = 6e4;
|
|
5181
|
+
function retryAfterMs(header, now = Date.now()) {
|
|
5182
|
+
if (!header) return null;
|
|
5183
|
+
const secs = Number(header.trim());
|
|
5184
|
+
if (Number.isFinite(secs) && secs >= 0) return Math.min(secs * 1e3, RETRY_AFTER_CAP_MS);
|
|
5185
|
+
const at = Date.parse(header);
|
|
5186
|
+
if (Number.isNaN(at)) return null;
|
|
5187
|
+
return Math.min(Math.max(0, at - now), RETRY_AFTER_CAP_MS);
|
|
5188
|
+
}
|
|
5189
|
+
function throttleBackoff(attempt) {
|
|
5190
|
+
const base = Math.min(1e3 * 2 ** attempt, 16e3);
|
|
5191
|
+
return base + Math.floor(Math.random() * 400);
|
|
5192
|
+
}
|
|
5108
5193
|
async function httpGet(url, opts = {}) {
|
|
5109
5194
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
5110
5195
|
const retries = opts.retries ?? 3;
|
|
5111
5196
|
const timeoutMs = opts.timeoutMs ?? 25e3;
|
|
5112
5197
|
let lastErr;
|
|
5198
|
+
const host = hostOf(url);
|
|
5113
5199
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
5200
|
+
await waitForHost(host, opts.signal);
|
|
5201
|
+
if (opts.signal?.aborted) throw new Error("aborted");
|
|
5114
5202
|
const ctrl = new AbortController();
|
|
5115
5203
|
const onAbort = () => ctrl.abort();
|
|
5116
5204
|
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
|
@@ -5142,8 +5230,19 @@ async function httpGet(url, opts = {}) {
|
|
|
5142
5230
|
} catch {
|
|
5143
5231
|
}
|
|
5144
5232
|
}
|
|
5145
|
-
if (
|
|
5146
|
-
|
|
5233
|
+
if (res.status === 429 || res.status === 503) {
|
|
5234
|
+
const asked = retryAfterMs(res.headers.get("retry-after"));
|
|
5235
|
+
const wait = asked ?? throttleBackoff(attempt);
|
|
5236
|
+
coolHost(host, wait);
|
|
5237
|
+
if (attempt < retries) {
|
|
5238
|
+
await res.body?.cancel().catch(() => {
|
|
5239
|
+
});
|
|
5240
|
+
continue;
|
|
5241
|
+
}
|
|
5242
|
+
return res;
|
|
5243
|
+
}
|
|
5244
|
+
if (res.status >= 500 && attempt < retries) {
|
|
5245
|
+
await sleep4(throttleBackoff(attempt));
|
|
5147
5246
|
continue;
|
|
5148
5247
|
}
|
|
5149
5248
|
return res;
|
|
@@ -5417,6 +5516,9 @@ async function fetchProductPages(ctx, urls, opts = {}) {
|
|
|
5417
5516
|
const out = [];
|
|
5418
5517
|
const seen = /* @__PURE__ */ new Set();
|
|
5419
5518
|
let kept = 0;
|
|
5519
|
+
let refusedRun = 0;
|
|
5520
|
+
const giveUpAt = opts.giveUpAfterRefusals ?? Number.POSITIVE_INFINITY;
|
|
5521
|
+
let givenUp = false;
|
|
5420
5522
|
const want = opts.want ?? Number.POSITIVE_INFINITY;
|
|
5421
5523
|
const ceiling = opts.maxPages ?? (Number.isFinite(want) ? want * 3 : urls.length);
|
|
5422
5524
|
const take = urls.slice(0, ceiling);
|
|
@@ -5425,12 +5527,12 @@ async function fetchProductPages(ctx, urls, opts = {}) {
|
|
|
5425
5527
|
take,
|
|
5426
5528
|
opts.delayMs ? 1 : opts.concurrency ?? 5,
|
|
5427
5529
|
async (u) => {
|
|
5428
|
-
if (kept >= want) return;
|
|
5530
|
+
if (kept >= want || givenUp) return;
|
|
5429
5531
|
if (opts.deadline != null && Date.now() > opts.deadline) return;
|
|
5430
5532
|
if (opts.maxTotalBytes != null && bytes >= opts.maxTotalBytes) return;
|
|
5431
5533
|
try {
|
|
5432
5534
|
if (opts.delayMs) await sleep5(opts.delayMs);
|
|
5433
|
-
const { ok, text, url } = await httpText(u, {
|
|
5535
|
+
const { ok, status, text, url } = await httpText(u, {
|
|
5434
5536
|
fetchImpl: ctx.fetchImpl,
|
|
5435
5537
|
signal: ctx.signal,
|
|
5436
5538
|
accept: "text/html",
|
|
@@ -5442,9 +5544,14 @@ async function fetchProductPages(ctx, urls, opts = {}) {
|
|
|
5442
5544
|
opts.stats.bytes = bytes;
|
|
5443
5545
|
}
|
|
5444
5546
|
if (!ok) {
|
|
5445
|
-
if (opts.stats)
|
|
5547
|
+
if (opts.stats) {
|
|
5548
|
+
opts.stats.refused = (opts.stats.refused ?? 0) + 1;
|
|
5549
|
+
if (opts.stats.reasons) tally(opts.stats.reasons, pageFailure(status));
|
|
5550
|
+
}
|
|
5551
|
+
if (++refusedRun >= giveUpAt) givenUp = true;
|
|
5446
5552
|
return;
|
|
5447
5553
|
}
|
|
5554
|
+
refusedRun = 0;
|
|
5448
5555
|
for (const p of productsFromPage(text, url)) {
|
|
5449
5556
|
if (seen.has(p.externalKey)) continue;
|
|
5450
5557
|
seen.add(p.externalKey);
|
|
@@ -5453,8 +5560,11 @@ async function fetchProductPages(ctx, urls, opts = {}) {
|
|
|
5453
5560
|
else out.push(p);
|
|
5454
5561
|
}
|
|
5455
5562
|
opts.onProduct?.(kept);
|
|
5456
|
-
} catch {
|
|
5457
|
-
if (opts.stats)
|
|
5563
|
+
} catch (err) {
|
|
5564
|
+
if (opts.stats) {
|
|
5565
|
+
opts.stats.refused = (opts.stats.refused ?? 0) + 1;
|
|
5566
|
+
if (opts.stats.reasons) tally(opts.stats.reasons, thrownFailure(err, ctx.signal?.aborted));
|
|
5567
|
+
}
|
|
5458
5568
|
}
|
|
5459
5569
|
},
|
|
5460
5570
|
ctx.signal
|
|
@@ -6442,7 +6552,7 @@ async function runJob(deps, jobId, brandId, url, signal, only) {
|
|
|
6442
6552
|
const { core, fetchImpl } = deps;
|
|
6443
6553
|
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6444
6554
|
try {
|
|
6445
|
-
const
|
|
6555
|
+
const tally2 = { fetched: 0, upserted: 0, imagesDone: 0, imagesTotal: 0, errors: [], seenKeys: [] };
|
|
6446
6556
|
const ctx = { fetchImpl: fetchImpl ?? fetch, baseUrl: url, signal };
|
|
6447
6557
|
let urls = [];
|
|
6448
6558
|
let bulk = null;
|
|
@@ -6506,10 +6616,11 @@ async function runJob(deps, jobId, brandId, url, signal, only) {
|
|
|
6506
6616
|
patch2({ stage: "fetching_products", discovered, message: `Reading ${discovered.toLocaleString()} products` });
|
|
6507
6617
|
}
|
|
6508
6618
|
}
|
|
6509
|
-
const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform,
|
|
6510
|
-
const pictures = drainPictures(deps, jobId, brandId,
|
|
6619
|
+
const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered);
|
|
6620
|
+
const pictures = drainPictures(deps, jobId, brandId, tally2, signal);
|
|
6511
6621
|
let pictureErrors;
|
|
6512
|
-
|
|
6622
|
+
let pictureReasons = {};
|
|
6623
|
+
const stats = { pages: 0, bytes: 0, refused: 0, reasons: {} };
|
|
6513
6624
|
try {
|
|
6514
6625
|
if (bulk) for (const p of bulk) run2.write(p);
|
|
6515
6626
|
else
|
|
@@ -6517,46 +6628,43 @@ async function runJob(deps, jobId, brandId, url, signal, only) {
|
|
|
6517
6628
|
concurrency: IMPORT_CONCURRENCY,
|
|
6518
6629
|
maxBytes: 15e5,
|
|
6519
6630
|
onEach: run2.write,
|
|
6520
|
-
stats
|
|
6631
|
+
stats,
|
|
6632
|
+
// Past this many refusals in a row the store is not going to change
|
|
6633
|
+
// its mind inside this import, and waiting out its cooldown for the
|
|
6634
|
+
// rest of the catalogue is time nobody gets a product for.
|
|
6635
|
+
giveUpAfterRefusals: 24
|
|
6521
6636
|
});
|
|
6522
|
-
if (!signal.aborted &&
|
|
6637
|
+
if (!signal.aborted && tally2.upserted) {
|
|
6523
6638
|
patch2({
|
|
6524
6639
|
stage: "processing_assets",
|
|
6525
|
-
message: `Downloading ${
|
|
6526
|
-
upserted:
|
|
6527
|
-
fetched:
|
|
6640
|
+
message: `Downloading ${tally2.imagesTotal.toLocaleString()} pictures`,
|
|
6641
|
+
upserted: tally2.upserted,
|
|
6642
|
+
fetched: tally2.fetched
|
|
6528
6643
|
});
|
|
6529
6644
|
}
|
|
6530
6645
|
} finally {
|
|
6531
6646
|
pictures.stop();
|
|
6532
|
-
|
|
6647
|
+
const settled = await pictures.done;
|
|
6648
|
+
pictureErrors = settled.errors;
|
|
6649
|
+
pictureReasons = settled.reasons;
|
|
6533
6650
|
}
|
|
6534
6651
|
if (signal.aborted) {
|
|
6535
6652
|
patch2({
|
|
6536
6653
|
stage: "cancelled",
|
|
6537
6654
|
errors: [],
|
|
6538
|
-
message:
|
|
6655
|
+
message: tally2.upserted ? `Stopped after saving ${tally2.upserted.toLocaleString()} products` : "Stopped before anything was saved",
|
|
6539
6656
|
finished: true
|
|
6540
6657
|
});
|
|
6541
6658
|
core.catalog.setSourceStatus(run2.sourceId, "partial", true);
|
|
6542
6659
|
return;
|
|
6543
6660
|
}
|
|
6544
|
-
if (!tally.upserted) {
|
|
6545
|
-
const message = only?.length ? "None of the chosen products could be read" : `Found pages on this ${platform} store but could not read a product from any of them. The store may be blocking automated readers.`;
|
|
6546
|
-
patch2({
|
|
6547
|
-
stage: "failed",
|
|
6548
|
-
message,
|
|
6549
|
-
errors: [{ code: "no_products_fetched", message }],
|
|
6550
|
-
finished: true
|
|
6551
|
-
});
|
|
6552
|
-
core.catalog.setSourceStatus(run2.sourceId, "failed", true);
|
|
6553
|
-
return;
|
|
6554
|
-
}
|
|
6555
6661
|
run2.finish({
|
|
6556
6662
|
sweep,
|
|
6557
6663
|
warnings,
|
|
6558
6664
|
errors: pictureErrors,
|
|
6559
6665
|
refused: stats.refused,
|
|
6666
|
+
reasons: { ...stats.reasons, ...pictureReasons },
|
|
6667
|
+
asked: bulk ? bulk.length : urls.length,
|
|
6560
6668
|
// Every address was read. A bulk API hands the catalogue over whole, so
|
|
6561
6669
|
// there is nothing to cover.
|
|
6562
6670
|
covered: bulk ? true : stats.pages >= urls.length
|
|
@@ -6578,7 +6686,7 @@ var IMAGES_PER_PRODUCT = 3;
|
|
|
6578
6686
|
var TERMINAL = /* @__PURE__ */ new Set(["completed", "partial", "no_catalog", "cancelled", "failed"]);
|
|
6579
6687
|
var IMAGE_CONCURRENCY = 4;
|
|
6580
6688
|
var sleep6 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
6581
|
-
function beginWrite(deps, jobId, brandId, baseUrl, platform,
|
|
6689
|
+
function beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered) {
|
|
6582
6690
|
const { core } = deps;
|
|
6583
6691
|
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6584
6692
|
const source = core.catalog.upsertSource(brandId, baseUrl, platform);
|
|
@@ -6616,44 +6724,50 @@ function beginWrite(deps, jobId, brandId, baseUrl, platform, tally, discovered)
|
|
|
6616
6724
|
})),
|
|
6617
6725
|
collections: (p.collections ?? []).map((c) => ({ externalKey: c, title: c }))
|
|
6618
6726
|
});
|
|
6619
|
-
|
|
6620
|
-
|
|
6621
|
-
|
|
6622
|
-
patch2({ upserted:
|
|
6727
|
+
tally2.seenKeys.push(p.externalKey);
|
|
6728
|
+
tally2.upserted++;
|
|
6729
|
+
tally2.fetched++;
|
|
6730
|
+
patch2({ upserted: tally2.upserted, fetched: tally2.fetched });
|
|
6623
6731
|
},
|
|
6624
6732
|
finish({
|
|
6625
6733
|
sweep,
|
|
6626
6734
|
warnings,
|
|
6627
6735
|
errors,
|
|
6628
6736
|
refused = 0,
|
|
6629
|
-
covered = true
|
|
6737
|
+
covered = true,
|
|
6738
|
+
reasons = {},
|
|
6739
|
+
asked = 0
|
|
6630
6740
|
}) {
|
|
6631
|
-
|
|
6632
|
-
|
|
6741
|
+
const mayRetire = sweep && covered && refused === 0 && tally2.upserted > 0;
|
|
6742
|
+
if (mayRetire) core.catalog.markMissingUnavailable(source.id, tally2.seenKeys);
|
|
6743
|
+
tally2.errors = errors;
|
|
6633
6744
|
const partial = !covered || refused > 0;
|
|
6634
|
-
const
|
|
6635
|
-
const
|
|
6745
|
+
const savedNothing = tally2.upserted === 0;
|
|
6746
|
+
const said = summarise(reasons, tally2.upserted, asked || discovered || tally2.fetched);
|
|
6747
|
+
const stage = savedNothing ? "failed" : partial ? "partial" : "completed";
|
|
6748
|
+
const message = savedNothing ? said ?? "We found the products on this site, but none of them could be imported." : said ?? (partial ? `Imported ${tally2.upserted.toLocaleString()} products with ${(errors.length + refused).toLocaleString()} issue${errors.length + refused === 1 ? "" : "s"}` : `Imported ${tally2.upserted.toLocaleString()} products`);
|
|
6636
6749
|
patch2({
|
|
6637
|
-
stage
|
|
6638
|
-
upserted:
|
|
6639
|
-
fetched:
|
|
6640
|
-
imagesDone:
|
|
6641
|
-
imagesTotal:
|
|
6642
|
-
errors
|
|
6750
|
+
stage,
|
|
6751
|
+
upserted: tally2.upserted,
|
|
6752
|
+
fetched: tally2.fetched,
|
|
6753
|
+
imagesDone: tally2.imagesDone,
|
|
6754
|
+
imagesTotal: tally2.imagesTotal,
|
|
6755
|
+
errors,
|
|
6643
6756
|
warnings,
|
|
6644
6757
|
message,
|
|
6645
6758
|
finished: true
|
|
6646
6759
|
});
|
|
6647
|
-
core.catalog.setSourceStatus(source.id, partial ? "partial" : "ready", true);
|
|
6760
|
+
core.catalog.setSourceStatus(source.id, savedNothing ? "failed" : partial ? "partial" : "ready", true);
|
|
6648
6761
|
}
|
|
6649
6762
|
};
|
|
6650
6763
|
}
|
|
6651
6764
|
var IMPORT_CONCURRENCY = 4;
|
|
6652
6765
|
var PICTURE_ROUND = 60;
|
|
6653
|
-
function drainPictures(deps, jobId, brandId,
|
|
6766
|
+
function drainPictures(deps, jobId, brandId, tally2, signal, imagesPerProduct = IMAGES_PER_PRODUCT) {
|
|
6654
6767
|
const { core, fetchImpl } = deps;
|
|
6655
6768
|
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6656
6769
|
const errors = [...core.catalog.getJob(jobId)?.errors ?? []];
|
|
6770
|
+
const reasons = {};
|
|
6657
6771
|
let stopped = false;
|
|
6658
6772
|
const done = (async () => {
|
|
6659
6773
|
while (!signal.aborted) {
|
|
@@ -6663,7 +6777,7 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
|
|
|
6663
6777
|
await sleep6(150);
|
|
6664
6778
|
continue;
|
|
6665
6779
|
}
|
|
6666
|
-
|
|
6780
|
+
tally2.imagesTotal += round.length;
|
|
6667
6781
|
await mapPool(
|
|
6668
6782
|
round,
|
|
6669
6783
|
IMAGE_CONCURRENCY,
|
|
@@ -6673,11 +6787,13 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
|
|
|
6673
6787
|
const res = await httpGet(img.sourceUrl, { fetchImpl, signal, timeoutMs: 4e4, retries: 2 });
|
|
6674
6788
|
if (!res.ok) {
|
|
6675
6789
|
errors.push({ code: "image_http", message: `HTTP ${res.status}`, url: img.sourceUrl });
|
|
6790
|
+
tally(reasons, imageFailure(res.status));
|
|
6676
6791
|
return;
|
|
6677
6792
|
}
|
|
6678
6793
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
6679
6794
|
if (!buf.length) {
|
|
6680
6795
|
errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
|
|
6796
|
+
tally(reasons, "IMAGE_EMPTY");
|
|
6681
6797
|
return;
|
|
6682
6798
|
}
|
|
6683
6799
|
const probe = await sharp21(buf).metadata();
|
|
@@ -6691,6 +6807,7 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
|
|
|
6691
6807
|
});
|
|
6692
6808
|
} catch (err) {
|
|
6693
6809
|
if (signal.aborted) return;
|
|
6810
|
+
tally(reasons, thrownFailure(err, signal.aborted));
|
|
6694
6811
|
errors.push({
|
|
6695
6812
|
code: "image_failed",
|
|
6696
6813
|
message: String(err?.message ?? err),
|
|
@@ -6698,8 +6815,8 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
|
|
|
6698
6815
|
retryable: true
|
|
6699
6816
|
});
|
|
6700
6817
|
} finally {
|
|
6701
|
-
|
|
6702
|
-
patch2({ imagesDone:
|
|
6818
|
+
tally2.imagesDone++;
|
|
6819
|
+
patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
|
|
6703
6820
|
}
|
|
6704
6821
|
},
|
|
6705
6822
|
signal
|
|
@@ -6710,8 +6827,8 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
|
|
|
6710
6827
|
break;
|
|
6711
6828
|
}
|
|
6712
6829
|
}
|
|
6713
|
-
patch2({ imagesDone:
|
|
6714
|
-
return errors;
|
|
6830
|
+
patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
|
|
6831
|
+
return { errors, reasons };
|
|
6715
6832
|
})();
|
|
6716
6833
|
return {
|
|
6717
6834
|
stop() {
|
|
@@ -9734,6 +9851,21 @@ function registerImageRoutes(app, deps) {
|
|
|
9734
9851
|
|
|
9735
9852
|
// src/release/notes.data.ts
|
|
9736
9853
|
var RELEASES = [
|
|
9854
|
+
{
|
|
9855
|
+
version: "0.9.4",
|
|
9856
|
+
date: "2026-09-14",
|
|
9857
|
+
title: "A store that asks Scenri to slow down no longer looks like an empty one.",
|
|
9858
|
+
sections: [
|
|
9859
|
+
{
|
|
9860
|
+
heading: "Products",
|
|
9861
|
+
body: "Some stores limit how fast anything can read them. Scenri now waits the way such a store asks and says what happened: an import that could not read anything tells you to try again in a few minutes, and one that got part way says how much is left. Products from an earlier import are never retired by a run that was refused, and a store that keeps saying no ends the import early rather than grinding on."
|
|
9862
|
+
},
|
|
9863
|
+
{
|
|
9864
|
+
heading: "Fixes",
|
|
9865
|
+
body: "A product picture saved in a newer format such as AVIF is found again instead of going missing."
|
|
9866
|
+
}
|
|
9867
|
+
]
|
|
9868
|
+
},
|
|
9737
9869
|
{
|
|
9738
9870
|
version: "0.9.3",
|
|
9739
9871
|
date: "2026-09-14",
|
|
@@ -10727,7 +10859,7 @@ function registerDesktopRoutes(app, deps) {
|
|
|
10727
10859
|
record: null
|
|
10728
10860
|
};
|
|
10729
10861
|
}
|
|
10730
|
-
const { installDeps } = await import('./cli-
|
|
10862
|
+
const { installDeps } = await import('./cli-MQO66C5O.js');
|
|
10731
10863
|
const { desktopStatus } = await import('./install-VXJPLYWM.js');
|
|
10732
10864
|
return desktopStatus(installDeps(runtime.entry));
|
|
10733
10865
|
});
|
|
@@ -10735,7 +10867,7 @@ function registerDesktopRoutes(app, deps) {
|
|
|
10735
10867
|
if (!runtime.entry) {
|
|
10736
10868
|
return { ok: false, reason: "unsupported", message: "Desktop shortcuts are not available on this system yet." };
|
|
10737
10869
|
}
|
|
10738
|
-
const { addToDesktop } = await import('./cli-
|
|
10870
|
+
const { addToDesktop } = await import('./cli-MQO66C5O.js');
|
|
10739
10871
|
return addToDesktop(runtime.entry);
|
|
10740
10872
|
});
|
|
10741
10873
|
app.get("/api/desktop", async () => {
|
|
@@ -12338,7 +12470,7 @@ async function run() {
|
|
|
12338
12470
|
}
|
|
12339
12471
|
}
|
|
12340
12472
|
const ownEntry = fileURLToPath(import.meta.url);
|
|
12341
|
-
const { addToDesktop, installDeps } = await import('./cli-
|
|
12473
|
+
const { addToDesktop, installDeps } = await import('./cli-MQO66C5O.js');
|
|
12342
12474
|
const { refreshLauncher } = await import('./refresh-5HBDLLLB.js');
|
|
12343
12475
|
const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-W6NJXMWT.js');
|
|
12344
12476
|
const { launcherInstalled } = await import('./paths-6ZXRZUHY.js');
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { ASPECT_TOLERANCE, BUDGET_EXHAUSTED, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, SCHEMA_VERSION, STEM_MIN, SchemaTooNewError, SpendCapError, TRIGRAM_MIN, budgetSize, createCatalogStore, createCore, createStore, defaultHome, fold, ftsMatch, matchesQuery, ratioLabel, searchTerms, termMatches, uniqueProjectSlug, uniqueSetSlug, uniqueSlug } from './chunk-
|
|
2
|
-
//# sourceMappingURL=src-
|
|
3
|
-
//# sourceMappingURL=src-
|
|
1
|
+
export { ASPECT_TOLERANCE, BUDGET_EXHAUSTED, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, SCHEMA_VERSION, STEM_MIN, SchemaTooNewError, SpendCapError, TRIGRAM_MIN, budgetSize, createCatalogStore, createCore, createStore, defaultHome, fold, ftsMatch, matchesQuery, ratioLabel, searchTerms, termMatches, uniqueProjectSlug, uniqueSetSlug, uniqueSlug } from './chunk-3ZRMKVBB.js';
|
|
2
|
+
//# sourceMappingURL=src-3ITTV2UE.js.map
|
|
3
|
+
//# sourceMappingURL=src-3ITTV2UE.js.map
|