scenri 0.9.3 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,16 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.9.4](https://github.com/tonygorb/Scenri/compare/v0.9.3...v0.9.4) (2026-09-14)
4
+
5
+
6
+ ### Bug Fixes
7
+
8
+ * a rate-limited store is not an empty one ([1eef6eb](https://github.com/tonygorb/Scenri/commit/1eef6eb7471e1d2de97ccedcafba7962a5d393ad))
9
+ * **catalog:** a rate-limited store is not an empty one ([2e7f45a](https://github.com/tonygorb/Scenri/commit/2e7f45a07adcb202722fd07972d4e5d63bbda944))
10
+ * **catalog:** keep an AVIF picture, and say why a page was refused ([7e08137](https://github.com/tonygorb/Scenri/commit/7e08137b543d6b491e91cc6c7f4de9aa67be4b9e))
11
+ * **catalog:** keep the enum out of the error list, and cut what nothing produces ([1201832](https://github.com/tonygorb/Scenri/commit/1201832ac7df892506647224e790dbbaea080092))
12
+ * **catalog:** stop waiting out a store that has made up its mind ([8f6751a](https://github.com/tonygorb/Scenri/commit/8f6751a6462b6e44b06b08a6b2dc2aa12ad456d1))
13
+
3
14
  ## [0.9.3](https://github.com/tonygorb/Scenri/compare/v0.9.2...v0.9.3) (2026-09-14)
4
15
 
5
16
 
@@ -551,7 +551,7 @@ function openDb(homeDir) {
551
551
  function createImageStore(homeDir) {
552
552
  const dir = join(homeDir, "images");
553
553
  mkdirSync(dir, { recursive: true, mode: 448 });
554
- const EXTS = ["png", "jpg", "jpeg", "webp", "gif", "avif"];
554
+ const EXTS = ["png", "jpg", "jpeg", "webp", "gif", "avif", "heif", "heic"];
555
555
  const fileFor = (hash, ext = "png") => join(dir, `${hash}.${ext}`);
556
556
  const resolve = (hash) => {
557
557
  for (const ext of EXTS) {
@@ -1737,11 +1737,18 @@ function productImportMethods(db) {
1737
1737
  ).run(sourceId);
1738
1738
  return r2.changes;
1739
1739
  }
1740
- const placeholders = seenExternalKeys.map(() => "?").join(",");
1740
+ db.exec("CREATE TEMP TABLE IF NOT EXISTS seen_keys (k TEXT PRIMARY KEY)");
1741
+ db.exec("DELETE FROM seen_keys");
1742
+ const add = db.prepare("INSERT OR IGNORE INTO seen_keys (k) VALUES (?)");
1743
+ db.transaction((keys) => {
1744
+ for (const k of keys) add.run(k);
1745
+ })(seenExternalKeys);
1741
1746
  const r = db.prepare(
1742
1747
  `UPDATE catalog_products SET status='unavailable', updated_at=datetime('now')
1743
- WHERE source_id=? AND status='active' AND external_key NOT IN (${placeholders})`
1744
- ).run(sourceId, ...seenExternalKeys);
1748
+ WHERE source_id=? AND status='active'
1749
+ AND external_key NOT IN (SELECT k FROM seen_keys)`
1750
+ ).run(sourceId);
1751
+ db.exec("DELETE FROM seen_keys");
1745
1752
  return r.changes;
1746
1753
  }
1747
1754
  };
@@ -2025,5 +2032,5 @@ function createCore(homeDir = defaultHome()) {
2025
2032
  }
2026
2033
 
2027
2034
  export { ASPECT_TOLERANCE, BUDGET_EXHAUSTED, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, SCHEMA_VERSION, STEM_MIN, SchemaTooNewError, SpendCapError, TRIGRAM_MIN, budgetSize, createCatalogStore, createCore, createStore, defaultHome, fold, ftsMatch, matchesQuery, ratioLabel, searchTerms, termMatches, uniqueProjectSlug, uniqueSetSlug, uniqueSlug };
2028
- //# sourceMappingURL=chunk-Y5L3KLX5.js.map
2029
- //# sourceMappingURL=chunk-Y5L3KLX5.js.map
2035
+ //# sourceMappingURL=chunk-3ZRMKVBB.js.map
2036
+ //# sourceMappingURL=chunk-3ZRMKVBB.js.map
@@ -463,7 +463,7 @@ async function addOrRepair(ownEntry, say, over) {
463
463
  }
464
464
  async function rememberDecline(home) {
465
465
  try {
466
- const { createCore } = await import('./src-F3KJZNZT.js');
466
+ const { createCore } = await import('./src-3ITTV2UE.js');
467
467
  const core = createCore(home);
468
468
  try {
469
469
  core.store.setSetting("desktop.prompt", "declined");
@@ -530,5 +530,5 @@ function tail(path, bytes = 4096) {
530
530
  }
531
531
 
532
532
  export { addToDesktop, assetsDirFor, installDeps, runDesktopCommand, runExecFile, runOpenCommand };
533
- //# sourceMappingURL=cli-SX46UQO6.js.map
534
- //# sourceMappingURL=cli-SX46UQO6.js.map
533
+ //# sourceMappingURL=cli-MQO66C5O.js.map
534
+ //# sourceMappingURL=cli-MQO66C5O.js.map
package/dist/index.js CHANGED
@@ -86,12 +86,12 @@ try {
86
86
  break;
87
87
  }
88
88
  case "desktop": {
89
- const { runDesktopCommand } = await import('./cli-SX46UQO6.js');
89
+ const { runDesktopCommand } = await import('./cli-MQO66C5O.js');
90
90
  process.exit(await runDesktopCommand(command, fileURLToPath(import.meta.url)));
91
91
  break;
92
92
  }
93
93
  case "open": {
94
- const { runOpenCommand } = await import('./cli-SX46UQO6.js');
94
+ const { runOpenCommand } = await import('./cli-MQO66C5O.js');
95
95
  process.exit(await runOpenCommand(fileURLToPath(import.meta.url)));
96
96
  break;
97
97
  }
package/dist/serve.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { createUpdateChecker, classify, isReleaseTriplet, findNpm, stageVersion } from './chunk-FYQ5BAFA.js';
2
2
  import { shouldAdoptRunning, anotherScenriLines, portBusyLines } from './chunk-CMLJGPYC.js';
3
- import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-Y5L3KLX5.js';
3
+ import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-3ZRMKVBB.js';
4
4
  import { detectInstallKind } from './chunk-PAIAXRAC.js';
5
5
  export { detectInstallKind } from './chunk-PAIAXRAC.js';
6
6
  import { readMeta, repoSlug } from './chunk-Y3ZPBPLP.js';
@@ -4887,6 +4887,57 @@ async function inspectMark(buf, toPng2) {
4887
4887
  };
4888
4888
  }
4889
4889
 
4890
+ // ../catalog/src/failures.ts
4891
+ function pageFailure(status) {
4892
+ if (status === 429 || status === 503) return "RATE_LIMITED";
4893
+ if (status === 401 || status === 403 || status === 405) return "PAGE_BLOCKED";
4894
+ if (status === 404 || status === 410) return "PAGE_NOT_FOUND";
4895
+ return "PAGE_UNREADABLE";
4896
+ }
4897
+ function imageFailure(status) {
4898
+ if (status === 429 || status === 503) return "RATE_LIMITED";
4899
+ if (status === 401 || status === 403) return "IMAGE_FORBIDDEN";
4900
+ if (status === 404 || status === 410) return "IMAGE_NOT_FOUND";
4901
+ return "DOWNLOAD_FAILED";
4902
+ }
4903
+ function thrownFailure(err, aborted = false) {
4904
+ if (aborted) return "ABORTED";
4905
+ const m = String(err?.message ?? err).toLowerCase();
4906
+ if (m.includes("abort") || m.includes("timed out") || m.includes("timeout")) return "IMAGE_TIMEOUT";
4907
+ return "DOWNLOAD_FAILED";
4908
+ }
4909
+ function tally(into, reason, n = 1) {
4910
+ into[reason] = (into[reason] ?? 0) + n;
4911
+ }
4912
+ function leadingReason(t) {
4913
+ const rows = Object.entries(t);
4914
+ if (!rows.length) return null;
4915
+ return rows.sort((a, b) => b[1] - a[1])[0][0];
4916
+ }
4917
+ function summarise(t, saved, asked) {
4918
+ const reason = leadingReason(t);
4919
+ if (!reason) return null;
4920
+ const none = saved === 0;
4921
+ switch (reason) {
4922
+ case "RATE_LIMITED":
4923
+ return none ? "The store asked us to slow down, so nothing could be read. Waiting a few minutes and trying again usually works." : `The store asked us to slow down partway, so ${asked - saved} of ${asked} products were skipped. Trying again picks up the rest.`;
4924
+ case "PAGE_BLOCKED":
4925
+ return none ? "This store would not let us read its product pages." : `${asked - saved} of ${asked} product pages would not open.`;
4926
+ case "PAGE_NOT_FOUND":
4927
+ return `${asked - saved} of ${asked} products are no longer on the site.`;
4928
+ case "IMAGE_FORBIDDEN":
4929
+ case "IMAGE_NOT_FOUND":
4930
+ case "IMAGE_TIMEOUT":
4931
+ case "IMAGE_EMPTY":
4932
+ case "DOWNLOAD_FAILED":
4933
+ return none ? "We found the products, but none of their pictures could be downloaded." : "Some pictures could not be downloaded. The products they belong to are still here.";
4934
+ case "ABORTED":
4935
+ return null;
4936
+ default:
4937
+ return none ? "The products on this site could not be read." : null;
4938
+ }
4939
+ }
4940
+
4890
4941
  // ../catalog/src/url.ts
4891
4942
  function normalizeStoreUrl(input) {
4892
4943
  let raw = input.trim();
@@ -5105,12 +5156,49 @@ async function readBounded2(res, maxBytes) {
5105
5156
  function sleep4(ms) {
5106
5157
  return new Promise((r) => setTimeout(r, ms));
5107
5158
  }
5159
+ var cooledUntil = /* @__PURE__ */ new Map();
5160
+ var hostOf = (url) => {
5161
+ try {
5162
+ return new URL(url).host;
5163
+ } catch {
5164
+ return "";
5165
+ }
5166
+ };
5167
+ async function waitForHost(host, signal) {
5168
+ if (!host) return;
5169
+ for (; ; ) {
5170
+ const left = (cooledUntil.get(host) ?? 0) - Date.now();
5171
+ if (left <= 0 || signal?.aborted) return;
5172
+ await sleep4(Math.min(left, 250));
5173
+ }
5174
+ }
5175
+ function coolHost(host, ms) {
5176
+ if (!host || ms <= 0) return;
5177
+ const until = Date.now() + ms;
5178
+ if (until > (cooledUntil.get(host) ?? 0)) cooledUntil.set(host, until);
5179
+ }
5180
+ var RETRY_AFTER_CAP_MS = 6e4;
5181
+ function retryAfterMs(header, now = Date.now()) {
5182
+ if (!header) return null;
5183
+ const secs = Number(header.trim());
5184
+ if (Number.isFinite(secs) && secs >= 0) return Math.min(secs * 1e3, RETRY_AFTER_CAP_MS);
5185
+ const at = Date.parse(header);
5186
+ if (Number.isNaN(at)) return null;
5187
+ return Math.min(Math.max(0, at - now), RETRY_AFTER_CAP_MS);
5188
+ }
5189
+ function throttleBackoff(attempt) {
5190
+ const base = Math.min(1e3 * 2 ** attempt, 16e3);
5191
+ return base + Math.floor(Math.random() * 400);
5192
+ }
5108
5193
  async function httpGet(url, opts = {}) {
5109
5194
  const fetchImpl = opts.fetchImpl ?? fetch;
5110
5195
  const retries = opts.retries ?? 3;
5111
5196
  const timeoutMs = opts.timeoutMs ?? 25e3;
5112
5197
  let lastErr;
5198
+ const host = hostOf(url);
5113
5199
  for (let attempt = 0; attempt <= retries; attempt++) {
5200
+ await waitForHost(host, opts.signal);
5201
+ if (opts.signal?.aborted) throw new Error("aborted");
5114
5202
  const ctrl = new AbortController();
5115
5203
  const onAbort = () => ctrl.abort();
5116
5204
  opts.signal?.addEventListener("abort", onAbort, { once: true });
@@ -5142,8 +5230,19 @@ async function httpGet(url, opts = {}) {
5142
5230
  } catch {
5143
5231
  }
5144
5232
  }
5145
- if ((res.status === 429 || res.status >= 500) && attempt < retries) {
5146
- await sleep4(400 * 2 ** attempt);
5233
+ if (res.status === 429 || res.status === 503) {
5234
+ const asked = retryAfterMs(res.headers.get("retry-after"));
5235
+ const wait = asked ?? throttleBackoff(attempt);
5236
+ coolHost(host, wait);
5237
+ if (attempt < retries) {
5238
+ await res.body?.cancel().catch(() => {
5239
+ });
5240
+ continue;
5241
+ }
5242
+ return res;
5243
+ }
5244
+ if (res.status >= 500 && attempt < retries) {
5245
+ await sleep4(throttleBackoff(attempt));
5147
5246
  continue;
5148
5247
  }
5149
5248
  return res;
@@ -5417,6 +5516,9 @@ async function fetchProductPages(ctx, urls, opts = {}) {
5417
5516
  const out = [];
5418
5517
  const seen = /* @__PURE__ */ new Set();
5419
5518
  let kept = 0;
5519
+ let refusedRun = 0;
5520
+ const giveUpAt = opts.giveUpAfterRefusals ?? Number.POSITIVE_INFINITY;
5521
+ let givenUp = false;
5420
5522
  const want = opts.want ?? Number.POSITIVE_INFINITY;
5421
5523
  const ceiling = opts.maxPages ?? (Number.isFinite(want) ? want * 3 : urls.length);
5422
5524
  const take = urls.slice(0, ceiling);
@@ -5425,12 +5527,12 @@ async function fetchProductPages(ctx, urls, opts = {}) {
5425
5527
  take,
5426
5528
  opts.delayMs ? 1 : opts.concurrency ?? 5,
5427
5529
  async (u) => {
5428
- if (kept >= want) return;
5530
+ if (kept >= want || givenUp) return;
5429
5531
  if (opts.deadline != null && Date.now() > opts.deadline) return;
5430
5532
  if (opts.maxTotalBytes != null && bytes >= opts.maxTotalBytes) return;
5431
5533
  try {
5432
5534
  if (opts.delayMs) await sleep5(opts.delayMs);
5433
- const { ok, text, url } = await httpText(u, {
5535
+ const { ok, status, text, url } = await httpText(u, {
5434
5536
  fetchImpl: ctx.fetchImpl,
5435
5537
  signal: ctx.signal,
5436
5538
  accept: "text/html",
@@ -5442,9 +5544,14 @@ async function fetchProductPages(ctx, urls, opts = {}) {
5442
5544
  opts.stats.bytes = bytes;
5443
5545
  }
5444
5546
  if (!ok) {
5445
- if (opts.stats) opts.stats.refused = (opts.stats.refused ?? 0) + 1;
5547
+ if (opts.stats) {
5548
+ opts.stats.refused = (opts.stats.refused ?? 0) + 1;
5549
+ if (opts.stats.reasons) tally(opts.stats.reasons, pageFailure(status));
5550
+ }
5551
+ if (++refusedRun >= giveUpAt) givenUp = true;
5446
5552
  return;
5447
5553
  }
5554
+ refusedRun = 0;
5448
5555
  for (const p of productsFromPage(text, url)) {
5449
5556
  if (seen.has(p.externalKey)) continue;
5450
5557
  seen.add(p.externalKey);
@@ -5453,8 +5560,11 @@ async function fetchProductPages(ctx, urls, opts = {}) {
5453
5560
  else out.push(p);
5454
5561
  }
5455
5562
  opts.onProduct?.(kept);
5456
- } catch {
5457
- if (opts.stats) opts.stats.refused = (opts.stats.refused ?? 0) + 1;
5563
+ } catch (err) {
5564
+ if (opts.stats) {
5565
+ opts.stats.refused = (opts.stats.refused ?? 0) + 1;
5566
+ if (opts.stats.reasons) tally(opts.stats.reasons, thrownFailure(err, ctx.signal?.aborted));
5567
+ }
5458
5568
  }
5459
5569
  },
5460
5570
  ctx.signal
@@ -6442,7 +6552,7 @@ async function runJob(deps, jobId, brandId, url, signal, only) {
6442
6552
  const { core, fetchImpl } = deps;
6443
6553
  const patch2 = (p) => core.catalog.updateJob(jobId, p);
6444
6554
  try {
6445
- const tally = { fetched: 0, upserted: 0, imagesDone: 0, imagesTotal: 0, errors: [], seenKeys: [] };
6555
+ const tally2 = { fetched: 0, upserted: 0, imagesDone: 0, imagesTotal: 0, errors: [], seenKeys: [] };
6446
6556
  const ctx = { fetchImpl: fetchImpl ?? fetch, baseUrl: url, signal };
6447
6557
  let urls = [];
6448
6558
  let bulk = null;
@@ -6506,10 +6616,11 @@ async function runJob(deps, jobId, brandId, url, signal, only) {
6506
6616
  patch2({ stage: "fetching_products", discovered, message: `Reading ${discovered.toLocaleString()} products` });
6507
6617
  }
6508
6618
  }
6509
- const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform, tally, discovered);
6510
- const pictures = drainPictures(deps, jobId, brandId, tally, signal);
6619
+ const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered);
6620
+ const pictures = drainPictures(deps, jobId, brandId, tally2, signal);
6511
6621
  let pictureErrors;
6512
- const stats = { pages: 0, bytes: 0, refused: 0 };
6622
+ let pictureReasons = {};
6623
+ const stats = { pages: 0, bytes: 0, refused: 0, reasons: {} };
6513
6624
  try {
6514
6625
  if (bulk) for (const p of bulk) run2.write(p);
6515
6626
  else
@@ -6517,46 +6628,43 @@ async function runJob(deps, jobId, brandId, url, signal, only) {
6517
6628
  concurrency: IMPORT_CONCURRENCY,
6518
6629
  maxBytes: 15e5,
6519
6630
  onEach: run2.write,
6520
- stats
6631
+ stats,
6632
+ // Past this many refusals in a row the store is not going to change
6633
+ // its mind inside this import, and waiting out its cooldown for the
6634
+ // rest of the catalogue is time nobody gets a product for.
6635
+ giveUpAfterRefusals: 24
6521
6636
  });
6522
- if (!signal.aborted && tally.upserted) {
6637
+ if (!signal.aborted && tally2.upserted) {
6523
6638
  patch2({
6524
6639
  stage: "processing_assets",
6525
- message: `Downloading ${tally.imagesTotal.toLocaleString()} pictures`,
6526
- upserted: tally.upserted,
6527
- fetched: tally.fetched
6640
+ message: `Downloading ${tally2.imagesTotal.toLocaleString()} pictures`,
6641
+ upserted: tally2.upserted,
6642
+ fetched: tally2.fetched
6528
6643
  });
6529
6644
  }
6530
6645
  } finally {
6531
6646
  pictures.stop();
6532
- pictureErrors = await pictures.done;
6647
+ const settled = await pictures.done;
6648
+ pictureErrors = settled.errors;
6649
+ pictureReasons = settled.reasons;
6533
6650
  }
6534
6651
  if (signal.aborted) {
6535
6652
  patch2({
6536
6653
  stage: "cancelled",
6537
6654
  errors: [],
6538
- message: tally.upserted ? `Stopped after saving ${tally.upserted.toLocaleString()} products` : "Stopped before anything was saved",
6655
+ message: tally2.upserted ? `Stopped after saving ${tally2.upserted.toLocaleString()} products` : "Stopped before anything was saved",
6539
6656
  finished: true
6540
6657
  });
6541
6658
  core.catalog.setSourceStatus(run2.sourceId, "partial", true);
6542
6659
  return;
6543
6660
  }
6544
- if (!tally.upserted) {
6545
- const message = only?.length ? "None of the chosen products could be read" : `Found pages on this ${platform} store but could not read a product from any of them. The store may be blocking automated readers.`;
6546
- patch2({
6547
- stage: "failed",
6548
- message,
6549
- errors: [{ code: "no_products_fetched", message }],
6550
- finished: true
6551
- });
6552
- core.catalog.setSourceStatus(run2.sourceId, "failed", true);
6553
- return;
6554
- }
6555
6661
  run2.finish({
6556
6662
  sweep,
6557
6663
  warnings,
6558
6664
  errors: pictureErrors,
6559
6665
  refused: stats.refused,
6666
+ reasons: { ...stats.reasons, ...pictureReasons },
6667
+ asked: bulk ? bulk.length : urls.length,
6560
6668
  // Every address was read. A bulk API hands the catalogue over whole, so
6561
6669
  // there is nothing to cover.
6562
6670
  covered: bulk ? true : stats.pages >= urls.length
@@ -6578,7 +6686,7 @@ var IMAGES_PER_PRODUCT = 3;
6578
6686
  var TERMINAL = /* @__PURE__ */ new Set(["completed", "partial", "no_catalog", "cancelled", "failed"]);
6579
6687
  var IMAGE_CONCURRENCY = 4;
6580
6688
  var sleep6 = (ms) => new Promise((r) => setTimeout(r, ms));
6581
- function beginWrite(deps, jobId, brandId, baseUrl, platform, tally, discovered) {
6689
+ function beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered) {
6582
6690
  const { core } = deps;
6583
6691
  const patch2 = (p) => core.catalog.updateJob(jobId, p);
6584
6692
  const source = core.catalog.upsertSource(brandId, baseUrl, platform);
@@ -6616,44 +6724,50 @@ function beginWrite(deps, jobId, brandId, baseUrl, platform, tally, discovered)
6616
6724
  })),
6617
6725
  collections: (p.collections ?? []).map((c) => ({ externalKey: c, title: c }))
6618
6726
  });
6619
- tally.seenKeys.push(p.externalKey);
6620
- tally.upserted++;
6621
- tally.fetched++;
6622
- patch2({ upserted: tally.upserted, fetched: tally.fetched });
6727
+ tally2.seenKeys.push(p.externalKey);
6728
+ tally2.upserted++;
6729
+ tally2.fetched++;
6730
+ patch2({ upserted: tally2.upserted, fetched: tally2.fetched });
6623
6731
  },
6624
6732
  finish({
6625
6733
  sweep,
6626
6734
  warnings,
6627
6735
  errors,
6628
6736
  refused = 0,
6629
- covered = true
6737
+ covered = true,
6738
+ reasons = {},
6739
+ asked = 0
6630
6740
  }) {
6631
- if (sweep) core.catalog.markMissingUnavailable(source.id, tally.seenKeys);
6632
- tally.errors = errors;
6741
+ const mayRetire = sweep && covered && refused === 0 && tally2.upserted > 0;
6742
+ if (mayRetire) core.catalog.markMissingUnavailable(source.id, tally2.seenKeys);
6743
+ tally2.errors = errors;
6633
6744
  const partial = !covered || refused > 0;
6634
- const shut = refused > 0 && refused >= Math.max(20, tally.upserted * 0.25);
6635
- const message = shut ? `The store stopped answering after ${tally.upserted.toLocaleString()} of ${discovered.toLocaleString()} products. Try again later.` : partial ? `Imported ${tally.upserted.toLocaleString()} products with ${(errors.length + refused).toLocaleString()} issue${errors.length + refused === 1 ? "" : "s"}` : `Imported ${tally.upserted.toLocaleString()} products`;
6745
+ const savedNothing = tally2.upserted === 0;
6746
+ const said = summarise(reasons, tally2.upserted, asked || discovered || tally2.fetched);
6747
+ const stage = savedNothing ? "failed" : partial ? "partial" : "completed";
6748
+ const message = savedNothing ? said ?? "We found the products on this site, but none of them could be imported." : said ?? (partial ? `Imported ${tally2.upserted.toLocaleString()} products with ${(errors.length + refused).toLocaleString()} issue${errors.length + refused === 1 ? "" : "s"}` : `Imported ${tally2.upserted.toLocaleString()} products`);
6636
6749
  patch2({
6637
- stage: partial ? "partial" : "completed",
6638
- upserted: tally.upserted,
6639
- fetched: tally.fetched,
6640
- imagesDone: tally.imagesDone,
6641
- imagesTotal: tally.imagesTotal,
6642
- errors: shut ? [...errors, { code: "store_refused", message: `${refused.toLocaleString()} pages were refused` }] : errors,
6750
+ stage,
6751
+ upserted: tally2.upserted,
6752
+ fetched: tally2.fetched,
6753
+ imagesDone: tally2.imagesDone,
6754
+ imagesTotal: tally2.imagesTotal,
6755
+ errors,
6643
6756
  warnings,
6644
6757
  message,
6645
6758
  finished: true
6646
6759
  });
6647
- core.catalog.setSourceStatus(source.id, partial ? "partial" : "ready", true);
6760
+ core.catalog.setSourceStatus(source.id, savedNothing ? "failed" : partial ? "partial" : "ready", true);
6648
6761
  }
6649
6762
  };
6650
6763
  }
6651
6764
  var IMPORT_CONCURRENCY = 4;
6652
6765
  var PICTURE_ROUND = 60;
6653
- function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = IMAGES_PER_PRODUCT) {
6766
+ function drainPictures(deps, jobId, brandId, tally2, signal, imagesPerProduct = IMAGES_PER_PRODUCT) {
6654
6767
  const { core, fetchImpl } = deps;
6655
6768
  const patch2 = (p) => core.catalog.updateJob(jobId, p);
6656
6769
  const errors = [...core.catalog.getJob(jobId)?.errors ?? []];
6770
+ const reasons = {};
6657
6771
  let stopped = false;
6658
6772
  const done = (async () => {
6659
6773
  while (!signal.aborted) {
@@ -6663,7 +6777,7 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
6663
6777
  await sleep6(150);
6664
6778
  continue;
6665
6779
  }
6666
- tally.imagesTotal += round.length;
6780
+ tally2.imagesTotal += round.length;
6667
6781
  await mapPool(
6668
6782
  round,
6669
6783
  IMAGE_CONCURRENCY,
@@ -6673,11 +6787,13 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
6673
6787
  const res = await httpGet(img.sourceUrl, { fetchImpl, signal, timeoutMs: 4e4, retries: 2 });
6674
6788
  if (!res.ok) {
6675
6789
  errors.push({ code: "image_http", message: `HTTP ${res.status}`, url: img.sourceUrl });
6790
+ tally(reasons, imageFailure(res.status));
6676
6791
  return;
6677
6792
  }
6678
6793
  const buf = Buffer.from(await res.arrayBuffer());
6679
6794
  if (!buf.length) {
6680
6795
  errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
6796
+ tally(reasons, "IMAGE_EMPTY");
6681
6797
  return;
6682
6798
  }
6683
6799
  const probe = await sharp21(buf).metadata();
@@ -6691,6 +6807,7 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
6691
6807
  });
6692
6808
  } catch (err) {
6693
6809
  if (signal.aborted) return;
6810
+ tally(reasons, thrownFailure(err, signal.aborted));
6694
6811
  errors.push({
6695
6812
  code: "image_failed",
6696
6813
  message: String(err?.message ?? err),
@@ -6698,8 +6815,8 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
6698
6815
  retryable: true
6699
6816
  });
6700
6817
  } finally {
6701
- tally.imagesDone++;
6702
- patch2({ imagesDone: tally.imagesDone, imagesTotal: tally.imagesTotal, errors });
6818
+ tally2.imagesDone++;
6819
+ patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
6703
6820
  }
6704
6821
  },
6705
6822
  signal
@@ -6710,8 +6827,8 @@ function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = I
6710
6827
  break;
6711
6828
  }
6712
6829
  }
6713
- patch2({ imagesDone: tally.imagesDone, imagesTotal: tally.imagesTotal, errors });
6714
- return errors;
6830
+ patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
6831
+ return { errors, reasons };
6715
6832
  })();
6716
6833
  return {
6717
6834
  stop() {
@@ -9734,6 +9851,21 @@ function registerImageRoutes(app, deps) {
9734
9851
 
9735
9852
  // src/release/notes.data.ts
9736
9853
  var RELEASES = [
9854
+ {
9855
+ version: "0.9.4",
9856
+ date: "2026-09-14",
9857
+ title: "A store that asks Scenri to slow down no longer looks like an empty one.",
9858
+ sections: [
9859
+ {
9860
+ heading: "Products",
9861
+ body: "Some stores limit how fast anything can read them. Scenri now waits the way such a store asks and says what happened: an import that could not read anything tells you to try again in a few minutes, and one that got part way says how much is left. Products from an earlier import are never retired by a run that was refused, and a store that keeps saying no ends the import early rather than grinding on."
9862
+ },
9863
+ {
9864
+ heading: "Fixes",
9865
+ body: "A product picture saved in a newer format such as AVIF is found again instead of going missing."
9866
+ }
9867
+ ]
9868
+ },
9737
9869
  {
9738
9870
  version: "0.9.3",
9739
9871
  date: "2026-09-14",
@@ -10727,7 +10859,7 @@ function registerDesktopRoutes(app, deps) {
10727
10859
  record: null
10728
10860
  };
10729
10861
  }
10730
- const { installDeps } = await import('./cli-SX46UQO6.js');
10862
+ const { installDeps } = await import('./cli-MQO66C5O.js');
10731
10863
  const { desktopStatus } = await import('./install-VXJPLYWM.js');
10732
10864
  return desktopStatus(installDeps(runtime.entry));
10733
10865
  });
@@ -10735,7 +10867,7 @@ function registerDesktopRoutes(app, deps) {
10735
10867
  if (!runtime.entry) {
10736
10868
  return { ok: false, reason: "unsupported", message: "Desktop shortcuts are not available on this system yet." };
10737
10869
  }
10738
- const { addToDesktop } = await import('./cli-SX46UQO6.js');
10870
+ const { addToDesktop } = await import('./cli-MQO66C5O.js');
10739
10871
  return addToDesktop(runtime.entry);
10740
10872
  });
10741
10873
  app.get("/api/desktop", async () => {
@@ -12338,7 +12470,7 @@ async function run() {
12338
12470
  }
12339
12471
  }
12340
12472
  const ownEntry = fileURLToPath(import.meta.url);
12341
- const { addToDesktop, installDeps } = await import('./cli-SX46UQO6.js');
12473
+ const { addToDesktop, installDeps } = await import('./cli-MQO66C5O.js');
12342
12474
  const { refreshLauncher } = await import('./refresh-5HBDLLLB.js');
12343
12475
  const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-W6NJXMWT.js');
12344
12476
  const { launcherInstalled } = await import('./paths-6ZXRZUHY.js');
@@ -1,3 +1,3 @@
1
- export { ASPECT_TOLERANCE, BUDGET_EXHAUSTED, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, SCHEMA_VERSION, STEM_MIN, SchemaTooNewError, SpendCapError, TRIGRAM_MIN, budgetSize, createCatalogStore, createCore, createStore, defaultHome, fold, ftsMatch, matchesQuery, ratioLabel, searchTerms, termMatches, uniqueProjectSlug, uniqueSetSlug, uniqueSlug } from './chunk-Y5L3KLX5.js';
2
- //# sourceMappingURL=src-F3KJZNZT.js.map
3
- //# sourceMappingURL=src-F3KJZNZT.js.map
1
+ export { ASPECT_TOLERANCE, BUDGET_EXHAUSTED, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, SCHEMA_VERSION, STEM_MIN, SchemaTooNewError, SpendCapError, TRIGRAM_MIN, budgetSize, createCatalogStore, createCore, createStore, defaultHome, fold, ftsMatch, matchesQuery, ratioLabel, searchTerms, termMatches, uniqueProjectSlug, uniqueSetSlug, uniqueSlug } from './chunk-3ZRMKVBB.js';
2
+ //# sourceMappingURL=src-3ITTV2UE.js.map
3
+ //# sourceMappingURL=src-3ITTV2UE.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scenri",
3
- "version": "0.9.3",
3
+ "version": "0.9.4",
4
4
  "author": "Tony Gorb <hello@scenri.co>",
5
5
  "license": "AGPL-3.0-only",
6
6
  "type": "module",