scenri 0.9.2 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/serve.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { createUpdateChecker, classify, isReleaseTriplet, findNpm, stageVersion } from './chunk-FYQ5BAFA.js';
2
2
  import { shouldAdoptRunning, anotherScenriLines, portBusyLines } from './chunk-CMLJGPYC.js';
3
- import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-OJAG3FRX.js';
3
+ import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-3ZRMKVBB.js';
4
4
  import { detectInstallKind } from './chunk-PAIAXRAC.js';
5
5
  export { detectInstallKind } from './chunk-PAIAXRAC.js';
6
6
  import { readMeta, repoSlug } from './chunk-Y3ZPBPLP.js';
@@ -12,7 +12,7 @@ import { homedir, networkInterfaces, tmpdir } from 'os';
12
12
  import { randomBytes, randomUUID, timingSafeEqual, createHash } from 'crypto';
13
13
  import { readFile, copyFile, stat, readdir, mkdtemp, rm, access, rename, unlink, writeFile } from 'fs/promises';
14
14
  import { spawn } from 'child_process';
15
- import sharp20 from 'sharp';
15
+ import sharp21 from 'sharp';
16
16
  import Fastify from 'fastify';
17
17
  import fastifyStatic from '@fastify/static';
18
18
  import fastifyMultipart from '@fastify/multipart';
@@ -20,6 +20,8 @@ import JSZip from 'jszip';
20
20
  import { Ajv2020 } from 'ajv/dist/2020.js';
21
21
  import addFormats from 'ajv-formats';
22
22
  import * as cheerio from 'cheerio';
23
+ import { lookup } from 'dns/promises';
24
+ import { isIP } from 'net';
23
25
  import { parse } from 'node-html-parser';
24
26
  import pixelmatch from 'pixelmatch';
25
27
  import { PNG } from 'pngjs';
@@ -687,6 +689,108 @@ function resolveCodex(platform, spawnImpl, timeoutMs = WHERE_TIMEOUT_MS) {
687
689
  });
688
690
  }
689
691
 
692
+ // ../engines/codex/src/childEnv.ts
693
+ function buildChildEnv(parent, drop) {
694
+ if (drop.length === 0) return { ...parent };
695
+ const unwanted = new Set(drop.map((name) => name.toUpperCase()));
696
+ const child = {};
697
+ for (const [name, value] of Object.entries(parent)) {
698
+ if (unwanted.has(name.toUpperCase())) continue;
699
+ child[name] = value;
700
+ }
701
+ return child;
702
+ }
703
+
704
+ // ../engines/codex/src/classify.ts
705
+ var CONFLICT_ENV_KEYS = ["CODEX_API_KEY", "CODEX_ACCESS_TOKEN", "OPENAI_API_KEY"];
706
+ function presentConflictKeys(env, ignored = []) {
707
+ const dropped = new Set(ignored.map((n) => n.toUpperCase()));
708
+ const present = [];
709
+ for (const name of CONFLICT_ENV_KEYS) {
710
+ if (dropped.has(name)) continue;
711
+ const value = env[name];
712
+ if (typeof value === "string" && value.trim() !== "") present.push(name);
713
+ }
714
+ return present;
715
+ }
716
+ function listKeys(keys) {
717
+ if (keys.length === 0) return "";
718
+ if (keys.length === 1) return keys[0];
719
+ return `${keys.slice(0, -1).join(", ")} and ${keys[keys.length - 1]}`;
720
+ }
721
+ function conflictReason(keys) {
722
+ return `Codex is signed in, but ${listKeys(keys)} in this computer's environment is overriding that sign-in and OpenAI rejected it.`;
723
+ }
724
+ var NOT_FOUND = /failed to spawn codex|\bENOENT\b|command not found|is not recognized as an internal/i;
725
+ var TOO_OLD = /requires a newer version of Codex|is too old/i;
726
+ var USAGE_LIMIT = /usage limit/i;
727
+ var SIGNED_OUT = /not logged in|login required|run .{0,3}codex login|no credentials found/i;
728
+ var UNAUTHORIZED = /\b401\b|unauthorized/i;
729
+ var BAD_KEY = /incorrect api key provided|invalid_api_key|invalid api key/i;
730
+ var NETWORK = /ENOTFOUND|ECONNREFUSED|ECONNRESET|EAI_AGAIN|getaddrinfo|error sending request|dns error|tls handshake/i;
731
+ var NO_OUTPUT = /produced no output for \d+s/i;
732
+ var TIMED_OUT = /timed out after/i;
733
+ var ALREADY_CONFLICT = /environment is overriding that sign-in/i;
734
+ function classifyCodexFailure(input) {
735
+ const text = input.text ?? "";
736
+ const conflictKeys = presentConflictKeys(input.env, input.ignored);
737
+ const at = (code, reason) => ({ code, conflictKeys, reason });
738
+ if (ALREADY_CONFLICT.test(text)) return at("CODEX_AUTH_CONFLICT", conflictReason(conflictKeys));
739
+ if (NOT_FOUND.test(text)) return at("CODEX_NOT_FOUND", "Codex CLI is not installed on this computer.");
740
+ if (TOO_OLD.test(text)) return at("CODEX_VERSION_TOO_OLD", "Codex CLI on this computer is too old.");
741
+ if (USAGE_LIMIT.test(text)) return at("CODEX_USAGE_LIMIT", "Your Codex plan's usage limit is used up.");
742
+ if (SIGNED_OUT.test(text)) return at("CODEX_NOT_AUTHENTICATED", "Codex CLI is signed out on this computer.");
743
+ if (UNAUTHORIZED.test(text)) {
744
+ if (BAD_KEY.test(text) && conflictKeys.length > 0) return at("CODEX_AUTH_CONFLICT", conflictReason(conflictKeys));
745
+ return at("CODEX_401", "OpenAI did not accept the Codex sign-in on this computer.");
746
+ }
747
+ if (NETWORK.test(text)) return at("NETWORK_FAILURE", "Codex could not reach OpenAI.");
748
+ if (NO_OUTPUT.test(text)) return at("CODEX_NO_OUTPUT", "Codex never started answering.");
749
+ if (TIMED_OUT.test(text)) return at("CODEX_TIMEOUT", "Codex ran out of time.");
750
+ if (text.trim() !== "") return at("CODEX_EXECUTION_FAILED", "Codex stopped with an error.");
751
+ return at("UNKNOWN", "Codex stopped for a reason it did not give.");
752
+ }
753
+
754
+ // ../engines/codex/src/connect.ts
755
+ var CONNECT_PROMPT = "Reply with the single word READY. Do not use any tools. Do not generate an image. Do not read or write any files.";
756
+ var CONNECT_TIMEOUT_MS = 45e3;
757
+ var CONNECT_TTL_MS = 10 * 6e4;
758
+ function connectFingerprint(p) {
759
+ return [p.command, p.version ?? "unknown", [...p.present].sort().join("+"), [...p.ignored].sort().join("+")].join(
760
+ "|"
761
+ );
762
+ }
763
+ function outcomeFor(code) {
764
+ switch (code) {
765
+ case "CODEX_AUTH_CONFLICT":
766
+ case "CODEX_401":
767
+ case "CODEX_NOT_AUTHENTICATED":
768
+ case "CODEX_NOT_FOUND":
769
+ case "CODEX_VERSION_TOO_OLD":
770
+ return "refused";
771
+ default:
772
+ return "unproven";
773
+ }
774
+ }
775
+ function availabilityFrom(shallow, conn) {
776
+ if (!shallow.ok) return shallow;
777
+ if (!conn || conn.outcome !== "refused" || !conn.failure) return shallow;
778
+ const { code, reason } = conn.failure;
779
+ switch (code) {
780
+ case "CODEX_AUTH_CONFLICT":
781
+ return { ok: false, reason, code: "env-conflict" };
782
+ case "CODEX_401":
783
+ case "CODEX_NOT_AUTHENTICATED":
784
+ return { ok: false, reason, code: "not-authenticated" };
785
+ case "CODEX_VERSION_TOO_OLD":
786
+ return { ok: false, reason, code: "update-needed" };
787
+ case "CODEX_NOT_FOUND":
788
+ return { ok: false, reason, code: "not-installed" };
789
+ default:
790
+ return shallow;
791
+ }
792
+ }
793
+
690
794
  // ../engines/codex/src/run.ts
691
795
  var NOT_INSTALLED_REASON = "Codex CLI is not installed on this computer";
692
796
  var NOT_AUTHENTICATED_REASON = "Codex CLI is installed but not signed in";
@@ -741,6 +845,7 @@ function codexFailureDetail(stderr, stdout) {
741
845
  return tailOf(marked) || tailOf(body) || tailOf(stdout.trim()) || tailOf(stderr.trim());
742
846
  }
743
847
  var MODEL_NEEDS_NEWER_CODEX = /The '([^']+)' model requires a newer version of Codex/;
848
+ var BAD_KEY_401 = /incorrect api key provided|invalid_api_key|invalid api key/i;
744
849
  function tooOldForModel(version, model) {
745
850
  return `Codex CLI ${version ?? "on this computer"} is too old for the model it is set to, ${model}.`;
746
851
  }
@@ -781,6 +886,10 @@ function createRunner(opts = {}) {
781
886
  const probeTtlMs = opts.probeTtlMs ?? PROBE_TTL_MS;
782
887
  const firstOutputMs = opts.firstOutputMs ?? FIRST_OUTPUT_TIMEOUT_MS;
783
888
  const platform = opts.platform ?? process.platform;
889
+ const connectTtlMs = opts.connectTtlMs ?? CONNECT_TTL_MS;
890
+ const connectTimeoutMs = opts.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
891
+ const parentEnv = opts.env ?? process.env;
892
+ const ignoreEnvKeys = opts.ignoreEnvKeys ?? (() => []);
784
893
  const killCodex = (child) => killTree(child, platform, spawnImpl);
785
894
  let resolved = null;
786
895
  async function resolution() {
@@ -792,7 +901,8 @@ function createRunner(opts = {}) {
792
901
  const winArg = (a) => `"${a.replace(/[\r\n]+/g, " ").replace(/"/g, "'").replace(/%/g, " percent ")}"`;
793
902
  const spawnCodex = (exe, args, stdinOpen) => {
794
903
  const stdio = [stdinOpen ? "pipe" : "ignore", "pipe", "pipe"];
795
- return exe.direct ? spawnImpl(exe.command, args, { stdio, ...platform !== "win32" ? { detached: true } : {} }) : spawnImpl([exe.command, ...args.map(winArg)].join(" "), [], { stdio, shell: true });
904
+ const env = buildChildEnv(parentEnv, ignoreEnvKeys());
905
+ return exe.direct ? spawnImpl(exe.command, args, { stdio, env, ...platform !== "win32" ? { detached: true } : {} }) : spawnImpl([exe.command, ...args.map(winArg)].join(" "), [], { stdio, env, shell: true });
796
906
  };
797
907
  async function run2(args, signal, io) {
798
908
  const exe = await resolution();
@@ -907,6 +1017,16 @@ function createRunner(opts = {}) {
907
1017
  );
908
1018
  return;
909
1019
  }
1020
+ const conflicting = presentConflictKeys(parentEnv, ignoreEnvKeys());
1021
+ if (conflicting.length > 0 && /\b401\b|unauthorized/i.test(stderr) && BAD_KEY_401.test(stderr)) {
1022
+ noteConnection("refused", {
1023
+ code: "CODEX_AUTH_CONFLICT",
1024
+ conflictKeys: conflicting,
1025
+ reason: conflictReason(conflicting)
1026
+ });
1027
+ finish(`exit-${code ?? "unknown"}`, () => reject(new Error(conflictReason(conflicting))));
1028
+ return;
1029
+ }
910
1030
  const snippet2 = codexFailureDetail(stderr, stdout);
911
1031
  finish(
912
1032
  `exit-${code ?? "unknown"}`,
@@ -966,6 +1086,23 @@ function createRunner(opts = {}) {
966
1086
  return avail;
967
1087
  }
968
1088
  let cached = null;
1089
+ let connCache = null;
1090
+ let connectInFlight = null;
1091
+ function fingerprintFor(exe, version) {
1092
+ const ignored = ignoreEnvKeys();
1093
+ return connectFingerprint({
1094
+ command: exe.command,
1095
+ version,
1096
+ present: presentConflictKeys(parentEnv, ignored),
1097
+ ignored
1098
+ });
1099
+ }
1100
+ function freshConnection(exe, version) {
1101
+ if (!connCache || connectTtlMs <= 0) return null;
1102
+ if (Date.now() - connCache.at >= connectTtlMs) return null;
1103
+ if (connCache.fingerprint !== fingerprintFor(exe, version)) return null;
1104
+ return connCache;
1105
+ }
969
1106
  async function probe() {
970
1107
  if (process.env.SCENRI_NO_CODEX === "1") {
971
1108
  return { ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" };
@@ -978,52 +1115,128 @@ function createRunner(opts = {}) {
978
1115
  return value;
979
1116
  }
980
1117
  async function probeUncached() {
1118
+ const { avail, exe, version } = await probeLadder();
1119
+ return availabilityFrom(avail, freshConnection(exe, version));
1120
+ }
1121
+ async function probeLadder() {
981
1122
  resolved = await resolveCodex(platform, spawnImpl);
982
1123
  const exe = resolved;
983
1124
  const ver = await probeSpawn(exe, ["--version"]);
984
1125
  if (ver.outcome === "timeout") {
985
- return verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, null);
1126
+ return {
1127
+ avail: verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, null),
1128
+ exe,
1129
+ version: null
1130
+ };
986
1131
  }
987
1132
  if (ver.outcome !== "ok") {
988
- return verdict({ ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" }, exe, null);
1133
+ return {
1134
+ avail: verdict({ ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" }, exe, null),
1135
+ exe,
1136
+ version: null
1137
+ };
989
1138
  }
990
1139
  const version = parseCodexVersion(ver.stdout);
991
1140
  knownVersion = version;
992
1141
  if (version && !versionAtLeast(version, MIN_CODEX_VERSION)) {
993
- return verdict(
994
- {
995
- ok: false,
996
- reason: `Codex CLI ${version} is too old. Scenri needs ${MIN_CODEX_VERSION} or newer.`,
997
- code: "update-needed"
998
- },
1142
+ return {
1143
+ avail: verdict(
1144
+ {
1145
+ ok: false,
1146
+ reason: `Codex CLI ${version} is too old. Scenri needs ${MIN_CODEX_VERSION} or newer.`,
1147
+ code: "update-needed"
1148
+ },
1149
+ exe,
1150
+ version
1151
+ ),
999
1152
  exe,
1000
1153
  version
1001
- );
1154
+ };
1002
1155
  }
1003
1156
  if (tooOldFor) {
1004
1157
  if (version === tooOldFor.version) {
1005
- return verdict(
1006
- { ok: false, reason: tooOldForModel(version, tooOldFor.model), code: "update-needed" },
1158
+ return {
1159
+ avail: verdict(
1160
+ { ok: false, reason: tooOldForModel(version, tooOldFor.model), code: "update-needed" },
1161
+ exe,
1162
+ version
1163
+ ),
1007
1164
  exe,
1008
1165
  version
1009
- );
1166
+ };
1010
1167
  }
1011
1168
  tooOldFor = null;
1012
1169
  }
1013
1170
  const login = await probeSpawn(exe, ["login", "status"]);
1014
1171
  if (login.outcome === "ok") {
1015
- return verdict({ ok: true }, exe, version);
1172
+ return { avail: verdict({ ok: true }, exe, version), exe, version };
1016
1173
  }
1017
1174
  if (login.outcome === "nonzero") {
1018
- return verdict({ ok: false, reason: NOT_AUTHENTICATED_REASON, code: "not-authenticated" }, exe, version);
1175
+ return {
1176
+ avail: verdict({ ok: false, reason: NOT_AUTHENTICATED_REASON, code: "not-authenticated" }, exe, version),
1177
+ exe,
1178
+ version
1179
+ };
1180
+ }
1181
+ return { avail: verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, version), exe, version };
1182
+ }
1183
+ async function connect(o = {}) {
1184
+ if (connectInFlight) return connectInFlight;
1185
+ const pending = (async () => {
1186
+ const { avail, exe, version } = await probeLadder();
1187
+ const fingerprint = fingerprintFor(exe, version);
1188
+ if (!o.force) {
1189
+ const fresh = freshConnection(exe, version);
1190
+ if (fresh) return fresh;
1191
+ }
1192
+ if (!avail.ok) {
1193
+ return { outcome: "unproven", at: Date.now(), fingerprint };
1194
+ }
1195
+ let result;
1196
+ try {
1197
+ await withWorkDir(
1198
+ (dir) => run2(execArgs(dir), void 0, {
1199
+ stdin: CONNECT_PROMPT,
1200
+ timeoutMs: connectTimeoutMs,
1201
+ label: "connect"
1202
+ })
1203
+ );
1204
+ result = { outcome: "proven", at: Date.now(), fingerprint };
1205
+ } catch (err) {
1206
+ const failure = classifyCodexFailure({
1207
+ text: err instanceof Error ? err.message : String(err),
1208
+ env: parentEnv,
1209
+ ignored: ignoreEnvKeys()
1210
+ });
1211
+ result = { outcome: outcomeFor(failure.code), failure, at: Date.now(), fingerprint };
1212
+ }
1213
+ connCache = result;
1214
+ cached = null;
1215
+ return result;
1216
+ })();
1217
+ connectInFlight = pending;
1218
+ try {
1219
+ return await pending;
1220
+ } finally {
1221
+ connectInFlight = null;
1019
1222
  }
1020
- return verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, version);
1223
+ }
1224
+ function noteConnection(outcome, failure) {
1225
+ if (outcome === "unproven") return;
1226
+ const exe = resolved;
1227
+ if (!exe) return;
1228
+ connCache = { outcome, failure, at: Date.now(), fingerprint: fingerprintFor(exe, knownVersion) };
1229
+ cached = null;
1021
1230
  }
1022
1231
  function invalidateProbe() {
1023
1232
  cached = null;
1024
1233
  resolved = null;
1025
1234
  }
1026
- return { run: run2, withWorkDir, probe, invalidateProbe };
1235
+ function invalidateConnection() {
1236
+ connCache = null;
1237
+ cached = null;
1238
+ }
1239
+ return { run: run2, withWorkDir, probe, invalidateProbe, connect, invalidateConnection, noteConnection };
1027
1240
  }
1028
1241
  var OUT_FILE = "analysis.json";
1029
1242
  function createCodexAnalyzer(opts = {}) {
@@ -1182,6 +1395,7 @@ function pick(v, allowed, max) {
1182
1395
  var INSTALL_COMMAND = "npm install -g @openai/codex";
1183
1396
  var INSTALL_DOCS_URL = "https://developers.openai.com/codex/cli";
1184
1397
  var INSTALL_COMMAND_SUDO = "sudo npm install -g @openai/codex";
1398
+ var INSTALL_COMMAND_WINDOWS = 'powershell -ExecutionPolicy ByPass -c "irm https://chatgpt.com/codex/install.ps1 | iex"';
1185
1399
  var DEFAULT_INSTALL_TIMEOUT_MS = 18e4;
1186
1400
  function stateFrom(avail) {
1187
1401
  if (avail.ok) return "ready";
@@ -1189,6 +1403,7 @@ function stateFrom(avail) {
1189
1403
  case "not-authenticated":
1190
1404
  case "update-needed":
1191
1405
  case "unverified":
1406
+ case "env-conflict":
1192
1407
  return avail.code;
1193
1408
  default:
1194
1409
  return "not-installed";
@@ -1199,6 +1414,7 @@ function createCodexSetup(opts = {}) {
1199
1414
  const platform = opts.platform ?? process.platform;
1200
1415
  const runner = opts.runner ?? createRunner(opts);
1201
1416
  const installTimeoutMs = opts.installTimeoutMs ?? DEFAULT_INSTALL_TIMEOUT_MS;
1417
+ const ignoreEnvKeys = opts.ignoreEnvKeys ?? (() => []);
1202
1418
  function run2(cmd, args, timeoutMs) {
1203
1419
  return new Promise((resolve) => {
1204
1420
  let settled = false;
@@ -1230,11 +1446,22 @@ function createCodexSetup(opts = {}) {
1230
1446
  });
1231
1447
  }
1232
1448
  return {
1233
- async status() {
1449
+ async status(o = {}) {
1234
1450
  runner.invalidateProbe();
1451
+ if (o.force) runner.invalidateConnection();
1452
+ const conn = await runner.connect();
1235
1453
  const avail = await runner.probe();
1454
+ const state = stateFrom(avail);
1236
1455
  const setupPlatform = platform === "win32" ? "windows" : platform === "darwin" ? "mac" : "linux";
1237
- return { state: stateFrom(avail), reason: avail.reason, platform: setupPlatform };
1456
+ return {
1457
+ state,
1458
+ reason: avail.reason,
1459
+ platform: setupPlatform,
1460
+ conflictKeys: state === "env-conflict" ? conn.failure?.conflictKeys ?? [] : [],
1461
+ ignoredKeys: [...ignoreEnvKeys()].filter(
1462
+ (k) => CONFLICT_ENV_KEYS.includes(k)
1463
+ )
1464
+ };
1238
1465
  },
1239
1466
  async install() {
1240
1467
  const res = await run2("npm", ["install", "-g", "@openai/codex"], installTimeoutMs);
@@ -1251,6 +1478,14 @@ function createCodexSetup(opts = {}) {
1251
1478
  }
1252
1479
  return { ok: true };
1253
1480
  }
1481
+ if (platform === "win32" && /EPERM|EACCES|EBUSY|permission denied|operation not permitted/i.test(res.stderr)) {
1482
+ return {
1483
+ ok: false,
1484
+ fallbackCommand: INSTALL_COMMAND_WINDOWS,
1485
+ docsUrl: INSTALL_DOCS_URL,
1486
+ detail: "npm could not write its files. This is almost never about administrator rights: close Codex and any terminal window using it, then try again. If it keeps failing, the command below is OpenAI\u2019s own installer, which does not go through npm."
1487
+ };
1488
+ }
1254
1489
  if (platform !== "win32" && /EACCES|permission denied/i.test(res.stderr)) {
1255
1490
  return {
1256
1491
  ok: false,
@@ -1452,6 +1687,7 @@ function createCodexEngine(opts) {
1452
1687
  fatal = err;
1453
1688
  inner.abort();
1454
1689
  runner.invalidateProbe();
1690
+ noteFromError(err);
1455
1691
  }
1456
1692
  }
1457
1693
  }
@@ -1514,7 +1750,10 @@ function createCodexEngine(opts) {
1514
1750
  try {
1515
1751
  await runCodex(args, signal, { stdin: promptText, label: `edit refs=${editRefs.length}` });
1516
1752
  } catch (err) {
1517
- if (isFatalSetupError(err)) runner.invalidateProbe();
1753
+ if (isFatalSetupError(err)) {
1754
+ runner.invalidateProbe();
1755
+ noteFromError(err);
1756
+ }
1518
1757
  throw err;
1519
1758
  }
1520
1759
  const images = await collectImages(dir, before);
@@ -1523,10 +1762,17 @@ function createCodexEngine(opts) {
1523
1762
  }
1524
1763
  };
1525
1764
  function isFatalSetupError(err) {
1526
- return /failed to spawn|ENOENT|not logged in|login required|401|unauthorized|is too old/i.test(
1765
+ return /failed to spawn|ENOENT|not logged in|login required|401|unauthorized|is too old|environment is overriding/i.test(
1527
1766
  String(err?.message ?? err)
1528
1767
  );
1529
1768
  }
1769
+ function noteFromError(err) {
1770
+ const failure = classifyCodexFailure({
1771
+ text: String(err?.message ?? err),
1772
+ env: process.env
1773
+ });
1774
+ runner.noteConnection(outcomeFor(failure.code), failure);
1775
+ }
1530
1776
  function buildPrompt2(req, index, roles) {
1531
1777
  const variation = req.variations?.[index] ?? "";
1532
1778
  const roleDirective = REFERENCE_ROLE_DIRECTIVE;
@@ -1566,9 +1812,15 @@ function createCodexEngine(opts) {
1566
1812
  function keyGetter(core, settingKey, envVar) {
1567
1813
  return () => core.store.getSetting(settingKey) || process.env[envVar] || null;
1568
1814
  }
1815
+ var IGNORE_ENV_KEYS_SETTING = "codex.ignore_env_keys";
1816
+ function ignoreEnvKeysGetter(core) {
1817
+ return () => (core.store.getSetting(IGNORE_ENV_KEYS_SETTING) ?? "").split(",").map((name) => name.trim().toUpperCase()).filter(
1818
+ (name) => CONFLICT_ENV_KEYS.includes(name)
1819
+ );
1820
+ }
1569
1821
  function createEngineRegistry(core, extra = []) {
1570
1822
  const saveImage = (buf) => core.images.save(buf);
1571
- const codexRunner = createRunner();
1823
+ const codexRunner = createRunner({ ignoreEnvKeys: ignoreEnvKeysGetter(core) });
1572
1824
  const adapters = [
1573
1825
  createOpenRouterEngine({ getKey: keyGetter(core, "openrouter_api_key", "OPENROUTER_API_KEY"), saveImage }),
1574
1826
  createReplicateEngine({ getKey: keyGetter(core, "replicate_api_token", "REPLICATE_API_TOKEN"), saveImage }),
@@ -1627,7 +1879,7 @@ function createDemoEngine(saveImage, opts = {}) {
1627
1879
  <text x="24" y="${h - 48}" font-family="Helvetica, Arial" font-size="${Math.max(14, Math.round(w / 42))}" fill="#ffffff" opacity="0.92">${esc(label)}</text>
1628
1880
  <text x="24" y="${h - 22}" font-family="Helvetica, Arial" font-size="12" fill="#ffffff" opacity="0.6">Scenri demo engine</text>
1629
1881
  </svg>`;
1630
- return sharp20(Buffer.from(svg)).png().toBuffer();
1882
+ return sharp21(Buffer.from(svg)).png().toBuffer();
1631
1883
  }
1632
1884
  return {
1633
1885
  capabilities() {
@@ -1818,7 +2070,7 @@ var resolvedRefs = /* @__PURE__ */ new Map();
1818
2070
  async function refHash(core, path) {
1819
2071
  const hit = resolvedRefs.get(path);
1820
2072
  if (hit && core.images.has(hit)) return hit;
1821
- const hash = core.images.save(await sharp20(readFileSync(path)).png().toBuffer());
2073
+ const hash = core.images.save(await sharp21(readFileSync(path)).png().toBuffer());
1822
2074
  resolvedRefs.set(path, hash);
1823
2075
  return hash;
1824
2076
  }
@@ -1922,7 +2174,7 @@ var resolvedRefs2 = /* @__PURE__ */ new Map();
1922
2174
  async function refHash2(core, path) {
1923
2175
  const hit = resolvedRefs2.get(path);
1924
2176
  if (hit && core.images.has(hit)) return hit;
1925
- const hash = core.images.save(await sharp20(readFileSync(path)).png().toBuffer());
2177
+ const hash = core.images.save(await sharp21(readFileSync(path)).png().toBuffer());
1926
2178
  resolvedRefs2.set(path, hash);
1927
2179
  return hash;
1928
2180
  }
@@ -2076,7 +2328,7 @@ function createThumbStore(core, opts = {}) {
2076
2328
  const pathFor = (key, w) => join(dir, `${key}-w${w}.webp`);
2077
2329
  const inflight = /* @__PURE__ */ new Map();
2078
2330
  const failed = /* @__PURE__ */ new Set();
2079
- const concurrency = Math.max(1, opts.concurrency ?? 2);
2331
+ const concurrency = Math.max(1, opts.concurrency ?? 4);
2080
2332
  let active = 0;
2081
2333
  const waiting = [];
2082
2334
  const acquire = () => new Promise((resolve) => {
@@ -2095,7 +2347,7 @@ function createThumbStore(core, opts = {}) {
2095
2347
  const tmp = `${final}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
2096
2348
  await acquire();
2097
2349
  try {
2098
- await sharp20(source).resize({ width: w, withoutEnlargement: true }).webp({ quality: QUALITY[w], effort: 4 }).toFile(tmp);
2350
+ await sharp21(source).resize({ width: w, withoutEnlargement: true }).webp({ quality: QUALITY[w], effort: 4 }).toFile(tmp);
2099
2351
  await rename(tmp, final);
2100
2352
  return final;
2101
2353
  } catch {
@@ -2185,7 +2437,7 @@ var assetHash = (ref) => {
2185
2437
  };
2186
2438
  var LOGO_ROLES = ["primary", "mark", "wordmark", "monochrome", "alternate"];
2187
2439
  var LOGO_BACKGROUNDS = ["light", "dark", "any"];
2188
- var toPng = (buf) => sharp20(buf).rotate().png().toBuffer();
2440
+ var toPng = (buf) => sharp21(buf).rotate().png().toBuffer();
2189
2441
  var COST_PROBE = {
2190
2442
  prompt: "",
2191
2443
  brand: { brand: {}, assetPaths: {} },
@@ -2198,11 +2450,11 @@ var MARK_MIN_EDGE = 1024;
2198
2450
  var MARK_TINY_EDGE = 256;
2199
2451
  var MARK_WARN_EDGE = 512;
2200
2452
  var toMarkPng = async (buf) => {
2201
- const out = await sharp20(buf, { density: 384 }).rotate().resize({ width: MARK_MAX_EDGE, height: MARK_MAX_EDGE, fit: "inside", withoutEnlargement: true }).png().toBuffer();
2202
- const meta = await sharp20(out).metadata();
2453
+ const out = await sharp21(buf, { density: 384 }).rotate().resize({ width: MARK_MAX_EDGE, height: MARK_MAX_EDGE, fit: "inside", withoutEnlargement: true }).png().toBuffer();
2454
+ const meta = await sharp21(out).metadata();
2203
2455
  const edge = Math.max(meta.width ?? 0, meta.height ?? 0);
2204
2456
  if (edge >= MARK_TINY_EDGE && edge < MARK_MIN_EDGE) {
2205
- return sharp20(out).resize({ width: MARK_MIN_EDGE, height: MARK_MIN_EDGE, fit: "inside", kernel: "lanczos3" }).png().toBuffer();
2457
+ return sharp21(out).resize({ width: MARK_MIN_EDGE, height: MARK_MIN_EDGE, fit: "inside", kernel: "lanczos3" }).png().toBuffer();
2206
2458
  }
2207
2459
  return out;
2208
2460
  };
@@ -2213,9 +2465,9 @@ async function capReferenceEdge(core, path, maxEdge) {
2213
2465
  if (hit) return hit;
2214
2466
  let out = path;
2215
2467
  try {
2216
- const meta = await sharp20(path).metadata();
2468
+ const meta = await sharp21(path).metadata();
2217
2469
  if ((meta.width ?? 0) > maxEdge || (meta.height ?? 0) > maxEdge) {
2218
- const buf = await sharp20(path).resize({ width: maxEdge, height: maxEdge, fit: "inside", withoutEnlargement: true }).png().toBuffer();
2470
+ const buf = await sharp21(path).resize({ width: maxEdge, height: maxEdge, fit: "inside", withoutEnlargement: true }).png().toBuffer();
2219
2471
  out = core.images.pathFor(core.images.save(buf));
2220
2472
  }
2221
2473
  } catch {
@@ -3491,13 +3743,122 @@ function validateBrand(json) {
3491
3743
  errors: valid ? [] : (compiled.errors ?? []).map((e) => `${e.instancePath || "/"} ${e.message}`)
3492
3744
  };
3493
3745
  }
3494
- var HEX_RE = /#[0-9a-fA-F]{6}\b/g;
3495
- var FETCH_TIMEOUT_MS = 15e3;
3746
+
3747
+ // ../brand-spec/src/colorParse.ts
3748
+ var CLAMP = (n) => Math.min(255, Math.max(0, Math.round(n)));
3749
+ var hex2 = (n) => CLAMP(n).toString(16).padStart(2, "0");
3750
+ var rgbHex = (r, g, b) => `#${hex2(r)}${hex2(g)}${hex2(b)}`;
3751
+ var MIN_ALPHA = 0.35;
3752
+ var NUM = String.raw`[-+]?[\d.]+%?`;
3753
+ var RGB_RE = new RegExp(String.raw`^rgba?\(\s*(${NUM})[\s,]+(${NUM})[\s,]+(${NUM})(?:[\s,/]+(${NUM}))?\s*\)$`, "i");
3754
+ var HSL_RE = new RegExp(
3755
+ String.raw`^hsla?\(\s*(${NUM}|[-+]?[\d.]+deg)[\s,]+(${NUM})[\s,]+(${NUM})(?:[\s,/]+(${NUM}))?\s*\)$`,
3756
+ "i"
3757
+ );
3758
+ var OKLCH_RE = new RegExp(
3759
+ String.raw`^oklch\(\s*(${NUM})[\s,]+(${NUM})[\s,]+([-+]?[\d.]+)(?:deg)?(?:[\s,/]+(${NUM}))?\s*\)$`,
3760
+ "i"
3761
+ );
3762
+ function scalar(token, full) {
3763
+ const t = token.trim();
3764
+ if (t.endsWith("%")) return Number.parseFloat(t) / 100 * full;
3765
+ return Number.parseFloat(t);
3766
+ }
3767
+ function alphaOk(token) {
3768
+ if (token == null) return true;
3769
+ const a = token.trim().endsWith("%") ? Number.parseFloat(token) / 100 : Number.parseFloat(token);
3770
+ return !Number.isNaN(a) && a >= MIN_ALPHA;
3771
+ }
3772
+ function toHex(value) {
3773
+ const v = value.trim().toLowerCase();
3774
+ const hex = /^#([0-9a-f]{3,8})$/.exec(v);
3775
+ if (hex) {
3776
+ const d = hex[1];
3777
+ if (d.length === 3 || d.length === 4) {
3778
+ if (d.length === 4 && !alphaOk(String(Number.parseInt(d[3] + d[3], 16) / 255))) return null;
3779
+ return `#${d[0]}${d[0]}${d[1]}${d[1]}${d[2]}${d[2]}`;
3780
+ }
3781
+ if (d.length === 6) return `#${d}`;
3782
+ if (d.length === 8) {
3783
+ if (!alphaOk(String(Number.parseInt(d.slice(6, 8), 16) / 255))) return null;
3784
+ return `#${d.slice(0, 6)}`;
3785
+ }
3786
+ return null;
3787
+ }
3788
+ const rgb = RGB_RE.exec(v);
3789
+ if (rgb) {
3790
+ if (!alphaOk(rgb[4])) return null;
3791
+ return rgbHex(scalar(rgb[1], 255), scalar(rgb[2], 255), scalar(rgb[3], 255));
3792
+ }
3793
+ const hsl = HSL_RE.exec(v);
3794
+ if (hsl) {
3795
+ if (!alphaOk(hsl[4])) return null;
3796
+ return hslHex(Number.parseFloat(hsl[1]), scalar(hsl[2], 1), scalar(hsl[3], 1));
3797
+ }
3798
+ const oklch = OKLCH_RE.exec(v);
3799
+ if (oklch) {
3800
+ if (!alphaOk(oklch[4])) return null;
3801
+ return oklchHex(scalar(oklch[1], 1), Number.parseFloat(oklch[2]), Number.parseFloat(oklch[3]));
3802
+ }
3803
+ return null;
3804
+ }
3805
+ function hslHex(h, s, l) {
3806
+ const c = (1 - Math.abs(2 * l - 1)) * s;
3807
+ const hp = (h % 360 + 360) % 360 / 60;
3808
+ const x = c * (1 - Math.abs(hp % 2 - 1));
3809
+ const [r, g, b] = hp < 1 ? [c, x, 0] : hp < 2 ? [x, c, 0] : hp < 3 ? [0, c, x] : hp < 4 ? [0, x, c] : hp < 5 ? [x, 0, c] : [c, 0, x];
3810
+ const m = l - c / 2;
3811
+ return rgbHex((r + m) * 255, (g + m) * 255, (b + m) * 255);
3812
+ }
3813
+ function oklchHex(L, C, hDeg) {
3814
+ const h = hDeg * Math.PI / 180;
3815
+ const a = C * Math.cos(h);
3816
+ const bb = C * Math.sin(h);
3817
+ const l_ = L + 0.3963377774 * a + 0.2158037573 * bb;
3818
+ const m_ = L - 0.1055613458 * a - 0.0638541728 * bb;
3819
+ const s_ = L - 0.0894841775 * a - 1.291485548 * bb;
3820
+ const l = l_ ** 3;
3821
+ const m = m_ ** 3;
3822
+ const s = s_ ** 3;
3823
+ const lr = 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s;
3824
+ const lg = -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s;
3825
+ const lb = -0.0041960863 * l - 0.7034186147 * m + 1.707614701 * s;
3826
+ const gamma = (u) => u <= 31308e-7 ? 12.92 * u : 1.055 * Math.abs(u) ** (1 / 2.4) - 0.055;
3827
+ return rgbHex(gamma(lr) * 255, gamma(lg) * 255, gamma(lb) * 255);
3828
+ }
3829
+ var DECL_RE = /(--[\w-]+|[\w-]+)\s*:\s*([^;{}]*?(?:#[0-9a-fA-F]{3,8}|rgba?\([^)]*\)|hsla?\([^)]*\)|oklch\([^)]*\))[^;{}]*)/g;
3830
+ var TOKEN_RE = /#[0-9a-fA-F]{3,8}\b|rgba?\([^)]*\)|hsla?\([^)]*\)|oklch\([^)]*\)/g;
3831
+ function extractColors(css) {
3832
+ const out = [];
3833
+ const blocks = css.split("}");
3834
+ for (const block of blocks) {
3835
+ const brace = block.lastIndexOf("{");
3836
+ const context = brace >= 0 ? block.slice(Math.max(0, brace - 200), brace) : "";
3837
+ const body = brace >= 0 ? block.slice(brace + 1) : block;
3838
+ DECL_RE.lastIndex = 0;
3839
+ let decl = DECL_RE.exec(body);
3840
+ while (decl !== null) {
3841
+ const property = decl[1];
3842
+ TOKEN_RE.lastIndex = 0;
3843
+ let token = TOKEN_RE.exec(decl[2]);
3844
+ while (token !== null) {
3845
+ const hex = toHex(token[0]);
3846
+ if (hex) out.push({ hex: hex.toLowerCase(), property, context });
3847
+ token = TOKEN_RE.exec(decl[2]);
3848
+ }
3849
+ decl = DECL_RE.exec(body);
3850
+ }
3851
+ }
3852
+ return out;
3853
+ }
3854
+
3855
+ // ../brand-spec/src/colors.ts
3496
3856
  function hexToHsl(hex) {
3497
- const r = parseInt(hex.slice(1, 3), 16) / 255;
3498
- const g = parseInt(hex.slice(3, 5), 16) / 255;
3499
- const b = parseInt(hex.slice(5, 7), 16) / 255;
3500
- const max = Math.max(r, g, b), min = Math.min(r, g, b);
3857
+ const r = Number.parseInt(hex.slice(1, 3), 16) / 255;
3858
+ const g = Number.parseInt(hex.slice(3, 5), 16) / 255;
3859
+ const b = Number.parseInt(hex.slice(5, 7), 16) / 255;
3860
+ const max = Math.max(r, g, b);
3861
+ const min = Math.min(r, g, b);
3501
3862
  const l = (max + min) / 2;
3502
3863
  if (max === min) return { h: 0, s: 0, l };
3503
3864
  const d = max - min;
@@ -3508,15 +3869,111 @@ function hexToHsl(hex) {
3508
3869
  else h = ((r - g) / d + 4) / 6;
3509
3870
  return { h: h * 360, s, l };
3510
3871
  }
3511
- function pickPalette(colors) {
3872
+ var BRANDISH_VAR = /--(?:[\w-]*)(brand|primary|accent|secondary|theme|main)/i;
3873
+ var FRAMEWORK_VAR = /^--(tw-|[\w-]*-(?:50|\d{3})$)/i;
3874
+ var UTILITY_CLASS = /\.(bg|text|border|fill|stroke|ring|from|via|to|shadow|outline|divide|accent|caret|decoration|placeholder)-[a-z]+-\d{2,3}\b/i;
3875
+ var INTERACTIVE = /(btn|button|cta|primary|brand|badge|header|nav\b|link|hero)/i;
3876
+ function weightOf(hit) {
3877
+ if (FRAMEWORK_VAR.test(hit.property) || UTILITY_CLASS.test(hit.context)) return 0;
3878
+ if (hit.property.startsWith("--")) return BRANDISH_VAR.test(hit.property) ? 6 : 2;
3879
+ if (INTERACTIVE.test(hit.context) || INTERACTIVE.test(hit.property)) return 3;
3880
+ return 1;
3881
+ }
3882
+ function liveClassTokens($) {
3883
+ const tokens = /* @__PURE__ */ new Set();
3884
+ for (const sel of ["html", "body"]) {
3885
+ for (const cls of ($(sel).first().attr("class") ?? "").split(/\s+/)) {
3886
+ if (cls) tokens.add(cls.toLowerCase());
3887
+ }
3888
+ }
3889
+ return [...tokens];
3890
+ }
3891
+ var familyOf = (cls) => cls.split("-")[0];
3892
+ function selectorIsLive(selector, live) {
3893
+ const sel = selector.toLowerCase();
3894
+ if (sel.includes(":root")) return true;
3895
+ const families = new Set(live.map(familyOf));
3896
+ return sel.split(",").some((branch) => {
3897
+ const positive = branch.replace(/:not\([^)]*\)/g, "").match(/\.[a-z0-9_-]+/g);
3898
+ if (!positive) return true;
3899
+ const leading = positive[0].slice(1);
3900
+ if (live.includes(leading)) return true;
3901
+ return !families.has(familyOf(leading));
3902
+ });
3903
+ }
3904
+ function roleOfProperty(property) {
3905
+ if (!property.startsWith("--")) return null;
3906
+ const name = property.slice(2).toLowerCase().replace(/^(brand|color|colour|theme|sc|c|ui)-/, "");
3907
+ const base = /^(primary|secondary|accent|dark|light)(?:-(\d+|dark|light|hover|active|soft|muted|contrast|foreground|fg|bg))?$/.exec(
3908
+ name
3909
+ );
3910
+ if (!base) return null;
3911
+ return { role: base[1], variant: Boolean(base[2]) };
3912
+ }
3913
+ function declaredPalette(hits, live) {
3914
+ const best = /* @__PURE__ */ new Map();
3915
+ for (const hit of hits) {
3916
+ const named = roleOfProperty(hit.property);
3917
+ if (!named || FRAMEWORK_VAR.test(hit.property) || !selectorIsLive(hit.context, live)) continue;
3918
+ const classes = hit.context.toLowerCase().match(/\.[a-z0-9_-]+/g) ?? [];
3919
+ const onLiveRoot = classes.some((c) => live.includes(c.slice(1)));
3920
+ const rank = (onLiveRoot ? 4 : classes.length > 0 ? 2 : 0) + (named.variant ? 0 : 1);
3921
+ const seen = best.get(named.role);
3922
+ if (seen && seen.rank > rank) continue;
3923
+ best.set(named.role, { hex: hit.hex, rank });
3924
+ }
3925
+ return {
3926
+ primary: best.get("primary")?.hex,
3927
+ secondary: best.get("secondary")?.hex,
3928
+ accent: best.get("accent")?.hex,
3929
+ dark: best.get("dark")?.hex,
3930
+ light: best.get("light")?.hex
3931
+ };
3932
+ }
3933
+ function collectColors(sources, live = []) {
3512
3934
  const counts = /* @__PURE__ */ new Map();
3513
- for (const c of colors) counts.set(c.toLowerCase(), (counts.get(c.toLowerCase()) ?? 0) + 1);
3514
- const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([c]) => c);
3935
+ for (const { css, weightScale = 1 } of sources) {
3936
+ for (const hit of extractColors(css)) {
3937
+ if (live.length > 0 && !selectorIsLive(hit.context, live)) continue;
3938
+ const weight = weightOf(hit) * weightScale;
3939
+ if (weight <= 0) continue;
3940
+ counts.set(hit.hex, (counts.get(hit.hex) ?? 0) + weight);
3941
+ }
3942
+ }
3943
+ return [...counts.entries()].map(([hex, weight]) => ({ hex, weight }));
3944
+ }
3945
+ function dedupeNearby(hits) {
3946
+ const buckets = /* @__PURE__ */ new Map();
3947
+ for (const hit of [...hits].sort((a, b) => b.weight - a.weight)) {
3948
+ const { h, s, l } = hexToHsl(hit.hex);
3949
+ const key = isNeutral(hit.hex) ? `grey:${Math.round(l * 8)}` : `${Math.round(h / 12)}:${Math.round(s * 8)}:${Math.round(l * 8)}`;
3950
+ const seen = buckets.get(key);
3951
+ if (seen) seen.weight += hit.weight;
3952
+ else buckets.set(key, { ...hit });
3953
+ }
3954
+ return [...buckets.values()].sort((a, b) => b.weight - a.weight);
3955
+ }
3956
+ function dropSingletons(hits) {
3957
+ if (hits.length < 12) return [...hits];
3958
+ const kept = hits.filter((h) => h.weight > 1);
3959
+ return kept.length > 0 ? kept : [...hits];
3960
+ }
3961
+ function chromaOf(hex) {
3962
+ const r = Number.parseInt(hex.slice(1, 3), 16) / 255;
3963
+ const g = Number.parseInt(hex.slice(3, 5), 16) / 255;
3964
+ const b = Number.parseInt(hex.slice(5, 7), 16) / 255;
3965
+ return Math.max(r, g, b) - Math.min(r, g, b);
3966
+ }
3967
+ function isNeutral(hex) {
3968
+ const { l } = hexToHsl(hex);
3969
+ return chromaOf(hex) < 0.1 || l < 0.06 || l > 0.96;
3970
+ }
3971
+ function pickPalette(hits) {
3972
+ const sorted = [...hits].sort((a, b) => b.weight - a.weight).map((h) => h.hex);
3515
3973
  const saturated = [];
3516
3974
  const neutrals = [];
3517
3975
  for (const c of sorted) {
3518
- const { s, l } = hexToHsl(c);
3519
- if (s < 0.12 || l < 0.06 || l > 0.96) neutrals.push(c);
3976
+ if (isNeutral(c)) neutrals.push(c);
3520
3977
  else saturated.push(c);
3521
3978
  }
3522
3979
  return {
@@ -3526,91 +3983,650 @@ function pickPalette(colors) {
3526
3983
  neutrals: neutrals.slice(0, 2)
3527
3984
  };
3528
3985
  }
3529
- async function buildFromUrl(url, opts = {}) {
3530
- const fetchImpl = opts.fetchImpl ?? fetch;
3531
- const warnings = [];
3532
- const origin = new URL(url);
3533
- const controller = new AbortController();
3534
- const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
3535
- let res;
3986
+ function paletteFrom(sources, live = []) {
3987
+ const hits = sources.flatMap((s) => extractColors(s.css));
3988
+ const declared = declaredPalette(hits, live);
3989
+ const counted = pickPalette(dropSingletons(dedupeNearby(collectColors(sources, live))));
3990
+ const taken = /* @__PURE__ */ new Set();
3991
+ const take = (hex) => {
3992
+ if (!hex || taken.has(hex)) return void 0;
3993
+ taken.add(hex);
3994
+ return hex;
3995
+ };
3996
+ const primary = take(declared.primary) ?? take(counted.primary);
3997
+ const secondary = take(declared.secondary) ?? take(counted.secondary);
3998
+ const named = Boolean(declared.primary || declared.secondary);
3999
+ const accent = (named ? [declared.accent, declared.dark, declared.light] : counted.accent).map(take).filter((h) => Boolean(h)).slice(0, 2);
4000
+ const neutrals = counted.neutrals.filter((h) => !taken.has(h)).slice(0, 2);
4001
+ return { primary, secondary, accent, neutrals };
4002
+ }
4003
+
4004
+ // ../brand-spec/src/logoCandidates.ts
4005
+ var BASE = {
4006
+ "json-ld": 170,
4007
+ manifest: 85,
4008
+ "apple-touch-icon": 80,
4009
+ "header-img": 75,
4010
+ "header-svg": 60,
4011
+ "link-icon": 45,
4012
+ "og-image": 10
4013
+ };
4014
+ var LOGOISH = /(^|[^a-z])(logo|wordmark|brandmark|lockup|masthead)([^a-z]|$)/i;
4015
+ var NOT_OURS = /(client|customer|partner|sponsor|press|award|integrat|collab|affiliat|featured-in|trusted-by|logos?-(wall|grid|cloud|strip)|testimonial|marquee|as-seen)/i;
4016
+ var SOCIAL = /(facebook|instagram|twitter|linkedin|youtube|tiktok|pinterest|threads|whatsapp|x-logo|social)/i;
4017
+ var FLAGGISH = /(\bflags?\b|locale|country|region|currency|\blang(uage)?\b)/i;
4018
+ var TRACKER = /(pixel|1x1|spacer|blank|beacon|analytics|doubleclick|facebook\.com\/tr)/i;
4019
+ var PHOTOISH = /(hero|banner|cover|photo|screenshot|slide|\bbg\b|background)/i;
4020
+ var DARKISH = /(dark|inverse|inverted|white|light-on)/i;
4021
+ var HEADER_SCOPE = 'header, nav, [role="banner"], .site-header, .navbar, .masthead, .topbar';
4022
+ var IMG_HORIZON = 40;
4023
+ function absolute(href, base) {
4024
+ if (!href) return null;
4025
+ const raw = href.trim();
4026
+ if (raw === "") return null;
4027
+ if (raw.startsWith("data:"))
4028
+ return raw.length <= 262144 && /^data:image\/(svg\+xml|png|jpeg|webp)/i.test(raw) ? raw : null;
3536
4029
  try {
3537
- res = await fetchImpl(url, {
3538
- redirect: "follow",
3539
- headers: { "user-agent": "scenri/0.1 (+https://scenri.co)" },
3540
- signal: controller.signal
4030
+ const u = new URL(raw, base);
4031
+ return u.protocol === "http:" || u.protocol === "https:" ? u.toString() : null;
4032
+ } catch {
4033
+ return null;
4034
+ }
4035
+ }
4036
+ function largestFromSrcset(srcset) {
4037
+ const entries = srcset.split(",").map((part) => part.trim()).filter(Boolean).map((part) => {
4038
+ const [url, descriptor = ""] = part.split(/\s+/);
4039
+ const w = /^(\d+)w$/.exec(descriptor);
4040
+ const x = /^([\d.]+)x$/.exec(descriptor);
4041
+ return { url, rank: w ? Number(w[1]) : x ? Number(x[1]) * 1e3 : 1 };
4042
+ });
4043
+ if (entries.length === 0) return null;
4044
+ return entries.sort((a, b) => b.rank - a.rank)[0].url;
4045
+ }
4046
+ function declaredEdgeOf(sizes, w, h) {
4047
+ const fromSizes = sizes ? Math.max(...(sizes.match(/\d+/g) ?? ["0"]).map(Number)) : 0;
4048
+ const fromAttrs = Math.max(Number(0) || 0, Number(0) || 0);
4049
+ const edge = Math.max(fromSizes, fromAttrs);
4050
+ return edge > 0 ? edge : null;
4051
+ }
4052
+ function layoutTerm(w, h, why) {
4053
+ const width = Number(w ?? 0) || 0;
4054
+ const height = Number(h ?? 0) || 0;
4055
+ if (width > 0 && height > 0 && width <= 32 && height <= 32) {
4056
+ why.push("icon-shaped");
4057
+ return -20;
4058
+ }
4059
+ return 0;
4060
+ }
4061
+ function sizeTerm(edge, why) {
4062
+ if (edge === null) return 0;
4063
+ if (edge >= 512) {
4064
+ why.push("large");
4065
+ return 15;
4066
+ }
4067
+ if (edge >= 256) {
4068
+ why.push("big enough");
4069
+ return 8;
4070
+ }
4071
+ if (edge <= 64) {
4072
+ why.push("favicon-sized");
4073
+ return -25;
4074
+ }
4075
+ return 0;
4076
+ }
4077
+ function logoCandidates($, base, manifest) {
4078
+ const out = [];
4079
+ const host = base.hostname.replace(/^www\./, "").split(".")[0];
4080
+ const push = (c) => {
4081
+ if (c.score <= 0) return;
4082
+ if (c.url && out.some((o) => o.url === c.url)) return;
4083
+ out.push(c);
4084
+ };
4085
+ for (const node of $('script[type="application/ld+json"]').toArray()) {
4086
+ for (const href of jsonLdLogos($(node).text())) {
4087
+ const url = absolute(href, base);
4088
+ if (url)
4089
+ push({
4090
+ url,
4091
+ source: "json-ld",
4092
+ score: BASE["json-ld"],
4093
+ declaredEdge: null,
4094
+ role: "primary",
4095
+ why: ["declared as the organisation logo"]
4096
+ });
4097
+ }
4098
+ }
4099
+ for (const icon of manifest?.icons ?? []) {
4100
+ const url = absolute(icon.src, base);
4101
+ const edge = declaredEdgeOf(icon.sizes);
4102
+ if (!url || (edge ?? 0) < 192) continue;
4103
+ const why = ["from the web app manifest"];
4104
+ push({
4105
+ url,
4106
+ source: "manifest",
4107
+ score: BASE.manifest + sizeTerm(edge, why),
4108
+ declaredEdge: edge,
4109
+ role: "primary",
4110
+ why
3541
4111
  });
3542
- } catch (err) {
3543
- if (err instanceof Error && err.name === "AbortError") {
3544
- throw new Error(`Timed out fetching ${url}`);
4112
+ }
4113
+ for (const el of $('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').toArray()) {
4114
+ const url = absolute($(el).attr("href"), base);
4115
+ const edge = declaredEdgeOf($(el).attr("sizes"));
4116
+ const why = ["a touch icon"];
4117
+ if (url)
4118
+ push({
4119
+ url,
4120
+ source: "apple-touch-icon",
4121
+ score: BASE["apple-touch-icon"] + sizeTerm(edge, why),
4122
+ declaredEdge: edge,
4123
+ role: "primary",
4124
+ why
4125
+ });
4126
+ }
4127
+ const imgs = $("img").toArray();
4128
+ imgs.forEach((el, index) => {
4129
+ const node = $(el);
4130
+ const inHeader = node.closest(HEADER_SCOPE).length > 0;
4131
+ if (!inHeader && index > IMG_HORIZON) return;
4132
+ const src = node.attr("src") ?? node.attr("data-src") ?? node.attr("data-lazy-src") ?? largestFromSrcset(node.attr("srcset") ?? node.attr("data-srcset") ?? "") ?? void 0;
4133
+ const url = absolute(src, base);
4134
+ if (!url) return;
4135
+ const text = [url, node.attr("alt") ?? "", node.attr("class") ?? "", node.attr("id") ?? ""].join(" ");
4136
+ const ancestry = node.parents().slice(0, 6).map((_i, p) => `${$(p).attr("class") ?? ""} ${$(p).attr("id") ?? ""}`).get().join(" ");
4137
+ const why = [];
4138
+ let score = inHeader ? BASE["header-img"] : BASE["header-img"] - 20;
4139
+ if (inHeader) why.push("in the header");
4140
+ if (index <= 2) {
4141
+ why.push("first image on the page");
4142
+ score += 20;
4143
+ } else if (index <= 6) {
4144
+ score += 6;
4145
+ }
4146
+ score += layoutTerm(node.attr("width"), node.attr("height"), why);
4147
+ score += semanticTerms(text, ancestry, host, why);
4148
+ if (node.parent().children("img").length >= 3) {
4149
+ why.push("one of a row of logos");
4150
+ score -= 50;
4151
+ }
4152
+ push({
4153
+ url,
4154
+ source: "header-img",
4155
+ score,
4156
+ declaredEdge: null,
4157
+ role: "primary",
4158
+ ...DARKISH.test(text) ? { background: "dark" } : {},
4159
+ why
4160
+ });
4161
+ });
4162
+ for (const el of $(`${HEADER_SCOPE}`).find("svg").toArray().slice(0, 4)) {
4163
+ const node = $(el);
4164
+ const text = [node.attr("class") ?? "", node.attr("id") ?? "", node.attr("aria-label") ?? ""].join(" ");
4165
+ const why = ["drawn in the header"];
4166
+ const score = BASE["header-svg"] + semanticTerms(text, "", host, why);
4167
+ const markup = $.html(el);
4168
+ if (markup) push({ url: null, svg: markup, source: "header-svg", score, declaredEdge: null, role: "primary", why });
4169
+ }
4170
+ for (const el of $('link[rel~="icon"]').toArray()) {
4171
+ const url = absolute($(el).attr("href"), base);
4172
+ const edge = declaredEdgeOf($(el).attr("sizes"));
4173
+ const why = ["the site icon"];
4174
+ if (url)
4175
+ push({
4176
+ url,
4177
+ source: "link-icon",
4178
+ score: BASE["link-icon"] + sizeTerm(edge, why),
4179
+ declaredEdge: edge,
4180
+ role: "primary",
4181
+ why
4182
+ });
4183
+ }
4184
+ const og = absolute($('meta[property="og:image"]').attr("content"), base);
4185
+ if (og)
4186
+ push({
4187
+ url: og,
4188
+ source: "og-image",
4189
+ score: BASE["og-image"],
4190
+ declaredEdge: null,
4191
+ role: "alternate",
4192
+ why: ["the social share image"]
4193
+ });
4194
+ return out.sort((a, b) => b.score - a.score);
4195
+ }
4196
+ function pathOf(text) {
4197
+ return text.replace(/https?:\/\/[^/\s]+/gi, "");
4198
+ }
4199
+ function semanticTerms(text, ancestry, host, why) {
4200
+ let score = 0;
4201
+ if (LOGOISH.test(text)) {
4202
+ why.push("called a logo");
4203
+ score += 40;
4204
+ }
4205
+ if (host.length >= 3 && pathOf(text).toLowerCase().includes(host.toLowerCase())) {
4206
+ why.push("named after the site");
4207
+ score += 6;
4208
+ }
4209
+ if (/\.svg(\?|$)/i.test(text)) score += 8;
4210
+ if (/\.ico(\?|$)/i.test(text)) score -= 15;
4211
+ if (/\.jpe?g(\?|$)/i.test(text)) score -= 10;
4212
+ if (NOT_OURS.test(ancestry) || NOT_OURS.test(text)) {
4213
+ why.push("inside a section of other companies");
4214
+ score -= 40;
4215
+ }
4216
+ if (SOCIAL.test(text)) {
4217
+ why.push("a social icon");
4218
+ score -= 60;
4219
+ }
4220
+ if (FLAGGISH.test(text)) {
4221
+ why.push("a flag or a region switcher");
4222
+ score -= 60;
4223
+ }
4224
+ if (TRACKER.test(text)) {
4225
+ why.push("a tracking pixel");
4226
+ score -= 80;
4227
+ }
4228
+ if (PHOTOISH.test(text)) {
4229
+ why.push("photography");
4230
+ score -= 30;
4231
+ }
4232
+ if (DARKISH.test(text)) score -= 15;
4233
+ return score;
4234
+ }
4235
+ function jsonLdLogos(text) {
4236
+ let data;
4237
+ try {
4238
+ data = JSON.parse(text);
4239
+ } catch {
4240
+ return [];
4241
+ }
4242
+ const found = [];
4243
+ const walk = (node, depth) => {
4244
+ if (depth > 6 || node === null || typeof node !== "object") return;
4245
+ if (Array.isArray(node)) {
4246
+ for (const item of node) walk(item, depth + 1);
4247
+ return;
3545
4248
  }
3546
- throw new Error(`Could not reach that site: ${url}`);
3547
- } finally {
3548
- clearTimeout(timeoutId);
4249
+ const obj = node;
4250
+ const logo = obj.logo;
4251
+ if (typeof logo === "string") found.push(logo);
4252
+ else if (logo && typeof logo === "object") {
4253
+ const url = logo.url;
4254
+ if (typeof url === "string") found.push(url);
4255
+ }
4256
+ for (const value of Object.values(obj)) walk(value, depth + 1);
4257
+ };
4258
+ walk(data, 0);
4259
+ return found;
4260
+ }
4261
+ function svgAsMark(markup, longEdge = 1024) {
4262
+ if (/<script|<foreignObject|(?:xlink:)?href\s*=\s*["']https?:/i.test(markup)) return null;
4263
+ const viewBox = /viewBox\s*=\s*["']\s*[-\d.]+[\s,]+[-\d.]+[\s,]+([\d.]+)[\s,]+([\d.]+)\s*["']/i.exec(markup);
4264
+ const width = /\bwidth\s*=\s*["']([\d.]+)/i.exec(markup);
4265
+ const height = /\bheight\s*=\s*["']([\d.]+)/i.exec(markup);
4266
+ const w = viewBox ? Number(viewBox[1]) : Number(width?.[1] ?? 0);
4267
+ const h = viewBox ? Number(viewBox[2]) : Number(height?.[1] ?? 0);
4268
+ if (!(w > 0 && h > 0)) return null;
4269
+ const scale = longEdge / Math.max(w, h);
4270
+ let out = markup.replace(/\s\bwidth\s*=\s*["'][^"']*["']/i, "").replace(/\s\bheight\s*=\s*["'][^"']*["']/i, "").replace(/<svg\b/i, `<svg width="${Math.round(w * scale)}" height="${Math.round(h * scale)}"`);
4271
+ if (!/xmlns\s*=/.test(out)) out = out.replace(/<svg\b/i, '<svg xmlns="http://www.w3.org/2000/svg"');
4272
+ return out;
4273
+ }
4274
+
4275
+ // ../brand-spec/src/scrapeError.ts
4276
+ var ScrapeError = class extends Error {
4277
+ constructor(code, message, statusCode = 502) {
4278
+ super(message);
4279
+ this.code = code;
4280
+ this.statusCode = statusCode;
4281
+ }
4282
+ code;
4283
+ statusCode;
4284
+ name = "ScrapeError";
4285
+ };
4286
+ function urlRefusal(reason, message) {
4287
+ return new ScrapeError(`url_${reason}`, message, 400);
4288
+ }
4289
+
4290
+ // ../brand-spec/src/safeFetch.ts
4291
+ var DEFAULTS = {
4292
+ budgetMs: 2e4,
4293
+ requestMs: 8e3,
4294
+ maxRedirects: 3,
4295
+ maxHtmlBytes: 3e6,
4296
+ maxCssBytes: 1e6,
4297
+ maxAssetBytes: 5e6
4298
+ };
4299
+ function isPrivateAddress(ip) {
4300
+ const kind = isIP(ip);
4301
+ if (kind === 4) return isPrivateV4(ip);
4302
+ if (kind === 6) return isPrivateV6(ip.toLowerCase());
4303
+ return true;
4304
+ }
4305
+ function isPrivateV4(ip) {
4306
+ const p = ip.split(".").map(Number);
4307
+ if (p.length !== 4 || p.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true;
4308
+ const [a, b] = p;
4309
+ if (a === 0 || a === 10 || a === 127) return true;
4310
+ if (a === 100 && b >= 64 && b <= 127) return true;
4311
+ if (a === 169 && b === 254) return true;
4312
+ if (a === 172 && b >= 16 && b <= 31) return true;
4313
+ if (a === 192 && b === 168) return true;
4314
+ if (a === 192 && b === 0) return true;
4315
+ if (a === 192 && b === 88) return true;
4316
+ if (a === 198 && (b === 18 || b === 19 || b === 51)) return true;
4317
+ if (a === 203 && b === 0) return true;
4318
+ if (a >= 224) return true;
4319
+ return false;
4320
+ }
4321
+ function isPrivateV6(ip) {
4322
+ const bare = ip.replace(/^\[|\]$/g, "");
4323
+ if (bare === "::" || bare === "::1") return true;
4324
+ const mapped = /^(?:::ffff:|64:ff9b::)(\d{1,3}(?:\.\d{1,3}){3})$/.exec(bare);
4325
+ if (mapped) return isPrivateV4(mapped[1]);
4326
+ const head = bare.split(":")[0];
4327
+ if (/^f[cd]/.test(head)) return true;
4328
+ if (/^fe[89ab]/.test(head)) return true;
4329
+ if (/^ff/.test(head)) return true;
4330
+ if (head === "2002") return true;
4331
+ return false;
4332
+ }
4333
+ var PRIVATE_SUFFIX = /(^|\.)(local|internal|localhost|home\.arpa)$/i;
4334
+ function assertPublicHost(host, addresses, allow) {
4335
+ if (allow) return;
4336
+ const bare = host.replace(/^\[|\]$/g, "");
4337
+ if (isIP(bare)) {
4338
+ if (isPrivateAddress(bare)) throw blocked(host);
4339
+ return;
4340
+ }
4341
+ if (!bare.includes(".") || PRIVATE_SUFFIX.test(bare)) throw blocked(host);
4342
+ if (addresses.length === 0) throw new ScrapeError("unreachable", `Could not reach ${host}.`);
4343
+ if (addresses.some((a) => isPrivateAddress(a))) throw blocked(host);
4344
+ }
4345
+ var blocked = (host) => new ScrapeError("blocked_host", `${host} points at this computer or a private network, not a public website.`, 400);
4346
+ var LIMIT = {
4347
+ html: "maxHtmlBytes",
4348
+ css: "maxCssBytes",
4349
+ asset: "maxAssetBytes"
4350
+ };
4351
+ var TRUNCATES = { html: true, css: true, asset: false };
4352
+ function createGuardedFetch(opts = {}) {
4353
+ const o = { ...DEFAULTS, ...opts };
4354
+ const fetchImpl = opts.fetchImpl ?? fetch;
4355
+ const resolve = opts.lookup ?? (async (host) => {
4356
+ if (isIP(host.replace(/^\[|\]$/g, ""))) return [host.replace(/^\[|\]$/g, "")];
4357
+ const all = await lookup(host, { all: true });
4358
+ return all.map((a) => a.address);
4359
+ });
4360
+ const deadline = Date.now() + o.budgetMs;
4361
+ const remaining = () => Math.max(0, deadline - Date.now());
4362
+ async function once(url, as) {
4363
+ let current = url;
4364
+ let hops = 0;
4365
+ let attempts = 0;
4366
+ for (; ; ) {
4367
+ const u = new URL(current);
4368
+ if (u.protocol !== "http:" && u.protocol !== "https:")
4369
+ throw new ScrapeError(
4370
+ "url_scheme",
4371
+ "Scenri reads web addresses only, the ones that start with http or https.",
4372
+ 400
4373
+ );
4374
+ let addresses = [];
4375
+ if (!o.allowPrivateHosts) {
4376
+ try {
4377
+ addresses = await resolve(u.hostname);
4378
+ } catch {
4379
+ throw new ScrapeError("unreachable", `Could not reach ${u.hostname}.`);
4380
+ }
4381
+ }
4382
+ assertPublicHost(u.hostname, addresses, Boolean(o.allowPrivateHosts));
4383
+ const left = remaining();
4384
+ if (left <= 0) throw new ScrapeError("timeout", "Reading that site took too long.");
4385
+ const signal = AbortSignal.any([AbortSignal.timeout(Math.min(o.requestMs, left))]);
4386
+ let res;
4387
+ try {
4388
+ res = await fetchImpl(current, {
4389
+ redirect: "manual",
4390
+ headers: { "user-agent": USER_AGENT, accept: ACCEPT[as] },
4391
+ signal
4392
+ });
4393
+ } catch (err) {
4394
+ if (err instanceof ScrapeError) throw err;
4395
+ const aborted = err instanceof Error && (err.name === "AbortError" || err.name === "TimeoutError");
4396
+ throw aborted ? new ScrapeError("timeout", `${u.hostname} took too long to answer.`) : new ScrapeError("unreachable", `Could not reach ${u.hostname}.`);
4397
+ }
4398
+ if (res.status >= 300 && res.status < 400) {
4399
+ const location = res.headers.get("location");
4400
+ await res.body?.cancel().catch(() => {
4401
+ });
4402
+ if (!location) throw new ScrapeError("http_status", `${u.hostname} answered ${res.status} with nowhere to go.`);
4403
+ if (++hops > o.maxRedirects)
4404
+ throw new ScrapeError(
4405
+ "too_many_redirects",
4406
+ `${new URL(url).hostname} kept redirecting, so nothing was read.`
4407
+ );
4408
+ current = new URL(location, current).toString();
4409
+ continue;
4410
+ }
4411
+ if (!res.ok) {
4412
+ if (RETRYABLE.has(res.status) && attempts < 1 && remaining() > 2e3) {
4413
+ await res.body?.cancel().catch(() => {
4414
+ });
4415
+ await sleep3(700);
4416
+ attempts++;
4417
+ continue;
4418
+ }
4419
+ throw new ScrapeError("http_status", statusSentence(u.hostname, res.status));
4420
+ }
4421
+ const contentType = (res.headers.get("content-type") ?? "").toLowerCase();
4422
+ const cap2 = o[LIMIT[as]];
4423
+ const { bytes, truncated } = await readBounded(res, cap2, TRUNCATES[as]);
4424
+ assertKind(as, contentType, bytes, u.hostname);
4425
+ return {
4426
+ finalUrl: res.url || current,
4427
+ status: res.status,
4428
+ contentType,
4429
+ bytes,
4430
+ text: as === "asset" ? "" : decode(bytes, contentType),
4431
+ truncated
4432
+ };
4433
+ }
4434
+ }
4435
+ const guarded = ((url, as) => once(url, as));
4436
+ guarded.remaining = remaining;
4437
+ return guarded;
4438
+ }
4439
+ var USER_AGENT = "scenri/0.1 (+https://scenri.co)";
4440
+ var sleep3 = (ms) => new Promise((r) => setTimeout(r, ms));
4441
+ var RETRYABLE = /* @__PURE__ */ new Set([403, 408, 425, 429, 500, 502, 503, 504]);
4442
+ function statusSentence(host, status) {
4443
+ if (status === 401 || status === 403)
4444
+ return `${host} would not let Scenri read it. Some sites block anything that is not a person in a browser; you can still add the logo and colours by hand.`;
4445
+ if (status === 404) return `There is no page at that address on ${host}.`;
4446
+ if (status === 429) return `${host} asked Scenri to slow down. Try again in a minute.`;
4447
+ if (status >= 500) return `${host} had trouble answering. Try again in a moment.`;
4448
+ return `${host} answered ${status}, so there was nothing to read.`;
4449
+ }
4450
+ var ACCEPT = {
4451
+ html: "text/html,application/xhtml+xml",
4452
+ css: "text/css,*/*;q=0.1",
4453
+ asset: "image/*"
4454
+ };
4455
+ async function readBounded(res, cap2, truncate) {
4456
+ const declared = Number(res.headers.get("content-length") ?? "0");
4457
+ if (!truncate && declared > cap2) {
4458
+ await res.body?.cancel().catch(() => {
4459
+ });
4460
+ throw new ScrapeError("too_large", "That file is too large to read.");
4461
+ }
4462
+ const reader = res.body?.getReader();
4463
+ if (!reader) return { bytes: Buffer.from(await res.arrayBuffer()), truncated: false };
4464
+ const chunks = [];
4465
+ let total = 0;
4466
+ for (; ; ) {
4467
+ const { done, value } = await reader.read();
4468
+ if (done) break;
4469
+ chunks.push(Buffer.from(value));
4470
+ total += value.byteLength;
4471
+ if (total >= cap2) {
4472
+ await reader.cancel().catch(() => {
4473
+ });
4474
+ if (!truncate) throw new ScrapeError("too_large", "That file is too large to read.");
4475
+ return { bytes: Buffer.concat(chunks).subarray(0, cap2), truncated: true };
4476
+ }
4477
+ }
4478
+ return { bytes: Buffer.concat(chunks), truncated: false };
4479
+ }
4480
+ var IMAGE_MAGIC = [
4481
+ ["png", (b) => b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))],
4482
+ ["jpeg", (b) => b[0] === 255 && b[1] === 216],
4483
+ ["gif", (b) => b.subarray(0, 3).toString("latin1") === "GIF"],
4484
+ ["webp", (b) => b.subarray(0, 4).toString("latin1") === "RIFF" && b.subarray(8, 12).toString("latin1") === "WEBP"],
4485
+ ["ico", (b) => b[0] === 0 && b[1] === 0 && (b[2] === 1 || b[2] === 2)],
4486
+ ["svg", (b) => /<svg[\s>]/i.test(b.subarray(0, 1024).toString("utf8"))]
4487
+ ];
4488
+ function assertKind(as, contentType, bytes, host) {
4489
+ if (as === "html") {
4490
+ const htmlish = /text\/html|application\/xhtml/.test(contentType);
4491
+ const vague = contentType === "" || /^text\/plain|^application\/octet-stream/.test(contentType);
4492
+ const looksLikeMarkup = vague && bytes.subarray(0, 512).toString("utf8").trimStart().startsWith("<");
4493
+ if (!htmlish && !looksLikeMarkup)
4494
+ throw new ScrapeError("not_a_page", `${host} answered with a file, not a web page.`);
4495
+ return;
4496
+ }
4497
+ if (as === "asset") {
4498
+ if (contentType.startsWith("image/")) return;
4499
+ const vague = contentType === "" || /^application\/octet-stream/.test(contentType);
4500
+ if (vague || IMAGE_MAGIC.some(([, test]) => test(bytes))) return;
4501
+ throw new ScrapeError("not_a_page", "That logo was not an image.");
4502
+ }
4503
+ }
4504
+ function decode(bytes, contentType) {
4505
+ const declared = /charset=([\w-]+)/i.exec(contentType)?.[1];
4506
+ const sniffed = declared ?? /charset=["']?([\w-]+)/i.exec(bytes.subarray(0, 1024).toString("latin1"))?.[1];
4507
+ const label = (sniffed ?? "utf-8").toLowerCase();
4508
+ if (label === "utf-8" || label === "utf8") return bytes.toString("utf8");
4509
+ try {
4510
+ return new TextDecoder(label, { fatal: false }).decode(bytes);
4511
+ } catch {
4512
+ return bytes.toString("utf8");
4513
+ }
4514
+ }
4515
+
4516
+ // ../brand-spec/src/siteUrl.ts
4517
+ var INVISIBLE = /[​-‍⁠­᠎‎‏‪-‮⁦-⁩]/g;
4518
+ var SPACES = /[\t\n\r\f\v   -    ]/g;
4519
+ var SCHEMED = /^[a-z][a-z0-9+.-]*:\/\//i;
4520
+ var OPAQUE = /^(javascript|data|mailto|tel|sms|file|about|blob|chrome|chrome-extension|view-source):/i;
4521
+ var HTTPISH = /^https?:\/\//i;
4522
+ var IP_LITERAL = /^(\d{1,3}(\.\d{1,3}){3}|\[[0-9a-f:]+\])$/i;
4523
+ var EXAMPLE = "acme.com";
4524
+ var MESSAGES = {
4525
+ empty: `Paste a website address, like ${EXAMPLE}.`,
4526
+ scheme: `Scenri reads web addresses only, the ones that start with http or https. Try ${EXAMPLE}.`,
4527
+ space: "A web address cannot contain a space.",
4528
+ host: `That does not look like a website address. Try ${EXAMPLE}.`,
4529
+ unparseable: `That does not look like a website address. Try ${EXAMPLE}.`
4530
+ };
4531
+ var no = (reason, suggestion) => ({
4532
+ ok: false,
4533
+ reason,
4534
+ message: suggestion && reason === "space" ? `${MESSAGES.space} Did you mean ${suggestion}?` : MESSAGES[reason],
4535
+ ...suggestion ? { suggestion } : {}
4536
+ });
4537
+ function unwrap(value) {
4538
+ const pairs = [
4539
+ ["<", ">"],
4540
+ ['"', '"'],
4541
+ ["'", "'"]
4542
+ ];
4543
+ for (const [open, close] of pairs) {
4544
+ if (value.length >= 2 && value.startsWith(open) && value.endsWith(close)) return value.slice(1, -1).trim();
4545
+ }
4546
+ return value;
4547
+ }
4548
+ function normalizeSiteUrl(input) {
4549
+ const cleaned = unwrap(
4550
+ String(input ?? "").replace(INVISIBLE, "").replace(SPACES, " ").trim()
4551
+ );
4552
+ if (cleaned === "") return no("empty");
4553
+ const addedScheme = !SCHEMED.test(cleaned) && !OPAQUE.test(cleaned);
4554
+ if (!addedScheme && !HTTPISH.test(cleaned)) return no("scheme");
4555
+ const candidate = addedScheme ? `https://${cleaned}` : cleaned;
4556
+ const authority = candidate.slice(candidate.indexOf("://") + 3).split(/[/?#]/)[0];
4557
+ if (authority.includes(" ")) {
4558
+ const squeezed = normalizeSiteUrl(cleaned.replace(/ /g, ""));
4559
+ return no("space", squeezed.ok ? squeezed.url.replace(/^https?:\/\//, "").replace(/\/$/, "") : void 0);
4560
+ }
4561
+ let u;
4562
+ try {
4563
+ u = new URL(candidate);
4564
+ } catch {
4565
+ return no("unparseable");
3549
4566
  }
3550
- if (!res.ok) throw new Error(`Could not fetch ${url}: HTTP ${res.status}`);
3551
- const html = await res.text();
3552
- const $ = cheerio.load(html);
3553
- const name = $('meta[property="og:site_name"]').attr("content")?.trim() || $("title").first().text().trim().split(/\s+[|–—-]\s+/)[0] || origin.hostname.replace(/^www\./, "");
4567
+ u.username = "";
4568
+ u.password = "";
4569
+ u.hash = "";
4570
+ const host = u.hostname.replace(/\.$/, "");
4571
+ u.hostname = host;
4572
+ if (!host.includes(".") && !IP_LITERAL.test(host) && host !== "localhost") return no("host");
4573
+ return { ok: true, url: u.toString(), host, addedScheme };
4574
+ }
4575
+
4576
+ // ../brand-spec/src/buildFromUrl.ts
4577
+ var MAX_PORTRAIT_RATIO = 2.5;
4578
+ var CONFIDENT_SCORE = 80;
4579
+ var ICON_SOURCES = /* @__PURE__ */ new Set(["json-ld", "manifest", "apple-touch-icon", "link-icon"]);
4580
+ var LOGO_TRIES = 3;
4581
+ var MAX_SHEETS = 6;
4582
+ var SHEET_RANK = /(root|variables|tokens|colou?rs|palette|theme|main|app|style|tailwind|global|site|brand|base)/i;
4583
+ var SHEET_SKIP = /(font|icon|fontawesome|bootstrap-icons|swiper|slick|slider|lightbox|print)/i;
4584
+ async function buildFromUrl(url, opts = {}) {
4585
+ const warnings = [];
4586
+ const normalized = normalizeSiteUrl(url);
4587
+ if (!normalized.ok) throw urlRefusal(normalized.reason, normalized.message);
4588
+ const get = createGuardedFetch({
4589
+ ...opts.guard,
4590
+ ...opts.fetchImpl ? { fetchImpl: opts.fetchImpl, allowPrivateHosts: true } : {}
4591
+ });
4592
+ const page = await get(normalized.url, "html");
4593
+ const origin = new URL(page.finalUrl);
4594
+ const $ = cheerio.load(page.text);
4595
+ const base = baseOf($, origin);
4596
+ const named = pickName($, origin);
3554
4597
  const tagline = $('meta[name="description"]').attr("content")?.trim() || $('meta[property="og:description"]').attr("content")?.trim();
3555
- const colorSources = [];
3556
- const themeColor = $('meta[name="theme-color"]').attr("content");
3557
- if (themeColor && /^#[0-9a-fA-F]{6}$/.test(themeColor.trim())) {
3558
- for (let i = 0; i < 5; i++) colorSources.push(themeColor.trim());
3559
- }
3560
- const styleText = $("[style]").map((_, el) => $(el).attr("style")).get().join("\n") + "\n" + $("style").map((_, el) => $(el).text()).get().join("\n");
3561
- colorSources.push(...styleText.match(HEX_RE) ?? []);
3562
- const cssHref = $('link[rel="stylesheet"]').first().attr("href");
3563
- if (cssHref) {
4598
+ const sources = [];
4599
+ const themeColor = $('meta[name="theme-color"]').attr("content")?.trim();
4600
+ const tileColor = $('meta[name="msapplication-TileColor"]').attr("content")?.trim();
4601
+ for (const [value, scale] of [
4602
+ [themeColor, 2],
4603
+ [tileColor, 1]
4604
+ ]) {
4605
+ if (value) sources.push({ css: `:root{--brand-theme:${value}}`, weightScale: scale });
4606
+ }
4607
+ sources.push({ css: inlineCss($) });
4608
+ for (const href of sheetHrefs($, base)) {
4609
+ if (get.remaining() <= 0) break;
3564
4610
  try {
3565
- const cssUrl = new URL(cssHref, origin).toString();
3566
- const cssRes = await fetchImpl(cssUrl, { redirect: "follow" });
3567
- if (cssRes.ok) colorSources.push(...(await cssRes.text()).match(HEX_RE) ?? []);
4611
+ const sheet = await get(href, "css");
4612
+ sources.push({ css: sheet.text });
3568
4613
  } catch {
3569
4614
  warnings.push("Stylesheet fetch failed; palette from inline styles only.");
3570
4615
  }
3571
4616
  }
3572
- const palette = pickPalette(colorSources);
4617
+ const palette = paletteFrom(sources, liveClassTokens($));
3573
4618
  if (!palette.primary) warnings.push("No confident palette found. Set colors manually.");
3574
- let logoRef;
3575
- let logoFromOg = false;
3576
- let logoTiny = false;
3577
- let iconHref = $('link[rel="apple-touch-icon"]').attr("href") || $('link[rel~="icon"]').attr("href");
3578
- if (!iconHref) {
3579
- iconHref = $('meta[property="og:image"]').attr("content");
3580
- logoFromOg = Boolean(iconHref);
3581
- }
3582
- if (iconHref && opts.saveAsset) {
3583
- try {
3584
- const iconRes = await fetchImpl(new URL(iconHref, origin).toString(), { redirect: "follow" });
3585
- if (iconRes.ok) {
3586
- const buf = Buffer.from(await iconRes.arrayBuffer());
3587
- if (buf.length > 0) {
3588
- logoRef = await opts.saveAsset(buf, "logo");
3589
- if (!logoFromOg && opts.probeLongEdge) {
3590
- const edge = await opts.probeLongEdge(buf).catch(() => null);
3591
- if (edge !== null && edge < 256) {
3592
- logoTiny = true;
3593
- warnings.push(
3594
- `The site icon is favicon-sized (${edge}px), so it was saved as an alternate mark, not the logo. Upload your real logo in Settings.`
3595
- );
3596
- }
3597
- }
3598
- }
3599
- }
3600
- } catch {
3601
- warnings.push("Logo download failed.");
3602
- }
3603
- }
3604
- if (!logoRef) warnings.push("No logo captured. Add one manually.");
3605
- else if (logoFromOg)
3606
- warnings.push(
3607
- "No site icon was found; the social share image was saved as an alternate mark. Check it before treating it as the logo."
3608
- );
4619
+ const colorCount = [palette.primary, palette.secondary, ...palette.accent, ...palette.neutrals].filter(
4620
+ Boolean
4621
+ ).length;
4622
+ const manifest = await readManifest($, base, get);
4623
+ const candidates = logoCandidates($, base, manifest);
4624
+ const picked = await downloadMark(candidates, get, opts, warnings);
3609
4625
  const brand = {
3610
4626
  specVersion: "0.1",
3611
4627
  meta: {
3612
- name,
3613
- slug: name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || origin.hostname,
4628
+ name: named.value,
4629
+ slug: named.value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || origin.hostname,
3614
4630
  ...tagline ? { tagline } : {},
3615
4631
  website: origin.origin,
3616
4632
  createdWith: opts.createdWith ?? "scenri",
@@ -3624,26 +4640,208 @@ async function buildFromUrl(url, opts = {}) {
3624
4640
  ...palette.neutrals.length ? { neutrals: palette.neutrals.map((hex) => ({ hex })) } : {}
3625
4641
  }
3626
4642
  } : {},
3627
- ...logoRef ? { logos: [{ role: logoFromOg || logoTiny ? "alternate" : "primary", file: logoRef }] } : {}
4643
+ ...picked.ref ? {
4644
+ logos: [
4645
+ {
4646
+ role: picked.role,
4647
+ file: picked.ref,
4648
+ ...picked.background ? { background: picked.background } : {}
4649
+ }
4650
+ ]
4651
+ } : {}
4652
+ };
4653
+ return {
4654
+ brand,
4655
+ warnings,
4656
+ report: {
4657
+ url: page.finalUrl,
4658
+ host: origin.hostname,
4659
+ name: named,
4660
+ tagline: tagline ?? null,
4661
+ logo: {
4662
+ status: picked.ref ? picked.role : "none",
4663
+ source: picked.source,
4664
+ ...picked.score != null ? { score: picked.score } : {},
4665
+ ...picked.note ? { note: picked.note } : {}
4666
+ },
4667
+ colors: { count: colorCount }
4668
+ }
3628
4669
  };
3629
- return { brand, warnings };
3630
- }
3631
-
3632
- // ../brand-spec/src/mergeScrape.ts
3633
- var str2 = (v) => typeof v === "string" ? v.trim() : "";
3634
- var arr = (v) => Array.isArray(v) ? v : [];
3635
- var hashOf = (ref) => {
3636
- const s = String(ref ?? "");
3637
- return s.startsWith("asset:") ? s.slice(6) : s;
3638
- };
3639
- function scrapedHexes(palette) {
3640
- return [palette?.primary, palette?.secondary, ...arr(palette?.accent), ...arr(palette?.neutrals)].filter((c) => typeof c?.hex === "string").map((c) => ({ hex: String(c.hex) }));
3641
4670
  }
3642
- var hasPalette = (palette) => scrapedHexes(palette).length > 0;
3643
- function mergeScrape(existing, scraped) {
3644
- const cur = existing ?? {};
3645
- const next = scraped ?? {};
3646
- const meta = { ...cur.meta ?? {} };
4671
+ function baseOf($, origin) {
4672
+ const declared = $("base[href]").attr("href");
4673
+ if (!declared) return origin;
4674
+ try {
4675
+ return new URL(declared, origin);
4676
+ } catch {
4677
+ return origin;
4678
+ }
4679
+ }
4680
+ function pickName($, origin) {
4681
+ const ld = jsonLdName($);
4682
+ if (ld) return { value: ld, source: "json-ld" };
4683
+ const og = $('meta[property="og:site_name"]').attr("content")?.trim();
4684
+ if (og) return { value: og, source: "og:site_name" };
4685
+ const title = nameFromTitle($("title").first().text());
4686
+ if (title) return { value: title, source: "title" };
4687
+ return { value: origin.hostname.replace(/^www\./, ""), source: "hostname" };
4688
+ }
4689
+ function nameFromTitle(raw) {
4690
+ const parts = raw.trim().split(/\s+[|\u00b7\u2022\u2013\u2014]\s+|\s+-\s+/).map((p) => p.trim()).filter(Boolean);
4691
+ if (parts.length < 2) return parts[0] ?? "";
4692
+ const first = parts[0];
4693
+ const last = parts[parts.length - 1];
4694
+ const words = (v) => v.split(/\s+/).length;
4695
+ if (/^(home|homepage|index|welcome|start|main)$/i.test(first)) return last;
4696
+ if (words(last) <= 4 && words(first) > words(last)) return last;
4697
+ return first;
4698
+ }
4699
+ function jsonLdName($) {
4700
+ for (const node of $('script[type="application/ld+json"]').toArray()) {
4701
+ try {
4702
+ const data = JSON.parse($(node).text());
4703
+ const found = findOrgName(data, 0);
4704
+ if (found) return found;
4705
+ } catch {
4706
+ }
4707
+ }
4708
+ return null;
4709
+ }
4710
+ function findOrgName(node, depth) {
4711
+ if (depth > 6 || node === null || typeof node !== "object") return null;
4712
+ if (Array.isArray(node)) {
4713
+ for (const item of node) {
4714
+ const found = findOrgName(item, depth + 1);
4715
+ if (found) return found;
4716
+ }
4717
+ return null;
4718
+ }
4719
+ const obj = node;
4720
+ const type = String(obj["@type"] ?? "");
4721
+ if (/Organization|Corporation|LocalBusiness/i.test(type) && typeof obj.name === "string" && obj.name.trim())
4722
+ return obj.name.trim();
4723
+ for (const value of Object.values(obj)) {
4724
+ const found = findOrgName(value, depth + 1);
4725
+ if (found) return found;
4726
+ }
4727
+ return null;
4728
+ }
4729
+ function inlineCss($) {
4730
+ const attrs = $("[style]").map((_, el) => `x{${$(el).attr("style")}}`).get().join("\n");
4731
+ const blocks = $("style").map((_, el) => $(el).text()).get().join("\n");
4732
+ return `${attrs}
4733
+ ${blocks}`;
4734
+ }
4735
+ function sheetHrefs($, base) {
4736
+ const hrefs = [];
4737
+ for (const el of $('link[rel="stylesheet"]').toArray()) {
4738
+ const href = $(el).attr("href");
4739
+ if (!href || SHEET_SKIP.test(href)) continue;
4740
+ try {
4741
+ hrefs.push(new URL(href, base).toString());
4742
+ } catch {
4743
+ }
4744
+ }
4745
+ return hrefs.sort((a, b) => Number(SHEET_RANK.test(b)) - Number(SHEET_RANK.test(a))).slice(0, MAX_SHEETS);
4746
+ }
4747
+ async function readManifest($, base, get) {
4748
+ const href = $('link[rel="manifest"]').attr("href");
4749
+ if (!href || get.remaining() <= 0) return void 0;
4750
+ try {
4751
+ const res = await get(new URL(href, base).toString(), "css");
4752
+ return JSON.parse(res.text);
4753
+ } catch {
4754
+ return void 0;
4755
+ }
4756
+ }
4757
+ async function downloadMark(candidates, get, opts, warnings) {
4758
+ if (!opts.saveAsset || candidates.length === 0) {
4759
+ if (candidates.length === 0) warnings.push("No logo captured. Add one manually.");
4760
+ return { role: "primary", source: null };
4761
+ }
4762
+ let fallback = null;
4763
+ let tried = 0;
4764
+ let failed = false;
4765
+ for (const candidate of candidates) {
4766
+ if (tried >= LOGO_TRIES || get.remaining() <= 0) break;
4767
+ let buf = null;
4768
+ if (candidate.svg) {
4769
+ const sized = svgAsMark(candidate.svg);
4770
+ if (!sized) continue;
4771
+ buf = Buffer.from(sized, "utf8");
4772
+ } else if (candidate.url?.startsWith("data:")) {
4773
+ const comma = candidate.url.indexOf(",");
4774
+ const body = candidate.url.slice(comma + 1);
4775
+ buf = candidate.url.slice(0, comma).includes(";base64") ? Buffer.from(body, "base64") : Buffer.from(decodeURIComponent(body), "utf8");
4776
+ } else if (candidate.url) {
4777
+ tried++;
4778
+ try {
4779
+ buf = (await get(candidate.url, "asset")).bytes;
4780
+ } catch {
4781
+ failed = true;
4782
+ continue;
4783
+ }
4784
+ }
4785
+ if (!buf || buf.length === 0) continue;
4786
+ let ref;
4787
+ try {
4788
+ ref = await opts.saveAsset(buf, "logo");
4789
+ } catch {
4790
+ failed = true;
4791
+ continue;
4792
+ }
4793
+ const shape = opts.inspectMark ? await opts.inspectMark(buf).catch(() => null) : null;
4794
+ if (shape?.blank) continue;
4795
+ const edge = candidate.svg ? null : shape?.longEdge ?? null;
4796
+ const tiny = edge !== null && edge < 256;
4797
+ const tall = shape?.width != null && shape?.height != null && shape.width > 0 && shape.height / shape.width > MAX_PORTRAIT_RATIO;
4798
+ const declaredIcon = ICON_SOURCES.has(candidate.source);
4799
+ const unmeasured = !candidate.svg && shape === null;
4800
+ const bigEnough = edge !== null ? declaredIcon && edge >= 512 : unmeasured;
4801
+ const unsure = candidate.score < CONFIDENT_SCORE && !bigEnough;
4802
+ const role = candidate.role === "alternate" || tiny || tall || unsure ? "alternate" : "primary";
4803
+ const picked = {
4804
+ ref,
4805
+ role,
4806
+ source: candidate.source,
4807
+ score: candidate.score,
4808
+ ...candidate.background ? { background: candidate.background } : {}
4809
+ };
4810
+ if (role === "primary") return picked;
4811
+ if (tiny)
4812
+ picked.note = `The site icon is favicon-sized (${edge}px), so it was saved as an alternate mark, not the logo. Upload your real logo in Settings.`;
4813
+ else if (tall)
4814
+ picked.note = "The only image we could find is much taller than it is wide, so it was saved as an alternate mark rather than the logo. Upload your real logo in Settings.";
4815
+ else if (unsure)
4816
+ picked.note = "We are not confident this is your logo, so it was saved as an alternate mark. Check it, or upload your real logo in Settings.";
4817
+ else if (candidate.source === "og-image")
4818
+ picked.note = "No site icon was found; the social share image was saved as an alternate mark. Check it before treating it as the logo.";
4819
+ fallback ??= picked;
4820
+ }
4821
+ if (fallback) {
4822
+ if (fallback.note) warnings.push(fallback.note);
4823
+ return fallback;
4824
+ }
4825
+ if (failed) warnings.push("Logo download failed.");
4826
+ warnings.push("No logo captured. Add one manually.");
4827
+ return { role: "primary", source: null };
4828
+ }
4829
+
4830
+ // ../brand-spec/src/mergeScrape.ts
4831
+ var str2 = (v) => typeof v === "string" ? v.trim() : "";
4832
+ var arr = (v) => Array.isArray(v) ? v : [];
4833
+ var hashOf = (ref) => {
4834
+ const s = String(ref ?? "");
4835
+ return s.startsWith("asset:") ? s.slice(6) : s;
4836
+ };
4837
+ function scrapedHexes(palette) {
4838
+ return [palette?.primary, palette?.secondary, ...arr(palette?.accent), ...arr(palette?.neutrals)].filter((c) => typeof c?.hex === "string").map((c) => ({ hex: String(c.hex) }));
4839
+ }
4840
+ var hasPalette = (palette) => scrapedHexes(palette).length > 0;
4841
+ function mergeScrape(existing, scraped) {
4842
+ const cur = existing ?? {};
4843
+ const next = scraped ?? {};
4844
+ const meta = { ...cur.meta ?? {} };
3647
4845
  if (!str2(meta.name) && str2(next.meta?.name)) meta.name = str2(next.meta.name);
3648
4846
  if (!str2(meta.tagline) && str2(next.meta?.tagline)) meta.tagline = str2(next.meta.tagline);
3649
4847
  if (str2(next.meta?.website)) meta.website = str2(next.meta.website);
@@ -3671,6 +4869,74 @@ function mergeScrape(existing, scraped) {
3671
4869
  }
3672
4870
  return { brand, suggestions: { palette: suggestions } };
3673
4871
  }
4872
+ async function inspectMark(buf, toPng2) {
4873
+ const png = await toPng2(buf);
4874
+ const meta = await sharp21(png).metadata();
4875
+ let blank = false;
4876
+ try {
4877
+ const flat = await sharp21(png).flatten({ background: "#ffffff" }).toBuffer();
4878
+ const stats = await sharp21(flat).stats();
4879
+ blank = stats.channels.slice(0, 3).every((c) => c.min > 200);
4880
+ } catch {
4881
+ }
4882
+ return {
4883
+ longEdge: Math.max(meta.width ?? 0, meta.height ?? 0) || null,
4884
+ width: meta.width ?? null,
4885
+ height: meta.height ?? null,
4886
+ blank
4887
+ };
4888
+ }
4889
+
4890
+ // ../catalog/src/failures.ts
4891
+ function pageFailure(status) {
4892
+ if (status === 429 || status === 503) return "RATE_LIMITED";
4893
+ if (status === 401 || status === 403 || status === 405) return "PAGE_BLOCKED";
4894
+ if (status === 404 || status === 410) return "PAGE_NOT_FOUND";
4895
+ return "PAGE_UNREADABLE";
4896
+ }
4897
+ function imageFailure(status) {
4898
+ if (status === 429 || status === 503) return "RATE_LIMITED";
4899
+ if (status === 401 || status === 403) return "IMAGE_FORBIDDEN";
4900
+ if (status === 404 || status === 410) return "IMAGE_NOT_FOUND";
4901
+ return "DOWNLOAD_FAILED";
4902
+ }
4903
+ function thrownFailure(err, aborted = false) {
4904
+ if (aborted) return "ABORTED";
4905
+ const m = String(err?.message ?? err).toLowerCase();
4906
+ if (m.includes("abort") || m.includes("timed out") || m.includes("timeout")) return "IMAGE_TIMEOUT";
4907
+ return "DOWNLOAD_FAILED";
4908
+ }
4909
+ function tally(into, reason, n = 1) {
4910
+ into[reason] = (into[reason] ?? 0) + n;
4911
+ }
4912
+ function leadingReason(t) {
4913
+ const rows = Object.entries(t);
4914
+ if (!rows.length) return null;
4915
+ return rows.sort((a, b) => b[1] - a[1])[0][0];
4916
+ }
4917
+ function summarise(t, saved, asked) {
4918
+ const reason = leadingReason(t);
4919
+ if (!reason) return null;
4920
+ const none = saved === 0;
4921
+ switch (reason) {
4922
+ case "RATE_LIMITED":
4923
+ return none ? "The store asked us to slow down, so nothing could be read. Waiting a few minutes and trying again usually works." : `The store asked us to slow down partway, so ${asked - saved} of ${asked} products were skipped. Trying again picks up the rest.`;
4924
+ case "PAGE_BLOCKED":
4925
+ return none ? "This store would not let us read its product pages." : `${asked - saved} of ${asked} product pages would not open.`;
4926
+ case "PAGE_NOT_FOUND":
4927
+ return `${asked - saved} of ${asked} products are no longer on the site.`;
4928
+ case "IMAGE_FORBIDDEN":
4929
+ case "IMAGE_NOT_FOUND":
4930
+ case "IMAGE_TIMEOUT":
4931
+ case "IMAGE_EMPTY":
4932
+ case "DOWNLOAD_FAILED":
4933
+ return none ? "We found the products, but none of their pictures could be downloaded." : "Some pictures could not be downloaded. The products they belong to are still here.";
4934
+ case "ABORTED":
4935
+ return null;
4936
+ default:
4937
+ return none ? "The products on this site could not be read." : null;
4938
+ }
4939
+ }
3674
4940
 
3675
4941
  // ../catalog/src/url.ts
3676
4942
  function normalizeStoreUrl(input) {
@@ -3717,6 +4983,28 @@ function upgradeImageUrl(url) {
3717
4983
  return url;
3718
4984
  }
3719
4985
  }
4986
+ var LOCALE_SEGMENT = /^\/[a-z]{2}(?:-[A-Za-z]{2})?\//;
4987
+ function preferCanonicalLocale(urls) {
4988
+ const canonical = /* @__PURE__ */ new Set();
4989
+ for (const u of urls) {
4990
+ try {
4991
+ const parsed = new URL(u);
4992
+ if (!LOCALE_SEGMENT.test(parsed.pathname)) canonical.add(`${parsed.origin}${parsed.pathname}`);
4993
+ } catch {
4994
+ }
4995
+ }
4996
+ if (!canonical.size) return urls;
4997
+ return urls.filter((u) => {
4998
+ try {
4999
+ const parsed = new URL(u);
5000
+ if (!LOCALE_SEGMENT.test(parsed.pathname)) return true;
5001
+ const stripped = parsed.pathname.replace(LOCALE_SEGMENT, "/");
5002
+ return !canonical.has(`${parsed.origin}${stripped}`);
5003
+ } catch {
5004
+ return true;
5005
+ }
5006
+ });
5007
+ }
3720
5008
 
3721
5009
  // ../catalog/src/normalize.ts
3722
5010
  function cleanText(s) {
@@ -3824,34 +5112,137 @@ function dedupeProducts(products) {
3824
5112
  }
3825
5113
  return [...byKey.values()];
3826
5114
  }
3827
-
3828
- // ../catalog/src/http/fetch.ts
3829
- var USER_AGENT = "scenri-catalog/0.1 (+https://scenri.co)";
3830
- function sleep3(ms) {
5115
+ var ALLOW_PRIVATE = process.env.SCENRI_SCRAPE_ALLOW_PRIVATE === "1";
5116
+ async function assertReachable(url, real) {
5117
+ const u = new URL(url);
5118
+ if (u.protocol !== "http:" && u.protocol !== "https:") {
5119
+ throw new Error(`Scenri reads http and https addresses only, not ${u.protocol.replace(":", "")}`);
5120
+ }
5121
+ if (ALLOW_PRIVATE) return;
5122
+ const host = u.hostname.replace(/^\[|\]$/g, "");
5123
+ if (isIP(host)) {
5124
+ assertPublicHost(u.hostname, [host], false);
5125
+ return;
5126
+ }
5127
+ if (!real) return;
5128
+ let addresses;
5129
+ try {
5130
+ addresses = (await lookup(host, { all: true })).map((a) => a.address);
5131
+ } catch {
5132
+ return;
5133
+ }
5134
+ assertPublicHost(u.hostname, addresses, false);
5135
+ }
5136
+ var USER_AGENT2 = "scenri-catalog/0.1 (+https://scenri.co)";
5137
+ async function readBounded2(res, maxBytes) {
5138
+ if (!maxBytes || !res.body) return res.text();
5139
+ const reader = res.body.getReader();
5140
+ const decoder = new TextDecoder();
5141
+ let out = "";
5142
+ let seen = 0;
5143
+ try {
5144
+ while (seen < maxBytes) {
5145
+ const { done, value } = await reader.read();
5146
+ if (done) break;
5147
+ seen += value.byteLength;
5148
+ out += decoder.decode(value, { stream: true });
5149
+ }
5150
+ } finally {
5151
+ await reader.cancel().catch(() => {
5152
+ });
5153
+ }
5154
+ return out + decoder.decode();
5155
+ }
5156
+ function sleep4(ms) {
3831
5157
  return new Promise((r) => setTimeout(r, ms));
3832
5158
  }
5159
+ var cooledUntil = /* @__PURE__ */ new Map();
5160
+ var hostOf = (url) => {
5161
+ try {
5162
+ return new URL(url).host;
5163
+ } catch {
5164
+ return "";
5165
+ }
5166
+ };
5167
+ async function waitForHost(host, signal) {
5168
+ if (!host) return;
5169
+ for (; ; ) {
5170
+ const left = (cooledUntil.get(host) ?? 0) - Date.now();
5171
+ if (left <= 0 || signal?.aborted) return;
5172
+ await sleep4(Math.min(left, 250));
5173
+ }
5174
+ }
5175
+ function coolHost(host, ms) {
5176
+ if (!host || ms <= 0) return;
5177
+ const until = Date.now() + ms;
5178
+ if (until > (cooledUntil.get(host) ?? 0)) cooledUntil.set(host, until);
5179
+ }
5180
+ var RETRY_AFTER_CAP_MS = 6e4;
5181
+ function retryAfterMs(header, now = Date.now()) {
5182
+ if (!header) return null;
5183
+ const secs = Number(header.trim());
5184
+ if (Number.isFinite(secs) && secs >= 0) return Math.min(secs * 1e3, RETRY_AFTER_CAP_MS);
5185
+ const at = Date.parse(header);
5186
+ if (Number.isNaN(at)) return null;
5187
+ return Math.min(Math.max(0, at - now), RETRY_AFTER_CAP_MS);
5188
+ }
5189
+ function throttleBackoff(attempt) {
5190
+ const base = Math.min(1e3 * 2 ** attempt, 16e3);
5191
+ return base + Math.floor(Math.random() * 400);
5192
+ }
3833
5193
  async function httpGet(url, opts = {}) {
3834
5194
  const fetchImpl = opts.fetchImpl ?? fetch;
3835
5195
  const retries = opts.retries ?? 3;
3836
5196
  const timeoutMs = opts.timeoutMs ?? 25e3;
3837
5197
  let lastErr;
5198
+ const host = hostOf(url);
3838
5199
  for (let attempt = 0; attempt <= retries; attempt++) {
5200
+ await waitForHost(host, opts.signal);
5201
+ if (opts.signal?.aborted) throw new Error("aborted");
3839
5202
  const ctrl = new AbortController();
3840
5203
  const onAbort = () => ctrl.abort();
3841
5204
  opts.signal?.addEventListener("abort", onAbort, { once: true });
3842
5205
  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
3843
5206
  try {
3844
- const res = await fetchImpl(url, {
3845
- redirect: "follow",
3846
- signal: ctrl.signal,
3847
- headers: {
3848
- "user-agent": USER_AGENT,
3849
- accept: opts.accept ?? "application/json, text/html, application/xml, text/xml, */*;q=0.8",
3850
- ...opts.headers ?? {}
5207
+ let current = url;
5208
+ let res;
5209
+ for (let hop = 0; ; hop++) {
5210
+ await assertReachable(current, !opts.fetchImpl);
5211
+ res = await fetchImpl(current, {
5212
+ redirect: "manual",
5213
+ signal: ctrl.signal,
5214
+ headers: {
5215
+ "user-agent": USER_AGENT2,
5216
+ accept: opts.accept ?? "application/json, text/html, application/xml, text/xml, */*;q=0.8",
5217
+ ...opts.headers ?? {}
5218
+ }
5219
+ });
5220
+ if (res.status < 300 || res.status >= 400) break;
5221
+ const location = res.headers.get("location");
5222
+ await res.body?.cancel().catch(() => {
5223
+ });
5224
+ if (!location || hop >= 5) break;
5225
+ current = new URL(location, current).toString();
5226
+ }
5227
+ if (res.url !== current) {
5228
+ try {
5229
+ Object.defineProperty(res, "url", { value: current, configurable: true });
5230
+ } catch {
3851
5231
  }
3852
- });
3853
- if ((res.status === 429 || res.status >= 500) && attempt < retries) {
3854
- await sleep3(400 * 2 ** attempt);
5232
+ }
5233
+ if (res.status === 429 || res.status === 503) {
5234
+ const asked = retryAfterMs(res.headers.get("retry-after"));
5235
+ const wait = asked ?? throttleBackoff(attempt);
5236
+ coolHost(host, wait);
5237
+ if (attempt < retries) {
5238
+ await res.body?.cancel().catch(() => {
5239
+ });
5240
+ continue;
5241
+ }
5242
+ return res;
5243
+ }
5244
+ if (res.status >= 500 && attempt < retries) {
5245
+ await sleep4(throttleBackoff(attempt));
3855
5246
  continue;
3856
5247
  }
3857
5248
  return res;
@@ -3859,7 +5250,7 @@ async function httpGet(url, opts = {}) {
3859
5250
  lastErr = err;
3860
5251
  if (opts.signal?.aborted) throw err;
3861
5252
  if (attempt < retries) {
3862
- await sleep3(400 * 2 ** attempt);
5253
+ await sleep4(400 * 2 ** attempt);
3863
5254
  continue;
3864
5255
  }
3865
5256
  throw err;
@@ -3872,7 +5263,7 @@ async function httpGet(url, opts = {}) {
3872
5263
  }
3873
5264
  async function httpText(url, opts = {}) {
3874
5265
  const res = await httpGet(url, opts);
3875
- const text = await res.text();
5266
+ const text = await readBounded2(res, opts.maxBytes);
3876
5267
  return { ok: res.ok, status: res.status, text, url: res.url || url };
3877
5268
  }
3878
5269
  async function httpJson(url, opts = {}) {
@@ -3902,6 +5293,284 @@ async function mapPool(items, concurrency, fn, signal) {
3902
5293
  await Promise.all(workers);
3903
5294
  return results;
3904
5295
  }
5296
+ function loadHtml(html) {
5297
+ return parse(html);
5298
+ }
5299
+ function attr(el, name) {
5300
+ return el?.getAttribute(name) ?? void 0;
5301
+ }
5302
+ function textOf(el) {
5303
+ return (el?.text ?? "").replace(/\s+/g, " ").trim();
5304
+ }
5305
+
5306
+ // ../catalog/src/adapters/productPage.ts
5307
+ function stableKey(url) {
5308
+ return createHash("sha256").update(url).digest("hex").slice(0, 16);
5309
+ }
5310
+ var typesOf = (obj) => {
5311
+ const t = obj["@type"];
5312
+ const list2 = Array.isArray(t) ? t : t ? [t] : [];
5313
+ return list2.map((x) => String(x).toLowerCase());
5314
+ };
5315
+ function walkJsonLd(node, groups, singles) {
5316
+ if (!node) return;
5317
+ if (Array.isArray(node)) {
5318
+ for (const n of node) walkJsonLd(n, groups, singles);
5319
+ return;
5320
+ }
5321
+ if (typeof node !== "object") return;
5322
+ const obj = node;
5323
+ const types = typesOf(obj);
5324
+ if (types.includes("productgroup")) groups.push(obj);
5325
+ else if (types.includes("product")) singles.push(obj);
5326
+ if (obj["@graph"]) walkJsonLd(obj["@graph"], groups, singles);
5327
+ for (const v of Object.values(obj)) {
5328
+ if (v && typeof v === "object") walkJsonLd(v, groups, singles);
5329
+ }
5330
+ }
5331
+ var asArray = (v) => Array.isArray(v) ? v : v == null ? [] : [v];
5332
+ function imagesOf(n, pageUrl) {
5333
+ return [].concat(n.image ?? []).flat().map((img) => typeof img === "string" ? img : img?.url ?? img?.contentUrl).filter(Boolean).map((u, i) => ({ url: absolutize(pageUrl, String(u)), position: i, width: null, height: null, alt: null })).filter((img) => img.url);
5334
+ }
5335
+ var offerOf = (n) => Array.isArray(n?.offers) ? n.offers[0] : n?.offers;
5336
+ function variesBy(group) {
5337
+ const named = asArray(group.variesBy).map((v) => String(v).split("/").pop() ?? "").filter(Boolean);
5338
+ return named.length ? named : ["size", "color"];
5339
+ }
5340
+ function variantsOf(group, keys) {
5341
+ return asArray(group.hasVariant).filter((v) => v && typeof v === "object").map((v, i) => {
5342
+ const offer = offerOf(v);
5343
+ const options = Object.fromEntries(
5344
+ keys.map((k) => [k, v[k]]).filter(([, val]) => val != null && val !== "")
5345
+ );
5346
+ const base = String(v.sku || v.mpn || v.gtin || `${group.productGroupID ?? group.name ?? "group"}:${i}`);
5347
+ const suffix = Object.values(options).join("/");
5348
+ return {
5349
+ externalKey: suffix ? `${base}:${suffix}` : base,
5350
+ title: [v.name, ...Object.values(options)].filter(Boolean).join(" "),
5351
+ sku: v.sku ? String(v.sku) : null,
5352
+ price: offer?.price != null ? Number(offer.price) : null,
5353
+ compareAtPrice: null,
5354
+ currency: offer?.priceCurrency ?? null,
5355
+ available: offer?.availability ? /instock/i.test(String(offer.availability)) : null,
5356
+ options
5357
+ };
5358
+ });
5359
+ }
5360
+ function fromGroup(group, pageUrl) {
5361
+ const keys = variesBy(group);
5362
+ const variants = variantsOf(group, keys);
5363
+ const url = absolutize(pageUrl, String(group.url ?? group["@id"] ?? pageUrl)) ?? pageUrl;
5364
+ const images = imagesOf(group, pageUrl);
5365
+ const fallback = images.length ? images : imagesOf(asArray(group.hasVariant)[0] ?? {}, pageUrl);
5366
+ return normalizeProduct({
5367
+ externalKey: String(group.productGroupID || group.sku || stableKey(url)),
5368
+ title: String(group.name ?? "Product"),
5369
+ descriptionHtml: group.description ? String(group.description) : null,
5370
+ url,
5371
+ vendor: group.brand?.name ?? (typeof group.brand === "string" ? group.brand : null),
5372
+ productType: group.category ? String(group.category) : null,
5373
+ category: group.category ? String(group.category) : null,
5374
+ price: variants.find((v) => v.price != null)?.price ?? null,
5375
+ compareAtPrice: null,
5376
+ currency: variants.find((v) => v.currency)?.currency ?? null,
5377
+ available: variants.some((v) => v.available) || null,
5378
+ tags: [],
5379
+ variants,
5380
+ images: fallback,
5381
+ raw: group
5382
+ });
5383
+ }
5384
+ function fromSingle(n, pageUrl) {
5385
+ const offers = offerOf(n);
5386
+ const url = String(n.url ?? n["@id"] ?? pageUrl);
5387
+ return normalizeProduct({
5388
+ externalKey: String(n.sku || n.productID || n.mpn || stableKey(url)),
5389
+ title: String(n.name ?? "Product"),
5390
+ descriptionHtml: n.description ? String(n.description) : null,
5391
+ url: absolutize(pageUrl, url) ?? pageUrl,
5392
+ vendor: n.brand?.name ?? (typeof n.brand === "string" ? n.brand : null),
5393
+ productType: n.category ? String(n.category) : null,
5394
+ category: n.category ? String(n.category) : null,
5395
+ price: offers?.price != null ? Number(offers.price) : null,
5396
+ compareAtPrice: null,
5397
+ currency: offers?.priceCurrency ?? null,
5398
+ available: offers?.availability ? /instock/i.test(String(offers.availability)) : null,
5399
+ tags: [],
5400
+ variants: [],
5401
+ images: imagesOf(n, pageUrl),
5402
+ raw: n
5403
+ });
5404
+ }
5405
+ function extractJsonLdProducts(html, pageUrl, doc) {
5406
+ const root = doc ?? loadHtml(html);
5407
+ const groups = [];
5408
+ const singles = [];
5409
+ for (const el of root.querySelectorAll('script[type="application/ld+json"]')) {
5410
+ const raw = el.innerHTML;
5411
+ if (!raw) continue;
5412
+ try {
5413
+ walkJsonLd(JSON.parse(raw), groups, singles);
5414
+ } catch {
5415
+ }
5416
+ }
5417
+ const claimed = /* @__PURE__ */ new Set();
5418
+ for (const g of groups) for (const v of asArray(g.hasVariant)) if (v && typeof v === "object") claimed.add(v);
5419
+ return [
5420
+ ...groups.map((g) => fromGroup(g, pageUrl)),
5421
+ ...singles.filter((n) => !claimed.has(n)).map((n) => fromSingle(n, pageUrl))
5422
+ ];
5423
+ }
5424
+ var PRODUCT_MARKERS = [
5425
+ 'meta[property="og:type"][content="product"]',
5426
+ 'meta[property="product:price:amount"]',
5427
+ 'meta[property="og:price:amount"]',
5428
+ '[itemtype*="schema.org/Product"]',
5429
+ '[itemprop="price"]',
5430
+ '[itemprop="offers"]',
5431
+ 'form[action*="/cart/add"]',
5432
+ '[name="add"]'
5433
+ ];
5434
+ var BUY_WORDS = /add to (cart|bag|basket)|buy now|add to my bag/i;
5435
+ var BUILD_ASSET = /\/_next\/static\/|\/static\/media\/|\/assets\/(icons|flags|ui)\//i;
5436
+ var NOT_A_PACKSHOT = /logo|icon|sprite|pixel|avatar|\bflags?\b|\/flags?\/|locale|country|currency|badge|payment|social/i;
5437
+ function looksLikeProduct(html, doc) {
5438
+ const $ = doc ?? loadHtml(html);
5439
+ for (const sel of PRODUCT_MARKERS) {
5440
+ try {
5441
+ if ($.querySelector(sel)) return true;
5442
+ } catch {
5443
+ }
5444
+ }
5445
+ for (const el of $.querySelectorAll('button, input[type="submit"], a')) {
5446
+ const words = `${textOf(el)} ${attr(el, "value") ?? ""} ${attr(el, "aria-label") ?? ""}`;
5447
+ if (BUY_WORDS.test(words)) return true;
5448
+ }
5449
+ return false;
5450
+ }
5451
+ function galleryImages($, pageUrl, cap2 = 12) {
5452
+ const images = [];
5453
+ const og = attr($.querySelector('meta[property="og:image"]'), "content");
5454
+ if (og) {
5455
+ const abs = absolutize(pageUrl, og);
5456
+ if (abs) images.push({ url: abs, position: 0, width: null, height: null, alt: null });
5457
+ }
5458
+ for (const el of $.querySelectorAll("img[src]")) {
5459
+ if (images.length >= cap2) break;
5460
+ const src = attr(el, "src") || attr(el, "data-src");
5461
+ const abs = src ? absolutize(pageUrl, src) : null;
5462
+ if (!abs || NOT_A_PACKSHOT.test(abs) || BUILD_ASSET.test(abs)) continue;
5463
+ if (images.some((x) => x.url === abs)) continue;
5464
+ images.push({ url: abs, position: images.length, width: null, height: null, alt: attr(el, "alt") ?? null });
5465
+ }
5466
+ return images;
5467
+ }
5468
+ function parseProductHtml(html, pageUrl, doc) {
5469
+ const $ = doc ?? loadHtml(html);
5470
+ if (!looksLikeProduct(html, $)) return null;
5471
+ const title = attr($.querySelector('meta[property="og:title"]'), "content") || textOf($.querySelector("h1")) || textOf($.querySelector("title"));
5472
+ if (!title) return null;
5473
+ const desc = attr($.querySelector('meta[property="og:description"]'), "content") || attr($.querySelector('meta[name="description"]'), "content") || null;
5474
+ const images = galleryImages($, pageUrl);
5475
+ const canonical = attr($.querySelector('link[rel="canonical"]'), "href");
5476
+ const url = canonical ? absolutize(pageUrl, canonical) ?? pageUrl : pageUrl;
5477
+ return normalizeProduct({
5478
+ externalKey: stableKey(url),
5479
+ title,
5480
+ descriptionHtml: desc,
5481
+ url,
5482
+ images,
5483
+ variants: [],
5484
+ tags: [],
5485
+ raw: { source: "html" }
5486
+ });
5487
+ }
5488
+ function sellsSomething(p) {
5489
+ const n = p.raw ?? {};
5490
+ return Boolean(n.offers || n.sku || n.gtin || n.mpn || n.hasVariant || p.price != null);
5491
+ }
5492
+ function withGallery(product, doc, pageUrl) {
5493
+ const have = new Set((product.images ?? []).map((i) => i.url));
5494
+ const extra = galleryImages(doc, pageUrl).filter((i) => !have.has(i.url));
5495
+ if (!extra.length) return product;
5496
+ const images = [...product.images ?? []];
5497
+ for (const img of extra) {
5498
+ if (images.length >= 12) break;
5499
+ images.push({ ...img, position: images.length });
5500
+ }
5501
+ return { ...product, images };
5502
+ }
5503
+ function productsFromPage(html, url) {
5504
+ const doc = loadHtml(html);
5505
+ const declared = looksLikeProduct(html, doc);
5506
+ const fromLd = extractJsonLdProducts(html, url, doc);
5507
+ const selling = declared ? fromLd : fromLd.filter(sellsSomething);
5508
+ if (selling.length === 1) return [withGallery(selling[0], doc, url)];
5509
+ if (selling.length) return selling;
5510
+ if (!declared) return [];
5511
+ const one = parseProductHtml(html, url, doc);
5512
+ return one ? [one] : [];
5513
+ }
5514
+ var sleep5 = (ms) => new Promise((r) => setTimeout(r, ms));
5515
+ async function fetchProductPages(ctx, urls, opts = {}) {
5516
+ const out = [];
5517
+ const seen = /* @__PURE__ */ new Set();
5518
+ let kept = 0;
5519
+ let refusedRun = 0;
5520
+ const giveUpAt = opts.giveUpAfterRefusals ?? Number.POSITIVE_INFINITY;
5521
+ let givenUp = false;
5522
+ const want = opts.want ?? Number.POSITIVE_INFINITY;
5523
+ const ceiling = opts.maxPages ?? (Number.isFinite(want) ? want * 3 : urls.length);
5524
+ const take = urls.slice(0, ceiling);
5525
+ let bytes = 0;
5526
+ await mapPool(
5527
+ take,
5528
+ opts.delayMs ? 1 : opts.concurrency ?? 5,
5529
+ async (u) => {
5530
+ if (kept >= want || givenUp) return;
5531
+ if (opts.deadline != null && Date.now() > opts.deadline) return;
5532
+ if (opts.maxTotalBytes != null && bytes >= opts.maxTotalBytes) return;
5533
+ try {
5534
+ if (opts.delayMs) await sleep5(opts.delayMs);
5535
+ const { ok, status, text, url } = await httpText(u, {
5536
+ fetchImpl: ctx.fetchImpl,
5537
+ signal: ctx.signal,
5538
+ accept: "text/html",
5539
+ maxBytes: opts.maxBytes
5540
+ });
5541
+ bytes += text.length;
5542
+ if (opts.stats) {
5543
+ opts.stats.pages += 1;
5544
+ opts.stats.bytes = bytes;
5545
+ }
5546
+ if (!ok) {
5547
+ if (opts.stats) {
5548
+ opts.stats.refused = (opts.stats.refused ?? 0) + 1;
5549
+ if (opts.stats.reasons) tally(opts.stats.reasons, pageFailure(status));
5550
+ }
5551
+ if (++refusedRun >= giveUpAt) givenUp = true;
5552
+ return;
5553
+ }
5554
+ refusedRun = 0;
5555
+ for (const p of productsFromPage(text, url)) {
5556
+ if (seen.has(p.externalKey)) continue;
5557
+ seen.add(p.externalKey);
5558
+ kept++;
5559
+ if (opts.onEach) opts.onEach(p);
5560
+ else out.push(p);
5561
+ }
5562
+ opts.onProduct?.(kept);
5563
+ } catch (err) {
5564
+ if (opts.stats) {
5565
+ opts.stats.refused = (opts.stats.refused ?? 0) + 1;
5566
+ if (opts.stats.reasons) tally(opts.stats.reasons, thrownFailure(err, ctx.signal?.aborted));
5567
+ }
5568
+ }
5569
+ },
5570
+ ctx.signal
5571
+ );
5572
+ return out;
5573
+ }
3905
5574
 
3906
5575
  // ../catalog/src/adapters/shopify.ts
3907
5576
  function mapShopifyProduct(base, p) {
@@ -3955,10 +5624,14 @@ function mapShopifyProduct(base, p) {
3955
5624
  async function fetchProductsJsonPage(ctx, page, limit = 250) {
3956
5625
  const origin = originOf(ctx.baseUrl);
3957
5626
  const url = `${origin}/products.json?limit=${limit}&page=${page}`;
3958
- const { ok, json } = await httpJson(url, { fetchImpl: ctx.fetchImpl, signal: ctx.signal });
3959
- if (!ok || !json?.products) return [];
3960
- return json.products;
5627
+ const { ok, status, json } = await httpJson(url, {
5628
+ fetchImpl: ctx.fetchImpl,
5629
+ signal: ctx.signal
5630
+ });
5631
+ if (ok && json?.products) return { products: json.products, blocked: false };
5632
+ return { products: [], blocked: status === 401 || status === 403 || status === 404 || status >= 500 };
3961
5633
  }
5634
+ var JSON_BLOCKED = "json-blocked";
3962
5635
  async function collectSitemapProductUrls(ctx) {
3963
5636
  const origin = originOf(ctx.baseUrl);
3964
5637
  const urls = /* @__PURE__ */ new Set();
@@ -3984,7 +5657,7 @@ async function collectSitemapProductUrls(ctx) {
3984
5657
  }
3985
5658
  }
3986
5659
  }
3987
- return [...urls];
5660
+ return preferCanonicalLocale([...urls]);
3988
5661
  }
3989
5662
  var shopifyAdapter = {
3990
5663
  platform: "shopify",
@@ -4016,6 +5689,7 @@ var shopifyAdapter = {
4016
5689
  },
4017
5690
  async discover(ctx) {
4018
5691
  const warnings = [];
5692
+ const hints = [];
4019
5693
  const keys = /* @__PURE__ */ new Set();
4020
5694
  const productUrls = /* @__PURE__ */ new Set();
4021
5695
  const origin = originOf(ctx.baseUrl);
@@ -4023,8 +5697,12 @@ var shopifyAdapter = {
4023
5697
  let emptyStreak = 0;
4024
5698
  while (emptyStreak < 1) {
4025
5699
  if (ctx.signal?.aborted) throw new Error("aborted");
4026
- const products = await fetchProductsJsonPage(ctx, page);
5700
+ const { products, blocked: blocked2 } = await fetchProductsJsonPage(ctx, page);
4027
5701
  ctx.onProgress?.({ stage: "discovering", discovered: keys.size, message: `Shopify page ${page}` });
5702
+ if (blocked2 && page === 1) {
5703
+ hints.push(JSON_BLOCKED);
5704
+ warnings.push("This store does not serve its product API, so the product pages were read instead");
5705
+ }
4028
5706
  if (!products.length) {
4029
5707
  emptyStreak++;
4030
5708
  break;
@@ -4053,17 +5731,23 @@ var shopifyAdapter = {
4053
5731
  productKeys: [...keys],
4054
5732
  productUrls: [...productUrls],
4055
5733
  estimatedTotal: keys.size || productUrls.size || null,
4056
- warnings
5734
+ // A store whose own product API answered but refused us leaves nothing
5735
+ // to read but the pages themselves, one request each. That is what
5736
+ // gymshark.com does, and it is the run worth batching.
5737
+ byPage: hints.includes(JSON_BLOCKED),
5738
+ warnings,
5739
+ hints
4057
5740
  };
4058
5741
  },
4059
5742
  async fetchAll(ctx, discovered) {
4060
5743
  const origin = originOf(ctx.baseUrl);
4061
5744
  const out = [];
4062
5745
  const seen = /* @__PURE__ */ new Set();
5746
+ const jsonBlocked = discovered.hints?.includes(JSON_BLOCKED) ?? false;
4063
5747
  let page = 1;
4064
- while (true) {
5748
+ while (!jsonBlocked) {
4065
5749
  if (ctx.signal?.aborted) throw new Error("aborted");
4066
- const products = await fetchProductsJsonPage(ctx, page);
5750
+ const { products } = await fetchProductsJsonPage(ctx, page);
4067
5751
  if (!products.length) break;
4068
5752
  for (const p of products) {
4069
5753
  const mapped = mapShopifyProduct(origin, p);
@@ -4080,48 +5764,56 @@ var shopifyAdapter = {
4080
5764
  page++;
4081
5765
  if (page > 1e4) break;
4082
5766
  }
4083
- const missingUrls = discovered.productUrls.filter((u) => {
4084
- const handle = /\/products\/([^/?#]+)/i.exec(u)?.[1];
4085
- return handle && !out.some((p) => p.handle === decodeURIComponent(handle));
5767
+ const handleOf = (u) => {
5768
+ const raw = /\/products\/([^/?#]+)/i.exec(u)?.[1];
5769
+ return raw ? decodeURIComponent(raw) : null;
5770
+ };
5771
+ const stillMissing = () => discovered.productUrls.filter((u) => {
5772
+ const handle = handleOf(u);
5773
+ return handle && !out.some((p) => p.handle === handle);
4086
5774
  });
4087
- if (missingUrls.length) {
4088
- await mapPool(
4089
- missingUrls,
4090
- 6,
4091
- async (u) => {
4092
- const handle = /\/products\/([^/?#]+)/i.exec(u)?.[1];
4093
- if (!handle) return;
4094
- const { ok, json } = await httpJson(`${origin}/products/${handle}.json`, {
4095
- fetchImpl: ctx.fetchImpl,
4096
- signal: ctx.signal
4097
- });
4098
- if (ok && json?.product) {
4099
- const mapped = mapShopifyProduct(origin, json.product);
4100
- if (!seen.has(mapped.externalKey)) {
4101
- seen.add(mapped.externalKey);
4102
- out.push(mapped);
4103
- ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
5775
+ if (!jsonBlocked) {
5776
+ const missingUrls = stillMissing();
5777
+ if (missingUrls.length) {
5778
+ await mapPool(
5779
+ missingUrls,
5780
+ 6,
5781
+ async (u) => {
5782
+ const handle = handleOf(u);
5783
+ if (!handle) return;
5784
+ const { ok, json } = await httpJson(`${origin}/products/${handle}.json`, {
5785
+ fetchImpl: ctx.fetchImpl,
5786
+ signal: ctx.signal
5787
+ });
5788
+ if (ok && json?.product) {
5789
+ const mapped = mapShopifyProduct(origin, json.product);
5790
+ if (!seen.has(mapped.externalKey)) {
5791
+ seen.add(mapped.externalKey);
5792
+ out.push(mapped);
5793
+ ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
5794
+ }
4104
5795
  }
4105
- }
4106
- },
4107
- ctx.signal
4108
- );
5796
+ },
5797
+ ctx.signal
5798
+ );
5799
+ }
5800
+ }
5801
+ const unread = stillMissing();
5802
+ if (unread.length) {
5803
+ for (const p of await fetchProductPages(ctx, unread, {
5804
+ concurrency: 4,
5805
+ onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched: out.length + fetched })
5806
+ })) {
5807
+ if (seen.has(p.externalKey)) continue;
5808
+ seen.add(p.externalKey);
5809
+ out.push(p);
5810
+ }
4109
5811
  }
4110
5812
  return out;
4111
5813
  }
4112
5814
  };
4113
- function loadHtml(html) {
4114
- return parse(html);
4115
- }
4116
- function attr(el, name) {
4117
- return el?.getAttribute(name) ?? void 0;
4118
- }
4119
- function textOf(el) {
4120
- return (el?.text ?? "").replace(/\s+/g, " ").trim();
4121
- }
4122
- function stableKey(url) {
4123
- return createHash("sha256").update(url).digest("hex").slice(0, 16);
4124
- }
5815
+
5816
+ // ../catalog/src/adapters/generic.ts
4125
5817
  async function extractSitemapUrls(ctx, filter = () => true) {
4126
5818
  const origin = originOf(ctx.baseUrl);
4127
5819
  const out = /* @__PURE__ */ new Set();
@@ -4153,89 +5845,7 @@ async function extractSitemapUrls(ctx, filter = () => true) {
4153
5845
  }
4154
5846
  if (seen.size > 200) break;
4155
5847
  }
4156
- return [...out];
4157
- }
4158
- function walkJsonLd(node, out) {
4159
- if (!node) return;
4160
- if (Array.isArray(node)) {
4161
- for (const n of node) walkJsonLd(n, out);
4162
- return;
4163
- }
4164
- if (typeof node !== "object") return;
4165
- const obj = node;
4166
- const type = obj["@type"];
4167
- const types = Array.isArray(type) ? type : type ? [type] : [];
4168
- if (types.some((t) => String(t).toLowerCase() === "product")) out.push(obj);
4169
- if (obj["@graph"]) walkJsonLd(obj["@graph"], out);
4170
- for (const v of Object.values(obj)) {
4171
- if (v && typeof v === "object") walkJsonLd(v, out);
4172
- }
4173
- }
4174
- function extractJsonLdProducts(html, pageUrl) {
4175
- const root = loadHtml(html);
4176
- const nodes = [];
4177
- for (const el of root.querySelectorAll('script[type="application/ld+json"]')) {
4178
- const raw = el.innerHTML;
4179
- if (!raw) continue;
4180
- try {
4181
- walkJsonLd(JSON.parse(raw), nodes);
4182
- } catch {
4183
- }
4184
- }
4185
- return nodes.map((n) => {
4186
- const offers = Array.isArray(n.offers) ? n.offers[0] : n.offers;
4187
- const images = [].concat(n.image ?? []).flat().map((img) => typeof img === "string" ? img : img?.url ?? img?.contentUrl).filter(Boolean).map((u, i) => ({ url: absolutize(pageUrl, String(u)), position: i, width: null, height: null, alt: null })).filter((img) => img.url);
4188
- const url = String(n.url ?? n["@id"] ?? pageUrl);
4189
- return normalizeProduct({
4190
- externalKey: String(n.sku || n.productID || n.mpn || stableKey(url)),
4191
- title: String(n.name ?? "Product"),
4192
- descriptionHtml: n.description ? String(n.description) : null,
4193
- url: absolutize(pageUrl, url) ?? pageUrl,
4194
- vendor: n.brand?.name ?? (typeof n.brand === "string" ? n.brand : null),
4195
- productType: n.category ? String(n.category) : null,
4196
- category: n.category ? String(n.category) : null,
4197
- price: offers?.price != null ? Number(offers.price) : null,
4198
- compareAtPrice: null,
4199
- currency: offers?.priceCurrency ?? null,
4200
- available: offers?.availability ? /instock/i.test(String(offers.availability)) : null,
4201
- tags: [],
4202
- variants: [],
4203
- images,
4204
- raw: n
4205
- });
4206
- });
4207
- }
4208
- function parseProductHtml(html, pageUrl) {
4209
- const $ = loadHtml(html);
4210
- const title = attr($.querySelector('meta[property="og:title"]'), "content") || textOf($.querySelector("h1")) || textOf($.querySelector("title"));
4211
- if (!title) return null;
4212
- const desc = attr($.querySelector('meta[property="og:description"]'), "content") || attr($.querySelector('meta[name="description"]'), "content") || null;
4213
- const images = [];
4214
- const og = attr($.querySelector('meta[property="og:image"]'), "content");
4215
- if (og) {
4216
- const abs = absolutize(pageUrl, og);
4217
- if (abs) images.push({ url: abs, position: 0, width: null, height: null, alt: null });
4218
- }
4219
- for (const el of $.querySelectorAll("img[src]")) {
4220
- if (images.length >= 12) break;
4221
- const src = attr(el, "src") || attr(el, "data-src");
4222
- const abs = src ? absolutize(pageUrl, src) : null;
4223
- if (!abs || /logo|icon|sprite|pixel|avatar/i.test(abs)) continue;
4224
- if (images.some((x) => x.url === abs)) continue;
4225
- images.push({ url: abs, position: images.length, width: null, height: null, alt: attr(el, "alt") ?? null });
4226
- }
4227
- const canonical = attr($.querySelector('link[rel="canonical"]'), "href");
4228
- const url = canonical ? absolutize(pageUrl, canonical) ?? pageUrl : pageUrl;
4229
- return normalizeProduct({
4230
- externalKey: stableKey(url),
4231
- title,
4232
- descriptionHtml: desc,
4233
- url,
4234
- images,
4235
- variants: [],
4236
- tags: [],
4237
- raw: { source: "html" }
4238
- });
5848
+ return preferCanonicalLocale([...out]);
4239
5849
  }
4240
5850
  async function extractFeedUrls(ctx) {
4241
5851
  const origin = originOf(ctx.baseUrl);
@@ -4339,41 +5949,15 @@ var genericAdapter = {
4339
5949
  productKeys: [...productUrls].map(stableKey),
4340
5950
  productUrls: [...productUrls],
4341
5951
  estimatedTotal: productUrls.size || null,
5952
+ // Nothing here but addresses: every product is its own page fetch.
5953
+ byPage: true,
4342
5954
  warnings
4343
5955
  };
4344
5956
  },
4345
5957
  async fetchAll(ctx, discovered) {
4346
- const out = [];
4347
- const seen = /* @__PURE__ */ new Set();
4348
- await mapPool(
4349
- discovered.productUrls,
4350
- 5,
4351
- async (u) => {
4352
- try {
4353
- const { ok, text, url } = await httpText(u, {
4354
- fetchImpl: ctx.fetchImpl,
4355
- signal: ctx.signal,
4356
- accept: "text/html"
4357
- });
4358
- if (!ok) return;
4359
- const fromLd = extractJsonLdProducts(text, url);
4360
- const list2 = fromLd.length ? fromLd : [parseProductHtml(text, url)].filter(Boolean);
4361
- for (const p of list2) {
4362
- if (seen.has(p.externalKey)) continue;
4363
- seen.add(p.externalKey);
4364
- out.push(p);
4365
- }
4366
- ctx.onProgress?.({
4367
- stage: "fetching_products",
4368
- fetched: out.length,
4369
- discovered: discovered.productUrls.length
4370
- });
4371
- } catch {
4372
- }
4373
- },
4374
- ctx.signal
4375
- );
4376
- return out;
5958
+ return fetchProductPages(ctx, discovered.productUrls, {
5959
+ onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched, discovered: discovered.productUrls.length })
5960
+ });
4377
5961
  }
4378
5962
  };
4379
5963
 
@@ -4481,6 +6065,9 @@ var woocommerceAdapter = {
4481
6065
  productKeys: [...keys],
4482
6066
  productUrls: [...productUrls],
4483
6067
  estimatedTotal: keys.size || productUrls.size || null,
6068
+ // The WooCommerce store API answers in pages of a hundred, so the whole
6069
+ // catalogue is a handful of requests and needs no batching of ours.
6070
+ byPage: false,
4484
6071
  warnings
4485
6072
  };
4486
6073
  },
@@ -4512,30 +6099,13 @@ var woocommerceAdapter = {
4512
6099
  }
4513
6100
  if (apiWorked && out.length) return out;
4514
6101
  const urls = discovered.productUrls.length ? discovered.productUrls : [...new Set(discovered.productKeys)];
4515
- await mapPool(
4516
- urls,
4517
- 5,
4518
- async (u) => {
4519
- try {
4520
- const {
4521
- ok,
4522
- text,
4523
- url: finalUrl
4524
- } = await httpText(u, { fetchImpl: ctx.fetchImpl, signal: ctx.signal, accept: "text/html" });
4525
- if (!ok) return;
4526
- const fromLd = extractJsonLdProducts(text, finalUrl);
4527
- const products = fromLd.length ? fromLd : [parseProductHtml(text, finalUrl)].filter(Boolean);
4528
- for (const p of products) {
4529
- if (seen.has(p.externalKey)) continue;
4530
- seen.add(p.externalKey);
4531
- out.push(p);
4532
- }
4533
- ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
4534
- } catch {
4535
- }
4536
- },
4537
- ctx.signal
4538
- );
6102
+ for (const p of await fetchProductPages(ctx, urls, {
6103
+ onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched: out.length + fetched })
6104
+ })) {
6105
+ if (seen.has(p.externalKey)) continue;
6106
+ seen.add(p.externalKey);
6107
+ out.push(p);
6108
+ }
4539
6109
  return out;
4540
6110
  }
4541
6111
  };
@@ -4603,42 +6173,16 @@ var webflowAdapter = {
4603
6173
  return {
4604
6174
  productKeys: [...productUrls],
4605
6175
  productUrls: [...productUrls],
4606
- estimatedTotal: productUrls.size || null,
4607
- warnings
4608
- };
4609
- },
4610
- async fetchAll(ctx, discovered) {
4611
- const out = [];
4612
- const seen = /* @__PURE__ */ new Set();
4613
- await mapPool(
4614
- discovered.productUrls,
4615
- 5,
4616
- async (u) => {
4617
- try {
4618
- const { ok, text, url } = await httpText(u, {
4619
- fetchImpl: ctx.fetchImpl,
4620
- signal: ctx.signal,
4621
- accept: "text/html"
4622
- });
4623
- if (!ok) return;
4624
- const fromLd = extractJsonLdProducts(text, url);
4625
- const list2 = fromLd.length ? fromLd : [parseProductHtml(text, url)].filter(Boolean);
4626
- for (const p of list2) {
4627
- if (seen.has(p.externalKey)) continue;
4628
- seen.add(p.externalKey);
4629
- out.push(p);
4630
- }
4631
- ctx.onProgress?.({
4632
- stage: "fetching_products",
4633
- fetched: out.length,
4634
- discovered: discovered.productUrls.length
4635
- });
4636
- } catch {
4637
- }
4638
- },
4639
- ctx.signal
4640
- );
4641
- return out;
6176
+ estimatedTotal: productUrls.size || null,
6177
+ // Webflow has no catalogue API to ask: every product is its own page.
6178
+ byPage: true,
6179
+ warnings
6180
+ };
6181
+ },
6182
+ async fetchAll(ctx, discovered) {
6183
+ return fetchProductPages(ctx, discovered.productUrls, {
6184
+ onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched, discovered: discovered.productUrls.length })
6185
+ });
4642
6186
  }
4643
6187
  };
4644
6188
 
@@ -4680,7 +6224,7 @@ function baseProgress(platform = "unknown") {
4680
6224
  warnings: []
4681
6225
  };
4682
6226
  }
4683
- async function runCatalogIngestion(opts) {
6227
+ async function discoverCatalog(opts) {
4684
6228
  const fetchImpl = opts.fetchImpl ?? fetch;
4685
6229
  const progress = baseProgress();
4686
6230
  const emit = (patch2) => {
@@ -4723,57 +6267,247 @@ async function runCatalogIngestion(opts) {
4723
6267
  warnings: [...progress.warnings, ...discovered.warnings],
4724
6268
  message: `Found ${discovered.estimatedTotal ?? discovered.productUrls.length} products`
4725
6269
  });
4726
- if (!(discovered.estimatedTotal ?? discovered.productUrls.length)) {
4727
- emit({
4728
- stage: "failed",
4729
- errors: [
4730
- ...progress.errors,
4731
- {
4732
- code: "empty_catalog",
4733
- message: detection.platform === "generic" ? "No public product catalog found. This store may be JavaScript-rendered or require authentication." : `No products discovered on this ${detection.platform} store.`
4734
- }
4735
- ]
4736
- });
4737
- return { baseUrl: detection.baseUrl, detection, products: [], progress };
6270
+ const estimatedTotal = discovered.estimatedTotal ?? discovered.productUrls.length;
6271
+ let empty = null;
6272
+ if (!estimatedTotal) {
6273
+ empty = {
6274
+ code: "empty_catalog",
6275
+ message: detection.platform === "generic" ? "No public product catalog found. This store may be JavaScript-rendered or require authentication." : `No products discovered on this ${detection.platform} store.`
6276
+ };
6277
+ emit({ stage: "failed", errors: [...progress.errors, empty] });
4738
6278
  }
4739
- emit({ stage: "fetching_products", message: "Fetching product details" });
4740
- let products = [];
6279
+ return {
6280
+ baseUrl: detection.baseUrl,
6281
+ detection,
6282
+ adapter,
6283
+ ctx: { ...ctx, baseUrl: detection.baseUrl },
6284
+ productUrls: discovered.productUrls,
6285
+ productKeys: discovered.productKeys,
6286
+ byPage: (discovered.byPage ?? false) && discovered.productUrls.length > 0,
6287
+ estimatedTotal,
6288
+ progress,
6289
+ empty,
6290
+ emit,
6291
+ async fetchAll() {
6292
+ emit({ stage: "fetching_products", message: "Fetching product details" });
6293
+ try {
6294
+ return dedupeProducts(await adapter.fetchAll({ ...ctx, baseUrl: detection.baseUrl }, discovered));
6295
+ } catch (err) {
6296
+ emit({
6297
+ stage: "partial",
6298
+ errors: [...progress.errors, { code: "fetch_failed", message: String(err?.message ?? err), retryable: true }]
6299
+ });
6300
+ return [];
6301
+ }
6302
+ }
6303
+ };
6304
+ }
6305
+
6306
+ // ../catalog/src/robots.ts
6307
+ var ALLOW_ALL = { rules: [], crawlDelayMs: 0 };
6308
+ function toRegExp(pattern) {
6309
+ const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
6310
+ const anchored = escaped.endsWith("\\$") ? `${escaped.slice(0, -2)}$` : escaped;
6311
+ return new RegExp(`^${anchored}`);
6312
+ }
6313
+ function parseRobots(text, agent = USER_AGENT2) {
6314
+ const lines = text.split(/\r?\n/).map((l) => l.replace(/#.*$/, "").trim());
6315
+ const groups = [];
6316
+ let current = null;
6317
+ let expectingAgents = false;
6318
+ for (const line of lines) {
6319
+ const at = line.indexOf(":");
6320
+ if (at < 0) continue;
6321
+ const field = line.slice(0, at).trim().toLowerCase();
6322
+ const value = line.slice(at + 1).trim();
6323
+ if (field === "user-agent") {
6324
+ if (!current || !expectingAgents) {
6325
+ current = { agents: [], rules: [], delay: 0 };
6326
+ groups.push(current);
6327
+ expectingAgents = true;
6328
+ }
6329
+ current.agents.push(value.toLowerCase());
6330
+ continue;
6331
+ }
6332
+ if (!current) continue;
6333
+ expectingAgents = false;
6334
+ if (field === "disallow" && value) current.rules.push({ allow: false, pattern: value });
6335
+ else if (field === "allow" && value) current.rules.push({ allow: true, pattern: value });
6336
+ else if (field === "crawl-delay") {
6337
+ const n = Number(value);
6338
+ if (Number.isFinite(n) && n > 0) current.delay = n;
6339
+ }
6340
+ }
6341
+ const token = agent.toLowerCase();
6342
+ const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && token.includes(a)));
6343
+ const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
6344
+ return {
6345
+ rules: chosen.flatMap((g) => g.rules),
6346
+ crawlDelayMs: Math.max(0, ...chosen.map((g) => g.delay)) * 1e3
6347
+ };
6348
+ }
6349
+ function isAllowed(robots, url) {
6350
+ let path;
4741
6351
  try {
4742
- products = await adapter.fetchAll({ ...ctx, baseUrl: detection.baseUrl }, discovered);
4743
- } catch (err) {
4744
- emit({
4745
- stage: "partial",
4746
- errors: [...progress.errors, { code: "fetch_failed", message: String(err?.message ?? err), retryable: true }]
4747
- });
6352
+ const parsed = new URL(url);
6353
+ path = `${parsed.pathname}${parsed.search}`;
6354
+ } catch {
6355
+ return true;
4748
6356
  }
4749
- products = dedupeProducts(products);
4750
- emit({
4751
- stage: products.length ? "fetching_products" : progress.stage,
4752
- fetched: products.length,
4753
- discovered: Math.max(progress.discovered, products.length),
4754
- message: `Fetched ${products.length} products`
4755
- });
4756
- if (!products.length) {
4757
- emit({
4758
- stage: "failed",
4759
- errors: [
4760
- ...progress.errors,
4761
- { code: "no_products_fetched", message: "Discovery found URLs but no product payloads could be parsed." }
4762
- ]
6357
+ let best = null;
6358
+ for (const rule of robots.rules) {
6359
+ if (!toRegExp(rule.pattern).test(path)) continue;
6360
+ const length = rule.pattern.replace(/[*$]/g, "").length;
6361
+ if (!best || length > best.length || length === best.length && rule.allow) {
6362
+ best = { allow: rule.allow, length };
6363
+ }
6364
+ }
6365
+ return best ? best.allow : true;
6366
+ }
6367
+ async function fetchRobots(ctx) {
6368
+ try {
6369
+ const { ok, text } = await httpText(`${originOf(ctx.baseUrl)}/robots.txt`, {
6370
+ fetchImpl: ctx.fetchImpl,
6371
+ signal: ctx.signal,
6372
+ retries: 0,
6373
+ accept: "text/plain",
6374
+ maxBytes: 512e3
4763
6375
  });
6376
+ if (!ok || !text) return ALLOW_ALL;
6377
+ return parseRobots(text);
6378
+ } catch {
6379
+ return ALLOW_ALL;
6380
+ }
6381
+ }
6382
+
6383
+ // ../catalog/src/candidates.ts
6384
+ var DEFAULT_SCAN_BUDGET = {
6385
+ maxPreviewPages: 24,
6386
+ maxBytesPerPage: 15e5,
6387
+ maxTotalBytes: 4e7,
6388
+ budgetMs: 25e3,
6389
+ concurrency: 4,
6390
+ /**
6391
+ * Time the preview is owed even when discovery has already spent the lot.
6392
+ *
6393
+ * Discovery on a large store can run long, and when it overran the whole
6394
+ * budget the preview read zero pages and the store was reported as one we
6395
+ * could not open - from a site that was answering every request with a 200.
6396
+ * Being slow to list a catalog is not the same as being shut.
6397
+ */
6398
+ previewFloorMs: 12e3
6399
+ };
6400
+ var COMMERCE_SIGNALS = /* @__PURE__ */ new Set([
6401
+ "products.json",
6402
+ "shopify-html",
6403
+ "wc-store-api",
6404
+ "woocommerce-html",
6405
+ "webflow-commerce"
6406
+ ]);
6407
+ function wwwVariant(baseUrl) {
6408
+ try {
6409
+ const url = new URL(baseUrl);
6410
+ const labels = url.hostname.split(".");
6411
+ if (labels.length !== 2 || /^\d+$/.test(labels[labels.length - 1])) return null;
6412
+ url.hostname = `www.${url.hostname}`;
6413
+ return url.origin;
6414
+ } catch {
6415
+ return null;
6416
+ }
6417
+ }
6418
+ async function scanForCandidates(opts) {
6419
+ const first = await scanOnce(opts, originOf(normalizeStoreUrl(opts.url)));
6420
+ if (first.verdict === "found") return first;
6421
+ const alternate = wwwVariant(first.baseUrl);
6422
+ if (!alternate) return first;
6423
+ const second = await scanOnce(opts, alternate);
6424
+ return second.verdict === "found" ? second : first;
6425
+ }
6426
+ async function scanOnce(opts, baseUrl) {
6427
+ const started = Date.now();
6428
+ const budget = { ...DEFAULT_SCAN_BUDGET, ...opts.budget };
6429
+ const deadline = started + budget.budgetMs;
6430
+ const fetchImpl = opts.fetchImpl ?? fetch;
6431
+ const ctx = {
6432
+ fetchImpl,
6433
+ baseUrl,
6434
+ signal: opts.signal,
6435
+ onProgress: opts.onProgress
6436
+ };
6437
+ const warnings = [];
6438
+ const stats = { pages: 0, bytes: 0 };
6439
+ const robots = await fetchRobots(ctx);
6440
+ if (robots.crawlDelayMs) warnings.push("This site asks readers to go slowly, so the preview is smaller");
6441
+ opts.onProgress?.({ stage: "discovering", message: "Looking for a shop" });
6442
+ const detection = await detectPlatform(ctx);
6443
+ const adapter = adapterFor(detection.platform);
6444
+ let discovered;
6445
+ try {
6446
+ discovered = await adapter.discover(ctx);
6447
+ } catch {
6448
+ return done("blocked", [], [], 0, "none", [...warnings, "The product list could not be read"]);
6449
+ }
6450
+ warnings.push(...discovered.warnings);
6451
+ const urls = discovered.productUrls.filter((u) => isAllowed(robots, u));
6452
+ const refused = discovered.productUrls.length - urls.length;
6453
+ if (refused > 0) warnings.push(`${refused} pages are disallowed by this site's robots.txt`);
6454
+ const fromApi = detection.platform !== "generic" && !discovered.hints?.includes("json-blocked");
6455
+ const countSource = urls.length === 0 ? "none" : fromApi ? "api" : "sitemap";
6456
+ if (urls.length === 0) {
6457
+ const commerce = detection.signals.some((sig) => COMMERCE_SIGNALS.has(sig));
6458
+ return done(commerce ? "likely" : "none", [], [], 0, countSource, warnings);
6459
+ }
6460
+ opts.onProgress?.({ stage: "discovering", discovered: urls.length, message: "Reading a few products" });
6461
+ const previewDeadline = Math.max(deadline, Date.now() + budget.previewFloorMs);
6462
+ const read = dedupeProducts(
6463
+ await fetchProductPages(ctx, urls, {
6464
+ want: budget.maxPreviewPages,
6465
+ stats,
6466
+ concurrency: budget.concurrency,
6467
+ maxBytes: budget.maxBytesPerPage,
6468
+ maxTotalBytes: budget.maxTotalBytes,
6469
+ delayMs: robots.crawlDelayMs,
6470
+ deadline: previewDeadline,
6471
+ onProduct: (fetched) => opts.onProgress?.({ stage: "fetching_products", fetched, discovered: urls.length })
6472
+ })
6473
+ );
6474
+ const preview = read.slice(0, budget.maxPreviewPages);
6475
+ const verdict = preview.length ? "found" : "blocked";
6476
+ return done(verdict, preview, urls, urls.length, countSource, warnings);
6477
+ function done(verdict2, candidates, candidateUrls, count, source, notes) {
6478
+ return {
6479
+ baseUrl,
6480
+ platform: detection?.platform ?? "unknown",
6481
+ signals: detection?.signals ?? [],
6482
+ verdict: verdict2,
6483
+ count,
6484
+ countSource: source,
6485
+ candidates,
6486
+ candidateUrls,
6487
+ truncated: count > candidates.length,
6488
+ warnings: notes,
6489
+ spent: { pages: stats.pages, bytes: stats.bytes, ms: Date.now() - started }
6490
+ };
4764
6491
  }
4765
- return { baseUrl: detection.baseUrl, detection, products, progress };
4766
6492
  }
4767
6493
 
4768
6494
  // src/catalogImport.ts
4769
6495
  var running = /* @__PURE__ */ new Map();
4770
6496
  function cancelCatalogImport(jobId) {
4771
- const ctrl = running.get(jobId);
4772
- if (!ctrl) return false;
4773
- ctrl.abort();
6497
+ const job = running.get(jobId);
6498
+ if (!job) return false;
6499
+ job.ctrl.abort();
4774
6500
  return true;
4775
6501
  }
4776
- function startCatalogImport(deps, brandId, url) {
6502
+ async function settleCatalogImports(timeoutMs = 5e3) {
6503
+ if (!running.size) return;
6504
+ for (const { ctrl } of running.values()) ctrl.abort();
6505
+ await Promise.race([
6506
+ Promise.allSettled([...running.values()].map((j) => j.done)),
6507
+ new Promise((r) => setTimeout(r, timeoutMs))
6508
+ ]);
6509
+ }
6510
+ function startCatalogImport(deps, brandId, url, opts = {}) {
4777
6511
  const { core } = deps;
4778
6512
  if (!core.store.getBrand(brandId)) throw Object.assign(new Error("brand not found"), { statusCode: 404 });
4779
6513
  let normalized;
@@ -4784,57 +6518,183 @@ function startCatalogImport(deps, brandId, url) {
4784
6518
  }
4785
6519
  const job = core.catalog.createJob({ brandId, url: normalized });
4786
6520
  const ctrl = new AbortController();
4787
- running.set(job.id, ctrl);
4788
- void runJob(deps, job.id, brandId, normalized, ctrl.signal).finally(() => {
6521
+ const done = runJob(deps, job.id, brandId, normalized, ctrl.signal, opts.only).finally(() => {
4789
6522
  running.delete(job.id);
4790
6523
  });
6524
+ void done.catch(() => {
6525
+ });
6526
+ running.set(job.id, { ctrl, done });
4791
6527
  return { jobId: job.id };
4792
6528
  }
4793
- async function runJob(deps, jobId, brandId, url, signal) {
6529
+ function progressWriter(patch2) {
6530
+ let lastStage = "";
6531
+ let lastMessage = null;
6532
+ return (p) => {
6533
+ const reported = p.stage === "queued" ? "discovering" : p.stage;
6534
+ const stage = TERMINAL.has(reported) ? "fetching_products" : reported;
6535
+ const message = p.message ?? null;
6536
+ const notable = stage !== lastStage || message !== lastMessage || p.fetched % 10 === 0;
6537
+ if (!notable) return;
6538
+ lastStage = stage;
6539
+ lastMessage = message;
6540
+ patch2({
6541
+ stage,
6542
+ platform: p.platform,
6543
+ discovered: p.discovered,
6544
+ fetched: p.fetched,
6545
+ warnings: p.warnings,
6546
+ errors: p.errors,
6547
+ message
6548
+ });
6549
+ };
6550
+ }
6551
+ async function runJob(deps, jobId, brandId, url, signal, only) {
4794
6552
  const { core, fetchImpl } = deps;
4795
6553
  const patch2 = (p) => core.catalog.updateJob(jobId, p);
4796
6554
  try {
4797
- patch2({ stage: "discovering", message: "Detecting store platform" });
4798
- const result = await runCatalogIngestion({
4799
- url,
4800
- fetchImpl,
4801
- signal,
4802
- onProgress: (p) => {
6555
+ const tally2 = { fetched: 0, upserted: 0, imagesDone: 0, imagesTotal: 0, errors: [], seenKeys: [] };
6556
+ const ctx = { fetchImpl: fetchImpl ?? fetch, baseUrl: url, signal };
6557
+ let urls = [];
6558
+ let bulk = null;
6559
+ let baseUrl;
6560
+ let platform;
6561
+ let discovered;
6562
+ let warnings = [];
6563
+ const sweep = !only?.length;
6564
+ if (only?.length) {
6565
+ patch2({ stage: "fetching_products", message: `Importing ${only.length} products`, discovered: only.length });
6566
+ const detection = await detectPlatform(ctx);
6567
+ urls = only;
6568
+ baseUrl = url;
6569
+ platform = detection.platform;
6570
+ discovered = only.length;
6571
+ } else {
6572
+ patch2({ stage: "discovering", message: "Detecting store platform" });
6573
+ const found = await discoverCatalog({ url, fetchImpl, signal, onProgress: progressWriter(patch2) });
6574
+ baseUrl = found.baseUrl;
6575
+ platform = found.detection.platform;
6576
+ discovered = found.estimatedTotal;
6577
+ warnings = found.progress.warnings;
6578
+ if (found.empty || !found.estimatedTotal) {
6579
+ const source = core.catalog.upsertSource(brandId, found.baseUrl, platform);
6580
+ patch2({ sourceId: source.id, platform });
4803
6581
  patch2({
4804
- stage: p.stage === "queued" ? "discovering" : p.stage,
4805
- platform: p.platform,
4806
- discovered: p.discovered,
4807
- fetched: p.fetched,
4808
- warnings: p.warnings,
4809
- errors: p.errors,
4810
- message: p.message ?? null
6582
+ stage: "no_catalog",
6583
+ errors: [],
6584
+ warnings: found.progress.warnings,
6585
+ message: "No shop found on this site",
6586
+ finished: true
4811
6587
  });
6588
+ core.catalog.setSourceStatus(source.id, "empty", true);
6589
+ return;
4812
6590
  }
4813
- });
4814
- if (signal.aborted) {
4815
- patch2({ stage: "failed", message: "Import cancelled", finished: true });
4816
- return;
6591
+ if (!found.byPage) {
6592
+ const products = await found.fetchAll();
6593
+ if (signal.aborted) {
6594
+ patch2({ stage: "cancelled", errors: [], message: "Stopped before anything was saved", finished: true });
6595
+ return;
6596
+ }
6597
+ if (!products.length) {
6598
+ const source = core.catalog.upsertSource(brandId, baseUrl, platform);
6599
+ patch2({ sourceId: source.id, platform });
6600
+ patch2({
6601
+ stage: "failed",
6602
+ errors: found.progress.errors,
6603
+ warnings: found.progress.warnings,
6604
+ message: found.progress.errors[0]?.message ?? "No products imported",
6605
+ finished: true
6606
+ });
6607
+ core.catalog.setSourceStatus(source.id, "failed", true);
6608
+ return;
6609
+ }
6610
+ bulk = products;
6611
+ discovered = found.progress.discovered;
6612
+ warnings = found.progress.warnings;
6613
+ }
6614
+ if (!bulk) {
6615
+ urls = found.productUrls;
6616
+ patch2({ stage: "fetching_products", discovered, message: `Reading ${discovered.toLocaleString()} products` });
6617
+ }
6618
+ }
6619
+ const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered);
6620
+ const pictures = drainPictures(deps, jobId, brandId, tally2, signal);
6621
+ let pictureErrors;
6622
+ let pictureReasons = {};
6623
+ const stats = { pages: 0, bytes: 0, refused: 0, reasons: {} };
6624
+ try {
6625
+ if (bulk) for (const p of bulk) run2.write(p);
6626
+ else
6627
+ await fetchProductPages(ctx, urls, {
6628
+ concurrency: IMPORT_CONCURRENCY,
6629
+ maxBytes: 15e5,
6630
+ onEach: run2.write,
6631
+ stats,
6632
+ // Past this many refusals in a row the store is not going to change
6633
+ // its mind inside this import, and waiting out its cooldown for the
6634
+ // rest of the catalogue is time nobody gets a product for.
6635
+ giveUpAfterRefusals: 24
6636
+ });
6637
+ if (!signal.aborted && tally2.upserted) {
6638
+ patch2({
6639
+ stage: "processing_assets",
6640
+ message: `Downloading ${tally2.imagesTotal.toLocaleString()} pictures`,
6641
+ upserted: tally2.upserted,
6642
+ fetched: tally2.fetched
6643
+ });
6644
+ }
6645
+ } finally {
6646
+ pictures.stop();
6647
+ const settled = await pictures.done;
6648
+ pictureErrors = settled.errors;
6649
+ pictureReasons = settled.reasons;
4817
6650
  }
4818
- const source = core.catalog.upsertSource(brandId, result.baseUrl, result.detection.platform);
4819
- patch2({ sourceId: source.id, platform: result.detection.platform });
4820
- core.catalog.setSourceStatus(source.id, "importing");
4821
- if (!result.products.length) {
4822
- const stage = result.progress.stage === "failed" ? "failed" : "failed";
6651
+ if (signal.aborted) {
4823
6652
  patch2({
4824
- stage,
4825
- errors: result.progress.errors,
4826
- warnings: result.progress.warnings,
4827
- message: result.progress.errors[0]?.message ?? "No products imported",
6653
+ stage: "cancelled",
6654
+ errors: [],
6655
+ message: tally2.upserted ? `Stopped after saving ${tally2.upserted.toLocaleString()} products` : "Stopped before anything was saved",
4828
6656
  finished: true
4829
6657
  });
4830
- core.catalog.setSourceStatus(source.id, "failed", true);
6658
+ core.catalog.setSourceStatus(run2.sourceId, "partial", true);
6659
+ return;
6660
+ }
6661
+ run2.finish({
6662
+ sweep,
6663
+ warnings,
6664
+ errors: pictureErrors,
6665
+ refused: stats.refused,
6666
+ reasons: { ...stats.reasons, ...pictureReasons },
6667
+ asked: bulk ? bulk.length : urls.length,
6668
+ // Every address was read. A bulk API hands the catalogue over whole, so
6669
+ // there is nothing to cover.
6670
+ covered: bulk ? true : stats.pages >= urls.length
6671
+ });
6672
+ } catch (err) {
6673
+ if (signal.aborted) {
6674
+ patch2({ stage: "cancelled", errors: [], message: "Stopped before anything was saved", finished: true });
4831
6675
  return;
4832
6676
  }
4833
- patch2({ stage: "fetching_products", fetched: result.products.length, message: "Saving products" });
4834
- let upserted = 0;
4835
- const seenKeys = [];
4836
- for (const p of result.products) {
4837
- if (signal.aborted) break;
6677
+ patch2({
6678
+ stage: "failed",
6679
+ message: String(err?.message ?? err),
6680
+ errors: [{ code: "import_failed", message: String(err?.message ?? err) }],
6681
+ finished: true
6682
+ });
6683
+ }
6684
+ }
6685
+ var IMAGES_PER_PRODUCT = 3;
6686
+ var TERMINAL = /* @__PURE__ */ new Set(["completed", "partial", "no_catalog", "cancelled", "failed"]);
6687
+ var IMAGE_CONCURRENCY = 4;
6688
+ var sleep6 = (ms) => new Promise((r) => setTimeout(r, ms));
6689
+ function beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered) {
6690
+ const { core } = deps;
6691
+ const patch2 = (p) => core.catalog.updateJob(jobId, p);
6692
+ const source = core.catalog.upsertSource(brandId, baseUrl, platform);
6693
+ patch2({ sourceId: source.id, platform });
6694
+ core.catalog.setSourceStatus(source.id, "importing");
6695
+ return {
6696
+ sourceId: source.id,
6697
+ write(p) {
4838
6698
  core.catalog.upsertProduct({
4839
6699
  sourceId: source.id,
4840
6700
  brandId,
@@ -4851,7 +6711,9 @@ async function runJob(deps, jobId, brandId, url, signal) {
4851
6711
  compareAtPrice: p.compareAtPrice,
4852
6712
  currency: p.currency,
4853
6713
  available: p.available,
4854
- raw: p.raw,
6714
+ // `raw` is the whole crawled payload, 14.2 KB a product on gymshark
6715
+ // and 32 MB across its catalog, written to sqlite and read by nothing.
6716
+ raw: null,
4855
6717
  variants: p.variants,
4856
6718
  images: (p.images ?? []).map((img) => ({
4857
6719
  sourceUrl: img.url,
@@ -4860,93 +6722,120 @@ async function runJob(deps, jobId, brandId, url, signal) {
4860
6722
  height: img.height,
4861
6723
  alt: img.alt
4862
6724
  })),
4863
- collections: (p.collections ?? []).map((c) => ({
4864
- externalKey: c,
4865
- title: c
4866
- }))
6725
+ collections: (p.collections ?? []).map((c) => ({ externalKey: c, title: c }))
6726
+ });
6727
+ tally2.seenKeys.push(p.externalKey);
6728
+ tally2.upserted++;
6729
+ tally2.fetched++;
6730
+ patch2({ upserted: tally2.upserted, fetched: tally2.fetched });
6731
+ },
6732
+ finish({
6733
+ sweep,
6734
+ warnings,
6735
+ errors,
6736
+ refused = 0,
6737
+ covered = true,
6738
+ reasons = {},
6739
+ asked = 0
6740
+ }) {
6741
+ const mayRetire = sweep && covered && refused === 0 && tally2.upserted > 0;
6742
+ if (mayRetire) core.catalog.markMissingUnavailable(source.id, tally2.seenKeys);
6743
+ tally2.errors = errors;
6744
+ const partial = !covered || refused > 0;
6745
+ const savedNothing = tally2.upserted === 0;
6746
+ const said = summarise(reasons, tally2.upserted, asked || discovered || tally2.fetched);
6747
+ const stage = savedNothing ? "failed" : partial ? "partial" : "completed";
6748
+ const message = savedNothing ? said ?? "We found the products on this site, but none of them could be imported." : said ?? (partial ? `Imported ${tally2.upserted.toLocaleString()} products with ${(errors.length + refused).toLocaleString()} issue${errors.length + refused === 1 ? "" : "s"}` : `Imported ${tally2.upserted.toLocaleString()} products`);
6749
+ patch2({
6750
+ stage,
6751
+ upserted: tally2.upserted,
6752
+ fetched: tally2.fetched,
6753
+ imagesDone: tally2.imagesDone,
6754
+ imagesTotal: tally2.imagesTotal,
6755
+ errors,
6756
+ warnings,
6757
+ message,
6758
+ finished: true
4867
6759
  });
4868
- seenKeys.push(p.externalKey);
4869
- upserted++;
4870
- if (upserted % 10 === 0) patch2({ upserted, fetched: result.products.length });
6760
+ core.catalog.setSourceStatus(source.id, savedNothing ? "failed" : partial ? "partial" : "ready", true);
4871
6761
  }
4872
- patch2({ upserted, fetched: result.products.length });
4873
- core.catalog.markMissingUnavailable(source.id, seenKeys);
4874
- const pending = core.catalog.listImagesNeedingAssets(brandId, 5e4);
4875
- patch2({
4876
- stage: "processing_assets",
4877
- imagesTotal: pending.length,
4878
- imagesDone: 0,
4879
- message: `Downloading ${pending.length} images`
4880
- });
4881
- const errors = [...core.catalog.getJob(jobId)?.errors ?? []];
4882
- let imagesDone = 0;
4883
- await mapPool(
4884
- pending,
4885
- 6,
4886
- async (img) => {
4887
- if (signal.aborted) return;
4888
- try {
4889
- const res = await httpGet(img.sourceUrl, { fetchImpl, signal, timeoutMs: 4e4, retries: 2 });
4890
- if (!res.ok) {
4891
- errors.push({ code: "image_http", message: `HTTP ${res.status}`, url: img.sourceUrl });
4892
- return;
4893
- }
4894
- const buf = Buffer.from(await res.arrayBuffer());
4895
- if (!buf.length) {
4896
- errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
4897
- return;
4898
- }
4899
- const png = await sharp20(buf).rotate().png().toBuffer();
4900
- const meta = await sharp20(png).metadata();
4901
- const hash = core.images.save(png);
4902
- core.catalog.setImageAsset(img.productId, img.sourceUrl, `asset:${hash}`, {
4903
- width: meta.width,
4904
- height: meta.height
4905
- });
4906
- } catch (err) {
6762
+ };
6763
+ }
6764
+ var IMPORT_CONCURRENCY = 4;
6765
+ var PICTURE_ROUND = 60;
6766
+ function drainPictures(deps, jobId, brandId, tally2, signal, imagesPerProduct = IMAGES_PER_PRODUCT) {
6767
+ const { core, fetchImpl } = deps;
6768
+ const patch2 = (p) => core.catalog.updateJob(jobId, p);
6769
+ const errors = [...core.catalog.getJob(jobId)?.errors ?? []];
6770
+ const reasons = {};
6771
+ let stopped = false;
6772
+ const done = (async () => {
6773
+ while (!signal.aborted) {
6774
+ const round = core.catalog.listImagesNeedingAssets(brandId, PICTURE_ROUND).filter((img) => img.position < imagesPerProduct);
6775
+ if (!round.length) {
6776
+ if (stopped) break;
6777
+ await sleep6(150);
6778
+ continue;
6779
+ }
6780
+ tally2.imagesTotal += round.length;
6781
+ await mapPool(
6782
+ round,
6783
+ IMAGE_CONCURRENCY,
6784
+ async (img) => {
4907
6785
  if (signal.aborted) return;
4908
- errors.push({
4909
- code: "image_failed",
4910
- message: String(err?.message ?? err),
4911
- url: img.sourceUrl,
4912
- retryable: true
4913
- });
4914
- } finally {
4915
- imagesDone++;
4916
- if (imagesDone % 5 === 0 || imagesDone === pending.length) {
4917
- patch2({ imagesDone, imagesTotal: pending.length, errors });
6786
+ try {
6787
+ const res = await httpGet(img.sourceUrl, { fetchImpl, signal, timeoutMs: 4e4, retries: 2 });
6788
+ if (!res.ok) {
6789
+ errors.push({ code: "image_http", message: `HTTP ${res.status}`, url: img.sourceUrl });
6790
+ tally(reasons, imageFailure(res.status));
6791
+ return;
6792
+ }
6793
+ const buf = Buffer.from(await res.arrayBuffer());
6794
+ if (!buf.length) {
6795
+ errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
6796
+ tally(reasons, "IMAGE_EMPTY");
6797
+ return;
6798
+ }
6799
+ const probe = await sharp21(buf).metadata();
6800
+ const turned = (probe.orientation ?? 1) > 1;
6801
+ const keep = turned ? await sharp21(buf).rotate().toBuffer() : buf;
6802
+ const meta = turned ? await sharp21(keep).metadata() : probe;
6803
+ const hash = core.images.save(keep, meta.format ?? probe.format ?? "png");
6804
+ core.catalog.setImageAsset(img.productId, img.sourceUrl, `asset:${hash}`, {
6805
+ width: meta.width,
6806
+ height: meta.height
6807
+ });
6808
+ } catch (err) {
6809
+ if (signal.aborted) return;
6810
+ tally(reasons, thrownFailure(err, signal.aborted));
6811
+ errors.push({
6812
+ code: "image_failed",
6813
+ message: String(err?.message ?? err),
6814
+ url: img.sourceUrl,
6815
+ retryable: true
6816
+ });
6817
+ } finally {
6818
+ tally2.imagesDone++;
6819
+ patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
4918
6820
  }
4919
- }
4920
- },
4921
- signal
4922
- );
4923
- if (signal.aborted) {
4924
- patch2({ stage: "failed", message: "Import cancelled", errors, finished: true });
4925
- core.catalog.setSourceStatus(source.id, "failed", true);
4926
- return;
6821
+ },
6822
+ signal
6823
+ );
6824
+ const stuck = core.catalog.listImagesNeedingAssets(brandId, PICTURE_ROUND).filter((img) => img.position < imagesPerProduct);
6825
+ if (stuck.length && stuck[0]?.id === round[0]?.id) {
6826
+ errors.push({ code: "images_stalled", message: "Some pictures could not be downloaded" });
6827
+ break;
6828
+ }
4927
6829
  }
4928
- const stillMissing = core.catalog.listImagesNeedingAssets(brandId, 1).length;
4929
- const productCount = result.products.length;
4930
- const partial = !!errors.length || stillMissing > 0 || result.progress.discovered > 0 && productCount < result.progress.discovered * 0.9;
4931
- patch2({
4932
- stage: partial ? "partial" : "completed",
4933
- upserted,
4934
- imagesDone,
4935
- imagesTotal: pending.length,
4936
- errors,
4937
- warnings: result.progress.warnings,
4938
- message: partial ? `Imported ${upserted} products with ${errors.length} issue${errors.length === 1 ? "" : "s"}` : `Imported ${upserted} products`,
4939
- finished: true
4940
- });
4941
- core.catalog.setSourceStatus(source.id, partial ? "partial" : "ready", true);
4942
- } catch (err) {
4943
- patch2({
4944
- stage: "failed",
4945
- message: String(err?.message ?? err),
4946
- errors: [{ code: "import_failed", message: String(err?.message ?? err) }],
4947
- finished: true
4948
- });
4949
- }
6830
+ patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
6831
+ return { errors, reasons };
6832
+ })();
6833
+ return {
6834
+ stop() {
6835
+ stopped = true;
6836
+ },
6837
+ done
6838
+ };
4950
6839
  }
4951
6840
  function resolveLibraryProduct(core, brandId, productId) {
4952
6841
  const brand = core.store.getBrand(brandId);
@@ -5374,7 +7263,7 @@ async function generateStudioSet(deps, job, who, sourcePaths, signal) {
5374
7263
  return { hashes: kept.map((f) => byAngle.get(f.angle)), angles: kept.map((f) => f.angle) };
5375
7264
  }
5376
7265
  async function edgeBarGeometry(buf) {
5377
- const { data, info } = await sharp20(buf).greyscale().raw().toBuffer({ resolveWithObject: true });
7266
+ const { data, info } = await sharp21(buf).greyscale().raw().toBuffer({ resolveWithObject: true });
5378
7267
  const W = info.width;
5379
7268
  const H = info.height;
5380
7269
  const scan = (len, cross, at) => {
@@ -5428,7 +7317,7 @@ async function trimEdgeBars(core, hash) {
5428
7317
  const width = g.right - g.left + 1;
5429
7318
  const height = g.bottom - g.top + 1;
5430
7319
  if (width < g.W * 0.6 || height < g.H * 0.6) return hash;
5431
- const png = await sharp20(buf).extract({ left: g.left, top: g.top, width, height }).png().toBuffer();
7320
+ const png = await sharp21(buf).extract({ left: g.left, top: g.top, width, height }).png().toBuffer();
5432
7321
  return core.images.save(png);
5433
7322
  } catch {
5434
7323
  return hash;
@@ -5499,7 +7388,7 @@ async function identityCrop(core, hash) {
5499
7388
  if (!out) return void 0;
5500
7389
  try {
5501
7390
  const height = Math.min(IDENTITY_TARGET_HEIGHT, Math.round(nativeHeight * IDENTITY_MAX_UPSCALE)) || IDENTITY_TARGET_HEIGHT;
5502
- const png = await sharp20(core.images.read(out)).resize({ height, fit: "inside", kernel: "lanczos3", withoutEnlargement: false }).png().toBuffer();
7391
+ const png = await sharp21(core.images.read(out)).resize({ height, fit: "inside", kernel: "lanczos3", withoutEnlargement: false }).png().toBuffer();
5503
7392
  const scaled = core.images.save(png);
5504
7393
  identityCrops.set(hash, scaled);
5505
7394
  return scaled;
@@ -5528,12 +7417,12 @@ async function brandJsonWithIdentityCrops(core, json, characterIds) {
5528
7417
  return changed ? { ...json, characters } : json;
5529
7418
  }
5530
7419
  async function figureBox(buf) {
5531
- const meta = await sharp20(buf).metadata();
7420
+ const meta = await sharp21(buf).metadata();
5532
7421
  const W = meta.width ?? 0;
5533
7422
  const H = meta.height ?? 0;
5534
7423
  if (!W || !H) return null;
5535
7424
  for (const threshold of FIGURE_TRIM_THRESHOLDS) {
5536
- const { info } = await sharp20(buf).trim({ threshold }).toBuffer({ resolveWithObject: true });
7425
+ const { info } = await sharp21(buf).trim({ threshold }).toBuffer({ resolveWithObject: true });
5537
7426
  const left = Math.abs(info.trimOffsetLeft ?? 0);
5538
7427
  const top = Math.abs(info.trimOffsetTop ?? 0);
5539
7428
  const width = info.width ?? 0;
@@ -5566,13 +7455,13 @@ async function smartCover(core, hash, box) {
5566
7455
  if (!hash || !core.images.has(hash)) return void 0;
5567
7456
  try {
5568
7457
  const buf = core.images.read(hash);
5569
- const meta = await sharp20(buf).metadata();
7458
+ const meta = await sharp21(buf).metadata();
5570
7459
  const w = meta.width ?? 0;
5571
7460
  const h = meta.height ?? 0;
5572
7461
  if (!w || !h) return void 0;
5573
7462
  const raw = box(w, h);
5574
7463
  const target = { width: Math.max(1, raw.width), height: Math.max(1, raw.height) };
5575
- const png = await sharp20(buf).resize(target.width, target.height, { fit: "cover", position: "attention" }).png().toBuffer();
7464
+ const png = await sharp21(buf).resize(target.width, target.height, { fit: "cover", position: "attention" }).png().toBuffer();
5576
7465
  return core.images.save(png);
5577
7466
  } catch {
5578
7467
  return void 0;
@@ -5581,11 +7470,11 @@ async function smartCover(core, hash, box) {
5581
7470
  async function crop(core, hash, region, cap2) {
5582
7471
  if (!hash || !core.images.has(hash)) return void 0;
5583
7472
  try {
5584
- const meta = await sharp20(core.images.read(hash)).metadata();
7473
+ const meta = await sharp21(core.images.read(hash)).metadata();
5585
7474
  const w = meta.width ?? 0;
5586
7475
  const h = meta.height ?? 0;
5587
7476
  if (!w || !h) return void 0;
5588
- let pipeline = sharp20(core.images.read(hash)).extract(region(w, h));
7477
+ let pipeline = sharp21(core.images.read(hash)).extract(region(w, h));
5589
7478
  if (cap2) pipeline = pipeline.resize(cap2, cap2, { fit: "inside", withoutEnlargement: true });
5590
7479
  const png = await pipeline.png().toBuffer();
5591
7480
  return core.images.save(png);
@@ -5932,7 +7821,7 @@ var fromLab = (l, a, bb) => {
5932
7821
  return [clamp(R), clamp(G), clamp(B)];
5933
7822
  };
5934
7823
  var rawAt = async (png, edge) => {
5935
- let img = sharp20(png);
7824
+ let img = sharp21(png);
5936
7825
  if (edge) img = img.resize(edge, edge, { fit: "fill" });
5937
7826
  const { data, info } = await img.removeAlpha().raw().toBuffer({ resolveWithObject: true });
5938
7827
  return { data, width: info.width, height: info.height };
@@ -5995,7 +7884,7 @@ async function gradeComposite(originalPng, modelInputPng, modelOutputPng) {
5995
7884
  if (residual > GRADE_GATE_MEAN_DELTA) return null;
5996
7885
  const full = await rawAt(originalPng);
5997
7886
  applyAffine(full, T);
5998
- const image = await sharp20(full.data, {
7887
+ const image = await sharp21(full.data, {
5999
7888
  raw: { width: full.width, height: full.height, channels: 3 }
6000
7889
  }).png().toBuffer();
6001
7890
  return { image, residual };
@@ -6169,7 +8058,7 @@ function fitExpandToBudget(plan, source, pixelBudget) {
6169
8058
  }
6170
8059
  async function attentionCropOrigin(srcBuf, source, plan) {
6171
8060
  try {
6172
- const { info } = await sharp20(srcBuf).resize(plan.width, plan.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
8061
+ const { info } = await sharp21(srcBuf).resize(plan.width, plan.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
6173
8062
  const attnLeft = typeof info.cropOffsetLeft === "number" ? Math.abs(info.cropOffsetLeft) : plan.left;
6174
8063
  const attnTop = typeof info.cropOffsetTop === "number" ? Math.abs(info.cropOffsetTop) : plan.top;
6175
8064
  const left = Math.round((attnLeft + plan.left) / 2);
@@ -6322,23 +8211,23 @@ function relax(grid, seam, fixedSweeps) {
6322
8211
 
6323
8212
  // src/expand.ts
6324
8213
  async function expandCanvas(source, plan) {
6325
- const bed = await sharp20(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
6326
- return sharp20(bed).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
8214
+ const bed = await sharp21(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
8215
+ return sharp21(bed).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
6327
8216
  }
6328
8217
  async function compositeExpand(engineImage, source, plan) {
6329
- const meta = await sharp20(engineImage).metadata();
8218
+ const meta = await sharp21(engineImage).metadata();
6330
8219
  const want = plan.width / plan.height;
6331
8220
  const got = meta.width && meta.height ? meta.width / meta.height : 0;
6332
8221
  const sameOrientation = got > 0 && got >= 1 === want >= 1;
6333
8222
  const aligned = sameOrientation;
6334
8223
  const exact = meta.width === plan.width && meta.height === plan.height;
6335
- const surround = aligned ? exact ? engineImage : await sharp20(engineImage).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).toBuffer() : await expandCanvasBedOnly(source, plan);
8224
+ const surround = aligned ? exact ? engineImage : await sharp21(engineImage).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).toBuffer() : await expandCanvasBedOnly(source, plan);
6336
8225
  const matched = aligned ? await matchMarginsToSeam(surround, source, plan) : surround;
6337
- const image = await sharp20(matched).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
8226
+ const image = await sharp21(matched).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
6338
8227
  return { image, aligned };
6339
8228
  }
6340
8229
  async function matchMarginsToSeam(surround, source, plan) {
6341
- const src = await sharp20(source).metadata();
8230
+ const src = await sharp21(source).metadata();
6342
8231
  if (!src.width || !src.height) return surround;
6343
8232
  const SW = src.width;
6344
8233
  const SH = src.height;
@@ -6385,8 +8274,8 @@ var MAX_CORRECTION = 60;
6385
8274
  async function reconcile(surround, source, side, axis) {
6386
8275
  const { margin } = side;
6387
8276
  if (margin.width < 1 || margin.height < 1) return surround;
6388
- const marginRaw = await sharp20(surround).extract(margin).removeAlpha().raw().toBuffer();
6389
- const edgeRaw = await sharp20(source).extract(side.srcEdge).removeAlpha().raw().toBuffer();
8277
+ const marginRaw = await sharp21(surround).extract(margin).removeAlpha().raw().toBuffer();
8278
+ const edgeRaw = await sharp21(source).extract(side.srcEdge).removeAlpha().raw().toBuffer();
6390
8279
  const W = margin.width;
6391
8280
  const H = margin.height;
6392
8281
  const along = axis === "width" ? H : W;
@@ -6440,11 +8329,11 @@ async function reconcile(surround, source, side, axis) {
6440
8329
  }
6441
8330
  }
6442
8331
  }
6443
- const patch2 = await sharp20(corrected, { raw: { width: W, height: H, channels: 3 } }).png().toBuffer();
6444
- return sharp20(surround).composite([{ input: patch2, left: margin.left, top: margin.top }]).toBuffer();
8332
+ const patch2 = await sharp21(corrected, { raw: { width: W, height: H, channels: 3 } }).png().toBuffer();
8333
+ return sharp21(surround).composite([{ input: patch2, left: margin.left, top: margin.top }]).toBuffer();
6445
8334
  }
6446
8335
  async function expandCanvasBedOnly(source, plan) {
6447
- return sharp20(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
8336
+ return sharp21(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
6448
8337
  }
6449
8338
  function medianOf(rgb, channel, from, to) {
6450
8339
  const n = to - from;
@@ -6455,17 +8344,17 @@ function medianOf(rgb, channel, from, to) {
6455
8344
  return n % 2 ? values[(n - 1) / 2] : (values[n / 2 - 1] + values[n / 2]) / 2;
6456
8345
  }
6457
8346
  async function reframeExpand(engineImage, plan) {
6458
- const meta = await sharp20(engineImage).metadata();
8347
+ const meta = await sharp21(engineImage).metadata();
6459
8348
  if (!(meta.width && meta.height)) return null;
6460
8349
  const want = plan.width / plan.height;
6461
8350
  const got = meta.width / meta.height;
6462
8351
  if (got >= 1 !== want >= 1) return null;
6463
8352
  if (meta.width === plan.width && meta.height === plan.height) return engineImage;
6464
8353
  const straight = Math.abs(got - want) / want <= 0.02;
6465
- return sharp20(engineImage).resize(plan.width, plan.height, { fit: straight ? "fill" : "cover", position: "centre" }).png().toBuffer();
8354
+ return sharp21(engineImage).resize(plan.width, plan.height, { fit: straight ? "fill" : "cover", position: "centre" }).png().toBuffer();
6466
8355
  }
6467
8356
  async function seamScore(image, plan, source) {
6468
- const { data, info } = await sharp20(image).removeAlpha().greyscale().raw().toBuffer({ resolveWithObject: true });
8357
+ const { data, info } = await sharp21(image).removeAlpha().greyscale().raw().toBuffer({ resolveWithObject: true });
6469
8358
  const W = info.width;
6470
8359
  const H = info.height;
6471
8360
  const horizontal = plan.axis === "width";
@@ -6498,7 +8387,7 @@ var SEAM_VISIBLE = 2.2;
6498
8387
  var OFFSET = 4;
6499
8388
  var RESIDUAL_VISIBLE = 15;
6500
8389
  async function seamResidual(image, plan, source) {
6501
- const { data, info } = await sharp20(image).removeAlpha().raw().toBuffer({ resolveWithObject: true });
8390
+ const { data, info } = await sharp21(image).removeAlpha().raw().toBuffer({ resolveWithObject: true });
6502
8391
  const W = info.width;
6503
8392
  const H = info.height;
6504
8393
  const ch = info.channels;
@@ -6533,7 +8422,7 @@ var MAX_SHARE = 0.8;
6533
8422
  async function subjectFraction(src, source, axis) {
6534
8423
  try {
6535
8424
  const window = axis === "width" ? { width: Math.max(8, Math.round(source.width * 0.5)), height: source.height } : { width: source.width, height: Math.max(8, Math.round(source.height * 0.5)) };
6536
- const { info } = await sharp20(src).resize(window.width, window.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
8425
+ const { info } = await sharp21(src).resize(window.width, window.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
6537
8426
  const offset = axis === "width" ? Math.abs(typeof info.cropOffsetLeft === "number" ? info.cropOffsetLeft : 0) : Math.abs(typeof info.cropOffsetTop === "number" ? info.cropOffsetTop : 0);
6538
8427
  const span = axis === "width" ? source.width : source.height;
6539
8428
  const extent = axis === "width" ? window.width : window.height;
@@ -6556,14 +8445,14 @@ function placeExpand(plan, source, fraction) {
6556
8445
  }
6557
8446
  var NEUTRAL = { r: 128, g: 128, b: 128 };
6558
8447
  async function conditioningCanvas(source, plan, fill = "edge") {
6559
- const meta = await sharp20(source).metadata();
8448
+ const meta = await sharp21(source).metadata();
6560
8449
  const sw = meta.width ?? 0;
6561
8450
  const sh = meta.height ?? 0;
6562
8451
  if (!(sw > 0 && sh > 0)) throw new Error("conditioningCanvas: source has no dimensions");
6563
8452
  const layers = [];
6564
8453
  if (fill === "edge") layers.push(...await edgeMargins(source, plan, { width: sw, height: sh }));
6565
8454
  layers.push({ input: source, left: plan.left, top: plan.top });
6566
- const canvas = sharp20({
8455
+ const canvas = sharp21({
6567
8456
  create: {
6568
8457
  width: plan.width,
6569
8458
  height: plan.height,
@@ -6575,7 +8464,7 @@ async function conditioningCanvas(source, plan, fill = "edge") {
6575
8464
  }
6576
8465
  async function edgeMargins(source, plan, size) {
6577
8466
  const out = [];
6578
- const strip = async (extract, width, height) => sharp20(source).extract(extract).resize(width, height, { fit: "fill" }).png().toBuffer();
8467
+ const strip = async (extract, width, height) => sharp21(source).extract(extract).resize(width, height, { fit: "fill" }).png().toBuffer();
6579
8468
  if (plan.axis === "width") {
6580
8469
  const before = plan.left;
6581
8470
  const after = plan.width - plan.left - size.width;
@@ -6666,12 +8555,12 @@ async function resolveOutpaintRoute(all, shot) {
6666
8555
  return { engine: shot, method: "reframe", crossed: false };
6667
8556
  }
6668
8557
  async function driftDiff(a, b) {
6669
- const metaA = await sharp20(a).metadata();
6670
- const metaB = await sharp20(b).metadata();
8558
+ const metaA = await sharp21(a).metadata();
8559
+ const metaB = await sharp21(b).metadata();
6671
8560
  const width = Math.min(metaA.width ?? 1, metaB.width ?? 1, 1024);
6672
8561
  const height = Math.min(metaA.height ?? 1, metaB.height ?? 1, 1024);
6673
8562
  const [rawA, rawB] = await Promise.all(
6674
- [a, b].map((buf) => sharp20(buf).resize(width, height, { fit: "cover" }).ensureAlpha().raw().toBuffer())
8563
+ [a, b].map((buf) => sharp21(buf).resize(width, height, { fit: "cover" }).ensureAlpha().raw().toBuffer())
6675
8564
  );
6676
8565
  const out = new PNG({ width, height });
6677
8566
  const changed = pixelmatch(rawA, rawB, out.data, width, height, { threshold: 0.1, diffColor: [255, 64, 64] });
@@ -6683,11 +8572,11 @@ async function driftDiff(a, b) {
6683
8572
  };
6684
8573
  }
6685
8574
  async function changeMask(a, b, cap2 = 1024) {
6686
- const metaA = await sharp20(a).metadata();
8575
+ const metaA = await sharp21(a).metadata();
6687
8576
  const width = Math.min(metaA.width ?? 1, cap2);
6688
8577
  const height = Math.min(metaA.height ?? 1, cap2);
6689
8578
  const [rawA, rawB] = await Promise.all(
6690
- [a, b].map((buf) => sharp20(buf).resize(width, height, { fit: "fill" }).ensureAlpha().raw().toBuffer())
8579
+ [a, b].map((buf) => sharp21(buf).resize(width, height, { fit: "fill" }).ensureAlpha().raw().toBuffer())
6691
8580
  );
6692
8581
  const out = new PNG({ width, height });
6693
8582
  pixelmatch(rawA, rawB, out.data, width, height, { threshold: 0.1, includeAA: true, diffMask: true });
@@ -6736,8 +8625,8 @@ function dilationFor(longEdge) {
6736
8625
  // src/localEdit.ts
6737
8626
  async function preserveOutsideChange(source, edited) {
6738
8627
  try {
6739
- const srcMeta = await sharp20(source).metadata();
6740
- const outMeta = await sharp20(edited).metadata();
8628
+ const srcMeta = await sharp21(source).metadata();
8629
+ const outMeta = await sharp21(edited).metadata();
6741
8630
  if (!srcMeta.width || !srcMeta.height || !outMeta.width || !outMeta.height)
6742
8631
  return { image: edited, outcome: "error", changed: 0 };
6743
8632
  const sameShape = Math.abs(outMeta.width / outMeta.height - srcMeta.width / srcMeta.height) / (srcMeta.width / srcMeta.height) <= 0.01;
@@ -6747,15 +8636,15 @@ async function preserveOutsideChange(source, edited) {
6747
8636
  if (outcome !== "composited") return { image: edited, outcome, changed: shape.changed };
6748
8637
  const r = dilationFor(Math.max(shape.width, shape.height));
6749
8638
  const rawShape = { raw: { width: shape.width, height: shape.height, channels: 1 } };
6750
- const spread = await sharp20(shape.mask, rawShape).blur(r).toColourspace("b-w").raw().toBuffer();
6751
- const dilated = await sharp20(spread, rawShape).threshold(8).toColourspace("b-w").raw().toBuffer();
6752
- const feathered = await sharp20(dilated, rawShape).blur(Math.max(2, r / 3)).toColourspace("b-w").raw().toBuffer();
6753
- const grown = await sharp20(feathered, rawShape).resize(srcMeta.width, srcMeta.height, { fit: "fill", kernel: "cubic" }).toColourspace("b-w").raw().toBuffer();
6754
- const editedRgb = await sharp20(edited).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).removeAlpha().raw().toBuffer();
6755
- const masked = await sharp20(editedRgb, {
8639
+ const spread = await sharp21(shape.mask, rawShape).blur(r).toColourspace("b-w").raw().toBuffer();
8640
+ const dilated = await sharp21(spread, rawShape).threshold(8).toColourspace("b-w").raw().toBuffer();
8641
+ const feathered = await sharp21(dilated, rawShape).blur(Math.max(2, r / 3)).toColourspace("b-w").raw().toBuffer();
8642
+ const grown = await sharp21(feathered, rawShape).resize(srcMeta.width, srcMeta.height, { fit: "fill", kernel: "cubic" }).toColourspace("b-w").raw().toBuffer();
8643
+ const editedRgb = await sharp21(edited).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).removeAlpha().raw().toBuffer();
8644
+ const masked = await sharp21(editedRgb, {
6756
8645
  raw: { width: srcMeta.width, height: srcMeta.height, channels: 3 }
6757
8646
  }).joinChannel(grown, { raw: { width: srcMeta.width, height: srcMeta.height, channels: 1 } }).png().toBuffer();
6758
- const image = await sharp20(source).removeAlpha().composite([{ input: masked }]).png().toBuffer();
8647
+ const image = await sharp21(source).removeAlpha().composite([{ input: masked }]).png().toBuffer();
6759
8648
  return { image, outcome: "composited", changed: shape.changed };
6760
8649
  } catch {
6761
8650
  return { image: edited, outcome: "error", changed: 0 };
@@ -6793,7 +8682,7 @@ function registerLogoRoutes(app, deps) {
6793
8682
  const v = validateBrand(json);
6794
8683
  if (!v.valid) return reply.status(400).send({ error: "brand became invalid", details: v.errors });
6795
8684
  const row = core.store.updateBrand(brand.id, json);
6796
- const meta = await sharp20(core.images.read(part.hash)).metadata().catch(() => null);
8685
+ const meta = await sharp21(core.images.read(part.hash)).metadata().catch(() => null);
6797
8686
  const logoEdge = meta ? Math.max(meta.width ?? 0, meta.height ?? 0) || null : null;
6798
8687
  return { ...row, logoHash: part.hash, logoEdge };
6799
8688
  });
@@ -6841,6 +8730,38 @@ function registerLogoRoutes(app, deps) {
6841
8730
  return core.store.updateBrand(brand.id, json);
6842
8731
  });
6843
8732
  }
8733
+ var scans = /* @__PURE__ */ new Map();
8734
+ var controllers = /* @__PURE__ */ new Map();
8735
+ var KEEP = 32;
8736
+ function remember(state) {
8737
+ scans.set(state.id, state);
8738
+ while (scans.size > KEEP) {
8739
+ const oldest = scans.keys().next().value;
8740
+ if (oldest === void 0) break;
8741
+ scans.delete(oldest);
8742
+ controllers.delete(oldest);
8743
+ }
8744
+ }
8745
+ function getScan(scanId) {
8746
+ return scans.get(scanId);
8747
+ }
8748
+ function startCatalogScan(deps, brandId, url) {
8749
+ const { core, fetchImpl } = deps;
8750
+ if (!core.store.getBrand(brandId)) throw Object.assign(new Error("brand not found"), { statusCode: 404 });
8751
+ const id = randomUUID();
8752
+ const ctrl = new AbortController();
8753
+ const state = { id, brandId, url, status: "running", startedAt: Date.now() };
8754
+ remember(state);
8755
+ controllers.set(id, ctrl);
8756
+ void scanForCandidates({ url, fetchImpl, signal: ctrl.signal }).then((result) => {
8757
+ state.result = result;
8758
+ state.status = "done";
8759
+ }).catch((err) => {
8760
+ state.status = "error";
8761
+ state.error = err instanceof Error ? err.message : String(err);
8762
+ }).finally(() => controllers.delete(id));
8763
+ return { scanId: id };
8764
+ }
6844
8765
 
6845
8766
  // src/routes/catalogImport.ts
6846
8767
  function registerCatalogImportRoutes(app, deps) {
@@ -6848,23 +8769,104 @@ function registerCatalogImportRoutes(app, deps) {
6848
8769
  app.get("/api/brands/:id/products-library", async (req, reply) => {
6849
8770
  const brand = core.store.getBrand(req.params.id);
6850
8771
  if (!brand) return reply.status(404).send({ error: "brand not found" });
6851
- const products = core.catalog.listLibraryProducts(brand.id, brand.json);
8772
+ const products = core.catalog.listLibraryIndex(brand.id, brand.json);
6852
8773
  const source = core.catalog.getSourceForBrand(brand.id);
6853
8774
  return { products, source };
6854
8775
  });
8776
+ app.get("/api/brands/:id/products-library/:productId", async (req, reply) => {
8777
+ const brand = core.store.getBrand(req.params.id);
8778
+ if (!brand) return reply.status(404).send({ error: "brand not found" });
8779
+ const product = core.catalog.libraryProduct(brand.id, brand.json, String(req.params.productId));
8780
+ if (!product) return reply.status(404).send({ error: "product not found" });
8781
+ return { product };
8782
+ });
6855
8783
  app.get("/api/brands/:id/catalog/source", async (req, reply) => {
6856
8784
  const brand = core.store.getBrand(req.params.id);
6857
8785
  if (!brand) return reply.status(404).send({ error: "brand not found" });
6858
8786
  return { source: core.catalog.getSourceForBrand(brand.id) };
6859
8787
  });
8788
+ app.post("/api/brands/:id/catalog/scan", async (req, reply) => {
8789
+ const brandId = req.params.id;
8790
+ const brand = core.store.getBrand(brandId);
8791
+ if (!brand) return reply.status(404).send({ error: "brand not found" });
8792
+ const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
8793
+ if (!url.trim()) return reply.status(400).send({ error: "url required" });
8794
+ try {
8795
+ return startCatalogScan({ core, fetchImpl }, brandId, url);
8796
+ } catch (err) {
8797
+ return reply.status(err.statusCode ?? 500).send({ error: err.message ?? "scan failed" });
8798
+ }
8799
+ });
8800
+ app.get("/api/brands/:id/catalog/scans/:scanId", async (req, reply) => {
8801
+ const brandId = req.params.id;
8802
+ const scan = getScan(req.params.scanId);
8803
+ if (!scan || scan.brandId !== brandId) return reply.status(404).send({ error: "scan not found" });
8804
+ return scan;
8805
+ });
8806
+ app.post("/api/brands/:id/catalog/details", async (req, reply) => {
8807
+ const brand = core.store.getBrand(req.params.id);
8808
+ if (!brand) return reply.status(404).send({ error: "brand not found" });
8809
+ const asked = req.body?.urls;
8810
+ if (!Array.isArray(asked) || asked.some((u) => typeof u !== "string")) {
8811
+ return reply.status(400).send({ error: "urls must be a list of product addresses" });
8812
+ }
8813
+ const urls = asked.slice(0, 48);
8814
+ if (!urls.length) return { products: [] };
8815
+ const site = String(brand.json?.meta?.website ?? "");
8816
+ let origin;
8817
+ try {
8818
+ origin = new URL(site.startsWith("http") ? site : `https://${site}`).origin;
8819
+ } catch {
8820
+ return reply.status(400).send({ error: "this brand has no website to read products from" });
8821
+ }
8822
+ const sameSite = urls.filter((u) => {
8823
+ try {
8824
+ return new URL(u).origin === origin;
8825
+ } catch {
8826
+ return false;
8827
+ }
8828
+ });
8829
+ if (!sameSite.length) {
8830
+ return reply.status(400).send({ error: "none of those products belong to this site" });
8831
+ }
8832
+ const products = await fetchProductPages({ fetchImpl: fetchImpl ?? fetch, baseUrl: origin }, sameSite, {
8833
+ concurrency: 4,
8834
+ maxBytes: 15e5
8835
+ });
8836
+ return { products };
8837
+ });
6860
8838
  app.post("/api/brands/:id/catalog/import", async (req, reply) => {
6861
8839
  const brandId = req.params.id;
6862
8840
  const brand = core.store.getBrand(brandId);
6863
8841
  if (!brand) return reply.status(404).send({ error: "brand not found" });
6864
8842
  const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
6865
8843
  if (!url.trim()) return reply.status(400).send({ error: "url required" });
8844
+ const asked = req.body?.urls;
8845
+ let only;
8846
+ if (asked != null) {
8847
+ if (!Array.isArray(asked) || asked.some((u) => typeof u !== "string")) {
8848
+ return reply.status(400).send({ error: "urls must be a list of product addresses" });
8849
+ }
8850
+ let origin;
8851
+ try {
8852
+ origin = new URL(url.startsWith("http") ? url : `https://${url}`).origin;
8853
+ } catch {
8854
+ return reply.status(400).send({ error: "url required" });
8855
+ }
8856
+ only = asked.filter((u) => {
8857
+ try {
8858
+ return new URL(u).origin === origin;
8859
+ } catch {
8860
+ return false;
8861
+ }
8862
+ });
8863
+ if (!only.length) return reply.status(400).send({ error: "none of those products belong to this site" });
8864
+ if (only.length > 2e3) {
8865
+ return reply.status(400).send({ error: "That is nearly the whole catalogue. Import everything instead." });
8866
+ }
8867
+ }
6866
8868
  try {
6867
- return startCatalogImport({ core, fetchImpl }, brandId, url);
8869
+ return startCatalogImport({ core, fetchImpl }, brandId, url, { only });
6868
8870
  } catch (err) {
6869
8871
  return reply.status(err.statusCode ?? 500).send({ error: err.message ?? "import failed" });
6870
8872
  }
@@ -6915,7 +8917,7 @@ async function vibrantColor(input) {
6915
8917
  let data;
6916
8918
  let channels;
6917
8919
  try {
6918
- const out = await sharp20(input).resize(48, 48, { fit: "inside" }).raw().toBuffer({ resolveWithObject: true });
8920
+ const out = await sharp21(input).resize(48, 48, { fit: "inside" }).raw().toBuffer({ resolveWithObject: true });
6919
8921
  data = out.data;
6920
8922
  channels = out.info.channels;
6921
8923
  } catch {
@@ -6938,13 +8940,13 @@ async function vibrantColor(input) {
6938
8940
  const best = buckets.reduce((a, b) => b.score > a.score ? b : a, buckets[0]);
6939
8941
  if (best.score <= 0) {
6940
8942
  try {
6941
- const { dominant } = await sharp20(input).stats();
6942
- return toHex(dominant.r, dominant.g, dominant.b);
8943
+ const { dominant } = await sharp21(input).stats();
8944
+ return toHex2(dominant.r, dominant.g, dominant.b);
6943
8945
  } catch {
6944
8946
  return null;
6945
8947
  }
6946
8948
  }
6947
- return toHex(best.r / best.score, best.g / best.score, best.b / best.score);
8949
+ return toHex2(best.r / best.score, best.g / best.score, best.b / best.score);
6948
8950
  }
6949
8951
  function rgbToHsl(r, g, b) {
6950
8952
  const rn = r / 255, gn = g / 255, bn = b / 255;
@@ -6959,7 +8961,7 @@ function rgbToHsl(r, g, b) {
6959
8961
  else h = ((rn - gn) / d + 4) * 60;
6960
8962
  return { h, s, l };
6961
8963
  }
6962
- var toHex = (r, g, b) => "#" + [r, g, b].map(
8964
+ var toHex2 = (r, g, b) => "#" + [r, g, b].map(
6963
8965
  (v) => Math.max(0, Math.min(255, Math.round(v))).toString(16).padStart(2, "0")
6964
8966
  ).join("");
6965
8967
 
@@ -7482,7 +9484,7 @@ function registerProjectRoutes(app, deps) {
7482
9484
  const brand = core.store.getBrand(req.params.id);
7483
9485
  if (!brand) return reply.status(404).send({ error: "brand not found" });
7484
9486
  const limit = Math.min(Number(req.query.limit) || 60, 200);
7485
- return { nodes: core.store.recentActivity(brand.id, limit), jobs: core.catalog.listJobs(brand.id) };
9487
+ return { nodes: core.store.recentActivity(brand.id, limit), jobs: core.catalog.listRecentJobs(brand.id) };
7486
9488
  });
7487
9489
  app.get("/api/brands/:id/workspace", async (req, reply) => {
7488
9490
  const brand = core.store.getBrand(req.params.id);
@@ -7591,7 +9593,10 @@ function registerProjectRoutes(app, deps) {
7591
9593
  function registerCodexSetupRoutes(app, deps) {
7592
9594
  const codexSetup = deps.codexSetup ?? createCodexSetup({ runner: deps.codexRunner });
7593
9595
  let codexSetupBusy = null;
7594
- app.get("/api/engines/codex/status", async () => codexSetup.status());
9596
+ app.get("/api/engines/codex/status", async (req) => {
9597
+ const force = req.query?.force === "1";
9598
+ return codexSetup.status({ force });
9599
+ });
7595
9600
  app.post("/api/engines/codex/install", async (_req, reply) => {
7596
9601
  if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
7597
9602
  codexSetupBusy = "install";
@@ -7614,6 +9619,35 @@ function registerCodexSetupRoutes(app, deps) {
7614
9619
  codexSetupBusy = null;
7615
9620
  }
7616
9621
  });
9622
+ app.post("/api/engines/codex/repair-env", async (req, reply) => {
9623
+ const repair = deps.envRepair;
9624
+ if (!repair) return reply.status(501).send({ error: "not available on this server" });
9625
+ const asked = req.body?.keys;
9626
+ if (!Array.isArray(asked) || asked.length === 0) return reply.status(400).send({ error: "keys required" });
9627
+ const keys = asked.map((k) => String(k).trim().toUpperCase());
9628
+ const unknown = keys.filter((k) => !CONFLICT_ENV_KEYS.includes(k));
9629
+ if (unknown.length > 0) return reply.status(400).send({ error: `not a Codex credential: ${unknown.join(", ")}` });
9630
+ if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
9631
+ codexSetupBusy = "repair";
9632
+ try {
9633
+ repair.set([.../* @__PURE__ */ new Set([...repair.get(), ...keys])]);
9634
+ return { ok: true, ...await codexSetup.status({ force: true }) };
9635
+ } finally {
9636
+ codexSetupBusy = null;
9637
+ }
9638
+ });
9639
+ app.post("/api/engines/codex/restore-env", async (_req, reply) => {
9640
+ const repair = deps.envRepair;
9641
+ if (!repair) return reply.status(501).send({ error: "not available on this server" });
9642
+ if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
9643
+ codexSetupBusy = "repair";
9644
+ try {
9645
+ repair.set([]);
9646
+ return { ok: true, ...await codexSetup.status({ force: true }) };
9647
+ } finally {
9648
+ codexSetupBusy = null;
9649
+ }
9650
+ });
7617
9651
  }
7618
9652
  var slug = (v, fallback) => {
7619
9653
  const s = String(v ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
@@ -7794,8 +9828,8 @@ function registerImageRoutes(app, deps) {
7794
9828
  if (!part) return reply.status(400).send({ error: "multipart file field required" });
7795
9829
  const buf = await part.toBuffer();
7796
9830
  if (buf.length === 0) return reply.status(400).send({ error: "empty file" });
7797
- const fmt = (await sharp20(buf).metadata().catch(() => null))?.format;
7798
- const png = fmt === "svg" ? await toMarkPng(buf) : await sharp20(buf).rotate().png().toBuffer();
9831
+ const fmt = (await sharp21(buf).metadata().catch(() => null))?.format;
9832
+ const png = fmt === "svg" ? await toMarkPng(buf) : await sharp21(buf).rotate().png().toBuffer();
7799
9833
  return { hash: core.images.save(png) };
7800
9834
  });
7801
9835
  app.post("/api/diff", async (req, reply) => {
@@ -7817,6 +9851,44 @@ function registerImageRoutes(app, deps) {
7817
9851
 
7818
9852
  // src/release/notes.data.ts
7819
9853
  var RELEASES = [
9854
+ {
9855
+ version: "0.9.4",
9856
+ date: "2026-09-14",
9857
+ title: "A store that asks Scenri to slow down no longer looks like an empty one.",
9858
+ sections: [
9859
+ {
9860
+ heading: "Products",
9861
+ body: "Some stores limit how fast anything can read them. Scenri now waits the way such a store asks and says what happened: an import that could not read anything tells you to try again in a few minutes, and one that got part way says how much is left. Products from an earlier import are never retired by a run that was refused, and a store that keeps saying no ends the import early rather than grinding on."
9862
+ },
9863
+ {
9864
+ heading: "Fixes",
9865
+ body: "A product picture saved in a newer format such as AVIF is found again instead of going missing."
9866
+ }
9867
+ ]
9868
+ },
9869
+ {
9870
+ version: "0.9.3",
9871
+ date: "2026-09-14",
9872
+ title: "Paste a website, get the brand and its products.",
9873
+ sections: [
9874
+ {
9875
+ heading: "Brand",
9876
+ body: "A website address is enough to start a brand. Scenri reads the logo, the wordmark and the palette a site actually declares, and says plainly what it found and what it could not."
9877
+ },
9878
+ {
9879
+ heading: "Products",
9880
+ body: "The same address brings in the store. Pick the products you want or take the whole catalogue, and watch them arrive: each one appears in your library as it lands, with its picture, newest first. A large store no longer runs out of memory partway through, and a shop that stops answering says so instead of looking empty."
9881
+ },
9882
+ {
9883
+ heading: "Create",
9884
+ body: "The insert menus reach everything. Typing $, @, / or # opens a shortlist that pages through the rest of that catalogue, with a count so nothing is quietly cut off, and the box stays where the caret is."
9885
+ },
9886
+ {
9887
+ heading: "Fixes",
9888
+ body: "A first run on Windows no longer stops at the desktop icon question, and never asks for an administrator it does not need. Codex reports itself ready only once a real run has authenticated."
9889
+ }
9890
+ ]
9891
+ },
7820
9892
  {
7821
9893
  version: "0.9.2",
7822
9894
  date: "2026-09-07",
@@ -8787,15 +10859,15 @@ function registerDesktopRoutes(app, deps) {
8787
10859
  record: null
8788
10860
  };
8789
10861
  }
8790
- const { installDeps } = await import('./cli-FYWM4HBN.js');
8791
- const { desktopStatus } = await import('./install-4W6AARK6.js');
10862
+ const { installDeps } = await import('./cli-MQO66C5O.js');
10863
+ const { desktopStatus } = await import('./install-VXJPLYWM.js');
8792
10864
  return desktopStatus(installDeps(runtime.entry));
8793
10865
  });
8794
10866
  const install = deps.installImpl ?? (async () => {
8795
10867
  if (!runtime.entry) {
8796
10868
  return { ok: false, reason: "unsupported", message: "Desktop shortcuts are not available on this system yet." };
8797
10869
  }
8798
- const { addToDesktop } = await import('./cli-FYWM4HBN.js');
10870
+ const { addToDesktop } = await import('./cli-MQO66C5O.js');
8799
10871
  return addToDesktop(runtime.entry);
8800
10872
  });
8801
10873
  app.get("/api/desktop", async () => {
@@ -8860,6 +10932,7 @@ function buildServer(opts) {
8860
10932
  registerAccessGuard(app, opts.access);
8861
10933
  const reserved = /* @__PURE__ */ new Map();
8862
10934
  const runningGenerations = /* @__PURE__ */ new Map();
10935
+ core.catalog.reconcileInterruptedJobs();
8863
10936
  const thumbs = createThumbStore(core);
8864
10937
  const { scenes } = loadScenes(opts.templatesDir);
8865
10938
  const resolveScene = sceneResolver(scenes);
@@ -8869,8 +10942,11 @@ function buildServer(opts) {
8869
10942
  const e = err;
8870
10943
  const status = err instanceof SpendCapError ? 402 : e.statusCode ?? 500;
8871
10944
  const leaksPath = typeof e.code === "string" && /^(ENOENT|EACCES|EPERM|EISDIR|ENOTDIR)$/.test(e.code);
8872
- reply.status(status).send({ error: leaksPath ? "unexpected error" : e.message ?? "unexpected error" });
10945
+ const rawRuntime = !e.statusCode && (err instanceof TypeError || err instanceof RangeError);
10946
+ if (rawRuntime) console.error("unexpected error:", err);
10947
+ reply.status(status).send({ error: leaksPath || rawRuntime ? "unexpected error" : e.message ?? "unexpected error" });
8873
10948
  });
10949
+ const scrapeGuard = () => ({ allowPrivateHosts: process.env.SCENRI_SCRAPE_ALLOW_PRIVATE === "1" });
8874
10950
  app.get("/api/brands", async () => core.store.listBrands());
8875
10951
  app.post("/api/brands", async (req, reply) => {
8876
10952
  const json = req.body?.brand;
@@ -8879,10 +10955,12 @@ function buildServer(opts) {
8879
10955
  return core.store.createBrand(json);
8880
10956
  });
8881
10957
  app.post("/api/brands/from-url", async (req, reply) => {
8882
- const url = String(req.body?.url ?? "");
8883
- if (!/^https?:\/\//.test(url)) return reply.status(400).send({ error: "url must be http(s)" });
8884
- const { brand, warnings } = await buildFromUrl(url, {
10958
+ const asked = normalizeSiteUrl(req.body?.url);
10959
+ if (!asked.ok) return reply.status(400).send({ error: asked.message });
10960
+ const url = asked.url;
10961
+ const { brand, warnings, report } = await buildFromUrl(url, {
8885
10962
  fetchImpl: opts.fetchImpl,
10963
+ guard: scrapeGuard(),
8886
10964
  // The store names every blob `<hash>.png` and /api/images/:hash always
8887
10965
  // serves image/png, so an un-normalized .ico or .svg here is a file lying
8888
10966
  // about its own format — broken in the marks grid, and mislabelled to any
@@ -8890,14 +10968,11 @@ function buildServer(opts) {
8890
10968
  saveAsset: async (buf) => `asset:${core.images.save(await toMarkPng(buf))}`,
8891
10969
  // Measured as stored (post-toMarkPng), so the scrape judges the same
8892
10970
  // pixels the compiler will one day attach.
8893
- probeLongEdge: async (buf) => {
8894
- const m = await sharp20(await toMarkPng(buf)).metadata();
8895
- return Math.max(m.width ?? 0, m.height ?? 0) || null;
8896
- },
10971
+ inspectMark: (buf) => inspectMark(buf, toMarkPng),
8897
10972
  createdWith: `${meta.name}/${meta.version}`
8898
10973
  });
8899
10974
  const row = core.store.createBrand(brand);
8900
- return { ...row, warnings };
10975
+ return { ...row, warnings, report };
8901
10976
  });
8902
10977
  app.put("/api/brands/:id", async (req, reply) => {
8903
10978
  const json = req.body?.brand;
@@ -8974,15 +11049,14 @@ function buildServer(opts) {
8974
11049
  app.post("/api/brands/:id/refresh-from-url", async (req, reply) => {
8975
11050
  const brand = core.store.getBrand(req.params.id);
8976
11051
  if (!brand) return reply.status(404).send({ error: "brand not found" });
8977
- const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
8978
- if (!/^https?:\/\//.test(url)) return reply.status(400).send({ error: "url must be http(s)" });
11052
+ const asked = normalizeSiteUrl(req.body?.url ?? brand.json?.meta?.website);
11053
+ if (!asked.ok) return reply.status(400).send({ error: asked.message });
11054
+ const url = asked.url;
8979
11055
  const { brand: scraped, warnings } = await buildFromUrl(url, {
8980
11056
  fetchImpl: opts.fetchImpl,
11057
+ guard: scrapeGuard(),
8981
11058
  saveAsset: async (buf) => `asset:${core.images.save(await toMarkPng(buf))}`,
8982
- probeLongEdge: async (buf) => {
8983
- const m = await sharp20(await toMarkPng(buf)).metadata();
8984
- return Math.max(m.width ?? 0, m.height ?? 0) || null;
8985
- },
11059
+ inspectMark: (buf) => inspectMark(buf, toMarkPng),
8986
11060
  createdWith: `${meta.name}/${meta.version}`
8987
11061
  });
8988
11062
  const { brand: merged, suggestions } = mergeScrape(brand.json, scraped);
@@ -9290,7 +11364,14 @@ function buildServer(opts) {
9290
11364
  ],
9291
11365
  engineNames: () => engines.all().map((e) => ({ id: e.capabilities().id, name: e.capabilities().displayName }))
9292
11366
  });
9293
- registerCodexSetupRoutes(app, { codexSetup: opts.codexSetup, codexRunner: engines.codexRunner });
11367
+ registerCodexSetupRoutes(app, {
11368
+ codexSetup: opts.codexSetup,
11369
+ codexRunner: engines.codexRunner,
11370
+ envRepair: {
11371
+ get: () => [...ignoreEnvKeysGetter(core)()],
11372
+ set: (keys) => core.store.setSetting(IGNORE_ENV_KEYS_SETTING, keys.join(","))
11373
+ }
11374
+ });
9294
11375
  app.get("/api/engines", async () => {
9295
11376
  const list2 = [];
9296
11377
  for (const e of engines.all()) {
@@ -9353,11 +11434,11 @@ function buildServer(opts) {
9353
11434
  const out = [];
9354
11435
  for (const h of images) {
9355
11436
  const buf = core.images.read(h);
9356
- const meta2 = await sharp20(buf).metadata().catch(() => null);
11437
+ const meta2 = await sharp21(buf).metadata().catch(() => null);
9357
11438
  if (!meta2?.width || !meta2.height) throw new Error("engine returned an undecodable image");
9358
11439
  const oriented = (meta2.orientation ?? 1) !== 1;
9359
11440
  out.push(
9360
- buf.subarray(0, 8).equals(PNG_SIG) && !oriented ? h : core.images.save(await sharp20(buf).rotate().png().toBuffer())
11441
+ buf.subarray(0, 8).equals(PNG_SIG) && !oriented ? h : core.images.save(await sharp21(buf).rotate().png().toBuffer())
9361
11442
  );
9362
11443
  }
9363
11444
  return out;
@@ -9369,7 +11450,7 @@ function buildServer(opts) {
9369
11450
  const out = [];
9370
11451
  for (const h of images) {
9371
11452
  const buf = core.images.read(h);
9372
- const meta2 = await sharp20(buf).metadata();
11453
+ const meta2 = await sharp21(buf).metadata();
9373
11454
  if (!meta2.width || !meta2.height) {
9374
11455
  out.push(h);
9375
11456
  continue;
@@ -9382,7 +11463,7 @@ function buildServer(opts) {
9382
11463
  }
9383
11464
  const w = got > target ? Math.round(meta2.height * target) : meta2.width;
9384
11465
  const hpx = got > target ? meta2.height : Math.round(meta2.width / target);
9385
- const cropped = await sharp20(buf).resize(w, hpx, { fit: "cover", position: "attention" }).png().toBuffer();
11466
+ const cropped = await sharp21(buf).resize(w, hpx, { fit: "cover", position: "attention" }).png().toBuffer();
9386
11467
  app.log.info(
9387
11468
  { nodeId, got: `${meta2.width}x${meta2.height}`, want: `${w}x${hpx}` },
9388
11469
  "canvas: cropped a drifted frame to the asked ratio"
@@ -9401,7 +11482,7 @@ function buildServer(opts) {
9401
11482
  async function assertAspect(images, expect) {
9402
11483
  const want = expect.width / expect.height;
9403
11484
  for (const h of images) {
9404
- const meta2 = await sharp20(core.images.read(h)).metadata();
11485
+ const meta2 = await sharp21(core.images.read(h)).metadata();
9405
11486
  if (!meta2.width || !meta2.height) continue;
9406
11487
  const got = meta2.width / meta2.height;
9407
11488
  if (Math.abs(got - want) / want > ASPECT_TOLERANCE)
@@ -9432,7 +11513,7 @@ function buildServer(opts) {
9432
11513
  if (post) own = await post(own);
9433
11514
  if (expect) await assertAspect(own, expect);
9434
11515
  try {
9435
- const meta2 = await sharp20(core.images.read(own[0])).metadata();
11516
+ const meta2 = await sharp21(core.images.read(own[0])).metadata();
9436
11517
  const node = core.store.getNode(id);
9437
11518
  if (node && meta2.width && meta2.height) {
9438
11519
  const brief = node.brief ?? {};
@@ -9541,7 +11622,7 @@ function buildServer(opts) {
9541
11622
  crop: window
9542
11623
  });
9543
11624
  const work2 = async () => ({
9544
- images: [core.images.save(await sharp20(args.srcBuf).extract(window).png().toBuffer())],
11625
+ images: [core.images.save(await sharp21(args.srcBuf).extract(window).png().toBuffer())],
9545
11626
  costUsd: 0
9546
11627
  });
9547
11628
  void runNode([node2.id], null, 0, work2, { width: plan2.width, height: plan2.height }).catch(
@@ -9566,7 +11647,7 @@ function buildServer(opts) {
9566
11647
  if (!srcHash || !core.images.has(String(srcHash)))
9567
11648
  return reply.status(400).send({ error: "edit needs a parent node with an image (sourceImage)" });
9568
11649
  const srcBuf = core.images.read(String(srcHash));
9569
- const srcMeta = await sharp20(srcBuf).metadata();
11650
+ const srcMeta = await sharp21(srcBuf).metadata();
9570
11651
  if (!srcMeta.width || !srcMeta.height) return reply.status(400).send({ error: "source image unreadable" });
9571
11652
  return runCropNode({
9572
11653
  parentId: cropParentId,
@@ -9781,7 +11862,7 @@ function buildServer(opts) {
9781
11862
  );
9782
11863
  }
9783
11864
  const srcBuf = core.images.read(String(srcHash));
9784
- const srcMeta = await sharp20(srcBuf).metadata();
11865
+ const srcMeta = await sharp21(srcBuf).metadata();
9785
11866
  if (srcMeta.width && srcMeta.height) expectShape = { width: srcMeta.width, height: srcMeta.height };
9786
11867
  const parentFormat = parent?.brief?.tokens?.find((t) => t?.t === "format");
9787
11868
  const parentNominal = parentFormat && Number(parentFormat.w) > 0 && Number(parentFormat.h) > 0 ? { width: Number(parentFormat.w), height: Number(parentFormat.h) } : null;
@@ -9815,7 +11896,7 @@ function buildServer(opts) {
9815
11896
  } else if (decision.op === "extend") {
9816
11897
  if (decision.assist) {
9817
11898
  expandAssist = { width: decision.assist.width, height: decision.assist.height };
9818
- workBuf = await sharp20(srcBuf).extract(decision.assist).png().toBuffer();
11899
+ workBuf = await sharp21(srcBuf).extract(decision.assist).png().toBuffer();
9819
11900
  workSize = { width: decision.assist.width, height: decision.assist.height };
9820
11901
  }
9821
11902
  expandPlan = planExpand(workSize, targetRatio);
@@ -9836,7 +11917,7 @@ function buildServer(opts) {
9836
11917
  const fit = fitExpandToBudget(expandPlan, workSize, runEngine.capabilities().editPixelBudget);
9837
11918
  if (fit.scale < 1) {
9838
11919
  expandPlan = fit.plan;
9839
- workBuf = await sharp20(workBuf).resize(fit.source.width, fit.source.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer();
11920
+ workBuf = await sharp21(workBuf).resize(fit.source.width, fit.source.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer();
9840
11921
  workSize = fit.source;
9841
11922
  extraWarnings.push(
9842
11923
  `${runEngine.capabilities().displayName} draws about ${((runEngine.capabilities().editPixelBudget ?? 0) / 1e6).toFixed(1)} megapixels, so this shape continues as a ${fit.plan.width}x${fit.plan.height} frame with the photograph riding inside it at ${fit.source.width}x${fit.source.height}. Nothing is upscaled; the stored size is the size the engine truly drew.`
@@ -9859,7 +11940,7 @@ function buildServer(opts) {
9859
11940
  if (editPixelBudget && stepped && (stepped.width !== srcMeta.width || stepped.height !== srcMeta.height)) {
9860
11941
  sentSize = stepped;
9861
11942
  budgetSourceHash = core.images.save(
9862
- await sharp20(srcBuf).resize(sentSize.width, sentSize.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer()
11943
+ await sharp21(srcBuf).resize(sentSize.width, sentSize.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer()
9863
11944
  );
9864
11945
  if (!gradeOnlyAsk)
9865
11946
  extraWarnings.push(
@@ -9999,11 +12080,11 @@ function buildServer(opts) {
9999
12080
  const original = editedFrom ? core.images.read(editedFrom) : null;
10000
12081
  const localScope = kind === "edit" && !plan && editScope === "local" && original;
10001
12082
  const enforceEditCanvas = async (images) => {
10002
- const srcMeta = await sharp20(original).metadata();
12083
+ const srcMeta = await sharp21(original).metadata();
10003
12084
  if (!srcMeta.width || !srcMeta.height) return images;
10004
12085
  const out = [];
10005
12086
  for (const h of images) {
10006
- const meta2 = await sharp20(core.images.read(h)).metadata();
12087
+ const meta2 = await sharp21(core.images.read(h)).metadata();
10007
12088
  const got = { width: meta2.width ?? 0, height: meta2.height ?? 0 };
10008
12089
  const verdict = judgeEditSize({ width: srcMeta.width, height: srcMeta.height }, got, {
10009
12090
  pixelBudget: runEngine.capabilities().editPixelBudget
@@ -10033,7 +12114,7 @@ function buildServer(opts) {
10033
12114
  );
10034
12115
  out.push(
10035
12116
  core.images.save(
10036
- await sharp20(core.images.read(h)).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).png().toBuffer()
12117
+ await sharp21(core.images.read(h)).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).png().toBuffer()
10037
12118
  )
10038
12119
  );
10039
12120
  try {
@@ -10055,7 +12136,7 @@ function buildServer(opts) {
10055
12136
  const out = [];
10056
12137
  for (const h of images) {
10057
12138
  const answer = core.images.read(h);
10058
- const got = await sharp20(answer).metadata();
12139
+ const got = await sharp21(answer).metadata();
10059
12140
  if (got.width !== plan.width || got.height !== plan.height)
10060
12141
  app.log.info(
10061
12142
  { nodeId: node.id, got: `${got.width}x${got.height}`, want: `${plan.width}x${plan.height}` },
@@ -10216,6 +12297,7 @@ function buildServer(opts) {
10216
12297
  while (runningGenerations.size > 0 && Date.now() < deadline) {
10217
12298
  await new Promise((r) => setTimeout(r, 25));
10218
12299
  }
12300
+ await settleCatalogImports();
10219
12301
  await thumbs.settle();
10220
12302
  await app.close();
10221
12303
  core.close();
@@ -10388,9 +12470,9 @@ async function run() {
10388
12470
  }
10389
12471
  }
10390
12472
  const ownEntry = fileURLToPath(import.meta.url);
10391
- const { addToDesktop, installDeps } = await import('./cli-FYWM4HBN.js');
10392
- const { refreshLauncher } = await import('./refresh-FKQABJOC.js');
10393
- const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-IVTNHIDJ.js');
12473
+ const { addToDesktop, installDeps } = await import('./cli-MQO66C5O.js');
12474
+ const { refreshLauncher } = await import('./refresh-5HBDLLLB.js');
12475
+ const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-W6NJXMWT.js');
10394
12476
  const { launcherInstalled } = await import('./paths-6ZXRZUHY.js');
10395
12477
  const meta = readMeta();
10396
12478
  void refreshLauncher({ ...installDeps(ownEntry), ownEntry, installKind, pkg: meta.name }).then((r) => {
@@ -10412,7 +12494,7 @@ async function run() {
10412
12494
  add: () => addToDesktop(ownEntry),
10413
12495
  decline: () => core.store.setSetting("desktop.prompt", "declined"),
10414
12496
  say: (line) => console.log(line)
10415
- });
12497
+ }).catch(() => console.log(" Could not ask about the desktop icon. Scenri is running anyway."));
10416
12498
  console.log("");
10417
12499
  }
10418
12500
  }
@@ -10422,8 +12504,8 @@ async function verify() {
10422
12504
  const db = new Database(":memory:");
10423
12505
  db.pragma("user_version");
10424
12506
  db.close();
10425
- const { default: sharp21 } = await import('sharp');
10426
- await sharp21({ create: { width: 1, height: 1, channels: 3, background: "#000" } }).png().toBuffer();
12507
+ const { default: sharp22 } = await import('sharp');
12508
+ await sharp22({ create: { width: 1, height: 1, channels: 3, background: "#000" } }).png().toBuffer();
10427
12509
  console.log(JSON.stringify({ ok: true, version: readMeta().version }));
10428
12510
  } catch (err) {
10429
12511
  console.log(JSON.stringify({ ok: false, error: String(err?.message ?? err) }));