scenri 0.9.2 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +3 -1
- package/dist/{chunk-SDVL5OJF.js → chunk-3BGDI4HI.js} +54 -16
- package/dist/{chunk-OJAG3FRX.js → chunk-3ZRMKVBB.js} +108 -12
- package/dist/{cli-FYWM4HBN.js → cli-MQO66C5O.js} +15 -7
- package/dist/index.js +2 -2
- package/dist/install-VXJPLYWM.js +4 -0
- package/dist/offer-W6NJXMWT.js +71 -0
- package/dist/{refresh-FKQABJOC.js → refresh-5HBDLLLB.js} +3 -3
- package/dist/serve.js +2707 -625
- package/dist/{src-CSR34EBZ.js → src-3ITTV2UE.js} +3 -3
- package/package.json +1 -1
- package/studio-dist/assets/{index-DfAHZRYM.css → index-BGts89lc.css} +1 -1
- package/studio-dist/assets/index-uiX1mW43.js +104 -0
- package/studio-dist/index.html +2 -2
- package/dist/install-4W6AARK6.js +0 -4
- package/dist/offer-IVTNHIDJ.js +0 -47
- package/studio-dist/assets/index-CqJgKSNM.js +0 -104
package/dist/serve.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createUpdateChecker, classify, isReleaseTriplet, findNpm, stageVersion } from './chunk-FYQ5BAFA.js';
|
|
2
2
|
import { shouldAdoptRunning, anotherScenriLines, portBusyLines } from './chunk-CMLJGPYC.js';
|
|
3
|
-
import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-
|
|
3
|
+
import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-3ZRMKVBB.js';
|
|
4
4
|
import { detectInstallKind } from './chunk-PAIAXRAC.js';
|
|
5
5
|
export { detectInstallKind } from './chunk-PAIAXRAC.js';
|
|
6
6
|
import { readMeta, repoSlug } from './chunk-Y3ZPBPLP.js';
|
|
@@ -12,7 +12,7 @@ import { homedir, networkInterfaces, tmpdir } from 'os';
|
|
|
12
12
|
import { randomBytes, randomUUID, timingSafeEqual, createHash } from 'crypto';
|
|
13
13
|
import { readFile, copyFile, stat, readdir, mkdtemp, rm, access, rename, unlink, writeFile } from 'fs/promises';
|
|
14
14
|
import { spawn } from 'child_process';
|
|
15
|
-
import
|
|
15
|
+
import sharp21 from 'sharp';
|
|
16
16
|
import Fastify from 'fastify';
|
|
17
17
|
import fastifyStatic from '@fastify/static';
|
|
18
18
|
import fastifyMultipart from '@fastify/multipart';
|
|
@@ -20,6 +20,8 @@ import JSZip from 'jszip';
|
|
|
20
20
|
import { Ajv2020 } from 'ajv/dist/2020.js';
|
|
21
21
|
import addFormats from 'ajv-formats';
|
|
22
22
|
import * as cheerio from 'cheerio';
|
|
23
|
+
import { lookup } from 'dns/promises';
|
|
24
|
+
import { isIP } from 'net';
|
|
23
25
|
import { parse } from 'node-html-parser';
|
|
24
26
|
import pixelmatch from 'pixelmatch';
|
|
25
27
|
import { PNG } from 'pngjs';
|
|
@@ -687,6 +689,108 @@ function resolveCodex(platform, spawnImpl, timeoutMs = WHERE_TIMEOUT_MS) {
|
|
|
687
689
|
});
|
|
688
690
|
}
|
|
689
691
|
|
|
692
|
+
// ../engines/codex/src/childEnv.ts
|
|
693
|
+
function buildChildEnv(parent, drop) {
|
|
694
|
+
if (drop.length === 0) return { ...parent };
|
|
695
|
+
const unwanted = new Set(drop.map((name) => name.toUpperCase()));
|
|
696
|
+
const child = {};
|
|
697
|
+
for (const [name, value] of Object.entries(parent)) {
|
|
698
|
+
if (unwanted.has(name.toUpperCase())) continue;
|
|
699
|
+
child[name] = value;
|
|
700
|
+
}
|
|
701
|
+
return child;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
// ../engines/codex/src/classify.ts
|
|
705
|
+
var CONFLICT_ENV_KEYS = ["CODEX_API_KEY", "CODEX_ACCESS_TOKEN", "OPENAI_API_KEY"];
|
|
706
|
+
function presentConflictKeys(env, ignored = []) {
|
|
707
|
+
const dropped = new Set(ignored.map((n) => n.toUpperCase()));
|
|
708
|
+
const present = [];
|
|
709
|
+
for (const name of CONFLICT_ENV_KEYS) {
|
|
710
|
+
if (dropped.has(name)) continue;
|
|
711
|
+
const value = env[name];
|
|
712
|
+
if (typeof value === "string" && value.trim() !== "") present.push(name);
|
|
713
|
+
}
|
|
714
|
+
return present;
|
|
715
|
+
}
|
|
716
|
+
function listKeys(keys) {
|
|
717
|
+
if (keys.length === 0) return "";
|
|
718
|
+
if (keys.length === 1) return keys[0];
|
|
719
|
+
return `${keys.slice(0, -1).join(", ")} and ${keys[keys.length - 1]}`;
|
|
720
|
+
}
|
|
721
|
+
function conflictReason(keys) {
|
|
722
|
+
return `Codex is signed in, but ${listKeys(keys)} in this computer's environment is overriding that sign-in and OpenAI rejected it.`;
|
|
723
|
+
}
|
|
724
|
+
var NOT_FOUND = /failed to spawn codex|\bENOENT\b|command not found|is not recognized as an internal/i;
|
|
725
|
+
var TOO_OLD = /requires a newer version of Codex|is too old/i;
|
|
726
|
+
var USAGE_LIMIT = /usage limit/i;
|
|
727
|
+
var SIGNED_OUT = /not logged in|login required|run .{0,3}codex login|no credentials found/i;
|
|
728
|
+
var UNAUTHORIZED = /\b401\b|unauthorized/i;
|
|
729
|
+
var BAD_KEY = /incorrect api key provided|invalid_api_key|invalid api key/i;
|
|
730
|
+
var NETWORK = /ENOTFOUND|ECONNREFUSED|ECONNRESET|EAI_AGAIN|getaddrinfo|error sending request|dns error|tls handshake/i;
|
|
731
|
+
var NO_OUTPUT = /produced no output for \d+s/i;
|
|
732
|
+
var TIMED_OUT = /timed out after/i;
|
|
733
|
+
var ALREADY_CONFLICT = /environment is overriding that sign-in/i;
|
|
734
|
+
function classifyCodexFailure(input) {
|
|
735
|
+
const text = input.text ?? "";
|
|
736
|
+
const conflictKeys = presentConflictKeys(input.env, input.ignored);
|
|
737
|
+
const at = (code, reason) => ({ code, conflictKeys, reason });
|
|
738
|
+
if (ALREADY_CONFLICT.test(text)) return at("CODEX_AUTH_CONFLICT", conflictReason(conflictKeys));
|
|
739
|
+
if (NOT_FOUND.test(text)) return at("CODEX_NOT_FOUND", "Codex CLI is not installed on this computer.");
|
|
740
|
+
if (TOO_OLD.test(text)) return at("CODEX_VERSION_TOO_OLD", "Codex CLI on this computer is too old.");
|
|
741
|
+
if (USAGE_LIMIT.test(text)) return at("CODEX_USAGE_LIMIT", "Your Codex plan's usage limit is used up.");
|
|
742
|
+
if (SIGNED_OUT.test(text)) return at("CODEX_NOT_AUTHENTICATED", "Codex CLI is signed out on this computer.");
|
|
743
|
+
if (UNAUTHORIZED.test(text)) {
|
|
744
|
+
if (BAD_KEY.test(text) && conflictKeys.length > 0) return at("CODEX_AUTH_CONFLICT", conflictReason(conflictKeys));
|
|
745
|
+
return at("CODEX_401", "OpenAI did not accept the Codex sign-in on this computer.");
|
|
746
|
+
}
|
|
747
|
+
if (NETWORK.test(text)) return at("NETWORK_FAILURE", "Codex could not reach OpenAI.");
|
|
748
|
+
if (NO_OUTPUT.test(text)) return at("CODEX_NO_OUTPUT", "Codex never started answering.");
|
|
749
|
+
if (TIMED_OUT.test(text)) return at("CODEX_TIMEOUT", "Codex ran out of time.");
|
|
750
|
+
if (text.trim() !== "") return at("CODEX_EXECUTION_FAILED", "Codex stopped with an error.");
|
|
751
|
+
return at("UNKNOWN", "Codex stopped for a reason it did not give.");
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// ../engines/codex/src/connect.ts
|
|
755
|
+
var CONNECT_PROMPT = "Reply with the single word READY. Do not use any tools. Do not generate an image. Do not read or write any files.";
|
|
756
|
+
var CONNECT_TIMEOUT_MS = 45e3;
|
|
757
|
+
var CONNECT_TTL_MS = 10 * 6e4;
|
|
758
|
+
function connectFingerprint(p) {
|
|
759
|
+
return [p.command, p.version ?? "unknown", [...p.present].sort().join("+"), [...p.ignored].sort().join("+")].join(
|
|
760
|
+
"|"
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
function outcomeFor(code) {
|
|
764
|
+
switch (code) {
|
|
765
|
+
case "CODEX_AUTH_CONFLICT":
|
|
766
|
+
case "CODEX_401":
|
|
767
|
+
case "CODEX_NOT_AUTHENTICATED":
|
|
768
|
+
case "CODEX_NOT_FOUND":
|
|
769
|
+
case "CODEX_VERSION_TOO_OLD":
|
|
770
|
+
return "refused";
|
|
771
|
+
default:
|
|
772
|
+
return "unproven";
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
function availabilityFrom(shallow, conn) {
|
|
776
|
+
if (!shallow.ok) return shallow;
|
|
777
|
+
if (!conn || conn.outcome !== "refused" || !conn.failure) return shallow;
|
|
778
|
+
const { code, reason } = conn.failure;
|
|
779
|
+
switch (code) {
|
|
780
|
+
case "CODEX_AUTH_CONFLICT":
|
|
781
|
+
return { ok: false, reason, code: "env-conflict" };
|
|
782
|
+
case "CODEX_401":
|
|
783
|
+
case "CODEX_NOT_AUTHENTICATED":
|
|
784
|
+
return { ok: false, reason, code: "not-authenticated" };
|
|
785
|
+
case "CODEX_VERSION_TOO_OLD":
|
|
786
|
+
return { ok: false, reason, code: "update-needed" };
|
|
787
|
+
case "CODEX_NOT_FOUND":
|
|
788
|
+
return { ok: false, reason, code: "not-installed" };
|
|
789
|
+
default:
|
|
790
|
+
return shallow;
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
|
|
690
794
|
// ../engines/codex/src/run.ts
|
|
691
795
|
var NOT_INSTALLED_REASON = "Codex CLI is not installed on this computer";
|
|
692
796
|
var NOT_AUTHENTICATED_REASON = "Codex CLI is installed but not signed in";
|
|
@@ -741,6 +845,7 @@ function codexFailureDetail(stderr, stdout) {
|
|
|
741
845
|
return tailOf(marked) || tailOf(body) || tailOf(stdout.trim()) || tailOf(stderr.trim());
|
|
742
846
|
}
|
|
743
847
|
var MODEL_NEEDS_NEWER_CODEX = /The '([^']+)' model requires a newer version of Codex/;
|
|
848
|
+
var BAD_KEY_401 = /incorrect api key provided|invalid_api_key|invalid api key/i;
|
|
744
849
|
function tooOldForModel(version, model) {
|
|
745
850
|
return `Codex CLI ${version ?? "on this computer"} is too old for the model it is set to, ${model}.`;
|
|
746
851
|
}
|
|
@@ -781,6 +886,10 @@ function createRunner(opts = {}) {
|
|
|
781
886
|
const probeTtlMs = opts.probeTtlMs ?? PROBE_TTL_MS;
|
|
782
887
|
const firstOutputMs = opts.firstOutputMs ?? FIRST_OUTPUT_TIMEOUT_MS;
|
|
783
888
|
const platform = opts.platform ?? process.platform;
|
|
889
|
+
const connectTtlMs = opts.connectTtlMs ?? CONNECT_TTL_MS;
|
|
890
|
+
const connectTimeoutMs = opts.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
|
|
891
|
+
const parentEnv = opts.env ?? process.env;
|
|
892
|
+
const ignoreEnvKeys = opts.ignoreEnvKeys ?? (() => []);
|
|
784
893
|
const killCodex = (child) => killTree(child, platform, spawnImpl);
|
|
785
894
|
let resolved = null;
|
|
786
895
|
async function resolution() {
|
|
@@ -792,7 +901,8 @@ function createRunner(opts = {}) {
|
|
|
792
901
|
const winArg = (a) => `"${a.replace(/[\r\n]+/g, " ").replace(/"/g, "'").replace(/%/g, " percent ")}"`;
|
|
793
902
|
const spawnCodex = (exe, args, stdinOpen) => {
|
|
794
903
|
const stdio = [stdinOpen ? "pipe" : "ignore", "pipe", "pipe"];
|
|
795
|
-
|
|
904
|
+
const env = buildChildEnv(parentEnv, ignoreEnvKeys());
|
|
905
|
+
return exe.direct ? spawnImpl(exe.command, args, { stdio, env, ...platform !== "win32" ? { detached: true } : {} }) : spawnImpl([exe.command, ...args.map(winArg)].join(" "), [], { stdio, env, shell: true });
|
|
796
906
|
};
|
|
797
907
|
async function run2(args, signal, io) {
|
|
798
908
|
const exe = await resolution();
|
|
@@ -907,6 +1017,16 @@ function createRunner(opts = {}) {
|
|
|
907
1017
|
);
|
|
908
1018
|
return;
|
|
909
1019
|
}
|
|
1020
|
+
const conflicting = presentConflictKeys(parentEnv, ignoreEnvKeys());
|
|
1021
|
+
if (conflicting.length > 0 && /\b401\b|unauthorized/i.test(stderr) && BAD_KEY_401.test(stderr)) {
|
|
1022
|
+
noteConnection("refused", {
|
|
1023
|
+
code: "CODEX_AUTH_CONFLICT",
|
|
1024
|
+
conflictKeys: conflicting,
|
|
1025
|
+
reason: conflictReason(conflicting)
|
|
1026
|
+
});
|
|
1027
|
+
finish(`exit-${code ?? "unknown"}`, () => reject(new Error(conflictReason(conflicting))));
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
910
1030
|
const snippet2 = codexFailureDetail(stderr, stdout);
|
|
911
1031
|
finish(
|
|
912
1032
|
`exit-${code ?? "unknown"}`,
|
|
@@ -966,6 +1086,23 @@ function createRunner(opts = {}) {
|
|
|
966
1086
|
return avail;
|
|
967
1087
|
}
|
|
968
1088
|
let cached = null;
|
|
1089
|
+
let connCache = null;
|
|
1090
|
+
let connectInFlight = null;
|
|
1091
|
+
function fingerprintFor(exe, version) {
|
|
1092
|
+
const ignored = ignoreEnvKeys();
|
|
1093
|
+
return connectFingerprint({
|
|
1094
|
+
command: exe.command,
|
|
1095
|
+
version,
|
|
1096
|
+
present: presentConflictKeys(parentEnv, ignored),
|
|
1097
|
+
ignored
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
function freshConnection(exe, version) {
|
|
1101
|
+
if (!connCache || connectTtlMs <= 0) return null;
|
|
1102
|
+
if (Date.now() - connCache.at >= connectTtlMs) return null;
|
|
1103
|
+
if (connCache.fingerprint !== fingerprintFor(exe, version)) return null;
|
|
1104
|
+
return connCache;
|
|
1105
|
+
}
|
|
969
1106
|
async function probe() {
|
|
970
1107
|
if (process.env.SCENRI_NO_CODEX === "1") {
|
|
971
1108
|
return { ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" };
|
|
@@ -978,52 +1115,128 @@ function createRunner(opts = {}) {
|
|
|
978
1115
|
return value;
|
|
979
1116
|
}
|
|
980
1117
|
async function probeUncached() {
|
|
1118
|
+
const { avail, exe, version } = await probeLadder();
|
|
1119
|
+
return availabilityFrom(avail, freshConnection(exe, version));
|
|
1120
|
+
}
|
|
1121
|
+
async function probeLadder() {
|
|
981
1122
|
resolved = await resolveCodex(platform, spawnImpl);
|
|
982
1123
|
const exe = resolved;
|
|
983
1124
|
const ver = await probeSpawn(exe, ["--version"]);
|
|
984
1125
|
if (ver.outcome === "timeout") {
|
|
985
|
-
return
|
|
1126
|
+
return {
|
|
1127
|
+
avail: verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, null),
|
|
1128
|
+
exe,
|
|
1129
|
+
version: null
|
|
1130
|
+
};
|
|
986
1131
|
}
|
|
987
1132
|
if (ver.outcome !== "ok") {
|
|
988
|
-
return
|
|
1133
|
+
return {
|
|
1134
|
+
avail: verdict({ ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" }, exe, null),
|
|
1135
|
+
exe,
|
|
1136
|
+
version: null
|
|
1137
|
+
};
|
|
989
1138
|
}
|
|
990
1139
|
const version = parseCodexVersion(ver.stdout);
|
|
991
1140
|
knownVersion = version;
|
|
992
1141
|
if (version && !versionAtLeast(version, MIN_CODEX_VERSION)) {
|
|
993
|
-
return
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1142
|
+
return {
|
|
1143
|
+
avail: verdict(
|
|
1144
|
+
{
|
|
1145
|
+
ok: false,
|
|
1146
|
+
reason: `Codex CLI ${version} is too old. Scenri needs ${MIN_CODEX_VERSION} or newer.`,
|
|
1147
|
+
code: "update-needed"
|
|
1148
|
+
},
|
|
1149
|
+
exe,
|
|
1150
|
+
version
|
|
1151
|
+
),
|
|
999
1152
|
exe,
|
|
1000
1153
|
version
|
|
1001
|
-
|
|
1154
|
+
};
|
|
1002
1155
|
}
|
|
1003
1156
|
if (tooOldFor) {
|
|
1004
1157
|
if (version === tooOldFor.version) {
|
|
1005
|
-
return
|
|
1006
|
-
|
|
1158
|
+
return {
|
|
1159
|
+
avail: verdict(
|
|
1160
|
+
{ ok: false, reason: tooOldForModel(version, tooOldFor.model), code: "update-needed" },
|
|
1161
|
+
exe,
|
|
1162
|
+
version
|
|
1163
|
+
),
|
|
1007
1164
|
exe,
|
|
1008
1165
|
version
|
|
1009
|
-
|
|
1166
|
+
};
|
|
1010
1167
|
}
|
|
1011
1168
|
tooOldFor = null;
|
|
1012
1169
|
}
|
|
1013
1170
|
const login = await probeSpawn(exe, ["login", "status"]);
|
|
1014
1171
|
if (login.outcome === "ok") {
|
|
1015
|
-
return verdict({ ok: true }, exe, version);
|
|
1172
|
+
return { avail: verdict({ ok: true }, exe, version), exe, version };
|
|
1016
1173
|
}
|
|
1017
1174
|
if (login.outcome === "nonzero") {
|
|
1018
|
-
return
|
|
1175
|
+
return {
|
|
1176
|
+
avail: verdict({ ok: false, reason: NOT_AUTHENTICATED_REASON, code: "not-authenticated" }, exe, version),
|
|
1177
|
+
exe,
|
|
1178
|
+
version
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
return { avail: verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, version), exe, version };
|
|
1182
|
+
}
|
|
1183
|
+
async function connect(o = {}) {
|
|
1184
|
+
if (connectInFlight) return connectInFlight;
|
|
1185
|
+
const pending = (async () => {
|
|
1186
|
+
const { avail, exe, version } = await probeLadder();
|
|
1187
|
+
const fingerprint = fingerprintFor(exe, version);
|
|
1188
|
+
if (!o.force) {
|
|
1189
|
+
const fresh = freshConnection(exe, version);
|
|
1190
|
+
if (fresh) return fresh;
|
|
1191
|
+
}
|
|
1192
|
+
if (!avail.ok) {
|
|
1193
|
+
return { outcome: "unproven", at: Date.now(), fingerprint };
|
|
1194
|
+
}
|
|
1195
|
+
let result;
|
|
1196
|
+
try {
|
|
1197
|
+
await withWorkDir(
|
|
1198
|
+
(dir) => run2(execArgs(dir), void 0, {
|
|
1199
|
+
stdin: CONNECT_PROMPT,
|
|
1200
|
+
timeoutMs: connectTimeoutMs,
|
|
1201
|
+
label: "connect"
|
|
1202
|
+
})
|
|
1203
|
+
);
|
|
1204
|
+
result = { outcome: "proven", at: Date.now(), fingerprint };
|
|
1205
|
+
} catch (err) {
|
|
1206
|
+
const failure = classifyCodexFailure({
|
|
1207
|
+
text: err instanceof Error ? err.message : String(err),
|
|
1208
|
+
env: parentEnv,
|
|
1209
|
+
ignored: ignoreEnvKeys()
|
|
1210
|
+
});
|
|
1211
|
+
result = { outcome: outcomeFor(failure.code), failure, at: Date.now(), fingerprint };
|
|
1212
|
+
}
|
|
1213
|
+
connCache = result;
|
|
1214
|
+
cached = null;
|
|
1215
|
+
return result;
|
|
1216
|
+
})();
|
|
1217
|
+
connectInFlight = pending;
|
|
1218
|
+
try {
|
|
1219
|
+
return await pending;
|
|
1220
|
+
} finally {
|
|
1221
|
+
connectInFlight = null;
|
|
1019
1222
|
}
|
|
1020
|
-
|
|
1223
|
+
}
|
|
1224
|
+
function noteConnection(outcome, failure) {
|
|
1225
|
+
if (outcome === "unproven") return;
|
|
1226
|
+
const exe = resolved;
|
|
1227
|
+
if (!exe) return;
|
|
1228
|
+
connCache = { outcome, failure, at: Date.now(), fingerprint: fingerprintFor(exe, knownVersion) };
|
|
1229
|
+
cached = null;
|
|
1021
1230
|
}
|
|
1022
1231
|
function invalidateProbe() {
|
|
1023
1232
|
cached = null;
|
|
1024
1233
|
resolved = null;
|
|
1025
1234
|
}
|
|
1026
|
-
|
|
1235
|
+
function invalidateConnection() {
|
|
1236
|
+
connCache = null;
|
|
1237
|
+
cached = null;
|
|
1238
|
+
}
|
|
1239
|
+
return { run: run2, withWorkDir, probe, invalidateProbe, connect, invalidateConnection, noteConnection };
|
|
1027
1240
|
}
|
|
1028
1241
|
var OUT_FILE = "analysis.json";
|
|
1029
1242
|
function createCodexAnalyzer(opts = {}) {
|
|
@@ -1182,6 +1395,7 @@ function pick(v, allowed, max) {
|
|
|
1182
1395
|
var INSTALL_COMMAND = "npm install -g @openai/codex";
|
|
1183
1396
|
var INSTALL_DOCS_URL = "https://developers.openai.com/codex/cli";
|
|
1184
1397
|
var INSTALL_COMMAND_SUDO = "sudo npm install -g @openai/codex";
|
|
1398
|
+
var INSTALL_COMMAND_WINDOWS = 'powershell -ExecutionPolicy ByPass -c "irm https://chatgpt.com/codex/install.ps1 | iex"';
|
|
1185
1399
|
var DEFAULT_INSTALL_TIMEOUT_MS = 18e4;
|
|
1186
1400
|
function stateFrom(avail) {
|
|
1187
1401
|
if (avail.ok) return "ready";
|
|
@@ -1189,6 +1403,7 @@ function stateFrom(avail) {
|
|
|
1189
1403
|
case "not-authenticated":
|
|
1190
1404
|
case "update-needed":
|
|
1191
1405
|
case "unverified":
|
|
1406
|
+
case "env-conflict":
|
|
1192
1407
|
return avail.code;
|
|
1193
1408
|
default:
|
|
1194
1409
|
return "not-installed";
|
|
@@ -1199,6 +1414,7 @@ function createCodexSetup(opts = {}) {
|
|
|
1199
1414
|
const platform = opts.platform ?? process.platform;
|
|
1200
1415
|
const runner = opts.runner ?? createRunner(opts);
|
|
1201
1416
|
const installTimeoutMs = opts.installTimeoutMs ?? DEFAULT_INSTALL_TIMEOUT_MS;
|
|
1417
|
+
const ignoreEnvKeys = opts.ignoreEnvKeys ?? (() => []);
|
|
1202
1418
|
function run2(cmd, args, timeoutMs) {
|
|
1203
1419
|
return new Promise((resolve) => {
|
|
1204
1420
|
let settled = false;
|
|
@@ -1230,11 +1446,22 @@ function createCodexSetup(opts = {}) {
|
|
|
1230
1446
|
});
|
|
1231
1447
|
}
|
|
1232
1448
|
return {
|
|
1233
|
-
async status() {
|
|
1449
|
+
async status(o = {}) {
|
|
1234
1450
|
runner.invalidateProbe();
|
|
1451
|
+
if (o.force) runner.invalidateConnection();
|
|
1452
|
+
const conn = await runner.connect();
|
|
1235
1453
|
const avail = await runner.probe();
|
|
1454
|
+
const state = stateFrom(avail);
|
|
1236
1455
|
const setupPlatform = platform === "win32" ? "windows" : platform === "darwin" ? "mac" : "linux";
|
|
1237
|
-
return {
|
|
1456
|
+
return {
|
|
1457
|
+
state,
|
|
1458
|
+
reason: avail.reason,
|
|
1459
|
+
platform: setupPlatform,
|
|
1460
|
+
conflictKeys: state === "env-conflict" ? conn.failure?.conflictKeys ?? [] : [],
|
|
1461
|
+
ignoredKeys: [...ignoreEnvKeys()].filter(
|
|
1462
|
+
(k) => CONFLICT_ENV_KEYS.includes(k)
|
|
1463
|
+
)
|
|
1464
|
+
};
|
|
1238
1465
|
},
|
|
1239
1466
|
async install() {
|
|
1240
1467
|
const res = await run2("npm", ["install", "-g", "@openai/codex"], installTimeoutMs);
|
|
@@ -1251,6 +1478,14 @@ function createCodexSetup(opts = {}) {
|
|
|
1251
1478
|
}
|
|
1252
1479
|
return { ok: true };
|
|
1253
1480
|
}
|
|
1481
|
+
if (platform === "win32" && /EPERM|EACCES|EBUSY|permission denied|operation not permitted/i.test(res.stderr)) {
|
|
1482
|
+
return {
|
|
1483
|
+
ok: false,
|
|
1484
|
+
fallbackCommand: INSTALL_COMMAND_WINDOWS,
|
|
1485
|
+
docsUrl: INSTALL_DOCS_URL,
|
|
1486
|
+
detail: "npm could not write its files. This is almost never about administrator rights: close Codex and any terminal window using it, then try again. If it keeps failing, the command below is OpenAI\u2019s own installer, which does not go through npm."
|
|
1487
|
+
};
|
|
1488
|
+
}
|
|
1254
1489
|
if (platform !== "win32" && /EACCES|permission denied/i.test(res.stderr)) {
|
|
1255
1490
|
return {
|
|
1256
1491
|
ok: false,
|
|
@@ -1452,6 +1687,7 @@ function createCodexEngine(opts) {
|
|
|
1452
1687
|
fatal = err;
|
|
1453
1688
|
inner.abort();
|
|
1454
1689
|
runner.invalidateProbe();
|
|
1690
|
+
noteFromError(err);
|
|
1455
1691
|
}
|
|
1456
1692
|
}
|
|
1457
1693
|
}
|
|
@@ -1514,7 +1750,10 @@ function createCodexEngine(opts) {
|
|
|
1514
1750
|
try {
|
|
1515
1751
|
await runCodex(args, signal, { stdin: promptText, label: `edit refs=${editRefs.length}` });
|
|
1516
1752
|
} catch (err) {
|
|
1517
|
-
if (isFatalSetupError(err))
|
|
1753
|
+
if (isFatalSetupError(err)) {
|
|
1754
|
+
runner.invalidateProbe();
|
|
1755
|
+
noteFromError(err);
|
|
1756
|
+
}
|
|
1518
1757
|
throw err;
|
|
1519
1758
|
}
|
|
1520
1759
|
const images = await collectImages(dir, before);
|
|
@@ -1523,10 +1762,17 @@ function createCodexEngine(opts) {
|
|
|
1523
1762
|
}
|
|
1524
1763
|
};
|
|
1525
1764
|
function isFatalSetupError(err) {
|
|
1526
|
-
return /failed to spawn|ENOENT|not logged in|login required|401|unauthorized|is too old/i.test(
|
|
1765
|
+
return /failed to spawn|ENOENT|not logged in|login required|401|unauthorized|is too old|environment is overriding/i.test(
|
|
1527
1766
|
String(err?.message ?? err)
|
|
1528
1767
|
);
|
|
1529
1768
|
}
|
|
1769
|
+
function noteFromError(err) {
|
|
1770
|
+
const failure = classifyCodexFailure({
|
|
1771
|
+
text: String(err?.message ?? err),
|
|
1772
|
+
env: process.env
|
|
1773
|
+
});
|
|
1774
|
+
runner.noteConnection(outcomeFor(failure.code), failure);
|
|
1775
|
+
}
|
|
1530
1776
|
function buildPrompt2(req, index, roles) {
|
|
1531
1777
|
const variation = req.variations?.[index] ?? "";
|
|
1532
1778
|
const roleDirective = REFERENCE_ROLE_DIRECTIVE;
|
|
@@ -1566,9 +1812,15 @@ function createCodexEngine(opts) {
|
|
|
1566
1812
|
function keyGetter(core, settingKey, envVar) {
|
|
1567
1813
|
return () => core.store.getSetting(settingKey) || process.env[envVar] || null;
|
|
1568
1814
|
}
|
|
1815
|
+
var IGNORE_ENV_KEYS_SETTING = "codex.ignore_env_keys";
|
|
1816
|
+
function ignoreEnvKeysGetter(core) {
|
|
1817
|
+
return () => (core.store.getSetting(IGNORE_ENV_KEYS_SETTING) ?? "").split(",").map((name) => name.trim().toUpperCase()).filter(
|
|
1818
|
+
(name) => CONFLICT_ENV_KEYS.includes(name)
|
|
1819
|
+
);
|
|
1820
|
+
}
|
|
1569
1821
|
function createEngineRegistry(core, extra = []) {
|
|
1570
1822
|
const saveImage = (buf) => core.images.save(buf);
|
|
1571
|
-
const codexRunner = createRunner();
|
|
1823
|
+
const codexRunner = createRunner({ ignoreEnvKeys: ignoreEnvKeysGetter(core) });
|
|
1572
1824
|
const adapters = [
|
|
1573
1825
|
createOpenRouterEngine({ getKey: keyGetter(core, "openrouter_api_key", "OPENROUTER_API_KEY"), saveImage }),
|
|
1574
1826
|
createReplicateEngine({ getKey: keyGetter(core, "replicate_api_token", "REPLICATE_API_TOKEN"), saveImage }),
|
|
@@ -1627,7 +1879,7 @@ function createDemoEngine(saveImage, opts = {}) {
|
|
|
1627
1879
|
<text x="24" y="${h - 48}" font-family="Helvetica, Arial" font-size="${Math.max(14, Math.round(w / 42))}" fill="#ffffff" opacity="0.92">${esc(label)}</text>
|
|
1628
1880
|
<text x="24" y="${h - 22}" font-family="Helvetica, Arial" font-size="12" fill="#ffffff" opacity="0.6">Scenri demo engine</text>
|
|
1629
1881
|
</svg>`;
|
|
1630
|
-
return
|
|
1882
|
+
return sharp21(Buffer.from(svg)).png().toBuffer();
|
|
1631
1883
|
}
|
|
1632
1884
|
return {
|
|
1633
1885
|
capabilities() {
|
|
@@ -1818,7 +2070,7 @@ var resolvedRefs = /* @__PURE__ */ new Map();
|
|
|
1818
2070
|
async function refHash(core, path) {
|
|
1819
2071
|
const hit = resolvedRefs.get(path);
|
|
1820
2072
|
if (hit && core.images.has(hit)) return hit;
|
|
1821
|
-
const hash = core.images.save(await
|
|
2073
|
+
const hash = core.images.save(await sharp21(readFileSync(path)).png().toBuffer());
|
|
1822
2074
|
resolvedRefs.set(path, hash);
|
|
1823
2075
|
return hash;
|
|
1824
2076
|
}
|
|
@@ -1922,7 +2174,7 @@ var resolvedRefs2 = /* @__PURE__ */ new Map();
|
|
|
1922
2174
|
async function refHash2(core, path) {
|
|
1923
2175
|
const hit = resolvedRefs2.get(path);
|
|
1924
2176
|
if (hit && core.images.has(hit)) return hit;
|
|
1925
|
-
const hash = core.images.save(await
|
|
2177
|
+
const hash = core.images.save(await sharp21(readFileSync(path)).png().toBuffer());
|
|
1926
2178
|
resolvedRefs2.set(path, hash);
|
|
1927
2179
|
return hash;
|
|
1928
2180
|
}
|
|
@@ -2076,7 +2328,7 @@ function createThumbStore(core, opts = {}) {
|
|
|
2076
2328
|
const pathFor = (key, w) => join(dir, `${key}-w${w}.webp`);
|
|
2077
2329
|
const inflight = /* @__PURE__ */ new Map();
|
|
2078
2330
|
const failed = /* @__PURE__ */ new Set();
|
|
2079
|
-
const concurrency = Math.max(1, opts.concurrency ??
|
|
2331
|
+
const concurrency = Math.max(1, opts.concurrency ?? 4);
|
|
2080
2332
|
let active = 0;
|
|
2081
2333
|
const waiting = [];
|
|
2082
2334
|
const acquire = () => new Promise((resolve) => {
|
|
@@ -2095,7 +2347,7 @@ function createThumbStore(core, opts = {}) {
|
|
|
2095
2347
|
const tmp = `${final}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
2096
2348
|
await acquire();
|
|
2097
2349
|
try {
|
|
2098
|
-
await
|
|
2350
|
+
await sharp21(source).resize({ width: w, withoutEnlargement: true }).webp({ quality: QUALITY[w], effort: 4 }).toFile(tmp);
|
|
2099
2351
|
await rename(tmp, final);
|
|
2100
2352
|
return final;
|
|
2101
2353
|
} catch {
|
|
@@ -2185,7 +2437,7 @@ var assetHash = (ref) => {
|
|
|
2185
2437
|
};
|
|
2186
2438
|
var LOGO_ROLES = ["primary", "mark", "wordmark", "monochrome", "alternate"];
|
|
2187
2439
|
var LOGO_BACKGROUNDS = ["light", "dark", "any"];
|
|
2188
|
-
var toPng = (buf) =>
|
|
2440
|
+
var toPng = (buf) => sharp21(buf).rotate().png().toBuffer();
|
|
2189
2441
|
var COST_PROBE = {
|
|
2190
2442
|
prompt: "",
|
|
2191
2443
|
brand: { brand: {}, assetPaths: {} },
|
|
@@ -2198,11 +2450,11 @@ var MARK_MIN_EDGE = 1024;
|
|
|
2198
2450
|
var MARK_TINY_EDGE = 256;
|
|
2199
2451
|
var MARK_WARN_EDGE = 512;
|
|
2200
2452
|
var toMarkPng = async (buf) => {
|
|
2201
|
-
const out = await
|
|
2202
|
-
const meta = await
|
|
2453
|
+
const out = await sharp21(buf, { density: 384 }).rotate().resize({ width: MARK_MAX_EDGE, height: MARK_MAX_EDGE, fit: "inside", withoutEnlargement: true }).png().toBuffer();
|
|
2454
|
+
const meta = await sharp21(out).metadata();
|
|
2203
2455
|
const edge = Math.max(meta.width ?? 0, meta.height ?? 0);
|
|
2204
2456
|
if (edge >= MARK_TINY_EDGE && edge < MARK_MIN_EDGE) {
|
|
2205
|
-
return
|
|
2457
|
+
return sharp21(out).resize({ width: MARK_MIN_EDGE, height: MARK_MIN_EDGE, fit: "inside", kernel: "lanczos3" }).png().toBuffer();
|
|
2206
2458
|
}
|
|
2207
2459
|
return out;
|
|
2208
2460
|
};
|
|
@@ -2213,9 +2465,9 @@ async function capReferenceEdge(core, path, maxEdge) {
|
|
|
2213
2465
|
if (hit) return hit;
|
|
2214
2466
|
let out = path;
|
|
2215
2467
|
try {
|
|
2216
|
-
const meta = await
|
|
2468
|
+
const meta = await sharp21(path).metadata();
|
|
2217
2469
|
if ((meta.width ?? 0) > maxEdge || (meta.height ?? 0) > maxEdge) {
|
|
2218
|
-
const buf = await
|
|
2470
|
+
const buf = await sharp21(path).resize({ width: maxEdge, height: maxEdge, fit: "inside", withoutEnlargement: true }).png().toBuffer();
|
|
2219
2471
|
out = core.images.pathFor(core.images.save(buf));
|
|
2220
2472
|
}
|
|
2221
2473
|
} catch {
|
|
@@ -3491,13 +3743,122 @@ function validateBrand(json) {
|
|
|
3491
3743
|
errors: valid ? [] : (compiled.errors ?? []).map((e) => `${e.instancePath || "/"} ${e.message}`)
|
|
3492
3744
|
};
|
|
3493
3745
|
}
|
|
3494
|
-
|
|
3495
|
-
|
|
3746
|
+
|
|
3747
|
+
// ../brand-spec/src/colorParse.ts
|
|
3748
|
+
var CLAMP = (n) => Math.min(255, Math.max(0, Math.round(n)));
|
|
3749
|
+
var hex2 = (n) => CLAMP(n).toString(16).padStart(2, "0");
|
|
3750
|
+
var rgbHex = (r, g, b) => `#${hex2(r)}${hex2(g)}${hex2(b)}`;
|
|
3751
|
+
var MIN_ALPHA = 0.35;
|
|
3752
|
+
var NUM = String.raw`[-+]?[\d.]+%?`;
|
|
3753
|
+
var RGB_RE = new RegExp(String.raw`^rgba?\(\s*(${NUM})[\s,]+(${NUM})[\s,]+(${NUM})(?:[\s,/]+(${NUM}))?\s*\)$`, "i");
|
|
3754
|
+
var HSL_RE = new RegExp(
|
|
3755
|
+
String.raw`^hsla?\(\s*(${NUM}|[-+]?[\d.]+deg)[\s,]+(${NUM})[\s,]+(${NUM})(?:[\s,/]+(${NUM}))?\s*\)$`,
|
|
3756
|
+
"i"
|
|
3757
|
+
);
|
|
3758
|
+
var OKLCH_RE = new RegExp(
|
|
3759
|
+
String.raw`^oklch\(\s*(${NUM})[\s,]+(${NUM})[\s,]+([-+]?[\d.]+)(?:deg)?(?:[\s,/]+(${NUM}))?\s*\)$`,
|
|
3760
|
+
"i"
|
|
3761
|
+
);
|
|
3762
|
+
function scalar(token, full) {
|
|
3763
|
+
const t = token.trim();
|
|
3764
|
+
if (t.endsWith("%")) return Number.parseFloat(t) / 100 * full;
|
|
3765
|
+
return Number.parseFloat(t);
|
|
3766
|
+
}
|
|
3767
|
+
function alphaOk(token) {
|
|
3768
|
+
if (token == null) return true;
|
|
3769
|
+
const a = token.trim().endsWith("%") ? Number.parseFloat(token) / 100 : Number.parseFloat(token);
|
|
3770
|
+
return !Number.isNaN(a) && a >= MIN_ALPHA;
|
|
3771
|
+
}
|
|
3772
|
+
function toHex(value) {
|
|
3773
|
+
const v = value.trim().toLowerCase();
|
|
3774
|
+
const hex = /^#([0-9a-f]{3,8})$/.exec(v);
|
|
3775
|
+
if (hex) {
|
|
3776
|
+
const d = hex[1];
|
|
3777
|
+
if (d.length === 3 || d.length === 4) {
|
|
3778
|
+
if (d.length === 4 && !alphaOk(String(Number.parseInt(d[3] + d[3], 16) / 255))) return null;
|
|
3779
|
+
return `#${d[0]}${d[0]}${d[1]}${d[1]}${d[2]}${d[2]}`;
|
|
3780
|
+
}
|
|
3781
|
+
if (d.length === 6) return `#${d}`;
|
|
3782
|
+
if (d.length === 8) {
|
|
3783
|
+
if (!alphaOk(String(Number.parseInt(d.slice(6, 8), 16) / 255))) return null;
|
|
3784
|
+
return `#${d.slice(0, 6)}`;
|
|
3785
|
+
}
|
|
3786
|
+
return null;
|
|
3787
|
+
}
|
|
3788
|
+
const rgb = RGB_RE.exec(v);
|
|
3789
|
+
if (rgb) {
|
|
3790
|
+
if (!alphaOk(rgb[4])) return null;
|
|
3791
|
+
return rgbHex(scalar(rgb[1], 255), scalar(rgb[2], 255), scalar(rgb[3], 255));
|
|
3792
|
+
}
|
|
3793
|
+
const hsl = HSL_RE.exec(v);
|
|
3794
|
+
if (hsl) {
|
|
3795
|
+
if (!alphaOk(hsl[4])) return null;
|
|
3796
|
+
return hslHex(Number.parseFloat(hsl[1]), scalar(hsl[2], 1), scalar(hsl[3], 1));
|
|
3797
|
+
}
|
|
3798
|
+
const oklch = OKLCH_RE.exec(v);
|
|
3799
|
+
if (oklch) {
|
|
3800
|
+
if (!alphaOk(oklch[4])) return null;
|
|
3801
|
+
return oklchHex(scalar(oklch[1], 1), Number.parseFloat(oklch[2]), Number.parseFloat(oklch[3]));
|
|
3802
|
+
}
|
|
3803
|
+
return null;
|
|
3804
|
+
}
|
|
3805
|
+
function hslHex(h, s, l) {
|
|
3806
|
+
const c = (1 - Math.abs(2 * l - 1)) * s;
|
|
3807
|
+
const hp = (h % 360 + 360) % 360 / 60;
|
|
3808
|
+
const x = c * (1 - Math.abs(hp % 2 - 1));
|
|
3809
|
+
const [r, g, b] = hp < 1 ? [c, x, 0] : hp < 2 ? [x, c, 0] : hp < 3 ? [0, c, x] : hp < 4 ? [0, x, c] : hp < 5 ? [x, 0, c] : [c, 0, x];
|
|
3810
|
+
const m = l - c / 2;
|
|
3811
|
+
return rgbHex((r + m) * 255, (g + m) * 255, (b + m) * 255);
|
|
3812
|
+
}
|
|
3813
|
+
function oklchHex(L, C, hDeg) {
|
|
3814
|
+
const h = hDeg * Math.PI / 180;
|
|
3815
|
+
const a = C * Math.cos(h);
|
|
3816
|
+
const bb = C * Math.sin(h);
|
|
3817
|
+
const l_ = L + 0.3963377774 * a + 0.2158037573 * bb;
|
|
3818
|
+
const m_ = L - 0.1055613458 * a - 0.0638541728 * bb;
|
|
3819
|
+
const s_ = L - 0.0894841775 * a - 1.291485548 * bb;
|
|
3820
|
+
const l = l_ ** 3;
|
|
3821
|
+
const m = m_ ** 3;
|
|
3822
|
+
const s = s_ ** 3;
|
|
3823
|
+
const lr = 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s;
|
|
3824
|
+
const lg = -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s;
|
|
3825
|
+
const lb = -0.0041960863 * l - 0.7034186147 * m + 1.707614701 * s;
|
|
3826
|
+
const gamma = (u) => u <= 31308e-7 ? 12.92 * u : 1.055 * Math.abs(u) ** (1 / 2.4) - 0.055;
|
|
3827
|
+
return rgbHex(gamma(lr) * 255, gamma(lg) * 255, gamma(lb) * 255);
|
|
3828
|
+
}
|
|
3829
|
+
var DECL_RE = /(--[\w-]+|[\w-]+)\s*:\s*([^;{}]*?(?:#[0-9a-fA-F]{3,8}|rgba?\([^)]*\)|hsla?\([^)]*\)|oklch\([^)]*\))[^;{}]*)/g;
|
|
3830
|
+
var TOKEN_RE = /#[0-9a-fA-F]{3,8}\b|rgba?\([^)]*\)|hsla?\([^)]*\)|oklch\([^)]*\)/g;
|
|
3831
|
+
function extractColors(css) {
|
|
3832
|
+
const out = [];
|
|
3833
|
+
const blocks = css.split("}");
|
|
3834
|
+
for (const block of blocks) {
|
|
3835
|
+
const brace = block.lastIndexOf("{");
|
|
3836
|
+
const context = brace >= 0 ? block.slice(Math.max(0, brace - 200), brace) : "";
|
|
3837
|
+
const body = brace >= 0 ? block.slice(brace + 1) : block;
|
|
3838
|
+
DECL_RE.lastIndex = 0;
|
|
3839
|
+
let decl = DECL_RE.exec(body);
|
|
3840
|
+
while (decl !== null) {
|
|
3841
|
+
const property = decl[1];
|
|
3842
|
+
TOKEN_RE.lastIndex = 0;
|
|
3843
|
+
let token = TOKEN_RE.exec(decl[2]);
|
|
3844
|
+
while (token !== null) {
|
|
3845
|
+
const hex = toHex(token[0]);
|
|
3846
|
+
if (hex) out.push({ hex: hex.toLowerCase(), property, context });
|
|
3847
|
+
token = TOKEN_RE.exec(decl[2]);
|
|
3848
|
+
}
|
|
3849
|
+
decl = DECL_RE.exec(body);
|
|
3850
|
+
}
|
|
3851
|
+
}
|
|
3852
|
+
return out;
|
|
3853
|
+
}
|
|
3854
|
+
|
|
3855
|
+
// ../brand-spec/src/colors.ts
|
|
3496
3856
|
function hexToHsl(hex) {
|
|
3497
|
-
const r = parseInt(hex.slice(1, 3), 16) / 255;
|
|
3498
|
-
const g = parseInt(hex.slice(3, 5), 16) / 255;
|
|
3499
|
-
const b = parseInt(hex.slice(5, 7), 16) / 255;
|
|
3500
|
-
const max = Math.max(r, g, b)
|
|
3857
|
+
const r = Number.parseInt(hex.slice(1, 3), 16) / 255;
|
|
3858
|
+
const g = Number.parseInt(hex.slice(3, 5), 16) / 255;
|
|
3859
|
+
const b = Number.parseInt(hex.slice(5, 7), 16) / 255;
|
|
3860
|
+
const max = Math.max(r, g, b);
|
|
3861
|
+
const min = Math.min(r, g, b);
|
|
3501
3862
|
const l = (max + min) / 2;
|
|
3502
3863
|
if (max === min) return { h: 0, s: 0, l };
|
|
3503
3864
|
const d = max - min;
|
|
@@ -3508,15 +3869,111 @@ function hexToHsl(hex) {
|
|
|
3508
3869
|
else h = ((r - g) / d + 4) / 6;
|
|
3509
3870
|
return { h: h * 360, s, l };
|
|
3510
3871
|
}
|
|
3511
|
-
|
|
3872
|
+
var BRANDISH_VAR = /--(?:[\w-]*)(brand|primary|accent|secondary|theme|main)/i;
|
|
3873
|
+
var FRAMEWORK_VAR = /^--(tw-|[\w-]*-(?:50|\d{3})$)/i;
|
|
3874
|
+
var UTILITY_CLASS = /\.(bg|text|border|fill|stroke|ring|from|via|to|shadow|outline|divide|accent|caret|decoration|placeholder)-[a-z]+-\d{2,3}\b/i;
|
|
3875
|
+
var INTERACTIVE = /(btn|button|cta|primary|brand|badge|header|nav\b|link|hero)/i;
|
|
3876
|
+
function weightOf(hit) {
|
|
3877
|
+
if (FRAMEWORK_VAR.test(hit.property) || UTILITY_CLASS.test(hit.context)) return 0;
|
|
3878
|
+
if (hit.property.startsWith("--")) return BRANDISH_VAR.test(hit.property) ? 6 : 2;
|
|
3879
|
+
if (INTERACTIVE.test(hit.context) || INTERACTIVE.test(hit.property)) return 3;
|
|
3880
|
+
return 1;
|
|
3881
|
+
}
|
|
3882
|
+
function liveClassTokens($) {
|
|
3883
|
+
const tokens = /* @__PURE__ */ new Set();
|
|
3884
|
+
for (const sel of ["html", "body"]) {
|
|
3885
|
+
for (const cls of ($(sel).first().attr("class") ?? "").split(/\s+/)) {
|
|
3886
|
+
if (cls) tokens.add(cls.toLowerCase());
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
return [...tokens];
|
|
3890
|
+
}
|
|
3891
|
+
var familyOf = (cls) => cls.split("-")[0];
|
|
3892
|
+
function selectorIsLive(selector, live) {
|
|
3893
|
+
const sel = selector.toLowerCase();
|
|
3894
|
+
if (sel.includes(":root")) return true;
|
|
3895
|
+
const families = new Set(live.map(familyOf));
|
|
3896
|
+
return sel.split(",").some((branch) => {
|
|
3897
|
+
const positive = branch.replace(/:not\([^)]*\)/g, "").match(/\.[a-z0-9_-]+/g);
|
|
3898
|
+
if (!positive) return true;
|
|
3899
|
+
const leading = positive[0].slice(1);
|
|
3900
|
+
if (live.includes(leading)) return true;
|
|
3901
|
+
return !families.has(familyOf(leading));
|
|
3902
|
+
});
|
|
3903
|
+
}
|
|
3904
|
+
function roleOfProperty(property) {
|
|
3905
|
+
if (!property.startsWith("--")) return null;
|
|
3906
|
+
const name = property.slice(2).toLowerCase().replace(/^(brand|color|colour|theme|sc|c|ui)-/, "");
|
|
3907
|
+
const base = /^(primary|secondary|accent|dark|light)(?:-(\d+|dark|light|hover|active|soft|muted|contrast|foreground|fg|bg))?$/.exec(
|
|
3908
|
+
name
|
|
3909
|
+
);
|
|
3910
|
+
if (!base) return null;
|
|
3911
|
+
return { role: base[1], variant: Boolean(base[2]) };
|
|
3912
|
+
}
|
|
3913
|
+
function declaredPalette(hits, live) {
|
|
3914
|
+
const best = /* @__PURE__ */ new Map();
|
|
3915
|
+
for (const hit of hits) {
|
|
3916
|
+
const named = roleOfProperty(hit.property);
|
|
3917
|
+
if (!named || FRAMEWORK_VAR.test(hit.property) || !selectorIsLive(hit.context, live)) continue;
|
|
3918
|
+
const classes = hit.context.toLowerCase().match(/\.[a-z0-9_-]+/g) ?? [];
|
|
3919
|
+
const onLiveRoot = classes.some((c) => live.includes(c.slice(1)));
|
|
3920
|
+
const rank = (onLiveRoot ? 4 : classes.length > 0 ? 2 : 0) + (named.variant ? 0 : 1);
|
|
3921
|
+
const seen = best.get(named.role);
|
|
3922
|
+
if (seen && seen.rank > rank) continue;
|
|
3923
|
+
best.set(named.role, { hex: hit.hex, rank });
|
|
3924
|
+
}
|
|
3925
|
+
return {
|
|
3926
|
+
primary: best.get("primary")?.hex,
|
|
3927
|
+
secondary: best.get("secondary")?.hex,
|
|
3928
|
+
accent: best.get("accent")?.hex,
|
|
3929
|
+
dark: best.get("dark")?.hex,
|
|
3930
|
+
light: best.get("light")?.hex
|
|
3931
|
+
};
|
|
3932
|
+
}
|
|
3933
|
+
function collectColors(sources, live = []) {
|
|
3512
3934
|
const counts = /* @__PURE__ */ new Map();
|
|
3513
|
-
for (const
|
|
3514
|
-
|
|
3935
|
+
for (const { css, weightScale = 1 } of sources) {
|
|
3936
|
+
for (const hit of extractColors(css)) {
|
|
3937
|
+
if (live.length > 0 && !selectorIsLive(hit.context, live)) continue;
|
|
3938
|
+
const weight = weightOf(hit) * weightScale;
|
|
3939
|
+
if (weight <= 0) continue;
|
|
3940
|
+
counts.set(hit.hex, (counts.get(hit.hex) ?? 0) + weight);
|
|
3941
|
+
}
|
|
3942
|
+
}
|
|
3943
|
+
return [...counts.entries()].map(([hex, weight]) => ({ hex, weight }));
|
|
3944
|
+
}
|
|
3945
|
+
function dedupeNearby(hits) {
|
|
3946
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
3947
|
+
for (const hit of [...hits].sort((a, b) => b.weight - a.weight)) {
|
|
3948
|
+
const { h, s, l } = hexToHsl(hit.hex);
|
|
3949
|
+
const key = isNeutral(hit.hex) ? `grey:${Math.round(l * 8)}` : `${Math.round(h / 12)}:${Math.round(s * 8)}:${Math.round(l * 8)}`;
|
|
3950
|
+
const seen = buckets.get(key);
|
|
3951
|
+
if (seen) seen.weight += hit.weight;
|
|
3952
|
+
else buckets.set(key, { ...hit });
|
|
3953
|
+
}
|
|
3954
|
+
return [...buckets.values()].sort((a, b) => b.weight - a.weight);
|
|
3955
|
+
}
|
|
3956
|
+
function dropSingletons(hits) {
|
|
3957
|
+
if (hits.length < 12) return [...hits];
|
|
3958
|
+
const kept = hits.filter((h) => h.weight > 1);
|
|
3959
|
+
return kept.length > 0 ? kept : [...hits];
|
|
3960
|
+
}
|
|
3961
|
+
function chromaOf(hex) {
|
|
3962
|
+
const r = Number.parseInt(hex.slice(1, 3), 16) / 255;
|
|
3963
|
+
const g = Number.parseInt(hex.slice(3, 5), 16) / 255;
|
|
3964
|
+
const b = Number.parseInt(hex.slice(5, 7), 16) / 255;
|
|
3965
|
+
return Math.max(r, g, b) - Math.min(r, g, b);
|
|
3966
|
+
}
|
|
3967
|
+
function isNeutral(hex) {
|
|
3968
|
+
const { l } = hexToHsl(hex);
|
|
3969
|
+
return chromaOf(hex) < 0.1 || l < 0.06 || l > 0.96;
|
|
3970
|
+
}
|
|
3971
|
+
function pickPalette(hits) {
|
|
3972
|
+
const sorted = [...hits].sort((a, b) => b.weight - a.weight).map((h) => h.hex);
|
|
3515
3973
|
const saturated = [];
|
|
3516
3974
|
const neutrals = [];
|
|
3517
3975
|
for (const c of sorted) {
|
|
3518
|
-
|
|
3519
|
-
if (s < 0.12 || l < 0.06 || l > 0.96) neutrals.push(c);
|
|
3976
|
+
if (isNeutral(c)) neutrals.push(c);
|
|
3520
3977
|
else saturated.push(c);
|
|
3521
3978
|
}
|
|
3522
3979
|
return {
|
|
@@ -3526,91 +3983,650 @@ function pickPalette(colors) {
|
|
|
3526
3983
|
neutrals: neutrals.slice(0, 2)
|
|
3527
3984
|
};
|
|
3528
3985
|
}
|
|
3529
|
-
|
|
3530
|
-
const
|
|
3531
|
-
const
|
|
3532
|
-
const
|
|
3533
|
-
const
|
|
3534
|
-
const
|
|
3535
|
-
|
|
3986
|
+
function paletteFrom(sources, live = []) {
|
|
3987
|
+
const hits = sources.flatMap((s) => extractColors(s.css));
|
|
3988
|
+
const declared = declaredPalette(hits, live);
|
|
3989
|
+
const counted = pickPalette(dropSingletons(dedupeNearby(collectColors(sources, live))));
|
|
3990
|
+
const taken = /* @__PURE__ */ new Set();
|
|
3991
|
+
const take = (hex) => {
|
|
3992
|
+
if (!hex || taken.has(hex)) return void 0;
|
|
3993
|
+
taken.add(hex);
|
|
3994
|
+
return hex;
|
|
3995
|
+
};
|
|
3996
|
+
const primary = take(declared.primary) ?? take(counted.primary);
|
|
3997
|
+
const secondary = take(declared.secondary) ?? take(counted.secondary);
|
|
3998
|
+
const named = Boolean(declared.primary || declared.secondary);
|
|
3999
|
+
const accent = (named ? [declared.accent, declared.dark, declared.light] : counted.accent).map(take).filter((h) => Boolean(h)).slice(0, 2);
|
|
4000
|
+
const neutrals = counted.neutrals.filter((h) => !taken.has(h)).slice(0, 2);
|
|
4001
|
+
return { primary, secondary, accent, neutrals };
|
|
4002
|
+
}
|
|
4003
|
+
|
|
4004
|
+
// ../brand-spec/src/logoCandidates.ts
|
|
4005
|
+
var BASE = {
|
|
4006
|
+
"json-ld": 170,
|
|
4007
|
+
manifest: 85,
|
|
4008
|
+
"apple-touch-icon": 80,
|
|
4009
|
+
"header-img": 75,
|
|
4010
|
+
"header-svg": 60,
|
|
4011
|
+
"link-icon": 45,
|
|
4012
|
+
"og-image": 10
|
|
4013
|
+
};
|
|
4014
|
+
var LOGOISH = /(^|[^a-z])(logo|wordmark|brandmark|lockup|masthead)([^a-z]|$)/i;
|
|
4015
|
+
var NOT_OURS = /(client|customer|partner|sponsor|press|award|integrat|collab|affiliat|featured-in|trusted-by|logos?-(wall|grid|cloud|strip)|testimonial|marquee|as-seen)/i;
|
|
4016
|
+
var SOCIAL = /(facebook|instagram|twitter|linkedin|youtube|tiktok|pinterest|threads|whatsapp|x-logo|social)/i;
|
|
4017
|
+
var FLAGGISH = /(\bflags?\b|locale|country|region|currency|\blang(uage)?\b)/i;
|
|
4018
|
+
var TRACKER = /(pixel|1x1|spacer|blank|beacon|analytics|doubleclick|facebook\.com\/tr)/i;
|
|
4019
|
+
var PHOTOISH = /(hero|banner|cover|photo|screenshot|slide|\bbg\b|background)/i;
|
|
4020
|
+
var DARKISH = /(dark|inverse|inverted|white|light-on)/i;
|
|
4021
|
+
var HEADER_SCOPE = 'header, nav, [role="banner"], .site-header, .navbar, .masthead, .topbar';
|
|
4022
|
+
var IMG_HORIZON = 40;
|
|
4023
|
+
function absolute(href, base) {
|
|
4024
|
+
if (!href) return null;
|
|
4025
|
+
const raw = href.trim();
|
|
4026
|
+
if (raw === "") return null;
|
|
4027
|
+
if (raw.startsWith("data:"))
|
|
4028
|
+
return raw.length <= 262144 && /^data:image\/(svg\+xml|png|jpeg|webp)/i.test(raw) ? raw : null;
|
|
3536
4029
|
try {
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
4030
|
+
const u = new URL(raw, base);
|
|
4031
|
+
return u.protocol === "http:" || u.protocol === "https:" ? u.toString() : null;
|
|
4032
|
+
} catch {
|
|
4033
|
+
return null;
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
function largestFromSrcset(srcset) {
|
|
4037
|
+
const entries = srcset.split(",").map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
4038
|
+
const [url, descriptor = ""] = part.split(/\s+/);
|
|
4039
|
+
const w = /^(\d+)w$/.exec(descriptor);
|
|
4040
|
+
const x = /^([\d.]+)x$/.exec(descriptor);
|
|
4041
|
+
return { url, rank: w ? Number(w[1]) : x ? Number(x[1]) * 1e3 : 1 };
|
|
4042
|
+
});
|
|
4043
|
+
if (entries.length === 0) return null;
|
|
4044
|
+
return entries.sort((a, b) => b.rank - a.rank)[0].url;
|
|
4045
|
+
}
|
|
4046
|
+
function declaredEdgeOf(sizes, w, h) {
|
|
4047
|
+
const fromSizes = sizes ? Math.max(...(sizes.match(/\d+/g) ?? ["0"]).map(Number)) : 0;
|
|
4048
|
+
const fromAttrs = Math.max(Number(0) || 0, Number(0) || 0);
|
|
4049
|
+
const edge = Math.max(fromSizes, fromAttrs);
|
|
4050
|
+
return edge > 0 ? edge : null;
|
|
4051
|
+
}
|
|
4052
|
+
function layoutTerm(w, h, why) {
|
|
4053
|
+
const width = Number(w ?? 0) || 0;
|
|
4054
|
+
const height = Number(h ?? 0) || 0;
|
|
4055
|
+
if (width > 0 && height > 0 && width <= 32 && height <= 32) {
|
|
4056
|
+
why.push("icon-shaped");
|
|
4057
|
+
return -20;
|
|
4058
|
+
}
|
|
4059
|
+
return 0;
|
|
4060
|
+
}
|
|
4061
|
+
function sizeTerm(edge, why) {
|
|
4062
|
+
if (edge === null) return 0;
|
|
4063
|
+
if (edge >= 512) {
|
|
4064
|
+
why.push("large");
|
|
4065
|
+
return 15;
|
|
4066
|
+
}
|
|
4067
|
+
if (edge >= 256) {
|
|
4068
|
+
why.push("big enough");
|
|
4069
|
+
return 8;
|
|
4070
|
+
}
|
|
4071
|
+
if (edge <= 64) {
|
|
4072
|
+
why.push("favicon-sized");
|
|
4073
|
+
return -25;
|
|
4074
|
+
}
|
|
4075
|
+
return 0;
|
|
4076
|
+
}
|
|
4077
|
+
function logoCandidates($, base, manifest) {
|
|
4078
|
+
const out = [];
|
|
4079
|
+
const host = base.hostname.replace(/^www\./, "").split(".")[0];
|
|
4080
|
+
const push = (c) => {
|
|
4081
|
+
if (c.score <= 0) return;
|
|
4082
|
+
if (c.url && out.some((o) => o.url === c.url)) return;
|
|
4083
|
+
out.push(c);
|
|
4084
|
+
};
|
|
4085
|
+
for (const node of $('script[type="application/ld+json"]').toArray()) {
|
|
4086
|
+
for (const href of jsonLdLogos($(node).text())) {
|
|
4087
|
+
const url = absolute(href, base);
|
|
4088
|
+
if (url)
|
|
4089
|
+
push({
|
|
4090
|
+
url,
|
|
4091
|
+
source: "json-ld",
|
|
4092
|
+
score: BASE["json-ld"],
|
|
4093
|
+
declaredEdge: null,
|
|
4094
|
+
role: "primary",
|
|
4095
|
+
why: ["declared as the organisation logo"]
|
|
4096
|
+
});
|
|
4097
|
+
}
|
|
4098
|
+
}
|
|
4099
|
+
for (const icon of manifest?.icons ?? []) {
|
|
4100
|
+
const url = absolute(icon.src, base);
|
|
4101
|
+
const edge = declaredEdgeOf(icon.sizes);
|
|
4102
|
+
if (!url || (edge ?? 0) < 192) continue;
|
|
4103
|
+
const why = ["from the web app manifest"];
|
|
4104
|
+
push({
|
|
4105
|
+
url,
|
|
4106
|
+
source: "manifest",
|
|
4107
|
+
score: BASE.manifest + sizeTerm(edge, why),
|
|
4108
|
+
declaredEdge: edge,
|
|
4109
|
+
role: "primary",
|
|
4110
|
+
why
|
|
3541
4111
|
});
|
|
3542
|
-
}
|
|
3543
|
-
|
|
3544
|
-
|
|
4112
|
+
}
|
|
4113
|
+
for (const el of $('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').toArray()) {
|
|
4114
|
+
const url = absolute($(el).attr("href"), base);
|
|
4115
|
+
const edge = declaredEdgeOf($(el).attr("sizes"));
|
|
4116
|
+
const why = ["a touch icon"];
|
|
4117
|
+
if (url)
|
|
4118
|
+
push({
|
|
4119
|
+
url,
|
|
4120
|
+
source: "apple-touch-icon",
|
|
4121
|
+
score: BASE["apple-touch-icon"] + sizeTerm(edge, why),
|
|
4122
|
+
declaredEdge: edge,
|
|
4123
|
+
role: "primary",
|
|
4124
|
+
why
|
|
4125
|
+
});
|
|
4126
|
+
}
|
|
4127
|
+
const imgs = $("img").toArray();
|
|
4128
|
+
imgs.forEach((el, index) => {
|
|
4129
|
+
const node = $(el);
|
|
4130
|
+
const inHeader = node.closest(HEADER_SCOPE).length > 0;
|
|
4131
|
+
if (!inHeader && index > IMG_HORIZON) return;
|
|
4132
|
+
const src = node.attr("src") ?? node.attr("data-src") ?? node.attr("data-lazy-src") ?? largestFromSrcset(node.attr("srcset") ?? node.attr("data-srcset") ?? "") ?? void 0;
|
|
4133
|
+
const url = absolute(src, base);
|
|
4134
|
+
if (!url) return;
|
|
4135
|
+
const text = [url, node.attr("alt") ?? "", node.attr("class") ?? "", node.attr("id") ?? ""].join(" ");
|
|
4136
|
+
const ancestry = node.parents().slice(0, 6).map((_i, p) => `${$(p).attr("class") ?? ""} ${$(p).attr("id") ?? ""}`).get().join(" ");
|
|
4137
|
+
const why = [];
|
|
4138
|
+
let score = inHeader ? BASE["header-img"] : BASE["header-img"] - 20;
|
|
4139
|
+
if (inHeader) why.push("in the header");
|
|
4140
|
+
if (index <= 2) {
|
|
4141
|
+
why.push("first image on the page");
|
|
4142
|
+
score += 20;
|
|
4143
|
+
} else if (index <= 6) {
|
|
4144
|
+
score += 6;
|
|
4145
|
+
}
|
|
4146
|
+
score += layoutTerm(node.attr("width"), node.attr("height"), why);
|
|
4147
|
+
score += semanticTerms(text, ancestry, host, why);
|
|
4148
|
+
if (node.parent().children("img").length >= 3) {
|
|
4149
|
+
why.push("one of a row of logos");
|
|
4150
|
+
score -= 50;
|
|
4151
|
+
}
|
|
4152
|
+
push({
|
|
4153
|
+
url,
|
|
4154
|
+
source: "header-img",
|
|
4155
|
+
score,
|
|
4156
|
+
declaredEdge: null,
|
|
4157
|
+
role: "primary",
|
|
4158
|
+
...DARKISH.test(text) ? { background: "dark" } : {},
|
|
4159
|
+
why
|
|
4160
|
+
});
|
|
4161
|
+
});
|
|
4162
|
+
for (const el of $(`${HEADER_SCOPE}`).find("svg").toArray().slice(0, 4)) {
|
|
4163
|
+
const node = $(el);
|
|
4164
|
+
const text = [node.attr("class") ?? "", node.attr("id") ?? "", node.attr("aria-label") ?? ""].join(" ");
|
|
4165
|
+
const why = ["drawn in the header"];
|
|
4166
|
+
const score = BASE["header-svg"] + semanticTerms(text, "", host, why);
|
|
4167
|
+
const markup = $.html(el);
|
|
4168
|
+
if (markup) push({ url: null, svg: markup, source: "header-svg", score, declaredEdge: null, role: "primary", why });
|
|
4169
|
+
}
|
|
4170
|
+
for (const el of $('link[rel~="icon"]').toArray()) {
|
|
4171
|
+
const url = absolute($(el).attr("href"), base);
|
|
4172
|
+
const edge = declaredEdgeOf($(el).attr("sizes"));
|
|
4173
|
+
const why = ["the site icon"];
|
|
4174
|
+
if (url)
|
|
4175
|
+
push({
|
|
4176
|
+
url,
|
|
4177
|
+
source: "link-icon",
|
|
4178
|
+
score: BASE["link-icon"] + sizeTerm(edge, why),
|
|
4179
|
+
declaredEdge: edge,
|
|
4180
|
+
role: "primary",
|
|
4181
|
+
why
|
|
4182
|
+
});
|
|
4183
|
+
}
|
|
4184
|
+
const og = absolute($('meta[property="og:image"]').attr("content"), base);
|
|
4185
|
+
if (og)
|
|
4186
|
+
push({
|
|
4187
|
+
url: og,
|
|
4188
|
+
source: "og-image",
|
|
4189
|
+
score: BASE["og-image"],
|
|
4190
|
+
declaredEdge: null,
|
|
4191
|
+
role: "alternate",
|
|
4192
|
+
why: ["the social share image"]
|
|
4193
|
+
});
|
|
4194
|
+
return out.sort((a, b) => b.score - a.score);
|
|
4195
|
+
}
|
|
4196
|
+
function pathOf(text) {
|
|
4197
|
+
return text.replace(/https?:\/\/[^/\s]+/gi, "");
|
|
4198
|
+
}
|
|
4199
|
+
function semanticTerms(text, ancestry, host, why) {
|
|
4200
|
+
let score = 0;
|
|
4201
|
+
if (LOGOISH.test(text)) {
|
|
4202
|
+
why.push("called a logo");
|
|
4203
|
+
score += 40;
|
|
4204
|
+
}
|
|
4205
|
+
if (host.length >= 3 && pathOf(text).toLowerCase().includes(host.toLowerCase())) {
|
|
4206
|
+
why.push("named after the site");
|
|
4207
|
+
score += 6;
|
|
4208
|
+
}
|
|
4209
|
+
if (/\.svg(\?|$)/i.test(text)) score += 8;
|
|
4210
|
+
if (/\.ico(\?|$)/i.test(text)) score -= 15;
|
|
4211
|
+
if (/\.jpe?g(\?|$)/i.test(text)) score -= 10;
|
|
4212
|
+
if (NOT_OURS.test(ancestry) || NOT_OURS.test(text)) {
|
|
4213
|
+
why.push("inside a section of other companies");
|
|
4214
|
+
score -= 40;
|
|
4215
|
+
}
|
|
4216
|
+
if (SOCIAL.test(text)) {
|
|
4217
|
+
why.push("a social icon");
|
|
4218
|
+
score -= 60;
|
|
4219
|
+
}
|
|
4220
|
+
if (FLAGGISH.test(text)) {
|
|
4221
|
+
why.push("a flag or a region switcher");
|
|
4222
|
+
score -= 60;
|
|
4223
|
+
}
|
|
4224
|
+
if (TRACKER.test(text)) {
|
|
4225
|
+
why.push("a tracking pixel");
|
|
4226
|
+
score -= 80;
|
|
4227
|
+
}
|
|
4228
|
+
if (PHOTOISH.test(text)) {
|
|
4229
|
+
why.push("photography");
|
|
4230
|
+
score -= 30;
|
|
4231
|
+
}
|
|
4232
|
+
if (DARKISH.test(text)) score -= 15;
|
|
4233
|
+
return score;
|
|
4234
|
+
}
|
|
4235
|
+
function jsonLdLogos(text) {
|
|
4236
|
+
let data;
|
|
4237
|
+
try {
|
|
4238
|
+
data = JSON.parse(text);
|
|
4239
|
+
} catch {
|
|
4240
|
+
return [];
|
|
4241
|
+
}
|
|
4242
|
+
const found = [];
|
|
4243
|
+
const walk = (node, depth) => {
|
|
4244
|
+
if (depth > 6 || node === null || typeof node !== "object") return;
|
|
4245
|
+
if (Array.isArray(node)) {
|
|
4246
|
+
for (const item of node) walk(item, depth + 1);
|
|
4247
|
+
return;
|
|
3545
4248
|
}
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
4249
|
+
const obj = node;
|
|
4250
|
+
const logo = obj.logo;
|
|
4251
|
+
if (typeof logo === "string") found.push(logo);
|
|
4252
|
+
else if (logo && typeof logo === "object") {
|
|
4253
|
+
const url = logo.url;
|
|
4254
|
+
if (typeof url === "string") found.push(url);
|
|
4255
|
+
}
|
|
4256
|
+
for (const value of Object.values(obj)) walk(value, depth + 1);
|
|
4257
|
+
};
|
|
4258
|
+
walk(data, 0);
|
|
4259
|
+
return found;
|
|
4260
|
+
}
|
|
4261
|
+
function svgAsMark(markup, longEdge = 1024) {
|
|
4262
|
+
if (/<script|<foreignObject|(?:xlink:)?href\s*=\s*["']https?:/i.test(markup)) return null;
|
|
4263
|
+
const viewBox = /viewBox\s*=\s*["']\s*[-\d.]+[\s,]+[-\d.]+[\s,]+([\d.]+)[\s,]+([\d.]+)\s*["']/i.exec(markup);
|
|
4264
|
+
const width = /\bwidth\s*=\s*["']([\d.]+)/i.exec(markup);
|
|
4265
|
+
const height = /\bheight\s*=\s*["']([\d.]+)/i.exec(markup);
|
|
4266
|
+
const w = viewBox ? Number(viewBox[1]) : Number(width?.[1] ?? 0);
|
|
4267
|
+
const h = viewBox ? Number(viewBox[2]) : Number(height?.[1] ?? 0);
|
|
4268
|
+
if (!(w > 0 && h > 0)) return null;
|
|
4269
|
+
const scale = longEdge / Math.max(w, h);
|
|
4270
|
+
let out = markup.replace(/\s\bwidth\s*=\s*["'][^"']*["']/i, "").replace(/\s\bheight\s*=\s*["'][^"']*["']/i, "").replace(/<svg\b/i, `<svg width="${Math.round(w * scale)}" height="${Math.round(h * scale)}"`);
|
|
4271
|
+
if (!/xmlns\s*=/.test(out)) out = out.replace(/<svg\b/i, '<svg xmlns="http://www.w3.org/2000/svg"');
|
|
4272
|
+
return out;
|
|
4273
|
+
}
|
|
4274
|
+
|
|
4275
|
+
// ../brand-spec/src/scrapeError.ts
|
|
4276
|
+
var ScrapeError = class extends Error {
|
|
4277
|
+
constructor(code, message, statusCode = 502) {
|
|
4278
|
+
super(message);
|
|
4279
|
+
this.code = code;
|
|
4280
|
+
this.statusCode = statusCode;
|
|
4281
|
+
}
|
|
4282
|
+
code;
|
|
4283
|
+
statusCode;
|
|
4284
|
+
name = "ScrapeError";
|
|
4285
|
+
};
|
|
4286
|
+
function urlRefusal(reason, message) {
|
|
4287
|
+
return new ScrapeError(`url_${reason}`, message, 400);
|
|
4288
|
+
}
|
|
4289
|
+
|
|
4290
|
+
// ../brand-spec/src/safeFetch.ts
|
|
4291
|
+
var DEFAULTS = {
|
|
4292
|
+
budgetMs: 2e4,
|
|
4293
|
+
requestMs: 8e3,
|
|
4294
|
+
maxRedirects: 3,
|
|
4295
|
+
maxHtmlBytes: 3e6,
|
|
4296
|
+
maxCssBytes: 1e6,
|
|
4297
|
+
maxAssetBytes: 5e6
|
|
4298
|
+
};
|
|
4299
|
+
function isPrivateAddress(ip) {
|
|
4300
|
+
const kind = isIP(ip);
|
|
4301
|
+
if (kind === 4) return isPrivateV4(ip);
|
|
4302
|
+
if (kind === 6) return isPrivateV6(ip.toLowerCase());
|
|
4303
|
+
return true;
|
|
4304
|
+
}
|
|
4305
|
+
function isPrivateV4(ip) {
|
|
4306
|
+
const p = ip.split(".").map(Number);
|
|
4307
|
+
if (p.length !== 4 || p.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true;
|
|
4308
|
+
const [a, b] = p;
|
|
4309
|
+
if (a === 0 || a === 10 || a === 127) return true;
|
|
4310
|
+
if (a === 100 && b >= 64 && b <= 127) return true;
|
|
4311
|
+
if (a === 169 && b === 254) return true;
|
|
4312
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
4313
|
+
if (a === 192 && b === 168) return true;
|
|
4314
|
+
if (a === 192 && b === 0) return true;
|
|
4315
|
+
if (a === 192 && b === 88) return true;
|
|
4316
|
+
if (a === 198 && (b === 18 || b === 19 || b === 51)) return true;
|
|
4317
|
+
if (a === 203 && b === 0) return true;
|
|
4318
|
+
if (a >= 224) return true;
|
|
4319
|
+
return false;
|
|
4320
|
+
}
|
|
4321
|
+
function isPrivateV6(ip) {
|
|
4322
|
+
const bare = ip.replace(/^\[|\]$/g, "");
|
|
4323
|
+
if (bare === "::" || bare === "::1") return true;
|
|
4324
|
+
const mapped = /^(?:::ffff:|64:ff9b::)(\d{1,3}(?:\.\d{1,3}){3})$/.exec(bare);
|
|
4325
|
+
if (mapped) return isPrivateV4(mapped[1]);
|
|
4326
|
+
const head = bare.split(":")[0];
|
|
4327
|
+
if (/^f[cd]/.test(head)) return true;
|
|
4328
|
+
if (/^fe[89ab]/.test(head)) return true;
|
|
4329
|
+
if (/^ff/.test(head)) return true;
|
|
4330
|
+
if (head === "2002") return true;
|
|
4331
|
+
return false;
|
|
4332
|
+
}
|
|
4333
|
+
var PRIVATE_SUFFIX = /(^|\.)(local|internal|localhost|home\.arpa)$/i;
|
|
4334
|
+
function assertPublicHost(host, addresses, allow) {
|
|
4335
|
+
if (allow) return;
|
|
4336
|
+
const bare = host.replace(/^\[|\]$/g, "");
|
|
4337
|
+
if (isIP(bare)) {
|
|
4338
|
+
if (isPrivateAddress(bare)) throw blocked(host);
|
|
4339
|
+
return;
|
|
4340
|
+
}
|
|
4341
|
+
if (!bare.includes(".") || PRIVATE_SUFFIX.test(bare)) throw blocked(host);
|
|
4342
|
+
if (addresses.length === 0) throw new ScrapeError("unreachable", `Could not reach ${host}.`);
|
|
4343
|
+
if (addresses.some((a) => isPrivateAddress(a))) throw blocked(host);
|
|
4344
|
+
}
|
|
4345
|
+
var blocked = (host) => new ScrapeError("blocked_host", `${host} points at this computer or a private network, not a public website.`, 400);
|
|
4346
|
+
var LIMIT = {
|
|
4347
|
+
html: "maxHtmlBytes",
|
|
4348
|
+
css: "maxCssBytes",
|
|
4349
|
+
asset: "maxAssetBytes"
|
|
4350
|
+
};
|
|
4351
|
+
var TRUNCATES = { html: true, css: true, asset: false };
|
|
4352
|
+
function createGuardedFetch(opts = {}) {
|
|
4353
|
+
const o = { ...DEFAULTS, ...opts };
|
|
4354
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
4355
|
+
const resolve = opts.lookup ?? (async (host) => {
|
|
4356
|
+
if (isIP(host.replace(/^\[|\]$/g, ""))) return [host.replace(/^\[|\]$/g, "")];
|
|
4357
|
+
const all = await lookup(host, { all: true });
|
|
4358
|
+
return all.map((a) => a.address);
|
|
4359
|
+
});
|
|
4360
|
+
const deadline = Date.now() + o.budgetMs;
|
|
4361
|
+
const remaining = () => Math.max(0, deadline - Date.now());
|
|
4362
|
+
async function once(url, as) {
|
|
4363
|
+
let current = url;
|
|
4364
|
+
let hops = 0;
|
|
4365
|
+
let attempts = 0;
|
|
4366
|
+
for (; ; ) {
|
|
4367
|
+
const u = new URL(current);
|
|
4368
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
4369
|
+
throw new ScrapeError(
|
|
4370
|
+
"url_scheme",
|
|
4371
|
+
"Scenri reads web addresses only, the ones that start with http or https.",
|
|
4372
|
+
400
|
|
4373
|
+
);
|
|
4374
|
+
let addresses = [];
|
|
4375
|
+
if (!o.allowPrivateHosts) {
|
|
4376
|
+
try {
|
|
4377
|
+
addresses = await resolve(u.hostname);
|
|
4378
|
+
} catch {
|
|
4379
|
+
throw new ScrapeError("unreachable", `Could not reach ${u.hostname}.`);
|
|
4380
|
+
}
|
|
4381
|
+
}
|
|
4382
|
+
assertPublicHost(u.hostname, addresses, Boolean(o.allowPrivateHosts));
|
|
4383
|
+
const left = remaining();
|
|
4384
|
+
if (left <= 0) throw new ScrapeError("timeout", "Reading that site took too long.");
|
|
4385
|
+
const signal = AbortSignal.any([AbortSignal.timeout(Math.min(o.requestMs, left))]);
|
|
4386
|
+
let res;
|
|
4387
|
+
try {
|
|
4388
|
+
res = await fetchImpl(current, {
|
|
4389
|
+
redirect: "manual",
|
|
4390
|
+
headers: { "user-agent": USER_AGENT, accept: ACCEPT[as] },
|
|
4391
|
+
signal
|
|
4392
|
+
});
|
|
4393
|
+
} catch (err) {
|
|
4394
|
+
if (err instanceof ScrapeError) throw err;
|
|
4395
|
+
const aborted = err instanceof Error && (err.name === "AbortError" || err.name === "TimeoutError");
|
|
4396
|
+
throw aborted ? new ScrapeError("timeout", `${u.hostname} took too long to answer.`) : new ScrapeError("unreachable", `Could not reach ${u.hostname}.`);
|
|
4397
|
+
}
|
|
4398
|
+
if (res.status >= 300 && res.status < 400) {
|
|
4399
|
+
const location = res.headers.get("location");
|
|
4400
|
+
await res.body?.cancel().catch(() => {
|
|
4401
|
+
});
|
|
4402
|
+
if (!location) throw new ScrapeError("http_status", `${u.hostname} answered ${res.status} with nowhere to go.`);
|
|
4403
|
+
if (++hops > o.maxRedirects)
|
|
4404
|
+
throw new ScrapeError(
|
|
4405
|
+
"too_many_redirects",
|
|
4406
|
+
`${new URL(url).hostname} kept redirecting, so nothing was read.`
|
|
4407
|
+
);
|
|
4408
|
+
current = new URL(location, current).toString();
|
|
4409
|
+
continue;
|
|
4410
|
+
}
|
|
4411
|
+
if (!res.ok) {
|
|
4412
|
+
if (RETRYABLE.has(res.status) && attempts < 1 && remaining() > 2e3) {
|
|
4413
|
+
await res.body?.cancel().catch(() => {
|
|
4414
|
+
});
|
|
4415
|
+
await sleep3(700);
|
|
4416
|
+
attempts++;
|
|
4417
|
+
continue;
|
|
4418
|
+
}
|
|
4419
|
+
throw new ScrapeError("http_status", statusSentence(u.hostname, res.status));
|
|
4420
|
+
}
|
|
4421
|
+
const contentType = (res.headers.get("content-type") ?? "").toLowerCase();
|
|
4422
|
+
const cap2 = o[LIMIT[as]];
|
|
4423
|
+
const { bytes, truncated } = await readBounded(res, cap2, TRUNCATES[as]);
|
|
4424
|
+
assertKind(as, contentType, bytes, u.hostname);
|
|
4425
|
+
return {
|
|
4426
|
+
finalUrl: res.url || current,
|
|
4427
|
+
status: res.status,
|
|
4428
|
+
contentType,
|
|
4429
|
+
bytes,
|
|
4430
|
+
text: as === "asset" ? "" : decode(bytes, contentType),
|
|
4431
|
+
truncated
|
|
4432
|
+
};
|
|
4433
|
+
}
|
|
4434
|
+
}
|
|
4435
|
+
const guarded = ((url, as) => once(url, as));
|
|
4436
|
+
guarded.remaining = remaining;
|
|
4437
|
+
return guarded;
|
|
4438
|
+
}
|
|
4439
|
+
var USER_AGENT = "scenri/0.1 (+https://scenri.co)";
|
|
4440
|
+
var sleep3 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
4441
|
+
var RETRYABLE = /* @__PURE__ */ new Set([403, 408, 425, 429, 500, 502, 503, 504]);
|
|
4442
|
+
function statusSentence(host, status) {
|
|
4443
|
+
if (status === 401 || status === 403)
|
|
4444
|
+
return `${host} would not let Scenri read it. Some sites block anything that is not a person in a browser; you can still add the logo and colours by hand.`;
|
|
4445
|
+
if (status === 404) return `There is no page at that address on ${host}.`;
|
|
4446
|
+
if (status === 429) return `${host} asked Scenri to slow down. Try again in a minute.`;
|
|
4447
|
+
if (status >= 500) return `${host} had trouble answering. Try again in a moment.`;
|
|
4448
|
+
return `${host} answered ${status}, so there was nothing to read.`;
|
|
4449
|
+
}
|
|
4450
|
+
var ACCEPT = {
|
|
4451
|
+
html: "text/html,application/xhtml+xml",
|
|
4452
|
+
css: "text/css,*/*;q=0.1",
|
|
4453
|
+
asset: "image/*"
|
|
4454
|
+
};
|
|
4455
|
+
async function readBounded(res, cap2, truncate) {
|
|
4456
|
+
const declared = Number(res.headers.get("content-length") ?? "0");
|
|
4457
|
+
if (!truncate && declared > cap2) {
|
|
4458
|
+
await res.body?.cancel().catch(() => {
|
|
4459
|
+
});
|
|
4460
|
+
throw new ScrapeError("too_large", "That file is too large to read.");
|
|
4461
|
+
}
|
|
4462
|
+
const reader = res.body?.getReader();
|
|
4463
|
+
if (!reader) return { bytes: Buffer.from(await res.arrayBuffer()), truncated: false };
|
|
4464
|
+
const chunks = [];
|
|
4465
|
+
let total = 0;
|
|
4466
|
+
for (; ; ) {
|
|
4467
|
+
const { done, value } = await reader.read();
|
|
4468
|
+
if (done) break;
|
|
4469
|
+
chunks.push(Buffer.from(value));
|
|
4470
|
+
total += value.byteLength;
|
|
4471
|
+
if (total >= cap2) {
|
|
4472
|
+
await reader.cancel().catch(() => {
|
|
4473
|
+
});
|
|
4474
|
+
if (!truncate) throw new ScrapeError("too_large", "That file is too large to read.");
|
|
4475
|
+
return { bytes: Buffer.concat(chunks).subarray(0, cap2), truncated: true };
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
return { bytes: Buffer.concat(chunks), truncated: false };
|
|
4479
|
+
}
|
|
4480
|
+
var IMAGE_MAGIC = [
|
|
4481
|
+
["png", (b) => b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))],
|
|
4482
|
+
["jpeg", (b) => b[0] === 255 && b[1] === 216],
|
|
4483
|
+
["gif", (b) => b.subarray(0, 3).toString("latin1") === "GIF"],
|
|
4484
|
+
["webp", (b) => b.subarray(0, 4).toString("latin1") === "RIFF" && b.subarray(8, 12).toString("latin1") === "WEBP"],
|
|
4485
|
+
["ico", (b) => b[0] === 0 && b[1] === 0 && (b[2] === 1 || b[2] === 2)],
|
|
4486
|
+
["svg", (b) => /<svg[\s>]/i.test(b.subarray(0, 1024).toString("utf8"))]
|
|
4487
|
+
];
|
|
4488
|
+
function assertKind(as, contentType, bytes, host) {
|
|
4489
|
+
if (as === "html") {
|
|
4490
|
+
const htmlish = /text\/html|application\/xhtml/.test(contentType);
|
|
4491
|
+
const vague = contentType === "" || /^text\/plain|^application\/octet-stream/.test(contentType);
|
|
4492
|
+
const looksLikeMarkup = vague && bytes.subarray(0, 512).toString("utf8").trimStart().startsWith("<");
|
|
4493
|
+
if (!htmlish && !looksLikeMarkup)
|
|
4494
|
+
throw new ScrapeError("not_a_page", `${host} answered with a file, not a web page.`);
|
|
4495
|
+
return;
|
|
4496
|
+
}
|
|
4497
|
+
if (as === "asset") {
|
|
4498
|
+
if (contentType.startsWith("image/")) return;
|
|
4499
|
+
const vague = contentType === "" || /^application\/octet-stream/.test(contentType);
|
|
4500
|
+
if (vague || IMAGE_MAGIC.some(([, test]) => test(bytes))) return;
|
|
4501
|
+
throw new ScrapeError("not_a_page", "That logo was not an image.");
|
|
4502
|
+
}
|
|
4503
|
+
}
|
|
4504
|
+
function decode(bytes, contentType) {
|
|
4505
|
+
const declared = /charset=([\w-]+)/i.exec(contentType)?.[1];
|
|
4506
|
+
const sniffed = declared ?? /charset=["']?([\w-]+)/i.exec(bytes.subarray(0, 1024).toString("latin1"))?.[1];
|
|
4507
|
+
const label = (sniffed ?? "utf-8").toLowerCase();
|
|
4508
|
+
if (label === "utf-8" || label === "utf8") return bytes.toString("utf8");
|
|
4509
|
+
try {
|
|
4510
|
+
return new TextDecoder(label, { fatal: false }).decode(bytes);
|
|
4511
|
+
} catch {
|
|
4512
|
+
return bytes.toString("utf8");
|
|
4513
|
+
}
|
|
4514
|
+
}
|
|
4515
|
+
|
|
4516
|
+
// ../brand-spec/src/siteUrl.ts
|
|
4517
|
+
var INVISIBLE = /[---]/g;
|
|
4518
|
+
var SPACES = /[\t\n\r\f\v - ]/g;
|
|
4519
|
+
var SCHEMED = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
4520
|
+
var OPAQUE = /^(javascript|data|mailto|tel|sms|file|about|blob|chrome|chrome-extension|view-source):/i;
|
|
4521
|
+
var HTTPISH = /^https?:\/\//i;
|
|
4522
|
+
var IP_LITERAL = /^(\d{1,3}(\.\d{1,3}){3}|\[[0-9a-f:]+\])$/i;
|
|
4523
|
+
var EXAMPLE = "acme.com";
|
|
4524
|
+
var MESSAGES = {
|
|
4525
|
+
empty: `Paste a website address, like ${EXAMPLE}.`,
|
|
4526
|
+
scheme: `Scenri reads web addresses only, the ones that start with http or https. Try ${EXAMPLE}.`,
|
|
4527
|
+
space: "A web address cannot contain a space.",
|
|
4528
|
+
host: `That does not look like a website address. Try ${EXAMPLE}.`,
|
|
4529
|
+
unparseable: `That does not look like a website address. Try ${EXAMPLE}.`
|
|
4530
|
+
};
|
|
4531
|
+
var no = (reason, suggestion) => ({
|
|
4532
|
+
ok: false,
|
|
4533
|
+
reason,
|
|
4534
|
+
message: suggestion && reason === "space" ? `${MESSAGES.space} Did you mean ${suggestion}?` : MESSAGES[reason],
|
|
4535
|
+
...suggestion ? { suggestion } : {}
|
|
4536
|
+
});
|
|
4537
|
+
function unwrap(value) {
|
|
4538
|
+
const pairs = [
|
|
4539
|
+
["<", ">"],
|
|
4540
|
+
['"', '"'],
|
|
4541
|
+
["'", "'"]
|
|
4542
|
+
];
|
|
4543
|
+
for (const [open, close] of pairs) {
|
|
4544
|
+
if (value.length >= 2 && value.startsWith(open) && value.endsWith(close)) return value.slice(1, -1).trim();
|
|
4545
|
+
}
|
|
4546
|
+
return value;
|
|
4547
|
+
}
|
|
4548
|
+
function normalizeSiteUrl(input) {
|
|
4549
|
+
const cleaned = unwrap(
|
|
4550
|
+
String(input ?? "").replace(INVISIBLE, "").replace(SPACES, " ").trim()
|
|
4551
|
+
);
|
|
4552
|
+
if (cleaned === "") return no("empty");
|
|
4553
|
+
const addedScheme = !SCHEMED.test(cleaned) && !OPAQUE.test(cleaned);
|
|
4554
|
+
if (!addedScheme && !HTTPISH.test(cleaned)) return no("scheme");
|
|
4555
|
+
const candidate = addedScheme ? `https://${cleaned}` : cleaned;
|
|
4556
|
+
const authority = candidate.slice(candidate.indexOf("://") + 3).split(/[/?#]/)[0];
|
|
4557
|
+
if (authority.includes(" ")) {
|
|
4558
|
+
const squeezed = normalizeSiteUrl(cleaned.replace(/ /g, ""));
|
|
4559
|
+
return no("space", squeezed.ok ? squeezed.url.replace(/^https?:\/\//, "").replace(/\/$/, "") : void 0);
|
|
4560
|
+
}
|
|
4561
|
+
let u;
|
|
4562
|
+
try {
|
|
4563
|
+
u = new URL(candidate);
|
|
4564
|
+
} catch {
|
|
4565
|
+
return no("unparseable");
|
|
3549
4566
|
}
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
const
|
|
4567
|
+
u.username = "";
|
|
4568
|
+
u.password = "";
|
|
4569
|
+
u.hash = "";
|
|
4570
|
+
const host = u.hostname.replace(/\.$/, "");
|
|
4571
|
+
u.hostname = host;
|
|
4572
|
+
if (!host.includes(".") && !IP_LITERAL.test(host) && host !== "localhost") return no("host");
|
|
4573
|
+
return { ok: true, url: u.toString(), host, addedScheme };
|
|
4574
|
+
}
|
|
4575
|
+
|
|
4576
|
+
// ../brand-spec/src/buildFromUrl.ts
|
|
4577
|
+
var MAX_PORTRAIT_RATIO = 2.5;
|
|
4578
|
+
var CONFIDENT_SCORE = 80;
|
|
4579
|
+
var ICON_SOURCES = /* @__PURE__ */ new Set(["json-ld", "manifest", "apple-touch-icon", "link-icon"]);
|
|
4580
|
+
var LOGO_TRIES = 3;
|
|
4581
|
+
var MAX_SHEETS = 6;
|
|
4582
|
+
var SHEET_RANK = /(root|variables|tokens|colou?rs|palette|theme|main|app|style|tailwind|global|site|brand|base)/i;
|
|
4583
|
+
var SHEET_SKIP = /(font|icon|fontawesome|bootstrap-icons|swiper|slick|slider|lightbox|print)/i;
|
|
4584
|
+
async function buildFromUrl(url, opts = {}) {
|
|
4585
|
+
const warnings = [];
|
|
4586
|
+
const normalized = normalizeSiteUrl(url);
|
|
4587
|
+
if (!normalized.ok) throw urlRefusal(normalized.reason, normalized.message);
|
|
4588
|
+
const get = createGuardedFetch({
|
|
4589
|
+
...opts.guard,
|
|
4590
|
+
...opts.fetchImpl ? { fetchImpl: opts.fetchImpl, allowPrivateHosts: true } : {}
|
|
4591
|
+
});
|
|
4592
|
+
const page = await get(normalized.url, "html");
|
|
4593
|
+
const origin = new URL(page.finalUrl);
|
|
4594
|
+
const $ = cheerio.load(page.text);
|
|
4595
|
+
const base = baseOf($, origin);
|
|
4596
|
+
const named = pickName($, origin);
|
|
3554
4597
|
const tagline = $('meta[name="description"]').attr("content")?.trim() || $('meta[property="og:description"]').attr("content")?.trim();
|
|
3555
|
-
const
|
|
3556
|
-
const themeColor = $('meta[name="theme-color"]').attr("content");
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
4598
|
+
const sources = [];
|
|
4599
|
+
const themeColor = $('meta[name="theme-color"]').attr("content")?.trim();
|
|
4600
|
+
const tileColor = $('meta[name="msapplication-TileColor"]').attr("content")?.trim();
|
|
4601
|
+
for (const [value, scale] of [
|
|
4602
|
+
[themeColor, 2],
|
|
4603
|
+
[tileColor, 1]
|
|
4604
|
+
]) {
|
|
4605
|
+
if (value) sources.push({ css: `:root{--brand-theme:${value}}`, weightScale: scale });
|
|
4606
|
+
}
|
|
4607
|
+
sources.push({ css: inlineCss($) });
|
|
4608
|
+
for (const href of sheetHrefs($, base)) {
|
|
4609
|
+
if (get.remaining() <= 0) break;
|
|
3564
4610
|
try {
|
|
3565
|
-
const
|
|
3566
|
-
|
|
3567
|
-
if (cssRes.ok) colorSources.push(...(await cssRes.text()).match(HEX_RE) ?? []);
|
|
4611
|
+
const sheet = await get(href, "css");
|
|
4612
|
+
sources.push({ css: sheet.text });
|
|
3568
4613
|
} catch {
|
|
3569
4614
|
warnings.push("Stylesheet fetch failed; palette from inline styles only.");
|
|
3570
4615
|
}
|
|
3571
4616
|
}
|
|
3572
|
-
const palette =
|
|
4617
|
+
const palette = paletteFrom(sources, liveClassTokens($));
|
|
3573
4618
|
if (!palette.primary) warnings.push("No confident palette found. Set colors manually.");
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
logoFromOg = Boolean(iconHref);
|
|
3581
|
-
}
|
|
3582
|
-
if (iconHref && opts.saveAsset) {
|
|
3583
|
-
try {
|
|
3584
|
-
const iconRes = await fetchImpl(new URL(iconHref, origin).toString(), { redirect: "follow" });
|
|
3585
|
-
if (iconRes.ok) {
|
|
3586
|
-
const buf = Buffer.from(await iconRes.arrayBuffer());
|
|
3587
|
-
if (buf.length > 0) {
|
|
3588
|
-
logoRef = await opts.saveAsset(buf, "logo");
|
|
3589
|
-
if (!logoFromOg && opts.probeLongEdge) {
|
|
3590
|
-
const edge = await opts.probeLongEdge(buf).catch(() => null);
|
|
3591
|
-
if (edge !== null && edge < 256) {
|
|
3592
|
-
logoTiny = true;
|
|
3593
|
-
warnings.push(
|
|
3594
|
-
`The site icon is favicon-sized (${edge}px), so it was saved as an alternate mark, not the logo. Upload your real logo in Settings.`
|
|
3595
|
-
);
|
|
3596
|
-
}
|
|
3597
|
-
}
|
|
3598
|
-
}
|
|
3599
|
-
}
|
|
3600
|
-
} catch {
|
|
3601
|
-
warnings.push("Logo download failed.");
|
|
3602
|
-
}
|
|
3603
|
-
}
|
|
3604
|
-
if (!logoRef) warnings.push("No logo captured. Add one manually.");
|
|
3605
|
-
else if (logoFromOg)
|
|
3606
|
-
warnings.push(
|
|
3607
|
-
"No site icon was found; the social share image was saved as an alternate mark. Check it before treating it as the logo."
|
|
3608
|
-
);
|
|
4619
|
+
const colorCount = [palette.primary, palette.secondary, ...palette.accent, ...palette.neutrals].filter(
|
|
4620
|
+
Boolean
|
|
4621
|
+
).length;
|
|
4622
|
+
const manifest = await readManifest($, base, get);
|
|
4623
|
+
const candidates = logoCandidates($, base, manifest);
|
|
4624
|
+
const picked = await downloadMark(candidates, get, opts, warnings);
|
|
3609
4625
|
const brand = {
|
|
3610
4626
|
specVersion: "0.1",
|
|
3611
4627
|
meta: {
|
|
3612
|
-
name,
|
|
3613
|
-
slug:
|
|
4628
|
+
name: named.value,
|
|
4629
|
+
slug: named.value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || origin.hostname,
|
|
3614
4630
|
...tagline ? { tagline } : {},
|
|
3615
4631
|
website: origin.origin,
|
|
3616
4632
|
createdWith: opts.createdWith ?? "scenri",
|
|
@@ -3624,26 +4640,208 @@ async function buildFromUrl(url, opts = {}) {
|
|
|
3624
4640
|
...palette.neutrals.length ? { neutrals: palette.neutrals.map((hex) => ({ hex })) } : {}
|
|
3625
4641
|
}
|
|
3626
4642
|
} : {},
|
|
3627
|
-
...
|
|
4643
|
+
...picked.ref ? {
|
|
4644
|
+
logos: [
|
|
4645
|
+
{
|
|
4646
|
+
role: picked.role,
|
|
4647
|
+
file: picked.ref,
|
|
4648
|
+
...picked.background ? { background: picked.background } : {}
|
|
4649
|
+
}
|
|
4650
|
+
]
|
|
4651
|
+
} : {}
|
|
4652
|
+
};
|
|
4653
|
+
return {
|
|
4654
|
+
brand,
|
|
4655
|
+
warnings,
|
|
4656
|
+
report: {
|
|
4657
|
+
url: page.finalUrl,
|
|
4658
|
+
host: origin.hostname,
|
|
4659
|
+
name: named,
|
|
4660
|
+
tagline: tagline ?? null,
|
|
4661
|
+
logo: {
|
|
4662
|
+
status: picked.ref ? picked.role : "none",
|
|
4663
|
+
source: picked.source,
|
|
4664
|
+
...picked.score != null ? { score: picked.score } : {},
|
|
4665
|
+
...picked.note ? { note: picked.note } : {}
|
|
4666
|
+
},
|
|
4667
|
+
colors: { count: colorCount }
|
|
4668
|
+
}
|
|
3628
4669
|
};
|
|
3629
|
-
return { brand, warnings };
|
|
3630
|
-
}
|
|
3631
|
-
|
|
3632
|
-
// ../brand-spec/src/mergeScrape.ts
|
|
3633
|
-
var str2 = (v) => typeof v === "string" ? v.trim() : "";
|
|
3634
|
-
var arr = (v) => Array.isArray(v) ? v : [];
|
|
3635
|
-
var hashOf = (ref) => {
|
|
3636
|
-
const s = String(ref ?? "");
|
|
3637
|
-
return s.startsWith("asset:") ? s.slice(6) : s;
|
|
3638
|
-
};
|
|
3639
|
-
function scrapedHexes(palette) {
|
|
3640
|
-
return [palette?.primary, palette?.secondary, ...arr(palette?.accent), ...arr(palette?.neutrals)].filter((c) => typeof c?.hex === "string").map((c) => ({ hex: String(c.hex) }));
|
|
3641
4670
|
}
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
4671
|
+
function baseOf($, origin) {
|
|
4672
|
+
const declared = $("base[href]").attr("href");
|
|
4673
|
+
if (!declared) return origin;
|
|
4674
|
+
try {
|
|
4675
|
+
return new URL(declared, origin);
|
|
4676
|
+
} catch {
|
|
4677
|
+
return origin;
|
|
4678
|
+
}
|
|
4679
|
+
}
|
|
4680
|
+
function pickName($, origin) {
|
|
4681
|
+
const ld = jsonLdName($);
|
|
4682
|
+
if (ld) return { value: ld, source: "json-ld" };
|
|
4683
|
+
const og = $('meta[property="og:site_name"]').attr("content")?.trim();
|
|
4684
|
+
if (og) return { value: og, source: "og:site_name" };
|
|
4685
|
+
const title = nameFromTitle($("title").first().text());
|
|
4686
|
+
if (title) return { value: title, source: "title" };
|
|
4687
|
+
return { value: origin.hostname.replace(/^www\./, ""), source: "hostname" };
|
|
4688
|
+
}
|
|
4689
|
+
function nameFromTitle(raw) {
|
|
4690
|
+
const parts = raw.trim().split(/\s+[|\u00b7\u2022\u2013\u2014]\s+|\s+-\s+/).map((p) => p.trim()).filter(Boolean);
|
|
4691
|
+
if (parts.length < 2) return parts[0] ?? "";
|
|
4692
|
+
const first = parts[0];
|
|
4693
|
+
const last = parts[parts.length - 1];
|
|
4694
|
+
const words = (v) => v.split(/\s+/).length;
|
|
4695
|
+
if (/^(home|homepage|index|welcome|start|main)$/i.test(first)) return last;
|
|
4696
|
+
if (words(last) <= 4 && words(first) > words(last)) return last;
|
|
4697
|
+
return first;
|
|
4698
|
+
}
|
|
4699
|
+
function jsonLdName($) {
|
|
4700
|
+
for (const node of $('script[type="application/ld+json"]').toArray()) {
|
|
4701
|
+
try {
|
|
4702
|
+
const data = JSON.parse($(node).text());
|
|
4703
|
+
const found = findOrgName(data, 0);
|
|
4704
|
+
if (found) return found;
|
|
4705
|
+
} catch {
|
|
4706
|
+
}
|
|
4707
|
+
}
|
|
4708
|
+
return null;
|
|
4709
|
+
}
|
|
4710
|
+
function findOrgName(node, depth) {
|
|
4711
|
+
if (depth > 6 || node === null || typeof node !== "object") return null;
|
|
4712
|
+
if (Array.isArray(node)) {
|
|
4713
|
+
for (const item of node) {
|
|
4714
|
+
const found = findOrgName(item, depth + 1);
|
|
4715
|
+
if (found) return found;
|
|
4716
|
+
}
|
|
4717
|
+
return null;
|
|
4718
|
+
}
|
|
4719
|
+
const obj = node;
|
|
4720
|
+
const type = String(obj["@type"] ?? "");
|
|
4721
|
+
if (/Organization|Corporation|LocalBusiness/i.test(type) && typeof obj.name === "string" && obj.name.trim())
|
|
4722
|
+
return obj.name.trim();
|
|
4723
|
+
for (const value of Object.values(obj)) {
|
|
4724
|
+
const found = findOrgName(value, depth + 1);
|
|
4725
|
+
if (found) return found;
|
|
4726
|
+
}
|
|
4727
|
+
return null;
|
|
4728
|
+
}
|
|
4729
|
+
function inlineCss($) {
|
|
4730
|
+
const attrs = $("[style]").map((_, el) => `x{${$(el).attr("style")}}`).get().join("\n");
|
|
4731
|
+
const blocks = $("style").map((_, el) => $(el).text()).get().join("\n");
|
|
4732
|
+
return `${attrs}
|
|
4733
|
+
${blocks}`;
|
|
4734
|
+
}
|
|
4735
|
+
function sheetHrefs($, base) {
|
|
4736
|
+
const hrefs = [];
|
|
4737
|
+
for (const el of $('link[rel="stylesheet"]').toArray()) {
|
|
4738
|
+
const href = $(el).attr("href");
|
|
4739
|
+
if (!href || SHEET_SKIP.test(href)) continue;
|
|
4740
|
+
try {
|
|
4741
|
+
hrefs.push(new URL(href, base).toString());
|
|
4742
|
+
} catch {
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
return hrefs.sort((a, b) => Number(SHEET_RANK.test(b)) - Number(SHEET_RANK.test(a))).slice(0, MAX_SHEETS);
|
|
4746
|
+
}
|
|
4747
|
+
async function readManifest($, base, get) {
|
|
4748
|
+
const href = $('link[rel="manifest"]').attr("href");
|
|
4749
|
+
if (!href || get.remaining() <= 0) return void 0;
|
|
4750
|
+
try {
|
|
4751
|
+
const res = await get(new URL(href, base).toString(), "css");
|
|
4752
|
+
return JSON.parse(res.text);
|
|
4753
|
+
} catch {
|
|
4754
|
+
return void 0;
|
|
4755
|
+
}
|
|
4756
|
+
}
|
|
4757
|
+
async function downloadMark(candidates, get, opts, warnings) {
|
|
4758
|
+
if (!opts.saveAsset || candidates.length === 0) {
|
|
4759
|
+
if (candidates.length === 0) warnings.push("No logo captured. Add one manually.");
|
|
4760
|
+
return { role: "primary", source: null };
|
|
4761
|
+
}
|
|
4762
|
+
let fallback = null;
|
|
4763
|
+
let tried = 0;
|
|
4764
|
+
let failed = false;
|
|
4765
|
+
for (const candidate of candidates) {
|
|
4766
|
+
if (tried >= LOGO_TRIES || get.remaining() <= 0) break;
|
|
4767
|
+
let buf = null;
|
|
4768
|
+
if (candidate.svg) {
|
|
4769
|
+
const sized = svgAsMark(candidate.svg);
|
|
4770
|
+
if (!sized) continue;
|
|
4771
|
+
buf = Buffer.from(sized, "utf8");
|
|
4772
|
+
} else if (candidate.url?.startsWith("data:")) {
|
|
4773
|
+
const comma = candidate.url.indexOf(",");
|
|
4774
|
+
const body = candidate.url.slice(comma + 1);
|
|
4775
|
+
buf = candidate.url.slice(0, comma).includes(";base64") ? Buffer.from(body, "base64") : Buffer.from(decodeURIComponent(body), "utf8");
|
|
4776
|
+
} else if (candidate.url) {
|
|
4777
|
+
tried++;
|
|
4778
|
+
try {
|
|
4779
|
+
buf = (await get(candidate.url, "asset")).bytes;
|
|
4780
|
+
} catch {
|
|
4781
|
+
failed = true;
|
|
4782
|
+
continue;
|
|
4783
|
+
}
|
|
4784
|
+
}
|
|
4785
|
+
if (!buf || buf.length === 0) continue;
|
|
4786
|
+
let ref;
|
|
4787
|
+
try {
|
|
4788
|
+
ref = await opts.saveAsset(buf, "logo");
|
|
4789
|
+
} catch {
|
|
4790
|
+
failed = true;
|
|
4791
|
+
continue;
|
|
4792
|
+
}
|
|
4793
|
+
const shape = opts.inspectMark ? await opts.inspectMark(buf).catch(() => null) : null;
|
|
4794
|
+
if (shape?.blank) continue;
|
|
4795
|
+
const edge = candidate.svg ? null : shape?.longEdge ?? null;
|
|
4796
|
+
const tiny = edge !== null && edge < 256;
|
|
4797
|
+
const tall = shape?.width != null && shape?.height != null && shape.width > 0 && shape.height / shape.width > MAX_PORTRAIT_RATIO;
|
|
4798
|
+
const declaredIcon = ICON_SOURCES.has(candidate.source);
|
|
4799
|
+
const unmeasured = !candidate.svg && shape === null;
|
|
4800
|
+
const bigEnough = edge !== null ? declaredIcon && edge >= 512 : unmeasured;
|
|
4801
|
+
const unsure = candidate.score < CONFIDENT_SCORE && !bigEnough;
|
|
4802
|
+
const role = candidate.role === "alternate" || tiny || tall || unsure ? "alternate" : "primary";
|
|
4803
|
+
const picked = {
|
|
4804
|
+
ref,
|
|
4805
|
+
role,
|
|
4806
|
+
source: candidate.source,
|
|
4807
|
+
score: candidate.score,
|
|
4808
|
+
...candidate.background ? { background: candidate.background } : {}
|
|
4809
|
+
};
|
|
4810
|
+
if (role === "primary") return picked;
|
|
4811
|
+
if (tiny)
|
|
4812
|
+
picked.note = `The site icon is favicon-sized (${edge}px), so it was saved as an alternate mark, not the logo. Upload your real logo in Settings.`;
|
|
4813
|
+
else if (tall)
|
|
4814
|
+
picked.note = "The only image we could find is much taller than it is wide, so it was saved as an alternate mark rather than the logo. Upload your real logo in Settings.";
|
|
4815
|
+
else if (unsure)
|
|
4816
|
+
picked.note = "We are not confident this is your logo, so it was saved as an alternate mark. Check it, or upload your real logo in Settings.";
|
|
4817
|
+
else if (candidate.source === "og-image")
|
|
4818
|
+
picked.note = "No site icon was found; the social share image was saved as an alternate mark. Check it before treating it as the logo.";
|
|
4819
|
+
fallback ??= picked;
|
|
4820
|
+
}
|
|
4821
|
+
if (fallback) {
|
|
4822
|
+
if (fallback.note) warnings.push(fallback.note);
|
|
4823
|
+
return fallback;
|
|
4824
|
+
}
|
|
4825
|
+
if (failed) warnings.push("Logo download failed.");
|
|
4826
|
+
warnings.push("No logo captured. Add one manually.");
|
|
4827
|
+
return { role: "primary", source: null };
|
|
4828
|
+
}
|
|
4829
|
+
|
|
4830
|
+
// ../brand-spec/src/mergeScrape.ts
|
|
4831
|
+
var str2 = (v) => typeof v === "string" ? v.trim() : "";
|
|
4832
|
+
var arr = (v) => Array.isArray(v) ? v : [];
|
|
4833
|
+
var hashOf = (ref) => {
|
|
4834
|
+
const s = String(ref ?? "");
|
|
4835
|
+
return s.startsWith("asset:") ? s.slice(6) : s;
|
|
4836
|
+
};
|
|
4837
|
+
function scrapedHexes(palette) {
|
|
4838
|
+
return [palette?.primary, palette?.secondary, ...arr(palette?.accent), ...arr(palette?.neutrals)].filter((c) => typeof c?.hex === "string").map((c) => ({ hex: String(c.hex) }));
|
|
4839
|
+
}
|
|
4840
|
+
var hasPalette = (palette) => scrapedHexes(palette).length > 0;
|
|
4841
|
+
function mergeScrape(existing, scraped) {
|
|
4842
|
+
const cur = existing ?? {};
|
|
4843
|
+
const next = scraped ?? {};
|
|
4844
|
+
const meta = { ...cur.meta ?? {} };
|
|
3647
4845
|
if (!str2(meta.name) && str2(next.meta?.name)) meta.name = str2(next.meta.name);
|
|
3648
4846
|
if (!str2(meta.tagline) && str2(next.meta?.tagline)) meta.tagline = str2(next.meta.tagline);
|
|
3649
4847
|
if (str2(next.meta?.website)) meta.website = str2(next.meta.website);
|
|
@@ -3671,6 +4869,74 @@ function mergeScrape(existing, scraped) {
|
|
|
3671
4869
|
}
|
|
3672
4870
|
return { brand, suggestions: { palette: suggestions } };
|
|
3673
4871
|
}
|
|
4872
|
+
async function inspectMark(buf, toPng2) {
|
|
4873
|
+
const png = await toPng2(buf);
|
|
4874
|
+
const meta = await sharp21(png).metadata();
|
|
4875
|
+
let blank = false;
|
|
4876
|
+
try {
|
|
4877
|
+
const flat = await sharp21(png).flatten({ background: "#ffffff" }).toBuffer();
|
|
4878
|
+
const stats = await sharp21(flat).stats();
|
|
4879
|
+
blank = stats.channels.slice(0, 3).every((c) => c.min > 200);
|
|
4880
|
+
} catch {
|
|
4881
|
+
}
|
|
4882
|
+
return {
|
|
4883
|
+
longEdge: Math.max(meta.width ?? 0, meta.height ?? 0) || null,
|
|
4884
|
+
width: meta.width ?? null,
|
|
4885
|
+
height: meta.height ?? null,
|
|
4886
|
+
blank
|
|
4887
|
+
};
|
|
4888
|
+
}
|
|
4889
|
+
|
|
4890
|
+
// ../catalog/src/failures.ts
|
|
4891
|
+
function pageFailure(status) {
|
|
4892
|
+
if (status === 429 || status === 503) return "RATE_LIMITED";
|
|
4893
|
+
if (status === 401 || status === 403 || status === 405) return "PAGE_BLOCKED";
|
|
4894
|
+
if (status === 404 || status === 410) return "PAGE_NOT_FOUND";
|
|
4895
|
+
return "PAGE_UNREADABLE";
|
|
4896
|
+
}
|
|
4897
|
+
function imageFailure(status) {
|
|
4898
|
+
if (status === 429 || status === 503) return "RATE_LIMITED";
|
|
4899
|
+
if (status === 401 || status === 403) return "IMAGE_FORBIDDEN";
|
|
4900
|
+
if (status === 404 || status === 410) return "IMAGE_NOT_FOUND";
|
|
4901
|
+
return "DOWNLOAD_FAILED";
|
|
4902
|
+
}
|
|
4903
|
+
function thrownFailure(err, aborted = false) {
|
|
4904
|
+
if (aborted) return "ABORTED";
|
|
4905
|
+
const m = String(err?.message ?? err).toLowerCase();
|
|
4906
|
+
if (m.includes("abort") || m.includes("timed out") || m.includes("timeout")) return "IMAGE_TIMEOUT";
|
|
4907
|
+
return "DOWNLOAD_FAILED";
|
|
4908
|
+
}
|
|
4909
|
+
function tally(into, reason, n = 1) {
|
|
4910
|
+
into[reason] = (into[reason] ?? 0) + n;
|
|
4911
|
+
}
|
|
4912
|
+
function leadingReason(t) {
|
|
4913
|
+
const rows = Object.entries(t);
|
|
4914
|
+
if (!rows.length) return null;
|
|
4915
|
+
return rows.sort((a, b) => b[1] - a[1])[0][0];
|
|
4916
|
+
}
|
|
4917
|
+
function summarise(t, saved, asked) {
|
|
4918
|
+
const reason = leadingReason(t);
|
|
4919
|
+
if (!reason) return null;
|
|
4920
|
+
const none = saved === 0;
|
|
4921
|
+
switch (reason) {
|
|
4922
|
+
case "RATE_LIMITED":
|
|
4923
|
+
return none ? "The store asked us to slow down, so nothing could be read. Waiting a few minutes and trying again usually works." : `The store asked us to slow down partway, so ${asked - saved} of ${asked} products were skipped. Trying again picks up the rest.`;
|
|
4924
|
+
case "PAGE_BLOCKED":
|
|
4925
|
+
return none ? "This store would not let us read its product pages." : `${asked - saved} of ${asked} product pages would not open.`;
|
|
4926
|
+
case "PAGE_NOT_FOUND":
|
|
4927
|
+
return `${asked - saved} of ${asked} products are no longer on the site.`;
|
|
4928
|
+
case "IMAGE_FORBIDDEN":
|
|
4929
|
+
case "IMAGE_NOT_FOUND":
|
|
4930
|
+
case "IMAGE_TIMEOUT":
|
|
4931
|
+
case "IMAGE_EMPTY":
|
|
4932
|
+
case "DOWNLOAD_FAILED":
|
|
4933
|
+
return none ? "We found the products, but none of their pictures could be downloaded." : "Some pictures could not be downloaded. The products they belong to are still here.";
|
|
4934
|
+
case "ABORTED":
|
|
4935
|
+
return null;
|
|
4936
|
+
default:
|
|
4937
|
+
return none ? "The products on this site could not be read." : null;
|
|
4938
|
+
}
|
|
4939
|
+
}
|
|
3674
4940
|
|
|
3675
4941
|
// ../catalog/src/url.ts
|
|
3676
4942
|
function normalizeStoreUrl(input) {
|
|
@@ -3717,6 +4983,28 @@ function upgradeImageUrl(url) {
|
|
|
3717
4983
|
return url;
|
|
3718
4984
|
}
|
|
3719
4985
|
}
|
|
4986
|
+
var LOCALE_SEGMENT = /^\/[a-z]{2}(?:-[A-Za-z]{2})?\//;
|
|
4987
|
+
function preferCanonicalLocale(urls) {
|
|
4988
|
+
const canonical = /* @__PURE__ */ new Set();
|
|
4989
|
+
for (const u of urls) {
|
|
4990
|
+
try {
|
|
4991
|
+
const parsed = new URL(u);
|
|
4992
|
+
if (!LOCALE_SEGMENT.test(parsed.pathname)) canonical.add(`${parsed.origin}${parsed.pathname}`);
|
|
4993
|
+
} catch {
|
|
4994
|
+
}
|
|
4995
|
+
}
|
|
4996
|
+
if (!canonical.size) return urls;
|
|
4997
|
+
return urls.filter((u) => {
|
|
4998
|
+
try {
|
|
4999
|
+
const parsed = new URL(u);
|
|
5000
|
+
if (!LOCALE_SEGMENT.test(parsed.pathname)) return true;
|
|
5001
|
+
const stripped = parsed.pathname.replace(LOCALE_SEGMENT, "/");
|
|
5002
|
+
return !canonical.has(`${parsed.origin}${stripped}`);
|
|
5003
|
+
} catch {
|
|
5004
|
+
return true;
|
|
5005
|
+
}
|
|
5006
|
+
});
|
|
5007
|
+
}
|
|
3720
5008
|
|
|
3721
5009
|
// ../catalog/src/normalize.ts
|
|
3722
5010
|
function cleanText(s) {
|
|
@@ -3824,34 +5112,137 @@ function dedupeProducts(products) {
|
|
|
3824
5112
|
}
|
|
3825
5113
|
return [...byKey.values()];
|
|
3826
5114
|
}
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
5115
|
+
var ALLOW_PRIVATE = process.env.SCENRI_SCRAPE_ALLOW_PRIVATE === "1";
|
|
5116
|
+
async function assertReachable(url, real) {
|
|
5117
|
+
const u = new URL(url);
|
|
5118
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") {
|
|
5119
|
+
throw new Error(`Scenri reads http and https addresses only, not ${u.protocol.replace(":", "")}`);
|
|
5120
|
+
}
|
|
5121
|
+
if (ALLOW_PRIVATE) return;
|
|
5122
|
+
const host = u.hostname.replace(/^\[|\]$/g, "");
|
|
5123
|
+
if (isIP(host)) {
|
|
5124
|
+
assertPublicHost(u.hostname, [host], false);
|
|
5125
|
+
return;
|
|
5126
|
+
}
|
|
5127
|
+
if (!real) return;
|
|
5128
|
+
let addresses;
|
|
5129
|
+
try {
|
|
5130
|
+
addresses = (await lookup(host, { all: true })).map((a) => a.address);
|
|
5131
|
+
} catch {
|
|
5132
|
+
return;
|
|
5133
|
+
}
|
|
5134
|
+
assertPublicHost(u.hostname, addresses, false);
|
|
5135
|
+
}
|
|
5136
|
+
var USER_AGENT2 = "scenri-catalog/0.1 (+https://scenri.co)";
|
|
5137
|
+
async function readBounded2(res, maxBytes) {
|
|
5138
|
+
if (!maxBytes || !res.body) return res.text();
|
|
5139
|
+
const reader = res.body.getReader();
|
|
5140
|
+
const decoder = new TextDecoder();
|
|
5141
|
+
let out = "";
|
|
5142
|
+
let seen = 0;
|
|
5143
|
+
try {
|
|
5144
|
+
while (seen < maxBytes) {
|
|
5145
|
+
const { done, value } = await reader.read();
|
|
5146
|
+
if (done) break;
|
|
5147
|
+
seen += value.byteLength;
|
|
5148
|
+
out += decoder.decode(value, { stream: true });
|
|
5149
|
+
}
|
|
5150
|
+
} finally {
|
|
5151
|
+
await reader.cancel().catch(() => {
|
|
5152
|
+
});
|
|
5153
|
+
}
|
|
5154
|
+
return out + decoder.decode();
|
|
5155
|
+
}
|
|
5156
|
+
function sleep4(ms) {
|
|
3831
5157
|
return new Promise((r) => setTimeout(r, ms));
|
|
3832
5158
|
}
|
|
5159
|
+
var cooledUntil = /* @__PURE__ */ new Map();
|
|
5160
|
+
var hostOf = (url) => {
|
|
5161
|
+
try {
|
|
5162
|
+
return new URL(url).host;
|
|
5163
|
+
} catch {
|
|
5164
|
+
return "";
|
|
5165
|
+
}
|
|
5166
|
+
};
|
|
5167
|
+
async function waitForHost(host, signal) {
|
|
5168
|
+
if (!host) return;
|
|
5169
|
+
for (; ; ) {
|
|
5170
|
+
const left = (cooledUntil.get(host) ?? 0) - Date.now();
|
|
5171
|
+
if (left <= 0 || signal?.aborted) return;
|
|
5172
|
+
await sleep4(Math.min(left, 250));
|
|
5173
|
+
}
|
|
5174
|
+
}
|
|
5175
|
+
function coolHost(host, ms) {
|
|
5176
|
+
if (!host || ms <= 0) return;
|
|
5177
|
+
const until = Date.now() + ms;
|
|
5178
|
+
if (until > (cooledUntil.get(host) ?? 0)) cooledUntil.set(host, until);
|
|
5179
|
+
}
|
|
5180
|
+
var RETRY_AFTER_CAP_MS = 6e4;
|
|
5181
|
+
function retryAfterMs(header, now = Date.now()) {
|
|
5182
|
+
if (!header) return null;
|
|
5183
|
+
const secs = Number(header.trim());
|
|
5184
|
+
if (Number.isFinite(secs) && secs >= 0) return Math.min(secs * 1e3, RETRY_AFTER_CAP_MS);
|
|
5185
|
+
const at = Date.parse(header);
|
|
5186
|
+
if (Number.isNaN(at)) return null;
|
|
5187
|
+
return Math.min(Math.max(0, at - now), RETRY_AFTER_CAP_MS);
|
|
5188
|
+
}
|
|
5189
|
+
function throttleBackoff(attempt) {
|
|
5190
|
+
const base = Math.min(1e3 * 2 ** attempt, 16e3);
|
|
5191
|
+
return base + Math.floor(Math.random() * 400);
|
|
5192
|
+
}
|
|
3833
5193
|
async function httpGet(url, opts = {}) {
|
|
3834
5194
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
3835
5195
|
const retries = opts.retries ?? 3;
|
|
3836
5196
|
const timeoutMs = opts.timeoutMs ?? 25e3;
|
|
3837
5197
|
let lastErr;
|
|
5198
|
+
const host = hostOf(url);
|
|
3838
5199
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
5200
|
+
await waitForHost(host, opts.signal);
|
|
5201
|
+
if (opts.signal?.aborted) throw new Error("aborted");
|
|
3839
5202
|
const ctrl = new AbortController();
|
|
3840
5203
|
const onAbort = () => ctrl.abort();
|
|
3841
5204
|
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
|
3842
5205
|
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
3843
5206
|
try {
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
5207
|
+
let current = url;
|
|
5208
|
+
let res;
|
|
5209
|
+
for (let hop = 0; ; hop++) {
|
|
5210
|
+
await assertReachable(current, !opts.fetchImpl);
|
|
5211
|
+
res = await fetchImpl(current, {
|
|
5212
|
+
redirect: "manual",
|
|
5213
|
+
signal: ctrl.signal,
|
|
5214
|
+
headers: {
|
|
5215
|
+
"user-agent": USER_AGENT2,
|
|
5216
|
+
accept: opts.accept ?? "application/json, text/html, application/xml, text/xml, */*;q=0.8",
|
|
5217
|
+
...opts.headers ?? {}
|
|
5218
|
+
}
|
|
5219
|
+
});
|
|
5220
|
+
if (res.status < 300 || res.status >= 400) break;
|
|
5221
|
+
const location = res.headers.get("location");
|
|
5222
|
+
await res.body?.cancel().catch(() => {
|
|
5223
|
+
});
|
|
5224
|
+
if (!location || hop >= 5) break;
|
|
5225
|
+
current = new URL(location, current).toString();
|
|
5226
|
+
}
|
|
5227
|
+
if (res.url !== current) {
|
|
5228
|
+
try {
|
|
5229
|
+
Object.defineProperty(res, "url", { value: current, configurable: true });
|
|
5230
|
+
} catch {
|
|
3851
5231
|
}
|
|
3852
|
-
}
|
|
3853
|
-
if (
|
|
3854
|
-
|
|
5232
|
+
}
|
|
5233
|
+
if (res.status === 429 || res.status === 503) {
|
|
5234
|
+
const asked = retryAfterMs(res.headers.get("retry-after"));
|
|
5235
|
+
const wait = asked ?? throttleBackoff(attempt);
|
|
5236
|
+
coolHost(host, wait);
|
|
5237
|
+
if (attempt < retries) {
|
|
5238
|
+
await res.body?.cancel().catch(() => {
|
|
5239
|
+
});
|
|
5240
|
+
continue;
|
|
5241
|
+
}
|
|
5242
|
+
return res;
|
|
5243
|
+
}
|
|
5244
|
+
if (res.status >= 500 && attempt < retries) {
|
|
5245
|
+
await sleep4(throttleBackoff(attempt));
|
|
3855
5246
|
continue;
|
|
3856
5247
|
}
|
|
3857
5248
|
return res;
|
|
@@ -3859,7 +5250,7 @@ async function httpGet(url, opts = {}) {
|
|
|
3859
5250
|
lastErr = err;
|
|
3860
5251
|
if (opts.signal?.aborted) throw err;
|
|
3861
5252
|
if (attempt < retries) {
|
|
3862
|
-
await
|
|
5253
|
+
await sleep4(400 * 2 ** attempt);
|
|
3863
5254
|
continue;
|
|
3864
5255
|
}
|
|
3865
5256
|
throw err;
|
|
@@ -3872,7 +5263,7 @@ async function httpGet(url, opts = {}) {
|
|
|
3872
5263
|
}
|
|
3873
5264
|
async function httpText(url, opts = {}) {
|
|
3874
5265
|
const res = await httpGet(url, opts);
|
|
3875
|
-
const text = await res.
|
|
5266
|
+
const text = await readBounded2(res, opts.maxBytes);
|
|
3876
5267
|
return { ok: res.ok, status: res.status, text, url: res.url || url };
|
|
3877
5268
|
}
|
|
3878
5269
|
async function httpJson(url, opts = {}) {
|
|
@@ -3902,6 +5293,284 @@ async function mapPool(items, concurrency, fn, signal) {
|
|
|
3902
5293
|
await Promise.all(workers);
|
|
3903
5294
|
return results;
|
|
3904
5295
|
}
|
|
5296
|
+
function loadHtml(html) {
|
|
5297
|
+
return parse(html);
|
|
5298
|
+
}
|
|
5299
|
+
function attr(el, name) {
|
|
5300
|
+
return el?.getAttribute(name) ?? void 0;
|
|
5301
|
+
}
|
|
5302
|
+
function textOf(el) {
|
|
5303
|
+
return (el?.text ?? "").replace(/\s+/g, " ").trim();
|
|
5304
|
+
}
|
|
5305
|
+
|
|
5306
|
+
// ../catalog/src/adapters/productPage.ts
|
|
5307
|
+
function stableKey(url) {
|
|
5308
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
5309
|
+
}
|
|
5310
|
+
var typesOf = (obj) => {
|
|
5311
|
+
const t = obj["@type"];
|
|
5312
|
+
const list2 = Array.isArray(t) ? t : t ? [t] : [];
|
|
5313
|
+
return list2.map((x) => String(x).toLowerCase());
|
|
5314
|
+
};
|
|
5315
|
+
function walkJsonLd(node, groups, singles) {
|
|
5316
|
+
if (!node) return;
|
|
5317
|
+
if (Array.isArray(node)) {
|
|
5318
|
+
for (const n of node) walkJsonLd(n, groups, singles);
|
|
5319
|
+
return;
|
|
5320
|
+
}
|
|
5321
|
+
if (typeof node !== "object") return;
|
|
5322
|
+
const obj = node;
|
|
5323
|
+
const types = typesOf(obj);
|
|
5324
|
+
if (types.includes("productgroup")) groups.push(obj);
|
|
5325
|
+
else if (types.includes("product")) singles.push(obj);
|
|
5326
|
+
if (obj["@graph"]) walkJsonLd(obj["@graph"], groups, singles);
|
|
5327
|
+
for (const v of Object.values(obj)) {
|
|
5328
|
+
if (v && typeof v === "object") walkJsonLd(v, groups, singles);
|
|
5329
|
+
}
|
|
5330
|
+
}
|
|
5331
|
+
var asArray = (v) => Array.isArray(v) ? v : v == null ? [] : [v];
|
|
5332
|
+
function imagesOf(n, pageUrl) {
|
|
5333
|
+
return [].concat(n.image ?? []).flat().map((img) => typeof img === "string" ? img : img?.url ?? img?.contentUrl).filter(Boolean).map((u, i) => ({ url: absolutize(pageUrl, String(u)), position: i, width: null, height: null, alt: null })).filter((img) => img.url);
|
|
5334
|
+
}
|
|
5335
|
+
var offerOf = (n) => Array.isArray(n?.offers) ? n.offers[0] : n?.offers;
|
|
5336
|
+
function variesBy(group) {
|
|
5337
|
+
const named = asArray(group.variesBy).map((v) => String(v).split("/").pop() ?? "").filter(Boolean);
|
|
5338
|
+
return named.length ? named : ["size", "color"];
|
|
5339
|
+
}
|
|
5340
|
+
function variantsOf(group, keys) {
|
|
5341
|
+
return asArray(group.hasVariant).filter((v) => v && typeof v === "object").map((v, i) => {
|
|
5342
|
+
const offer = offerOf(v);
|
|
5343
|
+
const options = Object.fromEntries(
|
|
5344
|
+
keys.map((k) => [k, v[k]]).filter(([, val]) => val != null && val !== "")
|
|
5345
|
+
);
|
|
5346
|
+
const base = String(v.sku || v.mpn || v.gtin || `${group.productGroupID ?? group.name ?? "group"}:${i}`);
|
|
5347
|
+
const suffix = Object.values(options).join("/");
|
|
5348
|
+
return {
|
|
5349
|
+
externalKey: suffix ? `${base}:${suffix}` : base,
|
|
5350
|
+
title: [v.name, ...Object.values(options)].filter(Boolean).join(" "),
|
|
5351
|
+
sku: v.sku ? String(v.sku) : null,
|
|
5352
|
+
price: offer?.price != null ? Number(offer.price) : null,
|
|
5353
|
+
compareAtPrice: null,
|
|
5354
|
+
currency: offer?.priceCurrency ?? null,
|
|
5355
|
+
available: offer?.availability ? /instock/i.test(String(offer.availability)) : null,
|
|
5356
|
+
options
|
|
5357
|
+
};
|
|
5358
|
+
});
|
|
5359
|
+
}
|
|
5360
|
+
function fromGroup(group, pageUrl) {
|
|
5361
|
+
const keys = variesBy(group);
|
|
5362
|
+
const variants = variantsOf(group, keys);
|
|
5363
|
+
const url = absolutize(pageUrl, String(group.url ?? group["@id"] ?? pageUrl)) ?? pageUrl;
|
|
5364
|
+
const images = imagesOf(group, pageUrl);
|
|
5365
|
+
const fallback = images.length ? images : imagesOf(asArray(group.hasVariant)[0] ?? {}, pageUrl);
|
|
5366
|
+
return normalizeProduct({
|
|
5367
|
+
externalKey: String(group.productGroupID || group.sku || stableKey(url)),
|
|
5368
|
+
title: String(group.name ?? "Product"),
|
|
5369
|
+
descriptionHtml: group.description ? String(group.description) : null,
|
|
5370
|
+
url,
|
|
5371
|
+
vendor: group.brand?.name ?? (typeof group.brand === "string" ? group.brand : null),
|
|
5372
|
+
productType: group.category ? String(group.category) : null,
|
|
5373
|
+
category: group.category ? String(group.category) : null,
|
|
5374
|
+
price: variants.find((v) => v.price != null)?.price ?? null,
|
|
5375
|
+
compareAtPrice: null,
|
|
5376
|
+
currency: variants.find((v) => v.currency)?.currency ?? null,
|
|
5377
|
+
available: variants.some((v) => v.available) || null,
|
|
5378
|
+
tags: [],
|
|
5379
|
+
variants,
|
|
5380
|
+
images: fallback,
|
|
5381
|
+
raw: group
|
|
5382
|
+
});
|
|
5383
|
+
}
|
|
5384
|
+
function fromSingle(n, pageUrl) {
|
|
5385
|
+
const offers = offerOf(n);
|
|
5386
|
+
const url = String(n.url ?? n["@id"] ?? pageUrl);
|
|
5387
|
+
return normalizeProduct({
|
|
5388
|
+
externalKey: String(n.sku || n.productID || n.mpn || stableKey(url)),
|
|
5389
|
+
title: String(n.name ?? "Product"),
|
|
5390
|
+
descriptionHtml: n.description ? String(n.description) : null,
|
|
5391
|
+
url: absolutize(pageUrl, url) ?? pageUrl,
|
|
5392
|
+
vendor: n.brand?.name ?? (typeof n.brand === "string" ? n.brand : null),
|
|
5393
|
+
productType: n.category ? String(n.category) : null,
|
|
5394
|
+
category: n.category ? String(n.category) : null,
|
|
5395
|
+
price: offers?.price != null ? Number(offers.price) : null,
|
|
5396
|
+
compareAtPrice: null,
|
|
5397
|
+
currency: offers?.priceCurrency ?? null,
|
|
5398
|
+
available: offers?.availability ? /instock/i.test(String(offers.availability)) : null,
|
|
5399
|
+
tags: [],
|
|
5400
|
+
variants: [],
|
|
5401
|
+
images: imagesOf(n, pageUrl),
|
|
5402
|
+
raw: n
|
|
5403
|
+
});
|
|
5404
|
+
}
|
|
5405
|
+
function extractJsonLdProducts(html, pageUrl, doc) {
|
|
5406
|
+
const root = doc ?? loadHtml(html);
|
|
5407
|
+
const groups = [];
|
|
5408
|
+
const singles = [];
|
|
5409
|
+
for (const el of root.querySelectorAll('script[type="application/ld+json"]')) {
|
|
5410
|
+
const raw = el.innerHTML;
|
|
5411
|
+
if (!raw) continue;
|
|
5412
|
+
try {
|
|
5413
|
+
walkJsonLd(JSON.parse(raw), groups, singles);
|
|
5414
|
+
} catch {
|
|
5415
|
+
}
|
|
5416
|
+
}
|
|
5417
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
5418
|
+
for (const g of groups) for (const v of asArray(g.hasVariant)) if (v && typeof v === "object") claimed.add(v);
|
|
5419
|
+
return [
|
|
5420
|
+
...groups.map((g) => fromGroup(g, pageUrl)),
|
|
5421
|
+
...singles.filter((n) => !claimed.has(n)).map((n) => fromSingle(n, pageUrl))
|
|
5422
|
+
];
|
|
5423
|
+
}
|
|
5424
|
+
var PRODUCT_MARKERS = [
|
|
5425
|
+
'meta[property="og:type"][content="product"]',
|
|
5426
|
+
'meta[property="product:price:amount"]',
|
|
5427
|
+
'meta[property="og:price:amount"]',
|
|
5428
|
+
'[itemtype*="schema.org/Product"]',
|
|
5429
|
+
'[itemprop="price"]',
|
|
5430
|
+
'[itemprop="offers"]',
|
|
5431
|
+
'form[action*="/cart/add"]',
|
|
5432
|
+
'[name="add"]'
|
|
5433
|
+
];
|
|
5434
|
+
var BUY_WORDS = /add to (cart|bag|basket)|buy now|add to my bag/i;
|
|
5435
|
+
var BUILD_ASSET = /\/_next\/static\/|\/static\/media\/|\/assets\/(icons|flags|ui)\//i;
|
|
5436
|
+
var NOT_A_PACKSHOT = /logo|icon|sprite|pixel|avatar|\bflags?\b|\/flags?\/|locale|country|currency|badge|payment|social/i;
|
|
5437
|
+
function looksLikeProduct(html, doc) {
|
|
5438
|
+
const $ = doc ?? loadHtml(html);
|
|
5439
|
+
for (const sel of PRODUCT_MARKERS) {
|
|
5440
|
+
try {
|
|
5441
|
+
if ($.querySelector(sel)) return true;
|
|
5442
|
+
} catch {
|
|
5443
|
+
}
|
|
5444
|
+
}
|
|
5445
|
+
for (const el of $.querySelectorAll('button, input[type="submit"], a')) {
|
|
5446
|
+
const words = `${textOf(el)} ${attr(el, "value") ?? ""} ${attr(el, "aria-label") ?? ""}`;
|
|
5447
|
+
if (BUY_WORDS.test(words)) return true;
|
|
5448
|
+
}
|
|
5449
|
+
return false;
|
|
5450
|
+
}
|
|
5451
|
+
function galleryImages($, pageUrl, cap2 = 12) {
|
|
5452
|
+
const images = [];
|
|
5453
|
+
const og = attr($.querySelector('meta[property="og:image"]'), "content");
|
|
5454
|
+
if (og) {
|
|
5455
|
+
const abs = absolutize(pageUrl, og);
|
|
5456
|
+
if (abs) images.push({ url: abs, position: 0, width: null, height: null, alt: null });
|
|
5457
|
+
}
|
|
5458
|
+
for (const el of $.querySelectorAll("img[src]")) {
|
|
5459
|
+
if (images.length >= cap2) break;
|
|
5460
|
+
const src = attr(el, "src") || attr(el, "data-src");
|
|
5461
|
+
const abs = src ? absolutize(pageUrl, src) : null;
|
|
5462
|
+
if (!abs || NOT_A_PACKSHOT.test(abs) || BUILD_ASSET.test(abs)) continue;
|
|
5463
|
+
if (images.some((x) => x.url === abs)) continue;
|
|
5464
|
+
images.push({ url: abs, position: images.length, width: null, height: null, alt: attr(el, "alt") ?? null });
|
|
5465
|
+
}
|
|
5466
|
+
return images;
|
|
5467
|
+
}
|
|
5468
|
+
function parseProductHtml(html, pageUrl, doc) {
|
|
5469
|
+
const $ = doc ?? loadHtml(html);
|
|
5470
|
+
if (!looksLikeProduct(html, $)) return null;
|
|
5471
|
+
const title = attr($.querySelector('meta[property="og:title"]'), "content") || textOf($.querySelector("h1")) || textOf($.querySelector("title"));
|
|
5472
|
+
if (!title) return null;
|
|
5473
|
+
const desc = attr($.querySelector('meta[property="og:description"]'), "content") || attr($.querySelector('meta[name="description"]'), "content") || null;
|
|
5474
|
+
const images = galleryImages($, pageUrl);
|
|
5475
|
+
const canonical = attr($.querySelector('link[rel="canonical"]'), "href");
|
|
5476
|
+
const url = canonical ? absolutize(pageUrl, canonical) ?? pageUrl : pageUrl;
|
|
5477
|
+
return normalizeProduct({
|
|
5478
|
+
externalKey: stableKey(url),
|
|
5479
|
+
title,
|
|
5480
|
+
descriptionHtml: desc,
|
|
5481
|
+
url,
|
|
5482
|
+
images,
|
|
5483
|
+
variants: [],
|
|
5484
|
+
tags: [],
|
|
5485
|
+
raw: { source: "html" }
|
|
5486
|
+
});
|
|
5487
|
+
}
|
|
5488
|
+
function sellsSomething(p) {
|
|
5489
|
+
const n = p.raw ?? {};
|
|
5490
|
+
return Boolean(n.offers || n.sku || n.gtin || n.mpn || n.hasVariant || p.price != null);
|
|
5491
|
+
}
|
|
5492
|
+
function withGallery(product, doc, pageUrl) {
|
|
5493
|
+
const have = new Set((product.images ?? []).map((i) => i.url));
|
|
5494
|
+
const extra = galleryImages(doc, pageUrl).filter((i) => !have.has(i.url));
|
|
5495
|
+
if (!extra.length) return product;
|
|
5496
|
+
const images = [...product.images ?? []];
|
|
5497
|
+
for (const img of extra) {
|
|
5498
|
+
if (images.length >= 12) break;
|
|
5499
|
+
images.push({ ...img, position: images.length });
|
|
5500
|
+
}
|
|
5501
|
+
return { ...product, images };
|
|
5502
|
+
}
|
|
5503
|
+
function productsFromPage(html, url) {
|
|
5504
|
+
const doc = loadHtml(html);
|
|
5505
|
+
const declared = looksLikeProduct(html, doc);
|
|
5506
|
+
const fromLd = extractJsonLdProducts(html, url, doc);
|
|
5507
|
+
const selling = declared ? fromLd : fromLd.filter(sellsSomething);
|
|
5508
|
+
if (selling.length === 1) return [withGallery(selling[0], doc, url)];
|
|
5509
|
+
if (selling.length) return selling;
|
|
5510
|
+
if (!declared) return [];
|
|
5511
|
+
const one = parseProductHtml(html, url, doc);
|
|
5512
|
+
return one ? [one] : [];
|
|
5513
|
+
}
|
|
5514
|
+
var sleep5 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
5515
|
+
async function fetchProductPages(ctx, urls, opts = {}) {
|
|
5516
|
+
const out = [];
|
|
5517
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5518
|
+
let kept = 0;
|
|
5519
|
+
let refusedRun = 0;
|
|
5520
|
+
const giveUpAt = opts.giveUpAfterRefusals ?? Number.POSITIVE_INFINITY;
|
|
5521
|
+
let givenUp = false;
|
|
5522
|
+
const want = opts.want ?? Number.POSITIVE_INFINITY;
|
|
5523
|
+
const ceiling = opts.maxPages ?? (Number.isFinite(want) ? want * 3 : urls.length);
|
|
5524
|
+
const take = urls.slice(0, ceiling);
|
|
5525
|
+
let bytes = 0;
|
|
5526
|
+
await mapPool(
|
|
5527
|
+
take,
|
|
5528
|
+
opts.delayMs ? 1 : opts.concurrency ?? 5,
|
|
5529
|
+
async (u) => {
|
|
5530
|
+
if (kept >= want || givenUp) return;
|
|
5531
|
+
if (opts.deadline != null && Date.now() > opts.deadline) return;
|
|
5532
|
+
if (opts.maxTotalBytes != null && bytes >= opts.maxTotalBytes) return;
|
|
5533
|
+
try {
|
|
5534
|
+
if (opts.delayMs) await sleep5(opts.delayMs);
|
|
5535
|
+
const { ok, status, text, url } = await httpText(u, {
|
|
5536
|
+
fetchImpl: ctx.fetchImpl,
|
|
5537
|
+
signal: ctx.signal,
|
|
5538
|
+
accept: "text/html",
|
|
5539
|
+
maxBytes: opts.maxBytes
|
|
5540
|
+
});
|
|
5541
|
+
bytes += text.length;
|
|
5542
|
+
if (opts.stats) {
|
|
5543
|
+
opts.stats.pages += 1;
|
|
5544
|
+
opts.stats.bytes = bytes;
|
|
5545
|
+
}
|
|
5546
|
+
if (!ok) {
|
|
5547
|
+
if (opts.stats) {
|
|
5548
|
+
opts.stats.refused = (opts.stats.refused ?? 0) + 1;
|
|
5549
|
+
if (opts.stats.reasons) tally(opts.stats.reasons, pageFailure(status));
|
|
5550
|
+
}
|
|
5551
|
+
if (++refusedRun >= giveUpAt) givenUp = true;
|
|
5552
|
+
return;
|
|
5553
|
+
}
|
|
5554
|
+
refusedRun = 0;
|
|
5555
|
+
for (const p of productsFromPage(text, url)) {
|
|
5556
|
+
if (seen.has(p.externalKey)) continue;
|
|
5557
|
+
seen.add(p.externalKey);
|
|
5558
|
+
kept++;
|
|
5559
|
+
if (opts.onEach) opts.onEach(p);
|
|
5560
|
+
else out.push(p);
|
|
5561
|
+
}
|
|
5562
|
+
opts.onProduct?.(kept);
|
|
5563
|
+
} catch (err) {
|
|
5564
|
+
if (opts.stats) {
|
|
5565
|
+
opts.stats.refused = (opts.stats.refused ?? 0) + 1;
|
|
5566
|
+
if (opts.stats.reasons) tally(opts.stats.reasons, thrownFailure(err, ctx.signal?.aborted));
|
|
5567
|
+
}
|
|
5568
|
+
}
|
|
5569
|
+
},
|
|
5570
|
+
ctx.signal
|
|
5571
|
+
);
|
|
5572
|
+
return out;
|
|
5573
|
+
}
|
|
3905
5574
|
|
|
3906
5575
|
// ../catalog/src/adapters/shopify.ts
|
|
3907
5576
|
function mapShopifyProduct(base, p) {
|
|
@@ -3955,10 +5624,14 @@ function mapShopifyProduct(base, p) {
|
|
|
3955
5624
|
async function fetchProductsJsonPage(ctx, page, limit = 250) {
|
|
3956
5625
|
const origin = originOf(ctx.baseUrl);
|
|
3957
5626
|
const url = `${origin}/products.json?limit=${limit}&page=${page}`;
|
|
3958
|
-
const { ok, json } = await httpJson(url, {
|
|
3959
|
-
|
|
3960
|
-
|
|
5627
|
+
const { ok, status, json } = await httpJson(url, {
|
|
5628
|
+
fetchImpl: ctx.fetchImpl,
|
|
5629
|
+
signal: ctx.signal
|
|
5630
|
+
});
|
|
5631
|
+
if (ok && json?.products) return { products: json.products, blocked: false };
|
|
5632
|
+
return { products: [], blocked: status === 401 || status === 403 || status === 404 || status >= 500 };
|
|
3961
5633
|
}
|
|
5634
|
+
var JSON_BLOCKED = "json-blocked";
|
|
3962
5635
|
async function collectSitemapProductUrls(ctx) {
|
|
3963
5636
|
const origin = originOf(ctx.baseUrl);
|
|
3964
5637
|
const urls = /* @__PURE__ */ new Set();
|
|
@@ -3984,7 +5657,7 @@ async function collectSitemapProductUrls(ctx) {
|
|
|
3984
5657
|
}
|
|
3985
5658
|
}
|
|
3986
5659
|
}
|
|
3987
|
-
return [...urls];
|
|
5660
|
+
return preferCanonicalLocale([...urls]);
|
|
3988
5661
|
}
|
|
3989
5662
|
var shopifyAdapter = {
|
|
3990
5663
|
platform: "shopify",
|
|
@@ -4016,6 +5689,7 @@ var shopifyAdapter = {
|
|
|
4016
5689
|
},
|
|
4017
5690
|
async discover(ctx) {
|
|
4018
5691
|
const warnings = [];
|
|
5692
|
+
const hints = [];
|
|
4019
5693
|
const keys = /* @__PURE__ */ new Set();
|
|
4020
5694
|
const productUrls = /* @__PURE__ */ new Set();
|
|
4021
5695
|
const origin = originOf(ctx.baseUrl);
|
|
@@ -4023,8 +5697,12 @@ var shopifyAdapter = {
|
|
|
4023
5697
|
let emptyStreak = 0;
|
|
4024
5698
|
while (emptyStreak < 1) {
|
|
4025
5699
|
if (ctx.signal?.aborted) throw new Error("aborted");
|
|
4026
|
-
const products = await fetchProductsJsonPage(ctx, page);
|
|
5700
|
+
const { products, blocked: blocked2 } = await fetchProductsJsonPage(ctx, page);
|
|
4027
5701
|
ctx.onProgress?.({ stage: "discovering", discovered: keys.size, message: `Shopify page ${page}` });
|
|
5702
|
+
if (blocked2 && page === 1) {
|
|
5703
|
+
hints.push(JSON_BLOCKED);
|
|
5704
|
+
warnings.push("This store does not serve its product API, so the product pages were read instead");
|
|
5705
|
+
}
|
|
4028
5706
|
if (!products.length) {
|
|
4029
5707
|
emptyStreak++;
|
|
4030
5708
|
break;
|
|
@@ -4053,17 +5731,23 @@ var shopifyAdapter = {
|
|
|
4053
5731
|
productKeys: [...keys],
|
|
4054
5732
|
productUrls: [...productUrls],
|
|
4055
5733
|
estimatedTotal: keys.size || productUrls.size || null,
|
|
4056
|
-
|
|
5734
|
+
// A store whose own product API answered but refused us leaves nothing
|
|
5735
|
+
// to read but the pages themselves, one request each. That is what
|
|
5736
|
+
// gymshark.com does, and it is the run worth batching.
|
|
5737
|
+
byPage: hints.includes(JSON_BLOCKED),
|
|
5738
|
+
warnings,
|
|
5739
|
+
hints
|
|
4057
5740
|
};
|
|
4058
5741
|
},
|
|
4059
5742
|
async fetchAll(ctx, discovered) {
|
|
4060
5743
|
const origin = originOf(ctx.baseUrl);
|
|
4061
5744
|
const out = [];
|
|
4062
5745
|
const seen = /* @__PURE__ */ new Set();
|
|
5746
|
+
const jsonBlocked = discovered.hints?.includes(JSON_BLOCKED) ?? false;
|
|
4063
5747
|
let page = 1;
|
|
4064
|
-
while (
|
|
5748
|
+
while (!jsonBlocked) {
|
|
4065
5749
|
if (ctx.signal?.aborted) throw new Error("aborted");
|
|
4066
|
-
const products = await fetchProductsJsonPage(ctx, page);
|
|
5750
|
+
const { products } = await fetchProductsJsonPage(ctx, page);
|
|
4067
5751
|
if (!products.length) break;
|
|
4068
5752
|
for (const p of products) {
|
|
4069
5753
|
const mapped = mapShopifyProduct(origin, p);
|
|
@@ -4080,48 +5764,56 @@ var shopifyAdapter = {
|
|
|
4080
5764
|
page++;
|
|
4081
5765
|
if (page > 1e4) break;
|
|
4082
5766
|
}
|
|
4083
|
-
const
|
|
4084
|
-
const
|
|
4085
|
-
return
|
|
5767
|
+
const handleOf = (u) => {
|
|
5768
|
+
const raw = /\/products\/([^/?#]+)/i.exec(u)?.[1];
|
|
5769
|
+
return raw ? decodeURIComponent(raw) : null;
|
|
5770
|
+
};
|
|
5771
|
+
const stillMissing = () => discovered.productUrls.filter((u) => {
|
|
5772
|
+
const handle = handleOf(u);
|
|
5773
|
+
return handle && !out.some((p) => p.handle === handle);
|
|
4086
5774
|
});
|
|
4087
|
-
if (
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
if (
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
5775
|
+
if (!jsonBlocked) {
|
|
5776
|
+
const missingUrls = stillMissing();
|
|
5777
|
+
if (missingUrls.length) {
|
|
5778
|
+
await mapPool(
|
|
5779
|
+
missingUrls,
|
|
5780
|
+
6,
|
|
5781
|
+
async (u) => {
|
|
5782
|
+
const handle = handleOf(u);
|
|
5783
|
+
if (!handle) return;
|
|
5784
|
+
const { ok, json } = await httpJson(`${origin}/products/${handle}.json`, {
|
|
5785
|
+
fetchImpl: ctx.fetchImpl,
|
|
5786
|
+
signal: ctx.signal
|
|
5787
|
+
});
|
|
5788
|
+
if (ok && json?.product) {
|
|
5789
|
+
const mapped = mapShopifyProduct(origin, json.product);
|
|
5790
|
+
if (!seen.has(mapped.externalKey)) {
|
|
5791
|
+
seen.add(mapped.externalKey);
|
|
5792
|
+
out.push(mapped);
|
|
5793
|
+
ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
|
|
5794
|
+
}
|
|
4104
5795
|
}
|
|
4105
|
-
}
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
5796
|
+
},
|
|
5797
|
+
ctx.signal
|
|
5798
|
+
);
|
|
5799
|
+
}
|
|
5800
|
+
}
|
|
5801
|
+
const unread = stillMissing();
|
|
5802
|
+
if (unread.length) {
|
|
5803
|
+
for (const p of await fetchProductPages(ctx, unread, {
|
|
5804
|
+
concurrency: 4,
|
|
5805
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched: out.length + fetched })
|
|
5806
|
+
})) {
|
|
5807
|
+
if (seen.has(p.externalKey)) continue;
|
|
5808
|
+
seen.add(p.externalKey);
|
|
5809
|
+
out.push(p);
|
|
5810
|
+
}
|
|
4109
5811
|
}
|
|
4110
5812
|
return out;
|
|
4111
5813
|
}
|
|
4112
5814
|
};
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
}
|
|
4116
|
-
function attr(el, name) {
|
|
4117
|
-
return el?.getAttribute(name) ?? void 0;
|
|
4118
|
-
}
|
|
4119
|
-
function textOf(el) {
|
|
4120
|
-
return (el?.text ?? "").replace(/\s+/g, " ").trim();
|
|
4121
|
-
}
|
|
4122
|
-
function stableKey(url) {
|
|
4123
|
-
return createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
4124
|
-
}
|
|
5815
|
+
|
|
5816
|
+
// ../catalog/src/adapters/generic.ts
|
|
4125
5817
|
async function extractSitemapUrls(ctx, filter = () => true) {
|
|
4126
5818
|
const origin = originOf(ctx.baseUrl);
|
|
4127
5819
|
const out = /* @__PURE__ */ new Set();
|
|
@@ -4153,89 +5845,7 @@ async function extractSitemapUrls(ctx, filter = () => true) {
|
|
|
4153
5845
|
}
|
|
4154
5846
|
if (seen.size > 200) break;
|
|
4155
5847
|
}
|
|
4156
|
-
return [...out];
|
|
4157
|
-
}
|
|
4158
|
-
function walkJsonLd(node, out) {
|
|
4159
|
-
if (!node) return;
|
|
4160
|
-
if (Array.isArray(node)) {
|
|
4161
|
-
for (const n of node) walkJsonLd(n, out);
|
|
4162
|
-
return;
|
|
4163
|
-
}
|
|
4164
|
-
if (typeof node !== "object") return;
|
|
4165
|
-
const obj = node;
|
|
4166
|
-
const type = obj["@type"];
|
|
4167
|
-
const types = Array.isArray(type) ? type : type ? [type] : [];
|
|
4168
|
-
if (types.some((t) => String(t).toLowerCase() === "product")) out.push(obj);
|
|
4169
|
-
if (obj["@graph"]) walkJsonLd(obj["@graph"], out);
|
|
4170
|
-
for (const v of Object.values(obj)) {
|
|
4171
|
-
if (v && typeof v === "object") walkJsonLd(v, out);
|
|
4172
|
-
}
|
|
4173
|
-
}
|
|
4174
|
-
function extractJsonLdProducts(html, pageUrl) {
|
|
4175
|
-
const root = loadHtml(html);
|
|
4176
|
-
const nodes = [];
|
|
4177
|
-
for (const el of root.querySelectorAll('script[type="application/ld+json"]')) {
|
|
4178
|
-
const raw = el.innerHTML;
|
|
4179
|
-
if (!raw) continue;
|
|
4180
|
-
try {
|
|
4181
|
-
walkJsonLd(JSON.parse(raw), nodes);
|
|
4182
|
-
} catch {
|
|
4183
|
-
}
|
|
4184
|
-
}
|
|
4185
|
-
return nodes.map((n) => {
|
|
4186
|
-
const offers = Array.isArray(n.offers) ? n.offers[0] : n.offers;
|
|
4187
|
-
const images = [].concat(n.image ?? []).flat().map((img) => typeof img === "string" ? img : img?.url ?? img?.contentUrl).filter(Boolean).map((u, i) => ({ url: absolutize(pageUrl, String(u)), position: i, width: null, height: null, alt: null })).filter((img) => img.url);
|
|
4188
|
-
const url = String(n.url ?? n["@id"] ?? pageUrl);
|
|
4189
|
-
return normalizeProduct({
|
|
4190
|
-
externalKey: String(n.sku || n.productID || n.mpn || stableKey(url)),
|
|
4191
|
-
title: String(n.name ?? "Product"),
|
|
4192
|
-
descriptionHtml: n.description ? String(n.description) : null,
|
|
4193
|
-
url: absolutize(pageUrl, url) ?? pageUrl,
|
|
4194
|
-
vendor: n.brand?.name ?? (typeof n.brand === "string" ? n.brand : null),
|
|
4195
|
-
productType: n.category ? String(n.category) : null,
|
|
4196
|
-
category: n.category ? String(n.category) : null,
|
|
4197
|
-
price: offers?.price != null ? Number(offers.price) : null,
|
|
4198
|
-
compareAtPrice: null,
|
|
4199
|
-
currency: offers?.priceCurrency ?? null,
|
|
4200
|
-
available: offers?.availability ? /instock/i.test(String(offers.availability)) : null,
|
|
4201
|
-
tags: [],
|
|
4202
|
-
variants: [],
|
|
4203
|
-
images,
|
|
4204
|
-
raw: n
|
|
4205
|
-
});
|
|
4206
|
-
});
|
|
4207
|
-
}
|
|
4208
|
-
function parseProductHtml(html, pageUrl) {
|
|
4209
|
-
const $ = loadHtml(html);
|
|
4210
|
-
const title = attr($.querySelector('meta[property="og:title"]'), "content") || textOf($.querySelector("h1")) || textOf($.querySelector("title"));
|
|
4211
|
-
if (!title) return null;
|
|
4212
|
-
const desc = attr($.querySelector('meta[property="og:description"]'), "content") || attr($.querySelector('meta[name="description"]'), "content") || null;
|
|
4213
|
-
const images = [];
|
|
4214
|
-
const og = attr($.querySelector('meta[property="og:image"]'), "content");
|
|
4215
|
-
if (og) {
|
|
4216
|
-
const abs = absolutize(pageUrl, og);
|
|
4217
|
-
if (abs) images.push({ url: abs, position: 0, width: null, height: null, alt: null });
|
|
4218
|
-
}
|
|
4219
|
-
for (const el of $.querySelectorAll("img[src]")) {
|
|
4220
|
-
if (images.length >= 12) break;
|
|
4221
|
-
const src = attr(el, "src") || attr(el, "data-src");
|
|
4222
|
-
const abs = src ? absolutize(pageUrl, src) : null;
|
|
4223
|
-
if (!abs || /logo|icon|sprite|pixel|avatar/i.test(abs)) continue;
|
|
4224
|
-
if (images.some((x) => x.url === abs)) continue;
|
|
4225
|
-
images.push({ url: abs, position: images.length, width: null, height: null, alt: attr(el, "alt") ?? null });
|
|
4226
|
-
}
|
|
4227
|
-
const canonical = attr($.querySelector('link[rel="canonical"]'), "href");
|
|
4228
|
-
const url = canonical ? absolutize(pageUrl, canonical) ?? pageUrl : pageUrl;
|
|
4229
|
-
return normalizeProduct({
|
|
4230
|
-
externalKey: stableKey(url),
|
|
4231
|
-
title,
|
|
4232
|
-
descriptionHtml: desc,
|
|
4233
|
-
url,
|
|
4234
|
-
images,
|
|
4235
|
-
variants: [],
|
|
4236
|
-
tags: [],
|
|
4237
|
-
raw: { source: "html" }
|
|
4238
|
-
});
|
|
5848
|
+
return preferCanonicalLocale([...out]);
|
|
4239
5849
|
}
|
|
4240
5850
|
async function extractFeedUrls(ctx) {
|
|
4241
5851
|
const origin = originOf(ctx.baseUrl);
|
|
@@ -4339,41 +5949,15 @@ var genericAdapter = {
|
|
|
4339
5949
|
productKeys: [...productUrls].map(stableKey),
|
|
4340
5950
|
productUrls: [...productUrls],
|
|
4341
5951
|
estimatedTotal: productUrls.size || null,
|
|
5952
|
+
// Nothing here but addresses: every product is its own page fetch.
|
|
5953
|
+
byPage: true,
|
|
4342
5954
|
warnings
|
|
4343
5955
|
};
|
|
4344
5956
|
},
|
|
4345
5957
|
async fetchAll(ctx, discovered) {
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
discovered.productUrls,
|
|
4350
|
-
5,
|
|
4351
|
-
async (u) => {
|
|
4352
|
-
try {
|
|
4353
|
-
const { ok, text, url } = await httpText(u, {
|
|
4354
|
-
fetchImpl: ctx.fetchImpl,
|
|
4355
|
-
signal: ctx.signal,
|
|
4356
|
-
accept: "text/html"
|
|
4357
|
-
});
|
|
4358
|
-
if (!ok) return;
|
|
4359
|
-
const fromLd = extractJsonLdProducts(text, url);
|
|
4360
|
-
const list2 = fromLd.length ? fromLd : [parseProductHtml(text, url)].filter(Boolean);
|
|
4361
|
-
for (const p of list2) {
|
|
4362
|
-
if (seen.has(p.externalKey)) continue;
|
|
4363
|
-
seen.add(p.externalKey);
|
|
4364
|
-
out.push(p);
|
|
4365
|
-
}
|
|
4366
|
-
ctx.onProgress?.({
|
|
4367
|
-
stage: "fetching_products",
|
|
4368
|
-
fetched: out.length,
|
|
4369
|
-
discovered: discovered.productUrls.length
|
|
4370
|
-
});
|
|
4371
|
-
} catch {
|
|
4372
|
-
}
|
|
4373
|
-
},
|
|
4374
|
-
ctx.signal
|
|
4375
|
-
);
|
|
4376
|
-
return out;
|
|
5958
|
+
return fetchProductPages(ctx, discovered.productUrls, {
|
|
5959
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched, discovered: discovered.productUrls.length })
|
|
5960
|
+
});
|
|
4377
5961
|
}
|
|
4378
5962
|
};
|
|
4379
5963
|
|
|
@@ -4481,6 +6065,9 @@ var woocommerceAdapter = {
|
|
|
4481
6065
|
productKeys: [...keys],
|
|
4482
6066
|
productUrls: [...productUrls],
|
|
4483
6067
|
estimatedTotal: keys.size || productUrls.size || null,
|
|
6068
|
+
// The WooCommerce store API answers in pages of a hundred, so the whole
|
|
6069
|
+
// catalogue is a handful of requests and needs no batching of ours.
|
|
6070
|
+
byPage: false,
|
|
4484
6071
|
warnings
|
|
4485
6072
|
};
|
|
4486
6073
|
},
|
|
@@ -4512,30 +6099,13 @@ var woocommerceAdapter = {
|
|
|
4512
6099
|
}
|
|
4513
6100
|
if (apiWorked && out.length) return out;
|
|
4514
6101
|
const urls = discovered.productUrls.length ? discovered.productUrls : [...new Set(discovered.productKeys)];
|
|
4515
|
-
await
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
text,
|
|
4523
|
-
url: finalUrl
|
|
4524
|
-
} = await httpText(u, { fetchImpl: ctx.fetchImpl, signal: ctx.signal, accept: "text/html" });
|
|
4525
|
-
if (!ok) return;
|
|
4526
|
-
const fromLd = extractJsonLdProducts(text, finalUrl);
|
|
4527
|
-
const products = fromLd.length ? fromLd : [parseProductHtml(text, finalUrl)].filter(Boolean);
|
|
4528
|
-
for (const p of products) {
|
|
4529
|
-
if (seen.has(p.externalKey)) continue;
|
|
4530
|
-
seen.add(p.externalKey);
|
|
4531
|
-
out.push(p);
|
|
4532
|
-
}
|
|
4533
|
-
ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
|
|
4534
|
-
} catch {
|
|
4535
|
-
}
|
|
4536
|
-
},
|
|
4537
|
-
ctx.signal
|
|
4538
|
-
);
|
|
6102
|
+
for (const p of await fetchProductPages(ctx, urls, {
|
|
6103
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched: out.length + fetched })
|
|
6104
|
+
})) {
|
|
6105
|
+
if (seen.has(p.externalKey)) continue;
|
|
6106
|
+
seen.add(p.externalKey);
|
|
6107
|
+
out.push(p);
|
|
6108
|
+
}
|
|
4539
6109
|
return out;
|
|
4540
6110
|
}
|
|
4541
6111
|
};
|
|
@@ -4603,42 +6173,16 @@ var webflowAdapter = {
|
|
|
4603
6173
|
return {
|
|
4604
6174
|
productKeys: [...productUrls],
|
|
4605
6175
|
productUrls: [...productUrls],
|
|
4606
|
-
estimatedTotal: productUrls.size || null,
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
discovered.productUrls
|
|
4615
|
-
|
|
4616
|
-
async (u) => {
|
|
4617
|
-
try {
|
|
4618
|
-
const { ok, text, url } = await httpText(u, {
|
|
4619
|
-
fetchImpl: ctx.fetchImpl,
|
|
4620
|
-
signal: ctx.signal,
|
|
4621
|
-
accept: "text/html"
|
|
4622
|
-
});
|
|
4623
|
-
if (!ok) return;
|
|
4624
|
-
const fromLd = extractJsonLdProducts(text, url);
|
|
4625
|
-
const list2 = fromLd.length ? fromLd : [parseProductHtml(text, url)].filter(Boolean);
|
|
4626
|
-
for (const p of list2) {
|
|
4627
|
-
if (seen.has(p.externalKey)) continue;
|
|
4628
|
-
seen.add(p.externalKey);
|
|
4629
|
-
out.push(p);
|
|
4630
|
-
}
|
|
4631
|
-
ctx.onProgress?.({
|
|
4632
|
-
stage: "fetching_products",
|
|
4633
|
-
fetched: out.length,
|
|
4634
|
-
discovered: discovered.productUrls.length
|
|
4635
|
-
});
|
|
4636
|
-
} catch {
|
|
4637
|
-
}
|
|
4638
|
-
},
|
|
4639
|
-
ctx.signal
|
|
4640
|
-
);
|
|
4641
|
-
return out;
|
|
6176
|
+
estimatedTotal: productUrls.size || null,
|
|
6177
|
+
// Webflow has no catalogue API to ask: every product is its own page.
|
|
6178
|
+
byPage: true,
|
|
6179
|
+
warnings
|
|
6180
|
+
};
|
|
6181
|
+
},
|
|
6182
|
+
async fetchAll(ctx, discovered) {
|
|
6183
|
+
return fetchProductPages(ctx, discovered.productUrls, {
|
|
6184
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched, discovered: discovered.productUrls.length })
|
|
6185
|
+
});
|
|
4642
6186
|
}
|
|
4643
6187
|
};
|
|
4644
6188
|
|
|
@@ -4680,7 +6224,7 @@ function baseProgress(platform = "unknown") {
|
|
|
4680
6224
|
warnings: []
|
|
4681
6225
|
};
|
|
4682
6226
|
}
|
|
4683
|
-
async function
|
|
6227
|
+
async function discoverCatalog(opts) {
|
|
4684
6228
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
4685
6229
|
const progress = baseProgress();
|
|
4686
6230
|
const emit = (patch2) => {
|
|
@@ -4723,57 +6267,247 @@ async function runCatalogIngestion(opts) {
|
|
|
4723
6267
|
warnings: [...progress.warnings, ...discovered.warnings],
|
|
4724
6268
|
message: `Found ${discovered.estimatedTotal ?? discovered.productUrls.length} products`
|
|
4725
6269
|
});
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
}
|
|
4735
|
-
]
|
|
4736
|
-
});
|
|
4737
|
-
return { baseUrl: detection.baseUrl, detection, products: [], progress };
|
|
6270
|
+
const estimatedTotal = discovered.estimatedTotal ?? discovered.productUrls.length;
|
|
6271
|
+
let empty = null;
|
|
6272
|
+
if (!estimatedTotal) {
|
|
6273
|
+
empty = {
|
|
6274
|
+
code: "empty_catalog",
|
|
6275
|
+
message: detection.platform === "generic" ? "No public product catalog found. This store may be JavaScript-rendered or require authentication." : `No products discovered on this ${detection.platform} store.`
|
|
6276
|
+
};
|
|
6277
|
+
emit({ stage: "failed", errors: [...progress.errors, empty] });
|
|
4738
6278
|
}
|
|
4739
|
-
|
|
4740
|
-
|
|
6279
|
+
return {
|
|
6280
|
+
baseUrl: detection.baseUrl,
|
|
6281
|
+
detection,
|
|
6282
|
+
adapter,
|
|
6283
|
+
ctx: { ...ctx, baseUrl: detection.baseUrl },
|
|
6284
|
+
productUrls: discovered.productUrls,
|
|
6285
|
+
productKeys: discovered.productKeys,
|
|
6286
|
+
byPage: (discovered.byPage ?? false) && discovered.productUrls.length > 0,
|
|
6287
|
+
estimatedTotal,
|
|
6288
|
+
progress,
|
|
6289
|
+
empty,
|
|
6290
|
+
emit,
|
|
6291
|
+
async fetchAll() {
|
|
6292
|
+
emit({ stage: "fetching_products", message: "Fetching product details" });
|
|
6293
|
+
try {
|
|
6294
|
+
return dedupeProducts(await adapter.fetchAll({ ...ctx, baseUrl: detection.baseUrl }, discovered));
|
|
6295
|
+
} catch (err) {
|
|
6296
|
+
emit({
|
|
6297
|
+
stage: "partial",
|
|
6298
|
+
errors: [...progress.errors, { code: "fetch_failed", message: String(err?.message ?? err), retryable: true }]
|
|
6299
|
+
});
|
|
6300
|
+
return [];
|
|
6301
|
+
}
|
|
6302
|
+
}
|
|
6303
|
+
};
|
|
6304
|
+
}
|
|
6305
|
+
|
|
6306
|
+
// ../catalog/src/robots.ts
|
|
6307
|
+
var ALLOW_ALL = { rules: [], crawlDelayMs: 0 };
|
|
6308
|
+
function toRegExp(pattern) {
|
|
6309
|
+
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
6310
|
+
const anchored = escaped.endsWith("\\$") ? `${escaped.slice(0, -2)}$` : escaped;
|
|
6311
|
+
return new RegExp(`^${anchored}`);
|
|
6312
|
+
}
|
|
6313
|
+
function parseRobots(text, agent = USER_AGENT2) {
|
|
6314
|
+
const lines = text.split(/\r?\n/).map((l) => l.replace(/#.*$/, "").trim());
|
|
6315
|
+
const groups = [];
|
|
6316
|
+
let current = null;
|
|
6317
|
+
let expectingAgents = false;
|
|
6318
|
+
for (const line of lines) {
|
|
6319
|
+
const at = line.indexOf(":");
|
|
6320
|
+
if (at < 0) continue;
|
|
6321
|
+
const field = line.slice(0, at).trim().toLowerCase();
|
|
6322
|
+
const value = line.slice(at + 1).trim();
|
|
6323
|
+
if (field === "user-agent") {
|
|
6324
|
+
if (!current || !expectingAgents) {
|
|
6325
|
+
current = { agents: [], rules: [], delay: 0 };
|
|
6326
|
+
groups.push(current);
|
|
6327
|
+
expectingAgents = true;
|
|
6328
|
+
}
|
|
6329
|
+
current.agents.push(value.toLowerCase());
|
|
6330
|
+
continue;
|
|
6331
|
+
}
|
|
6332
|
+
if (!current) continue;
|
|
6333
|
+
expectingAgents = false;
|
|
6334
|
+
if (field === "disallow" && value) current.rules.push({ allow: false, pattern: value });
|
|
6335
|
+
else if (field === "allow" && value) current.rules.push({ allow: true, pattern: value });
|
|
6336
|
+
else if (field === "crawl-delay") {
|
|
6337
|
+
const n = Number(value);
|
|
6338
|
+
if (Number.isFinite(n) && n > 0) current.delay = n;
|
|
6339
|
+
}
|
|
6340
|
+
}
|
|
6341
|
+
const token = agent.toLowerCase();
|
|
6342
|
+
const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && token.includes(a)));
|
|
6343
|
+
const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
|
|
6344
|
+
return {
|
|
6345
|
+
rules: chosen.flatMap((g) => g.rules),
|
|
6346
|
+
crawlDelayMs: Math.max(0, ...chosen.map((g) => g.delay)) * 1e3
|
|
6347
|
+
};
|
|
6348
|
+
}
|
|
6349
|
+
function isAllowed(robots, url) {
|
|
6350
|
+
let path;
|
|
4741
6351
|
try {
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
errors: [...progress.errors, { code: "fetch_failed", message: String(err?.message ?? err), retryable: true }]
|
|
4747
|
-
});
|
|
6352
|
+
const parsed = new URL(url);
|
|
6353
|
+
path = `${parsed.pathname}${parsed.search}`;
|
|
6354
|
+
} catch {
|
|
6355
|
+
return true;
|
|
4748
6356
|
}
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
6357
|
+
let best = null;
|
|
6358
|
+
for (const rule of robots.rules) {
|
|
6359
|
+
if (!toRegExp(rule.pattern).test(path)) continue;
|
|
6360
|
+
const length = rule.pattern.replace(/[*$]/g, "").length;
|
|
6361
|
+
if (!best || length > best.length || length === best.length && rule.allow) {
|
|
6362
|
+
best = { allow: rule.allow, length };
|
|
6363
|
+
}
|
|
6364
|
+
}
|
|
6365
|
+
return best ? best.allow : true;
|
|
6366
|
+
}
|
|
6367
|
+
async function fetchRobots(ctx) {
|
|
6368
|
+
try {
|
|
6369
|
+
const { ok, text } = await httpText(`${originOf(ctx.baseUrl)}/robots.txt`, {
|
|
6370
|
+
fetchImpl: ctx.fetchImpl,
|
|
6371
|
+
signal: ctx.signal,
|
|
6372
|
+
retries: 0,
|
|
6373
|
+
accept: "text/plain",
|
|
6374
|
+
maxBytes: 512e3
|
|
4763
6375
|
});
|
|
6376
|
+
if (!ok || !text) return ALLOW_ALL;
|
|
6377
|
+
return parseRobots(text);
|
|
6378
|
+
} catch {
|
|
6379
|
+
return ALLOW_ALL;
|
|
6380
|
+
}
|
|
6381
|
+
}
|
|
6382
|
+
|
|
6383
|
+
// ../catalog/src/candidates.ts
|
|
6384
|
+
var DEFAULT_SCAN_BUDGET = {
|
|
6385
|
+
maxPreviewPages: 24,
|
|
6386
|
+
maxBytesPerPage: 15e5,
|
|
6387
|
+
maxTotalBytes: 4e7,
|
|
6388
|
+
budgetMs: 25e3,
|
|
6389
|
+
concurrency: 4,
|
|
6390
|
+
/**
|
|
6391
|
+
* Time the preview is owed even when discovery has already spent the lot.
|
|
6392
|
+
*
|
|
6393
|
+
* Discovery on a large store can run long, and when it overran the whole
|
|
6394
|
+
* budget the preview read zero pages and the store was reported as one we
|
|
6395
|
+
* could not open - from a site that was answering every request with a 200.
|
|
6396
|
+
* Being slow to list a catalog is not the same as being shut.
|
|
6397
|
+
*/
|
|
6398
|
+
previewFloorMs: 12e3
|
|
6399
|
+
};
|
|
6400
|
+
var COMMERCE_SIGNALS = /* @__PURE__ */ new Set([
|
|
6401
|
+
"products.json",
|
|
6402
|
+
"shopify-html",
|
|
6403
|
+
"wc-store-api",
|
|
6404
|
+
"woocommerce-html",
|
|
6405
|
+
"webflow-commerce"
|
|
6406
|
+
]);
|
|
6407
|
+
function wwwVariant(baseUrl) {
|
|
6408
|
+
try {
|
|
6409
|
+
const url = new URL(baseUrl);
|
|
6410
|
+
const labels = url.hostname.split(".");
|
|
6411
|
+
if (labels.length !== 2 || /^\d+$/.test(labels[labels.length - 1])) return null;
|
|
6412
|
+
url.hostname = `www.${url.hostname}`;
|
|
6413
|
+
return url.origin;
|
|
6414
|
+
} catch {
|
|
6415
|
+
return null;
|
|
6416
|
+
}
|
|
6417
|
+
}
|
|
6418
|
+
async function scanForCandidates(opts) {
|
|
6419
|
+
const first = await scanOnce(opts, originOf(normalizeStoreUrl(opts.url)));
|
|
6420
|
+
if (first.verdict === "found") return first;
|
|
6421
|
+
const alternate = wwwVariant(first.baseUrl);
|
|
6422
|
+
if (!alternate) return first;
|
|
6423
|
+
const second = await scanOnce(opts, alternate);
|
|
6424
|
+
return second.verdict === "found" ? second : first;
|
|
6425
|
+
}
|
|
6426
|
+
async function scanOnce(opts, baseUrl) {
|
|
6427
|
+
const started = Date.now();
|
|
6428
|
+
const budget = { ...DEFAULT_SCAN_BUDGET, ...opts.budget };
|
|
6429
|
+
const deadline = started + budget.budgetMs;
|
|
6430
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
6431
|
+
const ctx = {
|
|
6432
|
+
fetchImpl,
|
|
6433
|
+
baseUrl,
|
|
6434
|
+
signal: opts.signal,
|
|
6435
|
+
onProgress: opts.onProgress
|
|
6436
|
+
};
|
|
6437
|
+
const warnings = [];
|
|
6438
|
+
const stats = { pages: 0, bytes: 0 };
|
|
6439
|
+
const robots = await fetchRobots(ctx);
|
|
6440
|
+
if (robots.crawlDelayMs) warnings.push("This site asks readers to go slowly, so the preview is smaller");
|
|
6441
|
+
opts.onProgress?.({ stage: "discovering", message: "Looking for a shop" });
|
|
6442
|
+
const detection = await detectPlatform(ctx);
|
|
6443
|
+
const adapter = adapterFor(detection.platform);
|
|
6444
|
+
let discovered;
|
|
6445
|
+
try {
|
|
6446
|
+
discovered = await adapter.discover(ctx);
|
|
6447
|
+
} catch {
|
|
6448
|
+
return done("blocked", [], [], 0, "none", [...warnings, "The product list could not be read"]);
|
|
6449
|
+
}
|
|
6450
|
+
warnings.push(...discovered.warnings);
|
|
6451
|
+
const urls = discovered.productUrls.filter((u) => isAllowed(robots, u));
|
|
6452
|
+
const refused = discovered.productUrls.length - urls.length;
|
|
6453
|
+
if (refused > 0) warnings.push(`${refused} pages are disallowed by this site's robots.txt`);
|
|
6454
|
+
const fromApi = detection.platform !== "generic" && !discovered.hints?.includes("json-blocked");
|
|
6455
|
+
const countSource = urls.length === 0 ? "none" : fromApi ? "api" : "sitemap";
|
|
6456
|
+
if (urls.length === 0) {
|
|
6457
|
+
const commerce = detection.signals.some((sig) => COMMERCE_SIGNALS.has(sig));
|
|
6458
|
+
return done(commerce ? "likely" : "none", [], [], 0, countSource, warnings);
|
|
6459
|
+
}
|
|
6460
|
+
opts.onProgress?.({ stage: "discovering", discovered: urls.length, message: "Reading a few products" });
|
|
6461
|
+
const previewDeadline = Math.max(deadline, Date.now() + budget.previewFloorMs);
|
|
6462
|
+
const read = dedupeProducts(
|
|
6463
|
+
await fetchProductPages(ctx, urls, {
|
|
6464
|
+
want: budget.maxPreviewPages,
|
|
6465
|
+
stats,
|
|
6466
|
+
concurrency: budget.concurrency,
|
|
6467
|
+
maxBytes: budget.maxBytesPerPage,
|
|
6468
|
+
maxTotalBytes: budget.maxTotalBytes,
|
|
6469
|
+
delayMs: robots.crawlDelayMs,
|
|
6470
|
+
deadline: previewDeadline,
|
|
6471
|
+
onProduct: (fetched) => opts.onProgress?.({ stage: "fetching_products", fetched, discovered: urls.length })
|
|
6472
|
+
})
|
|
6473
|
+
);
|
|
6474
|
+
const preview = read.slice(0, budget.maxPreviewPages);
|
|
6475
|
+
const verdict = preview.length ? "found" : "blocked";
|
|
6476
|
+
return done(verdict, preview, urls, urls.length, countSource, warnings);
|
|
6477
|
+
function done(verdict2, candidates, candidateUrls, count, source, notes) {
|
|
6478
|
+
return {
|
|
6479
|
+
baseUrl,
|
|
6480
|
+
platform: detection?.platform ?? "unknown",
|
|
6481
|
+
signals: detection?.signals ?? [],
|
|
6482
|
+
verdict: verdict2,
|
|
6483
|
+
count,
|
|
6484
|
+
countSource: source,
|
|
6485
|
+
candidates,
|
|
6486
|
+
candidateUrls,
|
|
6487
|
+
truncated: count > candidates.length,
|
|
6488
|
+
warnings: notes,
|
|
6489
|
+
spent: { pages: stats.pages, bytes: stats.bytes, ms: Date.now() - started }
|
|
6490
|
+
};
|
|
4764
6491
|
}
|
|
4765
|
-
return { baseUrl: detection.baseUrl, detection, products, progress };
|
|
4766
6492
|
}
|
|
4767
6493
|
|
|
4768
6494
|
// src/catalogImport.ts
|
|
4769
6495
|
var running = /* @__PURE__ */ new Map();
|
|
4770
6496
|
function cancelCatalogImport(jobId) {
|
|
4771
|
-
const
|
|
4772
|
-
if (!
|
|
4773
|
-
ctrl.abort();
|
|
6497
|
+
const job = running.get(jobId);
|
|
6498
|
+
if (!job) return false;
|
|
6499
|
+
job.ctrl.abort();
|
|
4774
6500
|
return true;
|
|
4775
6501
|
}
|
|
4776
|
-
function
|
|
6502
|
+
async function settleCatalogImports(timeoutMs = 5e3) {
|
|
6503
|
+
if (!running.size) return;
|
|
6504
|
+
for (const { ctrl } of running.values()) ctrl.abort();
|
|
6505
|
+
await Promise.race([
|
|
6506
|
+
Promise.allSettled([...running.values()].map((j) => j.done)),
|
|
6507
|
+
new Promise((r) => setTimeout(r, timeoutMs))
|
|
6508
|
+
]);
|
|
6509
|
+
}
|
|
6510
|
+
function startCatalogImport(deps, brandId, url, opts = {}) {
|
|
4777
6511
|
const { core } = deps;
|
|
4778
6512
|
if (!core.store.getBrand(brandId)) throw Object.assign(new Error("brand not found"), { statusCode: 404 });
|
|
4779
6513
|
let normalized;
|
|
@@ -4784,57 +6518,183 @@ function startCatalogImport(deps, brandId, url) {
|
|
|
4784
6518
|
}
|
|
4785
6519
|
const job = core.catalog.createJob({ brandId, url: normalized });
|
|
4786
6520
|
const ctrl = new AbortController();
|
|
4787
|
-
|
|
4788
|
-
void runJob(deps, job.id, brandId, normalized, ctrl.signal).finally(() => {
|
|
6521
|
+
const done = runJob(deps, job.id, brandId, normalized, ctrl.signal, opts.only).finally(() => {
|
|
4789
6522
|
running.delete(job.id);
|
|
4790
6523
|
});
|
|
6524
|
+
void done.catch(() => {
|
|
6525
|
+
});
|
|
6526
|
+
running.set(job.id, { ctrl, done });
|
|
4791
6527
|
return { jobId: job.id };
|
|
4792
6528
|
}
|
|
4793
|
-
|
|
6529
|
+
function progressWriter(patch2) {
|
|
6530
|
+
let lastStage = "";
|
|
6531
|
+
let lastMessage = null;
|
|
6532
|
+
return (p) => {
|
|
6533
|
+
const reported = p.stage === "queued" ? "discovering" : p.stage;
|
|
6534
|
+
const stage = TERMINAL.has(reported) ? "fetching_products" : reported;
|
|
6535
|
+
const message = p.message ?? null;
|
|
6536
|
+
const notable = stage !== lastStage || message !== lastMessage || p.fetched % 10 === 0;
|
|
6537
|
+
if (!notable) return;
|
|
6538
|
+
lastStage = stage;
|
|
6539
|
+
lastMessage = message;
|
|
6540
|
+
patch2({
|
|
6541
|
+
stage,
|
|
6542
|
+
platform: p.platform,
|
|
6543
|
+
discovered: p.discovered,
|
|
6544
|
+
fetched: p.fetched,
|
|
6545
|
+
warnings: p.warnings,
|
|
6546
|
+
errors: p.errors,
|
|
6547
|
+
message
|
|
6548
|
+
});
|
|
6549
|
+
};
|
|
6550
|
+
}
|
|
6551
|
+
async function runJob(deps, jobId, brandId, url, signal, only) {
|
|
4794
6552
|
const { core, fetchImpl } = deps;
|
|
4795
6553
|
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
4796
6554
|
try {
|
|
4797
|
-
|
|
4798
|
-
const
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
6555
|
+
const tally2 = { fetched: 0, upserted: 0, imagesDone: 0, imagesTotal: 0, errors: [], seenKeys: [] };
|
|
6556
|
+
const ctx = { fetchImpl: fetchImpl ?? fetch, baseUrl: url, signal };
|
|
6557
|
+
let urls = [];
|
|
6558
|
+
let bulk = null;
|
|
6559
|
+
let baseUrl;
|
|
6560
|
+
let platform;
|
|
6561
|
+
let discovered;
|
|
6562
|
+
let warnings = [];
|
|
6563
|
+
const sweep = !only?.length;
|
|
6564
|
+
if (only?.length) {
|
|
6565
|
+
patch2({ stage: "fetching_products", message: `Importing ${only.length} products`, discovered: only.length });
|
|
6566
|
+
const detection = await detectPlatform(ctx);
|
|
6567
|
+
urls = only;
|
|
6568
|
+
baseUrl = url;
|
|
6569
|
+
platform = detection.platform;
|
|
6570
|
+
discovered = only.length;
|
|
6571
|
+
} else {
|
|
6572
|
+
patch2({ stage: "discovering", message: "Detecting store platform" });
|
|
6573
|
+
const found = await discoverCatalog({ url, fetchImpl, signal, onProgress: progressWriter(patch2) });
|
|
6574
|
+
baseUrl = found.baseUrl;
|
|
6575
|
+
platform = found.detection.platform;
|
|
6576
|
+
discovered = found.estimatedTotal;
|
|
6577
|
+
warnings = found.progress.warnings;
|
|
6578
|
+
if (found.empty || !found.estimatedTotal) {
|
|
6579
|
+
const source = core.catalog.upsertSource(brandId, found.baseUrl, platform);
|
|
6580
|
+
patch2({ sourceId: source.id, platform });
|
|
4803
6581
|
patch2({
|
|
4804
|
-
stage:
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
errors: p.errors,
|
|
4810
|
-
message: p.message ?? null
|
|
6582
|
+
stage: "no_catalog",
|
|
6583
|
+
errors: [],
|
|
6584
|
+
warnings: found.progress.warnings,
|
|
6585
|
+
message: "No shop found on this site",
|
|
6586
|
+
finished: true
|
|
4811
6587
|
});
|
|
6588
|
+
core.catalog.setSourceStatus(source.id, "empty", true);
|
|
6589
|
+
return;
|
|
4812
6590
|
}
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
6591
|
+
if (!found.byPage) {
|
|
6592
|
+
const products = await found.fetchAll();
|
|
6593
|
+
if (signal.aborted) {
|
|
6594
|
+
patch2({ stage: "cancelled", errors: [], message: "Stopped before anything was saved", finished: true });
|
|
6595
|
+
return;
|
|
6596
|
+
}
|
|
6597
|
+
if (!products.length) {
|
|
6598
|
+
const source = core.catalog.upsertSource(brandId, baseUrl, platform);
|
|
6599
|
+
patch2({ sourceId: source.id, platform });
|
|
6600
|
+
patch2({
|
|
6601
|
+
stage: "failed",
|
|
6602
|
+
errors: found.progress.errors,
|
|
6603
|
+
warnings: found.progress.warnings,
|
|
6604
|
+
message: found.progress.errors[0]?.message ?? "No products imported",
|
|
6605
|
+
finished: true
|
|
6606
|
+
});
|
|
6607
|
+
core.catalog.setSourceStatus(source.id, "failed", true);
|
|
6608
|
+
return;
|
|
6609
|
+
}
|
|
6610
|
+
bulk = products;
|
|
6611
|
+
discovered = found.progress.discovered;
|
|
6612
|
+
warnings = found.progress.warnings;
|
|
6613
|
+
}
|
|
6614
|
+
if (!bulk) {
|
|
6615
|
+
urls = found.productUrls;
|
|
6616
|
+
patch2({ stage: "fetching_products", discovered, message: `Reading ${discovered.toLocaleString()} products` });
|
|
6617
|
+
}
|
|
6618
|
+
}
|
|
6619
|
+
const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered);
|
|
6620
|
+
const pictures = drainPictures(deps, jobId, brandId, tally2, signal);
|
|
6621
|
+
let pictureErrors;
|
|
6622
|
+
let pictureReasons = {};
|
|
6623
|
+
const stats = { pages: 0, bytes: 0, refused: 0, reasons: {} };
|
|
6624
|
+
try {
|
|
6625
|
+
if (bulk) for (const p of bulk) run2.write(p);
|
|
6626
|
+
else
|
|
6627
|
+
await fetchProductPages(ctx, urls, {
|
|
6628
|
+
concurrency: IMPORT_CONCURRENCY,
|
|
6629
|
+
maxBytes: 15e5,
|
|
6630
|
+
onEach: run2.write,
|
|
6631
|
+
stats,
|
|
6632
|
+
// Past this many refusals in a row the store is not going to change
|
|
6633
|
+
// its mind inside this import, and waiting out its cooldown for the
|
|
6634
|
+
// rest of the catalogue is time nobody gets a product for.
|
|
6635
|
+
giveUpAfterRefusals: 24
|
|
6636
|
+
});
|
|
6637
|
+
if (!signal.aborted && tally2.upserted) {
|
|
6638
|
+
patch2({
|
|
6639
|
+
stage: "processing_assets",
|
|
6640
|
+
message: `Downloading ${tally2.imagesTotal.toLocaleString()} pictures`,
|
|
6641
|
+
upserted: tally2.upserted,
|
|
6642
|
+
fetched: tally2.fetched
|
|
6643
|
+
});
|
|
6644
|
+
}
|
|
6645
|
+
} finally {
|
|
6646
|
+
pictures.stop();
|
|
6647
|
+
const settled = await pictures.done;
|
|
6648
|
+
pictureErrors = settled.errors;
|
|
6649
|
+
pictureReasons = settled.reasons;
|
|
4817
6650
|
}
|
|
4818
|
-
|
|
4819
|
-
patch2({ sourceId: source.id, platform: result.detection.platform });
|
|
4820
|
-
core.catalog.setSourceStatus(source.id, "importing");
|
|
4821
|
-
if (!result.products.length) {
|
|
4822
|
-
const stage = result.progress.stage === "failed" ? "failed" : "failed";
|
|
6651
|
+
if (signal.aborted) {
|
|
4823
6652
|
patch2({
|
|
4824
|
-
stage,
|
|
4825
|
-
errors:
|
|
4826
|
-
|
|
4827
|
-
message: result.progress.errors[0]?.message ?? "No products imported",
|
|
6653
|
+
stage: "cancelled",
|
|
6654
|
+
errors: [],
|
|
6655
|
+
message: tally2.upserted ? `Stopped after saving ${tally2.upserted.toLocaleString()} products` : "Stopped before anything was saved",
|
|
4828
6656
|
finished: true
|
|
4829
6657
|
});
|
|
4830
|
-
core.catalog.setSourceStatus(
|
|
6658
|
+
core.catalog.setSourceStatus(run2.sourceId, "partial", true);
|
|
6659
|
+
return;
|
|
6660
|
+
}
|
|
6661
|
+
run2.finish({
|
|
6662
|
+
sweep,
|
|
6663
|
+
warnings,
|
|
6664
|
+
errors: pictureErrors,
|
|
6665
|
+
refused: stats.refused,
|
|
6666
|
+
reasons: { ...stats.reasons, ...pictureReasons },
|
|
6667
|
+
asked: bulk ? bulk.length : urls.length,
|
|
6668
|
+
// Every address was read. A bulk API hands the catalogue over whole, so
|
|
6669
|
+
// there is nothing to cover.
|
|
6670
|
+
covered: bulk ? true : stats.pages >= urls.length
|
|
6671
|
+
});
|
|
6672
|
+
} catch (err) {
|
|
6673
|
+
if (signal.aborted) {
|
|
6674
|
+
patch2({ stage: "cancelled", errors: [], message: "Stopped before anything was saved", finished: true });
|
|
4831
6675
|
return;
|
|
4832
6676
|
}
|
|
4833
|
-
patch2({
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
6677
|
+
patch2({
|
|
6678
|
+
stage: "failed",
|
|
6679
|
+
message: String(err?.message ?? err),
|
|
6680
|
+
errors: [{ code: "import_failed", message: String(err?.message ?? err) }],
|
|
6681
|
+
finished: true
|
|
6682
|
+
});
|
|
6683
|
+
}
|
|
6684
|
+
}
|
|
6685
|
+
var IMAGES_PER_PRODUCT = 3;
|
|
6686
|
+
var TERMINAL = /* @__PURE__ */ new Set(["completed", "partial", "no_catalog", "cancelled", "failed"]);
|
|
6687
|
+
var IMAGE_CONCURRENCY = 4;
|
|
6688
|
+
var sleep6 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
6689
|
+
function beginWrite(deps, jobId, brandId, baseUrl, platform, tally2, discovered) {
|
|
6690
|
+
const { core } = deps;
|
|
6691
|
+
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6692
|
+
const source = core.catalog.upsertSource(brandId, baseUrl, platform);
|
|
6693
|
+
patch2({ sourceId: source.id, platform });
|
|
6694
|
+
core.catalog.setSourceStatus(source.id, "importing");
|
|
6695
|
+
return {
|
|
6696
|
+
sourceId: source.id,
|
|
6697
|
+
write(p) {
|
|
4838
6698
|
core.catalog.upsertProduct({
|
|
4839
6699
|
sourceId: source.id,
|
|
4840
6700
|
brandId,
|
|
@@ -4851,7 +6711,9 @@ async function runJob(deps, jobId, brandId, url, signal) {
|
|
|
4851
6711
|
compareAtPrice: p.compareAtPrice,
|
|
4852
6712
|
currency: p.currency,
|
|
4853
6713
|
available: p.available,
|
|
4854
|
-
raw
|
|
6714
|
+
// `raw` is the whole crawled payload, 14.2 KB a product on gymshark
|
|
6715
|
+
// and 32 MB across its catalog, written to sqlite and read by nothing.
|
|
6716
|
+
raw: null,
|
|
4855
6717
|
variants: p.variants,
|
|
4856
6718
|
images: (p.images ?? []).map((img) => ({
|
|
4857
6719
|
sourceUrl: img.url,
|
|
@@ -4860,93 +6722,120 @@ async function runJob(deps, jobId, brandId, url, signal) {
|
|
|
4860
6722
|
height: img.height,
|
|
4861
6723
|
alt: img.alt
|
|
4862
6724
|
})),
|
|
4863
|
-
collections: (p.collections ?? []).map((c) => ({
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
6725
|
+
collections: (p.collections ?? []).map((c) => ({ externalKey: c, title: c }))
|
|
6726
|
+
});
|
|
6727
|
+
tally2.seenKeys.push(p.externalKey);
|
|
6728
|
+
tally2.upserted++;
|
|
6729
|
+
tally2.fetched++;
|
|
6730
|
+
patch2({ upserted: tally2.upserted, fetched: tally2.fetched });
|
|
6731
|
+
},
|
|
6732
|
+
finish({
|
|
6733
|
+
sweep,
|
|
6734
|
+
warnings,
|
|
6735
|
+
errors,
|
|
6736
|
+
refused = 0,
|
|
6737
|
+
covered = true,
|
|
6738
|
+
reasons = {},
|
|
6739
|
+
asked = 0
|
|
6740
|
+
}) {
|
|
6741
|
+
const mayRetire = sweep && covered && refused === 0 && tally2.upserted > 0;
|
|
6742
|
+
if (mayRetire) core.catalog.markMissingUnavailable(source.id, tally2.seenKeys);
|
|
6743
|
+
tally2.errors = errors;
|
|
6744
|
+
const partial = !covered || refused > 0;
|
|
6745
|
+
const savedNothing = tally2.upserted === 0;
|
|
6746
|
+
const said = summarise(reasons, tally2.upserted, asked || discovered || tally2.fetched);
|
|
6747
|
+
const stage = savedNothing ? "failed" : partial ? "partial" : "completed";
|
|
6748
|
+
const message = savedNothing ? said ?? "We found the products on this site, but none of them could be imported." : said ?? (partial ? `Imported ${tally2.upserted.toLocaleString()} products with ${(errors.length + refused).toLocaleString()} issue${errors.length + refused === 1 ? "" : "s"}` : `Imported ${tally2.upserted.toLocaleString()} products`);
|
|
6749
|
+
patch2({
|
|
6750
|
+
stage,
|
|
6751
|
+
upserted: tally2.upserted,
|
|
6752
|
+
fetched: tally2.fetched,
|
|
6753
|
+
imagesDone: tally2.imagesDone,
|
|
6754
|
+
imagesTotal: tally2.imagesTotal,
|
|
6755
|
+
errors,
|
|
6756
|
+
warnings,
|
|
6757
|
+
message,
|
|
6758
|
+
finished: true
|
|
4867
6759
|
});
|
|
4868
|
-
|
|
4869
|
-
upserted++;
|
|
4870
|
-
if (upserted % 10 === 0) patch2({ upserted, fetched: result.products.length });
|
|
6760
|
+
core.catalog.setSourceStatus(source.id, savedNothing ? "failed" : partial ? "partial" : "ready", true);
|
|
4871
6761
|
}
|
|
4872
|
-
|
|
4873
|
-
|
|
4874
|
-
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
|
|
4878
|
-
|
|
4879
|
-
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
if (!buf.length) {
|
|
4896
|
-
errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
|
|
4897
|
-
return;
|
|
4898
|
-
}
|
|
4899
|
-
const png = await sharp20(buf).rotate().png().toBuffer();
|
|
4900
|
-
const meta = await sharp20(png).metadata();
|
|
4901
|
-
const hash = core.images.save(png);
|
|
4902
|
-
core.catalog.setImageAsset(img.productId, img.sourceUrl, `asset:${hash}`, {
|
|
4903
|
-
width: meta.width,
|
|
4904
|
-
height: meta.height
|
|
4905
|
-
});
|
|
4906
|
-
} catch (err) {
|
|
6762
|
+
};
|
|
6763
|
+
}
|
|
6764
|
+
var IMPORT_CONCURRENCY = 4;
|
|
6765
|
+
var PICTURE_ROUND = 60;
|
|
6766
|
+
function drainPictures(deps, jobId, brandId, tally2, signal, imagesPerProduct = IMAGES_PER_PRODUCT) {
|
|
6767
|
+
const { core, fetchImpl } = deps;
|
|
6768
|
+
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6769
|
+
const errors = [...core.catalog.getJob(jobId)?.errors ?? []];
|
|
6770
|
+
const reasons = {};
|
|
6771
|
+
let stopped = false;
|
|
6772
|
+
const done = (async () => {
|
|
6773
|
+
while (!signal.aborted) {
|
|
6774
|
+
const round = core.catalog.listImagesNeedingAssets(brandId, PICTURE_ROUND).filter((img) => img.position < imagesPerProduct);
|
|
6775
|
+
if (!round.length) {
|
|
6776
|
+
if (stopped) break;
|
|
6777
|
+
await sleep6(150);
|
|
6778
|
+
continue;
|
|
6779
|
+
}
|
|
6780
|
+
tally2.imagesTotal += round.length;
|
|
6781
|
+
await mapPool(
|
|
6782
|
+
round,
|
|
6783
|
+
IMAGE_CONCURRENCY,
|
|
6784
|
+
async (img) => {
|
|
4907
6785
|
if (signal.aborted) return;
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
6786
|
+
try {
|
|
6787
|
+
const res = await httpGet(img.sourceUrl, { fetchImpl, signal, timeoutMs: 4e4, retries: 2 });
|
|
6788
|
+
if (!res.ok) {
|
|
6789
|
+
errors.push({ code: "image_http", message: `HTTP ${res.status}`, url: img.sourceUrl });
|
|
6790
|
+
tally(reasons, imageFailure(res.status));
|
|
6791
|
+
return;
|
|
6792
|
+
}
|
|
6793
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
6794
|
+
if (!buf.length) {
|
|
6795
|
+
errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
|
|
6796
|
+
tally(reasons, "IMAGE_EMPTY");
|
|
6797
|
+
return;
|
|
6798
|
+
}
|
|
6799
|
+
const probe = await sharp21(buf).metadata();
|
|
6800
|
+
const turned = (probe.orientation ?? 1) > 1;
|
|
6801
|
+
const keep = turned ? await sharp21(buf).rotate().toBuffer() : buf;
|
|
6802
|
+
const meta = turned ? await sharp21(keep).metadata() : probe;
|
|
6803
|
+
const hash = core.images.save(keep, meta.format ?? probe.format ?? "png");
|
|
6804
|
+
core.catalog.setImageAsset(img.productId, img.sourceUrl, `asset:${hash}`, {
|
|
6805
|
+
width: meta.width,
|
|
6806
|
+
height: meta.height
|
|
6807
|
+
});
|
|
6808
|
+
} catch (err) {
|
|
6809
|
+
if (signal.aborted) return;
|
|
6810
|
+
tally(reasons, thrownFailure(err, signal.aborted));
|
|
6811
|
+
errors.push({
|
|
6812
|
+
code: "image_failed",
|
|
6813
|
+
message: String(err?.message ?? err),
|
|
6814
|
+
url: img.sourceUrl,
|
|
6815
|
+
retryable: true
|
|
6816
|
+
});
|
|
6817
|
+
} finally {
|
|
6818
|
+
tally2.imagesDone++;
|
|
6819
|
+
patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
|
|
4918
6820
|
}
|
|
4919
|
-
}
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
6821
|
+
},
|
|
6822
|
+
signal
|
|
6823
|
+
);
|
|
6824
|
+
const stuck = core.catalog.listImagesNeedingAssets(brandId, PICTURE_ROUND).filter((img) => img.position < imagesPerProduct);
|
|
6825
|
+
if (stuck.length && stuck[0]?.id === round[0]?.id) {
|
|
6826
|
+
errors.push({ code: "images_stalled", message: "Some pictures could not be downloaded" });
|
|
6827
|
+
break;
|
|
6828
|
+
}
|
|
4927
6829
|
}
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
warnings: result.progress.warnings,
|
|
4938
|
-
message: partial ? `Imported ${upserted} products with ${errors.length} issue${errors.length === 1 ? "" : "s"}` : `Imported ${upserted} products`,
|
|
4939
|
-
finished: true
|
|
4940
|
-
});
|
|
4941
|
-
core.catalog.setSourceStatus(source.id, partial ? "partial" : "ready", true);
|
|
4942
|
-
} catch (err) {
|
|
4943
|
-
patch2({
|
|
4944
|
-
stage: "failed",
|
|
4945
|
-
message: String(err?.message ?? err),
|
|
4946
|
-
errors: [{ code: "import_failed", message: String(err?.message ?? err) }],
|
|
4947
|
-
finished: true
|
|
4948
|
-
});
|
|
4949
|
-
}
|
|
6830
|
+
patch2({ imagesDone: tally2.imagesDone, imagesTotal: tally2.imagesTotal, errors });
|
|
6831
|
+
return { errors, reasons };
|
|
6832
|
+
})();
|
|
6833
|
+
return {
|
|
6834
|
+
stop() {
|
|
6835
|
+
stopped = true;
|
|
6836
|
+
},
|
|
6837
|
+
done
|
|
6838
|
+
};
|
|
4950
6839
|
}
|
|
4951
6840
|
function resolveLibraryProduct(core, brandId, productId) {
|
|
4952
6841
|
const brand = core.store.getBrand(brandId);
|
|
@@ -5374,7 +7263,7 @@ async function generateStudioSet(deps, job, who, sourcePaths, signal) {
|
|
|
5374
7263
|
return { hashes: kept.map((f) => byAngle.get(f.angle)), angles: kept.map((f) => f.angle) };
|
|
5375
7264
|
}
|
|
5376
7265
|
async function edgeBarGeometry(buf) {
|
|
5377
|
-
const { data, info } = await
|
|
7266
|
+
const { data, info } = await sharp21(buf).greyscale().raw().toBuffer({ resolveWithObject: true });
|
|
5378
7267
|
const W = info.width;
|
|
5379
7268
|
const H = info.height;
|
|
5380
7269
|
const scan = (len, cross, at) => {
|
|
@@ -5428,7 +7317,7 @@ async function trimEdgeBars(core, hash) {
|
|
|
5428
7317
|
const width = g.right - g.left + 1;
|
|
5429
7318
|
const height = g.bottom - g.top + 1;
|
|
5430
7319
|
if (width < g.W * 0.6 || height < g.H * 0.6) return hash;
|
|
5431
|
-
const png = await
|
|
7320
|
+
const png = await sharp21(buf).extract({ left: g.left, top: g.top, width, height }).png().toBuffer();
|
|
5432
7321
|
return core.images.save(png);
|
|
5433
7322
|
} catch {
|
|
5434
7323
|
return hash;
|
|
@@ -5499,7 +7388,7 @@ async function identityCrop(core, hash) {
|
|
|
5499
7388
|
if (!out) return void 0;
|
|
5500
7389
|
try {
|
|
5501
7390
|
const height = Math.min(IDENTITY_TARGET_HEIGHT, Math.round(nativeHeight * IDENTITY_MAX_UPSCALE)) || IDENTITY_TARGET_HEIGHT;
|
|
5502
|
-
const png = await
|
|
7391
|
+
const png = await sharp21(core.images.read(out)).resize({ height, fit: "inside", kernel: "lanczos3", withoutEnlargement: false }).png().toBuffer();
|
|
5503
7392
|
const scaled = core.images.save(png);
|
|
5504
7393
|
identityCrops.set(hash, scaled);
|
|
5505
7394
|
return scaled;
|
|
@@ -5528,12 +7417,12 @@ async function brandJsonWithIdentityCrops(core, json, characterIds) {
|
|
|
5528
7417
|
return changed ? { ...json, characters } : json;
|
|
5529
7418
|
}
|
|
5530
7419
|
async function figureBox(buf) {
|
|
5531
|
-
const meta = await
|
|
7420
|
+
const meta = await sharp21(buf).metadata();
|
|
5532
7421
|
const W = meta.width ?? 0;
|
|
5533
7422
|
const H = meta.height ?? 0;
|
|
5534
7423
|
if (!W || !H) return null;
|
|
5535
7424
|
for (const threshold of FIGURE_TRIM_THRESHOLDS) {
|
|
5536
|
-
const { info } = await
|
|
7425
|
+
const { info } = await sharp21(buf).trim({ threshold }).toBuffer({ resolveWithObject: true });
|
|
5537
7426
|
const left = Math.abs(info.trimOffsetLeft ?? 0);
|
|
5538
7427
|
const top = Math.abs(info.trimOffsetTop ?? 0);
|
|
5539
7428
|
const width = info.width ?? 0;
|
|
@@ -5566,13 +7455,13 @@ async function smartCover(core, hash, box) {
|
|
|
5566
7455
|
if (!hash || !core.images.has(hash)) return void 0;
|
|
5567
7456
|
try {
|
|
5568
7457
|
const buf = core.images.read(hash);
|
|
5569
|
-
const meta = await
|
|
7458
|
+
const meta = await sharp21(buf).metadata();
|
|
5570
7459
|
const w = meta.width ?? 0;
|
|
5571
7460
|
const h = meta.height ?? 0;
|
|
5572
7461
|
if (!w || !h) return void 0;
|
|
5573
7462
|
const raw = box(w, h);
|
|
5574
7463
|
const target = { width: Math.max(1, raw.width), height: Math.max(1, raw.height) };
|
|
5575
|
-
const png = await
|
|
7464
|
+
const png = await sharp21(buf).resize(target.width, target.height, { fit: "cover", position: "attention" }).png().toBuffer();
|
|
5576
7465
|
return core.images.save(png);
|
|
5577
7466
|
} catch {
|
|
5578
7467
|
return void 0;
|
|
@@ -5581,11 +7470,11 @@ async function smartCover(core, hash, box) {
|
|
|
5581
7470
|
async function crop(core, hash, region, cap2) {
|
|
5582
7471
|
if (!hash || !core.images.has(hash)) return void 0;
|
|
5583
7472
|
try {
|
|
5584
|
-
const meta = await
|
|
7473
|
+
const meta = await sharp21(core.images.read(hash)).metadata();
|
|
5585
7474
|
const w = meta.width ?? 0;
|
|
5586
7475
|
const h = meta.height ?? 0;
|
|
5587
7476
|
if (!w || !h) return void 0;
|
|
5588
|
-
let pipeline =
|
|
7477
|
+
let pipeline = sharp21(core.images.read(hash)).extract(region(w, h));
|
|
5589
7478
|
if (cap2) pipeline = pipeline.resize(cap2, cap2, { fit: "inside", withoutEnlargement: true });
|
|
5590
7479
|
const png = await pipeline.png().toBuffer();
|
|
5591
7480
|
return core.images.save(png);
|
|
@@ -5932,7 +7821,7 @@ var fromLab = (l, a, bb) => {
|
|
|
5932
7821
|
return [clamp(R), clamp(G), clamp(B)];
|
|
5933
7822
|
};
|
|
5934
7823
|
var rawAt = async (png, edge) => {
|
|
5935
|
-
let img =
|
|
7824
|
+
let img = sharp21(png);
|
|
5936
7825
|
if (edge) img = img.resize(edge, edge, { fit: "fill" });
|
|
5937
7826
|
const { data, info } = await img.removeAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
5938
7827
|
return { data, width: info.width, height: info.height };
|
|
@@ -5995,7 +7884,7 @@ async function gradeComposite(originalPng, modelInputPng, modelOutputPng) {
|
|
|
5995
7884
|
if (residual > GRADE_GATE_MEAN_DELTA) return null;
|
|
5996
7885
|
const full = await rawAt(originalPng);
|
|
5997
7886
|
applyAffine(full, T);
|
|
5998
|
-
const image = await
|
|
7887
|
+
const image = await sharp21(full.data, {
|
|
5999
7888
|
raw: { width: full.width, height: full.height, channels: 3 }
|
|
6000
7889
|
}).png().toBuffer();
|
|
6001
7890
|
return { image, residual };
|
|
@@ -6169,7 +8058,7 @@ function fitExpandToBudget(plan, source, pixelBudget) {
|
|
|
6169
8058
|
}
|
|
6170
8059
|
async function attentionCropOrigin(srcBuf, source, plan) {
|
|
6171
8060
|
try {
|
|
6172
|
-
const { info } = await
|
|
8061
|
+
const { info } = await sharp21(srcBuf).resize(plan.width, plan.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
|
|
6173
8062
|
const attnLeft = typeof info.cropOffsetLeft === "number" ? Math.abs(info.cropOffsetLeft) : plan.left;
|
|
6174
8063
|
const attnTop = typeof info.cropOffsetTop === "number" ? Math.abs(info.cropOffsetTop) : plan.top;
|
|
6175
8064
|
const left = Math.round((attnLeft + plan.left) / 2);
|
|
@@ -6322,23 +8211,23 @@ function relax(grid, seam, fixedSweeps) {
|
|
|
6322
8211
|
|
|
6323
8212
|
// src/expand.ts
|
|
6324
8213
|
async function expandCanvas(source, plan) {
|
|
6325
|
-
const bed = await
|
|
6326
|
-
return
|
|
8214
|
+
const bed = await sharp21(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
|
|
8215
|
+
return sharp21(bed).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
|
|
6327
8216
|
}
|
|
6328
8217
|
async function compositeExpand(engineImage, source, plan) {
|
|
6329
|
-
const meta = await
|
|
8218
|
+
const meta = await sharp21(engineImage).metadata();
|
|
6330
8219
|
const want = plan.width / plan.height;
|
|
6331
8220
|
const got = meta.width && meta.height ? meta.width / meta.height : 0;
|
|
6332
8221
|
const sameOrientation = got > 0 && got >= 1 === want >= 1;
|
|
6333
8222
|
const aligned = sameOrientation;
|
|
6334
8223
|
const exact = meta.width === plan.width && meta.height === plan.height;
|
|
6335
|
-
const surround = aligned ? exact ? engineImage : await
|
|
8224
|
+
const surround = aligned ? exact ? engineImage : await sharp21(engineImage).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).toBuffer() : await expandCanvasBedOnly(source, plan);
|
|
6336
8225
|
const matched = aligned ? await matchMarginsToSeam(surround, source, plan) : surround;
|
|
6337
|
-
const image = await
|
|
8226
|
+
const image = await sharp21(matched).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
|
|
6338
8227
|
return { image, aligned };
|
|
6339
8228
|
}
|
|
6340
8229
|
async function matchMarginsToSeam(surround, source, plan) {
|
|
6341
|
-
const src = await
|
|
8230
|
+
const src = await sharp21(source).metadata();
|
|
6342
8231
|
if (!src.width || !src.height) return surround;
|
|
6343
8232
|
const SW = src.width;
|
|
6344
8233
|
const SH = src.height;
|
|
@@ -6385,8 +8274,8 @@ var MAX_CORRECTION = 60;
|
|
|
6385
8274
|
async function reconcile(surround, source, side, axis) {
|
|
6386
8275
|
const { margin } = side;
|
|
6387
8276
|
if (margin.width < 1 || margin.height < 1) return surround;
|
|
6388
|
-
const marginRaw = await
|
|
6389
|
-
const edgeRaw = await
|
|
8277
|
+
const marginRaw = await sharp21(surround).extract(margin).removeAlpha().raw().toBuffer();
|
|
8278
|
+
const edgeRaw = await sharp21(source).extract(side.srcEdge).removeAlpha().raw().toBuffer();
|
|
6390
8279
|
const W = margin.width;
|
|
6391
8280
|
const H = margin.height;
|
|
6392
8281
|
const along = axis === "width" ? H : W;
|
|
@@ -6440,11 +8329,11 @@ async function reconcile(surround, source, side, axis) {
|
|
|
6440
8329
|
}
|
|
6441
8330
|
}
|
|
6442
8331
|
}
|
|
6443
|
-
const patch2 = await
|
|
6444
|
-
return
|
|
8332
|
+
const patch2 = await sharp21(corrected, { raw: { width: W, height: H, channels: 3 } }).png().toBuffer();
|
|
8333
|
+
return sharp21(surround).composite([{ input: patch2, left: margin.left, top: margin.top }]).toBuffer();
|
|
6445
8334
|
}
|
|
6446
8335
|
async function expandCanvasBedOnly(source, plan) {
|
|
6447
|
-
return
|
|
8336
|
+
return sharp21(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
|
|
6448
8337
|
}
|
|
6449
8338
|
function medianOf(rgb, channel, from, to) {
|
|
6450
8339
|
const n = to - from;
|
|
@@ -6455,17 +8344,17 @@ function medianOf(rgb, channel, from, to) {
|
|
|
6455
8344
|
return n % 2 ? values[(n - 1) / 2] : (values[n / 2 - 1] + values[n / 2]) / 2;
|
|
6456
8345
|
}
|
|
6457
8346
|
async function reframeExpand(engineImage, plan) {
|
|
6458
|
-
const meta = await
|
|
8347
|
+
const meta = await sharp21(engineImage).metadata();
|
|
6459
8348
|
if (!(meta.width && meta.height)) return null;
|
|
6460
8349
|
const want = plan.width / plan.height;
|
|
6461
8350
|
const got = meta.width / meta.height;
|
|
6462
8351
|
if (got >= 1 !== want >= 1) return null;
|
|
6463
8352
|
if (meta.width === plan.width && meta.height === plan.height) return engineImage;
|
|
6464
8353
|
const straight = Math.abs(got - want) / want <= 0.02;
|
|
6465
|
-
return
|
|
8354
|
+
return sharp21(engineImage).resize(plan.width, plan.height, { fit: straight ? "fill" : "cover", position: "centre" }).png().toBuffer();
|
|
6466
8355
|
}
|
|
6467
8356
|
async function seamScore(image, plan, source) {
|
|
6468
|
-
const { data, info } = await
|
|
8357
|
+
const { data, info } = await sharp21(image).removeAlpha().greyscale().raw().toBuffer({ resolveWithObject: true });
|
|
6469
8358
|
const W = info.width;
|
|
6470
8359
|
const H = info.height;
|
|
6471
8360
|
const horizontal = plan.axis === "width";
|
|
@@ -6498,7 +8387,7 @@ var SEAM_VISIBLE = 2.2;
|
|
|
6498
8387
|
var OFFSET = 4;
|
|
6499
8388
|
var RESIDUAL_VISIBLE = 15;
|
|
6500
8389
|
async function seamResidual(image, plan, source) {
|
|
6501
|
-
const { data, info } = await
|
|
8390
|
+
const { data, info } = await sharp21(image).removeAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
6502
8391
|
const W = info.width;
|
|
6503
8392
|
const H = info.height;
|
|
6504
8393
|
const ch = info.channels;
|
|
@@ -6533,7 +8422,7 @@ var MAX_SHARE = 0.8;
|
|
|
6533
8422
|
async function subjectFraction(src, source, axis) {
|
|
6534
8423
|
try {
|
|
6535
8424
|
const window = axis === "width" ? { width: Math.max(8, Math.round(source.width * 0.5)), height: source.height } : { width: source.width, height: Math.max(8, Math.round(source.height * 0.5)) };
|
|
6536
|
-
const { info } = await
|
|
8425
|
+
const { info } = await sharp21(src).resize(window.width, window.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
|
|
6537
8426
|
const offset = axis === "width" ? Math.abs(typeof info.cropOffsetLeft === "number" ? info.cropOffsetLeft : 0) : Math.abs(typeof info.cropOffsetTop === "number" ? info.cropOffsetTop : 0);
|
|
6538
8427
|
const span = axis === "width" ? source.width : source.height;
|
|
6539
8428
|
const extent = axis === "width" ? window.width : window.height;
|
|
@@ -6556,14 +8445,14 @@ function placeExpand(plan, source, fraction) {
|
|
|
6556
8445
|
}
|
|
6557
8446
|
var NEUTRAL = { r: 128, g: 128, b: 128 };
|
|
6558
8447
|
async function conditioningCanvas(source, plan, fill = "edge") {
|
|
6559
|
-
const meta = await
|
|
8448
|
+
const meta = await sharp21(source).metadata();
|
|
6560
8449
|
const sw = meta.width ?? 0;
|
|
6561
8450
|
const sh = meta.height ?? 0;
|
|
6562
8451
|
if (!(sw > 0 && sh > 0)) throw new Error("conditioningCanvas: source has no dimensions");
|
|
6563
8452
|
const layers = [];
|
|
6564
8453
|
if (fill === "edge") layers.push(...await edgeMargins(source, plan, { width: sw, height: sh }));
|
|
6565
8454
|
layers.push({ input: source, left: plan.left, top: plan.top });
|
|
6566
|
-
const canvas =
|
|
8455
|
+
const canvas = sharp21({
|
|
6567
8456
|
create: {
|
|
6568
8457
|
width: plan.width,
|
|
6569
8458
|
height: plan.height,
|
|
@@ -6575,7 +8464,7 @@ async function conditioningCanvas(source, plan, fill = "edge") {
|
|
|
6575
8464
|
}
|
|
6576
8465
|
async function edgeMargins(source, plan, size) {
|
|
6577
8466
|
const out = [];
|
|
6578
|
-
const strip = async (extract, width, height) =>
|
|
8467
|
+
const strip = async (extract, width, height) => sharp21(source).extract(extract).resize(width, height, { fit: "fill" }).png().toBuffer();
|
|
6579
8468
|
if (plan.axis === "width") {
|
|
6580
8469
|
const before = plan.left;
|
|
6581
8470
|
const after = plan.width - plan.left - size.width;
|
|
@@ -6666,12 +8555,12 @@ async function resolveOutpaintRoute(all, shot) {
|
|
|
6666
8555
|
return { engine: shot, method: "reframe", crossed: false };
|
|
6667
8556
|
}
|
|
6668
8557
|
async function driftDiff(a, b) {
|
|
6669
|
-
const metaA = await
|
|
6670
|
-
const metaB = await
|
|
8558
|
+
const metaA = await sharp21(a).metadata();
|
|
8559
|
+
const metaB = await sharp21(b).metadata();
|
|
6671
8560
|
const width = Math.min(metaA.width ?? 1, metaB.width ?? 1, 1024);
|
|
6672
8561
|
const height = Math.min(metaA.height ?? 1, metaB.height ?? 1, 1024);
|
|
6673
8562
|
const [rawA, rawB] = await Promise.all(
|
|
6674
|
-
[a, b].map((buf) =>
|
|
8563
|
+
[a, b].map((buf) => sharp21(buf).resize(width, height, { fit: "cover" }).ensureAlpha().raw().toBuffer())
|
|
6675
8564
|
);
|
|
6676
8565
|
const out = new PNG({ width, height });
|
|
6677
8566
|
const changed = pixelmatch(rawA, rawB, out.data, width, height, { threshold: 0.1, diffColor: [255, 64, 64] });
|
|
@@ -6683,11 +8572,11 @@ async function driftDiff(a, b) {
|
|
|
6683
8572
|
};
|
|
6684
8573
|
}
|
|
6685
8574
|
async function changeMask(a, b, cap2 = 1024) {
|
|
6686
|
-
const metaA = await
|
|
8575
|
+
const metaA = await sharp21(a).metadata();
|
|
6687
8576
|
const width = Math.min(metaA.width ?? 1, cap2);
|
|
6688
8577
|
const height = Math.min(metaA.height ?? 1, cap2);
|
|
6689
8578
|
const [rawA, rawB] = await Promise.all(
|
|
6690
|
-
[a, b].map((buf) =>
|
|
8579
|
+
[a, b].map((buf) => sharp21(buf).resize(width, height, { fit: "fill" }).ensureAlpha().raw().toBuffer())
|
|
6691
8580
|
);
|
|
6692
8581
|
const out = new PNG({ width, height });
|
|
6693
8582
|
pixelmatch(rawA, rawB, out.data, width, height, { threshold: 0.1, includeAA: true, diffMask: true });
|
|
@@ -6736,8 +8625,8 @@ function dilationFor(longEdge) {
|
|
|
6736
8625
|
// src/localEdit.ts
|
|
6737
8626
|
async function preserveOutsideChange(source, edited) {
|
|
6738
8627
|
try {
|
|
6739
|
-
const srcMeta = await
|
|
6740
|
-
const outMeta = await
|
|
8628
|
+
const srcMeta = await sharp21(source).metadata();
|
|
8629
|
+
const outMeta = await sharp21(edited).metadata();
|
|
6741
8630
|
if (!srcMeta.width || !srcMeta.height || !outMeta.width || !outMeta.height)
|
|
6742
8631
|
return { image: edited, outcome: "error", changed: 0 };
|
|
6743
8632
|
const sameShape = Math.abs(outMeta.width / outMeta.height - srcMeta.width / srcMeta.height) / (srcMeta.width / srcMeta.height) <= 0.01;
|
|
@@ -6747,15 +8636,15 @@ async function preserveOutsideChange(source, edited) {
|
|
|
6747
8636
|
if (outcome !== "composited") return { image: edited, outcome, changed: shape.changed };
|
|
6748
8637
|
const r = dilationFor(Math.max(shape.width, shape.height));
|
|
6749
8638
|
const rawShape = { raw: { width: shape.width, height: shape.height, channels: 1 } };
|
|
6750
|
-
const spread = await
|
|
6751
|
-
const dilated = await
|
|
6752
|
-
const feathered = await
|
|
6753
|
-
const grown = await
|
|
6754
|
-
const editedRgb = await
|
|
6755
|
-
const masked = await
|
|
8639
|
+
const spread = await sharp21(shape.mask, rawShape).blur(r).toColourspace("b-w").raw().toBuffer();
|
|
8640
|
+
const dilated = await sharp21(spread, rawShape).threshold(8).toColourspace("b-w").raw().toBuffer();
|
|
8641
|
+
const feathered = await sharp21(dilated, rawShape).blur(Math.max(2, r / 3)).toColourspace("b-w").raw().toBuffer();
|
|
8642
|
+
const grown = await sharp21(feathered, rawShape).resize(srcMeta.width, srcMeta.height, { fit: "fill", kernel: "cubic" }).toColourspace("b-w").raw().toBuffer();
|
|
8643
|
+
const editedRgb = await sharp21(edited).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).removeAlpha().raw().toBuffer();
|
|
8644
|
+
const masked = await sharp21(editedRgb, {
|
|
6756
8645
|
raw: { width: srcMeta.width, height: srcMeta.height, channels: 3 }
|
|
6757
8646
|
}).joinChannel(grown, { raw: { width: srcMeta.width, height: srcMeta.height, channels: 1 } }).png().toBuffer();
|
|
6758
|
-
const image = await
|
|
8647
|
+
const image = await sharp21(source).removeAlpha().composite([{ input: masked }]).png().toBuffer();
|
|
6759
8648
|
return { image, outcome: "composited", changed: shape.changed };
|
|
6760
8649
|
} catch {
|
|
6761
8650
|
return { image: edited, outcome: "error", changed: 0 };
|
|
@@ -6793,7 +8682,7 @@ function registerLogoRoutes(app, deps) {
|
|
|
6793
8682
|
const v = validateBrand(json);
|
|
6794
8683
|
if (!v.valid) return reply.status(400).send({ error: "brand became invalid", details: v.errors });
|
|
6795
8684
|
const row = core.store.updateBrand(brand.id, json);
|
|
6796
|
-
const meta = await
|
|
8685
|
+
const meta = await sharp21(core.images.read(part.hash)).metadata().catch(() => null);
|
|
6797
8686
|
const logoEdge = meta ? Math.max(meta.width ?? 0, meta.height ?? 0) || null : null;
|
|
6798
8687
|
return { ...row, logoHash: part.hash, logoEdge };
|
|
6799
8688
|
});
|
|
@@ -6841,6 +8730,38 @@ function registerLogoRoutes(app, deps) {
|
|
|
6841
8730
|
return core.store.updateBrand(brand.id, json);
|
|
6842
8731
|
});
|
|
6843
8732
|
}
|
|
8733
|
+
var scans = /* @__PURE__ */ new Map();
|
|
8734
|
+
var controllers = /* @__PURE__ */ new Map();
|
|
8735
|
+
var KEEP = 32;
|
|
8736
|
+
function remember(state) {
|
|
8737
|
+
scans.set(state.id, state);
|
|
8738
|
+
while (scans.size > KEEP) {
|
|
8739
|
+
const oldest = scans.keys().next().value;
|
|
8740
|
+
if (oldest === void 0) break;
|
|
8741
|
+
scans.delete(oldest);
|
|
8742
|
+
controllers.delete(oldest);
|
|
8743
|
+
}
|
|
8744
|
+
}
|
|
8745
|
+
function getScan(scanId) {
|
|
8746
|
+
return scans.get(scanId);
|
|
8747
|
+
}
|
|
8748
|
+
function startCatalogScan(deps, brandId, url) {
|
|
8749
|
+
const { core, fetchImpl } = deps;
|
|
8750
|
+
if (!core.store.getBrand(brandId)) throw Object.assign(new Error("brand not found"), { statusCode: 404 });
|
|
8751
|
+
const id = randomUUID();
|
|
8752
|
+
const ctrl = new AbortController();
|
|
8753
|
+
const state = { id, brandId, url, status: "running", startedAt: Date.now() };
|
|
8754
|
+
remember(state);
|
|
8755
|
+
controllers.set(id, ctrl);
|
|
8756
|
+
void scanForCandidates({ url, fetchImpl, signal: ctrl.signal }).then((result) => {
|
|
8757
|
+
state.result = result;
|
|
8758
|
+
state.status = "done";
|
|
8759
|
+
}).catch((err) => {
|
|
8760
|
+
state.status = "error";
|
|
8761
|
+
state.error = err instanceof Error ? err.message : String(err);
|
|
8762
|
+
}).finally(() => controllers.delete(id));
|
|
8763
|
+
return { scanId: id };
|
|
8764
|
+
}
|
|
6844
8765
|
|
|
6845
8766
|
// src/routes/catalogImport.ts
|
|
6846
8767
|
function registerCatalogImportRoutes(app, deps) {
|
|
@@ -6848,23 +8769,104 @@ function registerCatalogImportRoutes(app, deps) {
|
|
|
6848
8769
|
app.get("/api/brands/:id/products-library", async (req, reply) => {
|
|
6849
8770
|
const brand = core.store.getBrand(req.params.id);
|
|
6850
8771
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
6851
|
-
const products = core.catalog.
|
|
8772
|
+
const products = core.catalog.listLibraryIndex(brand.id, brand.json);
|
|
6852
8773
|
const source = core.catalog.getSourceForBrand(brand.id);
|
|
6853
8774
|
return { products, source };
|
|
6854
8775
|
});
|
|
8776
|
+
app.get("/api/brands/:id/products-library/:productId", async (req, reply) => {
|
|
8777
|
+
const brand = core.store.getBrand(req.params.id);
|
|
8778
|
+
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8779
|
+
const product = core.catalog.libraryProduct(brand.id, brand.json, String(req.params.productId));
|
|
8780
|
+
if (!product) return reply.status(404).send({ error: "product not found" });
|
|
8781
|
+
return { product };
|
|
8782
|
+
});
|
|
6855
8783
|
app.get("/api/brands/:id/catalog/source", async (req, reply) => {
|
|
6856
8784
|
const brand = core.store.getBrand(req.params.id);
|
|
6857
8785
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
6858
8786
|
return { source: core.catalog.getSourceForBrand(brand.id) };
|
|
6859
8787
|
});
|
|
8788
|
+
app.post("/api/brands/:id/catalog/scan", async (req, reply) => {
|
|
8789
|
+
const brandId = req.params.id;
|
|
8790
|
+
const brand = core.store.getBrand(brandId);
|
|
8791
|
+
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8792
|
+
const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
|
|
8793
|
+
if (!url.trim()) return reply.status(400).send({ error: "url required" });
|
|
8794
|
+
try {
|
|
8795
|
+
return startCatalogScan({ core, fetchImpl }, brandId, url);
|
|
8796
|
+
} catch (err) {
|
|
8797
|
+
return reply.status(err.statusCode ?? 500).send({ error: err.message ?? "scan failed" });
|
|
8798
|
+
}
|
|
8799
|
+
});
|
|
8800
|
+
app.get("/api/brands/:id/catalog/scans/:scanId", async (req, reply) => {
|
|
8801
|
+
const brandId = req.params.id;
|
|
8802
|
+
const scan = getScan(req.params.scanId);
|
|
8803
|
+
if (!scan || scan.brandId !== brandId) return reply.status(404).send({ error: "scan not found" });
|
|
8804
|
+
return scan;
|
|
8805
|
+
});
|
|
8806
|
+
app.post("/api/brands/:id/catalog/details", async (req, reply) => {
|
|
8807
|
+
const brand = core.store.getBrand(req.params.id);
|
|
8808
|
+
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8809
|
+
const asked = req.body?.urls;
|
|
8810
|
+
if (!Array.isArray(asked) || asked.some((u) => typeof u !== "string")) {
|
|
8811
|
+
return reply.status(400).send({ error: "urls must be a list of product addresses" });
|
|
8812
|
+
}
|
|
8813
|
+
const urls = asked.slice(0, 48);
|
|
8814
|
+
if (!urls.length) return { products: [] };
|
|
8815
|
+
const site = String(brand.json?.meta?.website ?? "");
|
|
8816
|
+
let origin;
|
|
8817
|
+
try {
|
|
8818
|
+
origin = new URL(site.startsWith("http") ? site : `https://${site}`).origin;
|
|
8819
|
+
} catch {
|
|
8820
|
+
return reply.status(400).send({ error: "this brand has no website to read products from" });
|
|
8821
|
+
}
|
|
8822
|
+
const sameSite = urls.filter((u) => {
|
|
8823
|
+
try {
|
|
8824
|
+
return new URL(u).origin === origin;
|
|
8825
|
+
} catch {
|
|
8826
|
+
return false;
|
|
8827
|
+
}
|
|
8828
|
+
});
|
|
8829
|
+
if (!sameSite.length) {
|
|
8830
|
+
return reply.status(400).send({ error: "none of those products belong to this site" });
|
|
8831
|
+
}
|
|
8832
|
+
const products = await fetchProductPages({ fetchImpl: fetchImpl ?? fetch, baseUrl: origin }, sameSite, {
|
|
8833
|
+
concurrency: 4,
|
|
8834
|
+
maxBytes: 15e5
|
|
8835
|
+
});
|
|
8836
|
+
return { products };
|
|
8837
|
+
});
|
|
6860
8838
|
app.post("/api/brands/:id/catalog/import", async (req, reply) => {
|
|
6861
8839
|
const brandId = req.params.id;
|
|
6862
8840
|
const brand = core.store.getBrand(brandId);
|
|
6863
8841
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
6864
8842
|
const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
|
|
6865
8843
|
if (!url.trim()) return reply.status(400).send({ error: "url required" });
|
|
8844
|
+
const asked = req.body?.urls;
|
|
8845
|
+
let only;
|
|
8846
|
+
if (asked != null) {
|
|
8847
|
+
if (!Array.isArray(asked) || asked.some((u) => typeof u !== "string")) {
|
|
8848
|
+
return reply.status(400).send({ error: "urls must be a list of product addresses" });
|
|
8849
|
+
}
|
|
8850
|
+
let origin;
|
|
8851
|
+
try {
|
|
8852
|
+
origin = new URL(url.startsWith("http") ? url : `https://${url}`).origin;
|
|
8853
|
+
} catch {
|
|
8854
|
+
return reply.status(400).send({ error: "url required" });
|
|
8855
|
+
}
|
|
8856
|
+
only = asked.filter((u) => {
|
|
8857
|
+
try {
|
|
8858
|
+
return new URL(u).origin === origin;
|
|
8859
|
+
} catch {
|
|
8860
|
+
return false;
|
|
8861
|
+
}
|
|
8862
|
+
});
|
|
8863
|
+
if (!only.length) return reply.status(400).send({ error: "none of those products belong to this site" });
|
|
8864
|
+
if (only.length > 2e3) {
|
|
8865
|
+
return reply.status(400).send({ error: "That is nearly the whole catalogue. Import everything instead." });
|
|
8866
|
+
}
|
|
8867
|
+
}
|
|
6866
8868
|
try {
|
|
6867
|
-
return startCatalogImport({ core, fetchImpl }, brandId, url);
|
|
8869
|
+
return startCatalogImport({ core, fetchImpl }, brandId, url, { only });
|
|
6868
8870
|
} catch (err) {
|
|
6869
8871
|
return reply.status(err.statusCode ?? 500).send({ error: err.message ?? "import failed" });
|
|
6870
8872
|
}
|
|
@@ -6915,7 +8917,7 @@ async function vibrantColor(input) {
|
|
|
6915
8917
|
let data;
|
|
6916
8918
|
let channels;
|
|
6917
8919
|
try {
|
|
6918
|
-
const out = await
|
|
8920
|
+
const out = await sharp21(input).resize(48, 48, { fit: "inside" }).raw().toBuffer({ resolveWithObject: true });
|
|
6919
8921
|
data = out.data;
|
|
6920
8922
|
channels = out.info.channels;
|
|
6921
8923
|
} catch {
|
|
@@ -6938,13 +8940,13 @@ async function vibrantColor(input) {
|
|
|
6938
8940
|
const best = buckets.reduce((a, b) => b.score > a.score ? b : a, buckets[0]);
|
|
6939
8941
|
if (best.score <= 0) {
|
|
6940
8942
|
try {
|
|
6941
|
-
const { dominant } = await
|
|
6942
|
-
return
|
|
8943
|
+
const { dominant } = await sharp21(input).stats();
|
|
8944
|
+
return toHex2(dominant.r, dominant.g, dominant.b);
|
|
6943
8945
|
} catch {
|
|
6944
8946
|
return null;
|
|
6945
8947
|
}
|
|
6946
8948
|
}
|
|
6947
|
-
return
|
|
8949
|
+
return toHex2(best.r / best.score, best.g / best.score, best.b / best.score);
|
|
6948
8950
|
}
|
|
6949
8951
|
function rgbToHsl(r, g, b) {
|
|
6950
8952
|
const rn = r / 255, gn = g / 255, bn = b / 255;
|
|
@@ -6959,7 +8961,7 @@ function rgbToHsl(r, g, b) {
|
|
|
6959
8961
|
else h = ((rn - gn) / d + 4) * 60;
|
|
6960
8962
|
return { h, s, l };
|
|
6961
8963
|
}
|
|
6962
|
-
var
|
|
8964
|
+
var toHex2 = (r, g, b) => "#" + [r, g, b].map(
|
|
6963
8965
|
(v) => Math.max(0, Math.min(255, Math.round(v))).toString(16).padStart(2, "0")
|
|
6964
8966
|
).join("");
|
|
6965
8967
|
|
|
@@ -7482,7 +9484,7 @@ function registerProjectRoutes(app, deps) {
|
|
|
7482
9484
|
const brand = core.store.getBrand(req.params.id);
|
|
7483
9485
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
7484
9486
|
const limit = Math.min(Number(req.query.limit) || 60, 200);
|
|
7485
|
-
return { nodes: core.store.recentActivity(brand.id, limit), jobs: core.catalog.
|
|
9487
|
+
return { nodes: core.store.recentActivity(brand.id, limit), jobs: core.catalog.listRecentJobs(brand.id) };
|
|
7486
9488
|
});
|
|
7487
9489
|
app.get("/api/brands/:id/workspace", async (req, reply) => {
|
|
7488
9490
|
const brand = core.store.getBrand(req.params.id);
|
|
@@ -7591,7 +9593,10 @@ function registerProjectRoutes(app, deps) {
|
|
|
7591
9593
|
function registerCodexSetupRoutes(app, deps) {
|
|
7592
9594
|
const codexSetup = deps.codexSetup ?? createCodexSetup({ runner: deps.codexRunner });
|
|
7593
9595
|
let codexSetupBusy = null;
|
|
7594
|
-
app.get("/api/engines/codex/status", async () =>
|
|
9596
|
+
app.get("/api/engines/codex/status", async (req) => {
|
|
9597
|
+
const force = req.query?.force === "1";
|
|
9598
|
+
return codexSetup.status({ force });
|
|
9599
|
+
});
|
|
7595
9600
|
app.post("/api/engines/codex/install", async (_req, reply) => {
|
|
7596
9601
|
if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
|
|
7597
9602
|
codexSetupBusy = "install";
|
|
@@ -7614,6 +9619,35 @@ function registerCodexSetupRoutes(app, deps) {
|
|
|
7614
9619
|
codexSetupBusy = null;
|
|
7615
9620
|
}
|
|
7616
9621
|
});
|
|
9622
|
+
app.post("/api/engines/codex/repair-env", async (req, reply) => {
|
|
9623
|
+
const repair = deps.envRepair;
|
|
9624
|
+
if (!repair) return reply.status(501).send({ error: "not available on this server" });
|
|
9625
|
+
const asked = req.body?.keys;
|
|
9626
|
+
if (!Array.isArray(asked) || asked.length === 0) return reply.status(400).send({ error: "keys required" });
|
|
9627
|
+
const keys = asked.map((k) => String(k).trim().toUpperCase());
|
|
9628
|
+
const unknown = keys.filter((k) => !CONFLICT_ENV_KEYS.includes(k));
|
|
9629
|
+
if (unknown.length > 0) return reply.status(400).send({ error: `not a Codex credential: ${unknown.join(", ")}` });
|
|
9630
|
+
if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
|
|
9631
|
+
codexSetupBusy = "repair";
|
|
9632
|
+
try {
|
|
9633
|
+
repair.set([.../* @__PURE__ */ new Set([...repair.get(), ...keys])]);
|
|
9634
|
+
return { ok: true, ...await codexSetup.status({ force: true }) };
|
|
9635
|
+
} finally {
|
|
9636
|
+
codexSetupBusy = null;
|
|
9637
|
+
}
|
|
9638
|
+
});
|
|
9639
|
+
app.post("/api/engines/codex/restore-env", async (_req, reply) => {
|
|
9640
|
+
const repair = deps.envRepair;
|
|
9641
|
+
if (!repair) return reply.status(501).send({ error: "not available on this server" });
|
|
9642
|
+
if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
|
|
9643
|
+
codexSetupBusy = "repair";
|
|
9644
|
+
try {
|
|
9645
|
+
repair.set([]);
|
|
9646
|
+
return { ok: true, ...await codexSetup.status({ force: true }) };
|
|
9647
|
+
} finally {
|
|
9648
|
+
codexSetupBusy = null;
|
|
9649
|
+
}
|
|
9650
|
+
});
|
|
7617
9651
|
}
|
|
7618
9652
|
var slug = (v, fallback) => {
|
|
7619
9653
|
const s = String(v ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
|
|
@@ -7794,8 +9828,8 @@ function registerImageRoutes(app, deps) {
|
|
|
7794
9828
|
if (!part) return reply.status(400).send({ error: "multipart file field required" });
|
|
7795
9829
|
const buf = await part.toBuffer();
|
|
7796
9830
|
if (buf.length === 0) return reply.status(400).send({ error: "empty file" });
|
|
7797
|
-
const fmt = (await
|
|
7798
|
-
const png = fmt === "svg" ? await toMarkPng(buf) : await
|
|
9831
|
+
const fmt = (await sharp21(buf).metadata().catch(() => null))?.format;
|
|
9832
|
+
const png = fmt === "svg" ? await toMarkPng(buf) : await sharp21(buf).rotate().png().toBuffer();
|
|
7799
9833
|
return { hash: core.images.save(png) };
|
|
7800
9834
|
});
|
|
7801
9835
|
app.post("/api/diff", async (req, reply) => {
|
|
@@ -7817,6 +9851,44 @@ function registerImageRoutes(app, deps) {
|
|
|
7817
9851
|
|
|
7818
9852
|
// src/release/notes.data.ts
|
|
7819
9853
|
var RELEASES = [
|
|
9854
|
+
{
|
|
9855
|
+
version: "0.9.4",
|
|
9856
|
+
date: "2026-09-14",
|
|
9857
|
+
title: "A store that asks Scenri to slow down no longer looks like an empty one.",
|
|
9858
|
+
sections: [
|
|
9859
|
+
{
|
|
9860
|
+
heading: "Products",
|
|
9861
|
+
body: "Some stores limit how fast anything can read them. Scenri now waits the way such a store asks and says what happened: an import that could not read anything tells you to try again in a few minutes, and one that got part way says how much is left. Products from an earlier import are never retired by a run that was refused, and a store that keeps saying no ends the import early rather than grinding on."
|
|
9862
|
+
},
|
|
9863
|
+
{
|
|
9864
|
+
heading: "Fixes",
|
|
9865
|
+
body: "A product picture saved in a newer format such as AVIF is found again instead of going missing."
|
|
9866
|
+
}
|
|
9867
|
+
]
|
|
9868
|
+
},
|
|
9869
|
+
{
|
|
9870
|
+
version: "0.9.3",
|
|
9871
|
+
date: "2026-09-14",
|
|
9872
|
+
title: "Paste a website, get the brand and its products.",
|
|
9873
|
+
sections: [
|
|
9874
|
+
{
|
|
9875
|
+
heading: "Brand",
|
|
9876
|
+
body: "A website address is enough to start a brand. Scenri reads the logo, the wordmark and the palette a site actually declares, and says plainly what it found and what it could not."
|
|
9877
|
+
},
|
|
9878
|
+
{
|
|
9879
|
+
heading: "Products",
|
|
9880
|
+
body: "The same address brings in the store. Pick the products you want or take the whole catalogue, and watch them arrive: each one appears in your library as it lands, with its picture, newest first. A large store no longer runs out of memory partway through, and a shop that stops answering says so instead of looking empty."
|
|
9881
|
+
},
|
|
9882
|
+
{
|
|
9883
|
+
heading: "Create",
|
|
9884
|
+
body: "The insert menus reach everything. Typing $, @, / or # opens a shortlist that pages through the rest of that catalogue, with a count so nothing is quietly cut off, and the box stays where the caret is."
|
|
9885
|
+
},
|
|
9886
|
+
{
|
|
9887
|
+
heading: "Fixes",
|
|
9888
|
+
body: "A first run on Windows no longer stops at the desktop icon question, and never asks for an administrator it does not need. Codex reports itself ready only once a real run has authenticated."
|
|
9889
|
+
}
|
|
9890
|
+
]
|
|
9891
|
+
},
|
|
7820
9892
|
{
|
|
7821
9893
|
version: "0.9.2",
|
|
7822
9894
|
date: "2026-09-07",
|
|
@@ -8787,15 +10859,15 @@ function registerDesktopRoutes(app, deps) {
|
|
|
8787
10859
|
record: null
|
|
8788
10860
|
};
|
|
8789
10861
|
}
|
|
8790
|
-
const { installDeps } = await import('./cli-
|
|
8791
|
-
const { desktopStatus } = await import('./install-
|
|
10862
|
+
const { installDeps } = await import('./cli-MQO66C5O.js');
|
|
10863
|
+
const { desktopStatus } = await import('./install-VXJPLYWM.js');
|
|
8792
10864
|
return desktopStatus(installDeps(runtime.entry));
|
|
8793
10865
|
});
|
|
8794
10866
|
const install = deps.installImpl ?? (async () => {
|
|
8795
10867
|
if (!runtime.entry) {
|
|
8796
10868
|
return { ok: false, reason: "unsupported", message: "Desktop shortcuts are not available on this system yet." };
|
|
8797
10869
|
}
|
|
8798
|
-
const { addToDesktop } = await import('./cli-
|
|
10870
|
+
const { addToDesktop } = await import('./cli-MQO66C5O.js');
|
|
8799
10871
|
return addToDesktop(runtime.entry);
|
|
8800
10872
|
});
|
|
8801
10873
|
app.get("/api/desktop", async () => {
|
|
@@ -8860,6 +10932,7 @@ function buildServer(opts) {
|
|
|
8860
10932
|
registerAccessGuard(app, opts.access);
|
|
8861
10933
|
const reserved = /* @__PURE__ */ new Map();
|
|
8862
10934
|
const runningGenerations = /* @__PURE__ */ new Map();
|
|
10935
|
+
core.catalog.reconcileInterruptedJobs();
|
|
8863
10936
|
const thumbs = createThumbStore(core);
|
|
8864
10937
|
const { scenes } = loadScenes(opts.templatesDir);
|
|
8865
10938
|
const resolveScene = sceneResolver(scenes);
|
|
@@ -8869,8 +10942,11 @@ function buildServer(opts) {
|
|
|
8869
10942
|
const e = err;
|
|
8870
10943
|
const status = err instanceof SpendCapError ? 402 : e.statusCode ?? 500;
|
|
8871
10944
|
const leaksPath = typeof e.code === "string" && /^(ENOENT|EACCES|EPERM|EISDIR|ENOTDIR)$/.test(e.code);
|
|
8872
|
-
|
|
10945
|
+
const rawRuntime = !e.statusCode && (err instanceof TypeError || err instanceof RangeError);
|
|
10946
|
+
if (rawRuntime) console.error("unexpected error:", err);
|
|
10947
|
+
reply.status(status).send({ error: leaksPath || rawRuntime ? "unexpected error" : e.message ?? "unexpected error" });
|
|
8873
10948
|
});
|
|
10949
|
+
const scrapeGuard = () => ({ allowPrivateHosts: process.env.SCENRI_SCRAPE_ALLOW_PRIVATE === "1" });
|
|
8874
10950
|
app.get("/api/brands", async () => core.store.listBrands());
|
|
8875
10951
|
app.post("/api/brands", async (req, reply) => {
|
|
8876
10952
|
const json = req.body?.brand;
|
|
@@ -8879,10 +10955,12 @@ function buildServer(opts) {
|
|
|
8879
10955
|
return core.store.createBrand(json);
|
|
8880
10956
|
});
|
|
8881
10957
|
app.post("/api/brands/from-url", async (req, reply) => {
|
|
8882
|
-
const
|
|
8883
|
-
if (
|
|
8884
|
-
const
|
|
10958
|
+
const asked = normalizeSiteUrl(req.body?.url);
|
|
10959
|
+
if (!asked.ok) return reply.status(400).send({ error: asked.message });
|
|
10960
|
+
const url = asked.url;
|
|
10961
|
+
const { brand, warnings, report } = await buildFromUrl(url, {
|
|
8885
10962
|
fetchImpl: opts.fetchImpl,
|
|
10963
|
+
guard: scrapeGuard(),
|
|
8886
10964
|
// The store names every blob `<hash>.png` and /api/images/:hash always
|
|
8887
10965
|
// serves image/png, so an un-normalized .ico or .svg here is a file lying
|
|
8888
10966
|
// about its own format — broken in the marks grid, and mislabelled to any
|
|
@@ -8890,14 +10968,11 @@ function buildServer(opts) {
|
|
|
8890
10968
|
saveAsset: async (buf) => `asset:${core.images.save(await toMarkPng(buf))}`,
|
|
8891
10969
|
// Measured as stored (post-toMarkPng), so the scrape judges the same
|
|
8892
10970
|
// pixels the compiler will one day attach.
|
|
8893
|
-
|
|
8894
|
-
const m = await sharp20(await toMarkPng(buf)).metadata();
|
|
8895
|
-
return Math.max(m.width ?? 0, m.height ?? 0) || null;
|
|
8896
|
-
},
|
|
10971
|
+
inspectMark: (buf) => inspectMark(buf, toMarkPng),
|
|
8897
10972
|
createdWith: `${meta.name}/${meta.version}`
|
|
8898
10973
|
});
|
|
8899
10974
|
const row = core.store.createBrand(brand);
|
|
8900
|
-
return { ...row, warnings };
|
|
10975
|
+
return { ...row, warnings, report };
|
|
8901
10976
|
});
|
|
8902
10977
|
app.put("/api/brands/:id", async (req, reply) => {
|
|
8903
10978
|
const json = req.body?.brand;
|
|
@@ -8974,15 +11049,14 @@ function buildServer(opts) {
|
|
|
8974
11049
|
app.post("/api/brands/:id/refresh-from-url", async (req, reply) => {
|
|
8975
11050
|
const brand = core.store.getBrand(req.params.id);
|
|
8976
11051
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8977
|
-
const
|
|
8978
|
-
if (
|
|
11052
|
+
const asked = normalizeSiteUrl(req.body?.url ?? brand.json?.meta?.website);
|
|
11053
|
+
if (!asked.ok) return reply.status(400).send({ error: asked.message });
|
|
11054
|
+
const url = asked.url;
|
|
8979
11055
|
const { brand: scraped, warnings } = await buildFromUrl(url, {
|
|
8980
11056
|
fetchImpl: opts.fetchImpl,
|
|
11057
|
+
guard: scrapeGuard(),
|
|
8981
11058
|
saveAsset: async (buf) => `asset:${core.images.save(await toMarkPng(buf))}`,
|
|
8982
|
-
|
|
8983
|
-
const m = await sharp20(await toMarkPng(buf)).metadata();
|
|
8984
|
-
return Math.max(m.width ?? 0, m.height ?? 0) || null;
|
|
8985
|
-
},
|
|
11059
|
+
inspectMark: (buf) => inspectMark(buf, toMarkPng),
|
|
8986
11060
|
createdWith: `${meta.name}/${meta.version}`
|
|
8987
11061
|
});
|
|
8988
11062
|
const { brand: merged, suggestions } = mergeScrape(brand.json, scraped);
|
|
@@ -9290,7 +11364,14 @@ function buildServer(opts) {
|
|
|
9290
11364
|
],
|
|
9291
11365
|
engineNames: () => engines.all().map((e) => ({ id: e.capabilities().id, name: e.capabilities().displayName }))
|
|
9292
11366
|
});
|
|
9293
|
-
registerCodexSetupRoutes(app, {
|
|
11367
|
+
registerCodexSetupRoutes(app, {
|
|
11368
|
+
codexSetup: opts.codexSetup,
|
|
11369
|
+
codexRunner: engines.codexRunner,
|
|
11370
|
+
envRepair: {
|
|
11371
|
+
get: () => [...ignoreEnvKeysGetter(core)()],
|
|
11372
|
+
set: (keys) => core.store.setSetting(IGNORE_ENV_KEYS_SETTING, keys.join(","))
|
|
11373
|
+
}
|
|
11374
|
+
});
|
|
9294
11375
|
app.get("/api/engines", async () => {
|
|
9295
11376
|
const list2 = [];
|
|
9296
11377
|
for (const e of engines.all()) {
|
|
@@ -9353,11 +11434,11 @@ function buildServer(opts) {
|
|
|
9353
11434
|
const out = [];
|
|
9354
11435
|
for (const h of images) {
|
|
9355
11436
|
const buf = core.images.read(h);
|
|
9356
|
-
const meta2 = await
|
|
11437
|
+
const meta2 = await sharp21(buf).metadata().catch(() => null);
|
|
9357
11438
|
if (!meta2?.width || !meta2.height) throw new Error("engine returned an undecodable image");
|
|
9358
11439
|
const oriented = (meta2.orientation ?? 1) !== 1;
|
|
9359
11440
|
out.push(
|
|
9360
|
-
buf.subarray(0, 8).equals(PNG_SIG) && !oriented ? h : core.images.save(await
|
|
11441
|
+
buf.subarray(0, 8).equals(PNG_SIG) && !oriented ? h : core.images.save(await sharp21(buf).rotate().png().toBuffer())
|
|
9361
11442
|
);
|
|
9362
11443
|
}
|
|
9363
11444
|
return out;
|
|
@@ -9369,7 +11450,7 @@ function buildServer(opts) {
|
|
|
9369
11450
|
const out = [];
|
|
9370
11451
|
for (const h of images) {
|
|
9371
11452
|
const buf = core.images.read(h);
|
|
9372
|
-
const meta2 = await
|
|
11453
|
+
const meta2 = await sharp21(buf).metadata();
|
|
9373
11454
|
if (!meta2.width || !meta2.height) {
|
|
9374
11455
|
out.push(h);
|
|
9375
11456
|
continue;
|
|
@@ -9382,7 +11463,7 @@ function buildServer(opts) {
|
|
|
9382
11463
|
}
|
|
9383
11464
|
const w = got > target ? Math.round(meta2.height * target) : meta2.width;
|
|
9384
11465
|
const hpx = got > target ? meta2.height : Math.round(meta2.width / target);
|
|
9385
|
-
const cropped = await
|
|
11466
|
+
const cropped = await sharp21(buf).resize(w, hpx, { fit: "cover", position: "attention" }).png().toBuffer();
|
|
9386
11467
|
app.log.info(
|
|
9387
11468
|
{ nodeId, got: `${meta2.width}x${meta2.height}`, want: `${w}x${hpx}` },
|
|
9388
11469
|
"canvas: cropped a drifted frame to the asked ratio"
|
|
@@ -9401,7 +11482,7 @@ function buildServer(opts) {
|
|
|
9401
11482
|
async function assertAspect(images, expect) {
|
|
9402
11483
|
const want = expect.width / expect.height;
|
|
9403
11484
|
for (const h of images) {
|
|
9404
|
-
const meta2 = await
|
|
11485
|
+
const meta2 = await sharp21(core.images.read(h)).metadata();
|
|
9405
11486
|
if (!meta2.width || !meta2.height) continue;
|
|
9406
11487
|
const got = meta2.width / meta2.height;
|
|
9407
11488
|
if (Math.abs(got - want) / want > ASPECT_TOLERANCE)
|
|
@@ -9432,7 +11513,7 @@ function buildServer(opts) {
|
|
|
9432
11513
|
if (post) own = await post(own);
|
|
9433
11514
|
if (expect) await assertAspect(own, expect);
|
|
9434
11515
|
try {
|
|
9435
|
-
const meta2 = await
|
|
11516
|
+
const meta2 = await sharp21(core.images.read(own[0])).metadata();
|
|
9436
11517
|
const node = core.store.getNode(id);
|
|
9437
11518
|
if (node && meta2.width && meta2.height) {
|
|
9438
11519
|
const brief = node.brief ?? {};
|
|
@@ -9541,7 +11622,7 @@ function buildServer(opts) {
|
|
|
9541
11622
|
crop: window
|
|
9542
11623
|
});
|
|
9543
11624
|
const work2 = async () => ({
|
|
9544
|
-
images: [core.images.save(await
|
|
11625
|
+
images: [core.images.save(await sharp21(args.srcBuf).extract(window).png().toBuffer())],
|
|
9545
11626
|
costUsd: 0
|
|
9546
11627
|
});
|
|
9547
11628
|
void runNode([node2.id], null, 0, work2, { width: plan2.width, height: plan2.height }).catch(
|
|
@@ -9566,7 +11647,7 @@ function buildServer(opts) {
|
|
|
9566
11647
|
if (!srcHash || !core.images.has(String(srcHash)))
|
|
9567
11648
|
return reply.status(400).send({ error: "edit needs a parent node with an image (sourceImage)" });
|
|
9568
11649
|
const srcBuf = core.images.read(String(srcHash));
|
|
9569
|
-
const srcMeta = await
|
|
11650
|
+
const srcMeta = await sharp21(srcBuf).metadata();
|
|
9570
11651
|
if (!srcMeta.width || !srcMeta.height) return reply.status(400).send({ error: "source image unreadable" });
|
|
9571
11652
|
return runCropNode({
|
|
9572
11653
|
parentId: cropParentId,
|
|
@@ -9781,7 +11862,7 @@ function buildServer(opts) {
|
|
|
9781
11862
|
);
|
|
9782
11863
|
}
|
|
9783
11864
|
const srcBuf = core.images.read(String(srcHash));
|
|
9784
|
-
const srcMeta = await
|
|
11865
|
+
const srcMeta = await sharp21(srcBuf).metadata();
|
|
9785
11866
|
if (srcMeta.width && srcMeta.height) expectShape = { width: srcMeta.width, height: srcMeta.height };
|
|
9786
11867
|
const parentFormat = parent?.brief?.tokens?.find((t) => t?.t === "format");
|
|
9787
11868
|
const parentNominal = parentFormat && Number(parentFormat.w) > 0 && Number(parentFormat.h) > 0 ? { width: Number(parentFormat.w), height: Number(parentFormat.h) } : null;
|
|
@@ -9815,7 +11896,7 @@ function buildServer(opts) {
|
|
|
9815
11896
|
} else if (decision.op === "extend") {
|
|
9816
11897
|
if (decision.assist) {
|
|
9817
11898
|
expandAssist = { width: decision.assist.width, height: decision.assist.height };
|
|
9818
|
-
workBuf = await
|
|
11899
|
+
workBuf = await sharp21(srcBuf).extract(decision.assist).png().toBuffer();
|
|
9819
11900
|
workSize = { width: decision.assist.width, height: decision.assist.height };
|
|
9820
11901
|
}
|
|
9821
11902
|
expandPlan = planExpand(workSize, targetRatio);
|
|
@@ -9836,7 +11917,7 @@ function buildServer(opts) {
|
|
|
9836
11917
|
const fit = fitExpandToBudget(expandPlan, workSize, runEngine.capabilities().editPixelBudget);
|
|
9837
11918
|
if (fit.scale < 1) {
|
|
9838
11919
|
expandPlan = fit.plan;
|
|
9839
|
-
workBuf = await
|
|
11920
|
+
workBuf = await sharp21(workBuf).resize(fit.source.width, fit.source.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer();
|
|
9840
11921
|
workSize = fit.source;
|
|
9841
11922
|
extraWarnings.push(
|
|
9842
11923
|
`${runEngine.capabilities().displayName} draws about ${((runEngine.capabilities().editPixelBudget ?? 0) / 1e6).toFixed(1)} megapixels, so this shape continues as a ${fit.plan.width}x${fit.plan.height} frame with the photograph riding inside it at ${fit.source.width}x${fit.source.height}. Nothing is upscaled; the stored size is the size the engine truly drew.`
|
|
@@ -9859,7 +11940,7 @@ function buildServer(opts) {
|
|
|
9859
11940
|
if (editPixelBudget && stepped && (stepped.width !== srcMeta.width || stepped.height !== srcMeta.height)) {
|
|
9860
11941
|
sentSize = stepped;
|
|
9861
11942
|
budgetSourceHash = core.images.save(
|
|
9862
|
-
await
|
|
11943
|
+
await sharp21(srcBuf).resize(sentSize.width, sentSize.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer()
|
|
9863
11944
|
);
|
|
9864
11945
|
if (!gradeOnlyAsk)
|
|
9865
11946
|
extraWarnings.push(
|
|
@@ -9999,11 +12080,11 @@ function buildServer(opts) {
|
|
|
9999
12080
|
const original = editedFrom ? core.images.read(editedFrom) : null;
|
|
10000
12081
|
const localScope = kind === "edit" && !plan && editScope === "local" && original;
|
|
10001
12082
|
const enforceEditCanvas = async (images) => {
|
|
10002
|
-
const srcMeta = await
|
|
12083
|
+
const srcMeta = await sharp21(original).metadata();
|
|
10003
12084
|
if (!srcMeta.width || !srcMeta.height) return images;
|
|
10004
12085
|
const out = [];
|
|
10005
12086
|
for (const h of images) {
|
|
10006
|
-
const meta2 = await
|
|
12087
|
+
const meta2 = await sharp21(core.images.read(h)).metadata();
|
|
10007
12088
|
const got = { width: meta2.width ?? 0, height: meta2.height ?? 0 };
|
|
10008
12089
|
const verdict = judgeEditSize({ width: srcMeta.width, height: srcMeta.height }, got, {
|
|
10009
12090
|
pixelBudget: runEngine.capabilities().editPixelBudget
|
|
@@ -10033,7 +12114,7 @@ function buildServer(opts) {
|
|
|
10033
12114
|
);
|
|
10034
12115
|
out.push(
|
|
10035
12116
|
core.images.save(
|
|
10036
|
-
await
|
|
12117
|
+
await sharp21(core.images.read(h)).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).png().toBuffer()
|
|
10037
12118
|
)
|
|
10038
12119
|
);
|
|
10039
12120
|
try {
|
|
@@ -10055,7 +12136,7 @@ function buildServer(opts) {
|
|
|
10055
12136
|
const out = [];
|
|
10056
12137
|
for (const h of images) {
|
|
10057
12138
|
const answer = core.images.read(h);
|
|
10058
|
-
const got = await
|
|
12139
|
+
const got = await sharp21(answer).metadata();
|
|
10059
12140
|
if (got.width !== plan.width || got.height !== plan.height)
|
|
10060
12141
|
app.log.info(
|
|
10061
12142
|
{ nodeId: node.id, got: `${got.width}x${got.height}`, want: `${plan.width}x${plan.height}` },
|
|
@@ -10216,6 +12297,7 @@ function buildServer(opts) {
|
|
|
10216
12297
|
while (runningGenerations.size > 0 && Date.now() < deadline) {
|
|
10217
12298
|
await new Promise((r) => setTimeout(r, 25));
|
|
10218
12299
|
}
|
|
12300
|
+
await settleCatalogImports();
|
|
10219
12301
|
await thumbs.settle();
|
|
10220
12302
|
await app.close();
|
|
10221
12303
|
core.close();
|
|
@@ -10388,9 +12470,9 @@ async function run() {
|
|
|
10388
12470
|
}
|
|
10389
12471
|
}
|
|
10390
12472
|
const ownEntry = fileURLToPath(import.meta.url);
|
|
10391
|
-
const { addToDesktop, installDeps } = await import('./cli-
|
|
10392
|
-
const { refreshLauncher } = await import('./refresh-
|
|
10393
|
-
const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-
|
|
12473
|
+
const { addToDesktop, installDeps } = await import('./cli-MQO66C5O.js');
|
|
12474
|
+
const { refreshLauncher } = await import('./refresh-5HBDLLLB.js');
|
|
12475
|
+
const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-W6NJXMWT.js');
|
|
10394
12476
|
const { launcherInstalled } = await import('./paths-6ZXRZUHY.js');
|
|
10395
12477
|
const meta = readMeta();
|
|
10396
12478
|
void refreshLauncher({ ...installDeps(ownEntry), ownEntry, installKind, pkg: meta.name }).then((r) => {
|
|
@@ -10412,7 +12494,7 @@ async function run() {
|
|
|
10412
12494
|
add: () => addToDesktop(ownEntry),
|
|
10413
12495
|
decline: () => core.store.setSetting("desktop.prompt", "declined"),
|
|
10414
12496
|
say: (line) => console.log(line)
|
|
10415
|
-
});
|
|
12497
|
+
}).catch(() => console.log(" Could not ask about the desktop icon. Scenri is running anyway."));
|
|
10416
12498
|
console.log("");
|
|
10417
12499
|
}
|
|
10418
12500
|
}
|
|
@@ -10422,8 +12504,8 @@ async function verify() {
|
|
|
10422
12504
|
const db = new Database(":memory:");
|
|
10423
12505
|
db.pragma("user_version");
|
|
10424
12506
|
db.close();
|
|
10425
|
-
const { default:
|
|
10426
|
-
await
|
|
12507
|
+
const { default: sharp22 } = await import('sharp');
|
|
12508
|
+
await sharp22({ create: { width: 1, height: 1, channels: 3, background: "#000" } }).png().toBuffer();
|
|
10427
12509
|
console.log(JSON.stringify({ ok: true, version: readMeta().version }));
|
|
10428
12510
|
} catch (err) {
|
|
10429
12511
|
console.log(JSON.stringify({ ok: false, error: String(err?.message ?? err) }));
|