scenri 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +3 -1
- package/dist/{chunk-SDVL5OJF.js → chunk-3BGDI4HI.js} +54 -16
- package/dist/{chunk-OJAG3FRX.js → chunk-Y5L3KLX5.js} +98 -9
- package/dist/{cli-FYWM4HBN.js → cli-SX46UQO6.js} +15 -7
- package/dist/index.js +2 -2
- package/dist/install-VXJPLYWM.js +4 -0
- package/dist/offer-W6NJXMWT.js +71 -0
- package/dist/{refresh-FKQABJOC.js → refresh-5HBDLLLB.js} +3 -3
- package/dist/serve.js +2571 -621
- package/dist/{src-CSR34EBZ.js → src-F3KJZNZT.js} +3 -3
- package/package.json +1 -1
- package/studio-dist/assets/{index-DfAHZRYM.css → index-BGts89lc.css} +1 -1
- package/studio-dist/assets/index-eKmAemHO.js +104 -0
- package/studio-dist/index.html +2 -2
- package/dist/install-4W6AARK6.js +0 -4
- package/dist/offer-IVTNHIDJ.js +0 -47
- package/studio-dist/assets/index-CqJgKSNM.js +0 -104
package/dist/serve.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createUpdateChecker, classify, isReleaseTriplet, findNpm, stageVersion } from './chunk-FYQ5BAFA.js';
|
|
2
2
|
import { shouldAdoptRunning, anotherScenriLines, portBusyLines } from './chunk-CMLJGPYC.js';
|
|
3
|
-
import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-
|
|
3
|
+
import { SchemaTooNewError, createCore, BUDGET_EXHAUSTED, SpendCapError, budgetSize, EDIT_REFERENCE_ROLE_DIRECTIVE, REFERENCE_ROLE_DIRECTIVE, ratioLabel, searchTerms, termMatches, SCHEMA_VERSION, ASPECT_TOLERANCE } from './chunk-Y5L3KLX5.js';
|
|
4
4
|
import { detectInstallKind } from './chunk-PAIAXRAC.js';
|
|
5
5
|
export { detectInstallKind } from './chunk-PAIAXRAC.js';
|
|
6
6
|
import { readMeta, repoSlug } from './chunk-Y3ZPBPLP.js';
|
|
@@ -12,7 +12,7 @@ import { homedir, networkInterfaces, tmpdir } from 'os';
|
|
|
12
12
|
import { randomBytes, randomUUID, timingSafeEqual, createHash } from 'crypto';
|
|
13
13
|
import { readFile, copyFile, stat, readdir, mkdtemp, rm, access, rename, unlink, writeFile } from 'fs/promises';
|
|
14
14
|
import { spawn } from 'child_process';
|
|
15
|
-
import
|
|
15
|
+
import sharp21 from 'sharp';
|
|
16
16
|
import Fastify from 'fastify';
|
|
17
17
|
import fastifyStatic from '@fastify/static';
|
|
18
18
|
import fastifyMultipart from '@fastify/multipart';
|
|
@@ -20,6 +20,8 @@ import JSZip from 'jszip';
|
|
|
20
20
|
import { Ajv2020 } from 'ajv/dist/2020.js';
|
|
21
21
|
import addFormats from 'ajv-formats';
|
|
22
22
|
import * as cheerio from 'cheerio';
|
|
23
|
+
import { lookup } from 'dns/promises';
|
|
24
|
+
import { isIP } from 'net';
|
|
23
25
|
import { parse } from 'node-html-parser';
|
|
24
26
|
import pixelmatch from 'pixelmatch';
|
|
25
27
|
import { PNG } from 'pngjs';
|
|
@@ -687,6 +689,108 @@ function resolveCodex(platform, spawnImpl, timeoutMs = WHERE_TIMEOUT_MS) {
|
|
|
687
689
|
});
|
|
688
690
|
}
|
|
689
691
|
|
|
692
|
+
// ../engines/codex/src/childEnv.ts
|
|
693
|
+
function buildChildEnv(parent, drop) {
|
|
694
|
+
if (drop.length === 0) return { ...parent };
|
|
695
|
+
const unwanted = new Set(drop.map((name) => name.toUpperCase()));
|
|
696
|
+
const child = {};
|
|
697
|
+
for (const [name, value] of Object.entries(parent)) {
|
|
698
|
+
if (unwanted.has(name.toUpperCase())) continue;
|
|
699
|
+
child[name] = value;
|
|
700
|
+
}
|
|
701
|
+
return child;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
// ../engines/codex/src/classify.ts
|
|
705
|
+
var CONFLICT_ENV_KEYS = ["CODEX_API_KEY", "CODEX_ACCESS_TOKEN", "OPENAI_API_KEY"];
|
|
706
|
+
function presentConflictKeys(env, ignored = []) {
|
|
707
|
+
const dropped = new Set(ignored.map((n) => n.toUpperCase()));
|
|
708
|
+
const present = [];
|
|
709
|
+
for (const name of CONFLICT_ENV_KEYS) {
|
|
710
|
+
if (dropped.has(name)) continue;
|
|
711
|
+
const value = env[name];
|
|
712
|
+
if (typeof value === "string" && value.trim() !== "") present.push(name);
|
|
713
|
+
}
|
|
714
|
+
return present;
|
|
715
|
+
}
|
|
716
|
+
function listKeys(keys) {
|
|
717
|
+
if (keys.length === 0) return "";
|
|
718
|
+
if (keys.length === 1) return keys[0];
|
|
719
|
+
return `${keys.slice(0, -1).join(", ")} and ${keys[keys.length - 1]}`;
|
|
720
|
+
}
|
|
721
|
+
function conflictReason(keys) {
|
|
722
|
+
return `Codex is signed in, but ${listKeys(keys)} in this computer's environment is overriding that sign-in and OpenAI rejected it.`;
|
|
723
|
+
}
|
|
724
|
+
var NOT_FOUND = /failed to spawn codex|\bENOENT\b|command not found|is not recognized as an internal/i;
|
|
725
|
+
var TOO_OLD = /requires a newer version of Codex|is too old/i;
|
|
726
|
+
var USAGE_LIMIT = /usage limit/i;
|
|
727
|
+
var SIGNED_OUT = /not logged in|login required|run .{0,3}codex login|no credentials found/i;
|
|
728
|
+
var UNAUTHORIZED = /\b401\b|unauthorized/i;
|
|
729
|
+
var BAD_KEY = /incorrect api key provided|invalid_api_key|invalid api key/i;
|
|
730
|
+
var NETWORK = /ENOTFOUND|ECONNREFUSED|ECONNRESET|EAI_AGAIN|getaddrinfo|error sending request|dns error|tls handshake/i;
|
|
731
|
+
var NO_OUTPUT = /produced no output for \d+s/i;
|
|
732
|
+
var TIMED_OUT = /timed out after/i;
|
|
733
|
+
var ALREADY_CONFLICT = /environment is overriding that sign-in/i;
|
|
734
|
+
function classifyCodexFailure(input) {
|
|
735
|
+
const text = input.text ?? "";
|
|
736
|
+
const conflictKeys = presentConflictKeys(input.env, input.ignored);
|
|
737
|
+
const at = (code, reason) => ({ code, conflictKeys, reason });
|
|
738
|
+
if (ALREADY_CONFLICT.test(text)) return at("CODEX_AUTH_CONFLICT", conflictReason(conflictKeys));
|
|
739
|
+
if (NOT_FOUND.test(text)) return at("CODEX_NOT_FOUND", "Codex CLI is not installed on this computer.");
|
|
740
|
+
if (TOO_OLD.test(text)) return at("CODEX_VERSION_TOO_OLD", "Codex CLI on this computer is too old.");
|
|
741
|
+
if (USAGE_LIMIT.test(text)) return at("CODEX_USAGE_LIMIT", "Your Codex plan's usage limit is used up.");
|
|
742
|
+
if (SIGNED_OUT.test(text)) return at("CODEX_NOT_AUTHENTICATED", "Codex CLI is signed out on this computer.");
|
|
743
|
+
if (UNAUTHORIZED.test(text)) {
|
|
744
|
+
if (BAD_KEY.test(text) && conflictKeys.length > 0) return at("CODEX_AUTH_CONFLICT", conflictReason(conflictKeys));
|
|
745
|
+
return at("CODEX_401", "OpenAI did not accept the Codex sign-in on this computer.");
|
|
746
|
+
}
|
|
747
|
+
if (NETWORK.test(text)) return at("NETWORK_FAILURE", "Codex could not reach OpenAI.");
|
|
748
|
+
if (NO_OUTPUT.test(text)) return at("CODEX_NO_OUTPUT", "Codex never started answering.");
|
|
749
|
+
if (TIMED_OUT.test(text)) return at("CODEX_TIMEOUT", "Codex ran out of time.");
|
|
750
|
+
if (text.trim() !== "") return at("CODEX_EXECUTION_FAILED", "Codex stopped with an error.");
|
|
751
|
+
return at("UNKNOWN", "Codex stopped for a reason it did not give.");
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// ../engines/codex/src/connect.ts
|
|
755
|
+
var CONNECT_PROMPT = "Reply with the single word READY. Do not use any tools. Do not generate an image. Do not read or write any files.";
|
|
756
|
+
var CONNECT_TIMEOUT_MS = 45e3;
|
|
757
|
+
var CONNECT_TTL_MS = 10 * 6e4;
|
|
758
|
+
function connectFingerprint(p) {
|
|
759
|
+
return [p.command, p.version ?? "unknown", [...p.present].sort().join("+"), [...p.ignored].sort().join("+")].join(
|
|
760
|
+
"|"
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
function outcomeFor(code) {
|
|
764
|
+
switch (code) {
|
|
765
|
+
case "CODEX_AUTH_CONFLICT":
|
|
766
|
+
case "CODEX_401":
|
|
767
|
+
case "CODEX_NOT_AUTHENTICATED":
|
|
768
|
+
case "CODEX_NOT_FOUND":
|
|
769
|
+
case "CODEX_VERSION_TOO_OLD":
|
|
770
|
+
return "refused";
|
|
771
|
+
default:
|
|
772
|
+
return "unproven";
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
function availabilityFrom(shallow, conn) {
|
|
776
|
+
if (!shallow.ok) return shallow;
|
|
777
|
+
if (!conn || conn.outcome !== "refused" || !conn.failure) return shallow;
|
|
778
|
+
const { code, reason } = conn.failure;
|
|
779
|
+
switch (code) {
|
|
780
|
+
case "CODEX_AUTH_CONFLICT":
|
|
781
|
+
return { ok: false, reason, code: "env-conflict" };
|
|
782
|
+
case "CODEX_401":
|
|
783
|
+
case "CODEX_NOT_AUTHENTICATED":
|
|
784
|
+
return { ok: false, reason, code: "not-authenticated" };
|
|
785
|
+
case "CODEX_VERSION_TOO_OLD":
|
|
786
|
+
return { ok: false, reason, code: "update-needed" };
|
|
787
|
+
case "CODEX_NOT_FOUND":
|
|
788
|
+
return { ok: false, reason, code: "not-installed" };
|
|
789
|
+
default:
|
|
790
|
+
return shallow;
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
|
|
690
794
|
// ../engines/codex/src/run.ts
|
|
691
795
|
var NOT_INSTALLED_REASON = "Codex CLI is not installed on this computer";
|
|
692
796
|
var NOT_AUTHENTICATED_REASON = "Codex CLI is installed but not signed in";
|
|
@@ -741,6 +845,7 @@ function codexFailureDetail(stderr, stdout) {
|
|
|
741
845
|
return tailOf(marked) || tailOf(body) || tailOf(stdout.trim()) || tailOf(stderr.trim());
|
|
742
846
|
}
|
|
743
847
|
var MODEL_NEEDS_NEWER_CODEX = /The '([^']+)' model requires a newer version of Codex/;
|
|
848
|
+
var BAD_KEY_401 = /incorrect api key provided|invalid_api_key|invalid api key/i;
|
|
744
849
|
function tooOldForModel(version, model) {
|
|
745
850
|
return `Codex CLI ${version ?? "on this computer"} is too old for the model it is set to, ${model}.`;
|
|
746
851
|
}
|
|
@@ -781,6 +886,10 @@ function createRunner(opts = {}) {
|
|
|
781
886
|
const probeTtlMs = opts.probeTtlMs ?? PROBE_TTL_MS;
|
|
782
887
|
const firstOutputMs = opts.firstOutputMs ?? FIRST_OUTPUT_TIMEOUT_MS;
|
|
783
888
|
const platform = opts.platform ?? process.platform;
|
|
889
|
+
const connectTtlMs = opts.connectTtlMs ?? CONNECT_TTL_MS;
|
|
890
|
+
const connectTimeoutMs = opts.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
|
|
891
|
+
const parentEnv = opts.env ?? process.env;
|
|
892
|
+
const ignoreEnvKeys = opts.ignoreEnvKeys ?? (() => []);
|
|
784
893
|
const killCodex = (child) => killTree(child, platform, spawnImpl);
|
|
785
894
|
let resolved = null;
|
|
786
895
|
async function resolution() {
|
|
@@ -792,7 +901,8 @@ function createRunner(opts = {}) {
|
|
|
792
901
|
const winArg = (a) => `"${a.replace(/[\r\n]+/g, " ").replace(/"/g, "'").replace(/%/g, " percent ")}"`;
|
|
793
902
|
const spawnCodex = (exe, args, stdinOpen) => {
|
|
794
903
|
const stdio = [stdinOpen ? "pipe" : "ignore", "pipe", "pipe"];
|
|
795
|
-
|
|
904
|
+
const env = buildChildEnv(parentEnv, ignoreEnvKeys());
|
|
905
|
+
return exe.direct ? spawnImpl(exe.command, args, { stdio, env, ...platform !== "win32" ? { detached: true } : {} }) : spawnImpl([exe.command, ...args.map(winArg)].join(" "), [], { stdio, env, shell: true });
|
|
796
906
|
};
|
|
797
907
|
async function run2(args, signal, io) {
|
|
798
908
|
const exe = await resolution();
|
|
@@ -907,6 +1017,16 @@ function createRunner(opts = {}) {
|
|
|
907
1017
|
);
|
|
908
1018
|
return;
|
|
909
1019
|
}
|
|
1020
|
+
const conflicting = presentConflictKeys(parentEnv, ignoreEnvKeys());
|
|
1021
|
+
if (conflicting.length > 0 && /\b401\b|unauthorized/i.test(stderr) && BAD_KEY_401.test(stderr)) {
|
|
1022
|
+
noteConnection("refused", {
|
|
1023
|
+
code: "CODEX_AUTH_CONFLICT",
|
|
1024
|
+
conflictKeys: conflicting,
|
|
1025
|
+
reason: conflictReason(conflicting)
|
|
1026
|
+
});
|
|
1027
|
+
finish(`exit-${code ?? "unknown"}`, () => reject(new Error(conflictReason(conflicting))));
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
910
1030
|
const snippet2 = codexFailureDetail(stderr, stdout);
|
|
911
1031
|
finish(
|
|
912
1032
|
`exit-${code ?? "unknown"}`,
|
|
@@ -966,6 +1086,23 @@ function createRunner(opts = {}) {
|
|
|
966
1086
|
return avail;
|
|
967
1087
|
}
|
|
968
1088
|
let cached = null;
|
|
1089
|
+
let connCache = null;
|
|
1090
|
+
let connectInFlight = null;
|
|
1091
|
+
function fingerprintFor(exe, version) {
|
|
1092
|
+
const ignored = ignoreEnvKeys();
|
|
1093
|
+
return connectFingerprint({
|
|
1094
|
+
command: exe.command,
|
|
1095
|
+
version,
|
|
1096
|
+
present: presentConflictKeys(parentEnv, ignored),
|
|
1097
|
+
ignored
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
function freshConnection(exe, version) {
|
|
1101
|
+
if (!connCache || connectTtlMs <= 0) return null;
|
|
1102
|
+
if (Date.now() - connCache.at >= connectTtlMs) return null;
|
|
1103
|
+
if (connCache.fingerprint !== fingerprintFor(exe, version)) return null;
|
|
1104
|
+
return connCache;
|
|
1105
|
+
}
|
|
969
1106
|
async function probe() {
|
|
970
1107
|
if (process.env.SCENRI_NO_CODEX === "1") {
|
|
971
1108
|
return { ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" };
|
|
@@ -978,52 +1115,128 @@ function createRunner(opts = {}) {
|
|
|
978
1115
|
return value;
|
|
979
1116
|
}
|
|
980
1117
|
async function probeUncached() {
|
|
1118
|
+
const { avail, exe, version } = await probeLadder();
|
|
1119
|
+
return availabilityFrom(avail, freshConnection(exe, version));
|
|
1120
|
+
}
|
|
1121
|
+
async function probeLadder() {
|
|
981
1122
|
resolved = await resolveCodex(platform, spawnImpl);
|
|
982
1123
|
const exe = resolved;
|
|
983
1124
|
const ver = await probeSpawn(exe, ["--version"]);
|
|
984
1125
|
if (ver.outcome === "timeout") {
|
|
985
|
-
return
|
|
1126
|
+
return {
|
|
1127
|
+
avail: verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, null),
|
|
1128
|
+
exe,
|
|
1129
|
+
version: null
|
|
1130
|
+
};
|
|
986
1131
|
}
|
|
987
1132
|
if (ver.outcome !== "ok") {
|
|
988
|
-
return
|
|
1133
|
+
return {
|
|
1134
|
+
avail: verdict({ ok: false, reason: NOT_INSTALLED_REASON, code: "not-installed" }, exe, null),
|
|
1135
|
+
exe,
|
|
1136
|
+
version: null
|
|
1137
|
+
};
|
|
989
1138
|
}
|
|
990
1139
|
const version = parseCodexVersion(ver.stdout);
|
|
991
1140
|
knownVersion = version;
|
|
992
1141
|
if (version && !versionAtLeast(version, MIN_CODEX_VERSION)) {
|
|
993
|
-
return
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1142
|
+
return {
|
|
1143
|
+
avail: verdict(
|
|
1144
|
+
{
|
|
1145
|
+
ok: false,
|
|
1146
|
+
reason: `Codex CLI ${version} is too old. Scenri needs ${MIN_CODEX_VERSION} or newer.`,
|
|
1147
|
+
code: "update-needed"
|
|
1148
|
+
},
|
|
1149
|
+
exe,
|
|
1150
|
+
version
|
|
1151
|
+
),
|
|
999
1152
|
exe,
|
|
1000
1153
|
version
|
|
1001
|
-
|
|
1154
|
+
};
|
|
1002
1155
|
}
|
|
1003
1156
|
if (tooOldFor) {
|
|
1004
1157
|
if (version === tooOldFor.version) {
|
|
1005
|
-
return
|
|
1006
|
-
|
|
1158
|
+
return {
|
|
1159
|
+
avail: verdict(
|
|
1160
|
+
{ ok: false, reason: tooOldForModel(version, tooOldFor.model), code: "update-needed" },
|
|
1161
|
+
exe,
|
|
1162
|
+
version
|
|
1163
|
+
),
|
|
1007
1164
|
exe,
|
|
1008
1165
|
version
|
|
1009
|
-
|
|
1166
|
+
};
|
|
1010
1167
|
}
|
|
1011
1168
|
tooOldFor = null;
|
|
1012
1169
|
}
|
|
1013
1170
|
const login = await probeSpawn(exe, ["login", "status"]);
|
|
1014
1171
|
if (login.outcome === "ok") {
|
|
1015
|
-
return verdict({ ok: true }, exe, version);
|
|
1172
|
+
return { avail: verdict({ ok: true }, exe, version), exe, version };
|
|
1016
1173
|
}
|
|
1017
1174
|
if (login.outcome === "nonzero") {
|
|
1018
|
-
return
|
|
1175
|
+
return {
|
|
1176
|
+
avail: verdict({ ok: false, reason: NOT_AUTHENTICATED_REASON, code: "not-authenticated" }, exe, version),
|
|
1177
|
+
exe,
|
|
1178
|
+
version
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
return { avail: verdict({ ok: false, reason: UNVERIFIED_REASON, code: "unverified" }, exe, version), exe, version };
|
|
1182
|
+
}
|
|
1183
|
+
async function connect(o = {}) {
|
|
1184
|
+
if (connectInFlight) return connectInFlight;
|
|
1185
|
+
const pending = (async () => {
|
|
1186
|
+
const { avail, exe, version } = await probeLadder();
|
|
1187
|
+
const fingerprint = fingerprintFor(exe, version);
|
|
1188
|
+
if (!o.force) {
|
|
1189
|
+
const fresh = freshConnection(exe, version);
|
|
1190
|
+
if (fresh) return fresh;
|
|
1191
|
+
}
|
|
1192
|
+
if (!avail.ok) {
|
|
1193
|
+
return { outcome: "unproven", at: Date.now(), fingerprint };
|
|
1194
|
+
}
|
|
1195
|
+
let result;
|
|
1196
|
+
try {
|
|
1197
|
+
await withWorkDir(
|
|
1198
|
+
(dir) => run2(execArgs(dir), void 0, {
|
|
1199
|
+
stdin: CONNECT_PROMPT,
|
|
1200
|
+
timeoutMs: connectTimeoutMs,
|
|
1201
|
+
label: "connect"
|
|
1202
|
+
})
|
|
1203
|
+
);
|
|
1204
|
+
result = { outcome: "proven", at: Date.now(), fingerprint };
|
|
1205
|
+
} catch (err) {
|
|
1206
|
+
const failure = classifyCodexFailure({
|
|
1207
|
+
text: err instanceof Error ? err.message : String(err),
|
|
1208
|
+
env: parentEnv,
|
|
1209
|
+
ignored: ignoreEnvKeys()
|
|
1210
|
+
});
|
|
1211
|
+
result = { outcome: outcomeFor(failure.code), failure, at: Date.now(), fingerprint };
|
|
1212
|
+
}
|
|
1213
|
+
connCache = result;
|
|
1214
|
+
cached = null;
|
|
1215
|
+
return result;
|
|
1216
|
+
})();
|
|
1217
|
+
connectInFlight = pending;
|
|
1218
|
+
try {
|
|
1219
|
+
return await pending;
|
|
1220
|
+
} finally {
|
|
1221
|
+
connectInFlight = null;
|
|
1019
1222
|
}
|
|
1020
|
-
|
|
1223
|
+
}
|
|
1224
|
+
function noteConnection(outcome, failure) {
|
|
1225
|
+
if (outcome === "unproven") return;
|
|
1226
|
+
const exe = resolved;
|
|
1227
|
+
if (!exe) return;
|
|
1228
|
+
connCache = { outcome, failure, at: Date.now(), fingerprint: fingerprintFor(exe, knownVersion) };
|
|
1229
|
+
cached = null;
|
|
1021
1230
|
}
|
|
1022
1231
|
function invalidateProbe() {
|
|
1023
1232
|
cached = null;
|
|
1024
1233
|
resolved = null;
|
|
1025
1234
|
}
|
|
1026
|
-
|
|
1235
|
+
function invalidateConnection() {
|
|
1236
|
+
connCache = null;
|
|
1237
|
+
cached = null;
|
|
1238
|
+
}
|
|
1239
|
+
return { run: run2, withWorkDir, probe, invalidateProbe, connect, invalidateConnection, noteConnection };
|
|
1027
1240
|
}
|
|
1028
1241
|
var OUT_FILE = "analysis.json";
|
|
1029
1242
|
function createCodexAnalyzer(opts = {}) {
|
|
@@ -1182,6 +1395,7 @@ function pick(v, allowed, max) {
|
|
|
1182
1395
|
var INSTALL_COMMAND = "npm install -g @openai/codex";
|
|
1183
1396
|
var INSTALL_DOCS_URL = "https://developers.openai.com/codex/cli";
|
|
1184
1397
|
var INSTALL_COMMAND_SUDO = "sudo npm install -g @openai/codex";
|
|
1398
|
+
var INSTALL_COMMAND_WINDOWS = 'powershell -ExecutionPolicy ByPass -c "irm https://chatgpt.com/codex/install.ps1 | iex"';
|
|
1185
1399
|
var DEFAULT_INSTALL_TIMEOUT_MS = 18e4;
|
|
1186
1400
|
function stateFrom(avail) {
|
|
1187
1401
|
if (avail.ok) return "ready";
|
|
@@ -1189,6 +1403,7 @@ function stateFrom(avail) {
|
|
|
1189
1403
|
case "not-authenticated":
|
|
1190
1404
|
case "update-needed":
|
|
1191
1405
|
case "unverified":
|
|
1406
|
+
case "env-conflict":
|
|
1192
1407
|
return avail.code;
|
|
1193
1408
|
default:
|
|
1194
1409
|
return "not-installed";
|
|
@@ -1199,6 +1414,7 @@ function createCodexSetup(opts = {}) {
|
|
|
1199
1414
|
const platform = opts.platform ?? process.platform;
|
|
1200
1415
|
const runner = opts.runner ?? createRunner(opts);
|
|
1201
1416
|
const installTimeoutMs = opts.installTimeoutMs ?? DEFAULT_INSTALL_TIMEOUT_MS;
|
|
1417
|
+
const ignoreEnvKeys = opts.ignoreEnvKeys ?? (() => []);
|
|
1202
1418
|
function run2(cmd, args, timeoutMs) {
|
|
1203
1419
|
return new Promise((resolve) => {
|
|
1204
1420
|
let settled = false;
|
|
@@ -1230,11 +1446,22 @@ function createCodexSetup(opts = {}) {
|
|
|
1230
1446
|
});
|
|
1231
1447
|
}
|
|
1232
1448
|
return {
|
|
1233
|
-
async status() {
|
|
1449
|
+
async status(o = {}) {
|
|
1234
1450
|
runner.invalidateProbe();
|
|
1451
|
+
if (o.force) runner.invalidateConnection();
|
|
1452
|
+
const conn = await runner.connect();
|
|
1235
1453
|
const avail = await runner.probe();
|
|
1454
|
+
const state = stateFrom(avail);
|
|
1236
1455
|
const setupPlatform = platform === "win32" ? "windows" : platform === "darwin" ? "mac" : "linux";
|
|
1237
|
-
return {
|
|
1456
|
+
return {
|
|
1457
|
+
state,
|
|
1458
|
+
reason: avail.reason,
|
|
1459
|
+
platform: setupPlatform,
|
|
1460
|
+
conflictKeys: state === "env-conflict" ? conn.failure?.conflictKeys ?? [] : [],
|
|
1461
|
+
ignoredKeys: [...ignoreEnvKeys()].filter(
|
|
1462
|
+
(k) => CONFLICT_ENV_KEYS.includes(k)
|
|
1463
|
+
)
|
|
1464
|
+
};
|
|
1238
1465
|
},
|
|
1239
1466
|
async install() {
|
|
1240
1467
|
const res = await run2("npm", ["install", "-g", "@openai/codex"], installTimeoutMs);
|
|
@@ -1251,6 +1478,14 @@ function createCodexSetup(opts = {}) {
|
|
|
1251
1478
|
}
|
|
1252
1479
|
return { ok: true };
|
|
1253
1480
|
}
|
|
1481
|
+
if (platform === "win32" && /EPERM|EACCES|EBUSY|permission denied|operation not permitted/i.test(res.stderr)) {
|
|
1482
|
+
return {
|
|
1483
|
+
ok: false,
|
|
1484
|
+
fallbackCommand: INSTALL_COMMAND_WINDOWS,
|
|
1485
|
+
docsUrl: INSTALL_DOCS_URL,
|
|
1486
|
+
detail: "npm could not write its files. This is almost never about administrator rights: close Codex and any terminal window using it, then try again. If it keeps failing, the command below is OpenAI\u2019s own installer, which does not go through npm."
|
|
1487
|
+
};
|
|
1488
|
+
}
|
|
1254
1489
|
if (platform !== "win32" && /EACCES|permission denied/i.test(res.stderr)) {
|
|
1255
1490
|
return {
|
|
1256
1491
|
ok: false,
|
|
@@ -1452,6 +1687,7 @@ function createCodexEngine(opts) {
|
|
|
1452
1687
|
fatal = err;
|
|
1453
1688
|
inner.abort();
|
|
1454
1689
|
runner.invalidateProbe();
|
|
1690
|
+
noteFromError(err);
|
|
1455
1691
|
}
|
|
1456
1692
|
}
|
|
1457
1693
|
}
|
|
@@ -1514,7 +1750,10 @@ function createCodexEngine(opts) {
|
|
|
1514
1750
|
try {
|
|
1515
1751
|
await runCodex(args, signal, { stdin: promptText, label: `edit refs=${editRefs.length}` });
|
|
1516
1752
|
} catch (err) {
|
|
1517
|
-
if (isFatalSetupError(err))
|
|
1753
|
+
if (isFatalSetupError(err)) {
|
|
1754
|
+
runner.invalidateProbe();
|
|
1755
|
+
noteFromError(err);
|
|
1756
|
+
}
|
|
1518
1757
|
throw err;
|
|
1519
1758
|
}
|
|
1520
1759
|
const images = await collectImages(dir, before);
|
|
@@ -1523,10 +1762,17 @@ function createCodexEngine(opts) {
|
|
|
1523
1762
|
}
|
|
1524
1763
|
};
|
|
1525
1764
|
function isFatalSetupError(err) {
|
|
1526
|
-
return /failed to spawn|ENOENT|not logged in|login required|401|unauthorized|is too old/i.test(
|
|
1765
|
+
return /failed to spawn|ENOENT|not logged in|login required|401|unauthorized|is too old|environment is overriding/i.test(
|
|
1527
1766
|
String(err?.message ?? err)
|
|
1528
1767
|
);
|
|
1529
1768
|
}
|
|
1769
|
+
function noteFromError(err) {
|
|
1770
|
+
const failure = classifyCodexFailure({
|
|
1771
|
+
text: String(err?.message ?? err),
|
|
1772
|
+
env: process.env
|
|
1773
|
+
});
|
|
1774
|
+
runner.noteConnection(outcomeFor(failure.code), failure);
|
|
1775
|
+
}
|
|
1530
1776
|
function buildPrompt2(req, index, roles) {
|
|
1531
1777
|
const variation = req.variations?.[index] ?? "";
|
|
1532
1778
|
const roleDirective = REFERENCE_ROLE_DIRECTIVE;
|
|
@@ -1566,9 +1812,15 @@ function createCodexEngine(opts) {
|
|
|
1566
1812
|
function keyGetter(core, settingKey, envVar) {
|
|
1567
1813
|
return () => core.store.getSetting(settingKey) || process.env[envVar] || null;
|
|
1568
1814
|
}
|
|
1815
|
+
var IGNORE_ENV_KEYS_SETTING = "codex.ignore_env_keys";
|
|
1816
|
+
function ignoreEnvKeysGetter(core) {
|
|
1817
|
+
return () => (core.store.getSetting(IGNORE_ENV_KEYS_SETTING) ?? "").split(",").map((name) => name.trim().toUpperCase()).filter(
|
|
1818
|
+
(name) => CONFLICT_ENV_KEYS.includes(name)
|
|
1819
|
+
);
|
|
1820
|
+
}
|
|
1569
1821
|
function createEngineRegistry(core, extra = []) {
|
|
1570
1822
|
const saveImage = (buf) => core.images.save(buf);
|
|
1571
|
-
const codexRunner = createRunner();
|
|
1823
|
+
const codexRunner = createRunner({ ignoreEnvKeys: ignoreEnvKeysGetter(core) });
|
|
1572
1824
|
const adapters = [
|
|
1573
1825
|
createOpenRouterEngine({ getKey: keyGetter(core, "openrouter_api_key", "OPENROUTER_API_KEY"), saveImage }),
|
|
1574
1826
|
createReplicateEngine({ getKey: keyGetter(core, "replicate_api_token", "REPLICATE_API_TOKEN"), saveImage }),
|
|
@@ -1627,7 +1879,7 @@ function createDemoEngine(saveImage, opts = {}) {
|
|
|
1627
1879
|
<text x="24" y="${h - 48}" font-family="Helvetica, Arial" font-size="${Math.max(14, Math.round(w / 42))}" fill="#ffffff" opacity="0.92">${esc(label)}</text>
|
|
1628
1880
|
<text x="24" y="${h - 22}" font-family="Helvetica, Arial" font-size="12" fill="#ffffff" opacity="0.6">Scenri demo engine</text>
|
|
1629
1881
|
</svg>`;
|
|
1630
|
-
return
|
|
1882
|
+
return sharp21(Buffer.from(svg)).png().toBuffer();
|
|
1631
1883
|
}
|
|
1632
1884
|
return {
|
|
1633
1885
|
capabilities() {
|
|
@@ -1818,7 +2070,7 @@ var resolvedRefs = /* @__PURE__ */ new Map();
|
|
|
1818
2070
|
async function refHash(core, path) {
|
|
1819
2071
|
const hit = resolvedRefs.get(path);
|
|
1820
2072
|
if (hit && core.images.has(hit)) return hit;
|
|
1821
|
-
const hash = core.images.save(await
|
|
2073
|
+
const hash = core.images.save(await sharp21(readFileSync(path)).png().toBuffer());
|
|
1822
2074
|
resolvedRefs.set(path, hash);
|
|
1823
2075
|
return hash;
|
|
1824
2076
|
}
|
|
@@ -1922,7 +2174,7 @@ var resolvedRefs2 = /* @__PURE__ */ new Map();
|
|
|
1922
2174
|
async function refHash2(core, path) {
|
|
1923
2175
|
const hit = resolvedRefs2.get(path);
|
|
1924
2176
|
if (hit && core.images.has(hit)) return hit;
|
|
1925
|
-
const hash = core.images.save(await
|
|
2177
|
+
const hash = core.images.save(await sharp21(readFileSync(path)).png().toBuffer());
|
|
1926
2178
|
resolvedRefs2.set(path, hash);
|
|
1927
2179
|
return hash;
|
|
1928
2180
|
}
|
|
@@ -2076,7 +2328,7 @@ function createThumbStore(core, opts = {}) {
|
|
|
2076
2328
|
const pathFor = (key, w) => join(dir, `${key}-w${w}.webp`);
|
|
2077
2329
|
const inflight = /* @__PURE__ */ new Map();
|
|
2078
2330
|
const failed = /* @__PURE__ */ new Set();
|
|
2079
|
-
const concurrency = Math.max(1, opts.concurrency ??
|
|
2331
|
+
const concurrency = Math.max(1, opts.concurrency ?? 4);
|
|
2080
2332
|
let active = 0;
|
|
2081
2333
|
const waiting = [];
|
|
2082
2334
|
const acquire = () => new Promise((resolve) => {
|
|
@@ -2095,7 +2347,7 @@ function createThumbStore(core, opts = {}) {
|
|
|
2095
2347
|
const tmp = `${final}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
2096
2348
|
await acquire();
|
|
2097
2349
|
try {
|
|
2098
|
-
await
|
|
2350
|
+
await sharp21(source).resize({ width: w, withoutEnlargement: true }).webp({ quality: QUALITY[w], effort: 4 }).toFile(tmp);
|
|
2099
2351
|
await rename(tmp, final);
|
|
2100
2352
|
return final;
|
|
2101
2353
|
} catch {
|
|
@@ -2185,7 +2437,7 @@ var assetHash = (ref) => {
|
|
|
2185
2437
|
};
|
|
2186
2438
|
var LOGO_ROLES = ["primary", "mark", "wordmark", "monochrome", "alternate"];
|
|
2187
2439
|
var LOGO_BACKGROUNDS = ["light", "dark", "any"];
|
|
2188
|
-
var toPng = (buf) =>
|
|
2440
|
+
var toPng = (buf) => sharp21(buf).rotate().png().toBuffer();
|
|
2189
2441
|
var COST_PROBE = {
|
|
2190
2442
|
prompt: "",
|
|
2191
2443
|
brand: { brand: {}, assetPaths: {} },
|
|
@@ -2198,11 +2450,11 @@ var MARK_MIN_EDGE = 1024;
|
|
|
2198
2450
|
var MARK_TINY_EDGE = 256;
|
|
2199
2451
|
var MARK_WARN_EDGE = 512;
|
|
2200
2452
|
var toMarkPng = async (buf) => {
|
|
2201
|
-
const out = await
|
|
2202
|
-
const meta = await
|
|
2453
|
+
const out = await sharp21(buf, { density: 384 }).rotate().resize({ width: MARK_MAX_EDGE, height: MARK_MAX_EDGE, fit: "inside", withoutEnlargement: true }).png().toBuffer();
|
|
2454
|
+
const meta = await sharp21(out).metadata();
|
|
2203
2455
|
const edge = Math.max(meta.width ?? 0, meta.height ?? 0);
|
|
2204
2456
|
if (edge >= MARK_TINY_EDGE && edge < MARK_MIN_EDGE) {
|
|
2205
|
-
return
|
|
2457
|
+
return sharp21(out).resize({ width: MARK_MIN_EDGE, height: MARK_MIN_EDGE, fit: "inside", kernel: "lanczos3" }).png().toBuffer();
|
|
2206
2458
|
}
|
|
2207
2459
|
return out;
|
|
2208
2460
|
};
|
|
@@ -2213,9 +2465,9 @@ async function capReferenceEdge(core, path, maxEdge) {
|
|
|
2213
2465
|
if (hit) return hit;
|
|
2214
2466
|
let out = path;
|
|
2215
2467
|
try {
|
|
2216
|
-
const meta = await
|
|
2468
|
+
const meta = await sharp21(path).metadata();
|
|
2217
2469
|
if ((meta.width ?? 0) > maxEdge || (meta.height ?? 0) > maxEdge) {
|
|
2218
|
-
const buf = await
|
|
2470
|
+
const buf = await sharp21(path).resize({ width: maxEdge, height: maxEdge, fit: "inside", withoutEnlargement: true }).png().toBuffer();
|
|
2219
2471
|
out = core.images.pathFor(core.images.save(buf));
|
|
2220
2472
|
}
|
|
2221
2473
|
} catch {
|
|
@@ -3491,13 +3743,122 @@ function validateBrand(json) {
|
|
|
3491
3743
|
errors: valid ? [] : (compiled.errors ?? []).map((e) => `${e.instancePath || "/"} ${e.message}`)
|
|
3492
3744
|
};
|
|
3493
3745
|
}
|
|
3494
|
-
|
|
3495
|
-
|
|
3746
|
+
|
|
3747
|
+
// ../brand-spec/src/colorParse.ts
|
|
3748
|
+
var CLAMP = (n) => Math.min(255, Math.max(0, Math.round(n)));
|
|
3749
|
+
var hex2 = (n) => CLAMP(n).toString(16).padStart(2, "0");
|
|
3750
|
+
var rgbHex = (r, g, b) => `#${hex2(r)}${hex2(g)}${hex2(b)}`;
|
|
3751
|
+
var MIN_ALPHA = 0.35;
|
|
3752
|
+
var NUM = String.raw`[-+]?[\d.]+%?`;
|
|
3753
|
+
var RGB_RE = new RegExp(String.raw`^rgba?\(\s*(${NUM})[\s,]+(${NUM})[\s,]+(${NUM})(?:[\s,/]+(${NUM}))?\s*\)$`, "i");
|
|
3754
|
+
var HSL_RE = new RegExp(
|
|
3755
|
+
String.raw`^hsla?\(\s*(${NUM}|[-+]?[\d.]+deg)[\s,]+(${NUM})[\s,]+(${NUM})(?:[\s,/]+(${NUM}))?\s*\)$`,
|
|
3756
|
+
"i"
|
|
3757
|
+
);
|
|
3758
|
+
var OKLCH_RE = new RegExp(
|
|
3759
|
+
String.raw`^oklch\(\s*(${NUM})[\s,]+(${NUM})[\s,]+([-+]?[\d.]+)(?:deg)?(?:[\s,/]+(${NUM}))?\s*\)$`,
|
|
3760
|
+
"i"
|
|
3761
|
+
);
|
|
3762
|
+
function scalar(token, full) {
|
|
3763
|
+
const t = token.trim();
|
|
3764
|
+
if (t.endsWith("%")) return Number.parseFloat(t) / 100 * full;
|
|
3765
|
+
return Number.parseFloat(t);
|
|
3766
|
+
}
|
|
3767
|
+
function alphaOk(token) {
|
|
3768
|
+
if (token == null) return true;
|
|
3769
|
+
const a = token.trim().endsWith("%") ? Number.parseFloat(token) / 100 : Number.parseFloat(token);
|
|
3770
|
+
return !Number.isNaN(a) && a >= MIN_ALPHA;
|
|
3771
|
+
}
|
|
3772
|
+
function toHex(value) {
|
|
3773
|
+
const v = value.trim().toLowerCase();
|
|
3774
|
+
const hex = /^#([0-9a-f]{3,8})$/.exec(v);
|
|
3775
|
+
if (hex) {
|
|
3776
|
+
const d = hex[1];
|
|
3777
|
+
if (d.length === 3 || d.length === 4) {
|
|
3778
|
+
if (d.length === 4 && !alphaOk(String(Number.parseInt(d[3] + d[3], 16) / 255))) return null;
|
|
3779
|
+
return `#${d[0]}${d[0]}${d[1]}${d[1]}${d[2]}${d[2]}`;
|
|
3780
|
+
}
|
|
3781
|
+
if (d.length === 6) return `#${d}`;
|
|
3782
|
+
if (d.length === 8) {
|
|
3783
|
+
if (!alphaOk(String(Number.parseInt(d.slice(6, 8), 16) / 255))) return null;
|
|
3784
|
+
return `#${d.slice(0, 6)}`;
|
|
3785
|
+
}
|
|
3786
|
+
return null;
|
|
3787
|
+
}
|
|
3788
|
+
const rgb = RGB_RE.exec(v);
|
|
3789
|
+
if (rgb) {
|
|
3790
|
+
if (!alphaOk(rgb[4])) return null;
|
|
3791
|
+
return rgbHex(scalar(rgb[1], 255), scalar(rgb[2], 255), scalar(rgb[3], 255));
|
|
3792
|
+
}
|
|
3793
|
+
const hsl = HSL_RE.exec(v);
|
|
3794
|
+
if (hsl) {
|
|
3795
|
+
if (!alphaOk(hsl[4])) return null;
|
|
3796
|
+
return hslHex(Number.parseFloat(hsl[1]), scalar(hsl[2], 1), scalar(hsl[3], 1));
|
|
3797
|
+
}
|
|
3798
|
+
const oklch = OKLCH_RE.exec(v);
|
|
3799
|
+
if (oklch) {
|
|
3800
|
+
if (!alphaOk(oklch[4])) return null;
|
|
3801
|
+
return oklchHex(scalar(oklch[1], 1), Number.parseFloat(oklch[2]), Number.parseFloat(oklch[3]));
|
|
3802
|
+
}
|
|
3803
|
+
return null;
|
|
3804
|
+
}
|
|
3805
|
+
function hslHex(h, s, l) {
|
|
3806
|
+
const c = (1 - Math.abs(2 * l - 1)) * s;
|
|
3807
|
+
const hp = (h % 360 + 360) % 360 / 60;
|
|
3808
|
+
const x = c * (1 - Math.abs(hp % 2 - 1));
|
|
3809
|
+
const [r, g, b] = hp < 1 ? [c, x, 0] : hp < 2 ? [x, c, 0] : hp < 3 ? [0, c, x] : hp < 4 ? [0, x, c] : hp < 5 ? [x, 0, c] : [c, 0, x];
|
|
3810
|
+
const m = l - c / 2;
|
|
3811
|
+
return rgbHex((r + m) * 255, (g + m) * 255, (b + m) * 255);
|
|
3812
|
+
}
|
|
3813
|
+
function oklchHex(L, C, hDeg) {
|
|
3814
|
+
const h = hDeg * Math.PI / 180;
|
|
3815
|
+
const a = C * Math.cos(h);
|
|
3816
|
+
const bb = C * Math.sin(h);
|
|
3817
|
+
const l_ = L + 0.3963377774 * a + 0.2158037573 * bb;
|
|
3818
|
+
const m_ = L - 0.1055613458 * a - 0.0638541728 * bb;
|
|
3819
|
+
const s_ = L - 0.0894841775 * a - 1.291485548 * bb;
|
|
3820
|
+
const l = l_ ** 3;
|
|
3821
|
+
const m = m_ ** 3;
|
|
3822
|
+
const s = s_ ** 3;
|
|
3823
|
+
const lr = 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s;
|
|
3824
|
+
const lg = -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s;
|
|
3825
|
+
const lb = -0.0041960863 * l - 0.7034186147 * m + 1.707614701 * s;
|
|
3826
|
+
const gamma = (u) => u <= 31308e-7 ? 12.92 * u : 1.055 * Math.abs(u) ** (1 / 2.4) - 0.055;
|
|
3827
|
+
return rgbHex(gamma(lr) * 255, gamma(lg) * 255, gamma(lb) * 255);
|
|
3828
|
+
}
|
|
3829
|
+
var DECL_RE = /(--[\w-]+|[\w-]+)\s*:\s*([^;{}]*?(?:#[0-9a-fA-F]{3,8}|rgba?\([^)]*\)|hsla?\([^)]*\)|oklch\([^)]*\))[^;{}]*)/g;
|
|
3830
|
+
var TOKEN_RE = /#[0-9a-fA-F]{3,8}\b|rgba?\([^)]*\)|hsla?\([^)]*\)|oklch\([^)]*\)/g;
|
|
3831
|
+
function extractColors(css) {
|
|
3832
|
+
const out = [];
|
|
3833
|
+
const blocks = css.split("}");
|
|
3834
|
+
for (const block of blocks) {
|
|
3835
|
+
const brace = block.lastIndexOf("{");
|
|
3836
|
+
const context = brace >= 0 ? block.slice(Math.max(0, brace - 200), brace) : "";
|
|
3837
|
+
const body = brace >= 0 ? block.slice(brace + 1) : block;
|
|
3838
|
+
DECL_RE.lastIndex = 0;
|
|
3839
|
+
let decl = DECL_RE.exec(body);
|
|
3840
|
+
while (decl !== null) {
|
|
3841
|
+
const property = decl[1];
|
|
3842
|
+
TOKEN_RE.lastIndex = 0;
|
|
3843
|
+
let token = TOKEN_RE.exec(decl[2]);
|
|
3844
|
+
while (token !== null) {
|
|
3845
|
+
const hex = toHex(token[0]);
|
|
3846
|
+
if (hex) out.push({ hex: hex.toLowerCase(), property, context });
|
|
3847
|
+
token = TOKEN_RE.exec(decl[2]);
|
|
3848
|
+
}
|
|
3849
|
+
decl = DECL_RE.exec(body);
|
|
3850
|
+
}
|
|
3851
|
+
}
|
|
3852
|
+
return out;
|
|
3853
|
+
}
|
|
3854
|
+
|
|
3855
|
+
// ../brand-spec/src/colors.ts
|
|
3496
3856
|
function hexToHsl(hex) {
|
|
3497
|
-
const r = parseInt(hex.slice(1, 3), 16) / 255;
|
|
3498
|
-
const g = parseInt(hex.slice(3, 5), 16) / 255;
|
|
3499
|
-
const b = parseInt(hex.slice(5, 7), 16) / 255;
|
|
3500
|
-
const max = Math.max(r, g, b)
|
|
3857
|
+
const r = Number.parseInt(hex.slice(1, 3), 16) / 255;
|
|
3858
|
+
const g = Number.parseInt(hex.slice(3, 5), 16) / 255;
|
|
3859
|
+
const b = Number.parseInt(hex.slice(5, 7), 16) / 255;
|
|
3860
|
+
const max = Math.max(r, g, b);
|
|
3861
|
+
const min = Math.min(r, g, b);
|
|
3501
3862
|
const l = (max + min) / 2;
|
|
3502
3863
|
if (max === min) return { h: 0, s: 0, l };
|
|
3503
3864
|
const d = max - min;
|
|
@@ -3508,15 +3869,111 @@ function hexToHsl(hex) {
|
|
|
3508
3869
|
else h = ((r - g) / d + 4) / 6;
|
|
3509
3870
|
return { h: h * 360, s, l };
|
|
3510
3871
|
}
|
|
3511
|
-
|
|
3872
|
+
var BRANDISH_VAR = /--(?:[\w-]*)(brand|primary|accent|secondary|theme|main)/i;
|
|
3873
|
+
var FRAMEWORK_VAR = /^--(tw-|[\w-]*-(?:50|\d{3})$)/i;
|
|
3874
|
+
var UTILITY_CLASS = /\.(bg|text|border|fill|stroke|ring|from|via|to|shadow|outline|divide|accent|caret|decoration|placeholder)-[a-z]+-\d{2,3}\b/i;
|
|
3875
|
+
var INTERACTIVE = /(btn|button|cta|primary|brand|badge|header|nav\b|link|hero)/i;
|
|
3876
|
+
function weightOf(hit) {
|
|
3877
|
+
if (FRAMEWORK_VAR.test(hit.property) || UTILITY_CLASS.test(hit.context)) return 0;
|
|
3878
|
+
if (hit.property.startsWith("--")) return BRANDISH_VAR.test(hit.property) ? 6 : 2;
|
|
3879
|
+
if (INTERACTIVE.test(hit.context) || INTERACTIVE.test(hit.property)) return 3;
|
|
3880
|
+
return 1;
|
|
3881
|
+
}
|
|
3882
|
+
function liveClassTokens($) {
|
|
3883
|
+
const tokens = /* @__PURE__ */ new Set();
|
|
3884
|
+
for (const sel of ["html", "body"]) {
|
|
3885
|
+
for (const cls of ($(sel).first().attr("class") ?? "").split(/\s+/)) {
|
|
3886
|
+
if (cls) tokens.add(cls.toLowerCase());
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3889
|
+
return [...tokens];
|
|
3890
|
+
}
|
|
3891
|
+
var familyOf = (cls) => cls.split("-")[0];
|
|
3892
|
+
function selectorIsLive(selector, live) {
|
|
3893
|
+
const sel = selector.toLowerCase();
|
|
3894
|
+
if (sel.includes(":root")) return true;
|
|
3895
|
+
const families = new Set(live.map(familyOf));
|
|
3896
|
+
return sel.split(",").some((branch) => {
|
|
3897
|
+
const positive = branch.replace(/:not\([^)]*\)/g, "").match(/\.[a-z0-9_-]+/g);
|
|
3898
|
+
if (!positive) return true;
|
|
3899
|
+
const leading = positive[0].slice(1);
|
|
3900
|
+
if (live.includes(leading)) return true;
|
|
3901
|
+
return !families.has(familyOf(leading));
|
|
3902
|
+
});
|
|
3903
|
+
}
|
|
3904
|
+
function roleOfProperty(property) {
|
|
3905
|
+
if (!property.startsWith("--")) return null;
|
|
3906
|
+
const name = property.slice(2).toLowerCase().replace(/^(brand|color|colour|theme|sc|c|ui)-/, "");
|
|
3907
|
+
const base = /^(primary|secondary|accent|dark|light)(?:-(\d+|dark|light|hover|active|soft|muted|contrast|foreground|fg|bg))?$/.exec(
|
|
3908
|
+
name
|
|
3909
|
+
);
|
|
3910
|
+
if (!base) return null;
|
|
3911
|
+
return { role: base[1], variant: Boolean(base[2]) };
|
|
3912
|
+
}
|
|
3913
|
+
function declaredPalette(hits, live) {
|
|
3914
|
+
const best = /* @__PURE__ */ new Map();
|
|
3915
|
+
for (const hit of hits) {
|
|
3916
|
+
const named = roleOfProperty(hit.property);
|
|
3917
|
+
if (!named || FRAMEWORK_VAR.test(hit.property) || !selectorIsLive(hit.context, live)) continue;
|
|
3918
|
+
const classes = hit.context.toLowerCase().match(/\.[a-z0-9_-]+/g) ?? [];
|
|
3919
|
+
const onLiveRoot = classes.some((c) => live.includes(c.slice(1)));
|
|
3920
|
+
const rank = (onLiveRoot ? 4 : classes.length > 0 ? 2 : 0) + (named.variant ? 0 : 1);
|
|
3921
|
+
const seen = best.get(named.role);
|
|
3922
|
+
if (seen && seen.rank > rank) continue;
|
|
3923
|
+
best.set(named.role, { hex: hit.hex, rank });
|
|
3924
|
+
}
|
|
3925
|
+
return {
|
|
3926
|
+
primary: best.get("primary")?.hex,
|
|
3927
|
+
secondary: best.get("secondary")?.hex,
|
|
3928
|
+
accent: best.get("accent")?.hex,
|
|
3929
|
+
dark: best.get("dark")?.hex,
|
|
3930
|
+
light: best.get("light")?.hex
|
|
3931
|
+
};
|
|
3932
|
+
}
|
|
3933
|
+
function collectColors(sources, live = []) {
|
|
3512
3934
|
const counts = /* @__PURE__ */ new Map();
|
|
3513
|
-
for (const
|
|
3514
|
-
|
|
3935
|
+
for (const { css, weightScale = 1 } of sources) {
|
|
3936
|
+
for (const hit of extractColors(css)) {
|
|
3937
|
+
if (live.length > 0 && !selectorIsLive(hit.context, live)) continue;
|
|
3938
|
+
const weight = weightOf(hit) * weightScale;
|
|
3939
|
+
if (weight <= 0) continue;
|
|
3940
|
+
counts.set(hit.hex, (counts.get(hit.hex) ?? 0) + weight);
|
|
3941
|
+
}
|
|
3942
|
+
}
|
|
3943
|
+
return [...counts.entries()].map(([hex, weight]) => ({ hex, weight }));
|
|
3944
|
+
}
|
|
3945
|
+
function dedupeNearby(hits) {
|
|
3946
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
3947
|
+
for (const hit of [...hits].sort((a, b) => b.weight - a.weight)) {
|
|
3948
|
+
const { h, s, l } = hexToHsl(hit.hex);
|
|
3949
|
+
const key = isNeutral(hit.hex) ? `grey:${Math.round(l * 8)}` : `${Math.round(h / 12)}:${Math.round(s * 8)}:${Math.round(l * 8)}`;
|
|
3950
|
+
const seen = buckets.get(key);
|
|
3951
|
+
if (seen) seen.weight += hit.weight;
|
|
3952
|
+
else buckets.set(key, { ...hit });
|
|
3953
|
+
}
|
|
3954
|
+
return [...buckets.values()].sort((a, b) => b.weight - a.weight);
|
|
3955
|
+
}
|
|
3956
|
+
function dropSingletons(hits) {
|
|
3957
|
+
if (hits.length < 12) return [...hits];
|
|
3958
|
+
const kept = hits.filter((h) => h.weight > 1);
|
|
3959
|
+
return kept.length > 0 ? kept : [...hits];
|
|
3960
|
+
}
|
|
3961
|
+
function chromaOf(hex) {
|
|
3962
|
+
const r = Number.parseInt(hex.slice(1, 3), 16) / 255;
|
|
3963
|
+
const g = Number.parseInt(hex.slice(3, 5), 16) / 255;
|
|
3964
|
+
const b = Number.parseInt(hex.slice(5, 7), 16) / 255;
|
|
3965
|
+
return Math.max(r, g, b) - Math.min(r, g, b);
|
|
3966
|
+
}
|
|
3967
|
+
function isNeutral(hex) {
|
|
3968
|
+
const { l } = hexToHsl(hex);
|
|
3969
|
+
return chromaOf(hex) < 0.1 || l < 0.06 || l > 0.96;
|
|
3970
|
+
}
|
|
3971
|
+
function pickPalette(hits) {
|
|
3972
|
+
const sorted = [...hits].sort((a, b) => b.weight - a.weight).map((h) => h.hex);
|
|
3515
3973
|
const saturated = [];
|
|
3516
3974
|
const neutrals = [];
|
|
3517
3975
|
for (const c of sorted) {
|
|
3518
|
-
|
|
3519
|
-
if (s < 0.12 || l < 0.06 || l > 0.96) neutrals.push(c);
|
|
3976
|
+
if (isNeutral(c)) neutrals.push(c);
|
|
3520
3977
|
else saturated.push(c);
|
|
3521
3978
|
}
|
|
3522
3979
|
return {
|
|
@@ -3526,91 +3983,650 @@ function pickPalette(colors) {
|
|
|
3526
3983
|
neutrals: neutrals.slice(0, 2)
|
|
3527
3984
|
};
|
|
3528
3985
|
}
|
|
3529
|
-
|
|
3530
|
-
const
|
|
3531
|
-
const
|
|
3532
|
-
const
|
|
3533
|
-
const
|
|
3534
|
-
const
|
|
3535
|
-
|
|
3986
|
+
function paletteFrom(sources, live = []) {
|
|
3987
|
+
const hits = sources.flatMap((s) => extractColors(s.css));
|
|
3988
|
+
const declared = declaredPalette(hits, live);
|
|
3989
|
+
const counted = pickPalette(dropSingletons(dedupeNearby(collectColors(sources, live))));
|
|
3990
|
+
const taken = /* @__PURE__ */ new Set();
|
|
3991
|
+
const take = (hex) => {
|
|
3992
|
+
if (!hex || taken.has(hex)) return void 0;
|
|
3993
|
+
taken.add(hex);
|
|
3994
|
+
return hex;
|
|
3995
|
+
};
|
|
3996
|
+
const primary = take(declared.primary) ?? take(counted.primary);
|
|
3997
|
+
const secondary = take(declared.secondary) ?? take(counted.secondary);
|
|
3998
|
+
const named = Boolean(declared.primary || declared.secondary);
|
|
3999
|
+
const accent = (named ? [declared.accent, declared.dark, declared.light] : counted.accent).map(take).filter((h) => Boolean(h)).slice(0, 2);
|
|
4000
|
+
const neutrals = counted.neutrals.filter((h) => !taken.has(h)).slice(0, 2);
|
|
4001
|
+
return { primary, secondary, accent, neutrals };
|
|
4002
|
+
}
|
|
4003
|
+
|
|
4004
|
+
// ../brand-spec/src/logoCandidates.ts
|
|
4005
|
+
var BASE = {
|
|
4006
|
+
"json-ld": 170,
|
|
4007
|
+
manifest: 85,
|
|
4008
|
+
"apple-touch-icon": 80,
|
|
4009
|
+
"header-img": 75,
|
|
4010
|
+
"header-svg": 60,
|
|
4011
|
+
"link-icon": 45,
|
|
4012
|
+
"og-image": 10
|
|
4013
|
+
};
|
|
4014
|
+
var LOGOISH = /(^|[^a-z])(logo|wordmark|brandmark|lockup|masthead)([^a-z]|$)/i;
|
|
4015
|
+
var NOT_OURS = /(client|customer|partner|sponsor|press|award|integrat|collab|affiliat|featured-in|trusted-by|logos?-(wall|grid|cloud|strip)|testimonial|marquee|as-seen)/i;
|
|
4016
|
+
var SOCIAL = /(facebook|instagram|twitter|linkedin|youtube|tiktok|pinterest|threads|whatsapp|x-logo|social)/i;
|
|
4017
|
+
var FLAGGISH = /(\bflags?\b|locale|country|region|currency|\blang(uage)?\b)/i;
|
|
4018
|
+
var TRACKER = /(pixel|1x1|spacer|blank|beacon|analytics|doubleclick|facebook\.com\/tr)/i;
|
|
4019
|
+
var PHOTOISH = /(hero|banner|cover|photo|screenshot|slide|\bbg\b|background)/i;
|
|
4020
|
+
var DARKISH = /(dark|inverse|inverted|white|light-on)/i;
|
|
4021
|
+
var HEADER_SCOPE = 'header, nav, [role="banner"], .site-header, .navbar, .masthead, .topbar';
|
|
4022
|
+
var IMG_HORIZON = 40;
|
|
4023
|
+
function absolute(href, base) {
|
|
4024
|
+
if (!href) return null;
|
|
4025
|
+
const raw = href.trim();
|
|
4026
|
+
if (raw === "") return null;
|
|
4027
|
+
if (raw.startsWith("data:"))
|
|
4028
|
+
return raw.length <= 262144 && /^data:image\/(svg\+xml|png|jpeg|webp)/i.test(raw) ? raw : null;
|
|
3536
4029
|
try {
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
4030
|
+
const u = new URL(raw, base);
|
|
4031
|
+
return u.protocol === "http:" || u.protocol === "https:" ? u.toString() : null;
|
|
4032
|
+
} catch {
|
|
4033
|
+
return null;
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
function largestFromSrcset(srcset) {
|
|
4037
|
+
const entries = srcset.split(",").map((part) => part.trim()).filter(Boolean).map((part) => {
|
|
4038
|
+
const [url, descriptor = ""] = part.split(/\s+/);
|
|
4039
|
+
const w = /^(\d+)w$/.exec(descriptor);
|
|
4040
|
+
const x = /^([\d.]+)x$/.exec(descriptor);
|
|
4041
|
+
return { url, rank: w ? Number(w[1]) : x ? Number(x[1]) * 1e3 : 1 };
|
|
4042
|
+
});
|
|
4043
|
+
if (entries.length === 0) return null;
|
|
4044
|
+
return entries.sort((a, b) => b.rank - a.rank)[0].url;
|
|
4045
|
+
}
|
|
4046
|
+
function declaredEdgeOf(sizes, w, h) {
|
|
4047
|
+
const fromSizes = sizes ? Math.max(...(sizes.match(/\d+/g) ?? ["0"]).map(Number)) : 0;
|
|
4048
|
+
const fromAttrs = Math.max(Number(0) || 0, Number(0) || 0);
|
|
4049
|
+
const edge = Math.max(fromSizes, fromAttrs);
|
|
4050
|
+
return edge > 0 ? edge : null;
|
|
4051
|
+
}
|
|
4052
|
+
function layoutTerm(w, h, why) {
|
|
4053
|
+
const width = Number(w ?? 0) || 0;
|
|
4054
|
+
const height = Number(h ?? 0) || 0;
|
|
4055
|
+
if (width > 0 && height > 0 && width <= 32 && height <= 32) {
|
|
4056
|
+
why.push("icon-shaped");
|
|
4057
|
+
return -20;
|
|
4058
|
+
}
|
|
4059
|
+
return 0;
|
|
4060
|
+
}
|
|
4061
|
+
function sizeTerm(edge, why) {
|
|
4062
|
+
if (edge === null) return 0;
|
|
4063
|
+
if (edge >= 512) {
|
|
4064
|
+
why.push("large");
|
|
4065
|
+
return 15;
|
|
4066
|
+
}
|
|
4067
|
+
if (edge >= 256) {
|
|
4068
|
+
why.push("big enough");
|
|
4069
|
+
return 8;
|
|
4070
|
+
}
|
|
4071
|
+
if (edge <= 64) {
|
|
4072
|
+
why.push("favicon-sized");
|
|
4073
|
+
return -25;
|
|
4074
|
+
}
|
|
4075
|
+
return 0;
|
|
4076
|
+
}
|
|
4077
|
+
function logoCandidates($, base, manifest) {
|
|
4078
|
+
const out = [];
|
|
4079
|
+
const host = base.hostname.replace(/^www\./, "").split(".")[0];
|
|
4080
|
+
const push = (c) => {
|
|
4081
|
+
if (c.score <= 0) return;
|
|
4082
|
+
if (c.url && out.some((o) => o.url === c.url)) return;
|
|
4083
|
+
out.push(c);
|
|
4084
|
+
};
|
|
4085
|
+
for (const node of $('script[type="application/ld+json"]').toArray()) {
|
|
4086
|
+
for (const href of jsonLdLogos($(node).text())) {
|
|
4087
|
+
const url = absolute(href, base);
|
|
4088
|
+
if (url)
|
|
4089
|
+
push({
|
|
4090
|
+
url,
|
|
4091
|
+
source: "json-ld",
|
|
4092
|
+
score: BASE["json-ld"],
|
|
4093
|
+
declaredEdge: null,
|
|
4094
|
+
role: "primary",
|
|
4095
|
+
why: ["declared as the organisation logo"]
|
|
4096
|
+
});
|
|
4097
|
+
}
|
|
4098
|
+
}
|
|
4099
|
+
for (const icon of manifest?.icons ?? []) {
|
|
4100
|
+
const url = absolute(icon.src, base);
|
|
4101
|
+
const edge = declaredEdgeOf(icon.sizes);
|
|
4102
|
+
if (!url || (edge ?? 0) < 192) continue;
|
|
4103
|
+
const why = ["from the web app manifest"];
|
|
4104
|
+
push({
|
|
4105
|
+
url,
|
|
4106
|
+
source: "manifest",
|
|
4107
|
+
score: BASE.manifest + sizeTerm(edge, why),
|
|
4108
|
+
declaredEdge: edge,
|
|
4109
|
+
role: "primary",
|
|
4110
|
+
why
|
|
3541
4111
|
});
|
|
3542
|
-
}
|
|
3543
|
-
|
|
3544
|
-
|
|
4112
|
+
}
|
|
4113
|
+
for (const el of $('link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"]').toArray()) {
|
|
4114
|
+
const url = absolute($(el).attr("href"), base);
|
|
4115
|
+
const edge = declaredEdgeOf($(el).attr("sizes"));
|
|
4116
|
+
const why = ["a touch icon"];
|
|
4117
|
+
if (url)
|
|
4118
|
+
push({
|
|
4119
|
+
url,
|
|
4120
|
+
source: "apple-touch-icon",
|
|
4121
|
+
score: BASE["apple-touch-icon"] + sizeTerm(edge, why),
|
|
4122
|
+
declaredEdge: edge,
|
|
4123
|
+
role: "primary",
|
|
4124
|
+
why
|
|
4125
|
+
});
|
|
4126
|
+
}
|
|
4127
|
+
const imgs = $("img").toArray();
|
|
4128
|
+
imgs.forEach((el, index) => {
|
|
4129
|
+
const node = $(el);
|
|
4130
|
+
const inHeader = node.closest(HEADER_SCOPE).length > 0;
|
|
4131
|
+
if (!inHeader && index > IMG_HORIZON) return;
|
|
4132
|
+
const src = node.attr("src") ?? node.attr("data-src") ?? node.attr("data-lazy-src") ?? largestFromSrcset(node.attr("srcset") ?? node.attr("data-srcset") ?? "") ?? void 0;
|
|
4133
|
+
const url = absolute(src, base);
|
|
4134
|
+
if (!url) return;
|
|
4135
|
+
const text = [url, node.attr("alt") ?? "", node.attr("class") ?? "", node.attr("id") ?? ""].join(" ");
|
|
4136
|
+
const ancestry = node.parents().slice(0, 6).map((_i, p) => `${$(p).attr("class") ?? ""} ${$(p).attr("id") ?? ""}`).get().join(" ");
|
|
4137
|
+
const why = [];
|
|
4138
|
+
let score = inHeader ? BASE["header-img"] : BASE["header-img"] - 20;
|
|
4139
|
+
if (inHeader) why.push("in the header");
|
|
4140
|
+
if (index <= 2) {
|
|
4141
|
+
why.push("first image on the page");
|
|
4142
|
+
score += 20;
|
|
4143
|
+
} else if (index <= 6) {
|
|
4144
|
+
score += 6;
|
|
4145
|
+
}
|
|
4146
|
+
score += layoutTerm(node.attr("width"), node.attr("height"), why);
|
|
4147
|
+
score += semanticTerms(text, ancestry, host, why);
|
|
4148
|
+
if (node.parent().children("img").length >= 3) {
|
|
4149
|
+
why.push("one of a row of logos");
|
|
4150
|
+
score -= 50;
|
|
4151
|
+
}
|
|
4152
|
+
push({
|
|
4153
|
+
url,
|
|
4154
|
+
source: "header-img",
|
|
4155
|
+
score,
|
|
4156
|
+
declaredEdge: null,
|
|
4157
|
+
role: "primary",
|
|
4158
|
+
...DARKISH.test(text) ? { background: "dark" } : {},
|
|
4159
|
+
why
|
|
4160
|
+
});
|
|
4161
|
+
});
|
|
4162
|
+
for (const el of $(`${HEADER_SCOPE}`).find("svg").toArray().slice(0, 4)) {
|
|
4163
|
+
const node = $(el);
|
|
4164
|
+
const text = [node.attr("class") ?? "", node.attr("id") ?? "", node.attr("aria-label") ?? ""].join(" ");
|
|
4165
|
+
const why = ["drawn in the header"];
|
|
4166
|
+
const score = BASE["header-svg"] + semanticTerms(text, "", host, why);
|
|
4167
|
+
const markup = $.html(el);
|
|
4168
|
+
if (markup) push({ url: null, svg: markup, source: "header-svg", score, declaredEdge: null, role: "primary", why });
|
|
4169
|
+
}
|
|
4170
|
+
for (const el of $('link[rel~="icon"]').toArray()) {
|
|
4171
|
+
const url = absolute($(el).attr("href"), base);
|
|
4172
|
+
const edge = declaredEdgeOf($(el).attr("sizes"));
|
|
4173
|
+
const why = ["the site icon"];
|
|
4174
|
+
if (url)
|
|
4175
|
+
push({
|
|
4176
|
+
url,
|
|
4177
|
+
source: "link-icon",
|
|
4178
|
+
score: BASE["link-icon"] + sizeTerm(edge, why),
|
|
4179
|
+
declaredEdge: edge,
|
|
4180
|
+
role: "primary",
|
|
4181
|
+
why
|
|
4182
|
+
});
|
|
4183
|
+
}
|
|
4184
|
+
const og = absolute($('meta[property="og:image"]').attr("content"), base);
|
|
4185
|
+
if (og)
|
|
4186
|
+
push({
|
|
4187
|
+
url: og,
|
|
4188
|
+
source: "og-image",
|
|
4189
|
+
score: BASE["og-image"],
|
|
4190
|
+
declaredEdge: null,
|
|
4191
|
+
role: "alternate",
|
|
4192
|
+
why: ["the social share image"]
|
|
4193
|
+
});
|
|
4194
|
+
return out.sort((a, b) => b.score - a.score);
|
|
4195
|
+
}
|
|
4196
|
+
function pathOf(text) {
|
|
4197
|
+
return text.replace(/https?:\/\/[^/\s]+/gi, "");
|
|
4198
|
+
}
|
|
4199
|
+
function semanticTerms(text, ancestry, host, why) {
|
|
4200
|
+
let score = 0;
|
|
4201
|
+
if (LOGOISH.test(text)) {
|
|
4202
|
+
why.push("called a logo");
|
|
4203
|
+
score += 40;
|
|
4204
|
+
}
|
|
4205
|
+
if (host.length >= 3 && pathOf(text).toLowerCase().includes(host.toLowerCase())) {
|
|
4206
|
+
why.push("named after the site");
|
|
4207
|
+
score += 6;
|
|
4208
|
+
}
|
|
4209
|
+
if (/\.svg(\?|$)/i.test(text)) score += 8;
|
|
4210
|
+
if (/\.ico(\?|$)/i.test(text)) score -= 15;
|
|
4211
|
+
if (/\.jpe?g(\?|$)/i.test(text)) score -= 10;
|
|
4212
|
+
if (NOT_OURS.test(ancestry) || NOT_OURS.test(text)) {
|
|
4213
|
+
why.push("inside a section of other companies");
|
|
4214
|
+
score -= 40;
|
|
4215
|
+
}
|
|
4216
|
+
if (SOCIAL.test(text)) {
|
|
4217
|
+
why.push("a social icon");
|
|
4218
|
+
score -= 60;
|
|
4219
|
+
}
|
|
4220
|
+
if (FLAGGISH.test(text)) {
|
|
4221
|
+
why.push("a flag or a region switcher");
|
|
4222
|
+
score -= 60;
|
|
4223
|
+
}
|
|
4224
|
+
if (TRACKER.test(text)) {
|
|
4225
|
+
why.push("a tracking pixel");
|
|
4226
|
+
score -= 80;
|
|
4227
|
+
}
|
|
4228
|
+
if (PHOTOISH.test(text)) {
|
|
4229
|
+
why.push("photography");
|
|
4230
|
+
score -= 30;
|
|
4231
|
+
}
|
|
4232
|
+
if (DARKISH.test(text)) score -= 15;
|
|
4233
|
+
return score;
|
|
4234
|
+
}
|
|
4235
|
+
function jsonLdLogos(text) {
|
|
4236
|
+
let data;
|
|
4237
|
+
try {
|
|
4238
|
+
data = JSON.parse(text);
|
|
4239
|
+
} catch {
|
|
4240
|
+
return [];
|
|
4241
|
+
}
|
|
4242
|
+
const found = [];
|
|
4243
|
+
const walk = (node, depth) => {
|
|
4244
|
+
if (depth > 6 || node === null || typeof node !== "object") return;
|
|
4245
|
+
if (Array.isArray(node)) {
|
|
4246
|
+
for (const item of node) walk(item, depth + 1);
|
|
4247
|
+
return;
|
|
3545
4248
|
}
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
4249
|
+
const obj = node;
|
|
4250
|
+
const logo = obj.logo;
|
|
4251
|
+
if (typeof logo === "string") found.push(logo);
|
|
4252
|
+
else if (logo && typeof logo === "object") {
|
|
4253
|
+
const url = logo.url;
|
|
4254
|
+
if (typeof url === "string") found.push(url);
|
|
4255
|
+
}
|
|
4256
|
+
for (const value of Object.values(obj)) walk(value, depth + 1);
|
|
4257
|
+
};
|
|
4258
|
+
walk(data, 0);
|
|
4259
|
+
return found;
|
|
4260
|
+
}
|
|
4261
|
+
function svgAsMark(markup, longEdge = 1024) {
|
|
4262
|
+
if (/<script|<foreignObject|(?:xlink:)?href\s*=\s*["']https?:/i.test(markup)) return null;
|
|
4263
|
+
const viewBox = /viewBox\s*=\s*["']\s*[-\d.]+[\s,]+[-\d.]+[\s,]+([\d.]+)[\s,]+([\d.]+)\s*["']/i.exec(markup);
|
|
4264
|
+
const width = /\bwidth\s*=\s*["']([\d.]+)/i.exec(markup);
|
|
4265
|
+
const height = /\bheight\s*=\s*["']([\d.]+)/i.exec(markup);
|
|
4266
|
+
const w = viewBox ? Number(viewBox[1]) : Number(width?.[1] ?? 0);
|
|
4267
|
+
const h = viewBox ? Number(viewBox[2]) : Number(height?.[1] ?? 0);
|
|
4268
|
+
if (!(w > 0 && h > 0)) return null;
|
|
4269
|
+
const scale = longEdge / Math.max(w, h);
|
|
4270
|
+
let out = markup.replace(/\s\bwidth\s*=\s*["'][^"']*["']/i, "").replace(/\s\bheight\s*=\s*["'][^"']*["']/i, "").replace(/<svg\b/i, `<svg width="${Math.round(w * scale)}" height="${Math.round(h * scale)}"`);
|
|
4271
|
+
if (!/xmlns\s*=/.test(out)) out = out.replace(/<svg\b/i, '<svg xmlns="http://www.w3.org/2000/svg"');
|
|
4272
|
+
return out;
|
|
4273
|
+
}
|
|
4274
|
+
|
|
4275
|
+
// ../brand-spec/src/scrapeError.ts
|
|
4276
|
+
var ScrapeError = class extends Error {
|
|
4277
|
+
constructor(code, message, statusCode = 502) {
|
|
4278
|
+
super(message);
|
|
4279
|
+
this.code = code;
|
|
4280
|
+
this.statusCode = statusCode;
|
|
4281
|
+
}
|
|
4282
|
+
code;
|
|
4283
|
+
statusCode;
|
|
4284
|
+
name = "ScrapeError";
|
|
4285
|
+
};
|
|
4286
|
+
function urlRefusal(reason, message) {
|
|
4287
|
+
return new ScrapeError(`url_${reason}`, message, 400);
|
|
4288
|
+
}
|
|
4289
|
+
|
|
4290
|
+
// ../brand-spec/src/safeFetch.ts
|
|
4291
|
+
var DEFAULTS = {
|
|
4292
|
+
budgetMs: 2e4,
|
|
4293
|
+
requestMs: 8e3,
|
|
4294
|
+
maxRedirects: 3,
|
|
4295
|
+
maxHtmlBytes: 3e6,
|
|
4296
|
+
maxCssBytes: 1e6,
|
|
4297
|
+
maxAssetBytes: 5e6
|
|
4298
|
+
};
|
|
4299
|
+
function isPrivateAddress(ip) {
|
|
4300
|
+
const kind = isIP(ip);
|
|
4301
|
+
if (kind === 4) return isPrivateV4(ip);
|
|
4302
|
+
if (kind === 6) return isPrivateV6(ip.toLowerCase());
|
|
4303
|
+
return true;
|
|
4304
|
+
}
|
|
4305
|
+
function isPrivateV4(ip) {
|
|
4306
|
+
const p = ip.split(".").map(Number);
|
|
4307
|
+
if (p.length !== 4 || p.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true;
|
|
4308
|
+
const [a, b] = p;
|
|
4309
|
+
if (a === 0 || a === 10 || a === 127) return true;
|
|
4310
|
+
if (a === 100 && b >= 64 && b <= 127) return true;
|
|
4311
|
+
if (a === 169 && b === 254) return true;
|
|
4312
|
+
if (a === 172 && b >= 16 && b <= 31) return true;
|
|
4313
|
+
if (a === 192 && b === 168) return true;
|
|
4314
|
+
if (a === 192 && b === 0) return true;
|
|
4315
|
+
if (a === 192 && b === 88) return true;
|
|
4316
|
+
if (a === 198 && (b === 18 || b === 19 || b === 51)) return true;
|
|
4317
|
+
if (a === 203 && b === 0) return true;
|
|
4318
|
+
if (a >= 224) return true;
|
|
4319
|
+
return false;
|
|
4320
|
+
}
|
|
4321
|
+
function isPrivateV6(ip) {
|
|
4322
|
+
const bare = ip.replace(/^\[|\]$/g, "");
|
|
4323
|
+
if (bare === "::" || bare === "::1") return true;
|
|
4324
|
+
const mapped = /^(?:::ffff:|64:ff9b::)(\d{1,3}(?:\.\d{1,3}){3})$/.exec(bare);
|
|
4325
|
+
if (mapped) return isPrivateV4(mapped[1]);
|
|
4326
|
+
const head = bare.split(":")[0];
|
|
4327
|
+
if (/^f[cd]/.test(head)) return true;
|
|
4328
|
+
if (/^fe[89ab]/.test(head)) return true;
|
|
4329
|
+
if (/^ff/.test(head)) return true;
|
|
4330
|
+
if (head === "2002") return true;
|
|
4331
|
+
return false;
|
|
4332
|
+
}
|
|
4333
|
+
var PRIVATE_SUFFIX = /(^|\.)(local|internal|localhost|home\.arpa)$/i;
|
|
4334
|
+
function assertPublicHost(host, addresses, allow) {
|
|
4335
|
+
if (allow) return;
|
|
4336
|
+
const bare = host.replace(/^\[|\]$/g, "");
|
|
4337
|
+
if (isIP(bare)) {
|
|
4338
|
+
if (isPrivateAddress(bare)) throw blocked(host);
|
|
4339
|
+
return;
|
|
4340
|
+
}
|
|
4341
|
+
if (!bare.includes(".") || PRIVATE_SUFFIX.test(bare)) throw blocked(host);
|
|
4342
|
+
if (addresses.length === 0) throw new ScrapeError("unreachable", `Could not reach ${host}.`);
|
|
4343
|
+
if (addresses.some((a) => isPrivateAddress(a))) throw blocked(host);
|
|
4344
|
+
}
|
|
4345
|
+
var blocked = (host) => new ScrapeError("blocked_host", `${host} points at this computer or a private network, not a public website.`, 400);
|
|
4346
|
+
var LIMIT = {
|
|
4347
|
+
html: "maxHtmlBytes",
|
|
4348
|
+
css: "maxCssBytes",
|
|
4349
|
+
asset: "maxAssetBytes"
|
|
4350
|
+
};
|
|
4351
|
+
var TRUNCATES = { html: true, css: true, asset: false };
|
|
4352
|
+
function createGuardedFetch(opts = {}) {
|
|
4353
|
+
const o = { ...DEFAULTS, ...opts };
|
|
4354
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
4355
|
+
const resolve = opts.lookup ?? (async (host) => {
|
|
4356
|
+
if (isIP(host.replace(/^\[|\]$/g, ""))) return [host.replace(/^\[|\]$/g, "")];
|
|
4357
|
+
const all = await lookup(host, { all: true });
|
|
4358
|
+
return all.map((a) => a.address);
|
|
4359
|
+
});
|
|
4360
|
+
const deadline = Date.now() + o.budgetMs;
|
|
4361
|
+
const remaining = () => Math.max(0, deadline - Date.now());
|
|
4362
|
+
async function once(url, as) {
|
|
4363
|
+
let current = url;
|
|
4364
|
+
let hops = 0;
|
|
4365
|
+
let attempts = 0;
|
|
4366
|
+
for (; ; ) {
|
|
4367
|
+
const u = new URL(current);
|
|
4368
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
4369
|
+
throw new ScrapeError(
|
|
4370
|
+
"url_scheme",
|
|
4371
|
+
"Scenri reads web addresses only, the ones that start with http or https.",
|
|
4372
|
+
400
|
|
4373
|
+
);
|
|
4374
|
+
let addresses = [];
|
|
4375
|
+
if (!o.allowPrivateHosts) {
|
|
4376
|
+
try {
|
|
4377
|
+
addresses = await resolve(u.hostname);
|
|
4378
|
+
} catch {
|
|
4379
|
+
throw new ScrapeError("unreachable", `Could not reach ${u.hostname}.`);
|
|
4380
|
+
}
|
|
4381
|
+
}
|
|
4382
|
+
assertPublicHost(u.hostname, addresses, Boolean(o.allowPrivateHosts));
|
|
4383
|
+
const left = remaining();
|
|
4384
|
+
if (left <= 0) throw new ScrapeError("timeout", "Reading that site took too long.");
|
|
4385
|
+
const signal = AbortSignal.any([AbortSignal.timeout(Math.min(o.requestMs, left))]);
|
|
4386
|
+
let res;
|
|
4387
|
+
try {
|
|
4388
|
+
res = await fetchImpl(current, {
|
|
4389
|
+
redirect: "manual",
|
|
4390
|
+
headers: { "user-agent": USER_AGENT, accept: ACCEPT[as] },
|
|
4391
|
+
signal
|
|
4392
|
+
});
|
|
4393
|
+
} catch (err) {
|
|
4394
|
+
if (err instanceof ScrapeError) throw err;
|
|
4395
|
+
const aborted = err instanceof Error && (err.name === "AbortError" || err.name === "TimeoutError");
|
|
4396
|
+
throw aborted ? new ScrapeError("timeout", `${u.hostname} took too long to answer.`) : new ScrapeError("unreachable", `Could not reach ${u.hostname}.`);
|
|
4397
|
+
}
|
|
4398
|
+
if (res.status >= 300 && res.status < 400) {
|
|
4399
|
+
const location = res.headers.get("location");
|
|
4400
|
+
await res.body?.cancel().catch(() => {
|
|
4401
|
+
});
|
|
4402
|
+
if (!location) throw new ScrapeError("http_status", `${u.hostname} answered ${res.status} with nowhere to go.`);
|
|
4403
|
+
if (++hops > o.maxRedirects)
|
|
4404
|
+
throw new ScrapeError(
|
|
4405
|
+
"too_many_redirects",
|
|
4406
|
+
`${new URL(url).hostname} kept redirecting, so nothing was read.`
|
|
4407
|
+
);
|
|
4408
|
+
current = new URL(location, current).toString();
|
|
4409
|
+
continue;
|
|
4410
|
+
}
|
|
4411
|
+
if (!res.ok) {
|
|
4412
|
+
if (RETRYABLE.has(res.status) && attempts < 1 && remaining() > 2e3) {
|
|
4413
|
+
await res.body?.cancel().catch(() => {
|
|
4414
|
+
});
|
|
4415
|
+
await sleep3(700);
|
|
4416
|
+
attempts++;
|
|
4417
|
+
continue;
|
|
4418
|
+
}
|
|
4419
|
+
throw new ScrapeError("http_status", statusSentence(u.hostname, res.status));
|
|
4420
|
+
}
|
|
4421
|
+
const contentType = (res.headers.get("content-type") ?? "").toLowerCase();
|
|
4422
|
+
const cap2 = o[LIMIT[as]];
|
|
4423
|
+
const { bytes, truncated } = await readBounded(res, cap2, TRUNCATES[as]);
|
|
4424
|
+
assertKind(as, contentType, bytes, u.hostname);
|
|
4425
|
+
return {
|
|
4426
|
+
finalUrl: res.url || current,
|
|
4427
|
+
status: res.status,
|
|
4428
|
+
contentType,
|
|
4429
|
+
bytes,
|
|
4430
|
+
text: as === "asset" ? "" : decode(bytes, contentType),
|
|
4431
|
+
truncated
|
|
4432
|
+
};
|
|
4433
|
+
}
|
|
4434
|
+
}
|
|
4435
|
+
const guarded = ((url, as) => once(url, as));
|
|
4436
|
+
guarded.remaining = remaining;
|
|
4437
|
+
return guarded;
|
|
4438
|
+
}
|
|
4439
|
+
var USER_AGENT = "scenri/0.1 (+https://scenri.co)";
|
|
4440
|
+
var sleep3 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
4441
|
+
var RETRYABLE = /* @__PURE__ */ new Set([403, 408, 425, 429, 500, 502, 503, 504]);
|
|
4442
|
+
function statusSentence(host, status) {
|
|
4443
|
+
if (status === 401 || status === 403)
|
|
4444
|
+
return `${host} would not let Scenri read it. Some sites block anything that is not a person in a browser; you can still add the logo and colours by hand.`;
|
|
4445
|
+
if (status === 404) return `There is no page at that address on ${host}.`;
|
|
4446
|
+
if (status === 429) return `${host} asked Scenri to slow down. Try again in a minute.`;
|
|
4447
|
+
if (status >= 500) return `${host} had trouble answering. Try again in a moment.`;
|
|
4448
|
+
return `${host} answered ${status}, so there was nothing to read.`;
|
|
4449
|
+
}
|
|
4450
|
+
var ACCEPT = {
|
|
4451
|
+
html: "text/html,application/xhtml+xml",
|
|
4452
|
+
css: "text/css,*/*;q=0.1",
|
|
4453
|
+
asset: "image/*"
|
|
4454
|
+
};
|
|
4455
|
+
async function readBounded(res, cap2, truncate) {
|
|
4456
|
+
const declared = Number(res.headers.get("content-length") ?? "0");
|
|
4457
|
+
if (!truncate && declared > cap2) {
|
|
4458
|
+
await res.body?.cancel().catch(() => {
|
|
4459
|
+
});
|
|
4460
|
+
throw new ScrapeError("too_large", "That file is too large to read.");
|
|
4461
|
+
}
|
|
4462
|
+
const reader = res.body?.getReader();
|
|
4463
|
+
if (!reader) return { bytes: Buffer.from(await res.arrayBuffer()), truncated: false };
|
|
4464
|
+
const chunks = [];
|
|
4465
|
+
let total = 0;
|
|
4466
|
+
for (; ; ) {
|
|
4467
|
+
const { done, value } = await reader.read();
|
|
4468
|
+
if (done) break;
|
|
4469
|
+
chunks.push(Buffer.from(value));
|
|
4470
|
+
total += value.byteLength;
|
|
4471
|
+
if (total >= cap2) {
|
|
4472
|
+
await reader.cancel().catch(() => {
|
|
4473
|
+
});
|
|
4474
|
+
if (!truncate) throw new ScrapeError("too_large", "That file is too large to read.");
|
|
4475
|
+
return { bytes: Buffer.concat(chunks).subarray(0, cap2), truncated: true };
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
return { bytes: Buffer.concat(chunks), truncated: false };
|
|
4479
|
+
}
|
|
4480
|
+
var IMAGE_MAGIC = [
|
|
4481
|
+
["png", (b) => b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))],
|
|
4482
|
+
["jpeg", (b) => b[0] === 255 && b[1] === 216],
|
|
4483
|
+
["gif", (b) => b.subarray(0, 3).toString("latin1") === "GIF"],
|
|
4484
|
+
["webp", (b) => b.subarray(0, 4).toString("latin1") === "RIFF" && b.subarray(8, 12).toString("latin1") === "WEBP"],
|
|
4485
|
+
["ico", (b) => b[0] === 0 && b[1] === 0 && (b[2] === 1 || b[2] === 2)],
|
|
4486
|
+
["svg", (b) => /<svg[\s>]/i.test(b.subarray(0, 1024).toString("utf8"))]
|
|
4487
|
+
];
|
|
4488
|
+
function assertKind(as, contentType, bytes, host) {
|
|
4489
|
+
if (as === "html") {
|
|
4490
|
+
const htmlish = /text\/html|application\/xhtml/.test(contentType);
|
|
4491
|
+
const vague = contentType === "" || /^text\/plain|^application\/octet-stream/.test(contentType);
|
|
4492
|
+
const looksLikeMarkup = vague && bytes.subarray(0, 512).toString("utf8").trimStart().startsWith("<");
|
|
4493
|
+
if (!htmlish && !looksLikeMarkup)
|
|
4494
|
+
throw new ScrapeError("not_a_page", `${host} answered with a file, not a web page.`);
|
|
4495
|
+
return;
|
|
4496
|
+
}
|
|
4497
|
+
if (as === "asset") {
|
|
4498
|
+
if (contentType.startsWith("image/")) return;
|
|
4499
|
+
const vague = contentType === "" || /^application\/octet-stream/.test(contentType);
|
|
4500
|
+
if (vague || IMAGE_MAGIC.some(([, test]) => test(bytes))) return;
|
|
4501
|
+
throw new ScrapeError("not_a_page", "That logo was not an image.");
|
|
4502
|
+
}
|
|
4503
|
+
}
|
|
4504
|
+
function decode(bytes, contentType) {
|
|
4505
|
+
const declared = /charset=([\w-]+)/i.exec(contentType)?.[1];
|
|
4506
|
+
const sniffed = declared ?? /charset=["']?([\w-]+)/i.exec(bytes.subarray(0, 1024).toString("latin1"))?.[1];
|
|
4507
|
+
const label = (sniffed ?? "utf-8").toLowerCase();
|
|
4508
|
+
if (label === "utf-8" || label === "utf8") return bytes.toString("utf8");
|
|
4509
|
+
try {
|
|
4510
|
+
return new TextDecoder(label, { fatal: false }).decode(bytes);
|
|
4511
|
+
} catch {
|
|
4512
|
+
return bytes.toString("utf8");
|
|
4513
|
+
}
|
|
4514
|
+
}
|
|
4515
|
+
|
|
4516
|
+
// ../brand-spec/src/siteUrl.ts
|
|
4517
|
+
var INVISIBLE = /[---]/g;
|
|
4518
|
+
var SPACES = /[\t\n\r\f\v - ]/g;
|
|
4519
|
+
var SCHEMED = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
4520
|
+
var OPAQUE = /^(javascript|data|mailto|tel|sms|file|about|blob|chrome|chrome-extension|view-source):/i;
|
|
4521
|
+
var HTTPISH = /^https?:\/\//i;
|
|
4522
|
+
var IP_LITERAL = /^(\d{1,3}(\.\d{1,3}){3}|\[[0-9a-f:]+\])$/i;
|
|
4523
|
+
var EXAMPLE = "acme.com";
|
|
4524
|
+
var MESSAGES = {
|
|
4525
|
+
empty: `Paste a website address, like ${EXAMPLE}.`,
|
|
4526
|
+
scheme: `Scenri reads web addresses only, the ones that start with http or https. Try ${EXAMPLE}.`,
|
|
4527
|
+
space: "A web address cannot contain a space.",
|
|
4528
|
+
host: `That does not look like a website address. Try ${EXAMPLE}.`,
|
|
4529
|
+
unparseable: `That does not look like a website address. Try ${EXAMPLE}.`
|
|
4530
|
+
};
|
|
4531
|
+
var no = (reason, suggestion) => ({
|
|
4532
|
+
ok: false,
|
|
4533
|
+
reason,
|
|
4534
|
+
message: suggestion && reason === "space" ? `${MESSAGES.space} Did you mean ${suggestion}?` : MESSAGES[reason],
|
|
4535
|
+
...suggestion ? { suggestion } : {}
|
|
4536
|
+
});
|
|
4537
|
+
function unwrap(value) {
|
|
4538
|
+
const pairs = [
|
|
4539
|
+
["<", ">"],
|
|
4540
|
+
['"', '"'],
|
|
4541
|
+
["'", "'"]
|
|
4542
|
+
];
|
|
4543
|
+
for (const [open, close] of pairs) {
|
|
4544
|
+
if (value.length >= 2 && value.startsWith(open) && value.endsWith(close)) return value.slice(1, -1).trim();
|
|
4545
|
+
}
|
|
4546
|
+
return value;
|
|
4547
|
+
}
|
|
4548
|
+
function normalizeSiteUrl(input) {
|
|
4549
|
+
const cleaned = unwrap(
|
|
4550
|
+
String(input ?? "").replace(INVISIBLE, "").replace(SPACES, " ").trim()
|
|
4551
|
+
);
|
|
4552
|
+
if (cleaned === "") return no("empty");
|
|
4553
|
+
const addedScheme = !SCHEMED.test(cleaned) && !OPAQUE.test(cleaned);
|
|
4554
|
+
if (!addedScheme && !HTTPISH.test(cleaned)) return no("scheme");
|
|
4555
|
+
const candidate = addedScheme ? `https://${cleaned}` : cleaned;
|
|
4556
|
+
const authority = candidate.slice(candidate.indexOf("://") + 3).split(/[/?#]/)[0];
|
|
4557
|
+
if (authority.includes(" ")) {
|
|
4558
|
+
const squeezed = normalizeSiteUrl(cleaned.replace(/ /g, ""));
|
|
4559
|
+
return no("space", squeezed.ok ? squeezed.url.replace(/^https?:\/\//, "").replace(/\/$/, "") : void 0);
|
|
4560
|
+
}
|
|
4561
|
+
let u;
|
|
4562
|
+
try {
|
|
4563
|
+
u = new URL(candidate);
|
|
4564
|
+
} catch {
|
|
4565
|
+
return no("unparseable");
|
|
3549
4566
|
}
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
const
|
|
4567
|
+
u.username = "";
|
|
4568
|
+
u.password = "";
|
|
4569
|
+
u.hash = "";
|
|
4570
|
+
const host = u.hostname.replace(/\.$/, "");
|
|
4571
|
+
u.hostname = host;
|
|
4572
|
+
if (!host.includes(".") && !IP_LITERAL.test(host) && host !== "localhost") return no("host");
|
|
4573
|
+
return { ok: true, url: u.toString(), host, addedScheme };
|
|
4574
|
+
}
|
|
4575
|
+
|
|
4576
|
+
// ../brand-spec/src/buildFromUrl.ts
|
|
4577
|
+
var MAX_PORTRAIT_RATIO = 2.5;
|
|
4578
|
+
var CONFIDENT_SCORE = 80;
|
|
4579
|
+
var ICON_SOURCES = /* @__PURE__ */ new Set(["json-ld", "manifest", "apple-touch-icon", "link-icon"]);
|
|
4580
|
+
var LOGO_TRIES = 3;
|
|
4581
|
+
var MAX_SHEETS = 6;
|
|
4582
|
+
var SHEET_RANK = /(root|variables|tokens|colou?rs|palette|theme|main|app|style|tailwind|global|site|brand|base)/i;
|
|
4583
|
+
var SHEET_SKIP = /(font|icon|fontawesome|bootstrap-icons|swiper|slick|slider|lightbox|print)/i;
|
|
4584
|
+
async function buildFromUrl(url, opts = {}) {
|
|
4585
|
+
const warnings = [];
|
|
4586
|
+
const normalized = normalizeSiteUrl(url);
|
|
4587
|
+
if (!normalized.ok) throw urlRefusal(normalized.reason, normalized.message);
|
|
4588
|
+
const get = createGuardedFetch({
|
|
4589
|
+
...opts.guard,
|
|
4590
|
+
...opts.fetchImpl ? { fetchImpl: opts.fetchImpl, allowPrivateHosts: true } : {}
|
|
4591
|
+
});
|
|
4592
|
+
const page = await get(normalized.url, "html");
|
|
4593
|
+
const origin = new URL(page.finalUrl);
|
|
4594
|
+
const $ = cheerio.load(page.text);
|
|
4595
|
+
const base = baseOf($, origin);
|
|
4596
|
+
const named = pickName($, origin);
|
|
3554
4597
|
const tagline = $('meta[name="description"]').attr("content")?.trim() || $('meta[property="og:description"]').attr("content")?.trim();
|
|
3555
|
-
const
|
|
3556
|
-
const themeColor = $('meta[name="theme-color"]').attr("content");
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
4598
|
+
const sources = [];
|
|
4599
|
+
const themeColor = $('meta[name="theme-color"]').attr("content")?.trim();
|
|
4600
|
+
const tileColor = $('meta[name="msapplication-TileColor"]').attr("content")?.trim();
|
|
4601
|
+
for (const [value, scale] of [
|
|
4602
|
+
[themeColor, 2],
|
|
4603
|
+
[tileColor, 1]
|
|
4604
|
+
]) {
|
|
4605
|
+
if (value) sources.push({ css: `:root{--brand-theme:${value}}`, weightScale: scale });
|
|
4606
|
+
}
|
|
4607
|
+
sources.push({ css: inlineCss($) });
|
|
4608
|
+
for (const href of sheetHrefs($, base)) {
|
|
4609
|
+
if (get.remaining() <= 0) break;
|
|
3564
4610
|
try {
|
|
3565
|
-
const
|
|
3566
|
-
|
|
3567
|
-
if (cssRes.ok) colorSources.push(...(await cssRes.text()).match(HEX_RE) ?? []);
|
|
4611
|
+
const sheet = await get(href, "css");
|
|
4612
|
+
sources.push({ css: sheet.text });
|
|
3568
4613
|
} catch {
|
|
3569
4614
|
warnings.push("Stylesheet fetch failed; palette from inline styles only.");
|
|
3570
4615
|
}
|
|
3571
4616
|
}
|
|
3572
|
-
const palette =
|
|
4617
|
+
const palette = paletteFrom(sources, liveClassTokens($));
|
|
3573
4618
|
if (!palette.primary) warnings.push("No confident palette found. Set colors manually.");
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
logoFromOg = Boolean(iconHref);
|
|
3581
|
-
}
|
|
3582
|
-
if (iconHref && opts.saveAsset) {
|
|
3583
|
-
try {
|
|
3584
|
-
const iconRes = await fetchImpl(new URL(iconHref, origin).toString(), { redirect: "follow" });
|
|
3585
|
-
if (iconRes.ok) {
|
|
3586
|
-
const buf = Buffer.from(await iconRes.arrayBuffer());
|
|
3587
|
-
if (buf.length > 0) {
|
|
3588
|
-
logoRef = await opts.saveAsset(buf, "logo");
|
|
3589
|
-
if (!logoFromOg && opts.probeLongEdge) {
|
|
3590
|
-
const edge = await opts.probeLongEdge(buf).catch(() => null);
|
|
3591
|
-
if (edge !== null && edge < 256) {
|
|
3592
|
-
logoTiny = true;
|
|
3593
|
-
warnings.push(
|
|
3594
|
-
`The site icon is favicon-sized (${edge}px), so it was saved as an alternate mark, not the logo. Upload your real logo in Settings.`
|
|
3595
|
-
);
|
|
3596
|
-
}
|
|
3597
|
-
}
|
|
3598
|
-
}
|
|
3599
|
-
}
|
|
3600
|
-
} catch {
|
|
3601
|
-
warnings.push("Logo download failed.");
|
|
3602
|
-
}
|
|
3603
|
-
}
|
|
3604
|
-
if (!logoRef) warnings.push("No logo captured. Add one manually.");
|
|
3605
|
-
else if (logoFromOg)
|
|
3606
|
-
warnings.push(
|
|
3607
|
-
"No site icon was found; the social share image was saved as an alternate mark. Check it before treating it as the logo."
|
|
3608
|
-
);
|
|
4619
|
+
const colorCount = [palette.primary, palette.secondary, ...palette.accent, ...palette.neutrals].filter(
|
|
4620
|
+
Boolean
|
|
4621
|
+
).length;
|
|
4622
|
+
const manifest = await readManifest($, base, get);
|
|
4623
|
+
const candidates = logoCandidates($, base, manifest);
|
|
4624
|
+
const picked = await downloadMark(candidates, get, opts, warnings);
|
|
3609
4625
|
const brand = {
|
|
3610
4626
|
specVersion: "0.1",
|
|
3611
4627
|
meta: {
|
|
3612
|
-
name,
|
|
3613
|
-
slug:
|
|
4628
|
+
name: named.value,
|
|
4629
|
+
slug: named.value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || origin.hostname,
|
|
3614
4630
|
...tagline ? { tagline } : {},
|
|
3615
4631
|
website: origin.origin,
|
|
3616
4632
|
createdWith: opts.createdWith ?? "scenri",
|
|
@@ -3624,26 +4640,208 @@ async function buildFromUrl(url, opts = {}) {
|
|
|
3624
4640
|
...palette.neutrals.length ? { neutrals: palette.neutrals.map((hex) => ({ hex })) } : {}
|
|
3625
4641
|
}
|
|
3626
4642
|
} : {},
|
|
3627
|
-
...
|
|
4643
|
+
...picked.ref ? {
|
|
4644
|
+
logos: [
|
|
4645
|
+
{
|
|
4646
|
+
role: picked.role,
|
|
4647
|
+
file: picked.ref,
|
|
4648
|
+
...picked.background ? { background: picked.background } : {}
|
|
4649
|
+
}
|
|
4650
|
+
]
|
|
4651
|
+
} : {}
|
|
4652
|
+
};
|
|
4653
|
+
return {
|
|
4654
|
+
brand,
|
|
4655
|
+
warnings,
|
|
4656
|
+
report: {
|
|
4657
|
+
url: page.finalUrl,
|
|
4658
|
+
host: origin.hostname,
|
|
4659
|
+
name: named,
|
|
4660
|
+
tagline: tagline ?? null,
|
|
4661
|
+
logo: {
|
|
4662
|
+
status: picked.ref ? picked.role : "none",
|
|
4663
|
+
source: picked.source,
|
|
4664
|
+
...picked.score != null ? { score: picked.score } : {},
|
|
4665
|
+
...picked.note ? { note: picked.note } : {}
|
|
4666
|
+
},
|
|
4667
|
+
colors: { count: colorCount }
|
|
4668
|
+
}
|
|
3628
4669
|
};
|
|
3629
|
-
return { brand, warnings };
|
|
3630
|
-
}
|
|
3631
|
-
|
|
3632
|
-
// ../brand-spec/src/mergeScrape.ts
|
|
3633
|
-
var str2 = (v) => typeof v === "string" ? v.trim() : "";
|
|
3634
|
-
var arr = (v) => Array.isArray(v) ? v : [];
|
|
3635
|
-
var hashOf = (ref) => {
|
|
3636
|
-
const s = String(ref ?? "");
|
|
3637
|
-
return s.startsWith("asset:") ? s.slice(6) : s;
|
|
3638
|
-
};
|
|
3639
|
-
function scrapedHexes(palette) {
|
|
3640
|
-
return [palette?.primary, palette?.secondary, ...arr(palette?.accent), ...arr(palette?.neutrals)].filter((c) => typeof c?.hex === "string").map((c) => ({ hex: String(c.hex) }));
|
|
3641
4670
|
}
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
4671
|
+
function baseOf($, origin) {
|
|
4672
|
+
const declared = $("base[href]").attr("href");
|
|
4673
|
+
if (!declared) return origin;
|
|
4674
|
+
try {
|
|
4675
|
+
return new URL(declared, origin);
|
|
4676
|
+
} catch {
|
|
4677
|
+
return origin;
|
|
4678
|
+
}
|
|
4679
|
+
}
|
|
4680
|
+
function pickName($, origin) {
|
|
4681
|
+
const ld = jsonLdName($);
|
|
4682
|
+
if (ld) return { value: ld, source: "json-ld" };
|
|
4683
|
+
const og = $('meta[property="og:site_name"]').attr("content")?.trim();
|
|
4684
|
+
if (og) return { value: og, source: "og:site_name" };
|
|
4685
|
+
const title = nameFromTitle($("title").first().text());
|
|
4686
|
+
if (title) return { value: title, source: "title" };
|
|
4687
|
+
return { value: origin.hostname.replace(/^www\./, ""), source: "hostname" };
|
|
4688
|
+
}
|
|
4689
|
+
function nameFromTitle(raw) {
|
|
4690
|
+
const parts = raw.trim().split(/\s+[|\u00b7\u2022\u2013\u2014]\s+|\s+-\s+/).map((p) => p.trim()).filter(Boolean);
|
|
4691
|
+
if (parts.length < 2) return parts[0] ?? "";
|
|
4692
|
+
const first = parts[0];
|
|
4693
|
+
const last = parts[parts.length - 1];
|
|
4694
|
+
const words = (v) => v.split(/\s+/).length;
|
|
4695
|
+
if (/^(home|homepage|index|welcome|start|main)$/i.test(first)) return last;
|
|
4696
|
+
if (words(last) <= 4 && words(first) > words(last)) return last;
|
|
4697
|
+
return first;
|
|
4698
|
+
}
|
|
4699
|
+
function jsonLdName($) {
|
|
4700
|
+
for (const node of $('script[type="application/ld+json"]').toArray()) {
|
|
4701
|
+
try {
|
|
4702
|
+
const data = JSON.parse($(node).text());
|
|
4703
|
+
const found = findOrgName(data, 0);
|
|
4704
|
+
if (found) return found;
|
|
4705
|
+
} catch {
|
|
4706
|
+
}
|
|
4707
|
+
}
|
|
4708
|
+
return null;
|
|
4709
|
+
}
|
|
4710
|
+
function findOrgName(node, depth) {
|
|
4711
|
+
if (depth > 6 || node === null || typeof node !== "object") return null;
|
|
4712
|
+
if (Array.isArray(node)) {
|
|
4713
|
+
for (const item of node) {
|
|
4714
|
+
const found = findOrgName(item, depth + 1);
|
|
4715
|
+
if (found) return found;
|
|
4716
|
+
}
|
|
4717
|
+
return null;
|
|
4718
|
+
}
|
|
4719
|
+
const obj = node;
|
|
4720
|
+
const type = String(obj["@type"] ?? "");
|
|
4721
|
+
if (/Organization|Corporation|LocalBusiness/i.test(type) && typeof obj.name === "string" && obj.name.trim())
|
|
4722
|
+
return obj.name.trim();
|
|
4723
|
+
for (const value of Object.values(obj)) {
|
|
4724
|
+
const found = findOrgName(value, depth + 1);
|
|
4725
|
+
if (found) return found;
|
|
4726
|
+
}
|
|
4727
|
+
return null;
|
|
4728
|
+
}
|
|
4729
|
+
function inlineCss($) {
|
|
4730
|
+
const attrs = $("[style]").map((_, el) => `x{${$(el).attr("style")}}`).get().join("\n");
|
|
4731
|
+
const blocks = $("style").map((_, el) => $(el).text()).get().join("\n");
|
|
4732
|
+
return `${attrs}
|
|
4733
|
+
${blocks}`;
|
|
4734
|
+
}
|
|
4735
|
+
function sheetHrefs($, base) {
|
|
4736
|
+
const hrefs = [];
|
|
4737
|
+
for (const el of $('link[rel="stylesheet"]').toArray()) {
|
|
4738
|
+
const href = $(el).attr("href");
|
|
4739
|
+
if (!href || SHEET_SKIP.test(href)) continue;
|
|
4740
|
+
try {
|
|
4741
|
+
hrefs.push(new URL(href, base).toString());
|
|
4742
|
+
} catch {
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
return hrefs.sort((a, b) => Number(SHEET_RANK.test(b)) - Number(SHEET_RANK.test(a))).slice(0, MAX_SHEETS);
|
|
4746
|
+
}
|
|
4747
|
+
async function readManifest($, base, get) {
|
|
4748
|
+
const href = $('link[rel="manifest"]').attr("href");
|
|
4749
|
+
if (!href || get.remaining() <= 0) return void 0;
|
|
4750
|
+
try {
|
|
4751
|
+
const res = await get(new URL(href, base).toString(), "css");
|
|
4752
|
+
return JSON.parse(res.text);
|
|
4753
|
+
} catch {
|
|
4754
|
+
return void 0;
|
|
4755
|
+
}
|
|
4756
|
+
}
|
|
4757
|
+
async function downloadMark(candidates, get, opts, warnings) {
|
|
4758
|
+
if (!opts.saveAsset || candidates.length === 0) {
|
|
4759
|
+
if (candidates.length === 0) warnings.push("No logo captured. Add one manually.");
|
|
4760
|
+
return { role: "primary", source: null };
|
|
4761
|
+
}
|
|
4762
|
+
let fallback = null;
|
|
4763
|
+
let tried = 0;
|
|
4764
|
+
let failed = false;
|
|
4765
|
+
for (const candidate of candidates) {
|
|
4766
|
+
if (tried >= LOGO_TRIES || get.remaining() <= 0) break;
|
|
4767
|
+
let buf = null;
|
|
4768
|
+
if (candidate.svg) {
|
|
4769
|
+
const sized = svgAsMark(candidate.svg);
|
|
4770
|
+
if (!sized) continue;
|
|
4771
|
+
buf = Buffer.from(sized, "utf8");
|
|
4772
|
+
} else if (candidate.url?.startsWith("data:")) {
|
|
4773
|
+
const comma = candidate.url.indexOf(",");
|
|
4774
|
+
const body = candidate.url.slice(comma + 1);
|
|
4775
|
+
buf = candidate.url.slice(0, comma).includes(";base64") ? Buffer.from(body, "base64") : Buffer.from(decodeURIComponent(body), "utf8");
|
|
4776
|
+
} else if (candidate.url) {
|
|
4777
|
+
tried++;
|
|
4778
|
+
try {
|
|
4779
|
+
buf = (await get(candidate.url, "asset")).bytes;
|
|
4780
|
+
} catch {
|
|
4781
|
+
failed = true;
|
|
4782
|
+
continue;
|
|
4783
|
+
}
|
|
4784
|
+
}
|
|
4785
|
+
if (!buf || buf.length === 0) continue;
|
|
4786
|
+
let ref;
|
|
4787
|
+
try {
|
|
4788
|
+
ref = await opts.saveAsset(buf, "logo");
|
|
4789
|
+
} catch {
|
|
4790
|
+
failed = true;
|
|
4791
|
+
continue;
|
|
4792
|
+
}
|
|
4793
|
+
const shape = opts.inspectMark ? await opts.inspectMark(buf).catch(() => null) : null;
|
|
4794
|
+
if (shape?.blank) continue;
|
|
4795
|
+
const edge = candidate.svg ? null : shape?.longEdge ?? null;
|
|
4796
|
+
const tiny = edge !== null && edge < 256;
|
|
4797
|
+
const tall = shape?.width != null && shape?.height != null && shape.width > 0 && shape.height / shape.width > MAX_PORTRAIT_RATIO;
|
|
4798
|
+
const declaredIcon = ICON_SOURCES.has(candidate.source);
|
|
4799
|
+
const unmeasured = !candidate.svg && shape === null;
|
|
4800
|
+
const bigEnough = edge !== null ? declaredIcon && edge >= 512 : unmeasured;
|
|
4801
|
+
const unsure = candidate.score < CONFIDENT_SCORE && !bigEnough;
|
|
4802
|
+
const role = candidate.role === "alternate" || tiny || tall || unsure ? "alternate" : "primary";
|
|
4803
|
+
const picked = {
|
|
4804
|
+
ref,
|
|
4805
|
+
role,
|
|
4806
|
+
source: candidate.source,
|
|
4807
|
+
score: candidate.score,
|
|
4808
|
+
...candidate.background ? { background: candidate.background } : {}
|
|
4809
|
+
};
|
|
4810
|
+
if (role === "primary") return picked;
|
|
4811
|
+
if (tiny)
|
|
4812
|
+
picked.note = `The site icon is favicon-sized (${edge}px), so it was saved as an alternate mark, not the logo. Upload your real logo in Settings.`;
|
|
4813
|
+
else if (tall)
|
|
4814
|
+
picked.note = "The only image we could find is much taller than it is wide, so it was saved as an alternate mark rather than the logo. Upload your real logo in Settings.";
|
|
4815
|
+
else if (unsure)
|
|
4816
|
+
picked.note = "We are not confident this is your logo, so it was saved as an alternate mark. Check it, or upload your real logo in Settings.";
|
|
4817
|
+
else if (candidate.source === "og-image")
|
|
4818
|
+
picked.note = "No site icon was found; the social share image was saved as an alternate mark. Check it before treating it as the logo.";
|
|
4819
|
+
fallback ??= picked;
|
|
4820
|
+
}
|
|
4821
|
+
if (fallback) {
|
|
4822
|
+
if (fallback.note) warnings.push(fallback.note);
|
|
4823
|
+
return fallback;
|
|
4824
|
+
}
|
|
4825
|
+
if (failed) warnings.push("Logo download failed.");
|
|
4826
|
+
warnings.push("No logo captured. Add one manually.");
|
|
4827
|
+
return { role: "primary", source: null };
|
|
4828
|
+
}
|
|
4829
|
+
|
|
4830
|
+
// ../brand-spec/src/mergeScrape.ts
|
|
4831
|
+
var str2 = (v) => typeof v === "string" ? v.trim() : "";
|
|
4832
|
+
var arr = (v) => Array.isArray(v) ? v : [];
|
|
4833
|
+
var hashOf = (ref) => {
|
|
4834
|
+
const s = String(ref ?? "");
|
|
4835
|
+
return s.startsWith("asset:") ? s.slice(6) : s;
|
|
4836
|
+
};
|
|
4837
|
+
function scrapedHexes(palette) {
|
|
4838
|
+
return [palette?.primary, palette?.secondary, ...arr(palette?.accent), ...arr(palette?.neutrals)].filter((c) => typeof c?.hex === "string").map((c) => ({ hex: String(c.hex) }));
|
|
4839
|
+
}
|
|
4840
|
+
var hasPalette = (palette) => scrapedHexes(palette).length > 0;
|
|
4841
|
+
function mergeScrape(existing, scraped) {
|
|
4842
|
+
const cur = existing ?? {};
|
|
4843
|
+
const next = scraped ?? {};
|
|
4844
|
+
const meta = { ...cur.meta ?? {} };
|
|
3647
4845
|
if (!str2(meta.name) && str2(next.meta?.name)) meta.name = str2(next.meta.name);
|
|
3648
4846
|
if (!str2(meta.tagline) && str2(next.meta?.tagline)) meta.tagline = str2(next.meta.tagline);
|
|
3649
4847
|
if (str2(next.meta?.website)) meta.website = str2(next.meta.website);
|
|
@@ -3671,6 +4869,23 @@ function mergeScrape(existing, scraped) {
|
|
|
3671
4869
|
}
|
|
3672
4870
|
return { brand, suggestions: { palette: suggestions } };
|
|
3673
4871
|
}
|
|
4872
|
+
async function inspectMark(buf, toPng2) {
|
|
4873
|
+
const png = await toPng2(buf);
|
|
4874
|
+
const meta = await sharp21(png).metadata();
|
|
4875
|
+
let blank = false;
|
|
4876
|
+
try {
|
|
4877
|
+
const flat = await sharp21(png).flatten({ background: "#ffffff" }).toBuffer();
|
|
4878
|
+
const stats = await sharp21(flat).stats();
|
|
4879
|
+
blank = stats.channels.slice(0, 3).every((c) => c.min > 200);
|
|
4880
|
+
} catch {
|
|
4881
|
+
}
|
|
4882
|
+
return {
|
|
4883
|
+
longEdge: Math.max(meta.width ?? 0, meta.height ?? 0) || null,
|
|
4884
|
+
width: meta.width ?? null,
|
|
4885
|
+
height: meta.height ?? null,
|
|
4886
|
+
blank
|
|
4887
|
+
};
|
|
4888
|
+
}
|
|
3674
4889
|
|
|
3675
4890
|
// ../catalog/src/url.ts
|
|
3676
4891
|
function normalizeStoreUrl(input) {
|
|
@@ -3717,6 +4932,28 @@ function upgradeImageUrl(url) {
|
|
|
3717
4932
|
return url;
|
|
3718
4933
|
}
|
|
3719
4934
|
}
|
|
4935
|
+
var LOCALE_SEGMENT = /^\/[a-z]{2}(?:-[A-Za-z]{2})?\//;
|
|
4936
|
+
function preferCanonicalLocale(urls) {
|
|
4937
|
+
const canonical = /* @__PURE__ */ new Set();
|
|
4938
|
+
for (const u of urls) {
|
|
4939
|
+
try {
|
|
4940
|
+
const parsed = new URL(u);
|
|
4941
|
+
if (!LOCALE_SEGMENT.test(parsed.pathname)) canonical.add(`${parsed.origin}${parsed.pathname}`);
|
|
4942
|
+
} catch {
|
|
4943
|
+
}
|
|
4944
|
+
}
|
|
4945
|
+
if (!canonical.size) return urls;
|
|
4946
|
+
return urls.filter((u) => {
|
|
4947
|
+
try {
|
|
4948
|
+
const parsed = new URL(u);
|
|
4949
|
+
if (!LOCALE_SEGMENT.test(parsed.pathname)) return true;
|
|
4950
|
+
const stripped = parsed.pathname.replace(LOCALE_SEGMENT, "/");
|
|
4951
|
+
return !canonical.has(`${parsed.origin}${stripped}`);
|
|
4952
|
+
} catch {
|
|
4953
|
+
return true;
|
|
4954
|
+
}
|
|
4955
|
+
});
|
|
4956
|
+
}
|
|
3720
4957
|
|
|
3721
4958
|
// ../catalog/src/normalize.ts
|
|
3722
4959
|
function cleanText(s) {
|
|
@@ -3824,10 +5061,48 @@ function dedupeProducts(products) {
|
|
|
3824
5061
|
}
|
|
3825
5062
|
return [...byKey.values()];
|
|
3826
5063
|
}
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
5064
|
+
var ALLOW_PRIVATE = process.env.SCENRI_SCRAPE_ALLOW_PRIVATE === "1";
|
|
5065
|
+
async function assertReachable(url, real) {
|
|
5066
|
+
const u = new URL(url);
|
|
5067
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") {
|
|
5068
|
+
throw new Error(`Scenri reads http and https addresses only, not ${u.protocol.replace(":", "")}`);
|
|
5069
|
+
}
|
|
5070
|
+
if (ALLOW_PRIVATE) return;
|
|
5071
|
+
const host = u.hostname.replace(/^\[|\]$/g, "");
|
|
5072
|
+
if (isIP(host)) {
|
|
5073
|
+
assertPublicHost(u.hostname, [host], false);
|
|
5074
|
+
return;
|
|
5075
|
+
}
|
|
5076
|
+
if (!real) return;
|
|
5077
|
+
let addresses;
|
|
5078
|
+
try {
|
|
5079
|
+
addresses = (await lookup(host, { all: true })).map((a) => a.address);
|
|
5080
|
+
} catch {
|
|
5081
|
+
return;
|
|
5082
|
+
}
|
|
5083
|
+
assertPublicHost(u.hostname, addresses, false);
|
|
5084
|
+
}
|
|
5085
|
+
var USER_AGENT2 = "scenri-catalog/0.1 (+https://scenri.co)";
|
|
5086
|
+
async function readBounded2(res, maxBytes) {
|
|
5087
|
+
if (!maxBytes || !res.body) return res.text();
|
|
5088
|
+
const reader = res.body.getReader();
|
|
5089
|
+
const decoder = new TextDecoder();
|
|
5090
|
+
let out = "";
|
|
5091
|
+
let seen = 0;
|
|
5092
|
+
try {
|
|
5093
|
+
while (seen < maxBytes) {
|
|
5094
|
+
const { done, value } = await reader.read();
|
|
5095
|
+
if (done) break;
|
|
5096
|
+
seen += value.byteLength;
|
|
5097
|
+
out += decoder.decode(value, { stream: true });
|
|
5098
|
+
}
|
|
5099
|
+
} finally {
|
|
5100
|
+
await reader.cancel().catch(() => {
|
|
5101
|
+
});
|
|
5102
|
+
}
|
|
5103
|
+
return out + decoder.decode();
|
|
5104
|
+
}
|
|
5105
|
+
function sleep4(ms) {
|
|
3831
5106
|
return new Promise((r) => setTimeout(r, ms));
|
|
3832
5107
|
}
|
|
3833
5108
|
async function httpGet(url, opts = {}) {
|
|
@@ -3841,17 +5116,34 @@ async function httpGet(url, opts = {}) {
|
|
|
3841
5116
|
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
|
3842
5117
|
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
3843
5118
|
try {
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
5119
|
+
let current = url;
|
|
5120
|
+
let res;
|
|
5121
|
+
for (let hop = 0; ; hop++) {
|
|
5122
|
+
await assertReachable(current, !opts.fetchImpl);
|
|
5123
|
+
res = await fetchImpl(current, {
|
|
5124
|
+
redirect: "manual",
|
|
5125
|
+
signal: ctrl.signal,
|
|
5126
|
+
headers: {
|
|
5127
|
+
"user-agent": USER_AGENT2,
|
|
5128
|
+
accept: opts.accept ?? "application/json, text/html, application/xml, text/xml, */*;q=0.8",
|
|
5129
|
+
...opts.headers ?? {}
|
|
5130
|
+
}
|
|
5131
|
+
});
|
|
5132
|
+
if (res.status < 300 || res.status >= 400) break;
|
|
5133
|
+
const location = res.headers.get("location");
|
|
5134
|
+
await res.body?.cancel().catch(() => {
|
|
5135
|
+
});
|
|
5136
|
+
if (!location || hop >= 5) break;
|
|
5137
|
+
current = new URL(location, current).toString();
|
|
5138
|
+
}
|
|
5139
|
+
if (res.url !== current) {
|
|
5140
|
+
try {
|
|
5141
|
+
Object.defineProperty(res, "url", { value: current, configurable: true });
|
|
5142
|
+
} catch {
|
|
3851
5143
|
}
|
|
3852
|
-
}
|
|
5144
|
+
}
|
|
3853
5145
|
if ((res.status === 429 || res.status >= 500) && attempt < retries) {
|
|
3854
|
-
await
|
|
5146
|
+
await sleep4(400 * 2 ** attempt);
|
|
3855
5147
|
continue;
|
|
3856
5148
|
}
|
|
3857
5149
|
return res;
|
|
@@ -3859,7 +5151,7 @@ async function httpGet(url, opts = {}) {
|
|
|
3859
5151
|
lastErr = err;
|
|
3860
5152
|
if (opts.signal?.aborted) throw err;
|
|
3861
5153
|
if (attempt < retries) {
|
|
3862
|
-
await
|
|
5154
|
+
await sleep4(400 * 2 ** attempt);
|
|
3863
5155
|
continue;
|
|
3864
5156
|
}
|
|
3865
5157
|
throw err;
|
|
@@ -3872,7 +5164,7 @@ async function httpGet(url, opts = {}) {
|
|
|
3872
5164
|
}
|
|
3873
5165
|
async function httpText(url, opts = {}) {
|
|
3874
5166
|
const res = await httpGet(url, opts);
|
|
3875
|
-
const text = await res.
|
|
5167
|
+
const text = await readBounded2(res, opts.maxBytes);
|
|
3876
5168
|
return { ok: res.ok, status: res.status, text, url: res.url || url };
|
|
3877
5169
|
}
|
|
3878
5170
|
async function httpJson(url, opts = {}) {
|
|
@@ -3902,6 +5194,273 @@ async function mapPool(items, concurrency, fn, signal) {
|
|
|
3902
5194
|
await Promise.all(workers);
|
|
3903
5195
|
return results;
|
|
3904
5196
|
}
|
|
5197
|
+
function loadHtml(html) {
|
|
5198
|
+
return parse(html);
|
|
5199
|
+
}
|
|
5200
|
+
function attr(el, name) {
|
|
5201
|
+
return el?.getAttribute(name) ?? void 0;
|
|
5202
|
+
}
|
|
5203
|
+
function textOf(el) {
|
|
5204
|
+
return (el?.text ?? "").replace(/\s+/g, " ").trim();
|
|
5205
|
+
}
|
|
5206
|
+
|
|
5207
|
+
// ../catalog/src/adapters/productPage.ts
|
|
5208
|
+
function stableKey(url) {
|
|
5209
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
5210
|
+
}
|
|
5211
|
+
var typesOf = (obj) => {
|
|
5212
|
+
const t = obj["@type"];
|
|
5213
|
+
const list2 = Array.isArray(t) ? t : t ? [t] : [];
|
|
5214
|
+
return list2.map((x) => String(x).toLowerCase());
|
|
5215
|
+
};
|
|
5216
|
+
function walkJsonLd(node, groups, singles) {
|
|
5217
|
+
if (!node) return;
|
|
5218
|
+
if (Array.isArray(node)) {
|
|
5219
|
+
for (const n of node) walkJsonLd(n, groups, singles);
|
|
5220
|
+
return;
|
|
5221
|
+
}
|
|
5222
|
+
if (typeof node !== "object") return;
|
|
5223
|
+
const obj = node;
|
|
5224
|
+
const types = typesOf(obj);
|
|
5225
|
+
if (types.includes("productgroup")) groups.push(obj);
|
|
5226
|
+
else if (types.includes("product")) singles.push(obj);
|
|
5227
|
+
if (obj["@graph"]) walkJsonLd(obj["@graph"], groups, singles);
|
|
5228
|
+
for (const v of Object.values(obj)) {
|
|
5229
|
+
if (v && typeof v === "object") walkJsonLd(v, groups, singles);
|
|
5230
|
+
}
|
|
5231
|
+
}
|
|
5232
|
+
var asArray = (v) => Array.isArray(v) ? v : v == null ? [] : [v];
|
|
5233
|
+
function imagesOf(n, pageUrl) {
|
|
5234
|
+
return [].concat(n.image ?? []).flat().map((img) => typeof img === "string" ? img : img?.url ?? img?.contentUrl).filter(Boolean).map((u, i) => ({ url: absolutize(pageUrl, String(u)), position: i, width: null, height: null, alt: null })).filter((img) => img.url);
|
|
5235
|
+
}
|
|
5236
|
+
var offerOf = (n) => Array.isArray(n?.offers) ? n.offers[0] : n?.offers;
|
|
5237
|
+
function variesBy(group) {
|
|
5238
|
+
const named = asArray(group.variesBy).map((v) => String(v).split("/").pop() ?? "").filter(Boolean);
|
|
5239
|
+
return named.length ? named : ["size", "color"];
|
|
5240
|
+
}
|
|
5241
|
+
function variantsOf(group, keys) {
|
|
5242
|
+
return asArray(group.hasVariant).filter((v) => v && typeof v === "object").map((v, i) => {
|
|
5243
|
+
const offer = offerOf(v);
|
|
5244
|
+
const options = Object.fromEntries(
|
|
5245
|
+
keys.map((k) => [k, v[k]]).filter(([, val]) => val != null && val !== "")
|
|
5246
|
+
);
|
|
5247
|
+
const base = String(v.sku || v.mpn || v.gtin || `${group.productGroupID ?? group.name ?? "group"}:${i}`);
|
|
5248
|
+
const suffix = Object.values(options).join("/");
|
|
5249
|
+
return {
|
|
5250
|
+
externalKey: suffix ? `${base}:${suffix}` : base,
|
|
5251
|
+
title: [v.name, ...Object.values(options)].filter(Boolean).join(" "),
|
|
5252
|
+
sku: v.sku ? String(v.sku) : null,
|
|
5253
|
+
price: offer?.price != null ? Number(offer.price) : null,
|
|
5254
|
+
compareAtPrice: null,
|
|
5255
|
+
currency: offer?.priceCurrency ?? null,
|
|
5256
|
+
available: offer?.availability ? /instock/i.test(String(offer.availability)) : null,
|
|
5257
|
+
options
|
|
5258
|
+
};
|
|
5259
|
+
});
|
|
5260
|
+
}
|
|
5261
|
+
function fromGroup(group, pageUrl) {
|
|
5262
|
+
const keys = variesBy(group);
|
|
5263
|
+
const variants = variantsOf(group, keys);
|
|
5264
|
+
const url = absolutize(pageUrl, String(group.url ?? group["@id"] ?? pageUrl)) ?? pageUrl;
|
|
5265
|
+
const images = imagesOf(group, pageUrl);
|
|
5266
|
+
const fallback = images.length ? images : imagesOf(asArray(group.hasVariant)[0] ?? {}, pageUrl);
|
|
5267
|
+
return normalizeProduct({
|
|
5268
|
+
externalKey: String(group.productGroupID || group.sku || stableKey(url)),
|
|
5269
|
+
title: String(group.name ?? "Product"),
|
|
5270
|
+
descriptionHtml: group.description ? String(group.description) : null,
|
|
5271
|
+
url,
|
|
5272
|
+
vendor: group.brand?.name ?? (typeof group.brand === "string" ? group.brand : null),
|
|
5273
|
+
productType: group.category ? String(group.category) : null,
|
|
5274
|
+
category: group.category ? String(group.category) : null,
|
|
5275
|
+
price: variants.find((v) => v.price != null)?.price ?? null,
|
|
5276
|
+
compareAtPrice: null,
|
|
5277
|
+
currency: variants.find((v) => v.currency)?.currency ?? null,
|
|
5278
|
+
available: variants.some((v) => v.available) || null,
|
|
5279
|
+
tags: [],
|
|
5280
|
+
variants,
|
|
5281
|
+
images: fallback,
|
|
5282
|
+
raw: group
|
|
5283
|
+
});
|
|
5284
|
+
}
|
|
5285
|
+
function fromSingle(n, pageUrl) {
|
|
5286
|
+
const offers = offerOf(n);
|
|
5287
|
+
const url = String(n.url ?? n["@id"] ?? pageUrl);
|
|
5288
|
+
return normalizeProduct({
|
|
5289
|
+
externalKey: String(n.sku || n.productID || n.mpn || stableKey(url)),
|
|
5290
|
+
title: String(n.name ?? "Product"),
|
|
5291
|
+
descriptionHtml: n.description ? String(n.description) : null,
|
|
5292
|
+
url: absolutize(pageUrl, url) ?? pageUrl,
|
|
5293
|
+
vendor: n.brand?.name ?? (typeof n.brand === "string" ? n.brand : null),
|
|
5294
|
+
productType: n.category ? String(n.category) : null,
|
|
5295
|
+
category: n.category ? String(n.category) : null,
|
|
5296
|
+
price: offers?.price != null ? Number(offers.price) : null,
|
|
5297
|
+
compareAtPrice: null,
|
|
5298
|
+
currency: offers?.priceCurrency ?? null,
|
|
5299
|
+
available: offers?.availability ? /instock/i.test(String(offers.availability)) : null,
|
|
5300
|
+
tags: [],
|
|
5301
|
+
variants: [],
|
|
5302
|
+
images: imagesOf(n, pageUrl),
|
|
5303
|
+
raw: n
|
|
5304
|
+
});
|
|
5305
|
+
}
|
|
5306
|
+
function extractJsonLdProducts(html, pageUrl, doc) {
|
|
5307
|
+
const root = doc ?? loadHtml(html);
|
|
5308
|
+
const groups = [];
|
|
5309
|
+
const singles = [];
|
|
5310
|
+
for (const el of root.querySelectorAll('script[type="application/ld+json"]')) {
|
|
5311
|
+
const raw = el.innerHTML;
|
|
5312
|
+
if (!raw) continue;
|
|
5313
|
+
try {
|
|
5314
|
+
walkJsonLd(JSON.parse(raw), groups, singles);
|
|
5315
|
+
} catch {
|
|
5316
|
+
}
|
|
5317
|
+
}
|
|
5318
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
5319
|
+
for (const g of groups) for (const v of asArray(g.hasVariant)) if (v && typeof v === "object") claimed.add(v);
|
|
5320
|
+
return [
|
|
5321
|
+
...groups.map((g) => fromGroup(g, pageUrl)),
|
|
5322
|
+
...singles.filter((n) => !claimed.has(n)).map((n) => fromSingle(n, pageUrl))
|
|
5323
|
+
];
|
|
5324
|
+
}
|
|
5325
|
+
var PRODUCT_MARKERS = [
|
|
5326
|
+
'meta[property="og:type"][content="product"]',
|
|
5327
|
+
'meta[property="product:price:amount"]',
|
|
5328
|
+
'meta[property="og:price:amount"]',
|
|
5329
|
+
'[itemtype*="schema.org/Product"]',
|
|
5330
|
+
'[itemprop="price"]',
|
|
5331
|
+
'[itemprop="offers"]',
|
|
5332
|
+
'form[action*="/cart/add"]',
|
|
5333
|
+
'[name="add"]'
|
|
5334
|
+
];
|
|
5335
|
+
var BUY_WORDS = /add to (cart|bag|basket)|buy now|add to my bag/i;
|
|
5336
|
+
var BUILD_ASSET = /\/_next\/static\/|\/static\/media\/|\/assets\/(icons|flags|ui)\//i;
|
|
5337
|
+
var NOT_A_PACKSHOT = /logo|icon|sprite|pixel|avatar|\bflags?\b|\/flags?\/|locale|country|currency|badge|payment|social/i;
|
|
5338
|
+
function looksLikeProduct(html, doc) {
|
|
5339
|
+
const $ = doc ?? loadHtml(html);
|
|
5340
|
+
for (const sel of PRODUCT_MARKERS) {
|
|
5341
|
+
try {
|
|
5342
|
+
if ($.querySelector(sel)) return true;
|
|
5343
|
+
} catch {
|
|
5344
|
+
}
|
|
5345
|
+
}
|
|
5346
|
+
for (const el of $.querySelectorAll('button, input[type="submit"], a')) {
|
|
5347
|
+
const words = `${textOf(el)} ${attr(el, "value") ?? ""} ${attr(el, "aria-label") ?? ""}`;
|
|
5348
|
+
if (BUY_WORDS.test(words)) return true;
|
|
5349
|
+
}
|
|
5350
|
+
return false;
|
|
5351
|
+
}
|
|
5352
|
+
function galleryImages($, pageUrl, cap2 = 12) {
|
|
5353
|
+
const images = [];
|
|
5354
|
+
const og = attr($.querySelector('meta[property="og:image"]'), "content");
|
|
5355
|
+
if (og) {
|
|
5356
|
+
const abs = absolutize(pageUrl, og);
|
|
5357
|
+
if (abs) images.push({ url: abs, position: 0, width: null, height: null, alt: null });
|
|
5358
|
+
}
|
|
5359
|
+
for (const el of $.querySelectorAll("img[src]")) {
|
|
5360
|
+
if (images.length >= cap2) break;
|
|
5361
|
+
const src = attr(el, "src") || attr(el, "data-src");
|
|
5362
|
+
const abs = src ? absolutize(pageUrl, src) : null;
|
|
5363
|
+
if (!abs || NOT_A_PACKSHOT.test(abs) || BUILD_ASSET.test(abs)) continue;
|
|
5364
|
+
if (images.some((x) => x.url === abs)) continue;
|
|
5365
|
+
images.push({ url: abs, position: images.length, width: null, height: null, alt: attr(el, "alt") ?? null });
|
|
5366
|
+
}
|
|
5367
|
+
return images;
|
|
5368
|
+
}
|
|
5369
|
+
function parseProductHtml(html, pageUrl, doc) {
|
|
5370
|
+
const $ = doc ?? loadHtml(html);
|
|
5371
|
+
if (!looksLikeProduct(html, $)) return null;
|
|
5372
|
+
const title = attr($.querySelector('meta[property="og:title"]'), "content") || textOf($.querySelector("h1")) || textOf($.querySelector("title"));
|
|
5373
|
+
if (!title) return null;
|
|
5374
|
+
const desc = attr($.querySelector('meta[property="og:description"]'), "content") || attr($.querySelector('meta[name="description"]'), "content") || null;
|
|
5375
|
+
const images = galleryImages($, pageUrl);
|
|
5376
|
+
const canonical = attr($.querySelector('link[rel="canonical"]'), "href");
|
|
5377
|
+
const url = canonical ? absolutize(pageUrl, canonical) ?? pageUrl : pageUrl;
|
|
5378
|
+
return normalizeProduct({
|
|
5379
|
+
externalKey: stableKey(url),
|
|
5380
|
+
title,
|
|
5381
|
+
descriptionHtml: desc,
|
|
5382
|
+
url,
|
|
5383
|
+
images,
|
|
5384
|
+
variants: [],
|
|
5385
|
+
tags: [],
|
|
5386
|
+
raw: { source: "html" }
|
|
5387
|
+
});
|
|
5388
|
+
}
|
|
5389
|
+
function sellsSomething(p) {
|
|
5390
|
+
const n = p.raw ?? {};
|
|
5391
|
+
return Boolean(n.offers || n.sku || n.gtin || n.mpn || n.hasVariant || p.price != null);
|
|
5392
|
+
}
|
|
5393
|
+
function withGallery(product, doc, pageUrl) {
|
|
5394
|
+
const have = new Set((product.images ?? []).map((i) => i.url));
|
|
5395
|
+
const extra = galleryImages(doc, pageUrl).filter((i) => !have.has(i.url));
|
|
5396
|
+
if (!extra.length) return product;
|
|
5397
|
+
const images = [...product.images ?? []];
|
|
5398
|
+
for (const img of extra) {
|
|
5399
|
+
if (images.length >= 12) break;
|
|
5400
|
+
images.push({ ...img, position: images.length });
|
|
5401
|
+
}
|
|
5402
|
+
return { ...product, images };
|
|
5403
|
+
}
|
|
5404
|
+
function productsFromPage(html, url) {
|
|
5405
|
+
const doc = loadHtml(html);
|
|
5406
|
+
const declared = looksLikeProduct(html, doc);
|
|
5407
|
+
const fromLd = extractJsonLdProducts(html, url, doc);
|
|
5408
|
+
const selling = declared ? fromLd : fromLd.filter(sellsSomething);
|
|
5409
|
+
if (selling.length === 1) return [withGallery(selling[0], doc, url)];
|
|
5410
|
+
if (selling.length) return selling;
|
|
5411
|
+
if (!declared) return [];
|
|
5412
|
+
const one = parseProductHtml(html, url, doc);
|
|
5413
|
+
return one ? [one] : [];
|
|
5414
|
+
}
|
|
5415
|
+
var sleep5 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
5416
|
+
async function fetchProductPages(ctx, urls, opts = {}) {
|
|
5417
|
+
const out = [];
|
|
5418
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5419
|
+
let kept = 0;
|
|
5420
|
+
const want = opts.want ?? Number.POSITIVE_INFINITY;
|
|
5421
|
+
const ceiling = opts.maxPages ?? (Number.isFinite(want) ? want * 3 : urls.length);
|
|
5422
|
+
const take = urls.slice(0, ceiling);
|
|
5423
|
+
let bytes = 0;
|
|
5424
|
+
await mapPool(
|
|
5425
|
+
take,
|
|
5426
|
+
opts.delayMs ? 1 : opts.concurrency ?? 5,
|
|
5427
|
+
async (u) => {
|
|
5428
|
+
if (kept >= want) return;
|
|
5429
|
+
if (opts.deadline != null && Date.now() > opts.deadline) return;
|
|
5430
|
+
if (opts.maxTotalBytes != null && bytes >= opts.maxTotalBytes) return;
|
|
5431
|
+
try {
|
|
5432
|
+
if (opts.delayMs) await sleep5(opts.delayMs);
|
|
5433
|
+
const { ok, text, url } = await httpText(u, {
|
|
5434
|
+
fetchImpl: ctx.fetchImpl,
|
|
5435
|
+
signal: ctx.signal,
|
|
5436
|
+
accept: "text/html",
|
|
5437
|
+
maxBytes: opts.maxBytes
|
|
5438
|
+
});
|
|
5439
|
+
bytes += text.length;
|
|
5440
|
+
if (opts.stats) {
|
|
5441
|
+
opts.stats.pages += 1;
|
|
5442
|
+
opts.stats.bytes = bytes;
|
|
5443
|
+
}
|
|
5444
|
+
if (!ok) {
|
|
5445
|
+
if (opts.stats) opts.stats.refused = (opts.stats.refused ?? 0) + 1;
|
|
5446
|
+
return;
|
|
5447
|
+
}
|
|
5448
|
+
for (const p of productsFromPage(text, url)) {
|
|
5449
|
+
if (seen.has(p.externalKey)) continue;
|
|
5450
|
+
seen.add(p.externalKey);
|
|
5451
|
+
kept++;
|
|
5452
|
+
if (opts.onEach) opts.onEach(p);
|
|
5453
|
+
else out.push(p);
|
|
5454
|
+
}
|
|
5455
|
+
opts.onProduct?.(kept);
|
|
5456
|
+
} catch {
|
|
5457
|
+
if (opts.stats) opts.stats.refused = (opts.stats.refused ?? 0) + 1;
|
|
5458
|
+
}
|
|
5459
|
+
},
|
|
5460
|
+
ctx.signal
|
|
5461
|
+
);
|
|
5462
|
+
return out;
|
|
5463
|
+
}
|
|
3905
5464
|
|
|
3906
5465
|
// ../catalog/src/adapters/shopify.ts
|
|
3907
5466
|
function mapShopifyProduct(base, p) {
|
|
@@ -3955,10 +5514,14 @@ function mapShopifyProduct(base, p) {
|
|
|
3955
5514
|
async function fetchProductsJsonPage(ctx, page, limit = 250) {
|
|
3956
5515
|
const origin = originOf(ctx.baseUrl);
|
|
3957
5516
|
const url = `${origin}/products.json?limit=${limit}&page=${page}`;
|
|
3958
|
-
const { ok, json } = await httpJson(url, {
|
|
3959
|
-
|
|
3960
|
-
|
|
5517
|
+
const { ok, status, json } = await httpJson(url, {
|
|
5518
|
+
fetchImpl: ctx.fetchImpl,
|
|
5519
|
+
signal: ctx.signal
|
|
5520
|
+
});
|
|
5521
|
+
if (ok && json?.products) return { products: json.products, blocked: false };
|
|
5522
|
+
return { products: [], blocked: status === 401 || status === 403 || status === 404 || status >= 500 };
|
|
3961
5523
|
}
|
|
5524
|
+
var JSON_BLOCKED = "json-blocked";
|
|
3962
5525
|
async function collectSitemapProductUrls(ctx) {
|
|
3963
5526
|
const origin = originOf(ctx.baseUrl);
|
|
3964
5527
|
const urls = /* @__PURE__ */ new Set();
|
|
@@ -3984,7 +5547,7 @@ async function collectSitemapProductUrls(ctx) {
|
|
|
3984
5547
|
}
|
|
3985
5548
|
}
|
|
3986
5549
|
}
|
|
3987
|
-
return [...urls];
|
|
5550
|
+
return preferCanonicalLocale([...urls]);
|
|
3988
5551
|
}
|
|
3989
5552
|
var shopifyAdapter = {
|
|
3990
5553
|
platform: "shopify",
|
|
@@ -4016,6 +5579,7 @@ var shopifyAdapter = {
|
|
|
4016
5579
|
},
|
|
4017
5580
|
async discover(ctx) {
|
|
4018
5581
|
const warnings = [];
|
|
5582
|
+
const hints = [];
|
|
4019
5583
|
const keys = /* @__PURE__ */ new Set();
|
|
4020
5584
|
const productUrls = /* @__PURE__ */ new Set();
|
|
4021
5585
|
const origin = originOf(ctx.baseUrl);
|
|
@@ -4023,8 +5587,12 @@ var shopifyAdapter = {
|
|
|
4023
5587
|
let emptyStreak = 0;
|
|
4024
5588
|
while (emptyStreak < 1) {
|
|
4025
5589
|
if (ctx.signal?.aborted) throw new Error("aborted");
|
|
4026
|
-
const products = await fetchProductsJsonPage(ctx, page);
|
|
5590
|
+
const { products, blocked: blocked2 } = await fetchProductsJsonPage(ctx, page);
|
|
4027
5591
|
ctx.onProgress?.({ stage: "discovering", discovered: keys.size, message: `Shopify page ${page}` });
|
|
5592
|
+
if (blocked2 && page === 1) {
|
|
5593
|
+
hints.push(JSON_BLOCKED);
|
|
5594
|
+
warnings.push("This store does not serve its product API, so the product pages were read instead");
|
|
5595
|
+
}
|
|
4028
5596
|
if (!products.length) {
|
|
4029
5597
|
emptyStreak++;
|
|
4030
5598
|
break;
|
|
@@ -4053,17 +5621,23 @@ var shopifyAdapter = {
|
|
|
4053
5621
|
productKeys: [...keys],
|
|
4054
5622
|
productUrls: [...productUrls],
|
|
4055
5623
|
estimatedTotal: keys.size || productUrls.size || null,
|
|
4056
|
-
|
|
5624
|
+
// A store whose own product API answered but refused us leaves nothing
|
|
5625
|
+
// to read but the pages themselves, one request each. That is what
|
|
5626
|
+
// gymshark.com does, and it is the run worth batching.
|
|
5627
|
+
byPage: hints.includes(JSON_BLOCKED),
|
|
5628
|
+
warnings,
|
|
5629
|
+
hints
|
|
4057
5630
|
};
|
|
4058
5631
|
},
|
|
4059
5632
|
async fetchAll(ctx, discovered) {
|
|
4060
5633
|
const origin = originOf(ctx.baseUrl);
|
|
4061
5634
|
const out = [];
|
|
4062
5635
|
const seen = /* @__PURE__ */ new Set();
|
|
5636
|
+
const jsonBlocked = discovered.hints?.includes(JSON_BLOCKED) ?? false;
|
|
4063
5637
|
let page = 1;
|
|
4064
|
-
while (
|
|
5638
|
+
while (!jsonBlocked) {
|
|
4065
5639
|
if (ctx.signal?.aborted) throw new Error("aborted");
|
|
4066
|
-
const products = await fetchProductsJsonPage(ctx, page);
|
|
5640
|
+
const { products } = await fetchProductsJsonPage(ctx, page);
|
|
4067
5641
|
if (!products.length) break;
|
|
4068
5642
|
for (const p of products) {
|
|
4069
5643
|
const mapped = mapShopifyProduct(origin, p);
|
|
@@ -4080,48 +5654,56 @@ var shopifyAdapter = {
|
|
|
4080
5654
|
page++;
|
|
4081
5655
|
if (page > 1e4) break;
|
|
4082
5656
|
}
|
|
4083
|
-
const
|
|
4084
|
-
const
|
|
4085
|
-
return
|
|
5657
|
+
const handleOf = (u) => {
|
|
5658
|
+
const raw = /\/products\/([^/?#]+)/i.exec(u)?.[1];
|
|
5659
|
+
return raw ? decodeURIComponent(raw) : null;
|
|
5660
|
+
};
|
|
5661
|
+
const stillMissing = () => discovered.productUrls.filter((u) => {
|
|
5662
|
+
const handle = handleOf(u);
|
|
5663
|
+
return handle && !out.some((p) => p.handle === handle);
|
|
4086
5664
|
});
|
|
4087
|
-
if (
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
if (
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
5665
|
+
if (!jsonBlocked) {
|
|
5666
|
+
const missingUrls = stillMissing();
|
|
5667
|
+
if (missingUrls.length) {
|
|
5668
|
+
await mapPool(
|
|
5669
|
+
missingUrls,
|
|
5670
|
+
6,
|
|
5671
|
+
async (u) => {
|
|
5672
|
+
const handle = handleOf(u);
|
|
5673
|
+
if (!handle) return;
|
|
5674
|
+
const { ok, json } = await httpJson(`${origin}/products/${handle}.json`, {
|
|
5675
|
+
fetchImpl: ctx.fetchImpl,
|
|
5676
|
+
signal: ctx.signal
|
|
5677
|
+
});
|
|
5678
|
+
if (ok && json?.product) {
|
|
5679
|
+
const mapped = mapShopifyProduct(origin, json.product);
|
|
5680
|
+
if (!seen.has(mapped.externalKey)) {
|
|
5681
|
+
seen.add(mapped.externalKey);
|
|
5682
|
+
out.push(mapped);
|
|
5683
|
+
ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
|
|
5684
|
+
}
|
|
4104
5685
|
}
|
|
4105
|
-
}
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
5686
|
+
},
|
|
5687
|
+
ctx.signal
|
|
5688
|
+
);
|
|
5689
|
+
}
|
|
5690
|
+
}
|
|
5691
|
+
const unread = stillMissing();
|
|
5692
|
+
if (unread.length) {
|
|
5693
|
+
for (const p of await fetchProductPages(ctx, unread, {
|
|
5694
|
+
concurrency: 4,
|
|
5695
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched: out.length + fetched })
|
|
5696
|
+
})) {
|
|
5697
|
+
if (seen.has(p.externalKey)) continue;
|
|
5698
|
+
seen.add(p.externalKey);
|
|
5699
|
+
out.push(p);
|
|
5700
|
+
}
|
|
4109
5701
|
}
|
|
4110
5702
|
return out;
|
|
4111
5703
|
}
|
|
4112
5704
|
};
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
}
|
|
4116
|
-
function attr(el, name) {
|
|
4117
|
-
return el?.getAttribute(name) ?? void 0;
|
|
4118
|
-
}
|
|
4119
|
-
function textOf(el) {
|
|
4120
|
-
return (el?.text ?? "").replace(/\s+/g, " ").trim();
|
|
4121
|
-
}
|
|
4122
|
-
function stableKey(url) {
|
|
4123
|
-
return createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
4124
|
-
}
|
|
5705
|
+
|
|
5706
|
+
// ../catalog/src/adapters/generic.ts
|
|
4125
5707
|
async function extractSitemapUrls(ctx, filter = () => true) {
|
|
4126
5708
|
const origin = originOf(ctx.baseUrl);
|
|
4127
5709
|
const out = /* @__PURE__ */ new Set();
|
|
@@ -4153,89 +5735,7 @@ async function extractSitemapUrls(ctx, filter = () => true) {
|
|
|
4153
5735
|
}
|
|
4154
5736
|
if (seen.size > 200) break;
|
|
4155
5737
|
}
|
|
4156
|
-
return [...out];
|
|
4157
|
-
}
|
|
4158
|
-
function walkJsonLd(node, out) {
|
|
4159
|
-
if (!node) return;
|
|
4160
|
-
if (Array.isArray(node)) {
|
|
4161
|
-
for (const n of node) walkJsonLd(n, out);
|
|
4162
|
-
return;
|
|
4163
|
-
}
|
|
4164
|
-
if (typeof node !== "object") return;
|
|
4165
|
-
const obj = node;
|
|
4166
|
-
const type = obj["@type"];
|
|
4167
|
-
const types = Array.isArray(type) ? type : type ? [type] : [];
|
|
4168
|
-
if (types.some((t) => String(t).toLowerCase() === "product")) out.push(obj);
|
|
4169
|
-
if (obj["@graph"]) walkJsonLd(obj["@graph"], out);
|
|
4170
|
-
for (const v of Object.values(obj)) {
|
|
4171
|
-
if (v && typeof v === "object") walkJsonLd(v, out);
|
|
4172
|
-
}
|
|
4173
|
-
}
|
|
4174
|
-
function extractJsonLdProducts(html, pageUrl) {
|
|
4175
|
-
const root = loadHtml(html);
|
|
4176
|
-
const nodes = [];
|
|
4177
|
-
for (const el of root.querySelectorAll('script[type="application/ld+json"]')) {
|
|
4178
|
-
const raw = el.innerHTML;
|
|
4179
|
-
if (!raw) continue;
|
|
4180
|
-
try {
|
|
4181
|
-
walkJsonLd(JSON.parse(raw), nodes);
|
|
4182
|
-
} catch {
|
|
4183
|
-
}
|
|
4184
|
-
}
|
|
4185
|
-
return nodes.map((n) => {
|
|
4186
|
-
const offers = Array.isArray(n.offers) ? n.offers[0] : n.offers;
|
|
4187
|
-
const images = [].concat(n.image ?? []).flat().map((img) => typeof img === "string" ? img : img?.url ?? img?.contentUrl).filter(Boolean).map((u, i) => ({ url: absolutize(pageUrl, String(u)), position: i, width: null, height: null, alt: null })).filter((img) => img.url);
|
|
4188
|
-
const url = String(n.url ?? n["@id"] ?? pageUrl);
|
|
4189
|
-
return normalizeProduct({
|
|
4190
|
-
externalKey: String(n.sku || n.productID || n.mpn || stableKey(url)),
|
|
4191
|
-
title: String(n.name ?? "Product"),
|
|
4192
|
-
descriptionHtml: n.description ? String(n.description) : null,
|
|
4193
|
-
url: absolutize(pageUrl, url) ?? pageUrl,
|
|
4194
|
-
vendor: n.brand?.name ?? (typeof n.brand === "string" ? n.brand : null),
|
|
4195
|
-
productType: n.category ? String(n.category) : null,
|
|
4196
|
-
category: n.category ? String(n.category) : null,
|
|
4197
|
-
price: offers?.price != null ? Number(offers.price) : null,
|
|
4198
|
-
compareAtPrice: null,
|
|
4199
|
-
currency: offers?.priceCurrency ?? null,
|
|
4200
|
-
available: offers?.availability ? /instock/i.test(String(offers.availability)) : null,
|
|
4201
|
-
tags: [],
|
|
4202
|
-
variants: [],
|
|
4203
|
-
images,
|
|
4204
|
-
raw: n
|
|
4205
|
-
});
|
|
4206
|
-
});
|
|
4207
|
-
}
|
|
4208
|
-
function parseProductHtml(html, pageUrl) {
|
|
4209
|
-
const $ = loadHtml(html);
|
|
4210
|
-
const title = attr($.querySelector('meta[property="og:title"]'), "content") || textOf($.querySelector("h1")) || textOf($.querySelector("title"));
|
|
4211
|
-
if (!title) return null;
|
|
4212
|
-
const desc = attr($.querySelector('meta[property="og:description"]'), "content") || attr($.querySelector('meta[name="description"]'), "content") || null;
|
|
4213
|
-
const images = [];
|
|
4214
|
-
const og = attr($.querySelector('meta[property="og:image"]'), "content");
|
|
4215
|
-
if (og) {
|
|
4216
|
-
const abs = absolutize(pageUrl, og);
|
|
4217
|
-
if (abs) images.push({ url: abs, position: 0, width: null, height: null, alt: null });
|
|
4218
|
-
}
|
|
4219
|
-
for (const el of $.querySelectorAll("img[src]")) {
|
|
4220
|
-
if (images.length >= 12) break;
|
|
4221
|
-
const src = attr(el, "src") || attr(el, "data-src");
|
|
4222
|
-
const abs = src ? absolutize(pageUrl, src) : null;
|
|
4223
|
-
if (!abs || /logo|icon|sprite|pixel|avatar/i.test(abs)) continue;
|
|
4224
|
-
if (images.some((x) => x.url === abs)) continue;
|
|
4225
|
-
images.push({ url: abs, position: images.length, width: null, height: null, alt: attr(el, "alt") ?? null });
|
|
4226
|
-
}
|
|
4227
|
-
const canonical = attr($.querySelector('link[rel="canonical"]'), "href");
|
|
4228
|
-
const url = canonical ? absolutize(pageUrl, canonical) ?? pageUrl : pageUrl;
|
|
4229
|
-
return normalizeProduct({
|
|
4230
|
-
externalKey: stableKey(url),
|
|
4231
|
-
title,
|
|
4232
|
-
descriptionHtml: desc,
|
|
4233
|
-
url,
|
|
4234
|
-
images,
|
|
4235
|
-
variants: [],
|
|
4236
|
-
tags: [],
|
|
4237
|
-
raw: { source: "html" }
|
|
4238
|
-
});
|
|
5738
|
+
return preferCanonicalLocale([...out]);
|
|
4239
5739
|
}
|
|
4240
5740
|
async function extractFeedUrls(ctx) {
|
|
4241
5741
|
const origin = originOf(ctx.baseUrl);
|
|
@@ -4339,41 +5839,15 @@ var genericAdapter = {
|
|
|
4339
5839
|
productKeys: [...productUrls].map(stableKey),
|
|
4340
5840
|
productUrls: [...productUrls],
|
|
4341
5841
|
estimatedTotal: productUrls.size || null,
|
|
5842
|
+
// Nothing here but addresses: every product is its own page fetch.
|
|
5843
|
+
byPage: true,
|
|
4342
5844
|
warnings
|
|
4343
5845
|
};
|
|
4344
5846
|
},
|
|
4345
5847
|
async fetchAll(ctx, discovered) {
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
discovered.productUrls,
|
|
4350
|
-
5,
|
|
4351
|
-
async (u) => {
|
|
4352
|
-
try {
|
|
4353
|
-
const { ok, text, url } = await httpText(u, {
|
|
4354
|
-
fetchImpl: ctx.fetchImpl,
|
|
4355
|
-
signal: ctx.signal,
|
|
4356
|
-
accept: "text/html"
|
|
4357
|
-
});
|
|
4358
|
-
if (!ok) return;
|
|
4359
|
-
const fromLd = extractJsonLdProducts(text, url);
|
|
4360
|
-
const list2 = fromLd.length ? fromLd : [parseProductHtml(text, url)].filter(Boolean);
|
|
4361
|
-
for (const p of list2) {
|
|
4362
|
-
if (seen.has(p.externalKey)) continue;
|
|
4363
|
-
seen.add(p.externalKey);
|
|
4364
|
-
out.push(p);
|
|
4365
|
-
}
|
|
4366
|
-
ctx.onProgress?.({
|
|
4367
|
-
stage: "fetching_products",
|
|
4368
|
-
fetched: out.length,
|
|
4369
|
-
discovered: discovered.productUrls.length
|
|
4370
|
-
});
|
|
4371
|
-
} catch {
|
|
4372
|
-
}
|
|
4373
|
-
},
|
|
4374
|
-
ctx.signal
|
|
4375
|
-
);
|
|
4376
|
-
return out;
|
|
5848
|
+
return fetchProductPages(ctx, discovered.productUrls, {
|
|
5849
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched, discovered: discovered.productUrls.length })
|
|
5850
|
+
});
|
|
4377
5851
|
}
|
|
4378
5852
|
};
|
|
4379
5853
|
|
|
@@ -4481,6 +5955,9 @@ var woocommerceAdapter = {
|
|
|
4481
5955
|
productKeys: [...keys],
|
|
4482
5956
|
productUrls: [...productUrls],
|
|
4483
5957
|
estimatedTotal: keys.size || productUrls.size || null,
|
|
5958
|
+
// The WooCommerce store API answers in pages of a hundred, so the whole
|
|
5959
|
+
// catalogue is a handful of requests and needs no batching of ours.
|
|
5960
|
+
byPage: false,
|
|
4484
5961
|
warnings
|
|
4485
5962
|
};
|
|
4486
5963
|
},
|
|
@@ -4508,34 +5985,17 @@ var woocommerceAdapter = {
|
|
|
4508
5985
|
});
|
|
4509
5986
|
if (batch.length < 100) break;
|
|
4510
5987
|
page++;
|
|
4511
|
-
if (page > 1e4) break;
|
|
4512
|
-
}
|
|
4513
|
-
if (apiWorked && out.length) return out;
|
|
4514
|
-
const urls = discovered.productUrls.length ? discovered.productUrls : [...new Set(discovered.productKeys)];
|
|
4515
|
-
await
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
text,
|
|
4523
|
-
url: finalUrl
|
|
4524
|
-
} = await httpText(u, { fetchImpl: ctx.fetchImpl, signal: ctx.signal, accept: "text/html" });
|
|
4525
|
-
if (!ok) return;
|
|
4526
|
-
const fromLd = extractJsonLdProducts(text, finalUrl);
|
|
4527
|
-
const products = fromLd.length ? fromLd : [parseProductHtml(text, finalUrl)].filter(Boolean);
|
|
4528
|
-
for (const p of products) {
|
|
4529
|
-
if (seen.has(p.externalKey)) continue;
|
|
4530
|
-
seen.add(p.externalKey);
|
|
4531
|
-
out.push(p);
|
|
4532
|
-
}
|
|
4533
|
-
ctx.onProgress?.({ stage: "fetching_products", fetched: out.length });
|
|
4534
|
-
} catch {
|
|
4535
|
-
}
|
|
4536
|
-
},
|
|
4537
|
-
ctx.signal
|
|
4538
|
-
);
|
|
5988
|
+
if (page > 1e4) break;
|
|
5989
|
+
}
|
|
5990
|
+
if (apiWorked && out.length) return out;
|
|
5991
|
+
const urls = discovered.productUrls.length ? discovered.productUrls : [...new Set(discovered.productKeys)];
|
|
5992
|
+
for (const p of await fetchProductPages(ctx, urls, {
|
|
5993
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched: out.length + fetched })
|
|
5994
|
+
})) {
|
|
5995
|
+
if (seen.has(p.externalKey)) continue;
|
|
5996
|
+
seen.add(p.externalKey);
|
|
5997
|
+
out.push(p);
|
|
5998
|
+
}
|
|
4539
5999
|
return out;
|
|
4540
6000
|
}
|
|
4541
6001
|
};
|
|
@@ -4604,41 +6064,15 @@ var webflowAdapter = {
|
|
|
4604
6064
|
productKeys: [...productUrls],
|
|
4605
6065
|
productUrls: [...productUrls],
|
|
4606
6066
|
estimatedTotal: productUrls.size || null,
|
|
6067
|
+
// Webflow has no catalogue API to ask: every product is its own page.
|
|
6068
|
+
byPage: true,
|
|
4607
6069
|
warnings
|
|
4608
6070
|
};
|
|
4609
6071
|
},
|
|
4610
6072
|
async fetchAll(ctx, discovered) {
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
discovered.productUrls,
|
|
4615
|
-
5,
|
|
4616
|
-
async (u) => {
|
|
4617
|
-
try {
|
|
4618
|
-
const { ok, text, url } = await httpText(u, {
|
|
4619
|
-
fetchImpl: ctx.fetchImpl,
|
|
4620
|
-
signal: ctx.signal,
|
|
4621
|
-
accept: "text/html"
|
|
4622
|
-
});
|
|
4623
|
-
if (!ok) return;
|
|
4624
|
-
const fromLd = extractJsonLdProducts(text, url);
|
|
4625
|
-
const list2 = fromLd.length ? fromLd : [parseProductHtml(text, url)].filter(Boolean);
|
|
4626
|
-
for (const p of list2) {
|
|
4627
|
-
if (seen.has(p.externalKey)) continue;
|
|
4628
|
-
seen.add(p.externalKey);
|
|
4629
|
-
out.push(p);
|
|
4630
|
-
}
|
|
4631
|
-
ctx.onProgress?.({
|
|
4632
|
-
stage: "fetching_products",
|
|
4633
|
-
fetched: out.length,
|
|
4634
|
-
discovered: discovered.productUrls.length
|
|
4635
|
-
});
|
|
4636
|
-
} catch {
|
|
4637
|
-
}
|
|
4638
|
-
},
|
|
4639
|
-
ctx.signal
|
|
4640
|
-
);
|
|
4641
|
-
return out;
|
|
6073
|
+
return fetchProductPages(ctx, discovered.productUrls, {
|
|
6074
|
+
onProduct: (fetched) => ctx.onProgress?.({ stage: "fetching_products", fetched, discovered: discovered.productUrls.length })
|
|
6075
|
+
});
|
|
4642
6076
|
}
|
|
4643
6077
|
};
|
|
4644
6078
|
|
|
@@ -4680,7 +6114,7 @@ function baseProgress(platform = "unknown") {
|
|
|
4680
6114
|
warnings: []
|
|
4681
6115
|
};
|
|
4682
6116
|
}
|
|
4683
|
-
async function
|
|
6117
|
+
async function discoverCatalog(opts) {
|
|
4684
6118
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
4685
6119
|
const progress = baseProgress();
|
|
4686
6120
|
const emit = (patch2) => {
|
|
@@ -4723,57 +6157,247 @@ async function runCatalogIngestion(opts) {
|
|
|
4723
6157
|
warnings: [...progress.warnings, ...discovered.warnings],
|
|
4724
6158
|
message: `Found ${discovered.estimatedTotal ?? discovered.productUrls.length} products`
|
|
4725
6159
|
});
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
6160
|
+
const estimatedTotal = discovered.estimatedTotal ?? discovered.productUrls.length;
|
|
6161
|
+
let empty = null;
|
|
6162
|
+
if (!estimatedTotal) {
|
|
6163
|
+
empty = {
|
|
6164
|
+
code: "empty_catalog",
|
|
6165
|
+
message: detection.platform === "generic" ? "No public product catalog found. This store may be JavaScript-rendered or require authentication." : `No products discovered on this ${detection.platform} store.`
|
|
6166
|
+
};
|
|
6167
|
+
emit({ stage: "failed", errors: [...progress.errors, empty] });
|
|
6168
|
+
}
|
|
6169
|
+
return {
|
|
6170
|
+
baseUrl: detection.baseUrl,
|
|
6171
|
+
detection,
|
|
6172
|
+
adapter,
|
|
6173
|
+
ctx: { ...ctx, baseUrl: detection.baseUrl },
|
|
6174
|
+
productUrls: discovered.productUrls,
|
|
6175
|
+
productKeys: discovered.productKeys,
|
|
6176
|
+
byPage: (discovered.byPage ?? false) && discovered.productUrls.length > 0,
|
|
6177
|
+
estimatedTotal,
|
|
6178
|
+
progress,
|
|
6179
|
+
empty,
|
|
6180
|
+
emit,
|
|
6181
|
+
async fetchAll() {
|
|
6182
|
+
emit({ stage: "fetching_products", message: "Fetching product details" });
|
|
6183
|
+
try {
|
|
6184
|
+
return dedupeProducts(await adapter.fetchAll({ ...ctx, baseUrl: detection.baseUrl }, discovered));
|
|
6185
|
+
} catch (err) {
|
|
6186
|
+
emit({
|
|
6187
|
+
stage: "partial",
|
|
6188
|
+
errors: [...progress.errors, { code: "fetch_failed", message: String(err?.message ?? err), retryable: true }]
|
|
6189
|
+
});
|
|
6190
|
+
return [];
|
|
6191
|
+
}
|
|
6192
|
+
}
|
|
6193
|
+
};
|
|
6194
|
+
}
|
|
6195
|
+
|
|
6196
|
+
// ../catalog/src/robots.ts
|
|
6197
|
+
var ALLOW_ALL = { rules: [], crawlDelayMs: 0 };
|
|
6198
|
+
function toRegExp(pattern) {
|
|
6199
|
+
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
6200
|
+
const anchored = escaped.endsWith("\\$") ? `${escaped.slice(0, -2)}$` : escaped;
|
|
6201
|
+
return new RegExp(`^${anchored}`);
|
|
6202
|
+
}
|
|
6203
|
+
function parseRobots(text, agent = USER_AGENT2) {
|
|
6204
|
+
const lines = text.split(/\r?\n/).map((l) => l.replace(/#.*$/, "").trim());
|
|
6205
|
+
const groups = [];
|
|
6206
|
+
let current = null;
|
|
6207
|
+
let expectingAgents = false;
|
|
6208
|
+
for (const line of lines) {
|
|
6209
|
+
const at = line.indexOf(":");
|
|
6210
|
+
if (at < 0) continue;
|
|
6211
|
+
const field = line.slice(0, at).trim().toLowerCase();
|
|
6212
|
+
const value = line.slice(at + 1).trim();
|
|
6213
|
+
if (field === "user-agent") {
|
|
6214
|
+
if (!current || !expectingAgents) {
|
|
6215
|
+
current = { agents: [], rules: [], delay: 0 };
|
|
6216
|
+
groups.push(current);
|
|
6217
|
+
expectingAgents = true;
|
|
6218
|
+
}
|
|
6219
|
+
current.agents.push(value.toLowerCase());
|
|
6220
|
+
continue;
|
|
6221
|
+
}
|
|
6222
|
+
if (!current) continue;
|
|
6223
|
+
expectingAgents = false;
|
|
6224
|
+
if (field === "disallow" && value) current.rules.push({ allow: false, pattern: value });
|
|
6225
|
+
else if (field === "allow" && value) current.rules.push({ allow: true, pattern: value });
|
|
6226
|
+
else if (field === "crawl-delay") {
|
|
6227
|
+
const n = Number(value);
|
|
6228
|
+
if (Number.isFinite(n) && n > 0) current.delay = n;
|
|
6229
|
+
}
|
|
6230
|
+
}
|
|
6231
|
+
const token = agent.toLowerCase();
|
|
6232
|
+
const mine = groups.filter((g) => g.agents.some((a) => a !== "*" && token.includes(a)));
|
|
6233
|
+
const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes("*"));
|
|
6234
|
+
return {
|
|
6235
|
+
rules: chosen.flatMap((g) => g.rules),
|
|
6236
|
+
crawlDelayMs: Math.max(0, ...chosen.map((g) => g.delay)) * 1e3
|
|
6237
|
+
};
|
|
6238
|
+
}
|
|
6239
|
+
function isAllowed(robots, url) {
|
|
6240
|
+
let path;
|
|
6241
|
+
try {
|
|
6242
|
+
const parsed = new URL(url);
|
|
6243
|
+
path = `${parsed.pathname}${parsed.search}`;
|
|
6244
|
+
} catch {
|
|
6245
|
+
return true;
|
|
6246
|
+
}
|
|
6247
|
+
let best = null;
|
|
6248
|
+
for (const rule of robots.rules) {
|
|
6249
|
+
if (!toRegExp(rule.pattern).test(path)) continue;
|
|
6250
|
+
const length = rule.pattern.replace(/[*$]/g, "").length;
|
|
6251
|
+
if (!best || length > best.length || length === best.length && rule.allow) {
|
|
6252
|
+
best = { allow: rule.allow, length };
|
|
6253
|
+
}
|
|
4738
6254
|
}
|
|
4739
|
-
|
|
4740
|
-
|
|
6255
|
+
return best ? best.allow : true;
|
|
6256
|
+
}
|
|
6257
|
+
async function fetchRobots(ctx) {
|
|
4741
6258
|
try {
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
6259
|
+
const { ok, text } = await httpText(`${originOf(ctx.baseUrl)}/robots.txt`, {
|
|
6260
|
+
fetchImpl: ctx.fetchImpl,
|
|
6261
|
+
signal: ctx.signal,
|
|
6262
|
+
retries: 0,
|
|
6263
|
+
accept: "text/plain",
|
|
6264
|
+
maxBytes: 512e3
|
|
4747
6265
|
});
|
|
6266
|
+
if (!ok || !text) return ALLOW_ALL;
|
|
6267
|
+
return parseRobots(text);
|
|
6268
|
+
} catch {
|
|
6269
|
+
return ALLOW_ALL;
|
|
4748
6270
|
}
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
6271
|
+
}
|
|
6272
|
+
|
|
6273
|
+
// ../catalog/src/candidates.ts
|
|
6274
|
+
var DEFAULT_SCAN_BUDGET = {
|
|
6275
|
+
maxPreviewPages: 24,
|
|
6276
|
+
maxBytesPerPage: 15e5,
|
|
6277
|
+
maxTotalBytes: 4e7,
|
|
6278
|
+
budgetMs: 25e3,
|
|
6279
|
+
concurrency: 4,
|
|
6280
|
+
/**
|
|
6281
|
+
* Time the preview is owed even when discovery has already spent the lot.
|
|
6282
|
+
*
|
|
6283
|
+
* Discovery on a large store can run long, and when it overran the whole
|
|
6284
|
+
* budget the preview read zero pages and the store was reported as one we
|
|
6285
|
+
* could not open - from a site that was answering every request with a 200.
|
|
6286
|
+
* Being slow to list a catalog is not the same as being shut.
|
|
6287
|
+
*/
|
|
6288
|
+
previewFloorMs: 12e3
|
|
6289
|
+
};
|
|
6290
|
+
var COMMERCE_SIGNALS = /* @__PURE__ */ new Set([
|
|
6291
|
+
"products.json",
|
|
6292
|
+
"shopify-html",
|
|
6293
|
+
"wc-store-api",
|
|
6294
|
+
"woocommerce-html",
|
|
6295
|
+
"webflow-commerce"
|
|
6296
|
+
]);
|
|
6297
|
+
function wwwVariant(baseUrl) {
|
|
6298
|
+
try {
|
|
6299
|
+
const url = new URL(baseUrl);
|
|
6300
|
+
const labels = url.hostname.split(".");
|
|
6301
|
+
if (labels.length !== 2 || /^\d+$/.test(labels[labels.length - 1])) return null;
|
|
6302
|
+
url.hostname = `www.${url.hostname}`;
|
|
6303
|
+
return url.origin;
|
|
6304
|
+
} catch {
|
|
6305
|
+
return null;
|
|
6306
|
+
}
|
|
6307
|
+
}
|
|
6308
|
+
async function scanForCandidates(opts) {
|
|
6309
|
+
const first = await scanOnce(opts, originOf(normalizeStoreUrl(opts.url)));
|
|
6310
|
+
if (first.verdict === "found") return first;
|
|
6311
|
+
const alternate = wwwVariant(first.baseUrl);
|
|
6312
|
+
if (!alternate) return first;
|
|
6313
|
+
const second = await scanOnce(opts, alternate);
|
|
6314
|
+
return second.verdict === "found" ? second : first;
|
|
6315
|
+
}
|
|
6316
|
+
async function scanOnce(opts, baseUrl) {
|
|
6317
|
+
const started = Date.now();
|
|
6318
|
+
const budget = { ...DEFAULT_SCAN_BUDGET, ...opts.budget };
|
|
6319
|
+
const deadline = started + budget.budgetMs;
|
|
6320
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
6321
|
+
const ctx = {
|
|
6322
|
+
fetchImpl,
|
|
6323
|
+
baseUrl,
|
|
6324
|
+
signal: opts.signal,
|
|
6325
|
+
onProgress: opts.onProgress
|
|
6326
|
+
};
|
|
6327
|
+
const warnings = [];
|
|
6328
|
+
const stats = { pages: 0, bytes: 0 };
|
|
6329
|
+
const robots = await fetchRobots(ctx);
|
|
6330
|
+
if (robots.crawlDelayMs) warnings.push("This site asks readers to go slowly, so the preview is smaller");
|
|
6331
|
+
opts.onProgress?.({ stage: "discovering", message: "Looking for a shop" });
|
|
6332
|
+
const detection = await detectPlatform(ctx);
|
|
6333
|
+
const adapter = adapterFor(detection.platform);
|
|
6334
|
+
let discovered;
|
|
6335
|
+
try {
|
|
6336
|
+
discovered = await adapter.discover(ctx);
|
|
6337
|
+
} catch {
|
|
6338
|
+
return done("blocked", [], [], 0, "none", [...warnings, "The product list could not be read"]);
|
|
6339
|
+
}
|
|
6340
|
+
warnings.push(...discovered.warnings);
|
|
6341
|
+
const urls = discovered.productUrls.filter((u) => isAllowed(robots, u));
|
|
6342
|
+
const refused = discovered.productUrls.length - urls.length;
|
|
6343
|
+
if (refused > 0) warnings.push(`${refused} pages are disallowed by this site's robots.txt`);
|
|
6344
|
+
const fromApi = detection.platform !== "generic" && !discovered.hints?.includes("json-blocked");
|
|
6345
|
+
const countSource = urls.length === 0 ? "none" : fromApi ? "api" : "sitemap";
|
|
6346
|
+
if (urls.length === 0) {
|
|
6347
|
+
const commerce = detection.signals.some((sig) => COMMERCE_SIGNALS.has(sig));
|
|
6348
|
+
return done(commerce ? "likely" : "none", [], [], 0, countSource, warnings);
|
|
6349
|
+
}
|
|
6350
|
+
opts.onProgress?.({ stage: "discovering", discovered: urls.length, message: "Reading a few products" });
|
|
6351
|
+
const previewDeadline = Math.max(deadline, Date.now() + budget.previewFloorMs);
|
|
6352
|
+
const read = dedupeProducts(
|
|
6353
|
+
await fetchProductPages(ctx, urls, {
|
|
6354
|
+
want: budget.maxPreviewPages,
|
|
6355
|
+
stats,
|
|
6356
|
+
concurrency: budget.concurrency,
|
|
6357
|
+
maxBytes: budget.maxBytesPerPage,
|
|
6358
|
+
maxTotalBytes: budget.maxTotalBytes,
|
|
6359
|
+
delayMs: robots.crawlDelayMs,
|
|
6360
|
+
deadline: previewDeadline,
|
|
6361
|
+
onProduct: (fetched) => opts.onProgress?.({ stage: "fetching_products", fetched, discovered: urls.length })
|
|
6362
|
+
})
|
|
6363
|
+
);
|
|
6364
|
+
const preview = read.slice(0, budget.maxPreviewPages);
|
|
6365
|
+
const verdict = preview.length ? "found" : "blocked";
|
|
6366
|
+
return done(verdict, preview, urls, urls.length, countSource, warnings);
|
|
6367
|
+
function done(verdict2, candidates, candidateUrls, count, source, notes) {
|
|
6368
|
+
return {
|
|
6369
|
+
baseUrl,
|
|
6370
|
+
platform: detection?.platform ?? "unknown",
|
|
6371
|
+
signals: detection?.signals ?? [],
|
|
6372
|
+
verdict: verdict2,
|
|
6373
|
+
count,
|
|
6374
|
+
countSource: source,
|
|
6375
|
+
candidates,
|
|
6376
|
+
candidateUrls,
|
|
6377
|
+
truncated: count > candidates.length,
|
|
6378
|
+
warnings: notes,
|
|
6379
|
+
spent: { pages: stats.pages, bytes: stats.bytes, ms: Date.now() - started }
|
|
6380
|
+
};
|
|
4764
6381
|
}
|
|
4765
|
-
return { baseUrl: detection.baseUrl, detection, products, progress };
|
|
4766
6382
|
}
|
|
4767
6383
|
|
|
4768
6384
|
// src/catalogImport.ts
|
|
4769
6385
|
var running = /* @__PURE__ */ new Map();
|
|
4770
6386
|
function cancelCatalogImport(jobId) {
|
|
4771
|
-
const
|
|
4772
|
-
if (!
|
|
4773
|
-
ctrl.abort();
|
|
6387
|
+
const job = running.get(jobId);
|
|
6388
|
+
if (!job) return false;
|
|
6389
|
+
job.ctrl.abort();
|
|
4774
6390
|
return true;
|
|
4775
6391
|
}
|
|
4776
|
-
function
|
|
6392
|
+
async function settleCatalogImports(timeoutMs = 5e3) {
|
|
6393
|
+
if (!running.size) return;
|
|
6394
|
+
for (const { ctrl } of running.values()) ctrl.abort();
|
|
6395
|
+
await Promise.race([
|
|
6396
|
+
Promise.allSettled([...running.values()].map((j) => j.done)),
|
|
6397
|
+
new Promise((r) => setTimeout(r, timeoutMs))
|
|
6398
|
+
]);
|
|
6399
|
+
}
|
|
6400
|
+
function startCatalogImport(deps, brandId, url, opts = {}) {
|
|
4777
6401
|
const { core } = deps;
|
|
4778
6402
|
if (!core.store.getBrand(brandId)) throw Object.assign(new Error("brand not found"), { statusCode: 404 });
|
|
4779
6403
|
let normalized;
|
|
@@ -4784,57 +6408,185 @@ function startCatalogImport(deps, brandId, url) {
|
|
|
4784
6408
|
}
|
|
4785
6409
|
const job = core.catalog.createJob({ brandId, url: normalized });
|
|
4786
6410
|
const ctrl = new AbortController();
|
|
4787
|
-
|
|
4788
|
-
void runJob(deps, job.id, brandId, normalized, ctrl.signal).finally(() => {
|
|
6411
|
+
const done = runJob(deps, job.id, brandId, normalized, ctrl.signal, opts.only).finally(() => {
|
|
4789
6412
|
running.delete(job.id);
|
|
4790
6413
|
});
|
|
6414
|
+
void done.catch(() => {
|
|
6415
|
+
});
|
|
6416
|
+
running.set(job.id, { ctrl, done });
|
|
4791
6417
|
return { jobId: job.id };
|
|
4792
6418
|
}
|
|
4793
|
-
|
|
6419
|
+
function progressWriter(patch2) {
|
|
6420
|
+
let lastStage = "";
|
|
6421
|
+
let lastMessage = null;
|
|
6422
|
+
return (p) => {
|
|
6423
|
+
const reported = p.stage === "queued" ? "discovering" : p.stage;
|
|
6424
|
+
const stage = TERMINAL.has(reported) ? "fetching_products" : reported;
|
|
6425
|
+
const message = p.message ?? null;
|
|
6426
|
+
const notable = stage !== lastStage || message !== lastMessage || p.fetched % 10 === 0;
|
|
6427
|
+
if (!notable) return;
|
|
6428
|
+
lastStage = stage;
|
|
6429
|
+
lastMessage = message;
|
|
6430
|
+
patch2({
|
|
6431
|
+
stage,
|
|
6432
|
+
platform: p.platform,
|
|
6433
|
+
discovered: p.discovered,
|
|
6434
|
+
fetched: p.fetched,
|
|
6435
|
+
warnings: p.warnings,
|
|
6436
|
+
errors: p.errors,
|
|
6437
|
+
message
|
|
6438
|
+
});
|
|
6439
|
+
};
|
|
6440
|
+
}
|
|
6441
|
+
async function runJob(deps, jobId, brandId, url, signal, only) {
|
|
4794
6442
|
const { core, fetchImpl } = deps;
|
|
4795
6443
|
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
4796
6444
|
try {
|
|
4797
|
-
|
|
4798
|
-
const
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
6445
|
+
const tally = { fetched: 0, upserted: 0, imagesDone: 0, imagesTotal: 0, errors: [], seenKeys: [] };
|
|
6446
|
+
const ctx = { fetchImpl: fetchImpl ?? fetch, baseUrl: url, signal };
|
|
6447
|
+
let urls = [];
|
|
6448
|
+
let bulk = null;
|
|
6449
|
+
let baseUrl;
|
|
6450
|
+
let platform;
|
|
6451
|
+
let discovered;
|
|
6452
|
+
let warnings = [];
|
|
6453
|
+
const sweep = !only?.length;
|
|
6454
|
+
if (only?.length) {
|
|
6455
|
+
patch2({ stage: "fetching_products", message: `Importing ${only.length} products`, discovered: only.length });
|
|
6456
|
+
const detection = await detectPlatform(ctx);
|
|
6457
|
+
urls = only;
|
|
6458
|
+
baseUrl = url;
|
|
6459
|
+
platform = detection.platform;
|
|
6460
|
+
discovered = only.length;
|
|
6461
|
+
} else {
|
|
6462
|
+
patch2({ stage: "discovering", message: "Detecting store platform" });
|
|
6463
|
+
const found = await discoverCatalog({ url, fetchImpl, signal, onProgress: progressWriter(patch2) });
|
|
6464
|
+
baseUrl = found.baseUrl;
|
|
6465
|
+
platform = found.detection.platform;
|
|
6466
|
+
discovered = found.estimatedTotal;
|
|
6467
|
+
warnings = found.progress.warnings;
|
|
6468
|
+
if (found.empty || !found.estimatedTotal) {
|
|
6469
|
+
const source = core.catalog.upsertSource(brandId, found.baseUrl, platform);
|
|
6470
|
+
patch2({ sourceId: source.id, platform });
|
|
4803
6471
|
patch2({
|
|
4804
|
-
stage:
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
errors: p.errors,
|
|
4810
|
-
message: p.message ?? null
|
|
6472
|
+
stage: "no_catalog",
|
|
6473
|
+
errors: [],
|
|
6474
|
+
warnings: found.progress.warnings,
|
|
6475
|
+
message: "No shop found on this site",
|
|
6476
|
+
finished: true
|
|
4811
6477
|
});
|
|
6478
|
+
core.catalog.setSourceStatus(source.id, "empty", true);
|
|
6479
|
+
return;
|
|
4812
6480
|
}
|
|
4813
|
-
|
|
6481
|
+
if (!found.byPage) {
|
|
6482
|
+
const products = await found.fetchAll();
|
|
6483
|
+
if (signal.aborted) {
|
|
6484
|
+
patch2({ stage: "cancelled", errors: [], message: "Stopped before anything was saved", finished: true });
|
|
6485
|
+
return;
|
|
6486
|
+
}
|
|
6487
|
+
if (!products.length) {
|
|
6488
|
+
const source = core.catalog.upsertSource(brandId, baseUrl, platform);
|
|
6489
|
+
patch2({ sourceId: source.id, platform });
|
|
6490
|
+
patch2({
|
|
6491
|
+
stage: "failed",
|
|
6492
|
+
errors: found.progress.errors,
|
|
6493
|
+
warnings: found.progress.warnings,
|
|
6494
|
+
message: found.progress.errors[0]?.message ?? "No products imported",
|
|
6495
|
+
finished: true
|
|
6496
|
+
});
|
|
6497
|
+
core.catalog.setSourceStatus(source.id, "failed", true);
|
|
6498
|
+
return;
|
|
6499
|
+
}
|
|
6500
|
+
bulk = products;
|
|
6501
|
+
discovered = found.progress.discovered;
|
|
6502
|
+
warnings = found.progress.warnings;
|
|
6503
|
+
}
|
|
6504
|
+
if (!bulk) {
|
|
6505
|
+
urls = found.productUrls;
|
|
6506
|
+
patch2({ stage: "fetching_products", discovered, message: `Reading ${discovered.toLocaleString()} products` });
|
|
6507
|
+
}
|
|
6508
|
+
}
|
|
6509
|
+
const run2 = beginWrite(deps, jobId, brandId, baseUrl, platform, tally, discovered);
|
|
6510
|
+
const pictures = drainPictures(deps, jobId, brandId, tally, signal);
|
|
6511
|
+
let pictureErrors;
|
|
6512
|
+
const stats = { pages: 0, bytes: 0, refused: 0 };
|
|
6513
|
+
try {
|
|
6514
|
+
if (bulk) for (const p of bulk) run2.write(p);
|
|
6515
|
+
else
|
|
6516
|
+
await fetchProductPages(ctx, urls, {
|
|
6517
|
+
concurrency: IMPORT_CONCURRENCY,
|
|
6518
|
+
maxBytes: 15e5,
|
|
6519
|
+
onEach: run2.write,
|
|
6520
|
+
stats
|
|
6521
|
+
});
|
|
6522
|
+
if (!signal.aborted && tally.upserted) {
|
|
6523
|
+
patch2({
|
|
6524
|
+
stage: "processing_assets",
|
|
6525
|
+
message: `Downloading ${tally.imagesTotal.toLocaleString()} pictures`,
|
|
6526
|
+
upserted: tally.upserted,
|
|
6527
|
+
fetched: tally.fetched
|
|
6528
|
+
});
|
|
6529
|
+
}
|
|
6530
|
+
} finally {
|
|
6531
|
+
pictures.stop();
|
|
6532
|
+
pictureErrors = await pictures.done;
|
|
6533
|
+
}
|
|
4814
6534
|
if (signal.aborted) {
|
|
4815
|
-
patch2({
|
|
6535
|
+
patch2({
|
|
6536
|
+
stage: "cancelled",
|
|
6537
|
+
errors: [],
|
|
6538
|
+
message: tally.upserted ? `Stopped after saving ${tally.upserted.toLocaleString()} products` : "Stopped before anything was saved",
|
|
6539
|
+
finished: true
|
|
6540
|
+
});
|
|
6541
|
+
core.catalog.setSourceStatus(run2.sourceId, "partial", true);
|
|
4816
6542
|
return;
|
|
4817
6543
|
}
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
core.catalog.setSourceStatus(source.id, "importing");
|
|
4821
|
-
if (!result.products.length) {
|
|
4822
|
-
const stage = result.progress.stage === "failed" ? "failed" : "failed";
|
|
6544
|
+
if (!tally.upserted) {
|
|
6545
|
+
const message = only?.length ? "None of the chosen products could be read" : `Found pages on this ${platform} store but could not read a product from any of them. The store may be blocking automated readers.`;
|
|
4823
6546
|
patch2({
|
|
4824
|
-
stage,
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
message: result.progress.errors[0]?.message ?? "No products imported",
|
|
6547
|
+
stage: "failed",
|
|
6548
|
+
message,
|
|
6549
|
+
errors: [{ code: "no_products_fetched", message }],
|
|
4828
6550
|
finished: true
|
|
4829
6551
|
});
|
|
4830
|
-
core.catalog.setSourceStatus(
|
|
6552
|
+
core.catalog.setSourceStatus(run2.sourceId, "failed", true);
|
|
6553
|
+
return;
|
|
6554
|
+
}
|
|
6555
|
+
run2.finish({
|
|
6556
|
+
sweep,
|
|
6557
|
+
warnings,
|
|
6558
|
+
errors: pictureErrors,
|
|
6559
|
+
refused: stats.refused,
|
|
6560
|
+
// Every address was read. A bulk API hands the catalogue over whole, so
|
|
6561
|
+
// there is nothing to cover.
|
|
6562
|
+
covered: bulk ? true : stats.pages >= urls.length
|
|
6563
|
+
});
|
|
6564
|
+
} catch (err) {
|
|
6565
|
+
if (signal.aborted) {
|
|
6566
|
+
patch2({ stage: "cancelled", errors: [], message: "Stopped before anything was saved", finished: true });
|
|
4831
6567
|
return;
|
|
4832
6568
|
}
|
|
4833
|
-
patch2({
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
6569
|
+
patch2({
|
|
6570
|
+
stage: "failed",
|
|
6571
|
+
message: String(err?.message ?? err),
|
|
6572
|
+
errors: [{ code: "import_failed", message: String(err?.message ?? err) }],
|
|
6573
|
+
finished: true
|
|
6574
|
+
});
|
|
6575
|
+
}
|
|
6576
|
+
}
|
|
6577
|
+
var IMAGES_PER_PRODUCT = 3;
|
|
6578
|
+
var TERMINAL = /* @__PURE__ */ new Set(["completed", "partial", "no_catalog", "cancelled", "failed"]);
|
|
6579
|
+
var IMAGE_CONCURRENCY = 4;
|
|
6580
|
+
var sleep6 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
6581
|
+
function beginWrite(deps, jobId, brandId, baseUrl, platform, tally, discovered) {
|
|
6582
|
+
const { core } = deps;
|
|
6583
|
+
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6584
|
+
const source = core.catalog.upsertSource(brandId, baseUrl, platform);
|
|
6585
|
+
patch2({ sourceId: source.id, platform });
|
|
6586
|
+
core.catalog.setSourceStatus(source.id, "importing");
|
|
6587
|
+
return {
|
|
6588
|
+
sourceId: source.id,
|
|
6589
|
+
write(p) {
|
|
4838
6590
|
core.catalog.upsertProduct({
|
|
4839
6591
|
sourceId: source.id,
|
|
4840
6592
|
brandId,
|
|
@@ -4851,7 +6603,9 @@ async function runJob(deps, jobId, brandId, url, signal) {
|
|
|
4851
6603
|
compareAtPrice: p.compareAtPrice,
|
|
4852
6604
|
currency: p.currency,
|
|
4853
6605
|
available: p.available,
|
|
4854
|
-
raw
|
|
6606
|
+
// `raw` is the whole crawled payload, 14.2 KB a product on gymshark
|
|
6607
|
+
// and 32 MB across its catalog, written to sqlite and read by nothing.
|
|
6608
|
+
raw: null,
|
|
4855
6609
|
variants: p.variants,
|
|
4856
6610
|
images: (p.images ?? []).map((img) => ({
|
|
4857
6611
|
sourceUrl: img.url,
|
|
@@ -4860,93 +6614,111 @@ async function runJob(deps, jobId, brandId, url, signal) {
|
|
|
4860
6614
|
height: img.height,
|
|
4861
6615
|
alt: img.alt
|
|
4862
6616
|
})),
|
|
4863
|
-
collections: (p.collections ?? []).map((c) => ({
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
6617
|
+
collections: (p.collections ?? []).map((c) => ({ externalKey: c, title: c }))
|
|
6618
|
+
});
|
|
6619
|
+
tally.seenKeys.push(p.externalKey);
|
|
6620
|
+
tally.upserted++;
|
|
6621
|
+
tally.fetched++;
|
|
6622
|
+
patch2({ upserted: tally.upserted, fetched: tally.fetched });
|
|
6623
|
+
},
|
|
6624
|
+
finish({
|
|
6625
|
+
sweep,
|
|
6626
|
+
warnings,
|
|
6627
|
+
errors,
|
|
6628
|
+
refused = 0,
|
|
6629
|
+
covered = true
|
|
6630
|
+
}) {
|
|
6631
|
+
if (sweep) core.catalog.markMissingUnavailable(source.id, tally.seenKeys);
|
|
6632
|
+
tally.errors = errors;
|
|
6633
|
+
const partial = !covered || refused > 0;
|
|
6634
|
+
const shut = refused > 0 && refused >= Math.max(20, tally.upserted * 0.25);
|
|
6635
|
+
const message = shut ? `The store stopped answering after ${tally.upserted.toLocaleString()} of ${discovered.toLocaleString()} products. Try again later.` : partial ? `Imported ${tally.upserted.toLocaleString()} products with ${(errors.length + refused).toLocaleString()} issue${errors.length + refused === 1 ? "" : "s"}` : `Imported ${tally.upserted.toLocaleString()} products`;
|
|
6636
|
+
patch2({
|
|
6637
|
+
stage: partial ? "partial" : "completed",
|
|
6638
|
+
upserted: tally.upserted,
|
|
6639
|
+
fetched: tally.fetched,
|
|
6640
|
+
imagesDone: tally.imagesDone,
|
|
6641
|
+
imagesTotal: tally.imagesTotal,
|
|
6642
|
+
errors: shut ? [...errors, { code: "store_refused", message: `${refused.toLocaleString()} pages were refused` }] : errors,
|
|
6643
|
+
warnings,
|
|
6644
|
+
message,
|
|
6645
|
+
finished: true
|
|
4867
6646
|
});
|
|
4868
|
-
|
|
4869
|
-
upserted++;
|
|
4870
|
-
if (upserted % 10 === 0) patch2({ upserted, fetched: result.products.length });
|
|
6647
|
+
core.catalog.setSourceStatus(source.id, partial ? "partial" : "ready", true);
|
|
4871
6648
|
}
|
|
4872
|
-
|
|
4873
|
-
|
|
4874
|
-
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
|
|
4878
|
-
|
|
4879
|
-
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
4895
|
-
if (!buf.length) {
|
|
4896
|
-
errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
|
|
4897
|
-
return;
|
|
4898
|
-
}
|
|
4899
|
-
const png = await sharp20(buf).rotate().png().toBuffer();
|
|
4900
|
-
const meta = await sharp20(png).metadata();
|
|
4901
|
-
const hash = core.images.save(png);
|
|
4902
|
-
core.catalog.setImageAsset(img.productId, img.sourceUrl, `asset:${hash}`, {
|
|
4903
|
-
width: meta.width,
|
|
4904
|
-
height: meta.height
|
|
4905
|
-
});
|
|
4906
|
-
} catch (err) {
|
|
6649
|
+
};
|
|
6650
|
+
}
|
|
6651
|
+
var IMPORT_CONCURRENCY = 4;
|
|
6652
|
+
var PICTURE_ROUND = 60;
|
|
6653
|
+
function drainPictures(deps, jobId, brandId, tally, signal, imagesPerProduct = IMAGES_PER_PRODUCT) {
|
|
6654
|
+
const { core, fetchImpl } = deps;
|
|
6655
|
+
const patch2 = (p) => core.catalog.updateJob(jobId, p);
|
|
6656
|
+
const errors = [...core.catalog.getJob(jobId)?.errors ?? []];
|
|
6657
|
+
let stopped = false;
|
|
6658
|
+
const done = (async () => {
|
|
6659
|
+
while (!signal.aborted) {
|
|
6660
|
+
const round = core.catalog.listImagesNeedingAssets(brandId, PICTURE_ROUND).filter((img) => img.position < imagesPerProduct);
|
|
6661
|
+
if (!round.length) {
|
|
6662
|
+
if (stopped) break;
|
|
6663
|
+
await sleep6(150);
|
|
6664
|
+
continue;
|
|
6665
|
+
}
|
|
6666
|
+
tally.imagesTotal += round.length;
|
|
6667
|
+
await mapPool(
|
|
6668
|
+
round,
|
|
6669
|
+
IMAGE_CONCURRENCY,
|
|
6670
|
+
async (img) => {
|
|
4907
6671
|
if (signal.aborted) return;
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
6672
|
+
try {
|
|
6673
|
+
const res = await httpGet(img.sourceUrl, { fetchImpl, signal, timeoutMs: 4e4, retries: 2 });
|
|
6674
|
+
if (!res.ok) {
|
|
6675
|
+
errors.push({ code: "image_http", message: `HTTP ${res.status}`, url: img.sourceUrl });
|
|
6676
|
+
return;
|
|
6677
|
+
}
|
|
6678
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
6679
|
+
if (!buf.length) {
|
|
6680
|
+
errors.push({ code: "image_empty", message: "Empty image", url: img.sourceUrl });
|
|
6681
|
+
return;
|
|
6682
|
+
}
|
|
6683
|
+
const probe = await sharp21(buf).metadata();
|
|
6684
|
+
const turned = (probe.orientation ?? 1) > 1;
|
|
6685
|
+
const keep = turned ? await sharp21(buf).rotate().toBuffer() : buf;
|
|
6686
|
+
const meta = turned ? await sharp21(keep).metadata() : probe;
|
|
6687
|
+
const hash = core.images.save(keep, meta.format ?? probe.format ?? "png");
|
|
6688
|
+
core.catalog.setImageAsset(img.productId, img.sourceUrl, `asset:${hash}`, {
|
|
6689
|
+
width: meta.width,
|
|
6690
|
+
height: meta.height
|
|
6691
|
+
});
|
|
6692
|
+
} catch (err) {
|
|
6693
|
+
if (signal.aborted) return;
|
|
6694
|
+
errors.push({
|
|
6695
|
+
code: "image_failed",
|
|
6696
|
+
message: String(err?.message ?? err),
|
|
6697
|
+
url: img.sourceUrl,
|
|
6698
|
+
retryable: true
|
|
6699
|
+
});
|
|
6700
|
+
} finally {
|
|
6701
|
+
tally.imagesDone++;
|
|
6702
|
+
patch2({ imagesDone: tally.imagesDone, imagesTotal: tally.imagesTotal, errors });
|
|
4918
6703
|
}
|
|
4919
|
-
}
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
6704
|
+
},
|
|
6705
|
+
signal
|
|
6706
|
+
);
|
|
6707
|
+
const stuck = core.catalog.listImagesNeedingAssets(brandId, PICTURE_ROUND).filter((img) => img.position < imagesPerProduct);
|
|
6708
|
+
if (stuck.length && stuck[0]?.id === round[0]?.id) {
|
|
6709
|
+
errors.push({ code: "images_stalled", message: "Some pictures could not be downloaded" });
|
|
6710
|
+
break;
|
|
6711
|
+
}
|
|
4927
6712
|
}
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
warnings: result.progress.warnings,
|
|
4938
|
-
message: partial ? `Imported ${upserted} products with ${errors.length} issue${errors.length === 1 ? "" : "s"}` : `Imported ${upserted} products`,
|
|
4939
|
-
finished: true
|
|
4940
|
-
});
|
|
4941
|
-
core.catalog.setSourceStatus(source.id, partial ? "partial" : "ready", true);
|
|
4942
|
-
} catch (err) {
|
|
4943
|
-
patch2({
|
|
4944
|
-
stage: "failed",
|
|
4945
|
-
message: String(err?.message ?? err),
|
|
4946
|
-
errors: [{ code: "import_failed", message: String(err?.message ?? err) }],
|
|
4947
|
-
finished: true
|
|
4948
|
-
});
|
|
4949
|
-
}
|
|
6713
|
+
patch2({ imagesDone: tally.imagesDone, imagesTotal: tally.imagesTotal, errors });
|
|
6714
|
+
return errors;
|
|
6715
|
+
})();
|
|
6716
|
+
return {
|
|
6717
|
+
stop() {
|
|
6718
|
+
stopped = true;
|
|
6719
|
+
},
|
|
6720
|
+
done
|
|
6721
|
+
};
|
|
4950
6722
|
}
|
|
4951
6723
|
function resolveLibraryProduct(core, brandId, productId) {
|
|
4952
6724
|
const brand = core.store.getBrand(brandId);
|
|
@@ -5374,7 +7146,7 @@ async function generateStudioSet(deps, job, who, sourcePaths, signal) {
|
|
|
5374
7146
|
return { hashes: kept.map((f) => byAngle.get(f.angle)), angles: kept.map((f) => f.angle) };
|
|
5375
7147
|
}
|
|
5376
7148
|
async function edgeBarGeometry(buf) {
|
|
5377
|
-
const { data, info } = await
|
|
7149
|
+
const { data, info } = await sharp21(buf).greyscale().raw().toBuffer({ resolveWithObject: true });
|
|
5378
7150
|
const W = info.width;
|
|
5379
7151
|
const H = info.height;
|
|
5380
7152
|
const scan = (len, cross, at) => {
|
|
@@ -5428,7 +7200,7 @@ async function trimEdgeBars(core, hash) {
|
|
|
5428
7200
|
const width = g.right - g.left + 1;
|
|
5429
7201
|
const height = g.bottom - g.top + 1;
|
|
5430
7202
|
if (width < g.W * 0.6 || height < g.H * 0.6) return hash;
|
|
5431
|
-
const png = await
|
|
7203
|
+
const png = await sharp21(buf).extract({ left: g.left, top: g.top, width, height }).png().toBuffer();
|
|
5432
7204
|
return core.images.save(png);
|
|
5433
7205
|
} catch {
|
|
5434
7206
|
return hash;
|
|
@@ -5499,7 +7271,7 @@ async function identityCrop(core, hash) {
|
|
|
5499
7271
|
if (!out) return void 0;
|
|
5500
7272
|
try {
|
|
5501
7273
|
const height = Math.min(IDENTITY_TARGET_HEIGHT, Math.round(nativeHeight * IDENTITY_MAX_UPSCALE)) || IDENTITY_TARGET_HEIGHT;
|
|
5502
|
-
const png = await
|
|
7274
|
+
const png = await sharp21(core.images.read(out)).resize({ height, fit: "inside", kernel: "lanczos3", withoutEnlargement: false }).png().toBuffer();
|
|
5503
7275
|
const scaled = core.images.save(png);
|
|
5504
7276
|
identityCrops.set(hash, scaled);
|
|
5505
7277
|
return scaled;
|
|
@@ -5528,12 +7300,12 @@ async function brandJsonWithIdentityCrops(core, json, characterIds) {
|
|
|
5528
7300
|
return changed ? { ...json, characters } : json;
|
|
5529
7301
|
}
|
|
5530
7302
|
async function figureBox(buf) {
|
|
5531
|
-
const meta = await
|
|
7303
|
+
const meta = await sharp21(buf).metadata();
|
|
5532
7304
|
const W = meta.width ?? 0;
|
|
5533
7305
|
const H = meta.height ?? 0;
|
|
5534
7306
|
if (!W || !H) return null;
|
|
5535
7307
|
for (const threshold of FIGURE_TRIM_THRESHOLDS) {
|
|
5536
|
-
const { info } = await
|
|
7308
|
+
const { info } = await sharp21(buf).trim({ threshold }).toBuffer({ resolveWithObject: true });
|
|
5537
7309
|
const left = Math.abs(info.trimOffsetLeft ?? 0);
|
|
5538
7310
|
const top = Math.abs(info.trimOffsetTop ?? 0);
|
|
5539
7311
|
const width = info.width ?? 0;
|
|
@@ -5566,13 +7338,13 @@ async function smartCover(core, hash, box) {
|
|
|
5566
7338
|
if (!hash || !core.images.has(hash)) return void 0;
|
|
5567
7339
|
try {
|
|
5568
7340
|
const buf = core.images.read(hash);
|
|
5569
|
-
const meta = await
|
|
7341
|
+
const meta = await sharp21(buf).metadata();
|
|
5570
7342
|
const w = meta.width ?? 0;
|
|
5571
7343
|
const h = meta.height ?? 0;
|
|
5572
7344
|
if (!w || !h) return void 0;
|
|
5573
7345
|
const raw = box(w, h);
|
|
5574
7346
|
const target = { width: Math.max(1, raw.width), height: Math.max(1, raw.height) };
|
|
5575
|
-
const png = await
|
|
7347
|
+
const png = await sharp21(buf).resize(target.width, target.height, { fit: "cover", position: "attention" }).png().toBuffer();
|
|
5576
7348
|
return core.images.save(png);
|
|
5577
7349
|
} catch {
|
|
5578
7350
|
return void 0;
|
|
@@ -5581,11 +7353,11 @@ async function smartCover(core, hash, box) {
|
|
|
5581
7353
|
async function crop(core, hash, region, cap2) {
|
|
5582
7354
|
if (!hash || !core.images.has(hash)) return void 0;
|
|
5583
7355
|
try {
|
|
5584
|
-
const meta = await
|
|
7356
|
+
const meta = await sharp21(core.images.read(hash)).metadata();
|
|
5585
7357
|
const w = meta.width ?? 0;
|
|
5586
7358
|
const h = meta.height ?? 0;
|
|
5587
7359
|
if (!w || !h) return void 0;
|
|
5588
|
-
let pipeline =
|
|
7360
|
+
let pipeline = sharp21(core.images.read(hash)).extract(region(w, h));
|
|
5589
7361
|
if (cap2) pipeline = pipeline.resize(cap2, cap2, { fit: "inside", withoutEnlargement: true });
|
|
5590
7362
|
const png = await pipeline.png().toBuffer();
|
|
5591
7363
|
return core.images.save(png);
|
|
@@ -5932,7 +7704,7 @@ var fromLab = (l, a, bb) => {
|
|
|
5932
7704
|
return [clamp(R), clamp(G), clamp(B)];
|
|
5933
7705
|
};
|
|
5934
7706
|
var rawAt = async (png, edge) => {
|
|
5935
|
-
let img =
|
|
7707
|
+
let img = sharp21(png);
|
|
5936
7708
|
if (edge) img = img.resize(edge, edge, { fit: "fill" });
|
|
5937
7709
|
const { data, info } = await img.removeAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
5938
7710
|
return { data, width: info.width, height: info.height };
|
|
@@ -5995,7 +7767,7 @@ async function gradeComposite(originalPng, modelInputPng, modelOutputPng) {
|
|
|
5995
7767
|
if (residual > GRADE_GATE_MEAN_DELTA) return null;
|
|
5996
7768
|
const full = await rawAt(originalPng);
|
|
5997
7769
|
applyAffine(full, T);
|
|
5998
|
-
const image = await
|
|
7770
|
+
const image = await sharp21(full.data, {
|
|
5999
7771
|
raw: { width: full.width, height: full.height, channels: 3 }
|
|
6000
7772
|
}).png().toBuffer();
|
|
6001
7773
|
return { image, residual };
|
|
@@ -6169,7 +7941,7 @@ function fitExpandToBudget(plan, source, pixelBudget) {
|
|
|
6169
7941
|
}
|
|
6170
7942
|
async function attentionCropOrigin(srcBuf, source, plan) {
|
|
6171
7943
|
try {
|
|
6172
|
-
const { info } = await
|
|
7944
|
+
const { info } = await sharp21(srcBuf).resize(plan.width, plan.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
|
|
6173
7945
|
const attnLeft = typeof info.cropOffsetLeft === "number" ? Math.abs(info.cropOffsetLeft) : plan.left;
|
|
6174
7946
|
const attnTop = typeof info.cropOffsetTop === "number" ? Math.abs(info.cropOffsetTop) : plan.top;
|
|
6175
7947
|
const left = Math.round((attnLeft + plan.left) / 2);
|
|
@@ -6322,23 +8094,23 @@ function relax(grid, seam, fixedSweeps) {
|
|
|
6322
8094
|
|
|
6323
8095
|
// src/expand.ts
|
|
6324
8096
|
async function expandCanvas(source, plan) {
|
|
6325
|
-
const bed = await
|
|
6326
|
-
return
|
|
8097
|
+
const bed = await sharp21(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
|
|
8098
|
+
return sharp21(bed).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
|
|
6327
8099
|
}
|
|
6328
8100
|
async function compositeExpand(engineImage, source, plan) {
|
|
6329
|
-
const meta = await
|
|
8101
|
+
const meta = await sharp21(engineImage).metadata();
|
|
6330
8102
|
const want = plan.width / plan.height;
|
|
6331
8103
|
const got = meta.width && meta.height ? meta.width / meta.height : 0;
|
|
6332
8104
|
const sameOrientation = got > 0 && got >= 1 === want >= 1;
|
|
6333
8105
|
const aligned = sameOrientation;
|
|
6334
8106
|
const exact = meta.width === plan.width && meta.height === plan.height;
|
|
6335
|
-
const surround = aligned ? exact ? engineImage : await
|
|
8107
|
+
const surround = aligned ? exact ? engineImage : await sharp21(engineImage).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).toBuffer() : await expandCanvasBedOnly(source, plan);
|
|
6336
8108
|
const matched = aligned ? await matchMarginsToSeam(surround, source, plan) : surround;
|
|
6337
|
-
const image = await
|
|
8109
|
+
const image = await sharp21(matched).composite([{ input: source, left: plan.left, top: plan.top }]).png().toBuffer();
|
|
6338
8110
|
return { image, aligned };
|
|
6339
8111
|
}
|
|
6340
8112
|
async function matchMarginsToSeam(surround, source, plan) {
|
|
6341
|
-
const src = await
|
|
8113
|
+
const src = await sharp21(source).metadata();
|
|
6342
8114
|
if (!src.width || !src.height) return surround;
|
|
6343
8115
|
const SW = src.width;
|
|
6344
8116
|
const SH = src.height;
|
|
@@ -6385,8 +8157,8 @@ var MAX_CORRECTION = 60;
|
|
|
6385
8157
|
async function reconcile(surround, source, side, axis) {
|
|
6386
8158
|
const { margin } = side;
|
|
6387
8159
|
if (margin.width < 1 || margin.height < 1) return surround;
|
|
6388
|
-
const marginRaw = await
|
|
6389
|
-
const edgeRaw = await
|
|
8160
|
+
const marginRaw = await sharp21(surround).extract(margin).removeAlpha().raw().toBuffer();
|
|
8161
|
+
const edgeRaw = await sharp21(source).extract(side.srcEdge).removeAlpha().raw().toBuffer();
|
|
6390
8162
|
const W = margin.width;
|
|
6391
8163
|
const H = margin.height;
|
|
6392
8164
|
const along = axis === "width" ? H : W;
|
|
@@ -6440,11 +8212,11 @@ async function reconcile(surround, source, side, axis) {
|
|
|
6440
8212
|
}
|
|
6441
8213
|
}
|
|
6442
8214
|
}
|
|
6443
|
-
const patch2 = await
|
|
6444
|
-
return
|
|
8215
|
+
const patch2 = await sharp21(corrected, { raw: { width: W, height: H, channels: 3 } }).png().toBuffer();
|
|
8216
|
+
return sharp21(surround).composite([{ input: patch2, left: margin.left, top: margin.top }]).toBuffer();
|
|
6445
8217
|
}
|
|
6446
8218
|
async function expandCanvasBedOnly(source, plan) {
|
|
6447
|
-
return
|
|
8219
|
+
return sharp21(source).resize(plan.width, plan.height, { fit: "cover", position: "centre" }).blur(Math.max(8, Math.round(Math.max(plan.width, plan.height) / 40))).toBuffer();
|
|
6448
8220
|
}
|
|
6449
8221
|
function medianOf(rgb, channel, from, to) {
|
|
6450
8222
|
const n = to - from;
|
|
@@ -6455,17 +8227,17 @@ function medianOf(rgb, channel, from, to) {
|
|
|
6455
8227
|
return n % 2 ? values[(n - 1) / 2] : (values[n / 2 - 1] + values[n / 2]) / 2;
|
|
6456
8228
|
}
|
|
6457
8229
|
async function reframeExpand(engineImage, plan) {
|
|
6458
|
-
const meta = await
|
|
8230
|
+
const meta = await sharp21(engineImage).metadata();
|
|
6459
8231
|
if (!(meta.width && meta.height)) return null;
|
|
6460
8232
|
const want = plan.width / plan.height;
|
|
6461
8233
|
const got = meta.width / meta.height;
|
|
6462
8234
|
if (got >= 1 !== want >= 1) return null;
|
|
6463
8235
|
if (meta.width === plan.width && meta.height === plan.height) return engineImage;
|
|
6464
8236
|
const straight = Math.abs(got - want) / want <= 0.02;
|
|
6465
|
-
return
|
|
8237
|
+
return sharp21(engineImage).resize(plan.width, plan.height, { fit: straight ? "fill" : "cover", position: "centre" }).png().toBuffer();
|
|
6466
8238
|
}
|
|
6467
8239
|
async function seamScore(image, plan, source) {
|
|
6468
|
-
const { data, info } = await
|
|
8240
|
+
const { data, info } = await sharp21(image).removeAlpha().greyscale().raw().toBuffer({ resolveWithObject: true });
|
|
6469
8241
|
const W = info.width;
|
|
6470
8242
|
const H = info.height;
|
|
6471
8243
|
const horizontal = plan.axis === "width";
|
|
@@ -6498,7 +8270,7 @@ var SEAM_VISIBLE = 2.2;
|
|
|
6498
8270
|
var OFFSET = 4;
|
|
6499
8271
|
var RESIDUAL_VISIBLE = 15;
|
|
6500
8272
|
async function seamResidual(image, plan, source) {
|
|
6501
|
-
const { data, info } = await
|
|
8273
|
+
const { data, info } = await sharp21(image).removeAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
6502
8274
|
const W = info.width;
|
|
6503
8275
|
const H = info.height;
|
|
6504
8276
|
const ch = info.channels;
|
|
@@ -6533,7 +8305,7 @@ var MAX_SHARE = 0.8;
|
|
|
6533
8305
|
async function subjectFraction(src, source, axis) {
|
|
6534
8306
|
try {
|
|
6535
8307
|
const window = axis === "width" ? { width: Math.max(8, Math.round(source.width * 0.5)), height: source.height } : { width: source.width, height: Math.max(8, Math.round(source.height * 0.5)) };
|
|
6536
|
-
const { info } = await
|
|
8308
|
+
const { info } = await sharp21(src).resize(window.width, window.height, { fit: "cover", position: "attention" }).toBuffer({ resolveWithObject: true });
|
|
6537
8309
|
const offset = axis === "width" ? Math.abs(typeof info.cropOffsetLeft === "number" ? info.cropOffsetLeft : 0) : Math.abs(typeof info.cropOffsetTop === "number" ? info.cropOffsetTop : 0);
|
|
6538
8310
|
const span = axis === "width" ? source.width : source.height;
|
|
6539
8311
|
const extent = axis === "width" ? window.width : window.height;
|
|
@@ -6556,14 +8328,14 @@ function placeExpand(plan, source, fraction) {
|
|
|
6556
8328
|
}
|
|
6557
8329
|
var NEUTRAL = { r: 128, g: 128, b: 128 };
|
|
6558
8330
|
async function conditioningCanvas(source, plan, fill = "edge") {
|
|
6559
|
-
const meta = await
|
|
8331
|
+
const meta = await sharp21(source).metadata();
|
|
6560
8332
|
const sw = meta.width ?? 0;
|
|
6561
8333
|
const sh = meta.height ?? 0;
|
|
6562
8334
|
if (!(sw > 0 && sh > 0)) throw new Error("conditioningCanvas: source has no dimensions");
|
|
6563
8335
|
const layers = [];
|
|
6564
8336
|
if (fill === "edge") layers.push(...await edgeMargins(source, plan, { width: sw, height: sh }));
|
|
6565
8337
|
layers.push({ input: source, left: plan.left, top: plan.top });
|
|
6566
|
-
const canvas =
|
|
8338
|
+
const canvas = sharp21({
|
|
6567
8339
|
create: {
|
|
6568
8340
|
width: plan.width,
|
|
6569
8341
|
height: plan.height,
|
|
@@ -6575,7 +8347,7 @@ async function conditioningCanvas(source, plan, fill = "edge") {
|
|
|
6575
8347
|
}
|
|
6576
8348
|
async function edgeMargins(source, plan, size) {
|
|
6577
8349
|
const out = [];
|
|
6578
|
-
const strip = async (extract, width, height) =>
|
|
8350
|
+
const strip = async (extract, width, height) => sharp21(source).extract(extract).resize(width, height, { fit: "fill" }).png().toBuffer();
|
|
6579
8351
|
if (plan.axis === "width") {
|
|
6580
8352
|
const before = plan.left;
|
|
6581
8353
|
const after = plan.width - plan.left - size.width;
|
|
@@ -6666,12 +8438,12 @@ async function resolveOutpaintRoute(all, shot) {
|
|
|
6666
8438
|
return { engine: shot, method: "reframe", crossed: false };
|
|
6667
8439
|
}
|
|
6668
8440
|
async function driftDiff(a, b) {
|
|
6669
|
-
const metaA = await
|
|
6670
|
-
const metaB = await
|
|
8441
|
+
const metaA = await sharp21(a).metadata();
|
|
8442
|
+
const metaB = await sharp21(b).metadata();
|
|
6671
8443
|
const width = Math.min(metaA.width ?? 1, metaB.width ?? 1, 1024);
|
|
6672
8444
|
const height = Math.min(metaA.height ?? 1, metaB.height ?? 1, 1024);
|
|
6673
8445
|
const [rawA, rawB] = await Promise.all(
|
|
6674
|
-
[a, b].map((buf) =>
|
|
8446
|
+
[a, b].map((buf) => sharp21(buf).resize(width, height, { fit: "cover" }).ensureAlpha().raw().toBuffer())
|
|
6675
8447
|
);
|
|
6676
8448
|
const out = new PNG({ width, height });
|
|
6677
8449
|
const changed = pixelmatch(rawA, rawB, out.data, width, height, { threshold: 0.1, diffColor: [255, 64, 64] });
|
|
@@ -6683,11 +8455,11 @@ async function driftDiff(a, b) {
|
|
|
6683
8455
|
};
|
|
6684
8456
|
}
|
|
6685
8457
|
async function changeMask(a, b, cap2 = 1024) {
|
|
6686
|
-
const metaA = await
|
|
8458
|
+
const metaA = await sharp21(a).metadata();
|
|
6687
8459
|
const width = Math.min(metaA.width ?? 1, cap2);
|
|
6688
8460
|
const height = Math.min(metaA.height ?? 1, cap2);
|
|
6689
8461
|
const [rawA, rawB] = await Promise.all(
|
|
6690
|
-
[a, b].map((buf) =>
|
|
8462
|
+
[a, b].map((buf) => sharp21(buf).resize(width, height, { fit: "fill" }).ensureAlpha().raw().toBuffer())
|
|
6691
8463
|
);
|
|
6692
8464
|
const out = new PNG({ width, height });
|
|
6693
8465
|
pixelmatch(rawA, rawB, out.data, width, height, { threshold: 0.1, includeAA: true, diffMask: true });
|
|
@@ -6736,8 +8508,8 @@ function dilationFor(longEdge) {
|
|
|
6736
8508
|
// src/localEdit.ts
|
|
6737
8509
|
async function preserveOutsideChange(source, edited) {
|
|
6738
8510
|
try {
|
|
6739
|
-
const srcMeta = await
|
|
6740
|
-
const outMeta = await
|
|
8511
|
+
const srcMeta = await sharp21(source).metadata();
|
|
8512
|
+
const outMeta = await sharp21(edited).metadata();
|
|
6741
8513
|
if (!srcMeta.width || !srcMeta.height || !outMeta.width || !outMeta.height)
|
|
6742
8514
|
return { image: edited, outcome: "error", changed: 0 };
|
|
6743
8515
|
const sameShape = Math.abs(outMeta.width / outMeta.height - srcMeta.width / srcMeta.height) / (srcMeta.width / srcMeta.height) <= 0.01;
|
|
@@ -6747,15 +8519,15 @@ async function preserveOutsideChange(source, edited) {
|
|
|
6747
8519
|
if (outcome !== "composited") return { image: edited, outcome, changed: shape.changed };
|
|
6748
8520
|
const r = dilationFor(Math.max(shape.width, shape.height));
|
|
6749
8521
|
const rawShape = { raw: { width: shape.width, height: shape.height, channels: 1 } };
|
|
6750
|
-
const spread = await
|
|
6751
|
-
const dilated = await
|
|
6752
|
-
const feathered = await
|
|
6753
|
-
const grown = await
|
|
6754
|
-
const editedRgb = await
|
|
6755
|
-
const masked = await
|
|
8522
|
+
const spread = await sharp21(shape.mask, rawShape).blur(r).toColourspace("b-w").raw().toBuffer();
|
|
8523
|
+
const dilated = await sharp21(spread, rawShape).threshold(8).toColourspace("b-w").raw().toBuffer();
|
|
8524
|
+
const feathered = await sharp21(dilated, rawShape).blur(Math.max(2, r / 3)).toColourspace("b-w").raw().toBuffer();
|
|
8525
|
+
const grown = await sharp21(feathered, rawShape).resize(srcMeta.width, srcMeta.height, { fit: "fill", kernel: "cubic" }).toColourspace("b-w").raw().toBuffer();
|
|
8526
|
+
const editedRgb = await sharp21(edited).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).removeAlpha().raw().toBuffer();
|
|
8527
|
+
const masked = await sharp21(editedRgb, {
|
|
6756
8528
|
raw: { width: srcMeta.width, height: srcMeta.height, channels: 3 }
|
|
6757
8529
|
}).joinChannel(grown, { raw: { width: srcMeta.width, height: srcMeta.height, channels: 1 } }).png().toBuffer();
|
|
6758
|
-
const image = await
|
|
8530
|
+
const image = await sharp21(source).removeAlpha().composite([{ input: masked }]).png().toBuffer();
|
|
6759
8531
|
return { image, outcome: "composited", changed: shape.changed };
|
|
6760
8532
|
} catch {
|
|
6761
8533
|
return { image: edited, outcome: "error", changed: 0 };
|
|
@@ -6793,7 +8565,7 @@ function registerLogoRoutes(app, deps) {
|
|
|
6793
8565
|
const v = validateBrand(json);
|
|
6794
8566
|
if (!v.valid) return reply.status(400).send({ error: "brand became invalid", details: v.errors });
|
|
6795
8567
|
const row = core.store.updateBrand(brand.id, json);
|
|
6796
|
-
const meta = await
|
|
8568
|
+
const meta = await sharp21(core.images.read(part.hash)).metadata().catch(() => null);
|
|
6797
8569
|
const logoEdge = meta ? Math.max(meta.width ?? 0, meta.height ?? 0) || null : null;
|
|
6798
8570
|
return { ...row, logoHash: part.hash, logoEdge };
|
|
6799
8571
|
});
|
|
@@ -6841,6 +8613,38 @@ function registerLogoRoutes(app, deps) {
|
|
|
6841
8613
|
return core.store.updateBrand(brand.id, json);
|
|
6842
8614
|
});
|
|
6843
8615
|
}
|
|
8616
|
+
var scans = /* @__PURE__ */ new Map();
|
|
8617
|
+
var controllers = /* @__PURE__ */ new Map();
|
|
8618
|
+
var KEEP = 32;
|
|
8619
|
+
function remember(state) {
|
|
8620
|
+
scans.set(state.id, state);
|
|
8621
|
+
while (scans.size > KEEP) {
|
|
8622
|
+
const oldest = scans.keys().next().value;
|
|
8623
|
+
if (oldest === void 0) break;
|
|
8624
|
+
scans.delete(oldest);
|
|
8625
|
+
controllers.delete(oldest);
|
|
8626
|
+
}
|
|
8627
|
+
}
|
|
8628
|
+
function getScan(scanId) {
|
|
8629
|
+
return scans.get(scanId);
|
|
8630
|
+
}
|
|
8631
|
+
function startCatalogScan(deps, brandId, url) {
|
|
8632
|
+
const { core, fetchImpl } = deps;
|
|
8633
|
+
if (!core.store.getBrand(brandId)) throw Object.assign(new Error("brand not found"), { statusCode: 404 });
|
|
8634
|
+
const id = randomUUID();
|
|
8635
|
+
const ctrl = new AbortController();
|
|
8636
|
+
const state = { id, brandId, url, status: "running", startedAt: Date.now() };
|
|
8637
|
+
remember(state);
|
|
8638
|
+
controllers.set(id, ctrl);
|
|
8639
|
+
void scanForCandidates({ url, fetchImpl, signal: ctrl.signal }).then((result) => {
|
|
8640
|
+
state.result = result;
|
|
8641
|
+
state.status = "done";
|
|
8642
|
+
}).catch((err) => {
|
|
8643
|
+
state.status = "error";
|
|
8644
|
+
state.error = err instanceof Error ? err.message : String(err);
|
|
8645
|
+
}).finally(() => controllers.delete(id));
|
|
8646
|
+
return { scanId: id };
|
|
8647
|
+
}
|
|
6844
8648
|
|
|
6845
8649
|
// src/routes/catalogImport.ts
|
|
6846
8650
|
function registerCatalogImportRoutes(app, deps) {
|
|
@@ -6848,23 +8652,104 @@ function registerCatalogImportRoutes(app, deps) {
|
|
|
6848
8652
|
app.get("/api/brands/:id/products-library", async (req, reply) => {
|
|
6849
8653
|
const brand = core.store.getBrand(req.params.id);
|
|
6850
8654
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
6851
|
-
const products = core.catalog.
|
|
8655
|
+
const products = core.catalog.listLibraryIndex(brand.id, brand.json);
|
|
6852
8656
|
const source = core.catalog.getSourceForBrand(brand.id);
|
|
6853
8657
|
return { products, source };
|
|
6854
8658
|
});
|
|
8659
|
+
app.get("/api/brands/:id/products-library/:productId", async (req, reply) => {
|
|
8660
|
+
const brand = core.store.getBrand(req.params.id);
|
|
8661
|
+
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8662
|
+
const product = core.catalog.libraryProduct(brand.id, brand.json, String(req.params.productId));
|
|
8663
|
+
if (!product) return reply.status(404).send({ error: "product not found" });
|
|
8664
|
+
return { product };
|
|
8665
|
+
});
|
|
6855
8666
|
app.get("/api/brands/:id/catalog/source", async (req, reply) => {
|
|
6856
8667
|
const brand = core.store.getBrand(req.params.id);
|
|
6857
8668
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
6858
8669
|
return { source: core.catalog.getSourceForBrand(brand.id) };
|
|
6859
8670
|
});
|
|
8671
|
+
app.post("/api/brands/:id/catalog/scan", async (req, reply) => {
|
|
8672
|
+
const brandId = req.params.id;
|
|
8673
|
+
const brand = core.store.getBrand(brandId);
|
|
8674
|
+
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8675
|
+
const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
|
|
8676
|
+
if (!url.trim()) return reply.status(400).send({ error: "url required" });
|
|
8677
|
+
try {
|
|
8678
|
+
return startCatalogScan({ core, fetchImpl }, brandId, url);
|
|
8679
|
+
} catch (err) {
|
|
8680
|
+
return reply.status(err.statusCode ?? 500).send({ error: err.message ?? "scan failed" });
|
|
8681
|
+
}
|
|
8682
|
+
});
|
|
8683
|
+
app.get("/api/brands/:id/catalog/scans/:scanId", async (req, reply) => {
|
|
8684
|
+
const brandId = req.params.id;
|
|
8685
|
+
const scan = getScan(req.params.scanId);
|
|
8686
|
+
if (!scan || scan.brandId !== brandId) return reply.status(404).send({ error: "scan not found" });
|
|
8687
|
+
return scan;
|
|
8688
|
+
});
|
|
8689
|
+
app.post("/api/brands/:id/catalog/details", async (req, reply) => {
|
|
8690
|
+
const brand = core.store.getBrand(req.params.id);
|
|
8691
|
+
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8692
|
+
const asked = req.body?.urls;
|
|
8693
|
+
if (!Array.isArray(asked) || asked.some((u) => typeof u !== "string")) {
|
|
8694
|
+
return reply.status(400).send({ error: "urls must be a list of product addresses" });
|
|
8695
|
+
}
|
|
8696
|
+
const urls = asked.slice(0, 48);
|
|
8697
|
+
if (!urls.length) return { products: [] };
|
|
8698
|
+
const site = String(brand.json?.meta?.website ?? "");
|
|
8699
|
+
let origin;
|
|
8700
|
+
try {
|
|
8701
|
+
origin = new URL(site.startsWith("http") ? site : `https://${site}`).origin;
|
|
8702
|
+
} catch {
|
|
8703
|
+
return reply.status(400).send({ error: "this brand has no website to read products from" });
|
|
8704
|
+
}
|
|
8705
|
+
const sameSite = urls.filter((u) => {
|
|
8706
|
+
try {
|
|
8707
|
+
return new URL(u).origin === origin;
|
|
8708
|
+
} catch {
|
|
8709
|
+
return false;
|
|
8710
|
+
}
|
|
8711
|
+
});
|
|
8712
|
+
if (!sameSite.length) {
|
|
8713
|
+
return reply.status(400).send({ error: "none of those products belong to this site" });
|
|
8714
|
+
}
|
|
8715
|
+
const products = await fetchProductPages({ fetchImpl: fetchImpl ?? fetch, baseUrl: origin }, sameSite, {
|
|
8716
|
+
concurrency: 4,
|
|
8717
|
+
maxBytes: 15e5
|
|
8718
|
+
});
|
|
8719
|
+
return { products };
|
|
8720
|
+
});
|
|
6860
8721
|
app.post("/api/brands/:id/catalog/import", async (req, reply) => {
|
|
6861
8722
|
const brandId = req.params.id;
|
|
6862
8723
|
const brand = core.store.getBrand(brandId);
|
|
6863
8724
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
6864
8725
|
const url = String(req.body?.url ?? brand.json?.meta?.website ?? "");
|
|
6865
8726
|
if (!url.trim()) return reply.status(400).send({ error: "url required" });
|
|
8727
|
+
const asked = req.body?.urls;
|
|
8728
|
+
let only;
|
|
8729
|
+
if (asked != null) {
|
|
8730
|
+
if (!Array.isArray(asked) || asked.some((u) => typeof u !== "string")) {
|
|
8731
|
+
return reply.status(400).send({ error: "urls must be a list of product addresses" });
|
|
8732
|
+
}
|
|
8733
|
+
let origin;
|
|
8734
|
+
try {
|
|
8735
|
+
origin = new URL(url.startsWith("http") ? url : `https://${url}`).origin;
|
|
8736
|
+
} catch {
|
|
8737
|
+
return reply.status(400).send({ error: "url required" });
|
|
8738
|
+
}
|
|
8739
|
+
only = asked.filter((u) => {
|
|
8740
|
+
try {
|
|
8741
|
+
return new URL(u).origin === origin;
|
|
8742
|
+
} catch {
|
|
8743
|
+
return false;
|
|
8744
|
+
}
|
|
8745
|
+
});
|
|
8746
|
+
if (!only.length) return reply.status(400).send({ error: "none of those products belong to this site" });
|
|
8747
|
+
if (only.length > 2e3) {
|
|
8748
|
+
return reply.status(400).send({ error: "That is nearly the whole catalogue. Import everything instead." });
|
|
8749
|
+
}
|
|
8750
|
+
}
|
|
6866
8751
|
try {
|
|
6867
|
-
return startCatalogImport({ core, fetchImpl }, brandId, url);
|
|
8752
|
+
return startCatalogImport({ core, fetchImpl }, brandId, url, { only });
|
|
6868
8753
|
} catch (err) {
|
|
6869
8754
|
return reply.status(err.statusCode ?? 500).send({ error: err.message ?? "import failed" });
|
|
6870
8755
|
}
|
|
@@ -6915,7 +8800,7 @@ async function vibrantColor(input) {
|
|
|
6915
8800
|
let data;
|
|
6916
8801
|
let channels;
|
|
6917
8802
|
try {
|
|
6918
|
-
const out = await
|
|
8803
|
+
const out = await sharp21(input).resize(48, 48, { fit: "inside" }).raw().toBuffer({ resolveWithObject: true });
|
|
6919
8804
|
data = out.data;
|
|
6920
8805
|
channels = out.info.channels;
|
|
6921
8806
|
} catch {
|
|
@@ -6938,13 +8823,13 @@ async function vibrantColor(input) {
|
|
|
6938
8823
|
const best = buckets.reduce((a, b) => b.score > a.score ? b : a, buckets[0]);
|
|
6939
8824
|
if (best.score <= 0) {
|
|
6940
8825
|
try {
|
|
6941
|
-
const { dominant } = await
|
|
6942
|
-
return
|
|
8826
|
+
const { dominant } = await sharp21(input).stats();
|
|
8827
|
+
return toHex2(dominant.r, dominant.g, dominant.b);
|
|
6943
8828
|
} catch {
|
|
6944
8829
|
return null;
|
|
6945
8830
|
}
|
|
6946
8831
|
}
|
|
6947
|
-
return
|
|
8832
|
+
return toHex2(best.r / best.score, best.g / best.score, best.b / best.score);
|
|
6948
8833
|
}
|
|
6949
8834
|
function rgbToHsl(r, g, b) {
|
|
6950
8835
|
const rn = r / 255, gn = g / 255, bn = b / 255;
|
|
@@ -6959,7 +8844,7 @@ function rgbToHsl(r, g, b) {
|
|
|
6959
8844
|
else h = ((rn - gn) / d + 4) * 60;
|
|
6960
8845
|
return { h, s, l };
|
|
6961
8846
|
}
|
|
6962
|
-
var
|
|
8847
|
+
var toHex2 = (r, g, b) => "#" + [r, g, b].map(
|
|
6963
8848
|
(v) => Math.max(0, Math.min(255, Math.round(v))).toString(16).padStart(2, "0")
|
|
6964
8849
|
).join("");
|
|
6965
8850
|
|
|
@@ -7482,7 +9367,7 @@ function registerProjectRoutes(app, deps) {
|
|
|
7482
9367
|
const brand = core.store.getBrand(req.params.id);
|
|
7483
9368
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
7484
9369
|
const limit = Math.min(Number(req.query.limit) || 60, 200);
|
|
7485
|
-
return { nodes: core.store.recentActivity(brand.id, limit), jobs: core.catalog.
|
|
9370
|
+
return { nodes: core.store.recentActivity(brand.id, limit), jobs: core.catalog.listRecentJobs(brand.id) };
|
|
7486
9371
|
});
|
|
7487
9372
|
app.get("/api/brands/:id/workspace", async (req, reply) => {
|
|
7488
9373
|
const brand = core.store.getBrand(req.params.id);
|
|
@@ -7591,7 +9476,10 @@ function registerProjectRoutes(app, deps) {
|
|
|
7591
9476
|
function registerCodexSetupRoutes(app, deps) {
|
|
7592
9477
|
const codexSetup = deps.codexSetup ?? createCodexSetup({ runner: deps.codexRunner });
|
|
7593
9478
|
let codexSetupBusy = null;
|
|
7594
|
-
app.get("/api/engines/codex/status", async () =>
|
|
9479
|
+
app.get("/api/engines/codex/status", async (req) => {
|
|
9480
|
+
const force = req.query?.force === "1";
|
|
9481
|
+
return codexSetup.status({ force });
|
|
9482
|
+
});
|
|
7595
9483
|
app.post("/api/engines/codex/install", async (_req, reply) => {
|
|
7596
9484
|
if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
|
|
7597
9485
|
codexSetupBusy = "install";
|
|
@@ -7614,6 +9502,35 @@ function registerCodexSetupRoutes(app, deps) {
|
|
|
7614
9502
|
codexSetupBusy = null;
|
|
7615
9503
|
}
|
|
7616
9504
|
});
|
|
9505
|
+
app.post("/api/engines/codex/repair-env", async (req, reply) => {
|
|
9506
|
+
const repair = deps.envRepair;
|
|
9507
|
+
if (!repair) return reply.status(501).send({ error: "not available on this server" });
|
|
9508
|
+
const asked = req.body?.keys;
|
|
9509
|
+
if (!Array.isArray(asked) || asked.length === 0) return reply.status(400).send({ error: "keys required" });
|
|
9510
|
+
const keys = asked.map((k) => String(k).trim().toUpperCase());
|
|
9511
|
+
const unknown = keys.filter((k) => !CONFLICT_ENV_KEYS.includes(k));
|
|
9512
|
+
if (unknown.length > 0) return reply.status(400).send({ error: `not a Codex credential: ${unknown.join(", ")}` });
|
|
9513
|
+
if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
|
|
9514
|
+
codexSetupBusy = "repair";
|
|
9515
|
+
try {
|
|
9516
|
+
repair.set([.../* @__PURE__ */ new Set([...repair.get(), ...keys])]);
|
|
9517
|
+
return { ok: true, ...await codexSetup.status({ force: true }) };
|
|
9518
|
+
} finally {
|
|
9519
|
+
codexSetupBusy = null;
|
|
9520
|
+
}
|
|
9521
|
+
});
|
|
9522
|
+
app.post("/api/engines/codex/restore-env", async (_req, reply) => {
|
|
9523
|
+
const repair = deps.envRepair;
|
|
9524
|
+
if (!repair) return reply.status(501).send({ error: "not available on this server" });
|
|
9525
|
+
if (codexSetupBusy) return reply.status(409).send({ error: `already running: ${codexSetupBusy}` });
|
|
9526
|
+
codexSetupBusy = "repair";
|
|
9527
|
+
try {
|
|
9528
|
+
repair.set([]);
|
|
9529
|
+
return { ok: true, ...await codexSetup.status({ force: true }) };
|
|
9530
|
+
} finally {
|
|
9531
|
+
codexSetupBusy = null;
|
|
9532
|
+
}
|
|
9533
|
+
});
|
|
7617
9534
|
}
|
|
7618
9535
|
var slug = (v, fallback) => {
|
|
7619
9536
|
const s = String(v ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
|
|
@@ -7794,8 +9711,8 @@ function registerImageRoutes(app, deps) {
|
|
|
7794
9711
|
if (!part) return reply.status(400).send({ error: "multipart file field required" });
|
|
7795
9712
|
const buf = await part.toBuffer();
|
|
7796
9713
|
if (buf.length === 0) return reply.status(400).send({ error: "empty file" });
|
|
7797
|
-
const fmt = (await
|
|
7798
|
-
const png = fmt === "svg" ? await toMarkPng(buf) : await
|
|
9714
|
+
const fmt = (await sharp21(buf).metadata().catch(() => null))?.format;
|
|
9715
|
+
const png = fmt === "svg" ? await toMarkPng(buf) : await sharp21(buf).rotate().png().toBuffer();
|
|
7799
9716
|
return { hash: core.images.save(png) };
|
|
7800
9717
|
});
|
|
7801
9718
|
app.post("/api/diff", async (req, reply) => {
|
|
@@ -7817,6 +9734,29 @@ function registerImageRoutes(app, deps) {
|
|
|
7817
9734
|
|
|
7818
9735
|
// src/release/notes.data.ts
|
|
7819
9736
|
var RELEASES = [
|
|
9737
|
+
{
|
|
9738
|
+
version: "0.9.3",
|
|
9739
|
+
date: "2026-09-14",
|
|
9740
|
+
title: "Paste a website, get the brand and its products.",
|
|
9741
|
+
sections: [
|
|
9742
|
+
{
|
|
9743
|
+
heading: "Brand",
|
|
9744
|
+
body: "A website address is enough to start a brand. Scenri reads the logo, the wordmark and the palette a site actually declares, and says plainly what it found and what it could not."
|
|
9745
|
+
},
|
|
9746
|
+
{
|
|
9747
|
+
heading: "Products",
|
|
9748
|
+
body: "The same address brings in the store. Pick the products you want or take the whole catalogue, and watch them arrive: each one appears in your library as it lands, with its picture, newest first. A large store no longer runs out of memory partway through, and a shop that stops answering says so instead of looking empty."
|
|
9749
|
+
},
|
|
9750
|
+
{
|
|
9751
|
+
heading: "Create",
|
|
9752
|
+
body: "The insert menus reach everything. Typing $, @, / or # opens a shortlist that pages through the rest of that catalogue, with a count so nothing is quietly cut off, and the box stays where the caret is."
|
|
9753
|
+
},
|
|
9754
|
+
{
|
|
9755
|
+
heading: "Fixes",
|
|
9756
|
+
body: "A first run on Windows no longer stops at the desktop icon question, and never asks for an administrator it does not need. Codex reports itself ready only once a real run has authenticated."
|
|
9757
|
+
}
|
|
9758
|
+
]
|
|
9759
|
+
},
|
|
7820
9760
|
{
|
|
7821
9761
|
version: "0.9.2",
|
|
7822
9762
|
date: "2026-09-07",
|
|
@@ -8787,15 +10727,15 @@ function registerDesktopRoutes(app, deps) {
|
|
|
8787
10727
|
record: null
|
|
8788
10728
|
};
|
|
8789
10729
|
}
|
|
8790
|
-
const { installDeps } = await import('./cli-
|
|
8791
|
-
const { desktopStatus } = await import('./install-
|
|
10730
|
+
const { installDeps } = await import('./cli-SX46UQO6.js');
|
|
10731
|
+
const { desktopStatus } = await import('./install-VXJPLYWM.js');
|
|
8792
10732
|
return desktopStatus(installDeps(runtime.entry));
|
|
8793
10733
|
});
|
|
8794
10734
|
const install = deps.installImpl ?? (async () => {
|
|
8795
10735
|
if (!runtime.entry) {
|
|
8796
10736
|
return { ok: false, reason: "unsupported", message: "Desktop shortcuts are not available on this system yet." };
|
|
8797
10737
|
}
|
|
8798
|
-
const { addToDesktop } = await import('./cli-
|
|
10738
|
+
const { addToDesktop } = await import('./cli-SX46UQO6.js');
|
|
8799
10739
|
return addToDesktop(runtime.entry);
|
|
8800
10740
|
});
|
|
8801
10741
|
app.get("/api/desktop", async () => {
|
|
@@ -8860,6 +10800,7 @@ function buildServer(opts) {
|
|
|
8860
10800
|
registerAccessGuard(app, opts.access);
|
|
8861
10801
|
const reserved = /* @__PURE__ */ new Map();
|
|
8862
10802
|
const runningGenerations = /* @__PURE__ */ new Map();
|
|
10803
|
+
core.catalog.reconcileInterruptedJobs();
|
|
8863
10804
|
const thumbs = createThumbStore(core);
|
|
8864
10805
|
const { scenes } = loadScenes(opts.templatesDir);
|
|
8865
10806
|
const resolveScene = sceneResolver(scenes);
|
|
@@ -8869,8 +10810,11 @@ function buildServer(opts) {
|
|
|
8869
10810
|
const e = err;
|
|
8870
10811
|
const status = err instanceof SpendCapError ? 402 : e.statusCode ?? 500;
|
|
8871
10812
|
const leaksPath = typeof e.code === "string" && /^(ENOENT|EACCES|EPERM|EISDIR|ENOTDIR)$/.test(e.code);
|
|
8872
|
-
|
|
10813
|
+
const rawRuntime = !e.statusCode && (err instanceof TypeError || err instanceof RangeError);
|
|
10814
|
+
if (rawRuntime) console.error("unexpected error:", err);
|
|
10815
|
+
reply.status(status).send({ error: leaksPath || rawRuntime ? "unexpected error" : e.message ?? "unexpected error" });
|
|
8873
10816
|
});
|
|
10817
|
+
const scrapeGuard = () => ({ allowPrivateHosts: process.env.SCENRI_SCRAPE_ALLOW_PRIVATE === "1" });
|
|
8874
10818
|
app.get("/api/brands", async () => core.store.listBrands());
|
|
8875
10819
|
app.post("/api/brands", async (req, reply) => {
|
|
8876
10820
|
const json = req.body?.brand;
|
|
@@ -8879,10 +10823,12 @@ function buildServer(opts) {
|
|
|
8879
10823
|
return core.store.createBrand(json);
|
|
8880
10824
|
});
|
|
8881
10825
|
app.post("/api/brands/from-url", async (req, reply) => {
|
|
8882
|
-
const
|
|
8883
|
-
if (
|
|
8884
|
-
const
|
|
10826
|
+
const asked = normalizeSiteUrl(req.body?.url);
|
|
10827
|
+
if (!asked.ok) return reply.status(400).send({ error: asked.message });
|
|
10828
|
+
const url = asked.url;
|
|
10829
|
+
const { brand, warnings, report } = await buildFromUrl(url, {
|
|
8885
10830
|
fetchImpl: opts.fetchImpl,
|
|
10831
|
+
guard: scrapeGuard(),
|
|
8886
10832
|
// The store names every blob `<hash>.png` and /api/images/:hash always
|
|
8887
10833
|
// serves image/png, so an un-normalized .ico or .svg here is a file lying
|
|
8888
10834
|
// about its own format — broken in the marks grid, and mislabelled to any
|
|
@@ -8890,14 +10836,11 @@ function buildServer(opts) {
|
|
|
8890
10836
|
saveAsset: async (buf) => `asset:${core.images.save(await toMarkPng(buf))}`,
|
|
8891
10837
|
// Measured as stored (post-toMarkPng), so the scrape judges the same
|
|
8892
10838
|
// pixels the compiler will one day attach.
|
|
8893
|
-
|
|
8894
|
-
const m = await sharp20(await toMarkPng(buf)).metadata();
|
|
8895
|
-
return Math.max(m.width ?? 0, m.height ?? 0) || null;
|
|
8896
|
-
},
|
|
10839
|
+
inspectMark: (buf) => inspectMark(buf, toMarkPng),
|
|
8897
10840
|
createdWith: `${meta.name}/${meta.version}`
|
|
8898
10841
|
});
|
|
8899
10842
|
const row = core.store.createBrand(brand);
|
|
8900
|
-
return { ...row, warnings };
|
|
10843
|
+
return { ...row, warnings, report };
|
|
8901
10844
|
});
|
|
8902
10845
|
app.put("/api/brands/:id", async (req, reply) => {
|
|
8903
10846
|
const json = req.body?.brand;
|
|
@@ -8974,15 +10917,14 @@ function buildServer(opts) {
|
|
|
8974
10917
|
app.post("/api/brands/:id/refresh-from-url", async (req, reply) => {
|
|
8975
10918
|
const brand = core.store.getBrand(req.params.id);
|
|
8976
10919
|
if (!brand) return reply.status(404).send({ error: "brand not found" });
|
|
8977
|
-
const
|
|
8978
|
-
if (
|
|
10920
|
+
const asked = normalizeSiteUrl(req.body?.url ?? brand.json?.meta?.website);
|
|
10921
|
+
if (!asked.ok) return reply.status(400).send({ error: asked.message });
|
|
10922
|
+
const url = asked.url;
|
|
8979
10923
|
const { brand: scraped, warnings } = await buildFromUrl(url, {
|
|
8980
10924
|
fetchImpl: opts.fetchImpl,
|
|
10925
|
+
guard: scrapeGuard(),
|
|
8981
10926
|
saveAsset: async (buf) => `asset:${core.images.save(await toMarkPng(buf))}`,
|
|
8982
|
-
|
|
8983
|
-
const m = await sharp20(await toMarkPng(buf)).metadata();
|
|
8984
|
-
return Math.max(m.width ?? 0, m.height ?? 0) || null;
|
|
8985
|
-
},
|
|
10927
|
+
inspectMark: (buf) => inspectMark(buf, toMarkPng),
|
|
8986
10928
|
createdWith: `${meta.name}/${meta.version}`
|
|
8987
10929
|
});
|
|
8988
10930
|
const { brand: merged, suggestions } = mergeScrape(brand.json, scraped);
|
|
@@ -9290,7 +11232,14 @@ function buildServer(opts) {
|
|
|
9290
11232
|
],
|
|
9291
11233
|
engineNames: () => engines.all().map((e) => ({ id: e.capabilities().id, name: e.capabilities().displayName }))
|
|
9292
11234
|
});
|
|
9293
|
-
registerCodexSetupRoutes(app, {
|
|
11235
|
+
registerCodexSetupRoutes(app, {
|
|
11236
|
+
codexSetup: opts.codexSetup,
|
|
11237
|
+
codexRunner: engines.codexRunner,
|
|
11238
|
+
envRepair: {
|
|
11239
|
+
get: () => [...ignoreEnvKeysGetter(core)()],
|
|
11240
|
+
set: (keys) => core.store.setSetting(IGNORE_ENV_KEYS_SETTING, keys.join(","))
|
|
11241
|
+
}
|
|
11242
|
+
});
|
|
9294
11243
|
app.get("/api/engines", async () => {
|
|
9295
11244
|
const list2 = [];
|
|
9296
11245
|
for (const e of engines.all()) {
|
|
@@ -9353,11 +11302,11 @@ function buildServer(opts) {
|
|
|
9353
11302
|
const out = [];
|
|
9354
11303
|
for (const h of images) {
|
|
9355
11304
|
const buf = core.images.read(h);
|
|
9356
|
-
const meta2 = await
|
|
11305
|
+
const meta2 = await sharp21(buf).metadata().catch(() => null);
|
|
9357
11306
|
if (!meta2?.width || !meta2.height) throw new Error("engine returned an undecodable image");
|
|
9358
11307
|
const oriented = (meta2.orientation ?? 1) !== 1;
|
|
9359
11308
|
out.push(
|
|
9360
|
-
buf.subarray(0, 8).equals(PNG_SIG) && !oriented ? h : core.images.save(await
|
|
11309
|
+
buf.subarray(0, 8).equals(PNG_SIG) && !oriented ? h : core.images.save(await sharp21(buf).rotate().png().toBuffer())
|
|
9361
11310
|
);
|
|
9362
11311
|
}
|
|
9363
11312
|
return out;
|
|
@@ -9369,7 +11318,7 @@ function buildServer(opts) {
|
|
|
9369
11318
|
const out = [];
|
|
9370
11319
|
for (const h of images) {
|
|
9371
11320
|
const buf = core.images.read(h);
|
|
9372
|
-
const meta2 = await
|
|
11321
|
+
const meta2 = await sharp21(buf).metadata();
|
|
9373
11322
|
if (!meta2.width || !meta2.height) {
|
|
9374
11323
|
out.push(h);
|
|
9375
11324
|
continue;
|
|
@@ -9382,7 +11331,7 @@ function buildServer(opts) {
|
|
|
9382
11331
|
}
|
|
9383
11332
|
const w = got > target ? Math.round(meta2.height * target) : meta2.width;
|
|
9384
11333
|
const hpx = got > target ? meta2.height : Math.round(meta2.width / target);
|
|
9385
|
-
const cropped = await
|
|
11334
|
+
const cropped = await sharp21(buf).resize(w, hpx, { fit: "cover", position: "attention" }).png().toBuffer();
|
|
9386
11335
|
app.log.info(
|
|
9387
11336
|
{ nodeId, got: `${meta2.width}x${meta2.height}`, want: `${w}x${hpx}` },
|
|
9388
11337
|
"canvas: cropped a drifted frame to the asked ratio"
|
|
@@ -9401,7 +11350,7 @@ function buildServer(opts) {
|
|
|
9401
11350
|
async function assertAspect(images, expect) {
|
|
9402
11351
|
const want = expect.width / expect.height;
|
|
9403
11352
|
for (const h of images) {
|
|
9404
|
-
const meta2 = await
|
|
11353
|
+
const meta2 = await sharp21(core.images.read(h)).metadata();
|
|
9405
11354
|
if (!meta2.width || !meta2.height) continue;
|
|
9406
11355
|
const got = meta2.width / meta2.height;
|
|
9407
11356
|
if (Math.abs(got - want) / want > ASPECT_TOLERANCE)
|
|
@@ -9432,7 +11381,7 @@ function buildServer(opts) {
|
|
|
9432
11381
|
if (post) own = await post(own);
|
|
9433
11382
|
if (expect) await assertAspect(own, expect);
|
|
9434
11383
|
try {
|
|
9435
|
-
const meta2 = await
|
|
11384
|
+
const meta2 = await sharp21(core.images.read(own[0])).metadata();
|
|
9436
11385
|
const node = core.store.getNode(id);
|
|
9437
11386
|
if (node && meta2.width && meta2.height) {
|
|
9438
11387
|
const brief = node.brief ?? {};
|
|
@@ -9541,7 +11490,7 @@ function buildServer(opts) {
|
|
|
9541
11490
|
crop: window
|
|
9542
11491
|
});
|
|
9543
11492
|
const work2 = async () => ({
|
|
9544
|
-
images: [core.images.save(await
|
|
11493
|
+
images: [core.images.save(await sharp21(args.srcBuf).extract(window).png().toBuffer())],
|
|
9545
11494
|
costUsd: 0
|
|
9546
11495
|
});
|
|
9547
11496
|
void runNode([node2.id], null, 0, work2, { width: plan2.width, height: plan2.height }).catch(
|
|
@@ -9566,7 +11515,7 @@ function buildServer(opts) {
|
|
|
9566
11515
|
if (!srcHash || !core.images.has(String(srcHash)))
|
|
9567
11516
|
return reply.status(400).send({ error: "edit needs a parent node with an image (sourceImage)" });
|
|
9568
11517
|
const srcBuf = core.images.read(String(srcHash));
|
|
9569
|
-
const srcMeta = await
|
|
11518
|
+
const srcMeta = await sharp21(srcBuf).metadata();
|
|
9570
11519
|
if (!srcMeta.width || !srcMeta.height) return reply.status(400).send({ error: "source image unreadable" });
|
|
9571
11520
|
return runCropNode({
|
|
9572
11521
|
parentId: cropParentId,
|
|
@@ -9781,7 +11730,7 @@ function buildServer(opts) {
|
|
|
9781
11730
|
);
|
|
9782
11731
|
}
|
|
9783
11732
|
const srcBuf = core.images.read(String(srcHash));
|
|
9784
|
-
const srcMeta = await
|
|
11733
|
+
const srcMeta = await sharp21(srcBuf).metadata();
|
|
9785
11734
|
if (srcMeta.width && srcMeta.height) expectShape = { width: srcMeta.width, height: srcMeta.height };
|
|
9786
11735
|
const parentFormat = parent?.brief?.tokens?.find((t) => t?.t === "format");
|
|
9787
11736
|
const parentNominal = parentFormat && Number(parentFormat.w) > 0 && Number(parentFormat.h) > 0 ? { width: Number(parentFormat.w), height: Number(parentFormat.h) } : null;
|
|
@@ -9815,7 +11764,7 @@ function buildServer(opts) {
|
|
|
9815
11764
|
} else if (decision.op === "extend") {
|
|
9816
11765
|
if (decision.assist) {
|
|
9817
11766
|
expandAssist = { width: decision.assist.width, height: decision.assist.height };
|
|
9818
|
-
workBuf = await
|
|
11767
|
+
workBuf = await sharp21(srcBuf).extract(decision.assist).png().toBuffer();
|
|
9819
11768
|
workSize = { width: decision.assist.width, height: decision.assist.height };
|
|
9820
11769
|
}
|
|
9821
11770
|
expandPlan = planExpand(workSize, targetRatio);
|
|
@@ -9836,7 +11785,7 @@ function buildServer(opts) {
|
|
|
9836
11785
|
const fit = fitExpandToBudget(expandPlan, workSize, runEngine.capabilities().editPixelBudget);
|
|
9837
11786
|
if (fit.scale < 1) {
|
|
9838
11787
|
expandPlan = fit.plan;
|
|
9839
|
-
workBuf = await
|
|
11788
|
+
workBuf = await sharp21(workBuf).resize(fit.source.width, fit.source.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer();
|
|
9840
11789
|
workSize = fit.source;
|
|
9841
11790
|
extraWarnings.push(
|
|
9842
11791
|
`${runEngine.capabilities().displayName} draws about ${((runEngine.capabilities().editPixelBudget ?? 0) / 1e6).toFixed(1)} megapixels, so this shape continues as a ${fit.plan.width}x${fit.plan.height} frame with the photograph riding inside it at ${fit.source.width}x${fit.source.height}. Nothing is upscaled; the stored size is the size the engine truly drew.`
|
|
@@ -9859,7 +11808,7 @@ function buildServer(opts) {
|
|
|
9859
11808
|
if (editPixelBudget && stepped && (stepped.width !== srcMeta.width || stepped.height !== srcMeta.height)) {
|
|
9860
11809
|
sentSize = stepped;
|
|
9861
11810
|
budgetSourceHash = core.images.save(
|
|
9862
|
-
await
|
|
11811
|
+
await sharp21(srcBuf).resize(sentSize.width, sentSize.height, { fit: "fill", kernel: "lanczos3" }).png().toBuffer()
|
|
9863
11812
|
);
|
|
9864
11813
|
if (!gradeOnlyAsk)
|
|
9865
11814
|
extraWarnings.push(
|
|
@@ -9999,11 +11948,11 @@ function buildServer(opts) {
|
|
|
9999
11948
|
const original = editedFrom ? core.images.read(editedFrom) : null;
|
|
10000
11949
|
const localScope = kind === "edit" && !plan && editScope === "local" && original;
|
|
10001
11950
|
const enforceEditCanvas = async (images) => {
|
|
10002
|
-
const srcMeta = await
|
|
11951
|
+
const srcMeta = await sharp21(original).metadata();
|
|
10003
11952
|
if (!srcMeta.width || !srcMeta.height) return images;
|
|
10004
11953
|
const out = [];
|
|
10005
11954
|
for (const h of images) {
|
|
10006
|
-
const meta2 = await
|
|
11955
|
+
const meta2 = await sharp21(core.images.read(h)).metadata();
|
|
10007
11956
|
const got = { width: meta2.width ?? 0, height: meta2.height ?? 0 };
|
|
10008
11957
|
const verdict = judgeEditSize({ width: srcMeta.width, height: srcMeta.height }, got, {
|
|
10009
11958
|
pixelBudget: runEngine.capabilities().editPixelBudget
|
|
@@ -10033,7 +11982,7 @@ function buildServer(opts) {
|
|
|
10033
11982
|
);
|
|
10034
11983
|
out.push(
|
|
10035
11984
|
core.images.save(
|
|
10036
|
-
await
|
|
11985
|
+
await sharp21(core.images.read(h)).resize(srcMeta.width, srcMeta.height, { fit: "fill" }).png().toBuffer()
|
|
10037
11986
|
)
|
|
10038
11987
|
);
|
|
10039
11988
|
try {
|
|
@@ -10055,7 +12004,7 @@ function buildServer(opts) {
|
|
|
10055
12004
|
const out = [];
|
|
10056
12005
|
for (const h of images) {
|
|
10057
12006
|
const answer = core.images.read(h);
|
|
10058
|
-
const got = await
|
|
12007
|
+
const got = await sharp21(answer).metadata();
|
|
10059
12008
|
if (got.width !== plan.width || got.height !== plan.height)
|
|
10060
12009
|
app.log.info(
|
|
10061
12010
|
{ nodeId: node.id, got: `${got.width}x${got.height}`, want: `${plan.width}x${plan.height}` },
|
|
@@ -10216,6 +12165,7 @@ function buildServer(opts) {
|
|
|
10216
12165
|
while (runningGenerations.size > 0 && Date.now() < deadline) {
|
|
10217
12166
|
await new Promise((r) => setTimeout(r, 25));
|
|
10218
12167
|
}
|
|
12168
|
+
await settleCatalogImports();
|
|
10219
12169
|
await thumbs.settle();
|
|
10220
12170
|
await app.close();
|
|
10221
12171
|
core.close();
|
|
@@ -10388,9 +12338,9 @@ async function run() {
|
|
|
10388
12338
|
}
|
|
10389
12339
|
}
|
|
10390
12340
|
const ownEntry = fileURLToPath(import.meta.url);
|
|
10391
|
-
const { addToDesktop, installDeps } = await import('./cli-
|
|
10392
|
-
const { refreshLauncher } = await import('./refresh-
|
|
10393
|
-
const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-
|
|
12341
|
+
const { addToDesktop, installDeps } = await import('./cli-SX46UQO6.js');
|
|
12342
|
+
const { refreshLauncher } = await import('./refresh-5HBDLLLB.js');
|
|
12343
|
+
const { askOnTerminal, offerDesktop, shouldOfferDesktop } = await import('./offer-W6NJXMWT.js');
|
|
10394
12344
|
const { launcherInstalled } = await import('./paths-6ZXRZUHY.js');
|
|
10395
12345
|
const meta = readMeta();
|
|
10396
12346
|
void refreshLauncher({ ...installDeps(ownEntry), ownEntry, installKind, pkg: meta.name }).then((r) => {
|
|
@@ -10412,7 +12362,7 @@ async function run() {
|
|
|
10412
12362
|
add: () => addToDesktop(ownEntry),
|
|
10413
12363
|
decline: () => core.store.setSetting("desktop.prompt", "declined"),
|
|
10414
12364
|
say: (line) => console.log(line)
|
|
10415
|
-
});
|
|
12365
|
+
}).catch(() => console.log(" Could not ask about the desktop icon. Scenri is running anyway."));
|
|
10416
12366
|
console.log("");
|
|
10417
12367
|
}
|
|
10418
12368
|
}
|
|
@@ -10422,8 +12372,8 @@ async function verify() {
|
|
|
10422
12372
|
const db = new Database(":memory:");
|
|
10423
12373
|
db.pragma("user_version");
|
|
10424
12374
|
db.close();
|
|
10425
|
-
const { default:
|
|
10426
|
-
await
|
|
12375
|
+
const { default: sharp22 } = await import('sharp');
|
|
12376
|
+
await sharp22({ create: { width: 1, height: 1, channels: 3, background: "#000" } }).png().toBuffer();
|
|
10427
12377
|
console.log(JSON.stringify({ ok: true, version: readMeta().version }));
|
|
10428
12378
|
} catch (err) {
|
|
10429
12379
|
console.log(JSON.stringify({ ok: false, error: String(err?.message ?? err) }));
|