tablefacts 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +313 -0
- package/bin/tablefacts.mjs +28 -0
- package/package.json +77 -0
- package/src/index.mjs +52 -0
- package/src/instagram/README.md +135 -0
- package/src/instagram/download.mjs +62 -0
- package/src/instagram/index.mjs +355 -0
- package/src/instagram/links.mjs +55 -0
- package/src/instagram/record.mjs +18 -0
- package/src/lib/edge.mjs +49 -0
- package/src/lib/env.mjs +35 -0
- package/src/lib/errors.mjs +39 -0
- package/src/lib/files.mjs +10 -0
- package/src/lib/images.mjs +20 -0
- package/src/lib/log.mjs +17 -0
- package/src/lib/photos.mjs +42 -0
- package/src/lib/playwright.mjs +13 -0
- package/src/lib/project.mjs +42 -0
- package/src/lib/text.mjs +7 -0
- package/src/lib/types.mjs +247 -0
- package/src/menu/README.md +97 -0
- package/src/menu/cluvi/config.mjs +33 -0
- package/src/menu/cluvi/extract.mjs +24 -0
- package/src/menu/cluvi/import.mjs +30 -0
- package/src/menu/cluvi/source.mjs +156 -0
- package/src/menu/index.mjs +9 -0
- package/src/menu/lib/db.mjs +119 -0
- package/src/menu/lib/import.mjs +126 -0
- package/src/menu/lib/menu.mjs +95 -0
- package/src/menu/lib/run.mjs +83 -0
- package/src/menu/raw/config.mjs +38 -0
- package/src/menu/raw/extract.mjs +72 -0
- package/src/menu/raw/import.mjs +99 -0
- package/src/menu/raw/normalize.mjs +120 -0
- package/src/menu/raw/source.mjs +116 -0
- package/src/menu/raw/vision.mjs +252 -0
- package/src/research/README.md +69 -0
- package/src/research/index.mjs +147 -0
- package/src/research/lib/google.mjs +93 -0
- package/src/research/lib/hours.mjs +109 -0
- package/src/research/lib/merge.mjs +119 -0
- package/src/research/lib/osm.mjs +49 -0
- package/src/research/lib/report.mjs +118 -0
- package/src/research/lib/social.mjs +49 -0
- package/src/research/lib/util.mjs +104 -0
- package/src/research/lib/website.mjs +285 -0
- package/src/research/research.mjs +67 -0
- package/src/tripadvisor/README.md +78 -0
- package/src/tripadvisor/index.mjs +178 -0
- package/src/tripadvisor/links.mjs +78 -0
- package/src/tripadvisor/photos.mjs +55 -0
- package/types/index.d.mts +65 -0
- package/types/instagram/download.d.mts +1 -0
- package/types/instagram/index.d.mts +13 -0
- package/types/instagram/links.d.mts +14 -0
- package/types/instagram/record.d.mts +1 -0
- package/types/lib/edge.d.mts +14 -0
- package/types/lib/env.d.mts +12 -0
- package/types/lib/errors.d.mts +25 -0
- package/types/lib/files.d.mts +1 -0
- package/types/lib/images.d.mts +6 -0
- package/types/lib/log.d.mts +5 -0
- package/types/lib/photos.d.mts +30 -0
- package/types/lib/playwright.d.mts +1677 -0
- package/types/lib/project.d.mts +32 -0
- package/types/lib/text.d.mts +4 -0
- package/types/lib/types.d.mts +668 -0
- package/types/menu/cluvi/config.d.mts +13 -0
- package/types/menu/cluvi/extract.d.mts +2 -0
- package/types/menu/cluvi/import.d.mts +35 -0
- package/types/menu/cluvi/source.d.mts +37 -0
- package/types/menu/index.d.mts +6 -0
- package/types/menu/lib/db.d.mts +23 -0
- package/types/menu/lib/import.d.mts +11 -0
- package/types/menu/lib/menu.d.mts +24 -0
- package/types/menu/lib/run.d.mts +27 -0
- package/types/menu/raw/config.d.mts +18 -0
- package/types/menu/raw/extract.d.mts +2 -0
- package/types/menu/raw/import.d.mts +64 -0
- package/types/menu/raw/normalize.d.mts +29 -0
- package/types/menu/raw/source.d.mts +19 -0
- package/types/menu/raw/vision.d.mts +13 -0
- package/types/research/index.d.mts +9 -0
- package/types/research/lib/google.d.mts +12 -0
- package/types/research/lib/hours.d.mts +22 -0
- package/types/research/lib/merge.d.mts +87 -0
- package/types/research/lib/osm.d.mts +36 -0
- package/types/research/lib/report.d.mts +6 -0
- package/types/research/lib/social.d.mts +118 -0
- package/types/research/lib/util.d.mts +45 -0
- package/types/research/lib/website.d.mts +283 -0
- package/types/research/research.d.mts +1 -0
- package/types/tripadvisor/index.d.mts +11 -0
- package/types/tripadvisor/links.d.mts +13 -0
- package/types/tripadvisor/photos.d.mts +1 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// The command every menu source is: extract, check, write to Supabase.
|
|
2
|
+
// A source script (src/menu/<source>/extract.mjs) only says how to get the
|
|
3
|
+
// menu; the flags, the checks and the write are the same for all of them.
|
|
4
|
+
import { parseArgs } from "node:util";
|
|
5
|
+
import { loadEnv } from "../../lib/env.mjs";
|
|
6
|
+
import { cliMessage, exitCodeFor } from "../../lib/errors.mjs";
|
|
7
|
+
import { importMenu } from "./import.mjs";
|
|
8
|
+
|
|
9
|
+
/** The option names the menu library uses in its messages, as the flags the CLI shows. */
|
|
10
|
+
export const menuFlags = {
|
|
11
|
+
only: "--only",
|
|
12
|
+
provider: "--provider",
|
|
13
|
+
model: "--model",
|
|
14
|
+
minWidth: "--min-width",
|
|
15
|
+
replaceAll: "--replace-all",
|
|
16
|
+
force: "--force",
|
|
17
|
+
dryRun: "--dry-run",
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
/** A library message with option names turned into flags. */
|
|
21
|
+
export const flagText = (message) => cliMessage({ message }, menuFlags);
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The CLI's logger: info and warn lines (progress, notes, "left untouched") go to stdout as
|
|
25
|
+
* they always did, errors to stderr, and option names in the text become flags.
|
|
26
|
+
*/
|
|
27
|
+
export const cliLog = (message, level = "info") => {
|
|
28
|
+
const text = flagText(message);
|
|
29
|
+
if (level === "error") console.error(text);
|
|
30
|
+
else console.log(text);
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const commonOptions = `
|
|
34
|
+
Options:
|
|
35
|
+
--dry-run extract and check, show what would change, write nothing
|
|
36
|
+
--json <file> also save the extracted menu as JSON
|
|
37
|
+
--replace-all replace the whole menu, not only the categories in this import
|
|
38
|
+
--force write even if the import has far fewer products than it replaces
|
|
39
|
+
-h, --help show this help
|
|
40
|
+
|
|
41
|
+
The database is SUPABASE_DB_URL in .env (see .env.example).
|
|
42
|
+
`;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* `fetchMenu({ values, positionals })` returns `{ menu, notes, title }`: the
|
|
46
|
+
* menu in the shape of lib/menu.mjs, what the source wants the user to know,
|
|
47
|
+
* and a heading. `options` are the source's own flags, in util.parseArgs form.
|
|
48
|
+
*/
|
|
49
|
+
export async function runImport({ usage, options = {}, fetchMenu }) {
|
|
50
|
+
try {
|
|
51
|
+
const { values, positionals } = parseArgs({
|
|
52
|
+
args: process.argv.slice(2),
|
|
53
|
+
allowPositionals: true,
|
|
54
|
+
options: {
|
|
55
|
+
"dry-run": { type: "boolean" },
|
|
56
|
+
json: { type: "string" },
|
|
57
|
+
"replace-all": { type: "boolean" },
|
|
58
|
+
force: { type: "boolean" },
|
|
59
|
+
help: { type: "boolean", short: "h" },
|
|
60
|
+
...options,
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
if (values.help) {
|
|
64
|
+
console.log(usage + commonOptions);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
loadEnv();
|
|
69
|
+
const fetched = await fetchMenu({ values, positionals });
|
|
70
|
+
await importMenu({
|
|
71
|
+
...fetched,
|
|
72
|
+
dryRun: !!values["dry-run"],
|
|
73
|
+
json: values.json,
|
|
74
|
+
replaceAll: !!values["replace-all"],
|
|
75
|
+
force: !!values.force,
|
|
76
|
+
log: cliLog,
|
|
77
|
+
});
|
|
78
|
+
} catch (error) {
|
|
79
|
+
console.error(`
|
|
80
|
+
${cliMessage(error, menuFlags)}`);
|
|
81
|
+
process.exitCode = exitCodeFor(error);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Everything restaurant-specific about the image-menu import. Edit this file,
|
|
2
|
+
// not the other scripts, when the menu is organised differently.
|
|
3
|
+
export default {
|
|
4
|
+
// The page that shows the menu pictures, or direct image URLs.
|
|
5
|
+
// `tablefacts menu raw <url> [<url>...]` overrides it for one run.
|
|
6
|
+
url: "https://www.mombasa.co/carta-restaurante-espanol/",
|
|
7
|
+
|
|
8
|
+
// Currency of the prices, an ISO code. Colombian menus print pesos as
|
|
9
|
+
// "$95.000": "." groups thousands and "," marks decimals.
|
|
10
|
+
currency: "COP",
|
|
11
|
+
thousands: ".",
|
|
12
|
+
decimal: ",",
|
|
13
|
+
// Multiply every price by this when the menu prints thousands short ("95"
|
|
14
|
+
// for 95.000).
|
|
15
|
+
scale: 1,
|
|
16
|
+
|
|
17
|
+
// Each section the model reads is tagged food or drink (or other, which is
|
|
18
|
+
// left out), and goes into the category that lists its group. The site's
|
|
19
|
+
// default categories are cocina and bar.
|
|
20
|
+
categories: [
|
|
21
|
+
{ slug: "cocina", name: "Comida", groups: ["food"] },
|
|
22
|
+
{ slug: "bar", name: "Bebidas", groups: ["drink"] },
|
|
23
|
+
],
|
|
24
|
+
|
|
25
|
+
// Section title (ignoring case and accents) > category slug, for a section
|
|
26
|
+
// the model tagged wrongly.
|
|
27
|
+
placeIn: {},
|
|
28
|
+
|
|
29
|
+
// Section title > the name stored, for ones that should not just be the
|
|
30
|
+
// title in UPPERCASE. "COCTELES DE AUTOR" is what the home page cocktail
|
|
31
|
+
// list reads (signatureMenu in content/site.ts).
|
|
32
|
+
sections: {
|
|
33
|
+
"Cócteles de Autor": "COCTELES DE AUTOR",
|
|
34
|
+
},
|
|
35
|
+
|
|
36
|
+
// Section titles to leave out entirely.
|
|
37
|
+
skipSections: [],
|
|
38
|
+
};
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Imports a restaurant's menu from pictures of its pages into the Supabase
|
|
3
|
+
// menu tables. See src/menu/README.md. Run from the project's folder:
|
|
4
|
+
// tablefacts menu raw --list
|
|
5
|
+
// tablefacts menu raw --dry-run
|
|
6
|
+
import { parseArgs } from "node:util";
|
|
7
|
+
import { cliLog, menuFlags, runImport } from "../lib/run.mjs";
|
|
8
|
+
import { cliMessage, exitCodeFor } from "../../lib/errors.mjs";
|
|
9
|
+
import { fetchImageMenu, findPages } from "./import.mjs";
|
|
10
|
+
import { defaultProvider, providers } from "./vision.mjs";
|
|
11
|
+
|
|
12
|
+
const providerNames = Object.keys(providers).join(", ");
|
|
13
|
+
const providerTable = Object.entries(providers)
|
|
14
|
+
.map(([name, { keyName, defaultModel }]) => ` ${name.padEnd(10)} ${keyName.padEnd(18)} ${defaultModel}`)
|
|
15
|
+
.join("\n");
|
|
16
|
+
|
|
17
|
+
const usage = `Usage: tablefacts menu raw [page-or-image-url...] [options]
|
|
18
|
+
|
|
19
|
+
Reads a menu that is only pictures (one image per page) by transcribing each
|
|
20
|
+
page with a vision model, and replaces the menu in Supabase.
|
|
21
|
+
The URLs are pages to scan for menu images, or direct image URLs
|
|
22
|
+
(default: config.mjs). The key of the provider that reads the pages goes in
|
|
23
|
+
.env:
|
|
24
|
+
|
|
25
|
+
provider key default model
|
|
26
|
+
${providerTable}
|
|
27
|
+
|
|
28
|
+
Image menu options:
|
|
29
|
+
--list show the pictures found and stop; nothing is read or written
|
|
30
|
+
--only <pages> only these pages, numbered as --list shows them (e.g. 1,3-5)
|
|
31
|
+
--provider <name> who reads the pages: ${providerNames}
|
|
32
|
+
(default: MENU_VISION_PROVIDER in .env, else ${defaultProvider})
|
|
33
|
+
--model <id> model of that provider (default: see the table)
|
|
34
|
+
--min-width <px> ignore images declaring a smaller width (default: 500)
|
|
35
|
+
--refresh read the pages again instead of using the saved transcriptions`;
|
|
36
|
+
|
|
37
|
+
const shared = {
|
|
38
|
+
"min-width": { type: "string", default: "500" },
|
|
39
|
+
only: { type: "string" },
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
if (process.argv.includes("--list")) {
|
|
43
|
+
try {
|
|
44
|
+
const { values, positionals } = parseArgs({ args: process.argv.slice(2), allowPositionals: true, options: { ...shared, list: { type: "boolean" }, help: { type: "boolean", short: "h" } }, strict: false });
|
|
45
|
+
const { pages, chosen } = await findPages({ urls: positionals, only: values.only, minWidth: values["min-width"] });
|
|
46
|
+
console.log(`${pages.length} images found:\n`);
|
|
47
|
+
for (const page of chosen) console.log(` ${String(page.number).padStart(2)} ${page.url}${page.alt ? `\n ${page.alt.slice(0, 100)}` : ""}`);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
console.error(`\n${cliMessage(error, menuFlags)}`);
|
|
50
|
+
process.exitCode = exitCodeFor(error);
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
await runImport({
|
|
54
|
+
usage,
|
|
55
|
+
options: {
|
|
56
|
+
...shared,
|
|
57
|
+
provider: { type: "string" },
|
|
58
|
+
model: { type: "string" },
|
|
59
|
+
refresh: { type: "boolean" },
|
|
60
|
+
},
|
|
61
|
+
fetchMenu: ({ values, positionals }) =>
|
|
62
|
+
fetchImageMenu({
|
|
63
|
+
urls: positionals,
|
|
64
|
+
only: values.only,
|
|
65
|
+
provider: values.provider, // else MENU_VISION_PROVIDER, which loadEnv has read by now
|
|
66
|
+
model: values.model,
|
|
67
|
+
minWidth: values["min-width"],
|
|
68
|
+
refresh: values.refresh,
|
|
69
|
+
log: cliLog,
|
|
70
|
+
}),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// Menus that are only pictures, as a library: find the pages, have a vision
|
|
2
|
+
// model transcribe them (saved in the cache), normalize and import.
|
|
3
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { resolveEnv } from "../../lib/env.mjs";
|
|
6
|
+
import { optionError, TablefactsError } from "../../lib/errors.mjs";
|
|
7
|
+
import { normalizeLog } from "../../lib/log.mjs";
|
|
8
|
+
import { importMenu } from "../lib/import.mjs";
|
|
9
|
+
import defaultConfig from "./config.mjs";
|
|
10
|
+
import { normalizePages } from "./normalize.mjs";
|
|
11
|
+
import { discoverPages, downloadPage } from "./source.mjs";
|
|
12
|
+
import { defaultProvider, providers, readPage } from "./vision.mjs";
|
|
13
|
+
|
|
14
|
+
const providerNames = Object.keys(providers).join(", ");
|
|
15
|
+
|
|
16
|
+
/** "1,3-5" (or a list of numbers) to Set {1,3,4,5}. */
|
|
17
|
+
function pageSet(only) {
|
|
18
|
+
if (Array.isArray(only)) return new Set(only.map(Number));
|
|
19
|
+
const wanted = new Set();
|
|
20
|
+
for (const part of String(only).split(",")) {
|
|
21
|
+
const m = part.trim().match(/^(\d+)(?:-(\d+))?$/);
|
|
22
|
+
if (!m) throw optionError("only", `\`only\` "${only}" is not a list of pages such as 1,3-5.`);
|
|
23
|
+
for (let n = Number(m[1]); n <= Number(m[2] ?? m[1]); n++) wanted.add(n);
|
|
24
|
+
}
|
|
25
|
+
return wanted;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Every image found, and the numbered ones `only` selects. `config` defaults to config.mjs. */
|
|
29
|
+
export async function findPages({ urls = [], only, minWidth = 500, config = defaultConfig } = {}) {
|
|
30
|
+
if (!urls.length && !config.url) throw new TablefactsError("No menu page to read: pass urls, or set url in the raw menu config.", "ECONFIG");
|
|
31
|
+
const pages = await discoverPages(urls.length ? urls : [config.url], { minWidth: Number(minWidth) });
|
|
32
|
+
const wanted = only === undefined || only === "" ? null : pageSet(only);
|
|
33
|
+
const chosen = pages.map((page, i) => ({ ...page, number: i + 1 })).filter((page) => !wanted || wanted.has(page.number));
|
|
34
|
+
if (!chosen.length) throw optionError("only", `\`only\` ${only} matches none of the ${pages.length} pages found.`);
|
|
35
|
+
return { pages, chosen };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The pictures the CLI's `--list` prints, numbered. `only` narrows them.
|
|
40
|
+
* @param {import('../../lib/types.mjs').ListMenuImagesOptions} [options]
|
|
41
|
+
* @returns {Promise<import('../../lib/types.mjs').MenuImage[]>}
|
|
42
|
+
*/
|
|
43
|
+
export async function listMenuImages(options = {}) {
|
|
44
|
+
return (await findPages(options)).chosen;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Reads the pages and normalizes them: `{ menu, notes, title }`, ready for importMenu. */
|
|
48
|
+
export async function fetchImageMenu({ urls = [], only, provider, model, minWidth = 500, refresh = false, apiKey, env, projectDir, config = defaultConfig, log: logOption } = {}) {
|
|
49
|
+
const log = normalizeLog(logOption);
|
|
50
|
+
provider = String(provider || resolveEnv(env).MENU_VISION_PROVIDER || defaultProvider).toLowerCase();
|
|
51
|
+
if (!Object.hasOwn(providers, provider)) throw optionError("provider", `Unknown provider "${provider}" (from \`provider\` or MENU_VISION_PROVIDER). Use one of: ${providerNames}.`, "ECONFIG");
|
|
52
|
+
model ||= providers[provider].defaultModel;
|
|
53
|
+
|
|
54
|
+
const { chosen } = await findPages({ urls, only, minWidth, config });
|
|
55
|
+
const host = new URL(urls[0] ?? config.url).hostname;
|
|
56
|
+
|
|
57
|
+
const transcriptions = new Array(chosen.length);
|
|
58
|
+
let next = 0;
|
|
59
|
+
let reading = 0;
|
|
60
|
+
const worker = async () => {
|
|
61
|
+
while (next < chosen.length) {
|
|
62
|
+
const slot = next++;
|
|
63
|
+
const page = chosen[slot];
|
|
64
|
+
const local = await downloadPage(page, host, { projectDir });
|
|
65
|
+
const cached = join(local.dir, `${local.id}.json`);
|
|
66
|
+
const saved = refresh ? null : await readFile(cached, "utf8").catch((error) => {
|
|
67
|
+
if (error.code === "ENOENT") return null;
|
|
68
|
+
throw error;
|
|
69
|
+
});
|
|
70
|
+
if (saved !== null) {
|
|
71
|
+
transcriptions[slot] = JSON.parse(saved);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
reading++;
|
|
75
|
+
log(`Reading page ${page.number}...`);
|
|
76
|
+
transcriptions[slot] = await readPage(local, { provider, model, apiKey, env });
|
|
77
|
+
await writeFile(cached, JSON.stringify(transcriptions[slot], null, 2) + "\n");
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
await Promise.all(Array.from({ length: Math.min(3, chosen.length) }, worker));
|
|
81
|
+
|
|
82
|
+
const { menu, notes, currency } = normalizePages(transcriptions.map((t, i) => ({ ...t, number: chosen[i].number })), config);
|
|
83
|
+
notes.unshift(`Prices were read from pictures: check them against the menu before writing. Transcriptions are saved in .tablefacts/cache/${host}/ (edit a .json there to fix a page, then run again).`);
|
|
84
|
+
return {
|
|
85
|
+
menu,
|
|
86
|
+
notes,
|
|
87
|
+
title: `Image menu: ${chosen.length} pages from ${host} (${reading} read with ${providers[provider].label} ${model}, ${chosen.length - reading} from the saved transcriptions), prices in ${currency}`,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Reads the menu from pictures with a vision model and imports it. Takes importMenu's options too.
|
|
93
|
+
* @param {import('../../lib/types.mjs').ImportImageMenuOptions} [options]
|
|
94
|
+
* @returns {Promise<import('../../lib/types.mjs').ImportResult>}
|
|
95
|
+
*/
|
|
96
|
+
export async function importImageMenu({ urls, only, provider, model, minWidth, refresh, apiKey, env, config, ...importOptions } = {}) {
|
|
97
|
+
const fetched = await fetchImageMenu({ urls, only, provider, model, minWidth, refresh, apiKey, env, config, projectDir: importOptions.projectDir, log: importOptions.log });
|
|
98
|
+
return importMenu({ ...fetched, ...importOptions, env });
|
|
99
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// Transcribed menu pages (vision.mjs) to the shape lib/menu.mjs describes.
|
|
2
|
+
import { cleanText, isCurrency, matchKey, sectionName } from "../lib/menu.mjs";
|
|
3
|
+
import { TablefactsError } from "../../lib/errors.mjs";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* "$95.000" to 95000. `thousands` and `decimal` are the separators the menu
|
|
7
|
+
* prints (Colombia: "." and ","); `scale` multiplies the result for menus that
|
|
8
|
+
* print "95" meaning 95.000. Returns null for text with no number.
|
|
9
|
+
* @param {string} text
|
|
10
|
+
* @param {import('../../lib/types.mjs').PriceFormat} [format]
|
|
11
|
+
* @returns {number | null}
|
|
12
|
+
*/
|
|
13
|
+
export function parsePrice(text, { thousands = ".", decimal = ",", scale = 1 } = {}) {
|
|
14
|
+
let digits = String(text ?? "").replace(/[^\d.,]/g, "");
|
|
15
|
+
if (!/\d/.test(digits)) return null;
|
|
16
|
+
digits = digits.split(thousands).join("").replace(decimal, ".");
|
|
17
|
+
const value = Number(digits);
|
|
18
|
+
if (!Number.isFinite(value)) return null;
|
|
19
|
+
return Math.round(value * scale * 100) / 100;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const listNames = (names) => names.slice(0, 6).join(", ") + (names.length > 6 ? `, and ${names.length - 6} more` : "");
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* `pages` is the transcription of each page, in page order. A section with no
|
|
26
|
+
* title continues the section before it (a list that runs onto the next page),
|
|
27
|
+
* and sections with the same title are merged. Each section is placed in the
|
|
28
|
+
* category of `config.categories` that lists its group (food, drink), unless
|
|
29
|
+
* `config.placeIn` names it. An item with several priced columns becomes one
|
|
30
|
+
* product per column, "Name (Botella)", because a product has one price.
|
|
31
|
+
* @param {{ number?: number, notes?: string[], sections?: any[] }[]} pages transcriptions, one per page
|
|
32
|
+
* @param {import('../../lib/types.mjs').RawConfig} config
|
|
33
|
+
* @returns {{ menu: import('../../lib/types.mjs').Menu, notes: string[], currency: string }}
|
|
34
|
+
*/
|
|
35
|
+
export function normalizePages(pages, config) {
|
|
36
|
+
const notes = [];
|
|
37
|
+
const currency = String(config.currency ?? "").toUpperCase();
|
|
38
|
+
if (!isCurrency(currency)) throw new TablefactsError(`config.currency "${config.currency}" is not a currency code (such as COP or USD).`, "ECONFIG");
|
|
39
|
+
|
|
40
|
+
const categories = (config.categories ?? []).map((c) => ({ ...c, sections: new Map() }));
|
|
41
|
+
const bySlug = new Map(categories.map((c) => [c.slug, c]));
|
|
42
|
+
const byGroup = new Map(categories.flatMap((c) => (c.groups ?? []).map((group) => [group, c])));
|
|
43
|
+
const renames = new Map(Object.entries(config.sections ?? {}).map(([from, to]) => [matchKey(from), to]));
|
|
44
|
+
const placeIn = new Map(Object.entries(config.placeIn ?? {}).map(([from, slug]) => [matchKey(from), slug]));
|
|
45
|
+
const skip = new Set((config.skipSections ?? []).map(matchKey));
|
|
46
|
+
for (const slug of placeIn.values()) if (!bySlug.has(slug)) throw new TablefactsError(`config.placeIn points at "${slug}", which is not in config.categories.`, "ECONFIG");
|
|
47
|
+
|
|
48
|
+
const unparsed = [];
|
|
49
|
+
const unlabeled = [];
|
|
50
|
+
const skipped = [];
|
|
51
|
+
let previous = null; // the last section placed: { category, name }
|
|
52
|
+
|
|
53
|
+
pages.forEach((page, index) => {
|
|
54
|
+
const label = `page ${page.number ?? index + 1}`;
|
|
55
|
+
for (const note of page.notes ?? []) notes.push(`${label}: ${cleanText(note)}`);
|
|
56
|
+
for (const section of page.sections ?? []) {
|
|
57
|
+
const title = cleanText(section.title);
|
|
58
|
+
const items = (section.items ?? []).filter((item) => cleanText(item.name));
|
|
59
|
+
if (!items.length) continue;
|
|
60
|
+
|
|
61
|
+
let target;
|
|
62
|
+
if (!title) {
|
|
63
|
+
if (!previous) {
|
|
64
|
+
notes.push(`${label}: ${items.length} items come before any section heading, so they were skipped: ${listNames(items.map((i) => i.name))}.`);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
target = previous;
|
|
68
|
+
} else {
|
|
69
|
+
if (skip.has(matchKey(title))) {
|
|
70
|
+
skipped.push(title);
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const category = bySlug.get(placeIn.get(matchKey(title))) ?? byGroup.get(section.group);
|
|
74
|
+
if (!category) {
|
|
75
|
+
skipped.push(`${title} (${section.group})`);
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
target = { category, name: renames.get(matchKey(title)) ?? sectionName(title) };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const { category, name } = target;
|
|
82
|
+
if (!category.sections.has(name)) category.sections.set(name, []);
|
|
83
|
+
const products = category.sections.get(name);
|
|
84
|
+
for (const item of items) {
|
|
85
|
+
const itemName = cleanText(item.name);
|
|
86
|
+
const prices = (item.prices ?? []).map((p) => ({ value: parsePrice(p.text, config), label: cleanText(p.label) })).filter((p) => p.value !== null);
|
|
87
|
+
if (!prices.length) {
|
|
88
|
+
unparsed.push(`${itemName} (${label})`);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const variants = prices.length > 1 && prices.every((p) => p.label) ? prices : prices.slice(0, 1);
|
|
92
|
+
if (prices.length > 1 && variants.length === 1) unlabeled.push(itemName);
|
|
93
|
+
for (const { value, label: variant } of variants) {
|
|
94
|
+
products.push({
|
|
95
|
+
name: variants.length > 1 ? `${itemName} (${variant})` : itemName,
|
|
96
|
+
description: cleanText(item.description) || null,
|
|
97
|
+
price: value,
|
|
98
|
+
currency,
|
|
99
|
+
image_url: null,
|
|
100
|
+
recommended: false,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
previous = target;
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
if (skipped.length) notes.push(`Sections not placed in any category and left out: ${listNames([...new Set(skipped)])}. Add them to config.categories groups, config.placeIn or config.skipSections.`);
|
|
109
|
+
if (unparsed.length) notes.push(`${unparsed.length} items have no readable price and were left out: ${listNames(unparsed)}.`);
|
|
110
|
+
if (unlabeled.length) notes.push(`${unlabeled.length} items show several prices without saying what each is for, so only the first was imported: ${listNames(unlabeled)}.`);
|
|
111
|
+
|
|
112
|
+
const menu = categories
|
|
113
|
+
.map((c) => ({
|
|
114
|
+
slug: c.slug,
|
|
115
|
+
name: c.name,
|
|
116
|
+
sections: [...c.sections].filter(([, list]) => list.length).map(([name, list]) => ({ name, products: list })),
|
|
117
|
+
}))
|
|
118
|
+
.filter((c) => c.sections.length);
|
|
119
|
+
return { menu, notes, currency };
|
|
120
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// Finds the menu pages of a restaurant whose menu is only pictures (a gallery
|
|
2
|
+
// of JPGs on a WordPress page, a menu photographed page by page) and downloads
|
|
3
|
+
// them. Nothing here is specific to one site: it takes the large images of a
|
|
4
|
+
// page's HTML in document order. Sites that load their pictures from
|
|
5
|
+
// JavaScript show up as "no images": pass the image URLs directly instead.
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
import { TablefactsError } from "../../lib/errors.mjs";
|
|
10
|
+
import { workDirIn } from "../../lib/project.mjs";
|
|
11
|
+
|
|
12
|
+
const headers = { "user-agent": "cannario-menu-sync/1.0 (restaurant menu importer)" };
|
|
13
|
+
const IMAGE = /\.(jpe?g|png|webp)(\?|$)/i;
|
|
14
|
+
const MAX_BYTES = 5 * 1024 * 1024; // the Claude API refuses larger images; Gemini and Groq take 20 MB, so this is the tightest limit
|
|
15
|
+
|
|
16
|
+
async function get(url, what) {
|
|
17
|
+
let failure;
|
|
18
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
19
|
+
try {
|
|
20
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(30_000) });
|
|
21
|
+
if (res.ok) return res;
|
|
22
|
+
failure = Object.assign(new TablefactsError(`HTTP ${res.status} fetching ${what} ${url}`, "EFAILED"), { status: res.status });
|
|
23
|
+
if (res.status < 500) break;
|
|
24
|
+
} catch (error) {
|
|
25
|
+
failure = error;
|
|
26
|
+
}
|
|
27
|
+
await new Promise((resolve) => setTimeout(resolve, attempt * 1000));
|
|
28
|
+
}
|
|
29
|
+
throw failure;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const attribute = (tag, name) => tag.match(new RegExp(`\\s${name}\\s*=\\s*("([^"]*)"|'([^']*)')`, "i"))?.slice(2).find((v) => v !== undefined);
|
|
33
|
+
|
|
34
|
+
/** WordPress serves "name-600x1199.jpg" next to the original "name.jpg": ask for the original. */
|
|
35
|
+
const original = (url) => url.replace(/-\d{2,4}x\d{2,4}(\.\w+)(\?|$)/, "$1$2");
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The picture URLs of a page, in document order. Lazy-loaders keep the real
|
|
39
|
+
* address in data-orig-src / data-src and put a placeholder in src, so those
|
|
40
|
+
* come first. Images declaring a width under `minWidth` (logos, icons) and
|
|
41
|
+
* anything that is not a JPG, PNG or WebP (SVG logos) are skipped.
|
|
42
|
+
*/
|
|
43
|
+
export function findImages(html, pageUrl, { minWidth = 500 } = {}) {
|
|
44
|
+
const found = new Map();
|
|
45
|
+
for (const [tag] of html.matchAll(/<img\b[^>]*>/gi)) {
|
|
46
|
+
const raw = [attribute(tag, "data-orig-src"), attribute(tag, "data-src"), attribute(tag, "src")].find((v) => v && !v.startsWith("data:"));
|
|
47
|
+
if (!raw) continue;
|
|
48
|
+
let url;
|
|
49
|
+
try {
|
|
50
|
+
url = original(new URL(raw.replace(/&/g, "&"), pageUrl).href);
|
|
51
|
+
} catch {
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (!IMAGE.test(url)) continue;
|
|
55
|
+
const width = Number(attribute(tag, "width"));
|
|
56
|
+
if (width && width < minWidth) continue;
|
|
57
|
+
if (!found.has(url)) found.set(url, { url, alt: attribute(tag, "alt") ?? "" });
|
|
58
|
+
}
|
|
59
|
+
return [...found.values()];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Each argument is a page to scan or a direct image URL. */
|
|
63
|
+
export async function discoverPages(inputs, options) {
|
|
64
|
+
for (const input of inputs) {
|
|
65
|
+
try {
|
|
66
|
+
new URL(input);
|
|
67
|
+
} catch {
|
|
68
|
+
throw new TablefactsError(`"${input}" is not a URL.`, "EUSAGE");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const found = await Promise.allSettled(
|
|
72
|
+
inputs.map(async (input) => {
|
|
73
|
+
if (IMAGE.test(new URL(input).pathname)) return [{ url: input, alt: "" }];
|
|
74
|
+
const html = await (await get(input, "page")).text();
|
|
75
|
+
const images = findImages(html, input, options);
|
|
76
|
+
if (!images.length) {
|
|
77
|
+
throw new TablefactsError(
|
|
78
|
+
`No menu images found in ${input} (looked for JPG, PNG or WebP at least ${options?.minWidth ?? 500}px wide).\n` +
|
|
79
|
+
"If the page loads its pictures with JavaScript, open it, copy the image addresses and pass them instead; use `minWidth` to lower the size filter.",
|
|
80
|
+
"EFAILED",
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
return images;
|
|
84
|
+
}),
|
|
85
|
+
);
|
|
86
|
+
const failed = found.find((r) => r.status === "rejected");
|
|
87
|
+
if (failed) throw failed.reason; // the first input's failure, whichever finished first
|
|
88
|
+
return found.flatMap((r) => r.value);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export const pageId = (url) => createHash("sha1").update(url).digest("hex").slice(0, 10);
|
|
92
|
+
|
|
93
|
+
/** Downloads one page image into the cache (once) and returns where it is and what it is. */
|
|
94
|
+
export async function downloadPage(page, host, { projectDir } = {}) {
|
|
95
|
+
const dir = join(workDirIn(projectDir, "cache"), host);
|
|
96
|
+
await mkdir(dir, { recursive: true });
|
|
97
|
+
const id = pageId(page.url);
|
|
98
|
+
const meta = join(dir, `${id}.type`);
|
|
99
|
+
const types = { "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp" };
|
|
100
|
+
// The saved media type names the extension, so a cache hit is one read and one access check.
|
|
101
|
+
const saved = await readFile(meta, "utf8").catch(() => null);
|
|
102
|
+
if (saved && types[saved]) {
|
|
103
|
+
const known = join(dir, `${id}.${types[saved]}`);
|
|
104
|
+
if (await access(known).then(() => true, () => false)) return { id, dir, file: known, mediaType: saved };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const res = await get(page.url, "image");
|
|
108
|
+
const mediaType = (res.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();
|
|
109
|
+
if (!types[mediaType]) throw new TablefactsError(`${page.url} is ${mediaType || "of unknown type"}, not a JPG, PNG or WebP image.`, "EFAILED");
|
|
110
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
111
|
+
if (bytes.length > MAX_BYTES) throw new TablefactsError(`${page.url} is ${(bytes.length / 1048576).toFixed(1)} MB; menu pages are limited to 5 MB so that every provider can read them. Save a smaller copy.`, "EFAILED");
|
|
112
|
+
const file = join(dir, `${id}.${types[mediaType]}`);
|
|
113
|
+
await writeFile(file, bytes);
|
|
114
|
+
await writeFile(meta, mediaType);
|
|
115
|
+
return { id, dir, file, mediaType };
|
|
116
|
+
}
|