tablefacts 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -1
- package/CHANGELOG.md +61 -0
- package/README.md +67 -19
- package/package.json +14 -3
- package/src/lib/types.mjs +16 -4
- package/src/menu/README.md +63 -15
- package/src/menu/cluvi/config.mjs +6 -0
- package/src/menu/cluvi/import.mjs +6 -2
- package/src/menu/lib/db.mjs +81 -17
- package/src/menu/lib/import.mjs +20 -3
- package/src/menu/lib/run.mjs +18 -0
- package/src/menu/lib/tables.mjs +44 -0
- package/src/menu/raw/config.mjs +12 -2
- package/src/menu/raw/extract.mjs +36 -15
- package/src/menu/raw/images.mjs +109 -0
- package/src/menu/raw/import.mjs +330 -51
- package/src/menu/raw/normalize.mjs +15 -4
- package/src/menu/raw/pdf.mjs +330 -0
- package/src/menu/raw/pdfjs.mjs +57 -0
- package/src/menu/raw/source.mjs +9 -2
- package/src/menu/raw/vision.mjs +124 -65
- package/src/research/README.md +23 -13
- package/src/research/index.mjs +52 -17
- package/src/research/lib/merge.mjs +11 -10
- package/src/research/lib/report.mjs +72 -19
- package/src/research/lib/search.mjs +127 -0
- package/src/research/lib/social.mjs +137 -26
- package/src/research/lib/util.mjs +22 -0
- package/src/research/lib/website.mjs +3 -14
- package/src/research/research.mjs +12 -8
- package/types/lib/types.d.mts +70 -6
- package/types/menu/cluvi/config.d.mts +1 -0
- package/types/menu/cluvi/import.d.mts +2 -0
- package/types/menu/lib/db.d.mts +31 -3
- package/types/menu/lib/import.d.mts +1 -1
- package/types/menu/lib/run.d.mts +6 -0
- package/types/menu/lib/tables.d.mts +18 -0
- package/types/menu/raw/config.d.mts +2 -0
- package/types/menu/raw/images.d.mts +27 -0
- package/types/menu/raw/import.d.mts +36 -7
- package/types/menu/raw/normalize.d.mts +7 -1
- package/types/menu/raw/pdf.d.mts +98 -0
- package/types/menu/raw/pdfjs.d.mts +12 -0
- package/types/menu/raw/source.d.mts +2 -0
- package/types/menu/raw/vision.d.mts +11 -1
- package/types/research/lib/merge.d.mts +4 -1
- package/types/research/lib/report.d.mts +14 -1
- package/types/research/lib/search.d.mts +58 -0
- package/types/research/lib/social.d.mts +32 -31
- package/types/research/lib/util.d.mts +5 -0
- package/types/research/lib/website.d.mts +20 -20
package/src/menu/lib/db.mjs
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
// Writes a menu (see lib/menu.mjs) into the Supabase tables of
|
|
2
2
|
// supabase/migrations/0001_menu.sql over a direct Postgres connection.
|
|
3
|
+
// The tables are the restaurant's own set (lib/tables.mjs): on a shared
|
|
4
|
+
// database each restaurant has a `<prefix>_menu_*` set, and assertTarget()
|
|
5
|
+
// refuses to guess before anything is written.
|
|
3
6
|
//
|
|
4
7
|
// The connection string (SUPABASE_DB_URL) is a privileged credential: it
|
|
5
8
|
// bypasses row level security, which only allows public reads. It lives in
|
|
@@ -8,6 +11,7 @@
|
|
|
8
11
|
import { randomUUID } from "node:crypto";
|
|
9
12
|
import { resolveEnv } from "../../lib/env.mjs";
|
|
10
13
|
import { TablefactsError } from "../../lib/errors.mjs";
|
|
14
|
+
import { menuTables, validateTablePrefix } from "./tables.mjs";
|
|
11
15
|
|
|
12
16
|
/** Opens the connection. `label` names the target without the credentials. */
|
|
13
17
|
export async function connect(url, { env } = {}) {
|
|
@@ -15,7 +19,7 @@ export async function connect(url, { env } = {}) {
|
|
|
15
19
|
if (!/^postgres(ql)?:\/\//i.test(url ?? "")) {
|
|
16
20
|
throw new TablefactsError(
|
|
17
21
|
"SUPABASE_DB_URL is not set to a postgres:// connection string.\n" +
|
|
18
|
-
"Copy .env.example to .env and paste the pooler URL from the Supabase dashboard (Connect >
|
|
22
|
+
"Copy .env.example to .env and paste the pooler URL from the Supabase dashboard (Connect > Session pooler).",
|
|
19
23
|
"ECONFIG",
|
|
20
24
|
);
|
|
21
25
|
}
|
|
@@ -42,31 +46,89 @@ export async function connect(url, { env } = {}) {
|
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
/** Says what is wrong when the migration has not been applied to this database. */
|
|
45
|
-
function explain(error) {
|
|
49
|
+
function explain(error, tables) {
|
|
46
50
|
if (error.code === "42P01") {
|
|
47
|
-
error.message =
|
|
51
|
+
error.message = `The menu tables do not exist. Apply supabase/migrations/0001_menu.sql to this database first (expected ${Object.values(tables).join(", ")}).`;
|
|
48
52
|
}
|
|
49
53
|
return error;
|
|
50
54
|
}
|
|
51
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Refuses to touch another site's tables before anything is written.
|
|
58
|
+
*
|
|
59
|
+
* Checks the three target tables exist (pointing at the migration when they do not). Then, when
|
|
60
|
+
* no prefix is set, lists the other restaurants' `<prefix>_menu_categories` tables: any of them
|
|
61
|
+
* means this database is shared, so the import stops unless the caller confirmed it targets the
|
|
62
|
+
* one unprefixed set with `allowUnprefixed`. A whole-menu `replaceAll` is refused in that case
|
|
63
|
+
* whatever the flags say, because deleting every unprefixed row could destroy another site.
|
|
64
|
+
* Only reads; returns `{ prefix, tables, others }`.
|
|
65
|
+
* @param {any} client
|
|
66
|
+
* @param {{ tablePrefix?: string, allowUnprefixed?: boolean, replaceAll?: boolean }} [options]
|
|
67
|
+
*/
|
|
68
|
+
export async function assertTarget(client, { tablePrefix = "", allowUnprefixed = false, replaceAll = false } = {}) {
|
|
69
|
+
const prefix = validateTablePrefix(tablePrefix);
|
|
70
|
+
const tables = menuTables(prefix);
|
|
71
|
+
const short = Object.values(tables).map((name) => name.replace(/^public\./, ""));
|
|
72
|
+
|
|
73
|
+
const { rows } = await client.query(
|
|
74
|
+
"select table_name from information_schema.tables where table_schema = 'public' and table_name = any($1::text[])",
|
|
75
|
+
[short],
|
|
76
|
+
);
|
|
77
|
+
const present = new Set(rows.map((row) => row.table_name));
|
|
78
|
+
const missing = short.filter((name) => !present.has(name));
|
|
79
|
+
if (missing.length) {
|
|
80
|
+
throw new TablefactsError(
|
|
81
|
+
`The menu tables do not exist. Apply supabase/migrations/0001_menu.sql to this database first (expected ${Object.values(tables).join(", ")}; missing ${missing.map((name) => `public.${name}`).join(", ")}).`,
|
|
82
|
+
"ECONFIG",
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (prefix) return { prefix, tables, others: [] };
|
|
87
|
+
|
|
88
|
+
// `_` is a LIKE wildcard, so the query is only a coarse filter: it can also return names such as
|
|
89
|
+
// "xmenu_categories". The endsWith check keeps only real `<prefix>_menu_categories` tables (and
|
|
90
|
+
// excludes the unprefixed "menu_categories" itself); without it a table like "xmenu_categories"
|
|
91
|
+
// would be mistaken for another restaurant's set.
|
|
92
|
+
const existing = await client.query(
|
|
93
|
+
"select table_name from information_schema.tables where table_schema = 'public' and table_name like '%_menu_categories' order by table_name",
|
|
94
|
+
);
|
|
95
|
+
const others = existing.rows.map((row) => row.table_name).filter((name) => name.endsWith("_menu_categories"));
|
|
96
|
+
if (!others.length) return { prefix, tables, others };
|
|
97
|
+
|
|
98
|
+
if (replaceAll) {
|
|
99
|
+
throw new TablefactsError(
|
|
100
|
+
`Refusing to replace the whole menu: this database has other restaurants' menu tables (${others.join(", ")}). Set \`tablePrefix\` in your config so the delete targets only this restaurant's tables.`,
|
|
101
|
+
"ECONFIG",
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
if (!allowUnprefixed) {
|
|
105
|
+
throw new TablefactsError(
|
|
106
|
+
`This database has other restaurants' menu tables (${others.join(", ")}). Set tablePrefix in your config or pass --table-prefix so the import cannot touch the wrong tables.`,
|
|
107
|
+
"ECONFIG",
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return { prefix, tables, others };
|
|
111
|
+
}
|
|
112
|
+
|
|
52
113
|
/** What an import would replace: the categories it writes, or the whole menu. */
|
|
53
|
-
export async function inspect(client, menu, { replaceAll = false } = {}) {
|
|
114
|
+
export async function inspect(client, menu, { replaceAll = false, tablePrefix = "" } = {}) {
|
|
115
|
+
const tables = menuTables(tablePrefix);
|
|
54
116
|
const slugs = replaceAll ? null : menu.map((category) => category.slug);
|
|
55
117
|
try {
|
|
56
118
|
const { rows } = await client.query(
|
|
57
119
|
`select count(distinct c.id)::int as categories, count(p.id)::int as products
|
|
58
|
-
from
|
|
59
|
-
left join
|
|
60
|
-
left join
|
|
120
|
+
from ${tables.categories} c
|
|
121
|
+
left join ${tables.sections} s on s.category_id = c.id
|
|
122
|
+
left join ${tables.products} p on p.section_id = s.id
|
|
61
123
|
where $1::text[] is null or c.slug = any($1)`,
|
|
62
124
|
[slugs],
|
|
63
125
|
);
|
|
64
126
|
const kept = slugs
|
|
65
|
-
? (await client.query(
|
|
127
|
+
? (await client.query(`select slug from ${tables.categories} where slug <> all($1) order by sort_order`, [slugs])).rows.map((row) => row.slug)
|
|
66
128
|
: [];
|
|
67
129
|
return { ...rows[0], kept };
|
|
68
130
|
} catch (error) {
|
|
69
|
-
throw explain(error);
|
|
131
|
+
throw explain(error, tables);
|
|
70
132
|
}
|
|
71
133
|
}
|
|
72
134
|
|
|
@@ -75,9 +137,11 @@ export async function inspect(client, menu, { replaceAll = false } = {}) {
|
|
|
75
137
|
* one and a failure changes nothing. By default only the categories in `menu`
|
|
76
138
|
* are replaced (deleting a category cascades to its sections and products);
|
|
77
139
|
* `replaceAll` empties the menu first. Ids are generated here so the rows can
|
|
78
|
-
* be inserted in bulk, a column at a time.
|
|
140
|
+
* be inserted in bulk, a column at a time. `tablePrefix` is the restaurant's
|
|
141
|
+
* own table set; the delete is scoped to it, never to another site's tables.
|
|
79
142
|
*/
|
|
80
|
-
export async function replaceMenu(client, menu, { replaceAll = false } = {}) {
|
|
143
|
+
export async function replaceMenu(client, menu, { replaceAll = false, tablePrefix = "" } = {}) {
|
|
144
|
+
const tables = menuTables(tablePrefix);
|
|
81
145
|
const categories = menu.map((c, i) => ({ id: randomUUID(), slug: c.slug, name: c.name, sort_order: i + 1 }));
|
|
82
146
|
const sections = menu.flatMap((c, ci) =>
|
|
83
147
|
c.sections.map((s, si) => ({ id: randomUUID(), category_id: categories[ci].id, name: s.name, sort_order: si + 1, products: s.products })),
|
|
@@ -88,20 +152,20 @@ export async function replaceMenu(client, menu, { replaceAll = false } = {}) {
|
|
|
88
152
|
|
|
89
153
|
try {
|
|
90
154
|
await client.query("begin");
|
|
91
|
-
if (replaceAll) await client.query(
|
|
92
|
-
else await client.query(
|
|
155
|
+
if (replaceAll) await client.query(`delete from ${tables.categories}`);
|
|
156
|
+
else await client.query(`delete from ${tables.categories} where slug = any($1::text[])`, [column(categories, "slug")]);
|
|
93
157
|
|
|
94
158
|
const inserted = [
|
|
95
159
|
await client.query(
|
|
96
|
-
|
|
160
|
+
`insert into ${tables.categories} (id, slug, name, sort_order) select * from unnest($1::uuid[], $2::text[], $3::text[], $4::int[])`,
|
|
97
161
|
["id", "slug", "name", "sort_order"].map((key) => column(categories, key)),
|
|
98
162
|
),
|
|
99
163
|
await client.query(
|
|
100
|
-
|
|
164
|
+
`insert into ${tables.sections} (id, category_id, name, sort_order) select * from unnest($1::uuid[], $2::uuid[], $3::text[], $4::int[])`,
|
|
101
165
|
["id", "category_id", "name", "sort_order"].map((key) => column(sections, key)),
|
|
102
166
|
),
|
|
103
167
|
await client.query(
|
|
104
|
-
`insert into
|
|
168
|
+
`insert into ${tables.products} (section_id, name, description, price, currency, image_url, recommended, sort_order)
|
|
105
169
|
select * from unnest($1::uuid[], $2::text[], $3::text[], $4::numeric[], $5::text[], $6::text[], $7::boolean[], $8::int[])`,
|
|
106
170
|
["section_id", "name", "description", "price", "currency", "image_url", "recommended", "sort_order"].map((key) => column(products, key)),
|
|
107
171
|
),
|
|
@@ -114,6 +178,6 @@ export async function replaceMenu(client, menu, { replaceAll = false } = {}) {
|
|
|
114
178
|
return { categories: categories.length, sections: sections.length, products: products.length };
|
|
115
179
|
} catch (error) {
|
|
116
180
|
await client.query("rollback").catch(() => {});
|
|
117
|
-
throw explain(error);
|
|
181
|
+
throw explain(error, tables);
|
|
118
182
|
}
|
|
119
183
|
}
|
package/src/menu/lib/import.mjs
CHANGED
|
@@ -7,8 +7,9 @@ import { resolveEnv } from "../../lib/env.mjs";
|
|
|
7
7
|
import { optionError, TablefactsError } from "../../lib/errors.mjs";
|
|
8
8
|
import { normalizeLog } from "../../lib/log.mjs";
|
|
9
9
|
import { projectRoot, resolveIn } from "../../lib/project.mjs";
|
|
10
|
-
import { connect, inspect, replaceMenu } from "./db.mjs";
|
|
10
|
+
import { assertTarget, connect, inspect, replaceMenu } from "./db.mjs";
|
|
11
11
|
import { countMenu, validateMenu } from "./menu.mjs";
|
|
12
|
+
import { validateTablePrefix } from "./tables.mjs";
|
|
12
13
|
|
|
13
14
|
/** Things the site needs that this menu does not give it, read from the template's own files. */
|
|
14
15
|
export async function templateHints(menu, { projectDir } = {}) {
|
|
@@ -67,6 +68,9 @@ export async function importMenu({
|
|
|
67
68
|
json,
|
|
68
69
|
replaceAll = false,
|
|
69
70
|
force = false,
|
|
71
|
+
tablePrefix = "",
|
|
72
|
+
allowUnprefixed = false,
|
|
73
|
+
yes = false,
|
|
70
74
|
databaseUrl,
|
|
71
75
|
env,
|
|
72
76
|
projectDir,
|
|
@@ -74,6 +78,8 @@ export async function importMenu({
|
|
|
74
78
|
} = {}) {
|
|
75
79
|
const log = normalizeLog(logOption);
|
|
76
80
|
databaseUrl ??= resolveEnv(env).SUPABASE_DB_URL;
|
|
81
|
+
// Before anything is opened: an invalid prefix must never reach the SQL builder.
|
|
82
|
+
const prefix = validateTablePrefix(tablePrefix);
|
|
77
83
|
let client;
|
|
78
84
|
try {
|
|
79
85
|
validateMenu(menu);
|
|
@@ -98,14 +104,25 @@ export async function importMenu({
|
|
|
98
104
|
log("\nDry run: SUPABASE_DB_URL is not set, so the database was not checked. Nothing was written.", "warn");
|
|
99
105
|
return result;
|
|
100
106
|
}
|
|
107
|
+
// A whole-menu replace empties every row of the target tables: it needs an explicit confirmation.
|
|
108
|
+
if (replaceAll && !yes && !dryRun) {
|
|
109
|
+
throw optionError("yes", "`replaceAll` empties every row in the target tables and needs confirmation: pass `yes`.", "EUSAGE");
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Only the fields that are set: this keeps the default call shape unchanged for callers and tests.
|
|
113
|
+
const scope = { replaceAll: !!replaceAll, ...(prefix ? { tablePrefix: prefix } : {}) };
|
|
101
114
|
|
|
102
|
-
const scope = { replaceAll: !!replaceAll };
|
|
103
115
|
let label;
|
|
104
116
|
({ client, label } = await connect(databaseUrl, { env }));
|
|
117
|
+
const target = await assertTarget(client, { tablePrefix: prefix, allowUnprefixed: !!allowUnprefixed, replaceAll: !!replaceAll });
|
|
105
118
|
const current = await inspect(client, menu, scope);
|
|
106
|
-
result.database = { label, current };
|
|
119
|
+
result.database = { label, tables: Object.values(target.tables), current };
|
|
107
120
|
log(`\nDatabase ${label}`);
|
|
121
|
+
log(` Target tables: ${Object.values(target.tables).join(", ")}`);
|
|
108
122
|
log(` ${scope.replaceAll ? "Replaces the whole menu" : `Replaces the categories this import writes`}: ${current.categories} categories and ${current.products} products now, ${totals.categories} and ${totals.products} after.`);
|
|
123
|
+
if (scope.replaceAll) {
|
|
124
|
+
log(` --replace-all will empty ${target.tables.categories}, ${target.tables.sections} and ${target.tables.products} (${current.categories} categories, ${current.products} products).`, "warn");
|
|
125
|
+
}
|
|
109
126
|
if (current.kept.length) log(` Left untouched (not part of this import): ${current.kept.join(", ")}. Use \`replaceAll\` to remove them.`, "warn");
|
|
110
127
|
|
|
111
128
|
if (current.products > 0 && totals.products < current.products / 2 && !force) {
|
package/src/menu/lib/run.mjs
CHANGED
|
@@ -12,6 +12,12 @@ export const menuFlags = {
|
|
|
12
12
|
provider: "--provider",
|
|
13
13
|
model: "--model",
|
|
14
14
|
minWidth: "--min-width",
|
|
15
|
+
imageDir: "--images <folder>",
|
|
16
|
+
imageBaseUrl: "--image-base-url <url>",
|
|
17
|
+
imageBoxes: "--image-boxes <auto|always>",
|
|
18
|
+
tablePrefix: "--table-prefix <prefix>",
|
|
19
|
+
allowUnprefixed: "--allow-unprefixed",
|
|
20
|
+
yes: "--yes",
|
|
15
21
|
replaceAll: "--replace-all",
|
|
16
22
|
force: "--force",
|
|
17
23
|
dryRun: "--dry-run",
|
|
@@ -34,7 +40,10 @@ const commonOptions = `
|
|
|
34
40
|
Options:
|
|
35
41
|
--dry-run extract and check, show what would change, write nothing
|
|
36
42
|
--json <file> also save the extracted menu as JSON
|
|
43
|
+
--table-prefix <prefix> this restaurant's table prefix (e.g. makibar_); overrides the config
|
|
44
|
+
--allow-unprefixed write the unprefixed menu_* tables on a single-restaurant database
|
|
37
45
|
--replace-all replace the whole menu, not only the categories in this import
|
|
46
|
+
--yes confirm --replace-all (it empties the target tables)
|
|
38
47
|
--force write even if the import has far fewer products than it replaces
|
|
39
48
|
-h, --help show this help
|
|
40
49
|
|
|
@@ -54,7 +63,10 @@ export async function runImport({ usage, options = {}, fetchMenu }) {
|
|
|
54
63
|
options: {
|
|
55
64
|
"dry-run": { type: "boolean" },
|
|
56
65
|
json: { type: "string" },
|
|
66
|
+
"table-prefix": { type: "string" },
|
|
67
|
+
"allow-unprefixed": { type: "boolean" },
|
|
57
68
|
"replace-all": { type: "boolean" },
|
|
69
|
+
yes: { type: "boolean" },
|
|
58
70
|
force: { type: "boolean" },
|
|
59
71
|
help: { type: "boolean", short: "h" },
|
|
60
72
|
...options,
|
|
@@ -67,12 +79,18 @@ export async function runImport({ usage, options = {}, fetchMenu }) {
|
|
|
67
79
|
|
|
68
80
|
loadEnv();
|
|
69
81
|
const fetched = await fetchMenu({ values, positionals });
|
|
82
|
+
// `tablePrefix` is only passed when the flag is given, so it overrides the source's config
|
|
83
|
+
// instead of replacing it with undefined.
|
|
84
|
+
const overrides = values["table-prefix"] === undefined ? {} : { tablePrefix: values["table-prefix"] };
|
|
70
85
|
await importMenu({
|
|
71
86
|
...fetched,
|
|
72
87
|
dryRun: !!values["dry-run"],
|
|
73
88
|
json: values.json,
|
|
74
89
|
replaceAll: !!values["replace-all"],
|
|
75
90
|
force: !!values.force,
|
|
91
|
+
allowUnprefixed: !!values["allow-unprefixed"],
|
|
92
|
+
yes: !!values.yes,
|
|
93
|
+
...overrides,
|
|
76
94
|
log: cliLog,
|
|
77
95
|
});
|
|
78
96
|
} catch (error) {
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// The table names a menu import writes. Several restaurants can share one
|
|
2
|
+
// Supabase database, so each restaurant's tables carry its own prefix
|
|
3
|
+
// (`cannario_menu_categories`, `makibar_menu_categories`, ...). The prefix is
|
|
4
|
+
// restaurant-specific and lives in the source's `config.mjs`; the CLI's
|
|
5
|
+
// `--table-prefix` overrides it for one run.
|
|
6
|
+
import { optionError } from "../../lib/errors.mjs";
|
|
7
|
+
|
|
8
|
+
// The only prefixes accepted: empty, or lowercase letters, digits and
|
|
9
|
+
// underscores ending in "_". A value that passed this allow-list is safe to
|
|
10
|
+
// build the table names from; nothing else is ever put into the SQL text.
|
|
11
|
+
const PREFIX = /^[a-z][a-z0-9_]*_$|^$/;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* `tablePrefix` when it is empty or a safe `<name>_` prefix (an ECONFIG error otherwise).
|
|
15
|
+
* @param {string} [tablePrefix]
|
|
16
|
+
* @returns {string}
|
|
17
|
+
*/
|
|
18
|
+
export function validateTablePrefix(tablePrefix) {
|
|
19
|
+
const value = tablePrefix ?? "";
|
|
20
|
+
if (!PREFIX.test(value)) {
|
|
21
|
+
throw optionError(
|
|
22
|
+
"tablePrefix",
|
|
23
|
+
`${JSON.stringify(value)} is not a valid table prefix: \`tablePrefix\` must be empty or lowercase letters, digits and underscores ending in "_", such as "makibar_".`,
|
|
24
|
+
"ECONFIG",
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The three `public` menu tables for a prefix, e.g.
|
|
32
|
+
* `{ categories: "public.makibar_menu_categories", sections: "public.makibar_menu_sections", products: "public.makibar_menu_products" }`.
|
|
33
|
+
* The default (empty prefix) is the unprefixed `public.menu_*` set.
|
|
34
|
+
* @param {string} [tablePrefix]
|
|
35
|
+
* @returns {{ categories: string, sections: string, products: string }}
|
|
36
|
+
*/
|
|
37
|
+
export function menuTables(tablePrefix) {
|
|
38
|
+
const prefix = validateTablePrefix(tablePrefix);
|
|
39
|
+
return {
|
|
40
|
+
categories: `public.${prefix}menu_categories`,
|
|
41
|
+
sections: `public.${prefix}menu_sections`,
|
|
42
|
+
products: `public.${prefix}menu_products`,
|
|
43
|
+
};
|
|
44
|
+
}
|
package/src/menu/raw/config.mjs
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
// Everything restaurant-specific about the image-menu import. Edit this file,
|
|
2
2
|
// not the other scripts, when the menu is organised differently.
|
|
3
3
|
export default {
|
|
4
|
-
//
|
|
5
|
-
//
|
|
4
|
+
// This restaurant's tables in a shared Supabase database: the import writes
|
|
5
|
+
// public.mombasa_menu_categories / _menu_sections / _menu_products. Change
|
|
6
|
+
// it (or pass `--table-prefix`) when importing another restaurant; leave it
|
|
7
|
+
// empty only when this restaurant owns the unprefixed menu_* tables.
|
|
8
|
+
tablePrefix: "mombasa_",
|
|
9
|
+
|
|
10
|
+
// The page that shows the menu pictures, a direct image URL, or a PDF file
|
|
11
|
+
// path/URL. `tablefacts menu raw <url> [<url>...]` overrides it for one run.
|
|
6
12
|
url: "https://www.mombasa.co/carta-restaurante-espanol/",
|
|
7
13
|
|
|
8
14
|
// Currency of the prices, an ISO code. Colombian menus print pesos as
|
|
@@ -14,6 +20,10 @@ export default {
|
|
|
14
20
|
// for 95.000).
|
|
15
21
|
scale: 1,
|
|
16
22
|
|
|
23
|
+
// Resolution a PDF page is rendered at before its printed product photos are
|
|
24
|
+
// screenshotted (2 means twice the page's size). Only used with `--images`.
|
|
25
|
+
imageScale: 2,
|
|
26
|
+
|
|
17
27
|
// Each section the model reads is tagged food or drink (or other, which is
|
|
18
28
|
// left out), and goes into the category that lists its group. The site's
|
|
19
29
|
// default categories are cocina and bar.
|
package/src/menu/raw/extract.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// Imports a restaurant's menu from pictures of its pages
|
|
3
|
-
// menu tables. See src/menu/README.md. Run from the project's folder:
|
|
2
|
+
// Imports a restaurant's menu from pictures of its pages, or from a PDF, into
|
|
3
|
+
// the Supabase menu tables. See src/menu/README.md. Run from the project's folder:
|
|
4
4
|
// tablefacts menu raw --list
|
|
5
5
|
// tablefacts menu raw --dry-run
|
|
6
|
+
// tablefacts menu raw carta.pdf --images ./photos --image-base-url https://cdn.example.com/menu/ --dry-run
|
|
6
7
|
import { parseArgs } from "node:util";
|
|
7
8
|
import { cliLog, menuFlags, runImport } from "../lib/run.mjs";
|
|
8
9
|
import { cliMessage, exitCodeFor } from "../../lib/errors.mjs";
|
|
@@ -14,25 +15,32 @@ const providerTable = Object.entries(providers)
|
|
|
14
15
|
.map(([name, { keyName, defaultModel }]) => ` ${name.padEnd(10)} ${keyName.padEnd(18)} ${defaultModel}`)
|
|
15
16
|
.join("\n");
|
|
16
17
|
|
|
17
|
-
const usage = `Usage: tablefacts menu raw [page-or-image-url...] [options]
|
|
18
|
+
const usage = `Usage: tablefacts menu raw [page-or-image-url...] [pdf-file-or-url...] [options]
|
|
18
19
|
|
|
19
|
-
Reads a menu that is only pictures (one image per page)
|
|
20
|
-
page
|
|
21
|
-
The URLs are pages to scan for
|
|
22
|
-
|
|
23
|
-
.
|
|
20
|
+
Reads a menu that is only pictures (one image per page) or a PDF, and replaces
|
|
21
|
+
the menu in Supabase. A PDF page is transcribed from its text when it has one,
|
|
22
|
+
and from a render of the page when it is a scan. The URLs are pages to scan for
|
|
23
|
+
menu images, or direct image URLs; an argument ending in .pdf is a PDF file
|
|
24
|
+
(local path) or URL. Default: config.mjs. The key of the provider that reads the
|
|
25
|
+
pages goes in .env:
|
|
24
26
|
|
|
25
27
|
provider key default model
|
|
26
28
|
${providerTable}
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
--list show the
|
|
30
|
+
Picture and PDF options:
|
|
31
|
+
--list show the pages found and stop; nothing is read or written
|
|
30
32
|
--only <pages> only these pages, numbered as --list shows them (e.g. 1,3-5)
|
|
31
33
|
--provider <name> who reads the pages: ${providerNames}
|
|
32
34
|
(default: MENU_VISION_PROVIDER in .env, else ${defaultProvider})
|
|
33
35
|
--model <id> model of that provider (default: see the table)
|
|
34
36
|
--min-width <px> ignore images declaring a smaller width (default: 500)
|
|
35
|
-
--refresh read the pages again instead of using the saved transcriptions
|
|
37
|
+
--refresh read the pages again instead of using the saved transcriptions
|
|
38
|
+
--images <folder> also save the dish photos printed on a PDF page
|
|
39
|
+
--image-base-url <url> https folder you will publish them at; fills image_url
|
|
40
|
+
--image-boxes <auto|always> auto (default) reads the page as text and matches
|
|
41
|
+
photos by position; always reads every page as a picture so
|
|
42
|
+
the model boxes every dish's photo (a vision call per page)
|
|
43
|
+
`;
|
|
36
44
|
|
|
37
45
|
const shared = {
|
|
38
46
|
"min-width": { type: "string", default: "500" },
|
|
@@ -41,10 +49,17 @@ const shared = {
|
|
|
41
49
|
|
|
42
50
|
if (process.argv.includes("--list")) {
|
|
43
51
|
try {
|
|
44
|
-
const { values, positionals } = parseArgs({ args: process.argv.slice(2), allowPositionals: true, options: { ...shared, list: { type: "boolean" }, help: { type: "boolean", short: "h" } }, strict: false });
|
|
45
|
-
const { pages, chosen } = await findPages({ urls: positionals, only: values.only, minWidth: values["min-width"] });
|
|
46
|
-
|
|
47
|
-
|
|
52
|
+
const { values, positionals } = parseArgs({ args: process.argv.slice(2), allowPositionals: true, options: { ...shared, list: { type: "boolean" }, help: { type: "boolean", short: "h" }, images: { type: "string" }, "image-base-url": { type: "string" }, "image-boxes": { type: "string" } }, strict: false });
|
|
53
|
+
const { pages, chosen, close } = await findPages({ urls: positionals, only: values.only, minWidth: values["min-width"] });
|
|
54
|
+
try {
|
|
55
|
+
console.log(`${pages.length} pages found:\n`);
|
|
56
|
+
for (const page of chosen) {
|
|
57
|
+
const detail = page.kind === "pdf" ? (page.chars >= 40 ? `${page.chars} characters of text` : "scanned; read as a picture") : page.alt?.slice(0, 100) ?? "";
|
|
58
|
+
console.log(` ${String(page.number).padStart(2)} ${page.url}${detail ? `\n ${detail}` : ""}`);
|
|
59
|
+
}
|
|
60
|
+
} finally {
|
|
61
|
+
await close();
|
|
62
|
+
}
|
|
48
63
|
} catch (error) {
|
|
49
64
|
console.error(`\n${cliMessage(error, menuFlags)}`);
|
|
50
65
|
process.exitCode = exitCodeFor(error);
|
|
@@ -57,6 +72,9 @@ if (process.argv.includes("--list")) {
|
|
|
57
72
|
provider: { type: "string" },
|
|
58
73
|
model: { type: "string" },
|
|
59
74
|
refresh: { type: "boolean" },
|
|
75
|
+
images: { type: "string" },
|
|
76
|
+
"image-base-url": { type: "string" },
|
|
77
|
+
"image-boxes": { type: "string" },
|
|
60
78
|
},
|
|
61
79
|
fetchMenu: ({ values, positionals }) =>
|
|
62
80
|
fetchImageMenu({
|
|
@@ -66,6 +84,9 @@ if (process.argv.includes("--list")) {
|
|
|
66
84
|
model: values.model,
|
|
67
85
|
minWidth: values["min-width"],
|
|
68
86
|
refresh: values.refresh,
|
|
87
|
+
imageDir: values.images,
|
|
88
|
+
imageBaseUrl: values["image-base-url"],
|
|
89
|
+
imageBoxes: values["image-boxes"],
|
|
69
90
|
log: cliLog,
|
|
70
91
|
}),
|
|
71
92
|
});
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// Places the product photos found on a menu page onto the products read from it.
|
|
2
|
+
//
|
|
3
|
+
// Two page kinds feed this:
|
|
4
|
+
// - a digital PDF page: pdf.mjs reports where each image sits on the page
|
|
5
|
+
// (`placed` crops with a rectangle), and the product's name is found among
|
|
6
|
+
// the page's text items to pick the nearest photo;
|
|
7
|
+
// - a scanned page: the vision model was asked for each item's photo box
|
|
8
|
+
// (`direct` crops already tied to an item name).
|
|
9
|
+
//
|
|
10
|
+
// Matching is pure, so it is tested with made-up pages; raw/import.mjs saves the
|
|
11
|
+
// crops and turns them into `image_url`s.
|
|
12
|
+
import { matchKey } from "../lib/menu.mjs";
|
|
13
|
+
|
|
14
|
+
const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
15
|
+
|
|
16
|
+
/** The gap between two rectangles, 0 when they touch or overlap. */
|
|
17
|
+
const gap = (a, b) => {
|
|
18
|
+
const dx = Math.max(a.x0 - b.x1, b.x0 - a.x1, 0);
|
|
19
|
+
const dy = Math.max(a.y0 - b.y1, b.y0 - a.y1, 0);
|
|
20
|
+
return Math.hypot(dx, dy);
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const listShort = (names) => names.slice(0, 6).join(", ") + (names.length > 6 ? `, and ${names.length - 6} more` : "");
|
|
24
|
+
|
|
25
|
+
const textRect = (item) => ({
|
|
26
|
+
x0: item.x ?? 0,
|
|
27
|
+
y0: item.y ?? 0,
|
|
28
|
+
x1: (item.x ?? 0) + Math.abs(item.width ?? 0),
|
|
29
|
+
y1: (item.y ?? 0) + Math.abs(item.height ?? 0),
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The page's text item that best matches `name`: an exact match first, then a
|
|
34
|
+
* line that contains it (preferring the shortest such line). null when no line
|
|
35
|
+
* holds the name, which happens when a heading was drawn as an outline.
|
|
36
|
+
*/
|
|
37
|
+
export function findNameItem(items, name) {
|
|
38
|
+
const wanted = matchKey(name);
|
|
39
|
+
if (!wanted) return null;
|
|
40
|
+
// Whole words only, so "Ana" does not match inside "Banana".
|
|
41
|
+
const bounded = new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRegExp(wanted)}(?=$|[^\\p{L}\\p{N}])`, "u");
|
|
42
|
+
let best = null;
|
|
43
|
+
for (const item of items ?? []) {
|
|
44
|
+
const key = matchKey(item.str);
|
|
45
|
+
const exact = key === wanted;
|
|
46
|
+
const contains = !exact && bounded.test(key);
|
|
47
|
+
if (!exact && !contains) continue;
|
|
48
|
+
const score = exact ? -1 : key.length;
|
|
49
|
+
if (!best || score < best.score) best = { item, score };
|
|
50
|
+
}
|
|
51
|
+
return best?.item ?? null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* One crop per product, in page order. `pages` maps a page number to
|
|
56
|
+
* `{ width, items, placed, direct }` (see the file comment). A crop is used
|
|
57
|
+
* once, so a single photo between two items goes to the closer one. Returns
|
|
58
|
+
* `{ matches, notes }`; matches carry the placement and the crop it won.
|
|
59
|
+
* @param {{ page: number, name: string, box: number[] | null, products: any[] }[]} placements
|
|
60
|
+
* @param {Map<number, any>} pages
|
|
61
|
+
* @returns {{ matches: { placement: any, crop: any }[], notes: string[] }}
|
|
62
|
+
*/
|
|
63
|
+
export function matchPlacements(placements, pages) {
|
|
64
|
+
const notes = [];
|
|
65
|
+
const matches = [];
|
|
66
|
+
const used = new Set();
|
|
67
|
+
const ambiguous = [];
|
|
68
|
+
|
|
69
|
+
for (const placement of placements) {
|
|
70
|
+
const page = pages.get(placement.page);
|
|
71
|
+
if (!page) continue;
|
|
72
|
+
let crop = null;
|
|
73
|
+
|
|
74
|
+
// A scanned page already tied each crop to the model's item name.
|
|
75
|
+
const direct = (page.direct ?? []).filter((c) => !used.has(c.id));
|
|
76
|
+
if (direct.length) crop = direct.find((c) => matchKey(c.name) === matchKey(placement.name)) ?? null;
|
|
77
|
+
|
|
78
|
+
if (!crop && (page.placed ?? []).length) {
|
|
79
|
+
const item = findNameItem(page.items, placement.name);
|
|
80
|
+
if (item) {
|
|
81
|
+
const rect = textRect(item);
|
|
82
|
+
const limit = (page.width ?? 0) * 0.2; // a photo farther than a fifth of the page is not this item's
|
|
83
|
+
const ranked = (page.placed ?? [])
|
|
84
|
+
.filter((c) => !used.has(c.id))
|
|
85
|
+
.map((c) => ({ c, d: gap(rect, c.rect) }))
|
|
86
|
+
.sort((a, b) => a.d - b.d);
|
|
87
|
+
if (ranked.length && ranked[0].d <= limit) {
|
|
88
|
+
crop = ranked[0].c;
|
|
89
|
+
// A near tie means the same photo could belong to either item: say so.
|
|
90
|
+
if (ranked[1] && ranked[0].d > 0 && Math.abs(ranked[1].d - ranked[0].d) < (page.width ?? 0) * 0.01) {
|
|
91
|
+
ambiguous.push(`${placement.name} (page ${placement.page})`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (crop) {
|
|
98
|
+
used.add(crop.id);
|
|
99
|
+
matches.push({ placement, crop });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// A dish without a photo is normal; only a photo with no dish is worth saying.
|
|
104
|
+
const allCrops = [...pages.values()].flatMap((page) => [...(page.placed ?? []), ...(page.direct ?? [])]);
|
|
105
|
+
const unused = allCrops.filter((crop) => !used.has(crop.id));
|
|
106
|
+
if (unused.length) notes.push(`${unused.length} printed photo(s) could not be matched to an item and were left out.`);
|
|
107
|
+
if (ambiguous.length) notes.push(`A printed photo sat between items and may be attached to the wrong one: ${listShort(ambiguous)}.`);
|
|
108
|
+
return { matches, notes };
|
|
109
|
+
}
|