tablefacts 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -1
- package/CHANGELOG.md +61 -0
- package/README.md +67 -19
- package/package.json +14 -3
- package/src/lib/types.mjs +16 -4
- package/src/menu/README.md +63 -15
- package/src/menu/cluvi/config.mjs +6 -0
- package/src/menu/cluvi/import.mjs +6 -2
- package/src/menu/lib/db.mjs +81 -17
- package/src/menu/lib/import.mjs +20 -3
- package/src/menu/lib/run.mjs +18 -0
- package/src/menu/lib/tables.mjs +44 -0
- package/src/menu/raw/config.mjs +12 -2
- package/src/menu/raw/extract.mjs +36 -15
- package/src/menu/raw/images.mjs +109 -0
- package/src/menu/raw/import.mjs +330 -51
- package/src/menu/raw/normalize.mjs +15 -4
- package/src/menu/raw/pdf.mjs +330 -0
- package/src/menu/raw/pdfjs.mjs +57 -0
- package/src/menu/raw/source.mjs +9 -2
- package/src/menu/raw/vision.mjs +124 -65
- package/src/research/README.md +23 -13
- package/src/research/index.mjs +52 -17
- package/src/research/lib/merge.mjs +11 -10
- package/src/research/lib/report.mjs +72 -19
- package/src/research/lib/search.mjs +127 -0
- package/src/research/lib/social.mjs +137 -26
- package/src/research/lib/util.mjs +22 -0
- package/src/research/lib/website.mjs +3 -14
- package/src/research/research.mjs +12 -8
- package/types/lib/types.d.mts +70 -6
- package/types/menu/cluvi/config.d.mts +1 -0
- package/types/menu/cluvi/import.d.mts +2 -0
- package/types/menu/lib/db.d.mts +31 -3
- package/types/menu/lib/import.d.mts +1 -1
- package/types/menu/lib/run.d.mts +6 -0
- package/types/menu/lib/tables.d.mts +18 -0
- package/types/menu/raw/config.d.mts +2 -0
- package/types/menu/raw/images.d.mts +27 -0
- package/types/menu/raw/import.d.mts +36 -7
- package/types/menu/raw/normalize.d.mts +7 -1
- package/types/menu/raw/pdf.d.mts +98 -0
- package/types/menu/raw/pdfjs.d.mts +12 -0
- package/types/menu/raw/source.d.mts +2 -0
- package/types/menu/raw/vision.d.mts +11 -1
- package/types/research/lib/merge.d.mts +4 -1
- package/types/research/lib/report.d.mts +14 -1
- package/types/research/lib/search.d.mts +58 -0
- package/types/research/lib/social.d.mts +32 -31
- package/types/research/lib/util.d.mts +5 -0
- package/types/research/lib/website.d.mts +20 -20
package/.env.example
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
# Supabase > Connect > Session pooler (this tool holds one connection and reads information_schema).
|
|
2
|
+
# Bypasses row level security: keep it here, never in a browser-exposed variable.
|
|
1
3
|
SUPABASE_DB_URL=SUPABASE_DB_POOLER_URL
|
|
2
4
|
|
|
3
|
-
# Menus that are only pictures (tablefacts menu raw) are read by a vision model. Fill in
|
|
5
|
+
# Menus that are only pictures or a PDF (tablefacts menu raw) are read by a vision model. Fill in
|
|
4
6
|
# the key of the provider you use: Claude, Gemini or Groq
|
|
5
7
|
ANTHROPIC_API_KEY=
|
|
6
8
|
GEMINI_API_KEY=
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,66 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
### Menu (`tablefacts menu raw`)
|
|
6
|
+
|
|
7
|
+
- The source now reads **PDF menus** as well as pictures. An argument ending in `.pdf` is a local file
|
|
8
|
+
(resolved against the project) or an http(s) URL. A page that carries its own text is transcribed from that
|
|
9
|
+
text (a two-column layout is read column by column); a page with no text layer (a scan) is rendered and read
|
|
10
|
+
like a picture. `--list` shows which each page is. New optional dependencies: `pdfjs-dist` for reading,
|
|
11
|
+
`@napi-rs/canvas` for rendering (loaded lazily, with an `EDEPENDENCY` error naming the install command when
|
|
12
|
+
missing).
|
|
13
|
+
- New options `--images <folder>` and `--image-base-url <url>` (`imageDir`, `imageBaseUrl` on
|
|
14
|
+
`importImageMenu`): the dish photos printed on a PDF page are screenshotted and saved. When the page places
|
|
15
|
+
each photo separately they are matched to the nearest printed dish name; when it does not (a flattened export,
|
|
16
|
+
vector art, a scan) the model is asked for each item's photo box instead, so photos can come from any page.
|
|
17
|
+
The new `--image-boxes auto|always` option controls this: `auto` (default) only reads a page as a picture when
|
|
18
|
+
it has no separately placed photo, while `always` reads every page as a picture so the model boxes each dish.
|
|
19
|
+
With a base URL each product's `image_url` is filled; without it the crops are saved and `image_url` stays
|
|
20
|
+
empty.
|
|
21
|
+
- `findPages`, `listMenuImages` and `normalizePages` now also know about PDF pages: `listMenuImages` reports
|
|
22
|
+
`kind` and `chars`, and `normalizePages` returns `placements` (which products each item built) so photos can
|
|
23
|
+
be attached.
|
|
24
|
+
|
|
25
|
+
## 0.2.0
|
|
26
|
+
|
|
27
|
+
- The menu importers no longer write hardcoded `public.menu_*` tables. A restaurant's config (or the
|
|
28
|
+
new `--table-prefix` flag) selects the prefixed table set, so several restaurants can share one
|
|
29
|
+
Supabase database without touching each other's tables. The shipped Cluvi and picture configs now
|
|
30
|
+
set `cannario_`/`mombasa_`, so those two commands target their prefixed tables by default.
|
|
31
|
+
- New safety guards before any write: the importer checks all three target tables exist, and with no
|
|
32
|
+
prefix stops when another restaurant's prefixed `_menu_categories` tables are present
|
|
33
|
+
(`--allow-unprefixed` overrides it for a single-restaurant database).
|
|
34
|
+
- `replaceAll` names the exact tables it will empty and requires `--yes`; it never runs on the
|
|
35
|
+
unprefixed tables while another restaurant's tables are present. `inspect()` reports the resolved
|
|
36
|
+
tables and the row counts it would delete.
|
|
37
|
+
- `--dry-run` runs the same checks and sends only `select`s (no write transaction).
|
|
38
|
+
- New options `tablePrefix`, `allowUnprefixed` and `yes` on `importMenu`/`importCluvi`/
|
|
39
|
+
`importImageMenu`; new `menuTables`/`validateTablePrefix` helpers in `src/menu/lib/tables.mjs`; the
|
|
40
|
+
result's `database` now carries `tables`.
|
|
41
|
+
- Docs recommend the Supabase **Session pooler** URL and explain the prefix convention; the full model,
|
|
42
|
+
checks, errors and a migration checklist are in [`docs/MENU_TABLE_PREFIX.md`](docs/MENU_TABLE_PREFIX.md).
|
|
43
|
+
|
|
44
|
+
### Research (`tablefacts research`)
|
|
45
|
+
|
|
46
|
+
- New key-free web-search source (DuckDuckGo Lite, with the HTML endpoint as fallback). It runs only when
|
|
47
|
+
Google Places and OpenStreetMap find no place or no website, and its candidate links (website, Instagram,
|
|
48
|
+
TripAdvisor, Maps, link-in-bio) feed the existing "sources find each other" chain, so a bare name and place
|
|
49
|
+
no longer produces an empty report.
|
|
50
|
+
- Instagram is read from the public share card, the web profile API and oEmbed, and its bio/counts are filled
|
|
51
|
+
from a search-engine snippet when a surface answers without them. Requests identify honestly: no login and
|
|
52
|
+
no user-agent impersonation; a walled profile stays blocked.
|
|
53
|
+
- A blocked TripAdvisor link is kept and stated at the top of the report, and its URL slug gives a name and
|
|
54
|
+
location hint without fetching the page.
|
|
55
|
+
- The CLI summary separates facts discovered from sources, values only echoed from your input, and
|
|
56
|
+
low-confidence guesses. Guesses are marked "low (guess)" and left blank in `setup-answers.txt`, so setup
|
|
57
|
+
keeps the template's placeholder.
|
|
58
|
+
- Similar-name runs (e.g. "Makibar" vs "Maki Bar") are noted, and `.tablefacts/research/latest.json` points at
|
|
59
|
+
the newest report; the pointer is only written in the default output layout.
|
|
60
|
+
- The report leads "Ask the client" with one prioritized next question, flattens web-sourced text so it cannot
|
|
61
|
+
forge report structure, and warns when Google returns several places with the same name.
|
|
62
|
+
- New shared `decodeHtml` and `instagramHandle` helpers in the research lib, reused by the website reader.
|
|
63
|
+
|
|
3
64
|
## 0.1.0
|
|
4
65
|
|
|
5
66
|
First release as an npm library.
|
package/README.md
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
Data extraction tools for restaurant sites, shared by every Cannario template. Give it a restaurant's name and
|
|
4
4
|
place and it gathers the public facts; point it at an Instagram or TripAdvisor page and it downloads the photos;
|
|
5
|
-
point it at a Cluvi menu
|
|
5
|
+
point it at a Cluvi menu, a PDF menu, or pictures of a paper menu and it loads the menu into Supabase.
|
|
6
|
+
|
|
7
|
+
Research needs no key: Google Places and OpenStreetMap run first, and a key-free web search (DuckDuckGo) fills in
|
|
8
|
+
candidate links when they find no place or no website, so a bare name and place still produces a useful report.
|
|
6
9
|
|
|
7
10
|
Everything works two ways: as a **command line** (`tablefacts <tool>`) and as a **JavaScript library**
|
|
8
11
|
(`import { research } from 'tablefacts'`). Both do the same work; the CLI is a thin layer over the functions.
|
|
@@ -13,7 +16,7 @@ Everything works two ways: as a **command line** (`tablefacts <tool>`) and as a
|
|
|
13
16
|
| Instagram photos | `tablefacts photos instagram` | `downloadInstagram()` | [src/instagram](src/instagram/README.md) |
|
|
14
17
|
| TripAdvisor photos | `tablefacts photos tripadvisor` | `downloadTripadvisor()` | [src/tripadvisor](src/tripadvisor/README.md) |
|
|
15
18
|
| Menu from Cluvi | `tablefacts menu cluvi` | `importCluvi()` | [src/menu](src/menu/README.md) |
|
|
16
|
-
| Menu from pictures | `tablefacts menu raw` | `importImageMenu()` | [src/menu](src/menu/README.md) |
|
|
19
|
+
| Menu from pictures or a PDF | `tablefacts menu raw` | `importImageMenu()` | [src/menu](src/menu/README.md) |
|
|
17
20
|
|
|
18
21
|
Contents: [Requirements](#requirements) · [Install](#install) · [Quick start](#quick-start) ·
|
|
19
22
|
[Command line](#command-line) · [Configuration](#configuration) · [Library](#use-it-as-a-library) ·
|
|
@@ -24,6 +27,9 @@ Contents: [Requirements](#requirements) · [Install](#install) · [Quick start](
|
|
|
24
27
|
|
|
25
28
|
- **Node.js 24 or newer** (`engines` in `package.json`). The tools use the built-in `fetch` and `util.parseArgs`.
|
|
26
29
|
- **Playwright** (optional) for the photo tools and `research --render`: `npm install --save-dev playwright`.
|
|
30
|
+
- **pdfjs-dist** and **@napi-rs/canvas** (optional) for PDF menus with `tablefacts menu raw`:
|
|
31
|
+
`npm install pdfjs-dist @napi-rs/canvas`. Only reading a PDF page's text needs `pdfjs-dist`; rendering a scanned
|
|
32
|
+
page or a product photo also needs the canvas package (it comes with `@napi-rs/canvas` on Windows, macOS and Linux).
|
|
27
33
|
- **Microsoft Edge on Windows** for the photo tools. They attach to a normal Edge window because the sites
|
|
28
34
|
guard their pages with bot checks that automated browsers fail (see the photo tool READMEs). Starting Edge
|
|
29
35
|
for you only works with Edge installed in its usual `Program Files` folder; elsewhere, start it yourself with
|
|
@@ -38,7 +44,8 @@ npx tablefacts --help
|
|
|
38
44
|
```
|
|
39
45
|
|
|
40
46
|
Playwright is an **optional** peer dependency. Without it the photo tools and `research --render` stop with an
|
|
41
|
-
`EDEPENDENCY` error that tells you to install it; everything else works.
|
|
47
|
+
`EDEPENDENCY` error that tells you to install it; everything else works. `pdfjs-dist` (and `@napi-rs/canvas` for
|
|
48
|
+
scans and product photos) is likewise optional and only `menu raw` with a PDF needs it.
|
|
42
49
|
|
|
43
50
|
## Quick start
|
|
44
51
|
|
|
@@ -76,8 +83,8 @@ npx tablefacts photos tripadvisor --out ./photos --dry-run <restaurant-link>
|
|
|
76
83
|
| `--no-google` | Skip Google even if `GOOGLE_PLACES_API_KEY` is set |
|
|
77
84
|
| `--out <dir>` | Output folder (default `.tablefacts/research/<slug>`; relative paths are relative to the project) |
|
|
78
85
|
|
|
79
|
-
Writes `report.md`, `profile.json` and `setup-answers.txt
|
|
80
|
-
[src/research/README.md](src/research/README.md).
|
|
86
|
+
Writes `report.md`, `profile.json` and `setup-answers.txt`, and keeps `latest.json` next to the `<slug>` folders
|
|
87
|
+
pointing at the newest run. Details, sources and trust rules: [src/research/README.md](src/research/README.md).
|
|
81
88
|
|
|
82
89
|
### `photos instagram`
|
|
83
90
|
|
|
@@ -116,12 +123,24 @@ Both share these options (and replace the menu in Supabase unless `--dry-run`):
|
|
|
116
123
|
| --- | --- |
|
|
117
124
|
| `--dry-run` | Extract and check, show what would change, write nothing |
|
|
118
125
|
| `--json <file>` | Also save the extracted menu as JSON |
|
|
119
|
-
| `--
|
|
126
|
+
| `--table-prefix <prefix>` | This restaurant's tables in a shared database, e.g. `makibar_` (overrides the config). Required when other restaurants' tables exist |
|
|
127
|
+
| `--allow-unprefixed` | Override the shared-database check and write the unprefixed `menu_*` tables, only when this restaurant owns them |
|
|
128
|
+
| `--replace-all` | Replace the whole menu, not only the categories in this import. Requires `--yes` (except with `--dry-run`) |
|
|
129
|
+
| `--yes` | Confirm `--replace-all`, which empties the target tables (`<prefix>menu_categories`, `<prefix>menu_sections`, `<prefix>menu_products`) |
|
|
120
130
|
| `--force` | Write even if the import has fewer than half the products it replaces |
|
|
131
|
+
| `--images <folder>` | With a PDF, also save the dish photos printed on its pages |
|
|
132
|
+
| `--image-base-url <url>` | https folder those photos will be published at; fills each product's `image_url` |
|
|
133
|
+
| `--image-boxes <auto\|always>` | How photos are found. `auto` (default) reads the page as text and matches separately placed photos by position, using the model's boxes only when the page has none; `always` reads every page as a picture so the model boxes every dish's photo |
|
|
121
134
|
|
|
122
135
|
`tablefacts menu cluvi [menu-url] [--service on_table|delivery|take_away] [--lang es]`
|
|
123
136
|
|
|
124
|
-
`tablefacts menu raw [page-or-image-url...] [--list] [--only 1,3-5] [--provider anthropic|gemini|groq] [--model <id>] [--min-width 500] [--refresh]`
|
|
137
|
+
`tablefacts menu raw [page-or-image-url...] [menu.pdf...] [--list] [--only 1,3-5] [--provider anthropic|gemini|groq] [--model <id>] [--min-width 500] [--refresh] [--images <folder>] [--image-base-url <url>] [--image-boxes auto|always]`
|
|
138
|
+
|
|
139
|
+
For a PDF, an argument ending in `.pdf` is a local file (resolved against the project) or an http(s) URL. A page
|
|
140
|
+
with text is transcribed from that text (a two-column layout is read column by column); a scanned page is rendered
|
|
141
|
+
and read like a picture. `--images` also saves the printed product photos: a page that places each photo
|
|
142
|
+
separately is matched by dish name, and a page that does not (a flattened export, vector art, a scan) is read as a
|
|
143
|
+
picture so the model can box each dish — `--image-boxes always` does that on every page.
|
|
125
144
|
|
|
126
145
|
Both read the restaurant-specific part (URL, category mapping, currency) from a `config.mjs` next to the tool.
|
|
127
146
|
**The shipped configs hold another restaurant's values: edit them first.** See [src/menu/README.md](src/menu/README.md).
|
|
@@ -143,7 +162,7 @@ environment win. [.env.example](.env.example) is the template.
|
|
|
143
162
|
|
|
144
163
|
| Variable | Used by | Meaning |
|
|
145
164
|
| --- | --- | --- |
|
|
146
|
-
| `SUPABASE_DB_URL` | `menu cluvi`, `menu raw` | Postgres connection string (Supabase *Transaction pooler*
|
|
165
|
+
| `SUPABASE_DB_URL` | `menu cluvi`, `menu raw` | Postgres connection string, with the real password. Use the **Session pooler** URL (Supabase dashboard > Connect > Session pooler): this tool holds one connection and reads `information_schema`, which the *Transaction pooler* is not meant for. **Bypasses row level security:** keep it in `.env`, never in a browser-exposed variable |
|
|
147
166
|
| `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `GROQ_API_KEY` | `menu raw` | Key of the vision provider that reads the pictures |
|
|
148
167
|
| `MENU_VISION_PROVIDER` | `menu raw` | Provider when `--provider` is not given: `anthropic` (default), `gemini` or `groq` |
|
|
149
168
|
| `GOOGLE_PLACES_API_KEY` | `research` | Optional. Enables Google Maps (Places API (New)); without it OpenStreetMap and the web pages still work |
|
|
@@ -188,10 +207,10 @@ default output folder, browser profiles, the menu transcription cache, and where
|
|
|
188
207
|
| `research({ name, location, country, website, instagram, tripadvisor, linktree, photos, render, google, googleKey, env, out, projectDir, log })` | Gathers public facts; writes `profile.json`, `report.md`, `setup-answers.txt`. Returns `{ profile, notes, photos, outDir, files }` | `GOOGLE_PLACES_API_KEY` is optional; `render` needs Playwright |
|
|
189
208
|
| `downloadInstagram({ links, file, out, profile, pages, cdp, edgeDir, viaGoogle, browser, userDataDir, headed, dryRun, debug, projectDir, log })` | Downloads post or profile photos (give `links`, `file` or `profile`). Returns `{ saved, skipped, failed: [{ item, reason }], found? }` | Playwright |
|
|
190
209
|
| `downloadTripadvisor({ links, out, cdp, edgeDir, max, dryRun, debug, projectDir, log })` | Downloads a restaurant page's photos. Same result | Playwright |
|
|
191
|
-
| `importMenu({ menu, notes, title, dryRun, json, replaceAll, force, databaseUrl, env, projectDir, log })` | Validates a menu and writes it to Supabase. Returns `{ totals, notes, written, dryRun, database }`, `database` being `{ label, current: { categories, products, kept } }` or `null` when it was not reached | `SUPABASE_DB_URL` unless `dryRun` |
|
|
210
|
+
| `importMenu({ menu, notes, title, dryRun, json, tablePrefix, allowUnprefixed, replaceAll, yes, force, databaseUrl, env, projectDir, log })` | Validates a menu and writes it to Supabase. Returns `{ totals, notes, written, dryRun, database }`, `database` being `{ label, tables, current: { categories, products, kept } }` or `null` when it was not reached | `SUPABASE_DB_URL` unless `dryRun` |
|
|
192
211
|
| `importCluvi({ url, service, lang, config, ...importMenu options })` | Reads a Cluvi menu, then `importMenu` | `SUPABASE_DB_URL` unless `dryRun` |
|
|
193
|
-
| `importImageMenu({ urls, only, provider, model, minWidth, refresh, apiKey, config, ...importMenu options })` | Transcribes menu pictures with a
|
|
194
|
-
| `listMenuImages({ urls, only, minWidth, config })` | Lists the
|
|
212
|
+
| `importImageMenu({ urls, only, provider, model, minWidth, refresh, imageDir, imageBaseUrl, imageBoxes, apiKey, config, ...importMenu options })` | Transcribes menu pictures or a PDF with a model, then `importMenu`. A PDF page uses its text when it has one, a render when it is a scan. `imageDir` also saves the printed product photos (`imageBaseUrl` fills their links; `imageBoxes` is `'auto'` or `'always'`) | A vision key (see [Configuration](#configuration)); a PDF needs `pdfjs-dist` (`@napi-rs/canvas` too for scans or photos) |
|
|
213
|
+
| `listMenuImages({ urls, only, minWidth, projectDir, config })` | Lists the pages found (pictures or PDF pages, numbered as `--list` shows) | none |
|
|
195
214
|
| `loadEnv({ projectDir, files, env })` | Loads `.env` files into `env` (default `process.env`); returns the files it read | none |
|
|
196
215
|
|
|
197
216
|
Also exported: `validateMenu`, `countMenu`, `parsePrice`, `normalizePages`, `providers`, `defaultProvider`,
|
|
@@ -227,7 +246,7 @@ Failures the library raises on purpose throw `TablefactsError`, which has a `cod
|
|
|
227
246
|
| --- | --- |
|
|
228
247
|
| `EUSAGE` | A missing or invalid argument (no `out`, no valid links, no `name`). CLIs exit with `2` |
|
|
229
248
|
| `ECONFIG` | Missing configuration: database URL, API key, unknown provider, no menu URL |
|
|
230
|
-
| `EDEPENDENCY` | An optional dependency is not installed (Playwright) |
|
|
249
|
+
| `EDEPENDENCY` | An optional dependency is not installed (Playwright, or `pdfjs-dist`/`@napi-rs/canvas` for a PDF menu) |
|
|
231
250
|
| `EFAILED` | A source failed or was blocked, data did not validate, or an import was refused (invalid menu, no products, far fewer products than it replaces) |
|
|
232
251
|
|
|
233
252
|
When the error is about one option, `err.option` names it (`'out'`, `'links'`, `'force'`...) and the message writes
|
|
@@ -255,8 +274,9 @@ folder tablefacts is installed in, so one install serves every template.
|
|
|
255
274
|
| What | Where |
|
|
256
275
|
| --- | --- |
|
|
257
276
|
| `.env` | `<project>/.env`, then `<project>/data/.env` (older templates) |
|
|
258
|
-
| Research output | `<project>/.tablefacts/research/<slug
|
|
277
|
+
| Research output | `<project>/.tablefacts/research/<slug>/`, with `latest.json` next to the folders |
|
|
259
278
|
| Menu transcription cache and downloaded page images | `<project>/.tablefacts/cache/<host>/` |
|
|
279
|
+
| Product photos saved from a PDF | The `--images` folder, or `<project>/.tablefacts/menu-images` |
|
|
260
280
|
| Fallback browser profile | `<project>/.tablefacts/instagram-profile` |
|
|
261
281
|
| Photos | The `--out` folder you give |
|
|
262
282
|
| Edge profile for the photo tools | `C:\ig-edge` (`--edge-dir` changes it) |
|
|
@@ -266,7 +286,7 @@ hosts) are read from the template's `frontend/src/content` and skipped silently
|
|
|
266
286
|
|
|
267
287
|
### Supabase tables
|
|
268
288
|
|
|
269
|
-
`menu cluvi` and `menu raw` write to
|
|
289
|
+
`menu cluvi` and `menu raw` write to three tables, which your project must already have (the Cannario
|
|
270
290
|
templates ship the migration `supabase/migrations/0001_menu.sql`):
|
|
271
291
|
|
|
272
292
|
| Table | Columns written |
|
|
@@ -275,9 +295,31 @@ templates ship the migration `supabase/migrations/0001_menu.sql`):
|
|
|
275
295
|
| `public.menu_sections` | `id` (uuid), `category_id`, `name`, `sort_order` |
|
|
276
296
|
| `public.menu_products` | `section_id`, `name`, `description`, `price` (numeric), `currency`, `image_url`, `recommended`, `sort_order` |
|
|
277
297
|
|
|
298
|
+
**Several restaurants can share one Supabase database.** Each restaurant then keeps its own prefixed copies
|
|
299
|
+
of those three tables — `cannario_menu_categories`, `cannario_menu_sections`, `cannario_menu_products`, and
|
|
300
|
+
likewise `mombasa_menu_*` and `makibar_menu_*` — while the unprefixed `menu_*` tables belong to a single,
|
|
301
|
+
older site. Set the restaurant's prefix in its `config.mjs` (`tablePrefix: "makibar_"`) or pass
|
|
302
|
+
`--table-prefix makibar_`; the flag overrides the config. The prefix must be lowercase letters, digits and
|
|
303
|
+
underscores ending in `_`, or empty.
|
|
304
|
+
|
|
305
|
+
Before it writes anything, the importer **checks which menu tables exist**:
|
|
306
|
+
|
|
307
|
+
- all three target tables must exist, or it stops and names the ones missing (apply the migration);
|
|
308
|
+
- with **no prefix**, it lists every `public` table ending in `_menu_categories`. If another restaurant's
|
|
309
|
+
prefixed set exists, it stops with `ECONFIG` and names those tables, so an import cannot silently write
|
|
310
|
+
into the wrong site. `--allow-unprefixed` overrides that check, and is only correct when this restaurant
|
|
311
|
+
really owns the unprefixed tables.
|
|
312
|
+
|
|
278
313
|
The write is **one transaction** and replaces only the categories in the import (their sections and products go
|
|
279
|
-
with them), so re-running never duplicates.
|
|
280
|
-
|
|
314
|
+
with them), so re-running never duplicates. The run prints the resolved target tables and the row counts it
|
|
315
|
+
would delete. `--replace-all` empties the whole menu and is refused without `--yes`; it also names the exact
|
|
316
|
+
tables it will empty, and never runs on the unprefixed tables while another restaurant's tables are present.
|
|
317
|
+
An import with no products, or with under half the products it replaces, is refused unless `--force`. Always
|
|
318
|
+
run `--dry-run` first: it runs the same checks and prints what would be written without opening a write
|
|
319
|
+
transaction (only `select`s are sent).
|
|
320
|
+
|
|
321
|
+
The full model — prefix rules, every check, the error codes and how to migrate a restaurant to its own prefixed
|
|
322
|
+
tables — is in [docs/MENU_TABLE_PREFIX.md](docs/MENU_TABLE_PREFIX.md).
|
|
281
323
|
|
|
282
324
|
## Limits
|
|
283
325
|
|
|
@@ -285,6 +327,12 @@ products, or with under half the products it replaces, is refused unless `--forc
|
|
|
285
327
|
past bot protection: a blocked source is reported, not worked around.
|
|
286
328
|
- Cluvi has no public API; the importer calls the two JSON endpoints its web app uses, which can change.
|
|
287
329
|
- Prices read from pictures can be wrong. Read the dry run against the original before writing.
|
|
330
|
+
- A PDF page with text is read as text; a scan is rendered and read as a picture, so small print can still be
|
|
331
|
+
misread. A two-column page is read column by column when the layout is clear, but an unusual layout is
|
|
332
|
+
flattened into single lines, so check the dry run. Product photos are screenshots cropped from the page: a
|
|
333
|
+
separately placed photo is matched to the nearest dish, and when the page has none (a flattened export, or
|
|
334
|
+
vector art) the model is asked for each dish's photo box instead — pass `--image-boxes always` to do that on
|
|
335
|
+
every page too. Boxes are approximate, so check the dry run.
|
|
288
336
|
- Only download photos the restaurant owns or has allowed you to use. Guest photos on TripAdvisor and
|
|
289
337
|
Instagram belong to their authors.
|
|
290
338
|
|
|
@@ -302,9 +350,9 @@ and its declarations fails the tests. Document options with JSDoc in `src/lib/ty
|
|
|
302
350
|
CI (`.github/workflows/ci.yml`) runs the tests on Ubuntu and Windows with Node 24 and checks `npm pack --dry-run`
|
|
303
351
|
on every push to `main` and every pull request. Dependabot (`.github/dependabot.yml`) keeps dependencies current.
|
|
304
352
|
|
|
305
|
-
**Release:**
|
|
306
|
-
|
|
307
|
-
|
|
353
|
+
**Release:** publishing is manual. Bump `version` in `package.json`, add the entry to `CHANGELOG.md`, commit, then
|
|
354
|
+
run `npm publish` from a clean checkout of that commit. Publishing is not automated in CI; `prepublishOnly` builds
|
|
355
|
+
`types/` and runs the tests first. Optionally tag the release commit (`git tag v0.1.1 && git push origin v0.1.1`).
|
|
308
356
|
|
|
309
357
|
For how the code is organised and how to add a source or a tool, see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md).
|
|
310
358
|
|
package/package.json
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tablefacts",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Data extraction tools for restaurant sites: public-facts research, photos (Instagram, TripAdvisor) and menu import",
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Data extraction tools for restaurant sites: public-facts research, photos (Instagram, TripAdvisor) and menu import from Cluvi, pictures or PDFs",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"restaurant",
|
|
7
7
|
"menu",
|
|
8
|
+
"pdf",
|
|
8
9
|
"scraper",
|
|
9
10
|
"tripadvisor",
|
|
10
11
|
"instagram",
|
|
@@ -61,15 +62,25 @@
|
|
|
61
62
|
"pg": "^8.23.1"
|
|
62
63
|
},
|
|
63
64
|
"devDependencies": {
|
|
64
|
-
"@
|
|
65
|
+
"@napi-rs/canvas": "^1.0.10",
|
|
66
|
+
"@types/node": "^26.6.4",
|
|
67
|
+
"pdfjs-dist": "~6.2.108",
|
|
65
68
|
"playwright": "^1.63.0",
|
|
66
69
|
"typescript": "^7.0.2",
|
|
67
70
|
"vitest": "^5.0.3"
|
|
68
71
|
},
|
|
69
72
|
"peerDependencies": {
|
|
73
|
+
"@napi-rs/canvas": "^1.0.10",
|
|
74
|
+
"pdfjs-dist": "~6.2.108",
|
|
70
75
|
"playwright": "^1.63.0"
|
|
71
76
|
},
|
|
72
77
|
"peerDependenciesMeta": {
|
|
78
|
+
"@napi-rs/canvas": {
|
|
79
|
+
"optional": true
|
|
80
|
+
},
|
|
81
|
+
"pdfjs-dist": {
|
|
82
|
+
"optional": true
|
|
83
|
+
},
|
|
73
84
|
"playwright": {
|
|
74
85
|
"optional": true
|
|
75
86
|
}
|
package/src/lib/types.mjs
CHANGED
|
@@ -137,7 +137,10 @@
|
|
|
137
137
|
* @typedef {object} ImportOptions
|
|
138
138
|
* @property {boolean} [dryRun] Check and report, write nothing.
|
|
139
139
|
* @property {string} [json] Also save the extracted menu as JSON at this path (resolved against projectDir).
|
|
140
|
-
* @property {
|
|
140
|
+
* @property {string} [tablePrefix] This restaurant's table prefix (e.g. "makibar_"); empty for the unprefixed menu_* tables. Default "": importCluvi/importImageMenu fall back to the source config's.
|
|
141
|
+
* @property {boolean} [allowUnprefixed] Write the unprefixed menu_* tables even when other restaurants' prefixed tables exist. Only for a single-restaurant database.
|
|
142
|
+
* @property {boolean} [yes] Confirm a destructive `replaceAll`, which empties the target tables.
|
|
143
|
+
* @property {boolean} [replaceAll] Replace the whole menu, not only the categories in this import. Needs `yes`.
|
|
141
144
|
* @property {boolean} [force] Write even if the import has far fewer products than it replaces.
|
|
142
145
|
* @property {string} [databaseUrl] Default: env.SUPABASE_DB_URL.
|
|
143
146
|
* @property {Env} [env] Environment the database URL and keys are read from. Default process.env.
|
|
@@ -151,7 +154,7 @@
|
|
|
151
154
|
* @property {string[]} notes
|
|
152
155
|
* @property {boolean} written
|
|
153
156
|
* @property {boolean} dryRun
|
|
154
|
-
* @property {{ label: string, current: { categories: number, products: number, kept: string[] } } | null} database Null when the database was not reached.
|
|
157
|
+
* @property {{ label: string, tables: string[], current: { categories: number, products: number, kept: string[] } } | null} database Null when the database was not reached.
|
|
155
158
|
*/
|
|
156
159
|
|
|
157
160
|
/**
|
|
@@ -161,6 +164,7 @@
|
|
|
161
164
|
/**
|
|
162
165
|
* Restaurant-specific part of the Cluvi source (src/menu/cluvi/config.mjs).
|
|
163
166
|
* @typedef {object} CluviConfig
|
|
167
|
+
* @property {string} [tablePrefix] This restaurant's table prefix in a shared database (e.g. "cannario_"); empty for the unprefixed menu_* tables.
|
|
164
168
|
* @property {string} [url] Any page of the restaurant's Cluvi menu.
|
|
165
169
|
* @property {{ slug: string, name: string, from: string[] }[]} [categories] Cluvi main categories folded into each site category.
|
|
166
170
|
* @property {Record<string, string>} [sections] Cluvi subcategory to section name.
|
|
@@ -169,11 +173,13 @@
|
|
|
169
173
|
/**
|
|
170
174
|
* Restaurant-specific part of the picture-menu source (src/menu/raw/config.mjs).
|
|
171
175
|
* @typedef {object} RawConfig
|
|
176
|
+
* @property {string} [tablePrefix] This restaurant's table prefix in a shared database (e.g. "mombasa_"); empty for the unprefixed menu_* tables.
|
|
172
177
|
* @property {string} [url] Page with the menu pictures, or a direct image URL.
|
|
173
178
|
* @property {string} currency ISO code of the prices.
|
|
174
179
|
* @property {string} [thousands] Thousands separator the menu prints. Default ".".
|
|
175
180
|
* @property {string} [decimal] Decimal separator the menu prints. Default ",".
|
|
176
181
|
* @property {number} [scale] Multiplies every price. Default 1.
|
|
182
|
+
* @property {number} [imageScale] Resolution a PDF page is rendered at before its product photos are screenshot. Default 2.
|
|
177
183
|
* @property {{ slug: string, name: string, groups?: ('food' | 'drink')[] }[]} [categories] Category that lists each group.
|
|
178
184
|
* @property {Record<string, string>} [placeIn] Section title to category slug.
|
|
179
185
|
* @property {Record<string, string>} [sections] Section title to the name stored.
|
|
@@ -194,13 +200,16 @@
|
|
|
194
200
|
|
|
195
201
|
/**
|
|
196
202
|
* @typedef {object} ImageMenuReadOptions
|
|
197
|
-
* @property {string[]} [urls] Pages or image URLs. Default: the config's url.
|
|
203
|
+
* @property {string[]} [urls] Pages or image URLs, or PDF file paths/URLs. Default: the config's url.
|
|
198
204
|
* @property {string | number[]} [only] Pages to read, e.g. '1,3-5' or [1, 3, 4, 5].
|
|
199
205
|
* @property {string} [provider] Vision provider, see `providers`. Default MENU_VISION_PROVIDER or `defaultProvider`.
|
|
200
206
|
* @property {string} [model]
|
|
201
207
|
* @property {number} [minWidth] Ignore images declaring a smaller width. Default 500.
|
|
202
208
|
* @property {string} [apiKey] Default: the provider's key in env.
|
|
203
209
|
* @property {boolean} [refresh] Read the pages again instead of using the saved transcriptions.
|
|
210
|
+
* @property {string} [imageDir] Also save the dish photos printed on a PDF page here (resolved against projectDir). Enables photo extraction.
|
|
211
|
+
* @property {string} [imageBaseUrl] https folder the saved photos will be published at; fills each product's image_url with it plus the file name.
|
|
212
|
+
* @property {'auto' | 'always'} [imageBoxes] How photos are found: 'auto' (default) reads a PDF page from its text and matches placed photos by position, using the model's boxes only when the page has none; 'always' reads every PDF page as a picture so the model boxes every dish's photo. Default 'auto'.
|
|
204
213
|
* @property {RawConfig} [config] Default: the raw config.mjs.
|
|
205
214
|
*/
|
|
206
215
|
|
|
@@ -211,14 +220,17 @@
|
|
|
211
220
|
* @property {string[]} [urls]
|
|
212
221
|
* @property {string | number[]} [only]
|
|
213
222
|
* @property {number} [minWidth]
|
|
223
|
+
* @property {string} [projectDir] Project folder. Default: TABLEFACTS_PROJECT or the current folder.
|
|
214
224
|
* @property {RawConfig} [config]
|
|
215
225
|
*/
|
|
216
226
|
|
|
217
227
|
/**
|
|
218
228
|
* @typedef {object} MenuImage
|
|
219
229
|
* @property {number} number Position as the CLI's --list numbers it.
|
|
220
|
-
* @property {string} url
|
|
230
|
+
* @property {string} url The image URL, or "<pdf path or URL>#<page number>" for a PDF page.
|
|
221
231
|
* @property {string} alt
|
|
232
|
+
* @property {'image' | 'pdf'} [kind] Where the page came from. Default: 'image'.
|
|
233
|
+
* @property {number} [chars] Characters of text a PDF page has (0 means it is a scan).
|
|
222
234
|
*/
|
|
223
235
|
|
|
224
236
|
/**
|
package/src/menu/README.md
CHANGED
|
@@ -6,6 +6,7 @@ Scripts that read a restaurant's menu from the website that hosts it and write i
|
|
|
6
6
|
| --- | --- | --- |
|
|
7
7
|
| Cluvi (`<restaurant>.cluvi.co`) | `cluvi/` | working |
|
|
8
8
|
| Menu that is only pictures (one image per page) | `raw/` | working, API calls untested |
|
|
9
|
+
| Menu as a PDF (text layer or scan) | `raw/pdf.mjs` | working, optional `pdfjs-dist` |
|
|
9
10
|
|
|
10
11
|
The same code is available as functions: `importCluvi`, `importImageMenu`, `listMenuImages` and `importMenu` (see the [main README](../../README.md#use-it-as-a-library)). Each source's `config.mjs` can be replaced with the `config` option, so a script can import a menu without editing the package:
|
|
11
12
|
|
|
@@ -22,34 +23,59 @@ await importCluvi({
|
|
|
22
23
|
const pictures = await listMenuImages({ urls: ['https://example.com/carta'], config: { currency: 'COP' } })
|
|
23
24
|
```
|
|
24
25
|
|
|
25
|
-
- **Cluvi `config`**: `{ url?, categories: [{ slug, name, from[] }], sections: { 'Cluvi subcategory': 'SECTION NAME' } }`.
|
|
26
|
-
- **Picture `config`**: `{ url?, currency, thousands, decimal, scale, categories: [{ slug, name, groups: ['food' | 'drink'] }], placeIn, sections, skipSections }`. `currency` is required.
|
|
27
|
-
- **
|
|
28
|
-
- **
|
|
29
|
-
- **
|
|
26
|
+
- **Cluvi `config`**: `{ tablePrefix?, url?, categories: [{ slug, name, from[] }], sections: { 'Cluvi subcategory': 'SECTION NAME' } }`.
|
|
27
|
+
- **Picture `config`**: `{ tablePrefix?, url?, currency, thousands, decimal, scale, imageScale?, categories: [{ slug, name, groups: ['food' | 'drink'] }], placeIn, sections, skipSections }`. `currency` is required; `imageScale` is the PDF render resolution for product photos (default 2).
|
|
28
|
+
- **Picture/PDF read options** (`importImageMenu`): `provider`, `model`, `minWidth`, `refresh`, `apiKey`, and the photo options `imageDir` (save the printed product photos), `imageBaseUrl` (fill their `image_url`) and `imageBoxes` (`'auto'` default, or `'always'`).
|
|
29
|
+
- **Options every import takes**: `dryRun`, `json` (relative paths resolve against `projectDir`), `tablePrefix` (overrides the config's; the restaurant's table set on a shared database), `allowUnprefixed` (override the shared-database check and write the unprefixed `menu_*` tables, only when this restaurant owns them), `replaceAll`, `yes` (confirms a whole-menu `replaceAll`), `force`, `databaseUrl` (default `env.SUPABASE_DB_URL`), `env`, `projectDir` and `log(message, level)` (`'info'`, `'warn'` for notes, `'error'`).
|
|
30
|
+
- **Result**: `{ totals, notes, written, dryRun, database }`; `database` is `{ label, tables, current: { categories, products, kept } }` once the database was inspected, and `null` when it was not reached (a dry run without a URL).
|
|
31
|
+
- **Errors** are `TablefactsError`: `ECONFIG` (no URL, unknown provider, no key, bad currency, a bad `tablePrefix`, another restaurant's tables on an unprefixed import), `EDEPENDENCY` (`pdfjs-dist`/`@napi-rs/canvas` missing for a PDF), `EFAILED` (invalid menu, no products, import refused without `force`), `EUSAGE` (bad `only`, a bad `imageBoxes`, `replaceAll` without `yes`). `err.option` names the option; the CLI shows the flag (`--force`, `--only`) and exits `2` for `EUSAGE`, `1` otherwise.
|
|
30
32
|
|
|
31
33
|
## Setup
|
|
32
34
|
|
|
33
35
|
1. Install the package in the project (`npm install --save-dev tablefacts`); it brings `pg`. Run the commands from the project's folder.
|
|
34
36
|
2. Apply the migration to your Supabase project.
|
|
35
|
-
3. `cp .env.example .env` and set `SUPABASE_DB_URL` to the pooler URL from the Supabase dashboard (Connect >
|
|
37
|
+
3. `cp .env.example .env` and set `SUPABASE_DB_URL` to the pooler URL from the Supabase dashboard (Connect > **Session pooler**), with the real password. Use the Session pooler, not the Transaction pooler: the importer holds one connection and reads `information_schema`. On a shared database, also set `tablePrefix` in the source's `config.mjs` (below).
|
|
36
38
|
|
|
37
39
|
`SUPABASE_DB_URL` bypasses row level security. It stays in `.env` (git-ignored). Never copy it into a browser-exposed variable (such as `NEXT_PUBLIC_*`): the site only uses the publishable key.
|
|
38
40
|
|
|
41
|
+
## Shared database
|
|
42
|
+
|
|
43
|
+
Several restaurants can live in one Supabase project. Each then owns prefixed tables — `cannario_menu_*`,
|
|
44
|
+
`mombasa_menu_*`, `makibar_menu_*` — and the unprefixed `menu_*` tables belong to a different, older site.
|
|
45
|
+
Set the restaurant's `tablePrefix` in its `config.mjs`, or pass `--table-prefix <prefix>` for one run (the
|
|
46
|
+
flag wins). The prefix may be empty, or lowercase letters, digits and underscores ending in `_`.
|
|
47
|
+
|
|
48
|
+
Before any write the importer:
|
|
49
|
+
|
|
50
|
+
- checks all three prefixed tables exist, or stops naming the missing ones and the migration to apply;
|
|
51
|
+
- with **no prefix**, lists the `public` tables ending in `_menu_categories` and, if another restaurant's set
|
|
52
|
+
is there, stops with an `ECONFIG` error that names them — rather than guessing which tables are yours.
|
|
53
|
+
`--allow-unprefixed` overrides that check only when this restaurant owns the unprefixed tables;
|
|
54
|
+
- prints the resolved table names and the row counts it would delete, and refuses `--replace-all` without
|
|
55
|
+
`--yes`. A whole-menu replace never runs on the unprefixed tables while another restaurant's tables exist.
|
|
56
|
+
|
|
57
|
+
`--dry-run` runs all of these checks and prints what would be written, but only sends `select`s (no `begin`).
|
|
58
|
+
|
|
59
|
+
The full model — prefix rules, every check, the error codes and a migration checklist — is in
|
|
60
|
+
[docs/MENU_TABLE_PREFIX.md](../../docs/MENU_TABLE_PREFIX.md).
|
|
61
|
+
|
|
39
62
|
## Run (Cluvi)
|
|
40
63
|
|
|
41
64
|
```bash
|
|
42
|
-
tablefacts menu cluvi --dry-run
|
|
43
|
-
tablefacts menu cluvi
|
|
44
|
-
tablefacts menu cluvi <menu-url>
|
|
65
|
+
tablefacts menu cluvi --dry-run # extract and check, show what would change, write nothing
|
|
66
|
+
tablefacts menu cluvi # replace this restaurant's menu in Supabase
|
|
67
|
+
tablefacts menu cluvi <menu-url> # another restaurant
|
|
68
|
+
tablefacts menu cluvi --table-prefix makibar_ # override the config's table prefix for one run
|
|
69
|
+
tablefacts menu cluvi --replace-all --yes # empty this restaurant's tables first, then write
|
|
70
|
+
tablefacts menu cluvi --allow-unprefixed # the unprefixed menu_* tables (single-restaurant database only)
|
|
45
71
|
tablefacts menu cluvi --help
|
|
46
72
|
```
|
|
47
73
|
|
|
48
74
|
Edit `cluvi/config.mjs` first: the menu URL, how Cluvi's main categories fold into the site's categories, and section renames. The run prints what the site still needs (dictionary keys, `content/qr.ts`).
|
|
49
75
|
|
|
50
|
-
## Run (menu that is only pictures)
|
|
76
|
+
## Run (menu that is only pictures or a PDF)
|
|
51
77
|
|
|
52
|
-
For a restaurant whose site shows the menu as a gallery of page images, such as <https://www.mombasa.co/carta-restaurante-espanol
|
|
78
|
+
For a restaurant whose site shows the menu as a gallery of page images, such as <https://www.mombasa.co/carta-restaurante-espanol/>, or that offers it as a PDF. Each page is transcribed by a model from one of three providers, so it needs that provider's key in `.env`:
|
|
53
79
|
|
|
54
80
|
| `--provider` | Key in `.env` | Default `--model` |
|
|
55
81
|
| --- | --- | --- |
|
|
@@ -64,14 +90,36 @@ tablefacts menu raw --list # show the pictures found; no key nee
|
|
|
64
90
|
tablefacts menu raw --dry-run # read the pages, show the menu and the checks, write nothing
|
|
65
91
|
tablefacts menu raw --only 3,9 --dry-run # try two pages first
|
|
66
92
|
tablefacts menu raw --provider gemini --dry-run # the same, read by Gemini
|
|
67
|
-
tablefacts menu raw # replace
|
|
93
|
+
tablefacts menu raw # replace this restaurant's menu in Supabase
|
|
68
94
|
tablefacts menu raw <page-or-image-url>... # another restaurant
|
|
95
|
+
tablefacts menu raw --table-prefix makibar_ # override the config's table prefix for one run
|
|
96
|
+
tablefacts menu raw --replace-all --yes # empty this restaurant's tables first, then write
|
|
97
|
+
tablefacts menu raw carta.pdf --list # pages of a PDF file (a .pdf URL works too)
|
|
98
|
+
tablefacts menu raw carta.pdf --dry-run # text pages from their text, scans from a render
|
|
99
|
+
tablefacts menu raw carta.pdf --images ./carta-fotos --image-base-url https://cdn.example.com/carta/ --dry-run
|
|
100
|
+
tablefacts menu raw carta.pdf --images ./carta-fotos --image-boxes always --dry-run
|
|
69
101
|
```
|
|
70
102
|
|
|
71
|
-
|
|
103
|
+
A PDF argument ends in `.pdf` and is a local file or an http(s) URL. A page that carries its own text is
|
|
104
|
+
transcribed from that text (cheaper and exact); a page with no text layer is rendered and read like a picture
|
|
105
|
+
(`chars` in `--list` tells them apart). Rendering needs `@napi-rs/canvas`; text-only PDFs need only `pdfjs-dist`.
|
|
106
|
+
|
|
107
|
+
`--images <folder>` also saves the dish photos printed on a PDF page, as PNG crops of the page render. Two ways
|
|
108
|
+
the crop finds its dish: when the page places each photo separately, each is matched to the product whose printed
|
|
109
|
+
name is nearest it; when it does not (a flattened export, vector art, a scan), the model is asked for each dish's
|
|
110
|
+
photo box instead. That fallback reads the page as a picture, so asking for photos adds a vision call to pages
|
|
111
|
+
with no separately placed photo. `--image-boxes always` reads **every** PDF page as a picture so the model boxes
|
|
112
|
+
each dish even when the page also places some photos separately (one vision call per page); `auto` (the default)
|
|
113
|
+
only does that when the page has no separately placed photo. `--image-base-url https://host/path/` then fills
|
|
114
|
+
each product's `image_url` with that URL plus the file name, so the import can use it. Without it the photos are
|
|
115
|
+
saved but `image_url` stays empty until you host them and re-run with the base URL.
|
|
116
|
+
|
|
117
|
+
Edit `raw/config.mjs` first: the URL (a PDF path/URL works there too), the currency, how the menu prints prices (`$95.000` is `thousands: "."`, `decimal: ","`) and which category each section goes to.
|
|
72
118
|
|
|
73
119
|
- `raw/source.mjs` takes the large JPG/PNG/WebP images of the page in document order (full size, not the thumbnails; logos and icons are skipped by width) and downloads them. A site that builds its gallery with JavaScript shows up as "no images": pass the image URLs instead.
|
|
74
|
-
- `raw/
|
|
120
|
+
- `raw/pdf.mjs` reads a PDF (a local file or URL) with the optional `pdfjs-dist`: a page's text and text positions, where each printed image sits on the page, and a render of the page. A page with enough text is sent to the model as text (a two-column layout is read column by column); a scan is rendered and sent as a picture. `raw/pdfjs.mjs` loads the dependency on use and turns a missing one into an `EDEPENDENCY` error with the install command.
|
|
121
|
+
- `raw/vision.mjs` has the model transcribe each page into sections and dishes with one request: a forced tool call for Anthropic, JSON held to the same schema for Gemini (`generateContent`) and Groq (strict structured output), so every provider hands back the same shape. The same providers read a PDF page's text (no picture) and can return each item's photo box. Prices come back as printed text and `raw/normalize.mjs` parses them, so a thousands separator is never guessed by the model. Adding a provider is one more entry in `providers` there.
|
|
122
|
+
- `raw/images.mjs` ties a product photo to a dish: by the model's box (a scan, or a page with no separately placed photo), or by matching the dish name to the page's text and taking the nearest placed image. `raw/import.mjs` saves the crops and fills `image_url` when a base URL is given.
|
|
75
123
|
- Groq serves a single vision model, `qwen/qwen3.8-27b`, and it is a preview one: if Groq retires it, pass its successor with `--model`.
|
|
76
124
|
- Rate limits (HTTP 429) are retried after the wait the service asks for, up to a minute; for a longer wait the run stops with the service's message. Pages already read are saved (below), so run it again later.
|
|
77
125
|
- The model tags each section food, drink or other; `config.categories` maps those to the site's categories. A section with no heading continues the one before it, even across pages. A dish with several price columns (bottle and glass) becomes one product per column, "Name (Botella)".
|
|
@@ -81,7 +129,7 @@ Edit `raw/config.mjs` first: the URL, the currency, how the menu prints prices (
|
|
|
81
129
|
## What it does (Cluvi)
|
|
82
130
|
|
|
83
131
|
- Cluvi main category > site category, subcategory > section (name in UPPERCASE), product > product. Products keep the order Cluvi shows (its `order` field). Cluvi's "important" flag is `recommended`. Descriptions are converted from HTML to text.
|
|
84
|
-
- The write is one transaction. By default only the categories in the import are replaced (their sections and products with them), so re-running never duplicates. `--replace-all`
|
|
132
|
+
- The write is one transaction. By default only the categories in the import are replaced (their sections and products with them), so re-running never duplicates. `--replace-all` empties the whole menu first, which also removes the template's sample categories; it needs `--yes` and is refused when it would empty the unprefixed tables on a shared database.
|
|
85
133
|
- It refuses an import with no products, or with under half the products it replaces (`--force` overrides), so a broken extraction cannot empty the menu.
|
|
86
134
|
- Photos stay on Cluvi's CDN: `image_url` holds the Cluvi URL, and your site must allow `images.cluvi.com` and `images-mini.cluvi.com` as image hosts (a template with `menuImageHosts` in `frontend/src/content/site.ts` gets a warning from the run when they are missing). If the restaurant leaves Cluvi, the photos must be re-hosted (Supabase Storage needs a different credential than the DB URL).
|
|
87
135
|
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
// Everything restaurant-specific about the Cluvi import. Edit this file, not
|
|
2
2
|
// source.mjs, when the menu is organised differently.
|
|
3
3
|
export default {
|
|
4
|
+
// This restaurant's tables in a shared Supabase database: the import writes
|
|
5
|
+
// public.cannario_menu_categories / _menu_sections / _menu_products. Change
|
|
6
|
+
// it (or pass `--table-prefix`) when importing another restaurant; leave it
|
|
7
|
+
// empty only when this restaurant owns the unprefixed menu_* tables.
|
|
8
|
+
tablePrefix: "cannario_",
|
|
9
|
+
|
|
4
10
|
// Any page of the restaurant's Cluvi menu. Only the first path segment is
|
|
5
11
|
// used (the supplier, here "cannario"). `tablefacts menu cluvi <url>`
|
|
6
12
|
// overrides it for one run.
|
|
@@ -17,6 +17,8 @@ export async function fetchCluviMenu({ url, service, lang, config = defaultConfi
|
|
|
17
17
|
menu,
|
|
18
18
|
notes,
|
|
19
19
|
title: `Cluvi: ${supplier.label} (id ${supplier.id}), ${fetched.service} menu in ${currency}`,
|
|
20
|
+
// The restaurant's own table set, so a shared database is never touched by accident.
|
|
21
|
+
tablePrefix: config.tablePrefix,
|
|
20
22
|
};
|
|
21
23
|
}
|
|
22
24
|
|
|
@@ -25,6 +27,8 @@ export async function fetchCluviMenu({ url, service, lang, config = defaultConfi
|
|
|
25
27
|
* @param {import('../../lib/types.mjs').ImportCluviOptions} [options]
|
|
26
28
|
* @returns {Promise<import('../../lib/types.mjs').ImportResult>}
|
|
27
29
|
*/
|
|
28
|
-
export async function importCluvi({ url, service, lang, config, ...importOptions } = {}) {
|
|
29
|
-
|
|
30
|
+
export async function importCluvi({ url, service, lang, config = defaultConfig, ...importOptions } = {}) {
|
|
31
|
+
const fetched = await fetchCluviMenu({ url, service, lang, config });
|
|
32
|
+
// An explicit `tablePrefix` (or the CLI's --table-prefix) wins over the config's.
|
|
33
|
+
return importMenu({ ...fetched, ...importOptions, tablePrefix: importOptions.tablePrefix ?? config.tablePrefix });
|
|
30
34
|
}
|